Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
39 commits
Select commit Hold shift + click to select a range
10eb7f5
resampler: added audio_frame2::resample_fake
MartinPulec Nov 23, 2021
0354ac4
Audio decoder: allow slight sample rate adjust
MartinPulec Nov 23, 2021
011c28a
messaging: added msg_universal C++ constructor
MartinPulec Dec 3, 2021
20f3c83
DO NOT MERGE DeckLink disp.: added deck_audio_drift_fixer
MartinPulec Nov 8, 2021
44ae96d
audio decoder: added option soft-resample
MartinPulec Dec 13, 2021
d4f9204
DeckLink cap.: use bmdNoVideoInputConversion for check
MartinPulec Apr 1, 2022
db20fa3
Updated the decklink with resample changes
aw-sohonet Apr 1, 2022
7af5c49
Updated jitter to be 50
aw-sohonet Apr 1, 2022
007331e
Updated audio resampling calculation to be correct
aw-sohonet Apr 2, 2022
8c33acd
Added latest audio resampling code
aw-sohonet Apr 5, 2022
bdf75e2
Latest updates for working resampling
aw-sohonet Apr 5, 2022
5ead15e
Removed unnecessary calls to check for zero data
aw-sohonet Apr 5, 2022
aae91eb
Added comments for the changes that were made to the resampling code …
aw-sohonet Apr 5, 2022
2727485
Removed set fps calls as they are no longer present. Fixed some casti…
aw-sohonet Apr 6, 2022
da436e4
Updated with some intial threading code and handling for 32bps audio …
aw-sohonet Apr 6, 2022
c5b31b2
Added missing commit
aw-sohonet Apr 6, 2022
6880a5a
Added some additional debug for 32bit audio resampling.
aw-sohonet Apr 6, 2022
ce03347
Updated to have the fix in for tail buffer when the decompressed audi…
aw-sohonet Apr 7, 2022
fb0ba11
Updated to introduce threading for the channels in resampling
aw-sohonet Apr 7, 2022
c852f61
Updated with comments and removal of some unnecessary log lines (and …
aw-sohonet Apr 7, 2022
49c5639
Fixed floating point calculations to be correct when converting 32bit…
aw-sohonet Apr 7, 2022
f674ea0
Changed the code to average over a smaller amount of time. To have a …
aw-sohonet Apr 8, 2022
f57c395
Removed some unecessary log lines from when the remainder from a resa…
aw-sohonet Apr 11, 2022
adbd605
Added an audio summary class to the decklink to get periodic updates …
aw-sohonet Apr 11, 2022
1ed3c8d
Reorganised so that the class doesn't need a forward declaration and …
aw-sohonet Apr 11, 2022
474c7c6
Resolved casting issues when the previous buffer has not been set for…
aw-sohonet Apr 11, 2022
6ac3c69
Moved the overflow code to be generic for both low-latency and non-lo…
aw-sohonet Apr 11, 2022
a4ed6c6
Updated audio summary struct variable name to be "audio_summary"
aw-sohonet Apr 11, 2022
ecba755
Updated to be able to variably set the minHz, maxHz, and target buffe…
aw-sohonet Apr 11, 2022
64161f9
Added comments to added functions that were missing them, and changed…
aw-sohonet Apr 11, 2022
e441300
Added some additional stats to capture
aw-sohonet Apr 12, 2022
54182f2
Fixed the correct way around for the time diff calculation
aw-sohonet Apr 12, 2022
47190f9
Have the decklink report stats more often
aw-sohonet Apr 12, 2022
c2c405d
Updated the decklink to always resample towards the target
aw-sohonet Apr 12, 2022
f94c554
Updated to fix some off by 1 errors which was causing major drifts in…
aw-sohonet Apr 13, 2022
872e2ec
Updated the jitter values to be far more forgiving
aw-sohonet Apr 13, 2022
590a235
Added a comment for the resampler.
aw-sohonet Apr 13, 2022
2a7931a
Fixed a boolean being the wrong way around
aw-sohonet Apr 14, 2022
11efbce
Updated dynamic resampler to use SOXR
aw-sohonet Jun 20, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .gitmodules
Original file line number Diff line number Diff line change
Expand Up @@ -4,3 +4,6 @@
[submodule "ext-deps/zfec"]
path = ext-deps/zfec
url = https://github.com/tahoe-lafs/zfec
[submodule "ext-deps/soxr"]
path = ext-deps/soxr
url = https://github.com/chirlu/soxr.git
21 changes: 21 additions & 0 deletions configure.ac
Original file line number Diff line number Diff line change
Expand Up @@ -2591,6 +2591,26 @@ elif test "$found_speexdsp" = no -a "$speexdsp_req" != no; then
UG_MSG_WARN([SpeexDSP was not found. Strongly recommending installing that, otherwise audio part of UG will be crippled.])
fi

# ---------------------------------------------------------------------------
# SOXR
# ---------------------------------------------------------------------------
soxr=no

AC_ARG_ENABLE(soxr,
AS_HELP_STRING([--disable-soxr], [disable Soxr (default is auto)]),
[soxr_req=$enableval],
[soxr_req=$build_default])
PKG_CHECK_MODULES([LIBSOXR], [soxr], [found_soxr=yes], [found_soxr=no])

if test "$found_soxr" = yes -a "$soxr_req" != no; then
LIBS="$LIBS $LIBSOXR_LIBS"
COMMON_FLAGS="$COMMON_FLAGS${SOXR_COMMON_FLAGS:+${COMMON_FLAGS:+ }}$SOXR_COMMON_FLAGS"
AC_DEFINE([HAVE_SOXR], [1], [Build with Soxr support])
soxr=yes
elif test "$soxr_req" = yes; then
AC_MSG_ERROR([Soxr not found]);
fi

# ---------------------------------------------------------------------------
# Zfec
# ---------------------------------------------------------------------------
Expand Down Expand Up @@ -3467,6 +3487,7 @@ RESULT=`add_column "$RESULT" "Profiling support" $profile $?`
RESULT=`add_column "$RESULT" "Qt GUI" $qt_gui $?`
RESULT=`add_column "$RESULT" "RT priority" $use_rt $?`
RESULT=`add_column "$RESULT" "SpeexDSP" $speexdsp $?`
RESULT=`add_column "$RESULT" "Soxr" $soxr $?`
RESULT=`add_column "$RESULT" "Standalone modules" $build_libraries $?`
RESULT=`add_column "$RESULT" "zfec" $zfec $?`
RESULT=`end_section "$RESULT"`
Expand Down
1 change: 1 addition & 0 deletions ext-deps/soxr
Submodule soxr added at 945b59
351 changes: 286 additions & 65 deletions src/audio/types.cpp

Large diffs are not rendered by default.

38 changes: 32 additions & 6 deletions src/audio/types.h
Original file line number Diff line number Diff line change
Expand Up @@ -113,20 +113,36 @@ typedef struct

#ifdef __cplusplus
#include <memory>
#include <tuple>
#include <utility>
#include <vector>

class audio_frame2;

class audio_frame2_resampler {
public:
audio_frame2_resampler();
~audio_frame2_resampler();
int get_resampler_numerator();
int get_resampler_denominator();
int get_resampler_output_latency();
int get_resampler_input_latency();
int get_resampler_from_sample_rate();
int get_resampler_initial_bps();
size_t get_resampler_channel_count();
bool resampler_is_set();
bool create_resampler(uint32_t original_sample_rate, uint32_t new_sample_rate_num, uint32_t new_sample_rate_den, size_t channel_size, int bps);
void resample_set_destroy_flag(bool destroy);
private:
void *resampler; // type is (SpeexResamplerState *)
int resample_from;
size_t resample_ch_count;
int resample_to;
void *resampler{nullptr}; // type is (SpeexResamplerState *)
int resample_from{0};
int resample_to_num{0};
int resample_to_den{1};
int resample_output_latency{0};
int resample_input_latency{0};
int resample_initial_bps{0};
size_t resample_ch_count{0};
bool destroy_resampler{false};
bool initial{};

friend class audio_frame2;
};
Expand Down Expand Up @@ -170,6 +186,9 @@ class audio_frame2
void set_fec_params(int channel, fec_desc const &);
static audio_frame2 copy_with_bps_change(audio_frame2 const &frame, int new_bps);
void change_bps(int new_bps);
void convert_int32_to_float();
void convert_float_to_int32();

/**
* @note
* bps of the frame needs to be 16 bits!
Expand All @@ -183,14 +202,21 @@ class audio_frame2
* properties.
* @retval false if SpeexDSP was not compiled in
*/
bool resample(audio_frame2_resampler &resampler_state, int new_sample_rate);
std::tuple<bool, bool> resample(audio_frame2_resampler &resampler_state, int new_sample_rate);

///@ resamples to new sample rate while keeping nominal sample rate intact
std::tuple<bool, bool, audio_frame2> resample_fake(audio_frame2_resampler & resampler_state, int new_sample_rate_num, int new_sample_rate_den);
private:
struct channel {
std::unique_ptr<char []> data;
size_t len;
size_t max_len;
struct fec_desc fec_params;
};
static void resample_channel(audio_frame2_resampler* resampler_state, int channel_index,
const uint16_t *in, uint32_t in_len, channel *new_channel, audio_frame2 *remainder);
static void resample_channel_float(audio_frame2_resampler* resampler_state, int channel_index,
const float *in, uint32_t in_len, channel *new_channel, audio_frame2 *remainder);
void reserve(int channel, size_t len);
int bps; /* bytes per sample */
int sample_rate;
Expand Down
12 changes: 12 additions & 0 deletions src/messaging.h
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,10 @@
#ifndef _MESSAGING_H
#define _MESSAGING_H

#ifdef __cplusplus
#include <cstring>
#endif

#include "types.h"

#ifdef __cplusplus
Expand All @@ -57,6 +61,8 @@ struct response;
#define RESPONSE_INT_SERV_ERR 500
#define RESPONSE_NOT_IMPL 501

#define RESPONSE_SUCCESSFUL(code) ((code) >= 200 && (code) < 300)

struct message;

struct message {
Expand Down Expand Up @@ -152,6 +158,12 @@ struct msg_stats {
* msg_universal::text to identify the receiving module.
*/
struct msg_universal {
#ifdef __cplusplus
msg_universal(const char *contents) {
memset(&m, 0, sizeof m);
strncpy(text, contents, sizeof text - 1);
}
#endif
struct message m;
char text[8192];
};
Expand Down
69 changes: 62 additions & 7 deletions src/rtp/audio_decoders.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@
#include "utils/worker.h"

#include <algorithm>
#include <atomic>
#include <cctype>
#include <chrono>
#include <cstring>
Expand All @@ -86,6 +87,7 @@ using rang::style;
using std::fixed;
using std::hex;
using std::map;
using std::move;
using std::ostringstream;
using std::pair;
using std::setprecision;
Expand Down Expand Up @@ -188,6 +190,7 @@ struct state_audio_decoder_summary {

struct state_audio_decoder {
uint32_t magic;
struct module mod;

struct timeval t0;

Expand All @@ -212,6 +215,7 @@ struct state_audio_decoder {
bool muted;

audio_frame2_resampler resampler;
bool resample_tail_buffer;

audio_playback_ctl_t audio_playback_ctl_func;
void *audio_playback_state;
Expand All @@ -221,6 +225,9 @@ struct state_audio_decoder {
fec_desc fec_state_desc;

struct state_audio_decoder_summary summary;

audio_frame2 resample_remainder;
std::atomic_uint64_t req_resample_to{0}; // hi 32 - numerator; lo 32 - denominator
};

constexpr double VOL_UP = 1.1;
Expand Down Expand Up @@ -250,6 +257,26 @@ static void compute_scale(struct scale_data *scale_data, double vol_avg, int sam
}
}

static void audio_decoder_process_message(struct module *m)
{
auto *s = static_cast<struct state_audio_decoder *>(m->priv_data);

while (auto *msg = reinterpret_cast<msg_universal *>(check_message(m))) {
struct response *r = nullptr;
if (strstr(msg->text, MSG_UNIVERSAL_TAG_AUDIO_DECODER) == msg->text) {
s->req_resample_to = strtoull(msg->text + strlen(MSG_UNIVERSAL_TAG_AUDIO_DECODER), nullptr, 0);
LOG(LOG_LEVEL_VERBOSE) << MOD_NAME << "Resampling sound to: " << (s->req_resample_to >> ADEC_CH_RATE_SHIFT) << "/" << (s->req_resample_to & ((1LU << ADEC_CH_RATE_SHIFT) - 1)) << "\n";
r = new_response(RESPONSE_OK, nullptr);
} else {
r = new_response(RESPONSE_NOT_FOUND, nullptr);
}
free_message(reinterpret_cast<message *>(msg), r);
}
}

ADD_TO_PARAM("soft-resample", "* soft-resample=<num>/<den>\n"
" Resample to specified sampling rate, eg. 12288128/256 for 48000.5 Hz\n");

void *audio_decoder_init(char *audio_channel_map, const char *audio_scale, const char *encryption, audio_playback_ctl_t c, void *p_state, struct module *parent)
{
struct state_audio_decoder *s;
Expand All @@ -264,6 +291,19 @@ void *audio_decoder_init(char *audio_channel_map, const char *audio_scale, const
s->audio_playback_ctl_func = c;
s->audio_playback_state = p_state;

s->resample_tail_buffer = false;

module_init_default(&s->mod);
s->mod.cls = MODULE_CLASS_DECODER;
s->mod.priv_data = s;
s->mod.new_message = audio_decoder_process_message;
module_register(&s->mod, parent);

if (const char *val = get_commandline_param("soft-resample")) {
assert(strchr(val, '/') != nullptr);
s->req_resample_to = atol(val) << 32LLU | atol(strchr(val, '/') + 1);
}

gettimeofday(&s->t0, NULL);
s->packet_counter = packet_counter_init(0);

Expand Down Expand Up @@ -403,7 +443,7 @@ void audio_decoder_destroy(void *state)
}

delete s->fec_state;

module_done(&s->mod);
delete s;
}

Expand Down Expand Up @@ -511,6 +551,7 @@ static bool audio_decoder_reconfigure(struct state_audio_decoder *decoder, struc
return false;
}

decoder->resample_remainder = {};
return true;
}

Expand Down Expand Up @@ -722,13 +763,27 @@ int decode_audio_frame(struct coded_data *cdata, void *pbuf_data, struct pbuf_st
return FALSE;
}

if (s->buffer.sample_rate != decompressed.get_sample_rate()) {
if (decompressed.get_bps() != 2) {
decompressed.change_bps(2);
// Perform a variable rate resample if any output device has requested it
if (decoder->req_resample_to != 0) {
// Set the BPS of the audio to be within the supported range. A value of 4 bytes has
// been selected as it'll give a higher accuracy for conversion.
if(decompressed.get_bps() != 2 || decompressed.get_bps() != 4) {
decompressed.change_bps(4);
}
if (!decompressed.resample(decoder->resampler, s->buffer.sample_rate)) {
LOG(LOG_LEVEL_INFO) << MOD_NAME << "You may try to set different sampling on sender.\n";
return FALSE;
if (decoder->req_resample_to != 0) {
auto [ret, reinitResampler, remainder] = decompressed.resample_fake(decoder->resampler, decoder->req_resample_to >> ADEC_CH_RATE_SHIFT, decoder->req_resample_to & ((1LU << ADEC_CH_RATE_SHIFT) - 1));
if (!ret) {
LOG(LOG_LEVEL_INFO) << MOD_NAME << "You may try to set different sampling on sender.\n";
return FALSE;
}
decoder->resample_remainder = move(remainder);
}
else {
auto [ret, reinitResampler] = decompressed.resample(decoder->resampler, s->buffer.sample_rate);
if (!ret) {
LOG(LOG_LEVEL_INFO) << MOD_NAME << "You may try to set different sampling on sender.\n";
return FALSE;
}
}
}

Expand Down
3 changes: 3 additions & 0 deletions src/rtp/audio_decoders.h
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,9 @@ void audio_decoder_set_volume(void *state, double val);

bool parse_audio_hdr(uint32_t *hdr, struct audio_desc *desc);

#define MSG_UNIVERSAL_TAG_AUDIO_DECODER "ADEC "
#define ADEC_CH_RATE_SHIFT 32LU

#ifdef __cplusplus
}
#endif
Expand Down
5 changes: 3 additions & 2 deletions src/video_capture/decklink.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1215,6 +1215,7 @@ vidcap_decklink_init(struct vidcap_params *params, void **state)

BMD_CONFIG_SET_INT(bmdDeckLinkConfigVideoInputConnection, s->connection);
BMD_CONFIG_SET_INT(bmdDeckLinkConfigVideoInputConversionMode, s->conversion_mode);
BMDVideoInputConversionMode supported_conversion_mode = s->conversion_mode ? s->conversion_mode : (BMDVideoInputConversionMode) bmdNoVideoInputConversion;
BMD_CONFIG_SET_INT(bmdDeckLinkConfigCapturePassThroughMode, s->passthrough);

if (s->link == 0) {
Expand Down Expand Up @@ -1281,7 +1282,7 @@ vidcap_decklink_init(struct vidcap_params *params, void **state)
}
BMDPixelFormat pf = it->second;
BMD_BOOL supported = 0;
EXIT_IF_FAILED(deckLinkInput->DoesSupportVideoMode(s->connection, displayMode->GetDisplayMode(), pf, s->conversion_mode, s->supported_flags, nullptr, &supported), "DoesSupportVideoMode");
EXIT_IF_FAILED(deckLinkInput->DoesSupportVideoMode(s->connection, displayMode->GetDisplayMode(), pf, supported_conversion_mode, s->supported_flags, nullptr, &supported), "DoesSupportVideoMode");
if (supported) {
break;
}
Expand Down Expand Up @@ -1343,7 +1344,7 @@ vidcap_decklink_init(struct vidcap_params *params, void **state)
}

BMD_BOOL supported = 0;
EXIT_IF_FAILED(deckLinkInput->DoesSupportVideoMode(s->connection, displayMode->GetDisplayMode(), pf, s->conversion_mode, s->supported_flags, nullptr, &supported), "DoesSupportVideoMode");
EXIT_IF_FAILED(deckLinkInput->DoesSupportVideoMode(s->connection, displayMode->GetDisplayMode(), pf, supported_conversion_mode, s->supported_flags, nullptr, &supported), "DoesSupportVideoMode");

if (!supported) {
LOG(LOG_LEVEL_ERROR) << MOD_NAME "Requested display mode not supported with the selected pixel format\n";
Expand Down
Loading