From d4b741cf4a0655f3fce5433ff22cb57be00624bc Mon Sep 17 00:00:00 2001 From: Quinn Damerell Date: Tue, 28 Feb 2017 17:47:46 -0800 Subject: [PATCH] Fixing two issues. 1) Ensuring we don't send any data until both audio and video are ready to send. 2) Making both of the rtp timestamps start at 0 for sync reasons and so it is easier to calculate the sync. --- libftl/ftl-sdk.c | 4 ++-- libftl/ftl_private.h | 2 ++ libftl/media.c | 46 ++++++++++++++++++++++++++++++++++++++------ 3 files changed, 44 insertions(+), 8 deletions(-) diff --git a/libftl/ftl-sdk.c b/libftl/ftl-sdk.c index 0335a4f..dccd727 100644 --- a/libftl/ftl-sdk.c +++ b/libftl/ftl-sdk.c @@ -11,8 +11,8 @@ static int _lookup_ingest_ip(const char *ingest_location, char *ingest_ip); char error_message[1000]; FTL_API const int FTL_VERSION_MAJOR = 0; -FTL_API const int FTL_VERSION_MINOR = 8; -FTL_API const int FTL_VERSION_MAINTENANCE = 16; +FTL_API const int FTL_VERSION_MINOR = 9; +FTL_API const int FTL_VERSION_MAINTENANCE = 0; // Initializes all sublibraries used by FTL FTL_API ftl_status_t ftl_init() { diff --git a/libftl/ftl_private.h b/libftl/ftl_private.h index bb9b189..3892f1e 100644 --- a/libftl/ftl_private.h +++ b/libftl/ftl_private.h @@ -208,6 +208,7 @@ typedef struct { int64_t dts_usec; ftl_media_component_common_t media_component; OS_MUTEX mutex; + BOOL is_ready_to_send; } ftl_audio_component_t; typedef struct { @@ -222,6 +223,7 @@ typedef struct { BOOL wait_for_idr_frame; ftl_media_component_common_t media_component; OS_MUTEX mutex; + BOOL has_sent_first_frame; } ftl_video_component_t; typedef struct { diff --git a/libftl/media.c b/libftl/media.c index 99fd695..e9f3857 100644 --- a/libftl/media.c +++ b/libftl/media.c @@ -79,17 +79,21 @@ ftl_status_t media_init(ftl_stream_configuration_private_t *ftl) { goto cleanup; } - comp->timestamp = 0; //TODO: should start at a random value + // According to RTP the time stamps should start at random values, + // but to help sync issues and to make sync easier to calcualte we + // start at 0. + comp->timestamp = 0; comp->producer = 0; comp->consumer = 0; + comp->prev_dts_usec = -1; _clear_stats(&comp->stats); } ftl->video.media_component.timestamp_clock = VIDEO_RTP_TS_CLOCK_HZ; ftl->audio.media_component.timestamp_clock = AUDIO_SAMPLE_RATE; - ftl->video.media_component.prev_dts_usec = -1; - ftl->audio.media_component.prev_dts_usec = -1; + ftl->audio.is_ready_to_send = FALSE; + ftl->video.has_sent_first_frame = FALSE; ftl->video.wait_for_idr_frame = TRUE; @@ -339,6 +343,7 @@ ftl_status_t media_speed_test(ftl_stream_configuration_private_t *ftl, int speed media_enable_nack(ftl, mc->ssrc, FALSE); ftl_set_state(ftl, FTL_DISABLE_TX_PING_PKTS); + ftl->video.has_sent_first_frame = TRUE; ping = (ping_pkt_t*)slot.packet; @@ -450,6 +455,7 @@ ftl_status_t media_speed_test(ftl_stream_configuration_private_t *ftl, int speed retval = FTL_SUCCESS; } + ftl->video.has_sent_first_frame = FALSE; media_enable_nack(ftl, mc->ssrc, TRUE); ftl_clear_state(ftl, FTL_DISABLE_TX_PING_PKTS); @@ -470,6 +476,15 @@ int media_send_audio(ftl_stream_configuration_private_t *ftl, int64_t dts_usec, int remaining = len; int retries = 0; + + // When we get our first audio packet, indicate that we are ready to send. + // However, don't send audio data until the video is also sending. + ftl->audio.is_ready_to_send = TRUE; + if (!ftl->video.has_sent_first_frame) + { + return 0; + } + if (os_trylock_mutex(&ftl->audio.mutex)) { if (ftl_get_state(ftl, FTL_MEDIA_READY)) { @@ -525,6 +540,17 @@ int media_send_video(ftl_stream_configuration_private_t *ftl, int64_t dts_usec, int remaining = len; int first_fu = 1; + // Before we send any video we want to make sure the audio stream + // is also ready to run. If the stream isn't ready drop this data. + if (!ftl->audio.is_ready_to_send) + { + if (end_of_frame) + { + mc->stats.dropped_frames++; + } + return bytes_queued; + } + if (os_trylock_mutex(&ftl->video.mutex)) { if (ftl_get_state(ftl, FTL_MEDIA_READY)) { @@ -532,12 +558,18 @@ int media_send_video(ftl_stream_configuration_private_t *ftl, int64_t dts_usec, nalu_type = data[0] & 0x1F; nri = (data[0] >> 5) & 0x3; - _update_timestamp(ftl, mc, dts_usec); - if (ftl->video.wait_for_idr_frame) { if (nalu_type == H264_NALU_TYPE_SPS) { - FTL_LOG(ftl, FTL_LOG_INFO, "Got key frame, continuing (dropped %d frames)\n", mc->stats.dropped_frames); + ftl->video.wait_for_idr_frame = FALSE; + + if (!ftl->video.has_sent_first_frame) { + FTL_LOG(ftl, FTL_LOG_INFO, "Audio is ready and we have the first iframe, starting stream. (dropped %d frames)\n", mc->stats.dropped_frames); + ftl->video.has_sent_first_frame = TRUE; + } + else { + FTL_LOG(ftl, FTL_LOG_INFO, "Got key frame, continuing (dropped %d frames)\n", mc->stats.dropped_frames); + } } else { if (end_of_frame) { @@ -548,6 +580,8 @@ int media_send_video(ftl_stream_configuration_private_t *ftl, int64_t dts_usec, } } + _update_timestamp(ftl, mc, dts_usec); + if (nalu_type == H264_NALU_TYPE_IDR) { mc->tmp_seq_num = mc->seq_num; }