FS-10249: [mod_av] Audio gradually falls behind video in recordings

src/include/private/switch_core_pvt.h

@@ -239,7 +239,7 @@ struct switch_media_bug {
 	switch_buffer_t *text_buffer;
 	char *text_framedata;
 	uint32_t text_framesize;
-
+	switch_mm_t mm;
 	struct switch_media_bug *next;
 };
 

src/include/switch_core.h

@@ -351,6 +351,9 @@ SWITCH_DECLARE(void) switch_core_media_bug_set_read_demux_frame(_In_ switch_medi
 */
 SWITCH_DECLARE(switch_core_session_t *) switch_core_media_bug_get_session(_In_ switch_media_bug_t *bug);
 
+SWITCH_DECLARE(void) switch_core_media_bug_set_media_params(switch_media_bug_t *bug, switch_mm_t *mm);
+SWITCH_DECLARE(void) switch_core_media_bug_get_media_params(switch_media_bug_t *bug, switch_mm_t *mm);
+
 SWITCH_DECLARE(const char *) switch_core_media_bug_get_text(switch_media_bug_t *bug);
 
 /*!

src/include/switch_module_interfaces.h

@@ -309,41 +309,6 @@ struct switch_file_interface {
 	struct switch_file_interface *next;
 };
 
-typedef enum {
-	SWITCH_VIDEO_ENCODE_SPEED_DEFAULT = 0,
-	SWITCH_VIDEO_ENCODE_SPEED_FAST = 0,
-	SWITCH_VIDEO_ENCODE_SPEED_MEDIUM,
-	SWITCH_VIDEO_ENCODE_SPEED_SLOW
-} switch_video_encode_speed_t;
-
-typedef enum {
-	SWITCH_VIDEO_PROFILE_BASELINE,
-	SWITCH_VIDEO_PROFILE_MAIN,
-	SWITCH_VIDEO_PROFILE_HIGH
-} switch_video_profile_t;
-
-typedef struct switch_mm_s {
-	int samplerate;
-	int channels;
-	int keyint;
-	int ab;
-	int vb;
-	int vw;
-	int vh;
-	int cbr;
-	float fps;
-	float source_fps;
-	int vbuf;
-	switch_video_profile_t vprofile;
-	switch_video_encode_speed_t vencspd;
-	uint8_t try_hardware_encoder;
-	int scale_w;
-	int scale_h;
-	switch_img_fmt_t fmt;
-	char *auth_username;
-	char *auth_password;
-} switch_mm_t;
-
 /*! an abstract representation of a file handle (some parameters based on compat with libsndfile) */
 struct switch_file_handle {
 	/*! the interface of the module that implemented the current file type */

src/include/switch_types.h

@@ -2670,6 +2670,42 @@ typedef struct switch_agc_s switch_agc_t;
 struct switch_chromakey_s;
 typedef struct switch_chromakey_s switch_chromakey_t;
 
+typedef enum {
+	SWITCH_VIDEO_ENCODE_SPEED_DEFAULT = 0,
+	SWITCH_VIDEO_ENCODE_SPEED_FAST = 0,
+	SWITCH_VIDEO_ENCODE_SPEED_MEDIUM,
+	SWITCH_VIDEO_ENCODE_SPEED_SLOW
+} switch_video_encode_speed_t;
+
+typedef enum {
+	SWITCH_VIDEO_PROFILE_BASELINE,
+	SWITCH_VIDEO_PROFILE_MAIN,
+	SWITCH_VIDEO_PROFILE_HIGH
+} switch_video_profile_t;
+
+typedef struct switch_mm_s {
+	int samplerate;
+	int channels;
+	int keyint;
+	int ab;
+	int vb;
+	int vw;
+	int vh;
+	int cbr;
+	float fps;
+	float source_fps;
+	int vbuf;
+	switch_video_profile_t vprofile;
+	switch_video_encode_speed_t vencspd;
+	uint8_t try_hardware_encoder;
+	int scale_w;
+	int scale_h;
+	switch_img_fmt_t fmt;
+	char *auth_username;
+	char *auth_password;
+} switch_mm_t;
+
+
 SWITCH_END_EXTERN_C
 #endif
 /* For Emacs:

src/switch_core_media_bug.c

@@ -83,6 +83,16 @@ SWITCH_DECLARE(uint32_t) switch_core_media_bug_clear_flag(switch_media_bug_t *bu
 	return switch_clear_flag(bug, flag);
 }
 
+SWITCH_DECLARE(void) switch_core_media_bug_set_media_params(switch_media_bug_t *bug, switch_mm_t *mm)
+{
+	bug->mm = *mm;
+}
+
+SWITCH_DECLARE(void) switch_core_media_bug_get_media_params(switch_media_bug_t *bug, switch_mm_t *mm)
+{
+	*mm = bug->mm;
+}
+
 SWITCH_DECLARE(switch_core_session_t *) switch_core_media_bug_get_session(switch_media_bug_t *bug)
 {
 	return bug->session;
@@ -553,6 +563,9 @@ static void *SWITCH_THREAD_FUNC video_bug_thread(switch_thread_t *thread, void *
 	uint8_t *buf;
 	switch_size_t buflen = SWITCH_RTP_MAX_BUF_LEN;
 	switch_frame_t frame = { 0 };
+	switch_timer_t timer = { 0 };
+	switch_mm_t mm = { 0 };
+	int fps = 15;
 
 	buf = switch_core_session_alloc(bug->session, buflen);
 	frame.packet = buf;
@@ -573,65 +586,114 @@ static void *SWITCH_THREAD_FUNC video_bug_thread(switch_thread_t *thread, void *
 		return NULL;
 	}
 
+	switch_core_media_bug_get_media_params(bug, &mm);
+
+	if (mm.fps) {
+		fps = (int) mm.fps;
+	}
+
+	switch_core_timer_init(&timer, "soft", 1000 / fps, (90000 / (1000 / fps)), NULL);
+
 	while (bug->ready) {
 		switch_status_t status;
-		int w = 0, h = 0, ok = 1;
+		int w = 0, h = 0, ok = 1, new_main = 0, new_other = 0, new_canvas = 0;
+		
+		switch_core_timer_next(&timer);
+
+		if (!switch_channel_test_flag(bug->session->channel, CF_ANSWERED) && switch_core_media_bug_test_flag(bug, SMBF_ANSWER_REQ)) {
+			flush_video_queue(main_q, 0);
+			if (other_q) flush_video_queue(other_q, 0);
+			continue;
+		}
+
+		flush_video_queue(main_q, 1);
+
+		if ((status = switch_queue_trypop(main_q, &pop)) == SWITCH_STATUS_SUCCESS) {
+			switch_img_free(&img);
 
-		if ((status = switch_queue_pop(main_q, &pop)) == SWITCH_STATUS_SUCCESS) {
 			if (!pop) {
 				goto end;
 			}
 
 			img = (switch_image_t *) pop;
+			new_main = 1;
 
 			w = img->d_w;
 			h = img->d_h;
+		}
+		
+		if (other_q) {
+			flush_video_queue(other_q, 1);
 			
-			if (other_q) {
-				flush_video_queue(other_q, 1);
-
-				if ((status = switch_queue_trypop(other_q, &other_pop)) == SWITCH_STATUS_SUCCESS) {
-					if (!(other_img = (switch_image_t *) other_pop)) {
-						goto end;
-					}
-				}
+			if ((status = switch_queue_trypop(other_q, &other_pop)) == SWITCH_STATUS_SUCCESS) {
+				switch_img_free(&other_img);
+				other_img = (switch_image_t *) other_pop;
+				new_other = 1;
+			}
 
-				if (other_img) {
-					if (other_img->d_w != w || other_img->d_h != h) {
-						switch_image_t *tmp_img = NULL;
+			if (other_img) {
+				if (!w) w = other_img->d_w;
+				if (!h) h = other_img->d_h;
+				
+				if (other_img->d_w != w || other_img->d_h != h) {
+					switch_image_t *tmp_img = NULL;
 
-						switch_img_scale(other_img, &tmp_img, w, h);
-						switch_img_free(&other_img);
-						other_img = tmp_img;
-					}
+					switch_img_scale(other_img, &tmp_img, w, h);
+					switch_img_free(&other_img);
+					other_img = tmp_img;
 				}
+			}
 
-				w *= 2;
+			if (!(w&&h)) continue;
 
-				if (!IMG || IMG->d_h != h || IMG->d_w != w) {
-					switch_img_free(&IMG);
-					IMG = switch_img_alloc(NULL, SWITCH_IMG_FMT_I420, w, h, 1);
+			if (img) {
+				if (img->d_w != w || img->d_h != h) {
+					switch_image_t *tmp_img = NULL;
+					
+					switch_img_scale(img, &tmp_img, w, h);
+					switch_img_free(&img);
+					img = tmp_img;
 				}
+			}
+			
+			w *= 2;
 
-				switch_img_patch(IMG, img, 0, 0);
+			if (!IMG || IMG->d_h != h || IMG->d_w != w) {
+				switch_rgb_color_t color = { 0 };
 
-				if (other_img) {
-					switch_img_patch(IMG, other_img, w / 2, 0);
-					switch_img_free(&other_img);
-				}
+				switch_img_free(&IMG);
+				IMG = switch_img_alloc(NULL, SWITCH_IMG_FMT_I420, w, h, 1);
+				new_canvas = 1;
+				switch_color_set_rgb(&color, "#000000");
+				switch_img_fill(IMG, 0, 0, IMG->d_w, IMG->d_h, &color);
 			}
+		}
 
+		
+		if (IMG) {
+			if (img && (new_canvas || new_main)) {
+				switch_img_patch(IMG, img, 0, 0);
+			}
+		
+			if (other_img && (new_canvas || new_other)) {
+				switch_img_patch(IMG, other_img, w / 2, 0);
+			}
+		}
+		
+		if (IMG || img) {
 			switch_thread_rwlock_rdlock(bug->session->bug_rwlock);
 			frame.img = other_q ? IMG : img;
+
 			bug->video_ping_frame = &frame;
+			
 			if (bug->callback) {
 				if (bug->callback(bug, bug->user_data, SWITCH_ABC_TYPE_STREAM_VIDEO_PING) == SWITCH_FALSE
 					|| (bug->stop_time && bug->stop_time <= switch_epoch_time_now(NULL))) {
 					ok = SWITCH_FALSE;
 				}
 			}
+
 			bug->video_ping_frame = NULL;
-			switch_img_free(&img);
 			switch_thread_rwlock_unlock(bug->session->bug_rwlock);
 
 			if (!ok) {
@@ -643,6 +705,8 @@ static void *SWITCH_THREAD_FUNC video_bug_thread(switch_thread_t *thread, void *
 
  end:
 
+	switch_core_timer_destroy(&timer);
+	
 	switch_img_free(&IMG);
 	switch_img_free(&img);
 	switch_img_free(&other_img);

src/switch_ivr_async.c

@@ -1229,6 +1229,11 @@ static switch_bool_t record_callback(switch_media_bug_t *bug, void *user_data, s
 			rh->completion_cause = NULL;
 
 			switch_core_session_get_read_impl(session, &rh->read_impl);
+
+			if (switch_core_file_has_video(rh->fh, SWITCH_TRUE)) {
+				switch_core_media_bug_set_media_params(bug, &rh->fh->mm);
+			}
+
 		}
 		break;
 	case SWITCH_ABC_TYPE_TAP_NATIVE_READ:
@@ -2536,7 +2541,6 @@ SWITCH_DECLARE(switch_status_t) switch_ivr_record_session(switch_core_session_t
 		file_flags |= SWITCH_FILE_WRITE_APPEND;
 	}
 
-
 	fh->samplerate = 0;
 	if ((vval = switch_channel_get_variable(channel, "record_sample_rate"))) {
 		int tmp = 0;
@@ -2554,6 +2558,8 @@ SWITCH_DECLARE(switch_status_t) switch_ivr_record_session(switch_core_session_t
 
 	fh->channels = channels;
 
+
+
 	if ((vval = switch_channel_get_variable(channel, "enable_file_write_buffering"))) {
 		int tmp = atoi(vval);
 
@@ -2640,11 +2646,13 @@ SWITCH_DECLARE(switch_status_t) switch_ivr_record_session(switch_core_session_t
 			//switch_core_media_set_video_file(session, fh, SWITCH_RW_READ);
 			//switch_channel_set_flag_recursive(session->channel, CF_VIDEO_DECODED_READ);
 
-			if ((vval = switch_channel_get_variable(channel, "record_concat_video")) && switch_true(vval)) {
+			if (switch_channel_var_true(channel, "record_concat_video")) {
 				flags |= SMBF_READ_VIDEO_STREAM;
 				flags |= SMBF_WRITE_VIDEO_STREAM;
+			} else if (switch_channel_var_true(channel, "record_bleg_video")) {
+				flags |= SMBF_WRITE_VIDEO_STREAM;
 			} else {
-				flags |= SMBF_READ_VIDEO_PING;
+				flags |= SMBF_READ_VIDEO_STREAM;
 			}
 		} else {
 			flags &= ~SMBF_READ_VIDEO_PING;