FS-7513: refactor conference video muxing to create one distinct encoder per…

FS-7513: refactor conference video muxing to create one distinct encoder per codec used and only create one encoded frame per distinct codec, store current image used by layer on the layer so it is not destroyed before the canvas is written, refactor and rearrange some functions

src/include/switch_core.h

@@ -1283,6 +1283,9 @@ SWITCH_DECLARE(switch_status_t) switch_core_session_read_video_frame(_In_ switch
 SWITCH_DECLARE(switch_status_t) switch_core_session_write_video_frame(_In_ switch_core_session_t *session, switch_frame_t *frame, switch_io_flag_t flags,
 																	  int stream_id);
 
+SWITCH_DECLARE(switch_status_t) switch_core_session_write_encoded_video_frame(switch_core_session_t *session, 
+																		switch_frame_t *frame, switch_io_flag_t flags, int stream_id);
+
 SWITCH_DECLARE(switch_status_t) switch_core_session_set_read_impl(switch_core_session_t *session, const switch_codec_implementation_t *impp);
 SWITCH_DECLARE(switch_status_t) switch_core_session_set_write_impl(switch_core_session_t *session, const switch_codec_implementation_t *impp);
 SWITCH_DECLARE(switch_status_t) switch_core_session_set_video_read_impl(switch_core_session_t *session, const switch_codec_implementation_t *impp);

src/include/switch_core_video.h

@@ -143,6 +143,9 @@ SWITCH_DECLARE(int) switch_img_set_rect(switch_image_t  *img,
 				   unsigned int  w,
 				   unsigned int  h);
 
+
+SWITCH_DECLARE(void) switch_img_patch(switch_image_t *IMG, switch_image_t *img, int x, int y);
+
 /*!\brief Copy image to a new image
 *
 * if new_img is NULL, a new image is allocated

src/switch_core_media.c

@@ -7319,8 +7319,8 @@ SWITCH_DECLARE(void) switch_core_media_gen_local_sdp(switch_core_session_t *sess
 				}
 
 				/* DFF nack pli etc */
-				//nack = v_engine->nack = 0;
-				//pli = v_engine->pli = 0;
+				nack = v_engine->nack = 0;
+				pli = v_engine->pli = 0;
 				
 				
 				for (pmap = v_engine->cur_payload_map; pmap && pmap->allocated; pmap = pmap->next) {
@@ -8130,7 +8130,9 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_receive_message(switch_core_se
 			if (v_engine->rtp_session) {
 				if (switch_rtp_test_flag(v_engine->rtp_session, SWITCH_RTP_FLAG_PLI)) {
 					switch_rtp_video_loss(v_engine->rtp_session);
-				} else if (switch_rtp_test_flag(v_engine->rtp_session, SWITCH_RTP_FLAG_FIR)) {
+				}
+
+				if (switch_rtp_test_flag(v_engine->rtp_session, SWITCH_RTP_FLAG_FIR)) {
 					switch_rtp_video_refresh(v_engine->rtp_session);
 				}
 			}
@@ -9744,6 +9746,8 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_codec_control(switch_core_sess
 
 			smh->last_codec_refresh = now;
 		}
+
+		switch_channel_set_flag(session->channel, CF_VIDEO_REFRESH_REQ);
 		return switch_core_codec_control(codec, cmd, ctype, cmd_data, rtype, ret_data);
 	}
 
@@ -9751,7 +9755,8 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_codec_control(switch_core_sess
 }
 
 
-static switch_status_t raw_write_video(switch_core_session_t *session, switch_frame_t *frame, switch_io_flag_t flags, int stream_id) 
+SWITCH_DECLARE(switch_status_t) switch_core_session_write_encoded_video_frame(switch_core_session_t *session, 
+																		switch_frame_t *frame, switch_io_flag_t flags, int stream_id)
 {
 	switch_io_event_hook_video_write_frame_t *ptr;
 	switch_status_t status = SWITCH_STATUS_FALSE;
@@ -9835,7 +9840,7 @@ SWITCH_DECLARE(switch_status_t) switch_core_session_write_video_frame(switch_cor
 	}	
 
 	if (!img) {
-		return raw_write_video(session, frame, flags, stream_id);
+		return switch_core_session_write_encoded_video_frame(session, frame, flags, stream_id);
 	}
 
 	write_frame = *frame;
@@ -9849,7 +9854,7 @@ SWITCH_DECLARE(switch_status_t) switch_core_session_write_video_frame(switch_cor
 			if (!smh->video_timer.timer_interface) {
 				switch_core_timer_init(&smh->video_timer, "soft", 1, 90, switch_core_session_get_pool(session));
 			}
-			
+			switch_core_timer_sync(&smh->video_timer);
 			timer = &smh->video_timer;
 		}
 
@@ -9873,7 +9878,7 @@ SWITCH_DECLARE(switch_status_t) switch_core_session_write_video_frame(switch_cor
 			}
 			
 			switch_set_flag(frame, SFF_RAW_RTP_PARSE_FRAME);
-			status = raw_write_video(session, frame, flags, stream_id);
+			status = switch_core_session_write_encoded_video_frame(session, frame, flags, stream_id);
 		}
 		
 	} while(status == SWITCH_STATUS_SUCCESS && encode_status == SWITCH_STATUS_MORE_DATA);

src/switch_core_video.c

@@ -72,9 +72,41 @@ SWITCH_DECLARE(void) switch_img_free(switch_image_t **img)
 	}
 }
 
+// simple implementation to patch a small img to a big IMG at position x,y
+SWITCH_DECLARE(void) switch_img_patch(switch_image_t *IMG, switch_image_t *img, int x, int y)
+{
+	int i, j, k;
+	int W = IMG->d_w;
+	int H = IMG->d_h;
+	int w = img->d_w;
+	int h = img->d_h;
+
+	switch_assert(img->fmt == SWITCH_IMG_FMT_I420);
+	switch_assert(IMG->fmt == SWITCH_IMG_FMT_I420);
+
+	for (i = y; i < (y + h) && i < H; i++) {
+		for (j = x; j < (x + w) && j < W; j++) {
+			IMG->planes[0][i * IMG->stride[0] + j] = img->planes[0][(i - y) * img->stride[0] + (j - x)];
+		}
+	}
+
+	for (i = y; i < (y + h) && i < H; i+=4) {
+		for (j = x; j < (x + w) && j < W; j+=4) {
+			for (k = 1; k <= 2; k++) {
+				IMG->planes[k][i/2 * IMG->stride[k] + j/2]         = img->planes[k][(i-y)/2 * img->stride[k] + (j-x)/2];
+				IMG->planes[k][i/2 * IMG->stride[k] + j/2 + 1]     = img->planes[k][(i-y)/2 * img->stride[k] + (j-x)/2 + 1];
+				IMG->planes[k][(i+2)/2 * IMG->stride[k] + j/2]     = img->planes[k][(i+2-y)/2 * img->stride[k] + (j-x)/2];
+				IMG->planes[k][(i+2)/2 * IMG->stride[k] + j/2 + 1] = img->planes[k][(i+2-y)/2 * img->stride[k] + (j-x)/2 + 1];
+			}
+		}
+	}
+}
+
+
+
 SWITCH_DECLARE(void) switch_img_copy(switch_image_t *img, switch_image_t **new_img)
 {
-	int i;
+	int i = 0;
 
 	switch_assert(img);
 	switch_assert(new_img);
@@ -101,6 +133,7 @@ SWITCH_DECLARE(void) switch_img_copy(switch_image_t *img, switch_image_t **new_i
 		memcpy((*new_img)->planes[SWITCH_PLANE_U] + (*new_img)->stride[SWITCH_PLANE_U] * i, img->planes[SWITCH_PLANE_U] + img->stride[SWITCH_PLANE_U] * i, img->d_w / 2);
 		memcpy((*new_img)->planes[SWITCH_PLANE_V] + (*new_img)->stride[SWITCH_PLANE_V] * i, img->planes[SWITCH_PLANE_V] + img->stride[SWITCH_PLANE_V] * i, img->d_w /2);
 	}
+
 }
 
 /* For Emacs:

src/switch_rtp.c

@@ -5242,6 +5242,7 @@ static void handle_nack(switch_rtp_t *rtp_session, uint32_t nack)
 	switch_size_t bytes = 0;
 	rtp_msg_t send_msg[1] = {{{0}}};
 	uint16_t seq = (uint16_t) (nack & 0xFFFF);
+	uint16_t blp = (uint16_t) (nack >> 16);
 	int i;
 	const char *tx_host = NULL;
 	const char *old_host = NULL;
@@ -5278,10 +5279,11 @@ static void handle_nack(switch_rtp_t *rtp_session, uint32_t nack)
 		switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG, "Cannot send NACK for seq %u\n", ntohs(seq));
 	}
 
+	blp = ntohs(blp);
 	for (i = 0; i < 16; i++) {
-		if ((nack & (1 << (16 + i)))) {
-			switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG, "Also Got NACK for seq %u\n", ntohs(seq) + i);
-			if (switch_vb_get_packet_by_seq(rtp_session->vbw, htons(ntohs(seq) + i), (switch_rtp_packet_t *) &send_msg, &bytes) == SWITCH_STATUS_SUCCESS) {
+		if (blp & (1 << i)) {
+			switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG, "Also Got NACK for seq %u\n", ntohs(seq) + i + 1);
+			if (switch_vb_get_packet_by_seq(rtp_session->vbw, htons(ntohs(seq) + i + 1), (switch_rtp_packet_t *) &send_msg, &bytes) == SWITCH_STATUS_SUCCESS) {
 				if (rtp_session->flags[SWITCH_RTP_FLAG_DEBUG_RTP_WRITE]) {
 					switch_log_printf(SWITCH_CHANNEL_SESSION_LOG_CLEAN(rtp_session->session), SWITCH_LOG_CONSOLE,
 									  "X %s b=%4ld %s:%u %s:%u %s:%u pt=%d ts=%u seq=%u m=%d\n",

src/switch_vidderbuffer.c

@@ -383,6 +383,7 @@ SWITCH_DECLARE(uint32_t) switch_vb_pop_nack(switch_vb_t *vb)
 {
 	switch_hash_index_t *hi = NULL;
 	uint32_t nack = 0;
+	uint16_t blp = 0;
 	uint16_t least = 0;
 	int i = 0;
 
@@ -405,15 +406,18 @@ SWITCH_DECLARE(uint32_t) switch_vb_pop_nack(switch_vb_t *vb)
 	if (least && switch_core_inthash_delete(vb->missing_seq_hash, (uint32_t)htons(least))) {
 		vb_debug(vb, 3, "Found smallest NACKABLE seq %u\n", least);
 		nack = (uint32_t) htons(least);
-	
-		for (i = 1; i > 17; i++) {
+		
+		for(i = 0; i < 16; i++) {
 			if (switch_core_inthash_delete(vb->missing_seq_hash, (uint32_t)htons(least + i))) {
-				vb_debug(vb, 3, "Found addtl NACKABLE seq %u\n", least + i);
-				nack |= (1 << (16 + i));
+				vb_debug(vb, 3, "Found addtl NACKABLE seq %u\n", least + i + 1);
+				blp |= (1 << i);
 			} else {
 				break;
 			}
 		}
+
+		blp = htons(blp);
+		nack |= (uint32_t) blp << 16;
 	}
 	
 	switch_mutex_unlock(vb->mutex);