Skip to content

F
freeswitch
提交 b29174e8 authored 作者: Seven Du's avatar Seven Du 提交者： Michael Jerris
浏览文件
操作

FS-7503 FS-7519: refactor to support video playback using libavformat 

need -lswscale to converto from non-I420 fmt to I420
上级 a8a2c32a
显示空白字符变更
内嵌 并排
正在显示 包含 660 行增加 和 166 行删除
src/mod/formats/mod_avformat/Makefile.sample
浏览文件 @ b29174e8
LOCAL_LDFLAGS=-L/opt/av/lib -lavformat -lavcodec -lavutil -lavresample
LOCAL_LDFLAGS=-L/opt/av/lib -lavformat -lavcodec -lavutil -lavresample -lswscale
LOCAL_CFLAGS=-I/opt/av/include
LOCAL_LIBADD=
......
src/mod/formats/mod_avformat/mod_avformat.c
浏览文件 @ b29174e8
......@@ -38,6 +38,9 @@
#include <libavutil/channel_layout.h>
// #include <libavutil/timestamp.h>
#include <libavresample/avresample.h>
#include <libswscale/swscale.h>
#define SCALE_FLAGS SWS_BICUBIC
#define DFT_RECORD_OFFSET 350
SWITCH_MODULE_LOAD_FUNCTION(mod_avformat_load);
......@@ -69,27 +72,72 @@ static void __attribute__((unused)) fill_avframe(AVFrame *pict, switch_image_t *
}
}
static void __attribute__((unused)) avframe2img(AVFrame *pict, switch_image_t *img)
{
int i;
uint8_t *y = pict->data[0];
uint8_t *u = pict->data[1];
uint8_t *v = pict->data[2];
/* Y */
for (i = 0; i < img->d_h; i++) {
memcpy(&img->planes[0][i * img->stride[0]], y + i * pict->linesize[0], img->d_w);
}
/* U/V */
for(i = 0; i < pict->height / 2; i++) {
memcpy(&img->planes[1][i * img->stride[1]], u + i * pict->linesize[1], img->d_w / 2);
memcpy(&img->planes[2][i * img->stride[2]], v + i * pict->linesize[2], img->d_w / 2);
}
}
static void __attribute__((unused)) avframe2fd(AVFrame *pict, int fd)
{
int i;
uint8_t *y = pict->data[0];
uint8_t *u = pict->data[1];
uint8_t *v = pict->data[2];
/* Y */
for (i = 0; i < pict->height; i++) {
write(fd, y + i * pict->linesize[0], pict->width);
}
/* U/V */
for(i = 0; i < pict->height / 2; i++) {
write(fd, u + i * pict->linesize[1], pict->width / 2);
}
for(i = 0; i < pict->height / 2; i++) {
write(fd, v + i * pict->linesize[2], pict->width / 2);
}
}
/* App interface */
// a wrapper around a single output AVStream
typedef struct OutputStream {
typedef struct MediaStream {
AVStream *st;
AVFrame *frame;
AVFrame *tmp_frame;
int64_t next_pts;
struct AVAudioResampleContext *resample_ctx;
// audio
int channels;
int sample_rate;
struct AVAudioResampleContext *resample_ctx;
//video
int width;
int height;
} OutputStream;
struct SwsContext *sws_ctx;
int64_t next_pts;
} MediaStream;
typedef struct record_helper_s {
switch_mutex_t *mutex;
AVFormatContext *oc;
OutputStream *video_st;
AVFormatContext *fc;
MediaStream *video_st;
switch_timer_t *timer;
int in_callback;
switch_queue_t *video_queue;
......@@ -172,7 +220,7 @@ static int write_frame(AVFormatContext *fmt_ctx, const AVRational *time_base, AV
}
/* Add an output stream. */
static switch_status_t add_stream(OutputStream *ost, AVFormatContext *oc, AVCodec **codec, enum AVCodecID codec_id, switch_mm_t *mm)
static switch_status_t add_stream(MediaStream *mst, AVFormatContext *fc, AVCodec **codec, enum AVCodecID codec_id, switch_mm_t *mm)
{
AVCodecContext *c;
switch_status_t status = SWITCH_STATUS_FALSE;
......@@ -186,14 +234,14 @@ static switch_status_t add_stream(OutputStream *ost, AVFormatContext *oc, AVCode
return status;
}
ost->st = avformat_new_stream(oc, *codec);
if (!ost->st) {
mst->st = avformat_new_stream(fc, *codec);
if (!mst->st) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not allocate stream\n");
return status;
}
ost->st->id = oc->nb_streams - 1;
c = ost->st->codec;
// switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "id:%d den:%d num:%d\n", ost->st->id, ost->st->time_base.den, ost->st->time_base.num);
mst->st->id = fc->nb_streams - 1;
c = mst->st->codec;
// switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "id:%d den:%d num:%d\n", mst->st->id, mst->st->time_base.den, mst->st->time_base.num);
if (threads > 4) {
threads = 4;
......@@ -203,8 +251,8 @@ static switch_status_t add_stream(OutputStream *ost, AVFormatContext *oc, AVCode
case AVMEDIA_TYPE_AUDIO:
c->sample_fmt = (*codec)->sample_fmts ? (*codec)->sample_fmts[0] : AV_SAMPLE_FMT_FLTP;
c->bit_rate = 128000;
c->sample_rate = ost->sample_rate = 44100;
c->channels = ost->channels;
c->sample_rate = mst->sample_rate = 44100;
c->channels = mst->channels;
c->channel_layout = av_get_default_channel_layout(c->channels);
if (mm) {
......@@ -212,7 +260,7 @@ static switch_status_t add_stream(OutputStream *ost, AVFormatContext *oc, AVCode
c->bit_rate = mm->ab * 1024;
}
if (mm->samplerate) {
c->sample_rate = ost->sample_rate = mm->samplerate;
c->sample_rate = mst->sample_rate = mm->samplerate;
}
}
......@@ -229,13 +277,13 @@ static switch_status_t add_stream(OutputStream *ost, AVFormatContext *oc, AVCode
c->codec_id = codec_id;
c->bit_rate = 1000000;
/* Resolution must be a multiple of two. */
c->width = ost->width;
c->height = ost->height;
ost->st->time_base.den = 1000;
ost->st->time_base.num = 1;
c->width = mst->width;
c->height = mst->height;
mst->st->time_base.den = 1000;
mst->st->time_base.num = 1;
c->time_base.den = 1000;
c->time_base.num = 1;
c->gop_size = 25; /* emit one intra frame every x frames at most */
c->gop_size = 25; /* emit one intra frame every x frames at mmst */
c->pix_fmt = AV_PIX_FMT_YUV420P;
c->thread_count = threads;
c->rc_initial_buffer_occupancy = buffer_bytes * 8;
......@@ -263,7 +311,7 @@ static switch_status_t add_stream(OutputStream *ost, AVFormatContext *oc, AVCode
}
/* Some formats want stream headers to be separate. */
if (oc->oformat->flags & AVFMT_GLOBALHEADER) {
if (fc->oformat->flags & AVFMT_GLOBALHEADER) {
c->flags |= CODEC_FLAG_GLOBAL_HEADER;
}
......@@ -292,10 +340,10 @@ static AVFrame *alloc_picture(enum AVPixelFormat pix_fmt, int width, int height)
return picture;
}
static switch_status_t open_video(AVFormatContext *oc, AVCodec *codec, OutputStream *ost)
static switch_status_t open_video(AVFormatContext *fc, AVCodec *codec, MediaStream *mst)
{
int ret;
AVCodecContext *c = ost->st->codec;
AVCodecContext *c = mst->st->codec;
switch_status_t status = SWITCH_STATUS_FALSE;
/* open the codec */
......@@ -306,8 +354,8 @@ static switch_status_t open_video(AVFormatContext *oc, AVCodec *codec, OutputStr
}
/* allocate and init a re-usable frame */
ost->frame = alloc_picture(c->pix_fmt, c->width, c->height);
switch_assert(ost->frame);
mst->frame = alloc_picture(c->pix_fmt, c->width, c->height);
switch_assert(mst->frame);
// switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "pix_fmt: %d\n", c->pix_fmt);
switch_assert(c->pix_fmt == AV_PIX_FMT_YUV420P); // always I420 for NOW
......@@ -315,13 +363,13 @@ static switch_status_t open_video(AVFormatContext *oc, AVCodec *codec, OutputStr
return SWITCH_STATUS_SUCCESS;
}
static switch_status_t open_audio(AVFormatContext *oc, AVCodec *codec, OutputStream *ost)
static switch_status_t open_audio(AVFormatContext *fc, AVCodec *codec, MediaStream *mst)
{
AVCodecContext *c;
int ret;
switch_status_t status = SWITCH_STATUS_FALSE;
c = ost->st->codec;
c = mst->st->codec;
ret = avcodec_open2(c, codec, NULL);
if (ret < 0) {
......@@ -329,64 +377,64 @@ static switch_status_t open_audio(AVFormatContext *oc, AVCodec *codec, OutputStr
return status;
}
ost->frame = av_frame_alloc();
switch_assert(ost->frame);
mst->frame = av_frame_alloc();
switch_assert(mst->frame);
ost->frame->sample_rate = c->sample_rate;
ost->frame->format = AV_SAMPLE_FMT_S16;
ost->frame->channel_layout = c->channel_layout;
mst->frame->sample_rate = c->sample_rate;
mst->frame->format = AV_SAMPLE_FMT_S16;
mst->frame->channel_layout = c->channel_layout;
if (c->codec->capabilities & CODEC_CAP_VARIABLE_FRAME_SIZE) {
ost->frame->nb_samples = 10000;
mst->frame->nb_samples = 10000;
} else {
ost->frame->nb_samples = c->frame_size;
mst->frame->nb_samples = c->frame_size;
}
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "sample_rate: %d nb_samples: %d\n", ost->frame->sample_rate, ost->frame->nb_samples);
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "sample_rate: %d nb_samples: %d\n", mst->frame->sample_rate, mst->frame->nb_samples);
if (c->sample_fmt != AV_SAMPLE_FMT_S16) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "sample_fmt %d != AV_SAMPLE_FMT_S16, start resampler\n", c->sample_fmt);
ost->resample_ctx = avresample_alloc_context();
mst->resample_ctx = avresample_alloc_context();
if (!ost->resample_ctx) {
if (!mst->resample_ctx) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not allocate resampler context\n");
return status;
}
/* set options */
av_opt_set_int(ost->resample_ctx, "in_channel_count", c->channels, 0);
av_opt_set_int(ost->resample_ctx, "in_sample_rate", c->sample_rate, 0);
av_opt_set_int(ost->resample_ctx, "in_sample_fmt", AV_SAMPLE_FMT_S16, 0);
av_opt_set_int(ost->resample_ctx, "in_channel_layout", c->channel_layout, 0);
av_opt_set_int(ost->resample_ctx, "out_channel_count", c->channels, 0);
av_opt_set_int(ost->resample_ctx, "out_sample_rate", c->sample_rate, 0);
av_opt_set_int(ost->resample_ctx, "out_sample_fmt", c->sample_fmt, 0);
av_opt_set_int(ost->resample_ctx, "out_channel_layout", c->channel_layout, 0);
if ((ret = avresample_open(ost->resample_ctx)) < 0) {
av_opt_set_int(mst->resample_ctx, "in_channel_count", c->channels, 0);
av_opt_set_int(mst->resample_ctx, "in_sample_rate", c->sample_rate, 0);
av_opt_set_int(mst->resample_ctx, "in_sample_fmt", AV_SAMPLE_FMT_S16, 0);
av_opt_set_int(mst->resample_ctx, "in_channel_layout", c->channel_layout, 0);
av_opt_set_int(mst->resample_ctx, "out_channel_count", c->channels, 0);
av_opt_set_int(mst->resample_ctx, "out_sample_rate", c->sample_rate, 0);
av_opt_set_int(mst->resample_ctx, "out_sample_fmt", c->sample_fmt, 0);
av_opt_set_int(mst->resample_ctx, "out_channel_layout", c->channel_layout, 0);
if ((ret = avresample_open(mst->resample_ctx)) < 0) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Failed to initialize the resampling context\n");
return status;
}
}
ret = av_frame_get_buffer(ost->frame, 0);
ret = av_frame_get_buffer(mst->frame, 0);
if (ret < 0) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not allocate audio frame.\n");
return status;
}
if (ost->resample_ctx) {
ost->tmp_frame = av_frame_alloc();
switch_assert(ost->tmp_frame);
if (mst->resample_ctx) {
mst->tmp_frame = av_frame_alloc();
switch_assert(mst->tmp_frame);
ost->tmp_frame->sample_rate = c->sample_rate;
ost->tmp_frame->format = c->sample_fmt;
ost->tmp_frame->channel_layout = c->channel_layout;
ost->tmp_frame->nb_samples = ost->frame->nb_samples;
mst->tmp_frame->sample_rate = c->sample_rate;
mst->tmp_frame->format = c->sample_fmt;
mst->tmp_frame->channel_layout = c->channel_layout;
mst->tmp_frame->nb_samples = mst->frame->nb_samples;
ret = av_frame_get_buffer(ost->tmp_frame, 0);
ret = av_frame_get_buffer(mst->tmp_frame, 0);
if (ret < 0) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not allocate audio frame.\n");
return status;
......@@ -473,7 +521,7 @@ static void *SWITCH_THREAD_FUNC video_thread_run(switch_thread_t *thread, void *
if (got_packet) {
switch_mutex_lock(eh->mutex);
ret = write_frame(eh->oc, &eh->video_st->st->codec->time_base, eh->video_st->st, &pkt);
ret = write_frame(eh->fc, &eh->video_st->st->codec->time_base, eh->video_st->st, &pkt);
switch_mutex_unlock(eh->mutex);
av_free_packet(&pkt);
}
......@@ -509,11 +557,14 @@ static switch_status_t video_read_callback(switch_core_session_t *session, switc
return SWITCH_STATUS_SUCCESS;;
}
static void close_stream(AVFormatContext *oc, OutputStream *ost)
static void close_stream(AVFormatContext *fc, MediaStream *mst)
{
avcodec_close(ost->st->codec);
av_frame_free(&ost->frame);
if (ost->tmp_frame) av_frame_free(&ost->tmp_frame);
if (mst->resample_ctx) avresample_close(mst->resample_ctx);
if (mst->sws_ctx) sws_freeContext(mst->sws_ctx);
if (mst->frame) av_frame_free(&mst->frame);
if (mst->tmp_frame) av_frame_free(&mst->tmp_frame);
avcodec_close(mst->st->codec);
}
SWITCH_STANDARD_APP(record_av_function)
......@@ -532,10 +583,10 @@ SWITCH_STANDARD_APP(record_av_function)
switch_vid_params_t vid_params = { 0 };
int force_sample_rate;
OutputStream video_st = { 0 }, audio_st = { 0 };
MediaStream video_st = { 0 }, audio_st = { 0 };
AVOutputFormat *fmt = NULL;
const char *format = NULL;
AVFormatContext *oc = NULL;
AVFormatContext *fc = NULL;
AVCodec *audio_codec, *video_codec;
int has_audio = 0, has_video = 0;
int ret;
......@@ -582,18 +633,18 @@ SWITCH_STANDARD_APP(record_av_function)
switch_buffer_create_dynamic(&buffer, 8192, 65536, 0);
av_register_all();
avformat_alloc_output_context2(&oc, NULL, format, data);
avformat_alloc_output_context2(&fc, NULL, format, data);
if (!oc) {
if (!fc) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "Could not deduce output format from file extension\n");
goto end;
}
fmt = oc->oformat;
fmt = fc->oformat;
/* open the output file, if needed */
if (!(fmt->flags & AVFMT_NOFILE)) {
ret = avio_open(&oc->pb, data, AVIO_FLAG_WRITE);
ret = avio_open(&fc->pb, data, AVIO_FLAG_WRITE);
if (ret < 0) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not open '%s': %s\n", data, get_error_text(ret));
goto end;
......@@ -621,8 +672,8 @@ SWITCH_STANDARD_APP(record_av_function)
video_st.width = vid_params.width;
video_st.height = vid_params.height;
video_st.next_pts = switch_time_now() / 1000;
if (add_stream(&video_st, oc, &video_codec, fmt->video_codec, NULL) == SWITCH_STATUS_SUCCESS &&
open_video(oc, video_codec, &video_st) == SWITCH_STATUS_SUCCESS) {
if (add_stream(&video_st, fc, &video_codec, fmt->video_codec, NULL) == SWITCH_STATUS_SUCCESS &&
open_video(fc, video_codec, &video_st) == SWITCH_STATUS_SUCCESS) {
avcodec_string(codec_str, sizeof(codec_str), video_st.st->codec, 1);
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "use video codec implementation %s\n", codec_str);
has_video = 1;
......@@ -633,18 +684,18 @@ SWITCH_STANDARD_APP(record_av_function)
audio_st.channels = read_impl.number_of_channels;
audio_st.sample_rate = force_sample_rate;
add_stream(&audio_st, oc, &audio_codec, fmt->audio_codec, NULL);
if (open_audio(oc, audio_codec, &audio_st) != SWITCH_STATUS_SUCCESS) {
add_stream(&audio_st, fc, &audio_codec, fmt->audio_codec, NULL);
if (open_audio(fc, audio_codec, &audio_st) != SWITCH_STATUS_SUCCESS) {
goto end;
}
has_audio = 1;
}
av_dump_format(oc, 0, data, 1);
av_dump_format(fc, 0, data, 1);
/* Write the stream header, if any. */
ret = avformat_write_header(oc, NULL);
ret = avformat_write_header(fc, NULL);
if (ret < 0) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error occurred when opening output file: %s\n", get_error_text(ret));
goto end;
......@@ -656,7 +707,7 @@ SWITCH_STANDARD_APP(record_av_function)
switch_mutex_init(&mutex, SWITCH_MUTEX_NESTED, switch_core_session_get_pool(session));
eh.mutex = mutex;
eh.video_st = &video_st;
eh.oc = oc;
eh.fc = fc;
if (switch_core_timer_init(&timer, "soft", 1, 1, switch_core_session_get_pool(session)) != SWITCH_STATUS_SUCCESS) {
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "Timer Activation Fail\n");
goto end;
......@@ -776,7 +827,7 @@ SWITCH_STANDARD_APP(record_av_function)
if (got_packet) {
// switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "got pkt: %d\n", pkt.size);
ret = write_frame(oc, &audio_st.st->codec->time_base, audio_st.st, &pkt);
ret = write_frame(fc, &audio_st.st->codec->time_base, audio_st.st, &pkt);
if (ret < 0) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error while writing audio frame: %s\n", get_error_text(ret));
goto end;
......@@ -812,31 +863,31 @@ SWITCH_STANDARD_APP(record_av_function)
}
if (got_packet) {
ret = write_frame(oc, &video_st.st->codec->time_base, video_st.st, &pkt);
ret = write_frame(fc, &video_st.st->codec->time_base, video_st.st, &pkt);
av_free_packet(&pkt);
goto again;
}
}
av_write_trailer(oc);
av_write_trailer(fc);
switch_channel_set_variable(channel, SWITCH_CURRENT_APPLICATION_RESPONSE_VARIABLE, "OK");
end:
if (oc) {
if (has_video) close_stream(oc, &video_st);
if (has_audio) close_stream(oc, &audio_st);
if (fc) {
if (has_video) close_stream(fc, &video_st);
if (has_audio) close_stream(fc, &audio_st);
if (fmt) {
if (!(fmt->flags & AVFMT_NOFILE)) {
avio_close(oc->pb);
avio_close(fc->pb);
} else {
avformat_network_deinit();
}
}
/* free the stream */
avformat_free_context(oc);
avformat_free_context(fc);
}
if (timer.interval) {
......@@ -1104,9 +1155,9 @@ struct av_file_context {
int vid_ready;
int audio_ready;
OutputStream video_st;
OutputStream audio_st;
AVFormatContext *oc;
MediaStream video_st;
MediaStream audio_st;
AVFormatContext *fc;
AVCodec *audio_codec;
AVCodec *video_codec;
......@@ -1114,10 +1165,293 @@ struct av_file_context {
int has_video;
record_helper_t eh;
switch_thread_t *file_read_thread;
int file_read_thread_running;
switch_time_t video_start_time;
};
typedef struct av_file_context av_file_context_t;
static switch_status_t open_input_file(av_file_context_t *context, switch_file_handle_t *handle, const char *filename)
{
AVCodec *audio_codec;
AVCodec *video_codec;
int error;
int i;
switch_status_t status = SWITCH_STATUS_SUCCESS;
/** Open the input file to read from it. */
if ((error = avformat_open_input(&context->fc, filename, NULL,
NULL)) < 0) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not open input file '%s' (error '%s')\n", filename, get_error_text(error));
switch_goto_status(SWITCH_STATUS_FALSE, err);
}
/** Get information on the input file (number of streams etc.). */
if ((error = avformat_find_stream_info(context->fc, NULL)) < 0) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not open find stream info (error '%s')\n", get_error_text(error));
switch_goto_status(SWITCH_STATUS_FALSE, err);
}
av_dump_format(context->fc, 0, filename, 0);
for (i = 0; i< context->fc->nb_streams; i++) {
if (context->fc->streams[i]->codec->codec_type == AVMEDIA_TYPE_AUDIO && !context->has_audio) {
context->audio_st.st = context->fc->streams[i];
context->has_audio = 1;
} else if (context->fc->streams[i]->codec->codec_type == AVMEDIA_TYPE_VIDEO && !context->has_video) {
context->video_st.st = context->fc->streams[i];
context->has_video = 1;
}
}
/** Find a decoder for the audio stream. */
if (context->has_audio && !(audio_codec = avcodec_find_decoder(context->audio_st.st->codec->codec_id))) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "Could not find input codec %d\n", context->audio_st.st->codec->codec_id);
context->has_audio = 0;
}
if (context->has_video && !(video_codec = avcodec_find_decoder(context->video_st.st->codec->codec_id))) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not find input codec %d\n", context->video_st.st->codec->codec_id);
context->has_video = 0;
}
if (context->has_audio && (error = avcodec_open2(context->audio_st.st->codec, audio_codec, NULL)) < 0) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not open input codec (error '%s')\n", get_error_text(error));
context->has_audio = 0;
}
if (context->has_video && (error = avcodec_open2(context->video_st.st->codec, video_codec, NULL)) < 0) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not open input codec (error '%s')\n", get_error_text(error));
context->has_video = 0;
}
// printf("has audio:%d has_video:%d\n", context->has_audio, context->has_video);
if ((!context->has_audio) && (!context->has_video)) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Neither audio nor video stream found in file %s\n", filename);
switch_goto_status(SWITCH_STATUS_FALSE, err);
}
if (context->has_audio) {
context->audio_st.frame = av_frame_alloc();
switch_assert(context->audio_st.frame);
AVCodecContext *c = context->audio_st.st->codec;
handle->channels = c->channels > 2 ? 2 : c->channels;
context->audio_st.channels = handle->channels;
context->audio_st.sample_rate = handle->samplerate;
if (context->audio_st.st->codec->sample_fmt != AV_SAMPLE_FMT_S16) {
AVAudioResampleContext *resample_ctx = avresample_alloc_context();
if (resample_ctx) {
int ret;
av_opt_set_int(resample_ctx, "in_channel_count", c->channels, 0);
av_opt_set_int(resample_ctx, "in_sample_rate", c->sample_rate, 0);
av_opt_set_int(resample_ctx, "in_sample_fmt", c->sample_fmt, 0);
av_opt_set_int(resample_ctx, "in_channel_layout", c->channel_layout, 0);
av_opt_set_int(resample_ctx, "out_channel_count", handle->channels, 0);
av_opt_set_int(resample_ctx, "out_sample_rate", handle->samplerate,0);
av_opt_set_int(resample_ctx, "out_sample_fmt", AV_SAMPLE_FMT_S16, 0);
av_opt_set_int(resample_ctx, "out_channel_layout", handle->channels == 2 ? AV_CH_LAYOUT_STEREO : AV_CH_LAYOUT_MONO, 0);
if ((ret = avresample_open(resample_ctx)) < 0) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Failed to initialize the resampling context\n");
av_free(resample_ctx);
switch_goto_status(SWITCH_STATUS_FALSE, err);
}
context->audio_st.resample_ctx = resample_ctx;
}
}
}
return status;
err:
if (context->fc) avformat_close_input(&context->fc);
return status;
}
static void *SWITCH_THREAD_FUNC file_read_thread_run(switch_thread_t *thread, void *obj)
{
av_file_context_t *context = (av_file_context_t *) obj;
AVPacket pkt = { 0 };
int got_data = 0;
int error;
context->file_read_thread_running = 1;
#define AUDIO_BUF_SEC 5
while (context->file_read_thread_running) {
if (switch_buffer_inuse(context->audio_buffer) > AUDIO_BUF_SEC * context->audio_st.sample_rate * context->audio_st.channels * 2) {
switch_yield(100000);
continue;
}
av_init_packet(&pkt);
pkt.data = NULL;
pkt.size = 0;
if ((error = av_read_frame(context->fc, &pkt)) < 0) {
if (error == AVERROR_EOF) break;
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not read frame (error '%s')\n", get_error_text(error));
break;
}
// switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "stream: %d, pkt size %d\n", pkt.stream_index, pkt.size);
if (context->has_video && pkt.stream_index == context->video_st.st->index) {
AVFrame *vframe = av_frame_alloc();
switch_image_t *img;
switch_assert(vframe);
if ((error = avcodec_decode_video2(context->video_st.st->codec, vframe, &got_data, &pkt)) < 0) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not decode frame (error '%s')\n", get_error_text(error));
av_free_packet(&pkt);
av_frame_free(&vframe);
break;
}
// switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "pkt: %d, pts: %lld dts: %lld\n", pkt.size, pkt.pts, pkt.dts);
av_free_packet(&pkt);
if (switch_queue_size(context->eh.video_queue) > 300) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "Dropping frames\n");
av_frame_free(&vframe);
continue;
}
if (got_data && error > 0) {
// switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "got picture %dx%d fmt: %d pktpts:%lld pktdts:%lld\n", vframe->width, vframe->height, vframe->format, vframe->pkt_pts, vframe->pkt_dts);
if (vframe->format != AV_PIX_FMT_YUV420P) {
AVFrame *frm = vframe;
int ret;
if (!context->video_st.sws_ctx) {
context->video_st.sws_ctx =
sws_getContext(frm->width, frm->height,
frm->format,
frm->width, frm->height,
AV_PIX_FMT_YUV420P,
SCALE_FLAGS, NULL, NULL, NULL);
if (!context->video_st.sws_ctx) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "Cannot init sws context\n");
av_frame_free(&frm);
continue;
}
}
vframe = av_frame_alloc();
switch_assert(vframe);
vframe->format = AV_PIX_FMT_YUV420P;
vframe->width = frm->width;
vframe->height = frm->height;
vframe->pts = frm->pts;
vframe->pkt_pts = frm->pkt_pts;
vframe->pkt_dts = frm->pkt_dts;
ret = av_frame_get_buffer(vframe, 32);
switch_assert(ret >= 0);
ret = sws_scale(context->video_st.sws_ctx, (const uint8_t *const *)frm->data, frm->linesize,
0, frm->height, vframe->data, vframe->linesize);
av_frame_free(&frm);
if (ret <= 0 ) {
av_frame_free(&vframe);
continue;
}
}
img = switch_img_alloc(NULL, SWITCH_IMG_FMT_I420, vframe->width, vframe->height, 1);
if (img) {
uint64_t *pts = malloc(sizeof(uint64_t));
if (pts) {
*pts = vframe->pkt_pts;
avframe2img(vframe, img);
img->user_priv = pts;
switch_queue_push(context->eh.video_queue, img);
}
}
av_frame_free(&vframe);
}
continue;
} else if (context->has_audio && pkt.stream_index == context->audio_st.st->index) {
AVFrame in_frame = { { 0 } };
if ((error = avcodec_decode_audio4(context->audio_st.st->codec, &in_frame, &got_data, &pkt)) < 0) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not decode frame (error '%s')\n", get_error_text(error));
av_free_packet(&pkt);
break;
}
// switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "pkt: %d, decodedddd: %d pts: %lld dts: %lld\n", pkt.size, error, pkt.pts, pkt.dts);
av_free_packet(&pkt);
if (got_data) {
// switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "got data frm->format: %d samples: %d\n", in_frame.format, in_frame.nb_samples);
if (context->audio_st.resample_ctx) {
int out_samples = avresample_get_out_samples(context->audio_st.resample_ctx, in_frame.nb_samples);
int ret;
uint8_t *data[2] = { 0 };
data[0] = malloc(out_samples * context->audio_st.channels * 2);
switch_assert(data[0]);
ret = avresample_convert(context->audio_st.resample_ctx, data, 0, out_samples,
in_frame.data, 0, in_frame.nb_samples);
// switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "out_samples: %d ret: %d delay: %d buffer: %zu\n", out_samples, ret, avresample_get_delay(context->audio_st.resample_ctx), switch_buffer_inuse(context->audio_buffer));
if (ret) {
switch_mutex_lock(context->mutex);
switch_buffer_write(context->audio_buffer, data[0], ret * 2 * context->audio_st.channels);
switch_mutex_unlock(context->mutex);
}
// switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "decoded samples: %d\n", ret);
free(data[0]);
// if (ret == 0 && avresample_get_delay(context->audio_st.resample_ctx)) {
// frameP = NULL;
// goto again;
// }
} else {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "this block is not tested samples: %d\n", in_frame.nb_samples);
switch_mutex_lock(context->mutex);
switch_buffer_write(context->audio_buffer, in_frame.data[0], in_frame.nb_samples * 2 * context->audio_st.channels);
switch_mutex_unlock(context->mutex);
}
}
}
}
if (context->has_video) switch_queue_push(context->eh.video_queue, NULL);
context->file_read_thread_running = 0;
return NULL;
}
static switch_status_t av_file_open(switch_file_handle_t *handle, const char *path)
{
av_file_context_t *context;
......@@ -1131,10 +1465,6 @@ static switch_status_t av_file_open(switch_file_handle_t *handle, const char *pa
switch_set_string(file, path);
if (switch_test_flag(handle, SWITCH_FILE_FLAG_READ)) {
return SWITCH_STATUS_FALSE;
}
if ((ext = strrchr((char *)path, '.')) == 0) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Invalid Format\n");
return SWITCH_STATUS_GENERR;
......@@ -1178,18 +1508,45 @@ static switch_status_t av_file_open(switch_file_handle_t *handle, const char *pa
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "sample rate: %d, channels: %d\n", handle->samplerate, handle->channels);
av_register_all();
avformat_alloc_output_context2(&context->oc, NULL, format, (char *)file);
if (!context->oc) {
if (switch_test_flag(handle, SWITCH_FILE_FLAG_READ)) {
if (open_input_file(context, handle, path) != SWITCH_STATUS_SUCCESS) {
//clean up;
return SWITCH_STATUS_GENERR;
}
handle->private_info = context;
context->pool = handle->memory_pool;
if (context->has_video) {
switch_queue_create(&context->eh.video_queue, SWITCH_CORE_QUEUE_LEN, handle->memory_pool);
switch_mutex_init(&context->eh.mutex, SWITCH_MUTEX_NESTED, handle->memory_pool);
}
{
switch_threadattr_t *thd_attr = NULL;
switch_threadattr_create(&thd_attr, handle->memory_pool);
switch_threadattr_stacksize_set(thd_attr, SWITCH_THREAD_STACKSIZE);
switch_thread_create(&context->file_read_thread, thd_attr, file_read_thread_run, context, handle->memory_pool);
}
return SWITCH_STATUS_SUCCESS;
}
avformat_alloc_output_context2(&context->fc, NULL, format, (char *)file);
if (!context->fc) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "Could not deduce output format from file extension\n");
goto end;
}
fmt = context->oc->oformat;
fmt = context->fc->oformat;
/* open the output file, if needed */
if (!(fmt->flags & AVFMT_NOFILE)) {
ret = avio_open(&context->oc->pb, file, AVIO_FLAG_WRITE);
ret = avio_open(&context->fc->pb, file, AVIO_FLAG_WRITE);
if (ret < 0) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not open '%s': %s\n", file, get_error_text(ret));
goto end;
......@@ -1256,15 +1613,15 @@ static switch_status_t av_file_open(switch_file_handle_t *handle, const char *pa
context->audio_st.channels = handle->channels;
context->audio_st.sample_rate = handle->samplerate;
add_stream(&context->audio_st, context->oc, &context->audio_codec, fmt->audio_codec, &handle->mm);
if (open_audio(context->oc, context->audio_codec, &context->audio_st) != SWITCH_STATUS_SUCCESS) {
add_stream(&context->audio_st, context->fc, &context->audio_codec, fmt->audio_codec, &handle->mm);
if (open_audio(context->fc, context->audio_codec, &context->audio_st) != SWITCH_STATUS_SUCCESS) {
goto end;
}
context->has_audio = 1;
}
av_dump_format(context->oc, 0, file, 1);
av_dump_format(context->fc, 0, file, 1);
handle->format = 0;
handle->sections = 0;
......@@ -1302,20 +1659,27 @@ static switch_status_t av_file_close(switch_file_handle_t *handle)
switch_thread_join(&status, context->eh.video_thread);
}
if (context->oc) {
av_write_trailer(context->oc);
if (context->file_read_thread_running && context->file_read_thread) {
context->file_read_thread_running = 0;
switch_thread_join(&status, context->file_read_thread);
}
if (context->fc) {
if (switch_test_flag(handle, SWITCH_FILE_FLAG_WRITE)) av_write_trailer(context->fc);
if (context->has_video) close_stream(context->oc, &context->video_st);
if (context->has_audio) close_stream(context->oc, &context->audio_st);
if (context->has_video) close_stream(context->fc, &context->video_st);
if (context->has_audio) close_stream(context->fc, &context->audio_st);
if (!(context->oc->oformat->flags & AVFMT_NOFILE)) {
avio_close(context->oc->pb);
if (context->fc->oformat) {
if (!(context->fc->oformat->flags & AVFMT_NOFILE)) {
avio_close(context->fc->pb);//todo
} else {
avformat_network_deinit();
}
}
/* free the stream */
avformat_free_context(context->oc);
avformat_free_context(context->fc);
}
if (context->timer.interval) {
......@@ -1335,8 +1699,38 @@ static switch_status_t av_file_seek(switch_file_handle_t *handle, unsigned int *
static switch_status_t av_file_read(switch_file_handle_t *handle, void *data, size_t *len)
{
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "read not implemented\n");
av_file_context_t *context = (av_file_context_t *)handle->private_info;
int size;
if (!context->has_audio && context->has_video && switch_queue_size(context->eh.video_queue) > 0) {
memset(data, 0, *len * handle->channels * 2);
return SWITCH_STATUS_SUCCESS;
}
again:
if (!context->file_read_thread_running && switch_buffer_inuse(context->audio_buffer) == 0) {
*len = 0;
return SWITCH_STATUS_FALSE;
}
switch_mutex_lock(context->mutex);
size = switch_buffer_inuse(context->audio_buffer);
if (size > *len * context->audio_st.channels * 2) size = *len * context->audio_st.channels * 2;
if (size) size = switch_buffer_read(context->audio_buffer, data, size);
switch_mutex_unlock(context->mutex);
if (size == 0) {
switch_yield(20000);
goto again;
}
*len = size / context->audio_st.channels / 2;
handle->pos += *len;
handle->sample_count += *len;
return *len == 0 ? SWITCH_STATUS_FALSE : SWITCH_STATUS_SUCCESS;
}
static switch_status_t av_file_write(switch_file_handle_t *handle, void *data, size_t *len)
......@@ -1419,7 +1813,7 @@ static switch_status_t av_file_write(switch_file_handle_t *handle, void *data, s
if (got_packet) {
if (context->mutex) switch_mutex_lock(context->mutex);
ret = write_frame(context->oc, &context->audio_st.st->codec->time_base, context->audio_st.st, &pkt);
ret = write_frame(context->fc, &context->audio_st.st->codec->time_base, context->audio_st.st, &pkt);
if (context->mutex) switch_mutex_unlock(context->mutex);
if (ret < 0) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error while writing audio frame: %s\n", get_error_text(ret));
......@@ -1438,7 +1832,106 @@ end:
static switch_status_t av_file_read_video(switch_file_handle_t *handle, switch_frame_t *frame, switch_video_read_flag_t flags)
{
av_file_context_t *context = (av_file_context_t *)handle->private_info;
void *pop;
MediaStream *mst = &context->video_st;
AVStream *st = mst->st;
int ticks = 0;
switch_status_t status = SWITCH_STATUS_SUCCESS;
if (!context->has_video) return SWITCH_STATUS_FALSE;
if (!context->file_read_thread_running && switch_queue_size(context->eh.video_queue) == 0) {
return SWITCH_STATUS_FALSE;
}
if (flags & SVR_FLUSH) {
while(switch_queue_size(context->eh.video_queue) > 1) {
if (switch_queue_trypop(context->eh.video_queue, &pop) == SWITCH_STATUS_SUCCESS) {
if (pop) {
switch_image_t *img = (switch_image_t *)pop;
switch_img_free(&img);
}
}
}
return SWITCH_STATUS_BREAK;
}
if (st->codec->time_base.num) {
ticks = st->parser ? st->parser->repeat_pict + 1 : st->codec->ticks_per_frame;
// mst->next_pts += ((int64_t)AV_TIME_BASE * st->codec->time_base.num * ticks) / st->codec->time_base.den;
}
if (!context->video_start_time) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "start: %lld ticks: %d ticks_per_frame: %d st num:%d st den:%d codec num:%d codec den:%d start: %lld, duration:%lld nb_frames:%lld q2d:%f\n",
context->video_start_time, ticks, st->codec->ticks_per_frame, st->time_base.num, st->time_base.den, st->codec->time_base.num, st->codec->time_base.den,
st->start_time, st->duration, st->nb_frames, av_q2d(st->time_base));
}
again: if (0) goto again;
if (flags & SVR_BLOCK) {
status = switch_queue_pop(context->eh.video_queue, &pop);
} else {
status = switch_queue_trypop(context->eh.video_queue, &pop);
}
if (pop && status == SWITCH_STATUS_SUCCESS) {
switch_image_t *img = (switch_image_t *)pop;
// #define YIELD 60000 // use a constant FPS
#ifdef YIELD
switch_yield(YIELD);
#else
uint64_t pts;
uint64_t now = switch_micro_time_now();
pts = av_rescale_q(*((uint64_t *)img->user_priv), st->time_base, AV_TIME_BASE_Q);
if (!context->video_start_time) {
context->video_start_time = now - pts;
}
if (st->time_base.num == 0) {
mst->next_pts = 0;
} else {
// uint64_t last_pts = mst->next_pts;
mst->next_pts = context->video_start_time + pts;
// switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "pts: %lld last_pts: %lld delta: %lld frame_pts: %lld nextpts: %lld, num: %d, den:%d num:%d den:%d sleep: %lld\n",
// pts, last_pts, mst->next_pts - last_pts, *((uint64_t *)img->user_priv), mst->next_pts, st->time_base.num, st->time_base.den, st->codec->time_base.num, st->codec->time_base.den, mst->next_pts - now);
}
if (pts == 0) mst->next_pts = 0;
if (mst->next_pts && switch_micro_time_now() - mst->next_pts > AV_TIME_BASE) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "picture is too late, diff: %lld %u\n", switch_micro_time_now() - mst->next_pts, switch_queue_size(context->eh.video_queue));
switch_img_free(&img);
// return SWITCH_STATUS_BREAK;
goto again;
}
while (switch_micro_time_now() - mst->next_pts < -10000LL / 2) {
if (!(flags & SVR_BLOCK)) {
switch_img_free(&img);
return SWITCH_STATUS_BREAK;
}
// switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "yield\n");
switch_yield(10000);
}
#endif
frame->img = img;
} else {
if ((flags & SVR_BLOCK)) {
switch_yield(10000);
}
return SWITCH_STATUS_BREAK;
}
return frame->img ? SWITCH_STATUS_SUCCESS : SWITCH_STATUS_FALSE;
}
static switch_status_t av_file_write_video(switch_file_handle_t *handle, switch_frame_t *frame)
......@@ -1454,8 +1947,8 @@ static switch_status_t av_file_write_video(switch_file_handle_t *handle, switch_
context->video_st.width = frame->img->d_w;
context->video_st.height = frame->img->d_h;
context->video_st.next_pts = switch_time_now() / 1000;
if (add_stream(&context->video_st, context->oc, &context->video_codec, context->oc->oformat->video_codec, &handle->mm) == SWITCH_STATUS_SUCCESS &&
open_video(context->oc, context->video_codec, &context->video_st) == SWITCH_STATUS_SUCCESS) {
if (add_stream(&context->video_st, context->fc, &context->video_codec, context->fc->oformat->video_codec, &handle->mm) == SWITCH_STATUS_SUCCESS &&
open_video(context->fc, context->video_codec, &context->video_st) == SWITCH_STATUS_SUCCESS) {
char codec_str[256];
int ret;
......@@ -1464,9 +1957,9 @@ static switch_status_t av_file_write_video(switch_file_handle_t *handle, switch_
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "use video codec implementation %s\n", codec_str);
context->has_video = 1;
// av_dump_format(context->oc, 0, "/tmp/av.mp4", 1);
// av_dump_format(context->fc, 0, "/tmp/av.mp4", 1);
ret = avformat_write_header(context->oc, NULL);
ret = avformat_write_header(context->fc, NULL);
if (ret < 0) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error occurred when opening output file: %s\n", get_error_text(ret));
switch_goto_status(SWITCH_STATUS_FALSE, end);
......@@ -1484,7 +1977,7 @@ static switch_status_t av_file_write_video(switch_file_handle_t *handle, switch_
switch_mutex_init(&context->mutex, SWITCH_MUTEX_NESTED, handle->memory_pool);
context->eh.mutex = context->mutex;
context->eh.video_st = &context->video_st;
context->eh.oc = context->oc;
context->eh.fc = context->fc;
if (switch_core_timer_init(&context->timer, "soft", 1, 1, handle->memory_pool) != SWITCH_STATUS_SUCCESS) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Timer Activation Fail\n");
switch_goto_status(SWITCH_STATUS_FALSE, end);
......@@ -1534,6 +2027,7 @@ SWITCH_MODULE_LOAD_FUNCTION(mod_avformat_load)
supported_formats[i++] = "av";
supported_formats[i++] = "rtmp";
supported_formats[i++] = "mp4";
supported_formats[i++] = "mov";
/* connect my internal structure to the blank pointer passed to me */
*module_interface = switch_loadable_module_create_module_interface(pool, modname);
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论