From 48606b2942c3bfdb81b6d8d20da35617f5f3d8a1 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 21 Feb 2021 17:39:59 +0100 Subject: [PATCH] Move producer/consumer frames out of video buffer The video buffer held 3 frames: - the producer frame (for decoding) - the pending frame (to exchange between the producer and consumer) - the consumer frame (for rendering) It worked well, but it prevented video buffers to be chained, because the consumer frame of the first video buffer must be the same as the producer frame of the second video buffer. To solve this problem, make the decoder and the screen handle their own frames, and keep only the pending frame in the video_buffer. This paves the way to support asynchronous swscale. --- app/src/decoder.c | 17 +++++++++++++---- app/src/decoder.h | 1 + app/src/screen.c | 13 ++++++++++++- app/src/screen.h | 2 ++ app/src/video_buffer.c | 37 +++++++++---------------------------- app/src/video_buffer.h | 13 ++++++------- 6 files changed, 43 insertions(+), 40 deletions(-) diff --git a/app/src/decoder.c b/app/src/decoder.c index 7da959c6..8c262412 100644 --- a/app/src/decoder.c +++ b/app/src/decoder.c @@ -30,11 +30,20 @@ decoder_open(struct decoder *decoder, const AVCodec *codec) { return false; } + decoder->frame = av_frame_alloc(); + if (!decoder->frame) { + LOGE("Could not create decoder frame"); + avcodec_close(decoder->codec_ctx); + avcodec_free_context(&decoder->codec_ctx); + return false; + } + return true; } void decoder_close(struct decoder *decoder) { + av_frame_free(&decoder->frame); avcodec_close(decoder->codec_ctx); avcodec_free_context(&decoder->codec_ctx); } @@ -49,11 +58,11 @@ decoder_push(struct decoder *decoder, const AVPacket *packet) { LOGE("Could not send video packet: %d", ret); return false; } - ret = avcodec_receive_frame(decoder->codec_ctx, - decoder->video_buffer->producer_frame); + ret = avcodec_receive_frame(decoder->codec_ctx, decoder->frame); if (!ret) { // a frame was received - video_buffer_producer_offer_frame(decoder->video_buffer); + video_buffer_producer_offer_frame(decoder->video_buffer, + &decoder->frame); } else if (ret != AVERROR(EAGAIN)) { LOGE("Could not receive video frame: %d", ret); return false; @@ -61,7 +70,7 @@ decoder_push(struct decoder *decoder, const AVPacket *packet) { #else int got_picture; int len = avcodec_decode_video2(decoder->codec_ctx, - decoder->video_buffer->decoding_frame, + decoder->frame, &got_picture, packet); if (len < 0) { diff --git a/app/src/decoder.h b/app/src/decoder.h index bbd7a9a7..69e06828 100644 --- a/app/src/decoder.h +++ b/app/src/decoder.h @@ -12,6 +12,7 @@ struct decoder { struct video_buffer *video_buffer; AVCodecContext *codec_ctx; + AVFrame *frame; }; void diff --git a/app/src/screen.c b/app/src/screen.c index aa6f32b7..b48dfb69 100644 --- a/app/src/screen.c +++ b/app/src/screen.c @@ -356,6 +356,15 @@ screen_init(struct screen *screen, struct video_buffer *vb, return false; } + screen->frame = av_frame_alloc(); + if (!screen->frame) { + LOGC("Could not create screen frame"); + SDL_DestroyTexture(screen->texture); + SDL_DestroyRenderer(screen->renderer); + SDL_DestroyWindow(screen->window); + return false; + } + // Reset the window size to trigger a SIZE_CHANGED event, to workaround // HiDPI issues with some SDL renderers when several displays having // different HiDPI scaling are connected @@ -373,6 +382,7 @@ screen_show_window(struct screen *screen) { void screen_destroy(struct screen *screen) { + av_frame_free(&screen->frame); SDL_DestroyTexture(screen->texture); SDL_DestroyRenderer(screen->renderer); SDL_DestroyWindow(screen->window); @@ -480,7 +490,8 @@ update_texture(struct screen *screen, const AVFrame *frame) { static bool screen_update_frame(struct screen *screen) { - const AVFrame *frame = video_buffer_consumer_take_frame(screen->vb); + video_buffer_consumer_take_frame(screen->vb, &screen->frame); + AVFrame *frame = screen->frame; fps_counter_add_rendered_frame(screen->fps_counter); diff --git a/app/src/screen.h b/app/src/screen.h index 4e1d5e63..ad7892c9 100644 --- a/app/src/screen.h +++ b/app/src/screen.h @@ -36,6 +36,8 @@ struct screen { bool fullscreen; bool maximized; bool mipmaps; + + AVFrame *frame; }; struct screen_params { diff --git a/app/src/video_buffer.c b/app/src/video_buffer.c index 94619840..4acabd27 100644 --- a/app/src/video_buffer.c +++ b/app/src/video_buffer.c @@ -8,24 +8,14 @@ bool video_buffer_init(struct video_buffer *vb, bool wait_consumer) { - vb->producer_frame = av_frame_alloc(); - if (!vb->producer_frame) { - goto error_0; - } - vb->pending_frame = av_frame_alloc(); if (!vb->pending_frame) { - goto error_1; - } - - vb->consumer_frame = av_frame_alloc(); - if (!vb->consumer_frame) { - goto error_2; + goto error_0; } bool ok = sc_mutex_init(&vb->mutex); if (!ok) { - goto error_3; + goto error_1; } vb->wait_consumer = wait_consumer; @@ -33,7 +23,7 @@ video_buffer_init(struct video_buffer *vb, bool wait_consumer) { ok = sc_cond_init(&vb->pending_frame_consumed_cond); if (!ok) { sc_mutex_destroy(&vb->mutex); - goto error_2; + goto error_1; } // interrupted is not used if wait_consumer is disabled since offering // a frame will never block @@ -49,12 +39,8 @@ video_buffer_init(struct video_buffer *vb, bool wait_consumer) { return true; -error_3: - av_frame_free(&vb->consumer_frame); -error_2: - av_frame_free(&vb->pending_frame); error_1: - av_frame_free(&vb->producer_frame); + av_frame_free(&vb->pending_frame); error_0: return false; } @@ -65,9 +51,7 @@ video_buffer_destroy(struct video_buffer *vb) { sc_cond_destroy(&vb->pending_frame_consumed_cond); } sc_mutex_destroy(&vb->mutex); - av_frame_free(&vb->consumer_frame); av_frame_free(&vb->pending_frame); - av_frame_free(&vb->producer_frame); } static inline void @@ -89,7 +73,7 @@ video_buffer_set_consumer_callbacks(struct video_buffer *vb, } void -video_buffer_producer_offer_frame(struct video_buffer *vb) { +video_buffer_producer_offer_frame(struct video_buffer *vb, AVFrame **pframe) { assert(vb->cbs); sc_mutex_lock(&vb->mutex); @@ -101,7 +85,7 @@ video_buffer_producer_offer_frame(struct video_buffer *vb) { } av_frame_unref(vb->pending_frame); - swap_frames(&vb->producer_frame, &vb->pending_frame); + swap_frames(pframe, &vb->pending_frame); bool skipped = !vb->pending_frame_consumed; vb->pending_frame_consumed = false; @@ -116,13 +100,13 @@ video_buffer_producer_offer_frame(struct video_buffer *vb) { } } -const AVFrame * -video_buffer_consumer_take_frame(struct video_buffer *vb) { +void +video_buffer_consumer_take_frame(struct video_buffer *vb, AVFrame **pframe) { sc_mutex_lock(&vb->mutex); assert(!vb->pending_frame_consumed); vb->pending_frame_consumed = true; - swap_frames(&vb->consumer_frame, &vb->pending_frame); + swap_frames(pframe, &vb->pending_frame); av_frame_unref(vb->pending_frame); if (vb->wait_consumer) { @@ -130,9 +114,6 @@ video_buffer_consumer_take_frame(struct video_buffer *vb) { sc_cond_signal(&vb->pending_frame_consumed_cond); } sc_mutex_unlock(&vb->mutex); - - // consumer_frame is only written from this thread, no need to lock - return vb->consumer_frame; } void diff --git a/app/src/video_buffer.h b/app/src/video_buffer.h index 4d11e3ab..a7d03368 100644 --- a/app/src/video_buffer.h +++ b/app/src/video_buffer.h @@ -29,9 +29,7 @@ typedef struct AVFrame AVFrame; */ struct video_buffer { - AVFrame *producer_frame; AVFrame *pending_frame; - AVFrame *consumer_frame; sc_mutex mutex; bool wait_consumer; // never overwrite a pending frame if it is not consumed @@ -67,13 +65,14 @@ video_buffer_set_consumer_callbacks(struct video_buffer *vb, void *cbs_userdata); // set the producer frame as ready for consuming +// the produced frame is exchanged with an unused allocated frame void -video_buffer_producer_offer_frame(struct video_buffer *vb); +video_buffer_producer_offer_frame(struct video_buffer *vb, AVFrame **pframe); -// mark the consumer frame as consumed and return it -// the frame is valid until the next call to this function -const AVFrame * -video_buffer_consumer_take_frame(struct video_buffer *vb); +// mark the consumer frame as consumed and exchange it with an unused allocated +// frame +void +video_buffer_consumer_take_frame(struct video_buffer *vb, AVFrame **pframe); // wake up and avoid any blocking call void