Discussion:
[PATCH v2 1/2] lavf: make overlay_qsv work based on framesync
(too old to reply)
Ruiling Song
2018-04-03 01:50:19 UTC
Permalink
The existing version which was cherry-picked from Libav does not work
with FFmpeg framework, because ff_request_frame() was totally
different between Libav (recursive) and FFmpeg (non-recursive).
The existing overlay_qsv implementation depends on the recursive version
of ff_request_frame to trigger immediate call to request_frame() on input pad.
But this has been removed in FFmpeg since "lavfi: make request_frame() non-recursive."
Now that we have handy framesync support in FFmpeg, so I make it work
based on framesync. Some other fixing which is also needed to make
overlay_qsv work are put in a separate patch.

v2:
add .preinit field to initilize framesync options.
export more options like vf_overlay.c

Signed-off-by: Ruiling Song <***@intel.com>
---
libavfilter/Makefile | 2 +-
libavfilter/vf_overlay_qsv.c | 213 ++++++++++++++++---------------------------
2 files changed, 78 insertions(+), 137 deletions(-)

diff --git a/libavfilter/Makefile b/libavfilter/Makefile
index a90ca30..7f2ad1f 100644
--- a/libavfilter/Makefile
+++ b/libavfilter/Makefile
@@ -267,7 +267,7 @@ OBJS-$(CONFIG_OSCILLOSCOPE_FILTER) += vf_datascope.o
OBJS-$(CONFIG_OVERLAY_FILTER) += vf_overlay.o framesync.o
OBJS-$(CONFIG_OVERLAY_OPENCL_FILTER) += vf_overlay_opencl.o opencl.o \
opencl/overlay.o framesync.o
-OBJS-$(CONFIG_OVERLAY_QSV_FILTER) += vf_overlay_qsv.o
+OBJS-$(CONFIG_OVERLAY_QSV_FILTER) += vf_overlay_qsv.o framesync.o
OBJS-$(CONFIG_OWDENOISE_FILTER) += vf_owdenoise.o
OBJS-$(CONFIG_PAD_FILTER) += vf_pad.o
OBJS-$(CONFIG_PALETTEGEN_FILTER) += vf_palettegen.o
diff --git a/libavfilter/vf_overlay_qsv.c b/libavfilter/vf_overlay_qsv.c
index 6c3efdb..2087178 100644
--- a/libavfilter/vf_overlay_qsv.c
+++ b/libavfilter/vf_overlay_qsv.c
@@ -36,6 +36,7 @@
#include "formats.h"
#include "video.h"

+#include "framesync.h"
#include "qsvvpp.h"

#define MAIN 0
@@ -56,14 +57,10 @@ enum var_name {
VAR_VARS_NB
};

-enum EOFAction {
- EOF_ACTION_REPEAT,
- EOF_ACTION_ENDALL
-};
-
typedef struct QSVOverlayContext {
const AVClass *class;

+ FFFrameSync fs;
QSVVPPContext *qsv;
QSVVPPParam qsv_param;
mfxExtVPPComposite comp_conf;
@@ -72,10 +69,6 @@ typedef struct QSVOverlayContext {
char *overlay_ox, *overlay_oy, *overlay_ow, *overlay_oh;
uint16_t overlay_alpha, overlay_pixel_alpha;

- enum EOFAction eof_action; /* action to take on EOF from source */
-
- AVFrame *main;
- AVFrame *over_prev, *over_next;
} QSVOverlayContext;

static const char *const var_names[] = {
@@ -90,20 +83,25 @@ static const char *const var_names[] = {
NULL
};

-static const AVOption options[] = {
+static const AVOption overlay_qsv_options[] = {
{ "x", "Overlay x position", OFFSET(overlay_ox), AV_OPT_TYPE_STRING, { .str="0"}, 0, 255, .flags = FLAGS},
{ "y", "Overlay y position", OFFSET(overlay_oy), AV_OPT_TYPE_STRING, { .str="0"}, 0, 255, .flags = FLAGS},
{ "w", "Overlay width", OFFSET(overlay_ow), AV_OPT_TYPE_STRING, { .str="overlay_iw"}, 0, 255, .flags = FLAGS},
{ "h", "Overlay height", OFFSET(overlay_oh), AV_OPT_TYPE_STRING, { .str="overlay_ih*w/overlay_iw"}, 0, 255, .flags = FLAGS},
{ "alpha", "Overlay global alpha", OFFSET(overlay_alpha), AV_OPT_TYPE_INT, { .i64 = 255}, 0, 255, .flags = FLAGS},
{ "eof_action", "Action to take when encountering EOF from secondary input ",
- OFFSET(eof_action), AV_OPT_TYPE_INT, { .i64 = EOF_ACTION_REPEAT },
- EOF_ACTION_REPEAT, EOF_ACTION_ENDALL, .flags = FLAGS, "eof_action" },
- { "repeat", "Repeat the previous frame.", 0, AV_OPT_TYPE_CONST, { .i64 = EOF_ACTION_REPEAT }, .flags = FLAGS, "eof_action" },
- { "endall", "End both streams.", 0, AV_OPT_TYPE_CONST, { .i64 = EOF_ACTION_ENDALL }, .flags = FLAGS, "eof_action" },
+ OFFSET(fs.opt_eof_action), AV_OPT_TYPE_INT, { .i64 = EOF_ACTION_REPEAT },
+ EOF_ACTION_REPEAT, EOF_ACTION_PASS, .flags = FLAGS, "eof_action" },
+ { "repeat", "Repeat the previous frame.", 0, AV_OPT_TYPE_CONST, { .i64 = EOF_ACTION_REPEAT }, .flags = FLAGS, "eof_action" },
+ { "endall", "End both streams.", 0, AV_OPT_TYPE_CONST, { .i64 = EOF_ACTION_ENDALL }, .flags = FLAGS, "eof_action" },
+ { "pass", "Pass through the main input.", 0, AV_OPT_TYPE_CONST, { .i64 = EOF_ACTION_PASS }, .flags = FLAGS, "eof_action" },
+ { "shortest", "force termination when the shortest input terminates", OFFSET(fs.opt_shortest), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, FLAGS },
+ { "repeatlast", "repeat overlay of the last overlay frame", OFFSET(fs.opt_repeatlast), AV_OPT_TYPE_BOOL, {.i64=1}, 0, 1, FLAGS },
{ NULL }
};

+FRAMESYNC_DEFINE_CLASS(overlay_qsv, QSVOverlayContext, fs);
+
static int eval_expr(AVFilterContext *ctx)
{
QSVOverlayContext *vpp = ctx->priv;
@@ -230,12 +228,53 @@ static int config_overlay_input(AVFilterLink *inlink)
return 0;
}

+static int process_frame(FFFrameSync *fs)
+{
+ AVFilterContext *ctx = fs->parent;
+ QSVOverlayContext *s = fs->opaque;
+ AVFrame *frame = NULL;
+ int ret = 0, i;
+
+ for (i = 0; i < ctx->nb_inputs; i++) {
+ ret = ff_framesync_get_frame(fs, i, &frame, 0);
+ if (ret == 0)
+ ret = ff_qsvvpp_filter_frame(s->qsv, ctx->inputs[i], frame);
+ if (ret < 0 && ret != AVERROR(EAGAIN))
+ break;
+ }
+
+ return ret;
+}
+
+static int init_framesync(AVFilterContext *ctx)
+{
+ QSVOverlayContext *s = ctx->priv;
+ int ret, i;
+
+ s->fs.on_event = process_frame;
+ s->fs.opaque = s;
+ ret = ff_framesync_init(&s->fs, ctx, ctx->nb_inputs);
+ if (ret < 0)
+ return ret;
+
+ for (i = 0; i < ctx->nb_inputs; i++) {
+ FFFrameSyncIn *in = &s->fs.in[i];
+ in->before = EXT_STOP;
+ in->after = EXT_INFINITY;
+ in->sync = i ? 1 : 2;
+ in->time_base = ctx->inputs[i]->time_base;
+ }
+
+ return ff_framesync_configure(&s->fs);
+}
+
static int config_output(AVFilterLink *outlink)
{
AVFilterContext *ctx = outlink->src;
QSVOverlayContext *vpp = ctx->priv;
AVFilterLink *in0 = ctx->inputs[0];
AVFilterLink *in1 = ctx->inputs[1];
+ int ret;

av_log(ctx, AV_LOG_DEBUG, "Output is of %s.\n", av_get_pix_fmt_name(outlink->format));
if ((in0->format == AV_PIX_FMT_QSV && in1->format != AV_PIX_FMT_QSV) ||
@@ -257,121 +296,27 @@ static int config_output(AVFilterLink *outlink)
outlink->frame_rate = in0->frame_rate;
outlink->time_base = av_inv_q(outlink->frame_rate);

- return ff_qsvvpp_create(ctx, &vpp->qsv, &vpp->qsv_param);
-}
-
-static int blend_frame(AVFilterContext *ctx, AVFrame *mpic, AVFrame *opic)
-{
- int ret = 0;
- QSVOverlayContext *vpp = ctx->priv;
- AVFrame *opic_copy = NULL;
-
- ret = ff_qsvvpp_filter_frame(vpp->qsv, ctx->inputs[0], mpic);
- if (ret == 0 || ret == AVERROR(EAGAIN)) {
- /* Reference the overlay frame. Because:
- * 1. ff_qsvvpp_filter_frame will take control of the given frame
- * 2. We need to repeat the overlay frame when 2nd input goes into EOF
- */
- opic_copy = av_frame_clone(opic);
- if (!opic_copy)
- return AVERROR(ENOMEM);
-
- ret = ff_qsvvpp_filter_frame(vpp->qsv, ctx->inputs[1], opic_copy);
- }
-
- return ret;
-}
-
-static int handle_overlay_eof(AVFilterContext *ctx)
-{
- int ret = 0;
- QSVOverlayContext *s = ctx->priv;
- /* Repeat previous frame on secondary input */
- if (s->over_prev && s->eof_action == EOF_ACTION_REPEAT)
- ret = blend_frame(ctx, s->main, s->over_prev);
- /* End both streams */
- else if (s->eof_action == EOF_ACTION_ENDALL)
- return AVERROR_EOF;
-
- s->main = NULL;
+ ret = init_framesync(ctx);
+ if (ret < 0)
+ return ret;

- return ret;
+ return ff_qsvvpp_create(ctx, &vpp->qsv, &vpp->qsv_param);
}

-static int request_frame(AVFilterLink *outlink)
-{
- AVFilterContext *ctx = outlink->src;
- QSVOverlayContext *s = ctx->priv;
- AVRational tb_main = ctx->inputs[MAIN]->time_base;
- AVRational tb_over = ctx->inputs[OVERLAY]->time_base;
- int ret = 0;
-
- /* get a frame on the main input */
- if (!s->main) {
- ret = ff_request_frame(ctx->inputs[MAIN]);
- if (ret < 0)
- return ret;
- }
-
- /* get a new frame on the overlay input, on EOF check setting 'eof_action' */
- if (!s->over_next) {
- ret = ff_request_frame(ctx->inputs[OVERLAY]);
- if (ret == AVERROR_EOF)
- return handle_overlay_eof(ctx);
- else if (ret < 0)
- return ret;
- }
-
- while (s->main->pts != AV_NOPTS_VALUE &&
- s->over_next->pts != AV_NOPTS_VALUE &&
- av_compare_ts(s->over_next->pts, tb_over, s->main->pts, tb_main) < 0) {
- av_frame_free(&s->over_prev);
- FFSWAP(AVFrame*, s->over_prev, s->over_next);
-
- ret = ff_request_frame(ctx->inputs[OVERLAY]);
- if (ret == AVERROR_EOF)
- return handle_overlay_eof(ctx);
- else if (ret < 0)
- return ret;
- }
-
- if (s->main->pts == AV_NOPTS_VALUE ||
- s->over_next->pts == AV_NOPTS_VALUE ||
- !av_compare_ts(s->over_next->pts, tb_over, s->main->pts, tb_main)) {
- ret = blend_frame(ctx, s->main, s->over_next);
- av_frame_free(&s->over_prev);
- FFSWAP(AVFrame*, s->over_prev, s->over_next);
- } else if (s->over_prev) {
- ret = blend_frame(ctx, s->main, s->over_prev);
- } else {
- av_frame_free(&s->main);
- ret = AVERROR(EAGAIN);
- }
-
- s->main = NULL;
-
- return ret;
-}
+/*
+ * Callback for qsvvpp
+ * @Note: qsvvpp composition does not generate PTS for result frame.
+ * so we assign the PTS from framesync to the output frame.
+ */

-static int filter_frame_main(AVFilterLink *inlink, AVFrame *frame)
+static int filter_callback(AVFilterLink *outlink, AVFrame *frame)
{
- QSVOverlayContext *s = inlink->dst->priv;
-
- av_assert0(!s->main);
- s->main = frame;
-
- return 0;
+ QSVOverlayContext *s = outlink->src->priv;
+ frame->pts = av_rescale_q(s->fs.pts,
+ s->fs.time_base, outlink->time_base);
+ return ff_filter_frame(outlink, frame);
}

-static int filter_frame_overlay(AVFilterLink *inlink, AVFrame *frame)
-{
- QSVOverlayContext *s = inlink->dst->priv;
-
- av_assert0(!s->over_next);
- s->over_next = frame;
-
- return 0;
-}

static int overlay_qsv_init(AVFilterContext *ctx)
{
@@ -387,7 +332,7 @@ static int overlay_qsv_init(AVFilterContext *ctx)
return AVERROR(ENOMEM);

/* initialize QSVVPP params */
- vpp->qsv_param.filter_frame = NULL;
+ vpp->qsv_param.filter_frame = filter_callback;
vpp->qsv_param.ext_buf = av_mallocz(sizeof(*vpp->qsv_param.ext_buf));
if (!vpp->qsv_param.ext_buf)
return AVERROR(ENOMEM);
@@ -404,14 +349,18 @@ static void overlay_qsv_uninit(AVFilterContext *ctx)
{
QSVOverlayContext *vpp = ctx->priv;

- av_frame_free(&vpp->main);
- av_frame_free(&vpp->over_prev);
- av_frame_free(&vpp->over_next);
ff_qsvvpp_free(&vpp->qsv);
+ ff_framesync_uninit(&vpp->fs);
av_freep(&vpp->comp_conf.InputStream);
av_freep(&vpp->qsv_param.ext_buf);
}

+static int activate(AVFilterContext *ctx)
+{
+ QSVOverlayContext *s = ctx->priv;
+ return ff_framesync_activate(&s->fs);
+}
+
static int overlay_qsv_query_formats(AVFilterContext *ctx)
{
int i;
@@ -444,25 +393,16 @@ static int overlay_qsv_query_formats(AVFilterContext *ctx)
return 0;
}

-static const AVClass overlay_qsv_class = {
- .class_name = "overlay_qsv",
- .item_name = av_default_item_name,
- .option = options,
- .version = LIBAVUTIL_VERSION_INT,
-};
-
static const AVFilterPad overlay_qsv_inputs[] = {
{
.name = "main",
.type = AVMEDIA_TYPE_VIDEO,
- .filter_frame = filter_frame_main,
.config_props = config_main_input,
.needs_fifo = 1,
},
{
.name = "overlay",
.type = AVMEDIA_TYPE_VIDEO,
- .filter_frame = filter_frame_overlay,
.config_props = config_overlay_input,
.needs_fifo = 1,
},
@@ -474,7 +414,6 @@ static const AVFilterPad overlay_qsv_outputs[] = {
.name = "default",
.type = AVMEDIA_TYPE_VIDEO,
.config_props = config_output,
- .request_frame = request_frame,
},
{ NULL }
};
@@ -484,8 +423,10 @@ AVFilter ff_vf_overlay_qsv = {
.description = NULL_IF_CONFIG_SMALL("Quick Sync Video overlay."),
.priv_size = sizeof(QSVOverlayContext),
.query_formats = overlay_qsv_query_formats,
+ .preinit = overlay_qsv_framesync_preinit,
.init = overlay_qsv_init,
.uninit = overlay_qsv_uninit,
+ .activate = activate,
.inputs = overlay_qsv_inputs,
.outputs = overlay_qsv_outputs,
.priv_class = &overlay_qsv_class,
--
2.7.4
Ruiling Song
2018-04-03 01:50:20 UTC
Permalink
For filters based on framesync, the input frame was managed
by framesync, so we should not directly keep and destroy it,
instead we make a clone of it here, or else double-free will occur.
But for other filters not based on framesync, we still need to
free the input frame inside filter_frame. That's why I made
this v2 to fix the side-effect on normal filters.

v2:
and one av_frame_free() in vf_vpp_qsv.c

Signed-off-by: Ruiling Song <***@intel.com>
---
libavfilter/qsvvpp.c | 4 ++--
libavfilter/vf_vpp_qsv.c | 5 ++++-
2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/libavfilter/qsvvpp.c b/libavfilter/qsvvpp.c
index f32b46d..86787c5 100644
--- a/libavfilter/qsvvpp.c
+++ b/libavfilter/qsvvpp.c
@@ -296,7 +296,7 @@ static QSVFrame *submit_frame(QSVVPPContext *s, AVFilterLink *inlink, AVFrame *p
av_log(ctx, AV_LOG_ERROR, "QSVVPP gets a wrong frame.\n");
return NULL;
}
- qsv_frame->frame = picref;
+ qsv_frame->frame = av_frame_clone(picref);
qsv_frame->surface = (mfxFrameSurface1 *)qsv_frame->frame->data[3];
} else {
/* make a copy if the input is not padded as libmfx requires */
@@ -318,7 +318,7 @@ static QSVFrame *submit_frame(QSVVPPContext *s, AVFilterLink *inlink, AVFrame *p
av_frame_copy_props(qsv_frame->frame, picref);
av_frame_free(&picref);
} else
- qsv_frame->frame = picref;
+ qsv_frame->frame = av_frame_clone(picref);

if (map_frame_to_surface(qsv_frame->frame,
&qsv_frame->surface_internal) < 0) {
diff --git a/libavfilter/vf_vpp_qsv.c b/libavfilter/vf_vpp_qsv.c
index bd5fc32..4ef5bfb 100644
--- a/libavfilter/vf_vpp_qsv.c
+++ b/libavfilter/vf_vpp_qsv.c
@@ -326,8 +326,11 @@ static int config_output(AVFilterLink *outlink)
static int filter_frame(AVFilterLink *inlink, AVFrame *picref)
{
VPPContext *vpp = inlink->dst->priv;
+ int ret = 0;

- return ff_qsvvpp_filter_frame(vpp->qsv, inlink, picref);
+ ret = ff_qsvvpp_filter_frame(vpp->qsv, inlink, picref);
+ av_frame_free(&picref);
+ return ret;
}

static int query_formats(AVFilterContext *ctx)
--
2.7.4
Song, Ruiling
2018-04-09 03:13:22 UTC
Permalink
-----Original Message-----
From: Song, Ruiling
Sent: Tuesday, April 3, 2018 9:50 AM
Subject: [PATCH v2 1/2] lavf: make overlay_qsv work based on framesync
The existing version which was cherry-picked from Libav does not work
with FFmpeg framework, because ff_request_frame() was totally
different between Libav (recursive) and FFmpeg (non-recursive).
The existing overlay_qsv implementation depends on the recursive version
of ff_request_frame to trigger immediate call to request_frame() on input pad.
But this has been removed in FFmpeg since "lavfi: make request_frame() non-
recursive."
Now that we have handy framesync support in FFmpeg, so I make it work
based on framesync. Some other fixing which is also needed to make
overlay_qsv work are put in a separate patch.
add .preinit field to initilize framesync options.
export more options like vf_overlay.c
---
libavfilter/Makefile | 2 +-
libavfilter/vf_overlay_qsv.c | 213 ++++++++++++++++---------------------------
2 files changed, 78 insertions(+), 137 deletions(-)
Ping?
Li, Zhong
2018-04-10 03:38:45 UTC
Permalink
Of Ruiling Song
Sent: Tuesday, April 3, 2018 9:50 AM
Subject: [FFmpeg-devel] [PATCH v2 1/2] lavf: make overlay_qsv work based
on framesync
The existing version which was cherry-picked from Libav does not work with
FFmpeg framework, because ff_request_frame() was totally different
between Libav (recursive) and FFmpeg (non-recursive).
The existing overlay_qsv implementation depends on the recursive version of
ff_request_frame to trigger immediate call to request_frame() on input pad.
But this has been removed in FFmpeg since "lavfi: make request_frame() non-recursive."
Now that we have handy framesync support in FFmpeg, so I make it work
based on framesync. Some other fixing which is also needed to make
overlay_qsv work are put in a separate patch.
add .preinit field to initilize framesync options.
export more options like vf_overlay.c
How about taking these options as a separated patch?
It doesn't take obvious effect to make qsv overlay work.
---
libavfilter/Makefile | 2 +-
libavfilter/vf_overlay_qsv.c | 213 ++++++++++++++++---------------------------
2 files changed, 78 insertions(+), 137 deletions(-)
diff --git a/libavfilter/Makefile b/libavfilter/Makefile index a90ca30..7f2ad1f
100644
--- a/libavfilter/Makefile
+++ b/libavfilter/Makefile
@@ -267,7 +267,7 @@ OBJS-$(CONFIG_OSCILLOSCOPE_FILTER)
+= vf_datascope.o
OBJS-$(CONFIG_OVERLAY_FILTER) += vf_overlay.o
framesync.o
OBJS-$(CONFIG_OVERLAY_OPENCL_FILTER) +=
vf_overlay_opencl.o opencl.o \
opencl/overlay.o framesync.o
-OBJS-$(CONFIG_OVERLAY_QSV_FILTER) += vf_overlay_qsv.o
+OBJS-$(CONFIG_OVERLAY_QSV_FILTER) += vf_overlay_qsv.o framesync.o
OBJS-$(CONFIG_OWDENOISE_FILTER) += vf_owdenoise.o
OBJS-$(CONFIG_PAD_FILTER) += vf_pad.o
OBJS-$(CONFIG_PALETTEGEN_FILTER) += vf_palettegen.o
diff --git a/libavfilter/vf_overlay_qsv.c b/libavfilter/vf_overlay_qsv.c index
6c3efdb..2087178 100644
--- a/libavfilter/vf_overlay_qsv.c
+++ b/libavfilter/vf_overlay_qsv.c
@@ -36,6 +36,7 @@
#include "formats.h"
#include "video.h"
+#include "framesync.h"
#include "qsvvpp.h"
#define MAIN 0
@@ -56,14 +57,10 @@ enum var_name {
VAR_VARS_NB
};
-enum EOFAction {
- EOF_ACTION_REPEAT,
- EOF_ACTION_ENDALL
-};
-
typedef struct QSVOverlayContext {
const AVClass *class;
+ FFFrameSync fs;
QSVVPPContext *qsv;
QSVVPPParam qsv_param;
mfxExtVPPComposite comp_conf;
@@ -72,10 +69,6 @@ typedef struct QSVOverlayContext {
char *overlay_ox, *overlay_oy, *overlay_ow, *overlay_oh;
uint16_t overlay_alpha, overlay_pixel_alpha;
- enum EOFAction eof_action; /* action to take on EOF from source */
-
- AVFrame *main;
- AVFrame *over_prev, *over_next;
} QSVOverlayContext;
const char *const var_names[] = {
NULL
};
-static const AVOption options[] = {
+static const AVOption overlay_qsv_options[] = {
{ "x", "Overlay x position", OFFSET(overlay_ox), AV_OPT_TYPE_STRING,
{ .str="0"}, 0, 255, .flags = FLAGS},
{ "y", "Overlay y position", OFFSET(overlay_oy), AV_OPT_TYPE_STRING,
{ .str="0"}, 0, 255, .flags = FLAGS},
{ "w", "Overlay width", OFFSET(overlay_ow),
AV_OPT_TYPE_STRING, { .str="overlay_iw"}, 0, 255, .flags = FLAGS},
{ "h", "Overlay height", OFFSET(overlay_oh),
AV_OPT_TYPE_STRING, { .str="overlay_ih*w/overlay_iw"}, 0, 255, .flags =
FLAGS},
{ "alpha", "Overlay global alpha", OFFSET(overlay_alpha),
AV_OPT_TYPE_INT, { .i64 = 255}, 0, 255, .flags = FLAGS},
{ "eof_action", "Action to take when encountering EOF from secondary input ",
- OFFSET(eof_action), AV_OPT_TYPE_INT, { .i64 =
EOF_ACTION_REPEAT },
- EOF_ACTION_REPEAT, EOF_ACTION_ENDALL, .flags = FLAGS,
"eof_action" },
- { "repeat", "Repeat the previous frame.", 0, AV_OPT_TYPE_CONST,
{ .i64 = EOF_ACTION_REPEAT }, .flags = FLAGS, "eof_action" },
- { "endall", "End both streams.", 0,
AV_OPT_TYPE_CONST, { .i64 = EOF_ACTION_ENDALL }, .flags = FLAGS,
"eof_action" },
+ OFFSET(fs.opt_eof_action), AV_OPT_TYPE_INT, { .i64 =
EOF_ACTION_REPEAT },
+ EOF_ACTION_REPEAT, EOF_ACTION_PASS, .flags = FLAGS,
"eof_action" },
+ { "repeat", "Repeat the previous frame.", 0,
AV_OPT_TYPE_CONST, { .i64 = EOF_ACTION_REPEAT }, .flags = FLAGS,
"eof_action" },
+ { "endall", "End both streams.", 0,
AV_OPT_TYPE_CONST, { .i64 = EOF_ACTION_ENDALL }, .flags = FLAGS,
"eof_action" },
+ { "pass", "Pass through the main input.", 0,
AV_OPT_TYPE_CONST, { .i64 = EOF_ACTION_PASS }, .flags = FLAGS,
"eof_action" },
+ { "shortest", "force termination when the shortest input terminates",
OFFSET(fs.opt_shortest), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, FLAGS },
+ { "repeatlast", "repeat overlay of the last overlay frame",
+ OFFSET(fs.opt_repeatlast), AV_OPT_TYPE_BOOL, {.i64=1}, 0, 1, FLAGS },
{ NULL }
};
+FRAMESYNC_DEFINE_CLASS(overlay_qsv, QSVOverlayContext, fs);
+
static int eval_expr(AVFilterContext *ctx) {
int config_overlay_input(AVFilterLink *inlink)
return 0;
}
+static int process_frame(FFFrameSync *fs) {
+ AVFilterContext *ctx = fs->parent;
+ QSVOverlayContext *s = fs->opaque;
+ AVFrame *frame = NULL;
+ int ret = 0, i;
+
+ for (i = 0; i < ctx->nb_inputs; i++) {
+ ret = ff_framesync_get_frame(fs, i, &frame, 0);
+ if (ret == 0)
+ ret = ff_qsvvpp_filter_frame(s->qsv, ctx->inputs[i], frame);
+ if (ret < 0 && ret != AVERROR(EAGAIN))
+ break;
+ }
+
+ return ret;
+}
+
+static int init_framesync(AVFilterContext *ctx) {
+ QSVOverlayContext *s = ctx->priv;
+ int ret, i;
+
+ s->fs.on_event = process_frame;
+ s->fs.opaque = s;
+ ret = ff_framesync_init(&s->fs, ctx, ctx->nb_inputs);
+ if (ret < 0)
+ return ret;
+
+ for (i = 0; i < ctx->nb_inputs; i++) {
+ FFFrameSyncIn *in = &s->fs.in[i];
+ in->before = EXT_STOP;
+ in->after = EXT_INFINITY;
+ in->sync = i ? 1 : 2;
+ in->time_base = ctx->inputs[i]->time_base;
+ }
+
+ return ff_framesync_configure(&s->fs); }
+
static int config_output(AVFilterLink *outlink) {
AVFilterContext *ctx = outlink->src;
QSVOverlayContext *vpp = ctx->priv;
AVFilterLink *in0 = ctx->inputs[0];
AVFilterLink *in1 = ctx->inputs[1];
+ int ret;
av_log(ctx, AV_LOG_DEBUG, "Output is of %s.\n",
av_get_pix_fmt_name(outlink->format));
if ((in0->format == AV_PIX_FMT_QSV && in1->format !=
config_output(AVFilterLink *outlink)
outlink->frame_rate = in0->frame_rate;
outlink->time_base = av_inv_q(outlink->frame_rate);
- return ff_qsvvpp_create(ctx, &vpp->qsv, &vpp->qsv_param);
-}
-
-static int blend_frame(AVFilterContext *ctx, AVFrame *mpic, AVFrame *opic)
-{
- int ret = 0;
- QSVOverlayContext *vpp = ctx->priv;
- AVFrame *opic_copy = NULL;
-
- ret = ff_qsvvpp_filter_frame(vpp->qsv, ctx->inputs[0], mpic);
- if (ret == 0 || ret == AVERROR(EAGAIN)) {
- * 1. ff_qsvvpp_filter_frame will take control of the given frame
- * 2. We need to repeat the overlay frame when 2nd input goes into EOF
- */
- opic_copy = av_frame_clone(opic);
- if (!opic_copy)
- return AVERROR(ENOMEM);
-
- ret = ff_qsvvpp_filter_frame(vpp->qsv, ctx->inputs[1], opic_copy);
- }
-
- return ret;
-}
-
-static int handle_overlay_eof(AVFilterContext *ctx) -{
- int ret = 0;
- QSVOverlayContext *s = ctx->priv;
- /* Repeat previous frame on secondary input */
- if (s->over_prev && s->eof_action == EOF_ACTION_REPEAT)
- ret = blend_frame(ctx, s->main, s->over_prev);
- /* End both streams */
- else if (s->eof_action == EOF_ACTION_ENDALL)
- return AVERROR_EOF;
-
- s->main = NULL;
+ ret = init_framesync(ctx);
+ if (ret < 0)
+ return ret;
- return ret;
+ return ff_qsvvpp_create(ctx, &vpp->qsv, &vpp->qsv_param);
}
-static int request_frame(AVFilterLink *outlink) -{
- AVFilterContext *ctx = outlink->src;
- QSVOverlayContext *s = ctx->priv;
- AVRational tb_main = ctx->inputs[MAIN]->time_base;
- AVRational tb_over = ctx->inputs[OVERLAY]->time_base;
- int ret = 0;
-
- /* get a frame on the main input */
- if (!s->main) {
- ret = ff_request_frame(ctx->inputs[MAIN]);
- if (ret < 0)
- return ret;
- }
-
- /* get a new frame on the overlay input, on EOF check setting 'eof_action' */
- if (!s->over_next) {
- ret = ff_request_frame(ctx->inputs[OVERLAY]);
- if (ret == AVERROR_EOF)
- return handle_overlay_eof(ctx);
- else if (ret < 0)
- return ret;
- }
-
- while (s->main->pts != AV_NOPTS_VALUE &&
- s->over_next->pts != AV_NOPTS_VALUE &&
- av_compare_ts(s->over_next->pts, tb_over, s->main->pts, tb_main) < 0) {
- av_frame_free(&s->over_prev);
- FFSWAP(AVFrame*, s->over_prev, s->over_next);
-
- ret = ff_request_frame(ctx->inputs[OVERLAY]);
- if (ret == AVERROR_EOF)
- return handle_overlay_eof(ctx);
- else if (ret < 0)
- return ret;
- }
-
- if (s->main->pts == AV_NOPTS_VALUE ||
- s->over_next->pts == AV_NOPTS_VALUE ||
- !av_compare_ts(s->over_next->pts, tb_over, s->main->pts, tb_main)) {
- ret = blend_frame(ctx, s->main, s->over_next);
- av_frame_free(&s->over_prev);
- FFSWAP(AVFrame*, s->over_prev, s->over_next);
- } else if (s->over_prev) {
- ret = blend_frame(ctx, s->main, s->over_prev);
- } else {
- av_frame_free(&s->main);
- ret = AVERROR(EAGAIN);
- }
-
- s->main = NULL;
-
- return ret;
-}
+/*
+ * Callback for qsvvpp
+ * so we assign the PTS from framesync to the output frame.
+ */
-static int filter_frame_main(AVFilterLink *inlink, AVFrame *frame)
+static int filter_callback(AVFilterLink *outlink, AVFrame *frame)
{
- QSVOverlayContext *s = inlink->dst->priv;
-
- av_assert0(!s->main);
- s->main = frame;
-
- return 0;
+ QSVOverlayContext *s = outlink->src->priv;
+ frame->pts = av_rescale_q(s->fs.pts,
+ s->fs.time_base, outlink->time_base);
+ return ff_filter_frame(outlink, frame);
}
-static int filter_frame_overlay(AVFilterLink *inlink, AVFrame *frame) -{
- QSVOverlayContext *s = inlink->dst->priv;
-
- av_assert0(!s->over_next);
- s->over_next = frame;
-
- return 0;
-}
static int overlay_qsv_init(AVFilterContext *ctx)
return AVERROR(ENOMEM);
/* initialize QSVVPP params */
- vpp->qsv_param.filter_frame = NULL;
+ vpp->qsv_param.filter_frame = filter_callback;
vpp->qsv_param.ext_buf =
av_mallocz(sizeof(*vpp->qsv_param.ext_buf));
if (!vpp->qsv_param.ext_buf)
return AVERROR(ENOMEM);
@@ -404,14 +349,18 @@ static void overlay_qsv_uninit(AVFilterContext
*ctx) {
QSVOverlayContext *vpp = ctx->priv;
- av_frame_free(&vpp->main);
- av_frame_free(&vpp->over_prev);
- av_frame_free(&vpp->over_next);
ff_qsvvpp_free(&vpp->qsv);
+ ff_framesync_uninit(&vpp->fs);
av_freep(&vpp->comp_conf.InputStream);
av_freep(&vpp->qsv_param.ext_buf);
}
+static int activate(AVFilterContext *ctx) {
+ QSVOverlayContext *s = ctx->priv;
+ return ff_framesync_activate(&s->fs); }
+
static int overlay_qsv_query_formats(AVFilterContext *ctx) {
int i;
@@ -444,25 +393,16 @@ static int
overlay_qsv_query_formats(AVFilterContext *ctx)
return 0;
}
-static const AVClass overlay_qsv_class = {
- .class_name = "overlay_qsv",
- .item_name = av_default_item_name,
- .option = options,
- .version = LIBAVUTIL_VERSION_INT,
-};
Why remove it but keep ".priv_class"?
The reset LGTM.
-
static const AVFilterPad overlay_qsv_inputs[] = {
{
.name = "main",
.type = AVMEDIA_TYPE_VIDEO,
- .filter_frame = filter_frame_main,
.config_props = config_main_input,
.needs_fifo = 1,
},
{
.name = "overlay",
.type = AVMEDIA_TYPE_VIDEO,
- .filter_frame = filter_frame_overlay,
.config_props = config_overlay_input,
.needs_fifo = 1,
},
@@ -474,7 +414,6 @@ static const AVFilterPad overlay_qsv_outputs[] = {
.name = "default",
.type = AVMEDIA_TYPE_VIDEO,
.config_props = config_output,
- .request_frame = request_frame,
},
{ NULL }
};
@@ -484,8 +423,10 @@ AVFilter ff_vf_overlay_qsv = {
.description = NULL_IF_CONFIG_SMALL("Quick Sync Video
overlay."),
.priv_size = sizeof(QSVOverlayContext),
.query_formats = overlay_qsv_query_formats,
+ .preinit = overlay_qsv_framesync_preinit,
.init = overlay_qsv_init,
.uninit = overlay_qsv_uninit,
+ .activate = activate,
.inputs = overlay_qsv_inputs,
.outputs = overlay_qsv_outputs,
.priv_class = &overlay_qsv_class,
--
2.7.4
_______________________________________________
ffmpeg-devel mailing list
http://ffmpeg.org/mailman/listinfo/ffmpeg-devel
Song, Ruiling
2018-04-12 01:51:04 UTC
Permalink
-----Original Message-----
From: Li, Zhong
Sent: Tuesday, April 10, 2018 11:39 AM
Subject: RE: [FFmpeg-devel] [PATCH v2 1/2] lavf: make overlay_qsv work based
on framesync
Of Ruiling Song
Sent: Tuesday, April 3, 2018 9:50 AM
Subject: [FFmpeg-devel] [PATCH v2 1/2] lavf: make overlay_qsv work based
on framesync
The existing version which was cherry-picked from Libav does not work with
FFmpeg framework, because ff_request_frame() was totally different
between Libav (recursive) and FFmpeg (non-recursive).
The existing overlay_qsv implementation depends on the recursive version of
ff_request_frame to trigger immediate call to request_frame() on input pad.
But this has been removed in FFmpeg since "lavfi: make request_frame() non-recursive."
Now that we have handy framesync support in FFmpeg, so I make it work
based on framesync. Some other fixing which is also needed to make
overlay_qsv work are put in a separate patch.
add .preinit field to initilize framesync options.
export more options like vf_overlay.c
How about taking these options as a separated patch?
It doesn't take obvious effect to make qsv overlay work.
Below are the options which I mainly mean here. Given that this is a new implementation,
I thought this minor change does not bring too much extra burden to the patch.
It just gives users more freedom to control the behavior of this filter.
But I am open to follow your comment if we think this is necessary to merge the patch.
I would like to wait if anybody else will give some comment on the overall patch.
Thanks for your review!
+ { "shortest", "force termination when the shortest input terminates",
OFFSET(fs.opt_shortest), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, FLAGS },
+ { "repeatlast", "repeat overlay of the last overlay frame",
+ OFFSET(fs.opt_repeatlast), AV_OPT_TYPE_BOOL, {.i64=1}, 0, 1, FLAGS },
{ NULL }
-static const AVClass overlay_qsv_class = {
- .class_name = "overlay_qsv",
- .item_name = av_default_item_name,
- .option = options,
- .version = LIBAVUTIL_VERSION_INT,
-};
Why remove it but keep ".priv_class"?
You can take a look at FRAMESYNC_DEFINE_CLASS macro definition. It already include this overlay_qsv_class definition.
The reset LGTM.
Jun Zhao
2018-04-12 03:08:02 UTC
Permalink
Post by Ruiling Song
The existing version which was cherry-picked from Libav does not work
with FFmpeg framework, because ff_request_frame() was totally
different between Libav (recursive) and FFmpeg (non-recursive).
The existing overlay_qsv implementation depends on the recursive version
of ff_request_frame to trigger immediate call to request_frame() on input pad.
But this has been removed in FFmpeg since "lavfi: make request_frame() non-recursive."
Now that we have handy framesync support in FFmpeg, so I make it work
based on framesync. Some other fixing which is also needed to make
overlay_qsv work are put in a separate patch.
add .preinit field to initilize framesync options.
export more options like vf_overlay.c
---
libavfilter/Makefile | 2 +-
libavfilter/vf_overlay_qsv.c | 213 ++++++++++++++++---------------------------
2 files changed, 78 insertions(+), 137 deletions(-)
diff --git a/libavfilter/Makefile b/libavfilter/Makefile
index a90ca30..7f2ad1f 100644
--- a/libavfilter/Makefile
+++ b/libavfilter/Makefile
@@ -267,7 +267,7 @@ OBJS-$(CONFIG_OSCILLOSCOPE_FILTER) += vf_datascope.o
OBJS-$(CONFIG_OVERLAY_FILTER) += vf_overlay.o framesync.o
OBJS-$(CONFIG_OVERLAY_OPENCL_FILTER) += vf_overlay_opencl.o opencl.o \
opencl/overlay.o framesync.o
-OBJS-$(CONFIG_OVERLAY_QSV_FILTER) += vf_overlay_qsv.o
+OBJS-$(CONFIG_OVERLAY_QSV_FILTER) += vf_overlay_qsv.o framesync.o
OBJS-$(CONFIG_OWDENOISE_FILTER) += vf_owdenoise.o
OBJS-$(CONFIG_PAD_FILTER) += vf_pad.o
OBJS-$(CONFIG_PALETTEGEN_FILTER) += vf_palettegen.o
diff --git a/libavfilter/vf_overlay_qsv.c b/libavfilter/vf_overlay_qsv.c
index 6c3efdb..2087178 100644
--- a/libavfilter/vf_overlay_qsv.c
+++ b/libavfilter/vf_overlay_qsv.c
@@ -36,6 +36,7 @@
#include "formats.h"
#include "video.h"
+#include "framesync.h"
#include "qsvvpp.h"
#define MAIN 0
@@ -56,14 +57,10 @@ enum var_name {
VAR_VARS_NB
};
-enum EOFAction {
- EOF_ACTION_REPEAT,
- EOF_ACTION_ENDALL
-};
-
typedef struct QSVOverlayContext {
const AVClass *class;
+ FFFrameSync fs;
QSVVPPContext *qsv;
QSVVPPParam qsv_param;
mfxExtVPPComposite comp_conf;
@@ -72,10 +69,6 @@ typedef struct QSVOverlayContext {
char *overlay_ox, *overlay_oy, *overlay_ow, *overlay_oh;
uint16_t overlay_alpha, overlay_pixel_alpha;
- enum EOFAction eof_action; /* action to take on EOF from source */
-
- AVFrame *main;
- AVFrame *over_prev, *over_next;
} QSVOverlayContext;
static const char *const var_names[] = {
@@ -90,20 +83,25 @@ static const char *const var_names[] = {
NULL
};
-static const AVOption options[] = {
+static const AVOption overlay_qsv_options[] = {
{ "x", "Overlay x position", OFFSET(overlay_ox), AV_OPT_TYPE_STRING, { .str="0"}, 0, 255, .flags = FLAGS},
{ "y", "Overlay y position", OFFSET(overlay_oy), AV_OPT_TYPE_STRING, { .str="0"}, 0, 255, .flags = FLAGS},
{ "w", "Overlay width", OFFSET(overlay_ow), AV_OPT_TYPE_STRING, { .str="overlay_iw"}, 0, 255, .flags = FLAGS},
{ "h", "Overlay height", OFFSET(overlay_oh), AV_OPT_TYPE_STRING, { .str="overlay_ih*w/overlay_iw"}, 0, 255, .flags = FLAGS},
{ "alpha", "Overlay global alpha", OFFSET(overlay_alpha), AV_OPT_TYPE_INT, { .i64 = 255}, 0, 255, .flags = FLAGS},
{ "eof_action", "Action to take when encountering EOF from secondary input ",
- OFFSET(eof_action), AV_OPT_TYPE_INT, { .i64 = EOF_ACTION_REPEAT },
- EOF_ACTION_REPEAT, EOF_ACTION_ENDALL, .flags = FLAGS, "eof_action" },
- { "repeat", "Repeat the previous frame.", 0, AV_OPT_TYPE_CONST, { .i64 = EOF_ACTION_REPEAT }, .flags = FLAGS, "eof_action" },
- { "endall", "End both streams.", 0, AV_OPT_TYPE_CONST, { .i64 = EOF_ACTION_ENDALL }, .flags = FLAGS, "eof_action" },
+ OFFSET(fs.opt_eof_action), AV_OPT_TYPE_INT, { .i64 = EOF_ACTION_REPEAT },
+ EOF_ACTION_REPEAT, EOF_ACTION_PASS, .flags = FLAGS, "eof_action" },
+ { "repeat", "Repeat the previous frame.", 0, AV_OPT_TYPE_CONST, { .i64 = EOF_ACTION_REPEAT }, .flags = FLAGS, "eof_action" },
+ { "endall", "End both streams.", 0, AV_OPT_TYPE_CONST, { .i64 = EOF_ACTION_ENDALL }, .flags = FLAGS, "eof_action" },
+ { "pass", "Pass through the main input.", 0, AV_OPT_TYPE_CONST, { .i64 = EOF_ACTION_PASS }, .flags = FLAGS, "eof_action" },
+ { "shortest", "force termination when the shortest input terminates", OFFSET(fs.opt_shortest), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, FLAGS },
+ { "repeatlast", "repeat overlay of the last overlay frame", OFFSET(fs.opt_repeatlast), AV_OPT_TYPE_BOOL, {.i64=1}, 0, 1, FLAGS },
{ NULL }
};
+FRAMESYNC_DEFINE_CLASS(overlay_qsv, QSVOverlayContext, fs);
+
static int eval_expr(AVFilterContext *ctx)
{
QSVOverlayContext *vpp = ctx->priv;
@@ -230,12 +228,53 @@ static int config_overlay_input(AVFilterLink *inlink)
return 0;
}
+static int process_frame(FFFrameSync *fs)
+{
+ AVFilterContext *ctx = fs->parent;
+ QSVOverlayContext *s = fs->opaque;
+ AVFrame *frame = NULL;
+ int ret = 0, i;
+
+ for (i = 0; i < ctx->nb_inputs; i++) {
+ ret = ff_framesync_get_frame(fs, i, &frame, 0);
+ if (ret == 0)
+ ret = ff_qsvvpp_filter_frame(s->qsv, ctx->inputs[i], frame);
+ if (ret < 0 && ret != AVERROR(EAGAIN))
+ break;
+ }
+
+ return ret;
+}
+
+static int init_framesync(AVFilterContext *ctx)
+{
+ QSVOverlayContext *s = ctx->priv;
+ int ret, i;
+
+ s->fs.on_event = process_frame;
+ s->fs.opaque = s;
+ ret = ff_framesync_init(&s->fs, ctx, ctx->nb_inputs);
+ if (ret < 0)
+ return ret;
+
+ for (i = 0; i < ctx->nb_inputs; i++) {
+ FFFrameSyncIn *in = &s->fs.in[i];
+ in->before = EXT_STOP;
+ in->after = EXT_INFINITY;
+ in->sync = i ? 1 : 2;
+ in->time_base = ctx->inputs[i]->time_base;
+ }
+
+ return ff_framesync_configure(&s->fs);
+}
+
static int config_output(AVFilterLink *outlink)
{
AVFilterContext *ctx = outlink->src;
QSVOverlayContext *vpp = ctx->priv;
AVFilterLink *in0 = ctx->inputs[0];
AVFilterLink *in1 = ctx->inputs[1];
+ int ret;
av_log(ctx, AV_LOG_DEBUG, "Output is of %s.\n", av_get_pix_fmt_name(outlink->format));
if ((in0->format == AV_PIX_FMT_QSV && in1->format != AV_PIX_FMT_QSV) ||
@@ -257,121 +296,27 @@ static int config_output(AVFilterLink *outlink)
outlink->frame_rate = in0->frame_rate;
outlink->time_base = av_inv_q(outlink->frame_rate);
- return ff_qsvvpp_create(ctx, &vpp->qsv, &vpp->qsv_param);
-}
-
-static int blend_frame(AVFilterContext *ctx, AVFrame *mpic, AVFrame *opic)
-{
- int ret = 0;
- QSVOverlayContext *vpp = ctx->priv;
- AVFrame *opic_copy = NULL;
-
- ret = ff_qsvvpp_filter_frame(vpp->qsv, ctx->inputs[0], mpic);
- if (ret == 0 || ret == AVERROR(EAGAIN)) {
- * 1. ff_qsvvpp_filter_frame will take control of the given frame
- * 2. We need to repeat the overlay frame when 2nd input goes into EOF
- */
- opic_copy = av_frame_clone(opic);
- if (!opic_copy)
- return AVERROR(ENOMEM);
-
- ret = ff_qsvvpp_filter_frame(vpp->qsv, ctx->inputs[1], opic_copy);
- }
-
- return ret;
-}
-
-static int handle_overlay_eof(AVFilterContext *ctx)
-{
- int ret = 0;
- QSVOverlayContext *s = ctx->priv;
- /* Repeat previous frame on secondary input */
- if (s->over_prev && s->eof_action == EOF_ACTION_REPEAT)
- ret = blend_frame(ctx, s->main, s->over_prev);
- /* End both streams */
- else if (s->eof_action == EOF_ACTION_ENDALL)
- return AVERROR_EOF;
-
- s->main = NULL;
+ ret = init_framesync(ctx);
+ if (ret < 0)
+ return ret;
- return ret;
+ return ff_qsvvpp_create(ctx, &vpp->qsv, &vpp->qsv_param);
}
-static int request_frame(AVFilterLink *outlink)
-{
- AVFilterContext *ctx = outlink->src;
- QSVOverlayContext *s = ctx->priv;
- AVRational tb_main = ctx->inputs[MAIN]->time_base;
- AVRational tb_over = ctx->inputs[OVERLAY]->time_base;
- int ret = 0;
-
- /* get a frame on the main input */
- if (!s->main) {
- ret = ff_request_frame(ctx->inputs[MAIN]);
- if (ret < 0)
- return ret;
- }
-
- /* get a new frame on the overlay input, on EOF check setting 'eof_action' */
- if (!s->over_next) {
- ret = ff_request_frame(ctx->inputs[OVERLAY]);
- if (ret == AVERROR_EOF)
- return handle_overlay_eof(ctx);
- else if (ret < 0)
- return ret;
- }
-
- while (s->main->pts != AV_NOPTS_VALUE &&
- s->over_next->pts != AV_NOPTS_VALUE &&
- av_compare_ts(s->over_next->pts, tb_over, s->main->pts, tb_main) < 0) {
- av_frame_free(&s->over_prev);
- FFSWAP(AVFrame*, s->over_prev, s->over_next);
-
- ret = ff_request_frame(ctx->inputs[OVERLAY]);
- if (ret == AVERROR_EOF)
- return handle_overlay_eof(ctx);
- else if (ret < 0)
- return ret;
- }
-
- if (s->main->pts == AV_NOPTS_VALUE ||
- s->over_next->pts == AV_NOPTS_VALUE ||
- !av_compare_ts(s->over_next->pts, tb_over, s->main->pts, tb_main)) {
- ret = blend_frame(ctx, s->main, s->over_next);
- av_frame_free(&s->over_prev);
- FFSWAP(AVFrame*, s->over_prev, s->over_next);
- } else if (s->over_prev) {
- ret = blend_frame(ctx, s->main, s->over_prev);
- } else {
- av_frame_free(&s->main);
- ret = AVERROR(EAGAIN);
- }
-
- s->main = NULL;
-
- return ret;
-}
+/*
+ * Callback for qsvvpp
+ * so we assign the PTS from framesync to the output frame.
+ */
-static int filter_frame_main(AVFilterLink *inlink, AVFrame *frame)
+static int filter_callback(AVFilterLink *outlink, AVFrame *frame)
{
- QSVOverlayContext *s = inlink->dst->priv;
-
- av_assert0(!s->main);
- s->main = frame;
-
- return 0;
+ QSVOverlayContext *s = outlink->src->priv;
+ frame->pts = av_rescale_q(s->fs.pts,
+ s->fs.time_base, outlink->time_base);
+ return ff_filter_frame(outlink, frame);
}
-static int filter_frame_overlay(AVFilterLink *inlink, AVFrame *frame)
-{
- QSVOverlayContext *s = inlink->dst->priv;
-
- av_assert0(!s->over_next);
- s->over_next = frame;
-
- return 0;
-}
static int overlay_qsv_init(AVFilterContext *ctx)
{
@@ -387,7 +332,7 @@ static int overlay_qsv_init(AVFilterContext *ctx)
return AVERROR(ENOMEM);
/* initialize QSVVPP params */
- vpp->qsv_param.filter_frame = NULL;
+ vpp->qsv_param.filter_frame = filter_callback;
vpp->qsv_param.ext_buf = av_mallocz(sizeof(*vpp->qsv_param.ext_buf));
if (!vpp->qsv_param.ext_buf)
return AVERROR(ENOMEM);
@@ -404,14 +349,18 @@ static void overlay_qsv_uninit(AVFilterContext *ctx)
{
QSVOverlayContext *vpp = ctx->priv;
- av_frame_free(&vpp->main);
- av_frame_free(&vpp->over_prev);
- av_frame_free(&vpp->over_next);
ff_qsvvpp_free(&vpp->qsv);
+ ff_framesync_uninit(&vpp->fs);
av_freep(&vpp->comp_conf.InputStream);
av_freep(&vpp->qsv_param.ext_buf);
}
+static int activate(AVFilterContext *ctx)
+{
+ QSVOverlayContext *s = ctx->priv;
+ return ff_framesync_activate(&s->fs);
+}
+
static int overlay_qsv_query_formats(AVFilterContext *ctx)
{
int i;
@@ -444,25 +393,16 @@ static int overlay_qsv_query_formats(AVFilterContext *ctx)
return 0;
}
-static const AVClass overlay_qsv_class = {
- .class_name = "overlay_qsv",
- .item_name = av_default_item_name,
- .option = options,
- .version = LIBAVUTIL_VERSION_INT,
-};
-
static const AVFilterPad overlay_qsv_inputs[] = {
{
.name = "main",
.type = AVMEDIA_TYPE_VIDEO,
- .filter_frame = filter_frame_main,
.config_props = config_main_input,
.needs_fifo = 1,
},
{
.name = "overlay",
.type = AVMEDIA_TYPE_VIDEO,
- .filter_frame = filter_frame_overlay,
.config_props = config_overlay_input,
.needs_fifo = 1,
},
@@ -474,7 +414,6 @@ static const AVFilterPad overlay_qsv_outputs[] = {
.name = "default",
.type = AVMEDIA_TYPE_VIDEO,
.config_props = config_output,
- .request_frame = request_frame,
},
{ NULL }
};
@@ -484,8 +423,10 @@ AVFilter ff_vf_overlay_qsv = {
.description = NULL_IF_CONFIG_SMALL("Quick Sync Video overlay."),
.priv_size = sizeof(QSVOverlayContext),
.query_formats = overlay_qsv_query_formats,
+ .preinit = overlay_qsv_framesync_preinit,
.init = overlay_qsv_init,
.uninit = overlay_qsv_uninit,
+ .activate = activate,
.inputs = overlay_qsv_inputs,
.outputs = overlay_qsv_outputs,
.priv_class = &overlay_qsv_class,
LGTM, In fact , it is a porting from libav with framesync.
Mark Thompson
2018-04-21 19:21:06 UTC
Permalink
Post by Ruiling Song
The existing version which was cherry-picked from Libav does not work
with FFmpeg framework, because ff_request_frame() was totally
different between Libav (recursive) and FFmpeg (non-recursive).
The existing overlay_qsv implementation depends on the recursive version
of ff_request_frame to trigger immediate call to request_frame() on input pad.
But this has been removed in FFmpeg since "lavfi: make request_frame() non-recursive."
Now that we have handy framesync support in FFmpeg, so I make it work
based on framesync. Some other fixing which is also needed to make
overlay_qsv work are put in a separate patch.
add .preinit field to initilize framesync options.
export more options like vf_overlay.c
---
libavfilter/Makefile | 2 +-
libavfilter/vf_overlay_qsv.c | 213 ++++++++++++++++---------------------------
2 files changed, 78 insertions(+), 137 deletions(-)
For filters based on framesync, the input frame was managed
by framesync, so we should not directly keep and destroy it,
instead we make a clone of it here, or else double-free will occur.
But for other filters not based on framesync, we still need to
free the input frame inside filter_frame. That's why I made
this v2 to fix the side-effect on normal filters.
and one av_frame_free() in vf_vpp_qsv.c
---
libavfilter/qsvvpp.c | 4 ++--
libavfilter/vf_vpp_qsv.c | 5 ++++-
2 files changed, 6 insertions(+), 3 deletions(-)
Tested, LGTM, set applied with one minor merge fixup - it collided with the change to pass through unmodified frames directly. (Apologies for the delay!)

Shall I apply this to the 4.0 branch as well?

Thanks,

- Mark
Li, Zhong
2018-04-23 01:49:20 UTC
Permalink
Of Mark Thompson
Sent: Sunday, April 22, 2018 3:21 AM
Subject: Re: [FFmpeg-devel] [PATCH v2 1/2] lavf: make overlay_qsv work
based on framesync
Post by Ruiling Song
The existing version which was cherry-picked from Libav does not work
with FFmpeg framework, because ff_request_frame() was totally
different between Libav (recursive) and FFmpeg (non-recursive).
The existing overlay_qsv implementation depends on the recursive
version of ff_request_frame to trigger immediate call to request_frame()
on input pad.
Post by Ruiling Song
But this has been removed in FFmpeg since "lavfi: make request_frame()
non-recursive."
Post by Ruiling Song
Now that we have handy framesync support in FFmpeg, so I make it work
based on framesync. Some other fixing which is also needed to make
overlay_qsv work are put in a separate patch.
add .preinit field to initilize framesync options.
export more options like vf_overlay.c
---
libavfilter/Makefile | 2 +-
libavfilter/vf_overlay_qsv.c | 213
++++++++++++++++---------------------------
2 files changed, 78 insertions(+), 137 deletions(-)
For filters based on framesync, the input frame was managed by
framesync, so we should not directly keep and destroy it, instead we
make a clone of it here, or else double-free will occur.
But for other filters not based on framesync, we still need to free
the input frame inside filter_frame. That's why I made this v2 to fix
the side-effect on normal filters.
and one av_frame_free() in vf_vpp_qsv.c
---
libavfilter/qsvvpp.c | 4 ++--
libavfilter/vf_vpp_qsv.c | 5 ++++-
2 files changed, 6 insertions(+), 3 deletions(-)
Tested, LGTM, set applied with one minor merge fixup - it collided with the
change to pass through unmodified frames directly. (Apologies for the
delay!)
Shall I apply this to the 4.0 branch as well?
Yes, merging it to 4.0 branch is a good idea (qsv_overly is declared but not workable now).
Thanks,
- Mark
Song, Ruiling
2018-04-23 07:22:23 UTC
Permalink
-----Original Message-----
Mark Thompson
Sent: Sunday, April 22, 2018 3:21 AM
Subject: Re: [FFmpeg-devel] [PATCH v2 1/2] lavf: make overlay_qsv work based
on framesync
Post by Ruiling Song
The existing version which was cherry-picked from Libav does not work
with FFmpeg framework, because ff_request_frame() was totally
different between Libav (recursive) and FFmpeg (non-recursive).
The existing overlay_qsv implementation depends on the recursive version
of ff_request_frame to trigger immediate call to request_frame() on input pad.
But this has been removed in FFmpeg since "lavfi: make request_frame() non-
recursive."
Post by Ruiling Song
Now that we have handy framesync support in FFmpeg, so I make it work
based on framesync. Some other fixing which is also needed to make
overlay_qsv work are put in a separate patch.
add .preinit field to initilize framesync options.
export more options like vf_overlay.c
---
libavfilter/Makefile | 2 +-
libavfilter/vf_overlay_qsv.c | 213 ++++++++++++++++---------------------------
2 files changed, 78 insertions(+), 137 deletions(-)
For filters based on framesync, the input frame was managed
by framesync, so we should not directly keep and destroy it,
instead we make a clone of it here, or else double-free will occur.
But for other filters not based on framesync, we still need to
free the input frame inside filter_frame. That's why I made
this v2 to fix the side-effect on normal filters.
and one av_frame_free() in vf_vpp_qsv.c
---
libavfilter/qsvvpp.c | 4 ++--
libavfilter/vf_vpp_qsv.c | 5 ++++-
2 files changed, 6 insertions(+), 3 deletions(-)
Tested, LGTM, set applied with one minor merge fixup - it collided with the
change to pass through unmodified frames directly. (Apologies for the delay!)
Shall I apply this to the 4.0 branch as well?
Sure, I think it's better to merge to 4.0. I noticed this feature was included in the release note.
Thanks Mark!

Ruiling
Thanks,
- Mark
_______________________________________________
ffmpeg-devel mailing list
http://ffmpeg.org/mailman/listinfo/ffmpeg-devel
Mark Thompson
2018-04-23 22:21:16 UTC
Permalink
Post by Song, Ruiling
-----Original Message-----
Mark Thompson
Sent: Sunday, April 22, 2018 3:21 AM
Subject: Re: [FFmpeg-devel] [PATCH v2 1/2] lavf: make overlay_qsv work based
on framesync
Post by Ruiling Song
The existing version which was cherry-picked from Libav does not work
with FFmpeg framework, because ff_request_frame() was totally
different between Libav (recursive) and FFmpeg (non-recursive).
The existing overlay_qsv implementation depends on the recursive version
of ff_request_frame to trigger immediate call to request_frame() on input pad.
But this has been removed in FFmpeg since "lavfi: make request_frame() non-
recursive."
Post by Ruiling Song
Now that we have handy framesync support in FFmpeg, so I make it work
based on framesync. Some other fixing which is also needed to make
overlay_qsv work are put in a separate patch.
add .preinit field to initilize framesync options.
export more options like vf_overlay.c
---
libavfilter/Makefile | 2 +-
libavfilter/vf_overlay_qsv.c | 213 ++++++++++++++++---------------------------
2 files changed, 78 insertions(+), 137 deletions(-)
For filters based on framesync, the input frame was managed
by framesync, so we should not directly keep and destroy it,
instead we make a clone of it here, or else double-free will occur.
But for other filters not based on framesync, we still need to
free the input frame inside filter_frame. That's why I made
this v2 to fix the side-effect on normal filters.
and one av_frame_free() in vf_vpp_qsv.c
---
libavfilter/qsvvpp.c | 4 ++--
libavfilter/vf_vpp_qsv.c | 5 ++++-
2 files changed, 6 insertions(+), 3 deletions(-)
Tested, LGTM, set applied with one minor merge fixup - it collided with the
change to pass through unmodified frames directly. (Apologies for the delay!)
Shall I apply this to the 4.0 branch as well?
Sure, I think it's better to merge to 4.0. I noticed this feature was included in the release note.
Ok, I've cherry-picked it to the 4.0 branch as well.

Thanks,

- Mark

Loading...