/* GStreamer * Copyright (C) 2020 Igalia, S.L. * Author: Víctor Jáquez * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Library General Public License for more details. * * You should have received a copy of the GNU Library General Public * License along with this library; if not, write to the0 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, * Boston, MA 02110-1301, USA. */ #include "gstvabasedec.h" #include #include #include #include "gstvacaps.h" #include "gstvapluginutils.h" #define GST_CAT_DEFAULT (base->debug_category) #define GST_VA_BASE_DEC_GET_PARENT_CLASS(obj) (GST_VA_BASE_DEC_GET_CLASS(obj)->parent_decoder_class) static void gst_va_base_dec_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec) { GstVaBaseDec *self = GST_VA_BASE_DEC (object); GstVaBaseDecClass *klass = GST_VA_BASE_DEC_GET_CLASS (self); switch (prop_id) { case GST_VA_DEC_PROP_DEVICE_PATH:{ if (!self->display) g_value_set_string (value, klass->render_device_path); else if (GST_IS_VA_DISPLAY_PLATFORM (self->display)) g_object_get_property (G_OBJECT (self->display), "path", value); else g_value_set_string (value, NULL); break; } default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); } } static gboolean gst_va_base_dec_open (GstVideoDecoder * decoder) { GstVaBaseDec *base = GST_VA_BASE_DEC (decoder); GstVaBaseDecClass *klass = GST_VA_BASE_DEC_GET_CLASS (decoder); gboolean ret = FALSE; if (!gst_va_ensure_element_data (decoder, klass->render_device_path, &base->display)) return FALSE; g_object_notify (G_OBJECT (decoder), "device-path"); if (!g_atomic_pointer_get (&base->decoder)) { GstVaDecoder *va_decoder; va_decoder = gst_va_decoder_new (base->display, klass->codec); if (va_decoder) ret = TRUE; gst_object_replace ((GstObject **) (&base->decoder), (GstObject *) va_decoder); gst_clear_object (&va_decoder); } else { ret = TRUE; } base->apply_video_crop = FALSE; return ret; } gboolean gst_va_base_dec_close (GstVideoDecoder * decoder) { GstVaBaseDec *base = GST_VA_BASE_DEC (decoder); gst_clear_object (&base->decoder); gst_clear_object (&base->display); g_object_notify (G_OBJECT (decoder), "device-path"); return TRUE; } static gboolean gst_va_base_dec_stop (GstVideoDecoder * decoder) { GstVaBaseDec *base = GST_VA_BASE_DEC (decoder); if (!gst_va_decoder_close (base->decoder)) return FALSE; g_clear_pointer (&base->output_state, gst_video_codec_state_unref); g_clear_pointer (&base->input_state, gst_video_codec_state_unref); if (base->other_pool) gst_buffer_pool_set_active (base->other_pool, FALSE); gst_clear_object (&base->other_pool); g_clear_pointer (&base->convert, gst_video_converter_free); return GST_VIDEO_DECODER_CLASS (GST_VA_BASE_DEC_GET_PARENT_CLASS (decoder))->stop (decoder); } static GstCaps * gst_va_base_dec_getcaps (GstVideoDecoder * decoder, GstCaps * filter) { GstCaps *caps = NULL, *tmp; GstVaBaseDec *base = GST_VA_BASE_DEC (decoder); GstVaDecoder *va_decoder = NULL; gst_object_replace ((GstObject **) & va_decoder, (GstObject *) base->decoder); if (va_decoder) { caps = gst_va_decoder_get_sinkpad_caps (va_decoder); gst_object_unref (va_decoder); } if (caps) { if (filter) { tmp = gst_caps_intersect_full (filter, caps, GST_CAPS_INTERSECT_FIRST); gst_caps_unref (caps); caps = tmp; } GST_LOG_OBJECT (base, "Returning caps %" GST_PTR_FORMAT, caps); } else { caps = gst_video_decoder_proxy_getcaps (decoder, NULL, filter); } return caps; } static gboolean _query_context (GstVaBaseDec * self, GstQuery * query) { GstVaDisplay *display = NULL; gboolean ret; gst_object_replace ((GstObject **) & display, (GstObject *) self->display); ret = gst_va_handle_context_query (GST_ELEMENT_CAST (self), query, display); gst_clear_object (&display); return ret; } static gboolean gst_va_base_dec_src_query (GstVideoDecoder * decoder, GstQuery * query) { GstVaBaseDec *base = GST_VA_BASE_DEC (decoder); gboolean ret = FALSE; switch (GST_QUERY_TYPE (query)) { case GST_QUERY_CONTEXT:{ ret = _query_context (base, query); break; } case GST_QUERY_CAPS:{ GstCaps *caps = NULL, *tmp, *filter = NULL; GstVaDecoder *va_decoder = NULL; gboolean fixed_caps; gst_object_replace ((GstObject **) & va_decoder, (GstObject *) base->decoder); gst_query_parse_caps (query, &filter); fixed_caps = GST_PAD_IS_FIXED_CAPS (GST_VIDEO_DECODER_SRC_PAD (decoder)); if (!fixed_caps && va_decoder) caps = gst_va_decoder_get_srcpad_caps (va_decoder); gst_clear_object (&va_decoder); if (caps) { if (filter) { tmp = gst_caps_intersect_full (filter, caps, GST_CAPS_INTERSECT_FIRST); gst_caps_unref (caps); caps = tmp; } GST_LOG_OBJECT (base, "Returning caps %" GST_PTR_FORMAT, caps); gst_query_set_caps_result (query, caps); gst_caps_unref (caps); ret = TRUE; break; } } /* FALLTHROUGH */ default: ret = GST_VIDEO_DECODER_CLASS (GST_VA_BASE_DEC_GET_PARENT_CLASS (decoder))->src_query (decoder, query); break; } return ret; } static gboolean gst_va_base_dec_sink_query (GstVideoDecoder * decoder, GstQuery * query) { if (GST_QUERY_TYPE (query) == GST_QUERY_CONTEXT) return _query_context (GST_VA_BASE_DEC (decoder), query); return GST_VIDEO_DECODER_CLASS (GST_VA_BASE_DEC_GET_PARENT_CLASS (decoder))->sink_query (decoder, query); } static GstAllocator * _create_allocator (GstVaBaseDec * base, GstCaps * caps) { GstAllocator *allocator = NULL; if (gst_video_is_dma_drm_caps (caps)) allocator = gst_va_dmabuf_allocator_new (base->display); else { GArray *surface_formats = gst_va_decoder_get_surface_formats (base->decoder); allocator = gst_va_allocator_new (base->display, surface_formats); gst_va_allocator_set_hacks (allocator, base->hacks); } return allocator; } static gboolean _need_video_crop (GstVaBaseDec * base) { if (base->need_valign && (base->valign.padding_left > 0 || base->valign.padding_top > 0)) return TRUE; return FALSE; } static GstVaFeature _use_derived (GstVaBaseDec * base) { /* i965 driver has a very poor performance reading derived images */ if (GST_VA_DISPLAY_IS_IMPLEMENTATION (base->display, INTEL_I965)) return GST_VA_FEATURE_DISABLED; return GST_VA_FEATURE_AUTO; } /* This path for pool setting is a little complicated but not commonly used. We deliberately separate it from the main path of pool setting. */ static gboolean _decide_allocation_for_video_crop (GstVideoDecoder * decoder, GstQuery * query, GstCaps * caps) { GstAllocator *allocator = NULL, *other_allocator = NULL; GstAllocationParams other_params = { 0, }, params = { 0, }; gboolean update_pool = FALSE, update_allocator = FALSE; GstBufferPool *pool = NULL, *other_pool = NULL; guint size = 0, min, max, usage_hint; GstVaBaseDec *base = GST_VA_BASE_DEC (decoder); gboolean ret = FALSE; gboolean dont_use_other_pool = FALSE; GstCaps *va_caps = NULL; /* If others provide a valid allocator, just use it. */ if (gst_query_get_n_allocation_params (query) > 0) { gst_query_parse_nth_allocation_param (query, 0, &other_allocator, &other_params); GstVaDisplay *display; display = gst_va_allocator_peek_display (other_allocator); /* We should not use allocator and pool from other display. */ if (display != base->display) { gst_clear_object (&other_allocator); dont_use_other_pool = TRUE; } update_allocator = TRUE; } else { gst_allocation_params_init (&other_params); } /* If others provide a valid pool, just use it. */ if (gst_query_get_n_allocation_pools (query) > 0) { gst_query_parse_nth_allocation_pool (query, 0, &other_pool, &size, &min, &max); if (dont_use_other_pool) gst_clear_object (&other_pool); min += base->min_buffers; update_pool = TRUE; } else { min = base->min_buffers; max = 0; } /* Ensure that the other pool is ready */ if (gst_caps_is_raw (caps)) { if (GST_IS_VA_POOL (other_pool)) { gst_clear_object (&other_pool); size = 0; } if (!other_pool) { if (other_allocator && (GST_IS_VA_DMABUF_ALLOCATOR (other_allocator) || GST_IS_VA_ALLOCATOR (other_allocator))) gst_clear_object (&other_allocator); GST_DEBUG_OBJECT (base, "making new other pool for copy"); base->other_pool = gst_va_create_other_pool (other_allocator, &other_params, caps, size); } else { gst_object_replace ((GstObject **) & base->other_pool, (GstObject *) other_pool); } if (!base->other_pool) { GST_ERROR_OBJECT (base, "Couldn't configure other pool for copy"); goto cleanup; } } else { GstStructure *other_config; if (!GST_IS_VA_POOL (other_pool)) gst_clear_object (&other_pool); if (!other_pool) other_pool = gst_va_pool_new (); if (other_allocator && !(GST_IS_VA_DMABUF_ALLOCATOR (other_allocator) || GST_IS_VA_ALLOCATOR (other_allocator))) gst_clear_object (&other_allocator); if (!other_allocator) { other_allocator = _create_allocator (base, caps); if (!other_allocator) goto cleanup; } other_config = gst_buffer_pool_get_config (other_pool); gst_buffer_pool_config_set_params (other_config, caps, 0, min, max); gst_buffer_pool_config_set_allocator (other_config, other_allocator, &other_params); /* Always support VideoMeta but no VideoCropMeta here. */ gst_buffer_pool_config_add_option (other_config, GST_BUFFER_POOL_OPTION_VIDEO_META); gst_buffer_pool_config_set_va_allocation_params (other_config, VA_SURFACE_ATTRIB_USAGE_HINT_GENERIC, _use_derived (base)); if (!gst_buffer_pool_set_config (other_pool, other_config)) goto cleanup; gst_object_replace ((GstObject **) & base->other_pool, (GstObject *) other_pool); } /* Now setup the buffer pool for decoder */ va_caps = gst_caps_copy (caps); gst_caps_set_features_simple (va_caps, gst_caps_features_new_single_static_str (GST_CAPS_FEATURE_MEMORY_VA)); if (!(allocator = _create_allocator (base, va_caps))) goto cleanup; gst_allocation_params_init (¶ms); pool = gst_va_pool_new (); { GstStructure *config = gst_buffer_pool_get_config (pool); gst_buffer_pool_config_set_params (config, caps, 0, min, max); gst_buffer_pool_config_set_allocator (config, allocator, ¶ms); gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META); if (_need_video_crop (base)) gst_buffer_pool_config_set_va_alignment (config, &base->valign); usage_hint = va_get_surface_usage_hint (base->display, VAEntrypointVLD, GST_PAD_SRC, gst_video_is_dma_drm_caps (caps)); gst_buffer_pool_config_set_va_allocation_params (config, usage_hint, _use_derived (base)); if (!gst_buffer_pool_set_config (pool, config)) goto cleanup; if (!gst_va_pool_get_buffer_size (pool, &size)) goto cleanup; } if (update_allocator) gst_query_set_nth_allocation_param (query, 0, allocator, ¶ms); else gst_query_add_allocation_param (query, allocator, ¶ms); if (update_pool) gst_query_set_nth_allocation_pool (query, 0, pool, size, min, max); else gst_query_add_allocation_pool (query, pool, size, min, max); GST_WARNING_OBJECT (base, "We need to copy the output buffer manually " "because of the top/left alignment, which may have low performance. " "The element which supports VideoCropMeta such as 'vapostproc' can " "avoid this."); base->copy_frames = TRUE; base->apply_video_crop = TRUE; ret = TRUE; cleanup: gst_clear_object (&allocator); gst_clear_object (&other_allocator); gst_clear_object (&pool); gst_clear_object (&other_pool); gst_clear_caps (&va_caps); return ret; } /* We only support system pool and va pool. For va pool, its allocator * should be va allocator or dma allocator. * If output caps is memory:VAMemory, the pool should be a va pool * with va allocator. * If output caps is memory:DMABuf, the pool should be a va pool * with dma allocator. * We may need the other_pool to copy the decoder picture to the * output buffer. We need to do this copy when: * 1). The output caps is raw(system mem), but the downstream does * not support VideoMeta and the strides and offsets of the va pool * are different from the system memory pool, which means that the * gst_video_frame_map() can not map the buffer correctly. Then we * need a va pool with va allocator as an the internal pool and create * a system pool as the other_pool to copy frames to system mem and * output it. * 2). The decoder has crop_top/left value > 0(e.g. the conformance * window in the H265). Which means that the real output picture * locates in the middle of the decoded buffer. If the downstream can * support VideoCropMeta, a VideoCropMeta is added to notify the * real picture's coordinate and size. But if not, we need to copy * it manually and the other_pool is needed. We always assume that * decoded picture starts from top-left corner, and so there is no * need to do this if crop_bottom/right value > 0. * * 1. if crop_top/left value > 0 and the downstream does not support the * VideoCropMeta, we always have the other_pool to do the copy(The pool * may be provided by the downstream element, or created by ourself if * no suitable one found). * 2. get allocator in query * 2.1 if allocator is not ours and caps is raw, keep it for other_pool. * 3. get pool in query * 3.1 if pool is not va, downstream doesn't support video meta and * caps are raw, keep it as other_pool. * 3.2 if there's no pool in query and and caps is raw, create other_pool * as GstVideoPool with the non-va from query and query's params. * 4. create our allocator and pool if they aren't in query * 5. add or update pool and allocator in query * 6. set our custom pool configuration */ static gboolean gst_va_base_dec_decide_allocation (GstVideoDecoder * decoder, GstQuery * query) { GstAllocator *allocator = NULL, *other_allocator = NULL; GstAllocationParams other_params = { 0, }, params = { 0, }; GstBufferPool *pool = NULL, *other_pool = NULL; GstCaps *caps = NULL; GstVaBaseDec *base = GST_VA_BASE_DEC (decoder); guint other_size = 0, size = 0, min, max, usage_hint; gboolean update_pool = FALSE, update_allocator = FALSE; gboolean has_videometa, has_video_crop_meta; gboolean dont_use_other_pool = FALSE, ret = FALSE; if (base->min_buffers == 0) { GST_DEBUG_OBJECT (base, "Don't know minimum number of required buffers yet"); return GST_VIDEO_DECODER_CLASS (GST_VA_BASE_DEC_GET_PARENT_CLASS (decoder))->decide_allocation (decoder, query); } gst_query_parse_allocation (query, &caps, NULL); if (!caps) goto wrong_caps; has_videometa = gst_query_find_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL); has_video_crop_meta = has_videometa && gst_query_find_allocation_meta (query, GST_VIDEO_CROP_META_API_TYPE, NULL); if (gst_video_is_dma_drm_caps (caps) && !has_videometa) { GST_ERROR_OBJECT (base, "DMABuf caps negotiated without the mandatory support of VideoMeta "); return FALSE; } /* 1. The output picture locates in the middle of the decoded buffer, but the downstream element does not support VideoCropMeta, we definitely need a copy. 2. Some codec such as H265, it does not clean the DPB when new SPS comes. The new SPS may set the crop window to top-left corner and so no video crop is needed here. But we may still have cached frames in DPB which need a copy. 3. For DMA kind memory, because we may not be able to map this buffer, just disable the copy for crop. This may cause some alignment garbage. */ if (!gst_video_is_dma_drm_caps (caps) && ((_need_video_crop (base) && !has_video_crop_meta) || base->apply_video_crop)) { return _decide_allocation_for_video_crop (decoder, query, caps); } if (gst_query_get_n_allocation_params (query) > 0) { GstVaDisplay *display; gst_query_parse_nth_allocation_param (query, 0, &allocator, &other_params); display = gst_va_allocator_peek_display (allocator); if (!display) { /* save the allocator for the other pool */ other_allocator = allocator; allocator = NULL; } else if (display != base->display) { /* The allocator and pool belong to other display, we should not use. */ gst_clear_object (&allocator); dont_use_other_pool = TRUE; } update_allocator = TRUE; } else { gst_allocation_params_init (&other_params); } gst_allocation_params_init (¶ms); if (gst_query_get_n_allocation_pools (query) > 0) { gst_query_parse_nth_allocation_pool (query, 0, &pool, &size, &min, &max); if (pool) { if (!GST_IS_VA_POOL (pool)) { GST_DEBUG_OBJECT (base, "may need other pool for copy frames %" GST_PTR_FORMAT, pool); other_pool = pool; pool = NULL; other_size = size; } else if (dont_use_other_pool) { gst_clear_object (&pool); } } min += base->min_buffers; update_pool = TRUE; } else { min = base->min_buffers; max = 0; } if (!allocator) { if (!(allocator = _create_allocator (base, caps))) goto cleanup; } if (!pool) pool = gst_va_pool_new (); { GstStructure *config = gst_buffer_pool_get_config (pool); gst_buffer_pool_config_set_params (config, caps, 0, min, max); gst_buffer_pool_config_set_allocator (config, allocator, ¶ms); gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META); if (base->need_valign) gst_buffer_pool_config_set_va_alignment (config, &base->valign); usage_hint = va_get_surface_usage_hint (base->display, VAEntrypointVLD, GST_PAD_SRC, gst_video_is_dma_drm_caps (caps)); gst_buffer_pool_config_set_va_allocation_params (config, usage_hint, _use_derived (base)); if (!gst_buffer_pool_set_config (pool, config)) goto cleanup; if (!gst_va_pool_get_buffer_size (pool, &size)) goto cleanup; } if (update_allocator) gst_query_set_nth_allocation_param (query, 0, allocator, ¶ms); else gst_query_add_allocation_param (query, allocator, ¶ms); if (update_pool) gst_query_set_nth_allocation_pool (query, 0, pool, size, min, max); else gst_query_add_allocation_pool (query, pool, size, min, max); base->copy_frames = (!has_videometa && gst_va_pool_requires_video_meta (pool) && gst_caps_is_raw (caps)); if (base->copy_frames) { if (other_pool) { gst_object_replace ((GstObject **) & base->other_pool, (GstObject *) other_pool); } else { gst_clear_object (&base->other_pool); base->other_pool = gst_va_create_other_pool (other_allocator, &other_params, caps, other_size); } if (!base->other_pool) { GST_ERROR_OBJECT (base, "Couldn't configure other pool for copy"); goto cleanup; } GST_DEBUG_OBJECT (base, "Use the other pool for copy %" GST_PTR_FORMAT, base->other_pool); } else { gst_clear_object (&base->other_pool); } ret = TRUE; cleanup: gst_clear_object (&allocator); gst_clear_object (&other_allocator); gst_clear_object (&pool); gst_clear_object (&other_pool); /* There's no need to chain decoder's method since all what is * needed is done. */ return ret; wrong_caps: { GST_WARNING_OBJECT (base, "No valid caps"); return FALSE; } } static void gst_va_base_dec_set_context (GstElement * element, GstContext * context) { GstVaDisplay *old_display, *new_display; GstVaBaseDec *base = GST_VA_BASE_DEC (element); GstVaBaseDecClass *klass = GST_VA_BASE_DEC_GET_CLASS (base); gboolean ret; old_display = base->display ? gst_object_ref (base->display) : NULL; ret = gst_va_handle_set_context (element, context, klass->render_device_path, &base->display); new_display = base->display ? gst_object_ref (base->display) : NULL; if (!ret || (old_display && new_display && old_display != new_display && base->decoder)) { GST_WARNING_OBJECT (base, "Can't replace VA display while operating"); } gst_clear_object (&old_display); gst_clear_object (&new_display); GST_ELEMENT_CLASS (GST_VA_BASE_DEC_GET_PARENT_CLASS (element))->set_context (element, context); } static gboolean gst_va_base_dec_negotiate (GstVideoDecoder * decoder) { GstVaBaseDec *base = GST_VA_BASE_DEC (decoder); /* Do not (re-)open the decoder in case the input state hasn't changed. */ if (!base->need_negotiation) { GST_DEBUG_OBJECT (decoder, "Input state hasn't changed, no need to (re-)open the decoder"); goto done; } base->need_negotiation = FALSE; if (!gst_va_decoder_config_is_equal (base->decoder, base->profile, base->rt_format, base->width, base->height)) { if (gst_va_decoder_is_open (base->decoder) && !gst_va_decoder_close (base->decoder)) { GST_WARNING_OBJECT (decoder, "Failed to close decoder"); return FALSE; } if (!gst_va_decoder_open (base->decoder, base->profile, base->rt_format)) { GST_WARNING_OBJECT (decoder, "Failed to open decoder"); return FALSE; } if (!gst_va_decoder_set_frame_size (base->decoder, base->width, base->height)) { GST_WARNING_OBJECT (decoder, "Failed to set frame size"); return FALSE; } } if (!gst_va_base_dec_set_output_state (base)) { GST_WARNING_OBJECT (decoder, "Failed to set output state"); return FALSE; } done: return GST_VIDEO_DECODER_CLASS (GST_VA_BASE_DEC_GET_PARENT_CLASS (decoder)) ->negotiate (decoder); } void gst_va_base_dec_init (GstVaBaseDec * base, GstDebugCategory * cat) { base->debug_category = cat; gst_video_info_init (&base->output_info); } void gst_va_base_dec_class_init (GstVaBaseDecClass * klass, GstVaCodecs codec, const gchar * render_device_path, GstCaps * sink_caps, GstCaps * src_caps, GstCaps * doc_src_caps, GstCaps * doc_sink_caps) { GstPadTemplate *sink_pad_templ, *src_pad_templ; GObjectClass *object_class = G_OBJECT_CLASS (klass); GstElementClass *element_class = GST_ELEMENT_CLASS (klass); GstVideoDecoderClass *decoder_class = GST_VIDEO_DECODER_CLASS (klass); klass->parent_decoder_class = g_type_class_peek_parent (klass); klass->codec = codec; klass->render_device_path = g_strdup (render_device_path); sink_pad_templ = gst_pad_template_new ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, sink_caps); gst_element_class_add_pad_template (element_class, sink_pad_templ); if (doc_sink_caps) { gst_pad_template_set_documentation_caps (sink_pad_templ, doc_sink_caps); gst_caps_unref (doc_sink_caps); } src_pad_templ = gst_pad_template_new ("src", GST_PAD_SRC, GST_PAD_ALWAYS, src_caps); gst_element_class_add_pad_template (element_class, src_pad_templ); if (doc_src_caps) { gst_pad_template_set_documentation_caps (src_pad_templ, doc_src_caps); gst_caps_unref (doc_src_caps); } object_class->get_property = gst_va_base_dec_get_property; element_class->set_context = GST_DEBUG_FUNCPTR (gst_va_base_dec_set_context); decoder_class->open = GST_DEBUG_FUNCPTR (gst_va_base_dec_open); decoder_class->close = GST_DEBUG_FUNCPTR (gst_va_base_dec_close); decoder_class->stop = GST_DEBUG_FUNCPTR (gst_va_base_dec_stop); decoder_class->getcaps = GST_DEBUG_FUNCPTR (gst_va_base_dec_getcaps); decoder_class->src_query = GST_DEBUG_FUNCPTR (gst_va_base_dec_src_query); decoder_class->sink_query = GST_DEBUG_FUNCPTR (gst_va_base_dec_sink_query); decoder_class->decide_allocation = GST_DEBUG_FUNCPTR (gst_va_base_dec_decide_allocation); decoder_class->negotiate = GST_DEBUG_FUNCPTR (gst_va_base_dec_negotiate); g_object_class_install_property (object_class, GST_VA_DEC_PROP_DEVICE_PATH, g_param_spec_string ("device-path", "Device Path", GST_VA_DEVICE_PATH_PROP_DESC, NULL, GST_PARAM_DOC_SHOW_DEFAULT | G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)); } static inline GstVideoFormat _get_video_format_from_value (const GValue * format, gboolean drm_format, guint64 * modifier) { guint32 fourcc; const char *fmt; g_assert (G_VALUE_HOLDS_STRING (format)); fmt = g_value_get_string (format); if (drm_format) { fourcc = gst_video_dma_drm_fourcc_from_string (fmt, modifier); return gst_va_video_format_from_drm_fourcc (fourcc); } return gst_video_format_from_string (fmt); } static GstVideoFormat _find_video_format_from_chroma (const GValue * formats, guint chroma_type, gboolean drm_format, guint64 * modifier) { GstVideoFormat fmt = GST_VIDEO_FORMAT_UNKNOWN; guint i, num_values; if (!formats) return GST_VIDEO_FORMAT_UNKNOWN; if (G_VALUE_HOLDS_STRING (formats)) { fmt = _get_video_format_from_value (formats, drm_format, modifier); if (gst_va_chroma_from_video_format (fmt) == chroma_type) return fmt; } else if (GST_VALUE_HOLDS_LIST (formats)) { const GValue *format; num_values = gst_value_list_get_size (formats); for (i = 0; i < num_values; i++) { format = gst_value_list_get_value (formats, i); fmt = _get_video_format_from_value (format, drm_format, modifier); if (gst_va_chroma_from_video_format (fmt) == chroma_type) return fmt; } } return GST_VIDEO_FORMAT_UNKNOWN; } static GstVideoFormat _caps_video_format_from_chroma (GstCaps * caps, guint chroma_type) { guint i, num_structures; GstCapsFeatures *feats; GstStructure *structure; const GValue *format; GstVideoFormat fmt; num_structures = gst_caps_get_size (caps); for (i = 0; i < num_structures; i++) { feats = gst_caps_get_features (caps, i); if (!gst_caps_features_is_equal (feats, GST_CAPS_FEATURES_MEMORY_SYSTEM_MEMORY)) continue; structure = gst_caps_get_structure (caps, i); format = gst_structure_get_value (structure, "format"); fmt = _find_video_format_from_chroma (format, chroma_type, FALSE, NULL); if (fmt == GST_VIDEO_FORMAT_UNKNOWN) continue; if (gst_va_chroma_from_video_format (fmt) == chroma_type) return fmt; } return GST_VIDEO_FORMAT_UNKNOWN; } /* ordered list of capsfeature preference */ enum { VA, DMABUF, SYSMEM }; void gst_va_base_dec_get_preferred_format_and_caps_features (GstVaBaseDec * base, GstVideoFormat * format, GstCapsFeatures ** capsfeatures, guint64 * modifier) { GstCaps *peer_caps, *allowed_caps; GstCapsFeatures *features; guint num_structures, i, j; gboolean is_any; GstIdStr sysmem = GST_ID_STR_INIT, dmabuf = GST_ID_STR_INIT, va = GST_ID_STR_INIT; const GstIdStr *feats[] = { &va, &dmabuf, &sysmem }; gst_id_str_set_static_str (&sysmem, GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY); gst_id_str_set_static_str (&dmabuf, GST_CAPS_FEATURE_MEMORY_DMABUF); gst_id_str_set_static_str (&va, GST_CAPS_FEATURE_MEMORY_VA); g_return_if_fail (base); /* verify if peer caps is any */ { peer_caps = gst_pad_peer_query_caps (GST_VIDEO_DECODER_SRC_PAD (base), NULL); is_any = gst_caps_is_any (peer_caps); gst_clear_caps (&peer_caps); } allowed_caps = gst_pad_get_allowed_caps (GST_VIDEO_DECODER_SRC_PAD (base)); GST_DEBUG_OBJECT (base, "Allowed caps %" GST_PTR_FORMAT, allowed_caps); /* if peer caps is any, returns format according memory to system caps in * allowed caps * * if downstream doesn't support system memory negotiation will fail later. */ if (is_any) { GstVideoFormat fmt = _caps_video_format_from_chroma (allowed_caps, base->rt_format); features = GST_CAPS_FEATURES_MEMORY_SYSTEM_MEMORY; if (format) *format = fmt; if (capsfeatures && fmt != GST_VIDEO_FORMAT_UNKNOWN) *capsfeatures = gst_caps_features_copy (features); goto bail; } /* iterate allowed caps to find the first "capable" capability according our * ordered list of preferred caps features */ num_structures = gst_caps_get_size (allowed_caps); for (i = 0; i < G_N_ELEMENTS (feats); i++) { for (j = 0; j < num_structures; j++) { GstStructure *structure; const GValue *formats; GstVideoFormat fmt; guint64 mod = 0; features = gst_caps_get_features (allowed_caps, j); if (!gst_caps_features_contains_id_str (features, feats[i])) continue; structure = gst_caps_get_structure (allowed_caps, j); if (i == DMABUF) formats = gst_structure_get_value (structure, "drm-format"); else formats = gst_structure_get_value (structure, "format"); fmt = _find_video_format_from_chroma (formats, base->rt_format, i == DMABUF, &mod); /* if doesn't found a proper format let's try other structure */ if (fmt == GST_VIDEO_FORMAT_UNKNOWN) continue; if (format) *format = fmt; if (i == DMABUF && modifier) *modifier = mod; if (capsfeatures) *capsfeatures = gst_caps_features_new_id_str (feats[i], NULL); goto bail; } } /* no matching format, let's fail */ if (i == G_N_ELEMENTS (feats)) *format = GST_VIDEO_FORMAT_UNKNOWN; bail: gst_caps_unref (allowed_caps); } static gboolean _copy_buffer_and_apply_video_crop (GstVaBaseDec * base, GstVideoFrame * src_frame, GstVideoFrame * dest_frame, GstVideoCropMeta * video_crop) { GstVideoInfo dst_info = dest_frame->info; dst_info.fps_n = src_frame->info.fps_n; dst_info.fps_d = src_frame->info.fps_d; if (base->convert) { gboolean new_convert = FALSE; gint x = 0, y = 0, width = 0, height = 0; const GstStructure *config = gst_video_converter_get_config (base->convert); if (!gst_structure_get_int (config, GST_VIDEO_CONVERTER_OPT_SRC_X, &x) || !gst_structure_get_int (config, GST_VIDEO_CONVERTER_OPT_SRC_Y, &y) || !gst_structure_get_int (config, GST_VIDEO_CONVERTER_OPT_SRC_WIDTH, &width) || !gst_structure_get_int (config, GST_VIDEO_CONVERTER_OPT_SRC_HEIGHT, &height)) new_convert = TRUE; new_convert |= (video_crop->x != x); new_convert |= (video_crop->y != y); new_convert |= (video_crop->width != width); new_convert |= (video_crop->height != height); /* No need to check dest, it always has (0,0) -> (width, height) */ if (new_convert) g_clear_pointer (&base->convert, gst_video_converter_free); } if (!base->convert) { base->convert = gst_video_converter_new (&src_frame->info, &dst_info, gst_structure_new ("options", GST_VIDEO_CONVERTER_OPT_DITHER_METHOD, GST_TYPE_VIDEO_DITHER_METHOD, GST_VIDEO_DITHER_NONE, GST_VIDEO_CONVERTER_OPT_DITHER_QUANTIZATION, G_TYPE_UINT, 0, GST_VIDEO_CONVERTER_OPT_CHROMA_MODE, GST_TYPE_VIDEO_CHROMA_MODE, GST_VIDEO_CHROMA_MODE_NONE, GST_VIDEO_CONVERTER_OPT_MATRIX_MODE, GST_TYPE_VIDEO_MATRIX_MODE, GST_VIDEO_MATRIX_MODE_NONE, GST_VIDEO_CONVERTER_OPT_SRC_X, G_TYPE_INT, video_crop->x, GST_VIDEO_CONVERTER_OPT_SRC_Y, G_TYPE_INT, video_crop->y, GST_VIDEO_CONVERTER_OPT_SRC_WIDTH, G_TYPE_INT, video_crop->width, GST_VIDEO_CONVERTER_OPT_SRC_HEIGHT, G_TYPE_INT, video_crop->height, GST_VIDEO_CONVERTER_OPT_DEST_X, G_TYPE_INT, 0, GST_VIDEO_CONVERTER_OPT_DEST_Y, G_TYPE_INT, 0, GST_VIDEO_CONVERTER_OPT_DEST_WIDTH, G_TYPE_INT, video_crop->width, GST_VIDEO_CONVERTER_OPT_DEST_HEIGHT, G_TYPE_INT, video_crop->height, NULL)); if (!base->convert) { GST_WARNING_OBJECT (base, "failed to create a video convert"); return FALSE; } } gst_video_converter_frame (base->convert, src_frame, dest_frame); return TRUE; } gboolean gst_va_base_dec_copy_output_buffer (GstVaBaseDec * base, GstVideoCodecFrame * codec_frame) { GstVideoFrame src_frame; GstVideoFrame dest_frame; GstVideoInfo dest_vinfo; GstVideoInfo *src_vinfo; GstBuffer *buffer = NULL; GstVideoCropMeta *video_crop; GstFlowReturn ret; g_return_val_if_fail (base && base->output_state, FALSE); if (!base->other_pool) return FALSE; if (!gst_buffer_pool_set_active (base->other_pool, TRUE)) return FALSE; src_vinfo = &base->output_state->info; gst_video_info_set_format (&dest_vinfo, GST_VIDEO_INFO_FORMAT (src_vinfo), GST_VIDEO_INFO_WIDTH (src_vinfo), GST_VIDEO_INFO_HEIGHT (src_vinfo)); ret = gst_buffer_pool_acquire_buffer (base->other_pool, &buffer, NULL); if (ret != GST_FLOW_OK) goto fail; if (!gst_video_frame_map (&src_frame, src_vinfo, codec_frame->output_buffer, GST_MAP_READ)) goto fail; if (!gst_video_frame_map (&dest_frame, &dest_vinfo, buffer, GST_MAP_WRITE)) { gst_video_frame_unmap (&src_frame); goto fail; } video_crop = gst_buffer_get_video_crop_meta (codec_frame->output_buffer); if (video_crop) { if (!_copy_buffer_and_apply_video_crop (base, &src_frame, &dest_frame, video_crop)) { gst_video_frame_unmap (&src_frame); gst_video_frame_unmap (&dest_frame); GST_ERROR_OBJECT (base, "fail to apply the video crop."); goto fail; } } else { /* gst_video_frame_copy can crop this, but does not know, so let * make it think it's all right */ GST_VIDEO_INFO_WIDTH (&src_frame.info) = GST_VIDEO_INFO_WIDTH (src_vinfo); GST_VIDEO_INFO_HEIGHT (&src_frame.info) = GST_VIDEO_INFO_HEIGHT (src_vinfo); if (!gst_video_frame_copy (&dest_frame, &src_frame)) { gst_video_frame_unmap (&src_frame); gst_video_frame_unmap (&dest_frame); goto fail; } } gst_video_frame_unmap (&src_frame); gst_video_frame_unmap (&dest_frame); gst_buffer_replace (&codec_frame->output_buffer, buffer); gst_buffer_unref (buffer); return TRUE; fail: if (buffer) gst_buffer_unref (buffer); GST_ERROR_OBJECT (base, "Failed copy output buffer."); return FALSE; } gboolean gst_va_base_dec_process_output (GstVaBaseDec * base, GstVideoCodecFrame * frame, GstVideoCodecState * input_state, GstVideoBufferFlags buffer_flags) { GstVideoDecoder *vdec = GST_VIDEO_DECODER (base); if (input_state) { g_clear_pointer (&base->input_state, gst_video_codec_state_unref); base->input_state = gst_video_codec_state_ref (input_state); base->need_negotiation = TRUE; if (!gst_video_decoder_negotiate (vdec)) { GST_ERROR_OBJECT (base, "Could not re-negotiate with updated state"); return FALSE; } } if (base->copy_frames) gst_va_base_dec_copy_output_buffer (base, frame); if (buffer_flags != 0) { #ifndef GST_DISABLE_GST_DEBUG gboolean interlaced = (buffer_flags & GST_VIDEO_BUFFER_FLAG_INTERLACED) != 0; gboolean tff = (buffer_flags & GST_VIDEO_BUFFER_FLAG_TFF) != 0; GST_TRACE_OBJECT (base, "apply buffer flags 0x%x (interlaced %d, top-field-first %d)", buffer_flags, interlaced, tff); #endif GST_BUFFER_FLAG_SET (frame->output_buffer, buffer_flags); } return TRUE; } GstFlowReturn gst_va_base_dec_prepare_output_frame (GstVaBaseDec * base, GstVideoCodecFrame * frame) { GstVideoDecoder *vdec = GST_VIDEO_DECODER (base); if (base->need_negotiation) { if (!gst_video_decoder_negotiate (vdec)) { GST_ERROR_OBJECT (base, "Failed to negotiate with downstream"); return GST_FLOW_NOT_NEGOTIATED; } } if (frame) return gst_video_decoder_allocate_output_frame (vdec, frame); return GST_FLOW_OK; } gboolean gst_va_base_dec_set_output_state (GstVaBaseDec * base) { GstVideoDecoder *decoder = GST_VIDEO_DECODER (base); GstVideoFormat format = GST_VIDEO_FORMAT_UNKNOWN; guint64 modifier; GstCapsFeatures *capsfeatures = NULL; GstVideoInfo *info = &base->output_info; if (base->output_state) gst_video_codec_state_unref (base->output_state); gst_va_base_dec_get_preferred_format_and_caps_features (base, &format, &capsfeatures, &modifier); if (format == GST_VIDEO_FORMAT_UNKNOWN) return FALSE; base->output_state = gst_video_decoder_set_interlaced_output_state (decoder, format, GST_VIDEO_INFO_INTERLACE_MODE (info), GST_VIDEO_INFO_WIDTH (info), GST_VIDEO_INFO_HEIGHT (info), base->input_state); /* set caps feature */ if (capsfeatures && gst_caps_features_contains (capsfeatures, GST_CAPS_FEATURE_MEMORY_DMABUF)) { base->output_state->caps = gst_va_video_info_to_dma_caps (&base->output_state->info, modifier); } else { base->output_state->caps = gst_video_info_to_caps (&base->output_state->info); } if (capsfeatures) gst_caps_set_features_simple (base->output_state->caps, capsfeatures); GST_INFO_OBJECT (base, "Negotiated caps %" GST_PTR_FORMAT, base->output_state->caps); return TRUE; }