/* GStreamer * * jpegparse: a parser for JPEG streams * * Copyright (C) <2009> Arnout Vandecappelle (Essensium/Mind) * <2022> Víctor Manuel Jáquez Leal * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, * Boston, MA 02110-1301, USA. */ /** * SECTION:element-jpegparse * @title: jpegparse * @short_description: JPEG parser * * Parses a JPEG stream into JPEG images. It looks for EOI boundaries to * split a continuous stream into single-frame buffers. Also reads the * image header searching for image properties such as width and height * among others. Jpegparse can also extract metadata (e.g. xmp). * * ## Example launch line * |[ * gst-launch-1.0 -v souphttpsrc location=... ! jpegparse ! matroskamux ! filesink location=... * ]| * The above pipeline fetches a motion JPEG stream from an IP camera over * HTTP and stores it in a matroska file. * */ /* FIXME: output plain JFIF APP marker only. This provides best code reuse. * JPEG decoders would not need to handle this part anymore. Also when remuxing * (... ! jpegparse ! ... ! jifmux ! ...) metadata consolidation would be * easier. */ /* TODO: * + APP2 -- ICC color profile * + APP3 -- meta (same as exif) * + APP12 -- Photoshop Save for Web: Ducky / Picture info * + APP13 -- Adobe IRB * + check for interlaced mjpeg */ #ifdef HAVE_CONFIG_H #include #endif #include #include #include #include #include "gstjpegparse.h" enum ParserState { GST_JPEG_PARSER_STATE_GOT_SOI = 1 << 0, GST_JPEG_PARSER_STATE_GOT_SOF = 1 << 1, GST_JPEG_PARSER_STATE_GOT_SOS = 1 << 2, GST_JPEG_PARSER_STATE_GOT_JFIF = 1 << 3, GST_JPEG_PARSER_STATE_GOT_ADOBE = 1 << 4, GST_JPEG_PARSER_STATE_GOT_METADATA = 1 << 5, GST_JPEG_PARSER_STATE_VALID_PICTURE = (GST_JPEG_PARSER_STATE_GOT_SOI | GST_JPEG_PARSER_STATE_GOT_SOF | GST_JPEG_PARSER_STATE_GOT_SOS), }; static GstStaticPadTemplate gst_jpeg_parse_src_pad_template = GST_STATIC_PAD_TEMPLATE ("src", GST_PAD_SRC, GST_PAD_ALWAYS, GST_STATIC_CAPS ("image/jpeg, " "framerate = (fraction) [ 0/1, MAX ], " "parsed = (boolean) true") ); static GstStaticPadTemplate gst_jpeg_parse_sink_pad_template = GST_STATIC_PAD_TEMPLATE ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, GST_STATIC_CAPS ("image/jpeg") ); GST_DEBUG_CATEGORY_STATIC (jpeg_parse_debug); #define GST_CAT_DEFAULT jpeg_parse_debug static GstFlowReturn gst_jpeg_parse_handle_frame (GstBaseParse * bparse, GstBaseParseFrame * frame, gint * skipsize); static gboolean gst_jpeg_parse_set_sink_caps (GstBaseParse * parse, GstCaps * caps); static gboolean gst_jpeg_parse_sink_event (GstBaseParse * parse, GstEvent * event); static gboolean gst_jpeg_parse_start (GstBaseParse * parse); static gboolean gst_jpeg_parse_stop (GstBaseParse * parse); #define gst_jpeg_parse_parent_class parent_class G_DEFINE_TYPE (GstJpegParse, gst_jpeg_parse, GST_TYPE_BASE_PARSE); GST_ELEMENT_REGISTER_DEFINE (jpegparse, "jpegparse", GST_RANK_PRIMARY, GST_TYPE_JPEG_PARSE); /* CIPA DC-x 007-2009 MPF spec states as TIFF */ #define MPF_LE 0x4949 #define MPF_BE 0x4D4D enum GstJPEGColorspace { GST_JPEG_COLORSPACE_NONE, GST_JPEG_COLORSPACE_RGB, GST_JPEG_COLORSPACE_YUV, GST_JPEG_COLORSPACE_GRAY, GST_JPEG_COLORSPACE_CMYK, GST_JPEG_COLORSPACE_YCCK, }; static const gchar *gst_jpeg_colorspace_strings[] = { [GST_JPEG_COLORSPACE_NONE] = NULL, [GST_JPEG_COLORSPACE_RGB] = "sRGB", [GST_JPEG_COLORSPACE_YUV] = "sYUV", [GST_JPEG_COLORSPACE_GRAY] = "GRAY", [GST_JPEG_COLORSPACE_CMYK] = "CMYK", [GST_JPEG_COLORSPACE_YCCK] = "YCCK", }; enum GstJPEGSampling { GST_JPEG_SAMPLING_NONE, GST_JPEG_SAMPLING_RGB, GST_JPEG_SAMPLING_BGR, GST_JPEG_SAMPLING_YBR444, GST_JPEG_SAMPLING_YBR422, GST_JPEG_SAMPLING_YBR420, GST_JPEG_SAMPLING_YBR440, GST_JPEG_SAMPLING_YBR410, GST_JPEG_SAMPLING_YBR411, GST_JPEG_SAMPLING_GRAYSCALE, }; static const gchar *gst_jpeg_sampling_strings[] = { [GST_JPEG_SAMPLING_NONE] = NULL, [GST_JPEG_SAMPLING_RGB] = "RGB", [GST_JPEG_SAMPLING_BGR] = "BGR", [GST_JPEG_SAMPLING_YBR444] = "YCbCr-4:4:4", [GST_JPEG_SAMPLING_YBR422] = "YCbCr-4:2:2", [GST_JPEG_SAMPLING_YBR420] = "YCbCr-4:2:0", [GST_JPEG_SAMPLING_YBR440] = "YCbCr-4:4:0", [GST_JPEG_SAMPLING_YBR410] = "YCbCr-4:1:0", [GST_JPEG_SAMPLING_YBR411] = "YCbCr-4:1:1", [GST_JPEG_SAMPLING_GRAYSCALE] = "GRAYSCALE", }; static void gst_jpeg_parse_class_init (GstJpegParseClass * klass) { GstBaseParseClass *gstbaseparse_class; GstElementClass *gstelement_class; gstbaseparse_class = (GstBaseParseClass *) klass; gstelement_class = (GstElementClass *) klass; gstbaseparse_class->start = gst_jpeg_parse_start; gstbaseparse_class->stop = gst_jpeg_parse_stop; gstbaseparse_class->set_sink_caps = gst_jpeg_parse_set_sink_caps; gstbaseparse_class->sink_event = gst_jpeg_parse_sink_event; gstbaseparse_class->handle_frame = gst_jpeg_parse_handle_frame; gst_element_class_add_static_pad_template (gstelement_class, &gst_jpeg_parse_src_pad_template); gst_element_class_add_static_pad_template (gstelement_class, &gst_jpeg_parse_sink_pad_template); gst_element_class_set_static_metadata (gstelement_class, "JPEG stream parser", "Codec/Parser/Image", "Parse JPEG images into single-frame buffers", "Víctor Jáquez "); GST_DEBUG_CATEGORY_INIT (jpeg_parse_debug, "jpegparse", 0, "JPEG parser"); } static void gst_jpeg_parse_init (GstJpegParse * parse) { parse->sof = -1; } static void parse_avid (GstJpegParse * parse, const guint8 * data, guint16 len) { parse->avid = 1; if (len > 14 && data[12] == 1) /* 1 - NTSC */ parse->field_order = GST_VIDEO_FIELD_ORDER_BOTTOM_FIELD_FIRST; if (len > 14 && data[12] == 2) /* 2 - PAL */ parse->field_order = GST_VIDEO_FIELD_ORDER_TOP_FIELD_FIRST; GST_INFO_OBJECT (parse, "AVID: %s", gst_video_field_order_to_string (parse->field_order)); } static gboolean gst_jpeg_parse_set_sink_caps (GstBaseParse * bparse, GstCaps * caps) { GstJpegParse *parse = GST_JPEG_PARSE_CAST (bparse); GstStructure *s = gst_caps_get_structure (caps, 0); const GValue *codec_data; const char *interlace_mode, *field_order; GST_DEBUG_OBJECT (parse, "get sink caps %" GST_PTR_FORMAT, caps); gst_structure_get_fraction (s, "framerate", &parse->framerate_numerator, &parse->framerate_denominator); gst_structure_get_int (s, "height", &parse->orig_height); gst_structure_get_int (s, "width", &parse->orig_width); gst_structure_get_fraction (s, "pixel-aspect-ration", &parse->par_num, &parse->par_den); codec_data = gst_structure_get_value (s, "codec_data"); if (codec_data && G_VALUE_TYPE (codec_data) == GST_TYPE_BUFFER) { GstMapInfo map; gst_clear_buffer (&parse->codec_data); parse->codec_data = GST_BUFFER (g_value_dup_boxed (codec_data)); if (gst_buffer_map (parse->codec_data, &map, GST_MAP_READ)) { if (map.size > 8 && map.data[0] == 0x2c && map.data[4] == 0x18) parse_avid (parse, map.data, map.size); gst_buffer_unmap (parse->codec_data, &map); } } interlace_mode = gst_structure_get_string (s, "interlace-mode"); if (interlace_mode) { parse->interlace_mode = gst_video_interlace_mode_from_string (interlace_mode); } if (parse->interlace_mode != GST_VIDEO_INTERLACE_MODE_PROGRESSIVE) { field_order = gst_structure_get_string (s, "field-order"); if (field_order) parse->field_order = gst_video_field_order_from_string (field_order); } g_clear_pointer (&parse->colorimetry, g_free); parse->colorimetry = g_strdup (gst_structure_get_string (s, "colorimetry")); return TRUE; } static inline gboolean valid_state (guint state, guint ref_state) { return (state & ref_state) == ref_state; } /* https://zpl.fi/chroma-subsampling-and-jpeg-sampling-factors/ */ /* *INDENT-OFF* */ static const struct { gint h[3]; gint v[3]; enum GstJPEGSampling sampling; } subsampling_map[] = { {{1, 1, 1}, {1, 1, 1}, GST_JPEG_SAMPLING_YBR444}, {{2, 2, 2}, {1, 1, 1}, GST_JPEG_SAMPLING_YBR444}, {{3, 3, 3}, {1, 1, 1}, GST_JPEG_SAMPLING_YBR444}, {{1, 1, 1}, {2, 2, 2}, GST_JPEG_SAMPLING_YBR444}, {{1, 1, 1}, {3, 3, 3}, GST_JPEG_SAMPLING_YBR444}, {{1, 1, 1}, {2, 1, 1}, GST_JPEG_SAMPLING_YBR440}, {{2, 2, 2}, {2, 1, 1}, GST_JPEG_SAMPLING_YBR440}, {{1, 1, 1}, {4, 2, 2}, GST_JPEG_SAMPLING_YBR440}, {{2, 1, 1}, {1, 1, 1}, GST_JPEG_SAMPLING_YBR422}, {{2, 1, 1}, {2, 2, 2}, GST_JPEG_SAMPLING_YBR422}, {{4, 2, 2}, {1, 1, 1}, GST_JPEG_SAMPLING_YBR422}, {{2, 1, 1}, {2, 1, 1}, GST_JPEG_SAMPLING_YBR420}, {{4, 1, 1}, {1, 1, 1}, GST_JPEG_SAMPLING_YBR411}, {{4, 1, 1}, {2, 1, 1}, GST_JPEG_SAMPLING_YBR410}, }; /* *INDENT-ON* */ static guint16 yuv_sampling (GstJpegFrameHdr * frame_hdr) { int i, h0, h1, h2, v0, v1, v2; g_return_val_if_fail (frame_hdr->num_components == 3, GST_JPEG_SAMPLING_NONE); h0 = frame_hdr->components[0].horizontal_factor; h1 = frame_hdr->components[1].horizontal_factor; h2 = frame_hdr->components[2].horizontal_factor; v0 = frame_hdr->components[0].vertical_factor; v1 = frame_hdr->components[1].vertical_factor; v2 = frame_hdr->components[2].vertical_factor; for (i = 0; i < G_N_ELEMENTS (subsampling_map); i++) { if (subsampling_map[i].h[0] == h0 && subsampling_map[i].h[1] == h1 && subsampling_map[i].h[2] == h2 && subsampling_map[i].v[0] == v0 && subsampling_map[i].v[1] == v1 && subsampling_map[i].v[2] == v2) return subsampling_map[i].sampling; } return GST_JPEG_SAMPLING_NONE; } static const gchar * colorspace_to_string (enum GstJPEGColorspace colorspace) { return gst_jpeg_colorspace_strings[colorspace]; } /* https://entropymine.wordpress.com/2018/10/22/how-is-a-jpeg-images-color-type-determined/ */ /* T-REC-T.872-201206 6.1 Colour encodings and associated values to define white and black */ static gboolean gst_jpeg_parse_sof (GstJpegParse * parse, GstJpegSegment * seg) { GstJpegFrameHdr hdr = { 0, }; guint colorspace; guint sampling; if (!gst_jpeg_segment_parse_frame_header (seg, &hdr)) { return FALSE; } if (parse->mpf.mode && parse->mpf.primary_image_index != parse->mpf.cur_image_index) { GST_DEBUG_OBJECT (parse, "Ignoring MPF SOF of picture %d", parse->mpf.cur_image_index); return TRUE; } colorspace = GST_JPEG_COLORSPACE_NONE; sampling = GST_JPEG_SAMPLING_NONE; switch (hdr.num_components) { case 1: colorspace = GST_JPEG_COLORSPACE_GRAY; sampling = GST_JPEG_SAMPLING_GRAYSCALE; break; case 3: if (valid_state (parse->state, GST_JPEG_PARSER_STATE_GOT_JFIF)) { colorspace = GST_JPEG_COLORSPACE_YUV; sampling = yuv_sampling (&hdr); } else { if (valid_state (parse->state, GST_JPEG_PARSER_STATE_GOT_ADOBE)) { if (parse->adobe_transform == 0) { colorspace = GST_JPEG_COLORSPACE_RGB; sampling = GST_JPEG_SAMPLING_RGB; } else if (parse->adobe_transform == 1) { colorspace = GST_JPEG_COLORSPACE_YUV;; sampling = yuv_sampling (&hdr); } else { GST_DEBUG_OBJECT (parse, "Unknown Adobe color transform code"); colorspace = GST_JPEG_COLORSPACE_YUV;; sampling = yuv_sampling (&hdr); } } else { int cid0, cid1, cid2; cid0 = hdr.components[0].identifier; cid1 = hdr.components[1].identifier; cid2 = hdr.components[2].identifier; if (cid0 == 1 && cid1 == 2 && cid2 == 3) { colorspace = GST_JPEG_COLORSPACE_YUV; sampling = yuv_sampling (&hdr); } else if (cid0 == 'R' && cid1 == 'G' && cid2 == 'B') { colorspace = GST_JPEG_COLORSPACE_RGB; sampling = GST_JPEG_SAMPLING_RGB; } else { GST_DEBUG_OBJECT (parse, "Unrecognized component IDs"); colorspace = GST_JPEG_COLORSPACE_YUV; sampling = yuv_sampling (&hdr); } } } break; case 4: if (valid_state (parse->state, GST_JPEG_PARSER_STATE_GOT_ADOBE)) { if (parse->adobe_transform == 0) { colorspace = GST_JPEG_COLORSPACE_CMYK; } else if (parse->adobe_transform == 2) { colorspace = GST_JPEG_COLORSPACE_YCCK; } else { GST_DEBUG_OBJECT (parse, "Unknown Adobe color transform code"); colorspace = GST_JPEG_COLORSPACE_YCCK; } } else { colorspace = GST_JPEG_COLORSPACE_CMYK; } break; default: GST_WARNING_OBJECT (parse, "Unknown color space"); break; } if (hdr.width != parse->width || hdr.height != parse->height || colorspace != parse->colorspace || sampling != parse->sampling) { parse->width = hdr.width; parse->height = hdr.height; parse->colorspace = colorspace; parse->sampling = sampling; if (parse->first_picture && !parse->multiscope) { if (parse->orig_height > 0 && parse->height < ((parse->orig_height * 3) / 4)) { parse->interlace_mode = GST_VIDEO_INTERLACE_MODE_INTERLEAVED; } else if (parse->avid) { /* if no container info, let's suppose it doubles its height */ if (parse->orig_height == 0) parse->orig_height = 2 * hdr.height; parse->interlace_mode = GST_VIDEO_INTERLACE_MODE_INTERLEAVED; } } parse->first_picture = FALSE; parse->renegotiate = TRUE; } GST_INFO_OBJECT (parse, "SOF [%dx%d] %d comp - %s", parse->width, parse->height, hdr.num_components, GST_STR_NULL (colorspace_to_string (parse->colorspace))); return TRUE; } static inline GstTagList * get_tag_list (GstJpegParse * parse) { if (!parse->tags) parse->tags = gst_tag_list_new_empty (); return parse->tags; } static gboolean gst_jpeg_parse_app0 (GstJpegParse * parse, GstJpegSegment * seg) { GstByteReader reader; guint16 xd, yd; guint8 unit, xt, yt; guint32 id; if (seg->size < 6) /* less than 6 means no id string */ return FALSE; gst_byte_reader_init (&reader, seg->data + seg->offset, seg->size); gst_byte_reader_skip_unchecked (&reader, 2); #if G_BYTE_ORDER == G_LITTLE_ENDIAN if (!gst_byte_reader_get_uint32_le (&reader, &id)) return FALSE; #else if (!gst_byte_reader_get_uint32_be (&reader, &id)) return FALSE; #endif if (!valid_state (parse->state, GST_JPEG_PARSER_STATE_GOT_JFIF) && GST_MAKE_FOURCC ('J', 'F', 'I', 'F') == id) { parse->state |= GST_JPEG_PARSER_STATE_GOT_JFIF; /* trailing zero-byte */ gst_byte_reader_skip_unchecked (&reader, 1); /* version */ gst_byte_reader_skip_unchecked (&reader, 2); /* units */ if (!gst_byte_reader_get_uint8 (&reader, &unit)) return FALSE; /* x density */ #if G_BYTE_ORDER == G_LITTLE_ENDIAN if (!gst_byte_reader_get_uint16_le (&reader, &xd)) return FALSE; /* y density */ if (!gst_byte_reader_get_uint16_le (&reader, &yd)) return FALSE; #else if (!gst_byte_reader_get_uint16_be (&reader, &xd)) return FALSE; /* y density */ if (!gst_byte_reader_get_uint16_be (&reader, &yd)) return FALSE; #endif /* x thumbnail */ if (!gst_byte_reader_get_uint8 (&reader, &xt)) return FALSE; /* y thumbnail */ if (!gst_byte_reader_get_uint8 (&reader, &yt)) return FALSE; if (unit == 0) { /* no units, X and Y specify the pixel aspect ratio */ if (parse->par_num != xd || parse->par_den != yd) { parse->renegotiate = TRUE; parse->par_num = xd; parse->par_den = yd; } } else if (unit == 1 || unit == 2) { /* tag pixel per inches */ double hppi = xd, vppi = yd; /* cm to in */ if (unit == 2) { hppi *= 2.54; vppi *= 2.54; } gst_tag_register_musicbrainz_tags (); gst_tag_list_add (get_tag_list (parse), GST_TAG_MERGE_REPLACE, GST_TAG_IMAGE_HORIZONTAL_PPI, hppi, GST_TAG_IMAGE_VERTICAL_PPI, vppi, NULL); } if (xt > 0 && yt > 0) GST_FIXME_OBJECT (parse, "embedded thumbnail ignored"); goto bail; } /* JFIF Extension */ if (GST_MAKE_FOURCC ('J', 'F', 'X', 'X') == id) { if (!valid_state (parse->state, GST_JPEG_PARSER_STATE_GOT_JFIF)) return FALSE; goto bail; } /* https://exiftool.org/TagNames/JPEG.html#AVI1 */ if (GST_MAKE_FOURCC ('A', 'V', 'I', '1') == id) { /* polarity */ if (!gst_byte_reader_get_uint8 (&reader, &unit)) return FALSE; parse->avid = (unit > 0); /* otherwise is not interleaved */ /* TODO: update caps for interlaced MJPEG */ GST_DEBUG_OBJECT (parse, "MJPEG interleaved field: %s", unit == 0 ? "not interleaved" : unit % 2 ? "Odd" : "Even"); goto bail; } GST_MEMDUMP_OBJECT (parse, "Unhandled app0", seg->data + seg->offset, seg->size); bail: parse->state |= GST_JPEG_PARSER_STATE_GOT_METADATA; return TRUE; } /* *INDENT-OFF* */ static const struct { const gchar *suffix; GstTagList *(*tag_func) (GstBuffer * buff); } TagMap[] = { {"Exif", gst_tag_list_from_exif_buffer_with_tiff_header}, {"http://ns.adobe.com/xap/1.0/", gst_tag_list_from_xmp_buffer}, }; /* *INDENT-ON* */ static gboolean gst_jpeg_parse_app1 (GstJpegParse * parse, GstJpegSegment * seg) { GstByteReader reader; GstBuffer *buf; guint16 size = 0; const gchar *id_str; const guint8 *data; gint i; if (seg->size < 6) /* less than 6 means no id string */ return FALSE; gst_byte_reader_init (&reader, seg->data + seg->offset, seg->size); gst_byte_reader_skip_unchecked (&reader, 2); if (!gst_byte_reader_get_string_utf8 (&reader, &id_str)) return FALSE; for (i = 0; i < G_N_ELEMENTS (TagMap); i++) { if (!g_str_has_suffix (id_str, TagMap[i].suffix)) continue; /* skip NUL only for Exif */ if (i == 0) { if (!gst_byte_reader_skip (&reader, 1)) return FALSE; } size = gst_byte_reader_get_remaining (&reader); if (!gst_byte_reader_get_data (&reader, size, &data)) return FALSE; /* add synthetic xpacket for xpm if it doesn't have it */ if (i == 1 && !g_strstr_len ((const char *) data, size, ""); g_string_append_len (xmp, (const char *) data, size); g_string_append (xmp, ""); len = xmp->len; #if GLIB_CHECK_VERSION (2, 76, 0) str = g_string_free_and_steal (xmp); #else str = g_string_free (xmp, FALSE); #endif buf = gst_buffer_new_wrapped_full (GST_MEMORY_FLAG_READONLY, str, len, 0, len, str, g_free); } else { buf = gst_buffer_new_wrapped_full (GST_MEMORY_FLAG_READONLY, (gpointer) data, size, 0, size, NULL, NULL); } if (buf) { GstTagList *tags; tags = TagMap[i].tag_func (buf); gst_buffer_unref (buf); if (tags) { GST_LOG_OBJECT (parse, "parsed marker %x: '%s' %" GST_PTR_FORMAT, GST_JPEG_MARKER_APP1, id_str, tags); gst_tag_list_insert (get_tag_list (parse), tags, GST_TAG_MERGE_REPLACE); gst_tag_list_unref (tags); } else { GST_INFO_OBJECT (parse, "failed to parse %s: %s", id_str, data); } } goto bail; } GST_MEMDUMP_OBJECT (parse, "Unhandled app1", seg->data + seg->offset, seg->size); bail: parse->state |= GST_JPEG_PARSER_STATE_GOT_METADATA; return TRUE; } struct MPF { guint32 num_images; guint32 individual_image_no; struct { gboolean parent; gboolean child; gboolean representative; enum { UNDEFINED = 0x0, BASELINE_PRIMARY = 0x30000, LARGE_THUMB_VGA = 0x10001, LARGE_THUMB_HD = 0x10002, MULTI_FRAME_PANO = 0x20001, MULTI_FRAME_DISPARITY = 0x20002, MULTI_FRAME_MULTI_ANGLE = 0x20003, } type; guint32 size; guint32 offset; } entries[32]; }; static gboolean gst_jpeg_parse_mpf (GstJpegParse * parse, GstByteReader * reader, struct MPF *mpf) { guint16 endianness; guint16 fortytwo; guint32 offset; guint16 num_entries; gsize offset_ref, offset_cur; offset_ref = gst_byte_reader_get_pos (reader); /* MP Header */ if (!gst_byte_reader_get_uint16_be (reader, &endianness)) return FALSE; if (endianness == MPF_LE) { if (!gst_byte_reader_get_uint16_le (reader, &fortytwo) || !gst_byte_reader_get_uint32_le (reader, &offset)) return FALSE; } else if (endianness == MPF_BE) { if (!gst_byte_reader_get_uint16_be (reader, &fortytwo) || !gst_byte_reader_get_uint32_be (reader, &offset)) return FALSE; } else { return FALSE; } /* endianness check */ if (fortytwo != 42) return FALSE; /* Skip to MP Index IFD */ offset_cur = gst_byte_reader_get_pos (reader); /* number of bytes to skip = (reference offset + new offset) - current offset */ if (!gst_byte_reader_skip (reader, offset_ref + offset - offset_cur)) return FALSE; while (TRUE) { /* MP Index IFD - number of entries */ if (endianness == MPF_LE) { if (!gst_byte_reader_get_uint16_le (reader, &num_entries)) return FALSE; } else { if (!gst_byte_reader_get_uint16_be (reader, &num_entries)) return FALSE; } GST_DEBUG_OBJECT (parse, "MPF: %d IFD entries", num_entries); for (int i = 0; i < num_entries; i++) { guint16 tag, type; guint32 count, value; if (endianness == MPF_LE) { if (!gst_byte_reader_get_uint16_le (reader, &tag) || !gst_byte_reader_get_uint16_le (reader, &type) || !gst_byte_reader_get_uint32_le (reader, &count) || !gst_byte_reader_get_uint32_le (reader, &value)) return FALSE; } else { if (!gst_byte_reader_get_uint16_be (reader, &tag) || !gst_byte_reader_get_uint16_be (reader, &type) || !gst_byte_reader_get_uint32_be (reader, &count) || !gst_byte_reader_get_uint32_be (reader, &value)) return FALSE; } switch (tag) { case 0XB000: /* MPF version # */ GST_DEBUG_OBJECT (parse, "MPF version %" GST_FOURCC_FORMAT, GST_FOURCC_ARGS (value)); break; case 0xB001: /* number of images */ mpf->num_images = value; GST_DEBUG_OBJECT (parse, "MPF number of images %d", mpf->num_images); break; case 0xB002:{ /* MP entries */ if (count / 16 != mpf->num_images) return FALSE; offset_cur = gst_byte_reader_get_pos (reader); if (!gst_byte_reader_skip (reader, offset_ref + value - offset_cur)) return FALSE; if (mpf->num_images > 32) { GST_WARNING_OBJECT (parse, "MPF has more than 32 pictures. Forced to 32"); mpf->num_images = 32; } for (int j = 0; j < mpf->num_images; j++) { guint32 attr, size, offset, dependencies; if (endianness == MPF_LE) { if (!gst_byte_reader_get_uint32_le (reader, &attr) || !gst_byte_reader_get_uint32_le (reader, &size) || !gst_byte_reader_get_uint32_le (reader, &offset) || !gst_byte_reader_get_uint32_le (reader, &dependencies)) return FALSE; } else { if (!gst_byte_reader_get_uint32_be (reader, &attr) || !gst_byte_reader_get_uint32_be (reader, &size) || !gst_byte_reader_get_uint32_be (reader, &offset) || !gst_byte_reader_get_uint32_be (reader, &dependencies)) return FALSE; } mpf->entries[j].parent = attr & 0x8000000000u; mpf->entries[j].child = attr & 0x4000000000u; mpf->entries[j].representative = attr & 0x2000000000u; mpf->entries[j].type = attr & 0xffffffu; mpf->entries[j].size = size; mpf->entries[j].offset = offset; GST_DEBUG_OBJECT (parse, "MPF entry image type 0x%x", mpf->entries[j].type); } break; } case 0xB101: /* individual image number */ mpf->individual_image_no = value; GST_DEBUG_OBJECT (parse, "MPF individual image %d", mpf->individual_image_no); break; case 0xB003: /* image uid list */ case 0xB004: /* total frames */ case 0xB201: /* panorama scanning orientation */ case 0xB202: /* panorama horiz overlap */ case 0xB203: /* panorama vert overlap */ case 0xB204: /* base viewpoint # */ case 0xB205: /* convergence angle */ case 0xB206: /* baseline length */ case 0xB207: /* divergence angle */ case 0xB208: /* horiz axis distance */ case 0xB209: /* vert axis distance */ case 0xB20A: /* collimation axis distance */ case 0xB20B: /* yaw angle */ case 0xB20C: /* pitch angle */ case 0xB20D: /* roll angle */ GST_DEBUG_OBJECT (parse, "unhandled MPF entry 0x%x", tag); break; default: return FALSE; }; } if (gst_byte_reader_get_remaining (reader) == 0) break; /* Next IFD offset */ if (endianness == MPF_LE) { if (!gst_byte_reader_get_uint32_le (reader, &offset)) return FALSE; } else { if (!gst_byte_reader_get_uint32_be (reader, &offset)) return FALSE; } if (offset == 0) break; offset_cur = gst_byte_reader_get_pos (reader); if (!gst_byte_reader_skip (reader, offset_ref + offset - offset_cur)) return FALSE; } return TRUE; } static gboolean gst_jpeg_parse_app2 (GstJpegParse * parse, GstJpegSegment * seg) { GstByteReader reader; const gchar *id_str; guint i; if (seg->size < 4) /* less than 6 means no id string */ return FALSE; gst_byte_reader_init (&reader, seg->data + seg->offset, seg->size); gst_byte_reader_skip_unchecked (&reader, 2); if (!gst_byte_reader_get_string_utf8 (&reader, &id_str)) return FALSE; if (g_str_has_suffix (id_str, "MPF")) { struct MPF mpf = { 0, }; if (!gst_jpeg_parse_mpf (parse, &reader, &mpf)) return FALSE; parse->mpf.mode = TRUE; parse->mpf.num_images = mpf.num_images; parse->mpf.cur_image_index = 0; for (i = 0; i < mpf.num_images; i++) { if (mpf.entries[i].type == BASELINE_PRIMARY) { parse->mpf.primary_image_index = i; break; } } if (i == mpf.num_images) { GST_WARNING_OBJECT (parse, "No baseline primary image found. Forcing the first"); parse->mpf.primary_image_index = 0; } } return TRUE; } static gboolean gst_jpeg_parse_app14 (GstJpegParse * parse, GstJpegSegment * seg) { GstByteReader reader; guint8 transform; const guint8 *id = NULL; const guint8 adobe_tag[] = { 'A', 'd', 'o', 'b', 'e' }; if (seg->size < 6) /* less than 6 means no id string */ return FALSE; gst_byte_reader_init (&reader, seg->data + seg->offset, seg->size); gst_byte_reader_skip_unchecked (&reader, 2); if (!gst_byte_reader_peek_data (&reader, 5, &id)) return FALSE; if (G_LIKELY (!memcmp (id, adobe_tag, 5))) { if (!gst_byte_reader_skip (&reader, 5)) return FALSE; } else { GST_MEMDUMP_OBJECT (parse, "Unhandled app14", seg->data + seg->offset, seg->size); goto bail; } /* skip version and flags */ if (!gst_byte_reader_skip (&reader, 6)) return FALSE; parse->state |= GST_JPEG_PARSER_STATE_GOT_ADOBE; /* transform bit might not exist */ if (!gst_byte_reader_get_uint8 (&reader, &transform)) goto bail; parse->adobe_transform = transform; bail: parse->state |= GST_JPEG_PARSER_STATE_GOT_METADATA; return TRUE; } static inline gchar * get_utf8_from_data (const guint8 * data, guint16 size) { const gchar *env_vars[] = { "GST_JPEG_TAG_ENCODING", "GST_TAG_ENCODING", NULL }; const char *str = (gchar *) data; char *ret; ret = gst_tag_freeform_string_to_utf8 (str, size, env_vars); if (!ret) GST_MEMDUMP ("non-parsed marker data", data, size); return ret; } /* read comment and post as tag */ static inline gboolean gst_jpeg_parse_com (GstJpegParse * parse, GstJpegSegment * seg) { GstByteReader reader; const guint8 *data = NULL; guint16 size; const gchar *buf; gst_byte_reader_init (&reader, seg->data + seg->offset, seg->size); gst_byte_reader_skip_unchecked (&reader, 2); size = gst_byte_reader_get_remaining (&reader); if (!gst_byte_reader_get_data (&reader, size, &data)) return FALSE; buf = (const gchar *) data; /* buggy avid, it puts EOI only at every 10th frame */ if (g_str_has_prefix (buf, "AVID")) { parse_avid (parse, data, size); } else if (g_str_has_prefix (buf, "MULTISCOPE II")) { parse->par_num = 1; parse->par_den = 2; parse->multiscope = TRUE; } else { gchar *comment; comment = get_utf8_from_data (data, size); if (!comment) return FALSE; GST_INFO_OBJECT (parse, "comment found: %s", comment); gst_tag_list_add (get_tag_list (parse), GST_TAG_MERGE_REPLACE, GST_TAG_COMMENT, comment, NULL); g_free (comment); } parse->state |= GST_JPEG_PARSER_STATE_GOT_METADATA; return TRUE; } /* reset per image */ static void gst_jpeg_parse_reset (GstJpegParse * parse) { parse->last_offset = 0; parse->state = 0; parse->adobe_transform = 0; parse->field = 0; if (parse->tags) { gst_tag_list_unref (parse->tags); parse->tags = NULL; } } static const gchar * sampling_to_string (enum GstJPEGSampling sampling) { return gst_jpeg_sampling_strings[sampling]; } static gboolean gst_jpeg_parse_set_new_caps (GstJpegParse * parse) { GstCaps *caps; GstEvent *event; gboolean res; if (!parse->renegotiate) return TRUE; caps = gst_caps_new_simple ("image/jpeg", "parsed", G_TYPE_BOOLEAN, TRUE, NULL); if (parse->width > 0) gst_caps_set_simple (caps, "width", G_TYPE_INT, parse->width, NULL); if (parse->orig_height > 0 && parse->orig_height > parse->height) gst_caps_set_simple (caps, "height", G_TYPE_INT, parse->orig_height, NULL); else if (parse->height > 0) gst_caps_set_simple (caps, "height", G_TYPE_INT, parse->height, NULL); if (parse->sof >= 0) gst_caps_set_simple (caps, "sof-marker", G_TYPE_INT, parse->sof, NULL); if (parse->colorspace != GST_JPEG_COLORSPACE_NONE) { gst_caps_set_simple (caps, "colorspace", G_TYPE_STRING, colorspace_to_string (parse->colorspace), NULL); } if (parse->sampling != GST_JPEG_SAMPLING_NONE) { gst_caps_set_simple (caps, "sampling", G_TYPE_STRING, sampling_to_string (parse->sampling), NULL); } if (parse->colorimetry) { gst_caps_set_simple (caps, "colorimetry", G_TYPE_STRING, parse->colorimetry, NULL); } gst_caps_set_simple (caps, "interlace-mode", G_TYPE_STRING, gst_video_interlace_mode_to_string (parse->interlace_mode), NULL); if (parse->interlace_mode == GST_VIDEO_INTERLACE_MODE_INTERLEAVED) { gst_caps_set_simple (caps, "field-order", G_TYPE_STRING, gst_video_field_order_to_string (parse->field_order), NULL); } gst_caps_set_simple (caps, "framerate", GST_TYPE_FRACTION, parse->framerate_numerator, parse->framerate_denominator, NULL); if (parse->par_num > 0 && parse->par_den > 0) { gst_caps_set_simple (caps, "pixel-aspect-ratio", GST_TYPE_FRACTION, parse->par_num, parse->par_den, NULL); } if (parse->codec_data) { gst_caps_set_simple (caps, "codec_data", GST_TYPE_BUFFER, parse->codec_data, NULL); } parse->renegotiate = FALSE; GST_DEBUG_OBJECT (parse, "setting downstream caps on %s:%s to %" GST_PTR_FORMAT, GST_DEBUG_PAD_NAME (GST_BASE_PARSE_SRC_PAD (parse)), caps); event = gst_event_new_caps (caps); res = gst_pad_push_event (GST_BASE_PARSE_SRC_PAD (parse), event); gst_caps_unref (caps); return res; } static GstFlowReturn gst_jpeg_parse_finish_frame (GstJpegParse * parse, GstBaseParseFrame * frame, gint size) { GstBaseParse *bparse = GST_BASE_PARSE (parse); GstFlowReturn ret; if (parse->tags) gst_base_parse_merge_tags (bparse, parse->tags, GST_TAG_MERGE_REPLACE); if (!gst_jpeg_parse_set_new_caps (parse)) return GST_FLOW_ERROR; if (!valid_state (parse->state, GST_JPEG_PARSER_STATE_VALID_PICTURE)) { /* this validation breaks unit tests */ /* frame->flags |= GST_BASE_PARSE_FRAME_FLAG_DROP; */ GST_WARNING_OBJECT (parse, "Potentially invalid picture"); } GST_TRACE_OBJECT (parse, "Finish frame %" GST_PTR_FORMAT, frame->buffer); ret = gst_base_parse_finish_frame (bparse, frame, size); gst_jpeg_parse_reset (parse); return ret; } static inline gboolean gst_jpeg_parse_should_finish_buffer (GstJpegParse * parse, GstJpegMarker marker) { guint field_to_check; if (parse->mpf.mode) return parse->mpf.cur_image_index + 1 == parse->mpf.num_images; if (marker == GST_JPEG_MARKER_SOI) field_to_check = 0; else if (marker == GST_JPEG_MARKER_EOI) field_to_check = 1; else g_assert_not_reached (); return parse->interlace_mode == GST_VIDEO_INTERLACE_MODE_PROGRESSIVE || parse->field == field_to_check; } static GstFlowReturn gst_jpeg_parse_handle_frame (GstBaseParse * bparse, GstBaseParseFrame * frame, gint * skipsize) { GstJpegParse *parse = GST_JPEG_PARSE_CAST (bparse); GstMapInfo mapinfo; GstJpegMarker marker; GstJpegSegment seg; guint offset; gint prev_state; GST_TRACE_OBJECT (parse, "frame %" GST_PTR_FORMAT, frame->buffer); if (!gst_buffer_map (frame->buffer, &mapinfo, GST_MAP_READ)) return GST_FLOW_ERROR; offset = parse->last_offset; if (offset > 0) offset -= 1; /* it migth be in the middle marker */ while (offset < mapinfo.size) { if (!gst_jpeg_parse (&seg, mapinfo.data, mapinfo.size, offset)) { if (!valid_state (parse->state, GST_JPEG_PARSER_STATE_GOT_SOI)) { /* Skip any garbage until SOI */ *skipsize = mapinfo.size; GST_INFO_OBJECT (parse, "skipping %d bytes", *skipsize); } else { /* Accept anything after SOI */ parse->last_offset = mapinfo.size; } goto beach; } offset = seg.offset; marker = seg.marker; if (!valid_state (parse->state, GST_JPEG_PARSER_STATE_GOT_SOI) && marker != GST_JPEG_MARKER_SOI) continue; /* check if the whole segment is available */ if (offset + seg.size > mapinfo.size) { GST_DEBUG_OBJECT (parse, "incomplete segment: %x [offset %d]", marker, offset); parse->last_offset = offset - 2; goto beach; } offset += seg.size; GST_LOG_OBJECT (parse, "marker found: %x [offset %d / size %" G_GSSIZE_FORMAT "]", marker, seg.offset, seg.size); switch (marker) { case GST_JPEG_MARKER_SOI: /* This means that new SOI comes without an previous EOI. */ if (offset > 2 && gst_jpeg_parse_should_finish_buffer (parse, marker)) { /* If already some data segment parsed, push it as a frame. */ if (valid_state (parse->state, GST_JPEG_PARSER_STATE_GOT_SOS)) { gst_buffer_unmap (frame->buffer, &mapinfo); frame->out_buffer = gst_buffer_copy_region (frame->buffer, GST_BUFFER_COPY_ALL, 0, seg.offset - 2); GST_MINI_OBJECT_FLAGS (frame->out_buffer) |= GST_BUFFER_FLAG_CORRUPTED; GST_WARNING_OBJECT (parse, "Push a frame without EOI, size %d", seg.offset - 2); return gst_jpeg_parse_finish_frame (parse, frame, seg.offset - 2); } prev_state = parse->state; gst_jpeg_parse_reset (parse); parse->state |= prev_state | GST_JPEG_PARSER_STATE_GOT_SOI; if (!valid_state (parse->state, GST_JPEG_PARSER_STATE_GOT_METADATA)) { /* unset tags */ gst_base_parse_merge_tags (bparse, NULL, GST_TAG_MERGE_UNDEFINED); *skipsize = offset - 2; GST_DEBUG_OBJECT (parse, "skipping %d bytes before SOI", *skipsize); parse->last_offset = 2; goto beach; } } /* unset tags */ gst_base_parse_merge_tags (bparse, NULL, GST_TAG_MERGE_UNDEFINED); parse->state |= GST_JPEG_PARSER_STATE_GOT_SOI; break; case GST_JPEG_MARKER_EOI: if (gst_jpeg_parse_should_finish_buffer (parse, marker)) { gst_buffer_unmap (frame->buffer, &mapinfo); return gst_jpeg_parse_finish_frame (parse, frame, seg.offset); } else if (parse->mpf.mode) { parse->mpf.cur_image_index++; GST_DEBUG_OBJECT (parse, "finished image number %d of %d", parse->mpf.cur_image_index, parse->mpf.num_images); /* reset the state to continue parsing */ parse->state = GST_JPEG_PARSER_STATE_GOT_METADATA; } else if (parse->interlace_mode == GST_VIDEO_INTERLACE_MODE_INTERLEAVED && parse->field == 0) { parse->field = 1; /* reset the state to continue parsing */ parse->state = GST_JPEG_PARSER_STATE_GOT_METADATA; } break; case GST_JPEG_MARKER_SOS: if (!valid_state (parse->state, GST_JPEG_PARSER_STATE_GOT_SOF)) GST_WARNING_OBJECT (parse, "SOS marker without SOF one"); parse->state |= GST_JPEG_PARSER_STATE_GOT_SOS; break; case GST_JPEG_MARKER_COM: if (!gst_jpeg_parse_com (parse, &seg)) GST_WARNING_OBJECT (parse, "Failed to parse com segment"); break; case GST_JPEG_MARKER_APP0: if (!gst_jpeg_parse_app0 (parse, &seg)) GST_WARNING_OBJECT (parse, "Failed to parse app0 segment"); break; case GST_JPEG_MARKER_APP1: if (!gst_jpeg_parse_app1 (parse, &seg)) GST_WARNING_OBJECT (parse, "Failed to parse app1 segment"); break; case GST_JPEG_MARKER_APP2: if (!gst_jpeg_parse_app2 (parse, &seg)) GST_WARNING_OBJECT (parse, "Failed to parse app2 segment"); break; case GST_JPEG_MARKER_APP14: if (!gst_jpeg_parse_app14 (parse, &seg)) GST_WARNING_OBJECT (parse, "Failed to parse app14 segment"); break; case GST_JPEG_MARKER_DHT: case GST_JPEG_MARKER_DAC: /* to avoid break the below SOF interval */ break; default: /* SOFn segments */ if (marker >= GST_JPEG_MARKER_SOF_MIN && marker <= GST_JPEG_MARKER_SOF_MAX) { if (!valid_state (parse->state, GST_JPEG_PARSER_STATE_GOT_SOF) && gst_jpeg_parse_sof (parse, &seg)) { gint8 sof; parse->state |= GST_JPEG_PARSER_STATE_GOT_SOF; sof = marker - 0xc0; if (parse->sof != sof) { parse->sof = sof; parse->renegotiate = TRUE; } } else { GST_ELEMENT_ERROR (parse, STREAM, FORMAT, ("Invalid data"), ("Duplicated or bad SOF marker")); gst_buffer_unmap (frame->buffer, &mapinfo); gst_jpeg_parse_reset (parse); return GST_FLOW_ERROR; } } break; } } parse->last_offset = offset; beach: gst_buffer_unmap (frame->buffer, &mapinfo); return GST_FLOW_OK; } static gboolean gst_jpeg_parse_sink_event (GstBaseParse * bparse, GstEvent * event) { GstJpegParse *parse = GST_JPEG_PARSE_CAST (bparse); GST_DEBUG_OBJECT (parse, "event : %s", GST_EVENT_TYPE_NAME (event)); switch (GST_EVENT_TYPE (event)) { case GST_EVENT_FLUSH_STOP: gst_jpeg_parse_reset (parse); break; default: break; } return GST_BASE_PARSE_CLASS (parent_class)->sink_event (bparse, event); } static gboolean gst_jpeg_parse_start (GstBaseParse * bparse) { GstJpegParse *parse = GST_JPEG_PARSE_CAST (bparse); parse->framerate_numerator = 0; parse->framerate_denominator = 1; parse->first_picture = TRUE; parse->renegotiate = TRUE; parse->par_num = parse->par_den = 1; parse->interlace_mode = GST_VIDEO_INTERLACE_MODE_PROGRESSIVE; parse->field_order = GST_VIDEO_FIELD_ORDER_TOP_FIELD_FIRST; gst_jpeg_parse_reset (parse); gst_base_parse_set_min_frame_size (bparse, 2); return TRUE; } static gboolean gst_jpeg_parse_stop (GstBaseParse * bparse) { GstJpegParse *parse = GST_JPEG_PARSE_CAST (bparse); if (parse->tags) { gst_tag_list_unref (parse->tags); parse->tags = NULL; } gst_clear_buffer (&parse->codec_data); gst_clear_caps (&parse->prev_caps); g_clear_pointer (&parse->colorimetry, g_free); return TRUE; }