libextractor

thumbnailffmpeg_extractor.c (44693B)

---

 /*
      This file is part of libextractor.
      Copyright (C) 2008, 2012, 2026 Vidyut Samanta and Christian Grothoff
 
      libextractor is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
      by the Free Software Foundation; either version 3, or (at your
      option) any later version.
 
      libextractor is distributed in the hope that it will be useful, but
      WITHOUT ANY WARRANTY; without even the implied warranty of
      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
      General Public License for more details.
 
      You should have received a copy of the GNU General Public License
      along with libextractor; see the file COPYING.  If not, write to the
      Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
      Boston, MA 02110-1301, USA.
  */
 /**
  * @file plugins/thumbnailffmpeg_extractor.c
  * @author Christian Grothoff
  * @brief this extractor produces a binary encoded thumbnail of a frame
  *        taken from a video, using FFmpeg for demuxing, decoding and
  *        scaling
  *
  * Unlike the gtk thumbnailer, which can only re-scale a still image, this
  * plugin seeks about 30s into a video and encodes the frame it finds
  * there as a small JPEG (or PNG).  A previous incarnation of this plugin
  * was removed in 2021 because it had accumulated compatibility code for
  * every libav and FFmpeg API since 2008; this one deliberately requires
  * FFmpeg 5.1 or newer and uses only the send/receive APIs, so that there
  * is nothing to keep in sync with upstream churn.
  *
  * Besides the thumbnail, the plugin always reports the language of every
  * subtitle stream.  Two options add more expensive output:
  *
  * - "srt:LANG" dumps the first text subtitle stream in language LANG as
  *   a complete SubRip file.  This demuxes the entire file (without
  *   decoding the video), and is done first, on a fresh demuxer.
  * - "gallery:N" replaces the single thumbnail by a contact sheet of N
  *   frames sampled evenly over the whole duration.  Frames that show
  *   nothing (black, white, a fade) are replaced by a nearby frame of the
  *   same slice of the video.  If the deadline expires while the sheet is
  *   being assembled, the sheet is emitted with the tiles obtained so far
  *   and black cells for the rest: a partial overview is more useful than
  *   a single frame, and trying to get a single frame once the budget is
  *   used up would not work anyway.  Without a known duration there is
  *   nothing to spread the samples over, and the plugin falls back to the
  *   single thumbnail.
  */
 #include "platform.h"
 #include "extractor.h"
 #include "mediautil.h"
 #include "mediaffmpeg.h"
 
 #include <libavutil/imgutils.h>
 #include <libswscale/swscale.h>
 
 /**
  * Give up after this many decoded frames, no matter what the timestamps
  * say.  A malformed file can otherwise keep us decoding forever.
  */
 #define MAX_FRAMES 2048
 
 /**
  * Mean absolute deviation below which we consider a frame to be a flat
  * surface (a fade-in, a black leader) and not worth showing.
  */
 #define FLAT_FRAME_MAD 2
 
 /**
  * How far to jump ahead when we hit a flat frame, in seconds.
  */
 #define FLAT_FRAME_SKIP 5
 
 /**
  * How often we are willing to do that.
  */
 #define MAX_FLAT_RETRIES 2
 
 /**
  * JPEG quantizer scale; smaller is better and bigger.
  */
 #define JPEG_QSCALE 5
 
 /**
  * Give up on reaching a gallery sample after decoding this many frames
  * past the keyframe we seeked to.
  */
 #define GALLERY_MAX_FRAMES 512
 
 /**
  * Largest contact sheet we assemble, in pixels.  Far below
  * #EXTRACTOR_MEDIA_MAX_PIXELS, because the sheet has to be encoded in
  * one go, without any I/O that would tell the parent process we are
  * still alive, and a sheet this size already cannot be squeezed into
  * #EXTRACTOR_MEDIA_MAX_GALLERY_BYTES as a PNG unless it is very simple.
  */
 #define GALLERY_MAX_SHEET_PIXELS EXTRACTOR_MEDIA_MAX_GALLERY_PIXELS
 
 /**
  * zlib level for PNG galleries; 9 costs a lot of time on a big sheet
  * for a few percent.
  */
 #define GALLERY_PNG_LEVEL 6
 
 /**
  * Most distinct subtitle languages we report.
  */
 #define MAX_LANGUAGES 64
 
 /**
  * Longest language tag we report (e.g. "zh-Hant-TW").
  */
 #define MAX_LANGUAGE_LEN 15
 
 /**
  * Most subtitle cues we collect for an SRT dump.
  */
 #define MAX_CUES 100000
 
 /**
  * How long a cue whose end is unknown and that has no successor stays
  * on screen, in milliseconds.
  */
 #define DEFAULT_CUE_MS 5000
 
 /**
  * Cap for any time stamp in an SRT dump (1000 hours), in milliseconds.
  */
 #define MAX_CUE_TIME_MS (1000ULL * 3600 * 1000)
 
 
 /**
  * Is @a frame a flat surface?  Cheap heuristic over the first plane; we
  * only want to catch the completely uniform black frame of a fade-in.
  *
  * @param frame frame to inspect
  * @return 1 if the frame carries no discernible detail
  */
 static int
 is_flat (const AVFrame *frame)
 {
   const uint8_t *p = frame->data[0];
   int stride = frame->linesize[0];
   uint64_t sum = 0;
   uint64_t dev = 0;
   unsigned int n = 0;
   int x;
   int y;
 
   if ( (NULL == p) ||
        (stride <= 0) ||
        (frame->height <= 0) )
     return 0;
   for (y = 0; y < frame->height; y += 4)
     for (x = 0; x < stride; x += 4)
     {
       sum += p[y * stride + x];
       n++;
     }
   if (0 == n)
     return 0;
   sum /= n;
   for (y = 0; y < frame->height; y += 4)
     for (x = 0; x < stride; x += 4)
     {
       uint8_t v = p[y * stride + x];
 
       dev += (v > sum) ? (v - sum) : (sum - v);
     }
   return (dev / n) < FLAT_FRAME_MAD;
 }
 
 
 /**
  * Scale @a frame to @a width x @a height and encode it as JPEG or PNG.
  *
  * @param frame the image to convert (any pixel format swscale accepts)
  * @param width width of the result
  * @param height height of the result
  * @param png 1 for PNG, 0 for JPEG
  * @param qscale JPEG quantizer scale (ignored for PNG)
  * @param level zlib level for PNG (ignored for JPEG)
  * @param max_bytes largest acceptable result
  * @param[out] output set to the encoded image, to be freed with av_free()
  * @param[out] output_size set to the number of bytes in @a output
  * @return 0 on success, -2 if the result would exceed @a max_bytes,
  *         -1 on other errors
  */
 static int
 encode_image (const AVFrame *frame,
               unsigned int width,
               unsigned int height,
               int png,
               int qscale,
               int level,
               size_t max_bytes,
               uint8_t **output,
               size_t *output_size)
 {
   const AVCodec *codec;
   AVCodecContext *ctx = NULL;
   struct SwsContext *sws = NULL;
   AVFrame *dst = NULL;
   AVPacket *pkt = NULL;
   enum AVPixelFormat dst_fmt;
   int ret = -1;
   int r;
 
   if ( (0 == width) ||
        (0 == height) ||
        (width > EXTRACTOR_MEDIA_MAX_DIMENSION) ||
        (height > EXTRACTOR_MEDIA_MAX_DIMENSION) )
     return -1;
   if (! sws_isSupportedInput (frame->format))
     return -1;
   if (png)
   {
     dst_fmt = AV_PIX_FMT_RGB24;
     codec = avcodec_find_encoder (AV_CODEC_ID_PNG);
   }
   else
   {
     dst_fmt = AV_PIX_FMT_YUVJ420P;
     codec = avcodec_find_encoder (AV_CODEC_ID_MJPEG);
   }
   if (NULL == codec)
     return -1;
   if (NULL == (sws = sws_getContext (frame->width,
                                      frame->height,
                                      frame->format,
                                      (int) width,
                                      (int) height,
                                      dst_fmt,
                                      SWS_BILINEAR,
                                      NULL,
                                      NULL,
                                      NULL)))
     return -1;
   if (NULL == (dst = av_frame_alloc ()))
     goto cleanup;
   dst->format = dst_fmt;
   dst->width = (int) width;
   dst->height = (int) height;
   dst->pts = 0;
   if (0 != av_frame_get_buffer (dst, 0))
     goto cleanup;
   if (0 > sws_scale (sws,
                      (const uint8_t *const *) frame->data,
                      frame->linesize,
                      0,
                      frame->height,
                      dst->data,
                      dst->linesize))
     goto cleanup;
   if (NULL == (ctx = avcodec_alloc_context3 (codec)))
     goto cleanup;
   ctx->width = (int) width;
   ctx->height = (int) height;
   ctx->pix_fmt = dst_fmt;
   ctx->time_base = (AVRational) { 1, 25 };
   ctx->thread_count = 1;
   if (png)
   {
     ctx->compression_level = level;
   }
   else
   {
     ctx->color_range = AVCOL_RANGE_JPEG;
     ctx->flags |= AV_CODEC_FLAG_QSCALE;
     ctx->global_quality = FF_QP2LAMBDA * qscale;
   }
   if (0 != avcodec_open2 (ctx, codec, NULL))
     goto cleanup;
   if (NULL == (pkt = av_packet_alloc ()))
     goto cleanup;
   dst->quality = ctx->global_quality;
   if (0 != avcodec_send_frame (ctx, dst))
     goto cleanup;
   r = avcodec_receive_packet (ctx, pkt);
   if (AVERROR (EAGAIN) == r)
   {
     /* encoder wants to be flushed before it hands anything back */
     if (0 != avcodec_send_frame (ctx, NULL))
       goto cleanup;
     r = avcodec_receive_packet (ctx, pkt);
   }
   if (0 != r)
     goto cleanup;
   if (pkt->size <= 0)
     goto cleanup;
   if ((size_t) pkt->size > max_bytes)
   {
     ret = -2;
     goto cleanup;
   }
   if (NULL == (*output = av_malloc (pkt->size)))
     goto cleanup;
   memcpy (*output,
           pkt->data,
           pkt->size);
   *output_size = (size_t) pkt->size;
   ret = 0;
 cleanup:
   if (NULL != pkt)
     av_packet_free (&pkt);
   if (NULL != ctx)
     avcodec_free_context (&ctx);
   if (NULL != dst)
     av_frame_free (&dst);
   sws_freeContext (sws);
   return ret;
 }
 
 
 /**
  * Scale @a frame down and encode it as JPEG or PNG.
  *
  * @param frame the decoded frame to convert
  * @param opt plugin options (target size and format)
  * @param[out] output set to the encoded image, to be freed with av_free()
  * @param[out] output_size set to the number of bytes in @a output
  * @return 0 on success
  */
 static int
 encode_thumbnail (const AVFrame *frame,
                   const struct EXTRACTOR_MediaOptions *opt,
                   uint8_t **output,
                   size_t *output_size)
 {
   unsigned int width;
   unsigned int height;
 
   if (! EXTRACTOR_media_thumbnail_size_ (frame->width,
                                          frame->height,
                                          opt->size,
                                          &width,
                                          &height))
     return -1;
   return (0 == encode_image (frame,
                              width,
                              height,
                              opt->png,
                              JPEG_QSCALE,
                              9,
                              EXTRACTOR_MEDIA_MAX_THUMBNAIL,
                              output,
                              output_size)) ? 0 : -1;
 }
 
 
 /**
  * Decode the video stream until we reach @a target seconds, and hand back
  * the frame we find there.  If we never get there (short file, broken
  * timestamps), the last frame we did decode is returned instead.
  *
  * Normally "there" is the first frame at or after @a target.  With
  * @a displayed, it is the frame on screen at @a target, i.e. the last
  * one starting at or before it; the difference matters for streams
  * that only carry a new frame when the picture changes.
  *
  * @param io open I/O state
  * @param ctx open decoder for @a stream
  * @param stream index of the video stream
  * @param target position we are aiming for, in seconds
  * @param skip_flat 1 to move on (a few times) if the frame at @a target
  *        is a flat surface
  * @param displayed 1 to return the frame displayed at @a target
  * @param max_frames give up after decoding this many frames
  * @param[out] frame frame to fill in
  * @return 0 if @a frame was filled in
  */
 static int
 decode_frame_at (struct EXTRACTOR_FFmpegIO *io,
                  AVCodecContext *ctx,
                  int stream,
                  double target,
                  int skip_flat,
                  int displayed,
                  unsigned int max_frames,
                  AVFrame *frame)
 {
   AVPacket *pkt;
   AVFrame *last;
   AVRational tb = io->fmt->streams[stream]->time_base;
   unsigned int frames = 0;
   unsigned int retries = 0;
   int have_last = 0;
   int overshot = 0;
   int ret = -1;
 
   if (NULL == (pkt = av_packet_alloc ()))
     return -1;
   if (NULL == (last = av_frame_alloc ()))
   {
     av_packet_free (&pkt);
     return -1;
   }
   while (frames < max_frames)
   {
     int r;
 
     if (EXTRACTOR_media_deadline_expired_ (&io->deadline))
       break;
     /* decoding can go on for a while without touching the file */
     (void) io->ec->seek (io->ec->cls,
                          0,
                          SEEK_CUR);
     r = av_read_frame (io->fmt, pkt);
     if (0 != r)
     {
       /* end of file: flush the decoder and take what is left */
       avcodec_send_packet (ctx, NULL);
     }
     else if (pkt->stream_index != stream)
     {
       av_packet_unref (pkt);
       continue;
     }
     else if (0 != avcodec_send_packet (ctx, pkt))
     {
       av_packet_unref (pkt);
       continue;
     }
     av_packet_unref (pkt);
     while (0 == avcodec_receive_frame (ctx, frame))
     {
       double ts;
 
       frames++;
       ts = (AV_NOPTS_VALUE == frame->best_effort_timestamp)
            ? -1.0
            : frame->best_effort_timestamp * av_q2d (tb);
       if ( (ts < target) &&
            (frames < max_frames) )
       {
         /* not there yet; hold on to it in case we never arrive */
         av_frame_unref (last);
         if (0 == av_frame_ref (last, frame))
           have_last = 1;
         av_frame_unref (frame);
         continue;
       }
       if (displayed &&
           have_last &&
           (ts > target) )
       {
         /* overshot: the previous frame is still on screen at target */
         av_frame_unref (frame);
         overshot = 1;
         break;
       }
       if (skip_flat &&
           (retries < MAX_FLAT_RETRIES) &&
           is_flat (frame) )
       {
         /* a black leader or a fade-in; try a bit further in */
         retries++;
         target = ((ts < 0.0) ? target : ts) + FLAT_FRAME_SKIP;
         av_frame_unref (last);
         if (0 == av_frame_ref (last, frame))
           have_last = 1;
         av_frame_unref (frame);
         continue;
       }
       av_frame_unref (last);
       av_frame_free (&last);
       av_packet_free (&pkt);
       return 0;
     }
     if (overshot)
       break;
     if (0 != r)
       break;   /* end of file, and the decoder had nothing left either */
   }
   if (have_last)
   {
     av_frame_unref (frame);
     if (0 == av_frame_ref (frame, last))
       ret = 0;
   }
   av_frame_unref (last);
   av_frame_free (&last);
   av_packet_free (&pkt);
   return ret;
 }
 
 
 /**
  * Is @a tag something we are willing to report as a language?  Stream
  * metadata comes straight from the file, so only accept what a language
  * tag can look like (letters, digits, '-' and '_'), and skip the
  * "undetermined" placeholder.
  *
  * @param tag language tag of a stream, may be NULL
  * @return 1 if @a tag should be reported
  */
 static int
 is_reportable_language (const char *tag)
 {
   size_t len;
   size_t i;
 
   if (NULL == tag)
     return 0;
   len = strlen (tag);
   if ( (0 == len) ||
        (len > MAX_LANGUAGE_LEN) )
     return 0;
   for (i = 0; i < len; i++)
     if (! ( ( (tag[i] >= 'a') && (tag[i] <= 'z') ) ||
             ( (tag[i] >= 'A') && (tag[i] <= 'Z') ) ||
             ( (tag[i] >= '0') && (tag[i] <= '9') ) ||
             ('-' == tag[i]) ||
             ('_' == tag[i]) ) )
       return 0;
   if (0 == strcasecmp (tag,
                        "und"))
     return 0;
   return 1;
 }
 
 
 /**
  * Report the language of every subtitle stream, each distinct language
  * once, in stream order.
  *
  * @param io open I/O state
  * @return 0 to continue, 1 if the application asked us to stop
  */
 static int
 emit_subtitle_languages (struct EXTRACTOR_FFmpegIO *io)
 {
   const char *seen[MAX_LANGUAGES];
   unsigned int nseen = 0;
   unsigned int i;
 
   for (i = 0; i < io->fmt->nb_streams; i++)
   {
     const AVStream *st = io->fmt->streams[i];
     const AVDictionaryEntry *e;
     unsigned int j;
 
     if (AVMEDIA_TYPE_SUBTITLE != st->codecpar->codec_type)
       continue;
     e = av_dict_get (st->metadata,
                      "language",
                      NULL,
                      0);
     if ( (NULL == e) ||
          (! is_reportable_language (e->value)) )
       continue;
     for (j = 0; j < nseen; j++)
       if (0 == strcasecmp (seen[j],
                            e->value))
         break;
     if (j < nseen)
       continue;
     if (nseen == MAX_LANGUAGES)
       break;
     seen[nseen++] = e->value;
     if (0 != io->ec->proc (io->ec->cls,
                            "thumbnailffmpeg",
                            EXTRACTOR_METATYPE_SUBTITLE_LANGUAGE,
                            EXTRACTOR_METAFORMAT_UTF8,
                            "text/plain",
                            e->value,
                            strlen (e->value) + 1))
       return 1;
   }
   return 0;
 }
 
 
 /**
  * One subtitle event, collected before we can write the SRT file.
  */
 struct Cue
 {
   /**
    * Start in milliseconds from the beginning of the file.
    */
   uint64_t start;
 
   /**
    * End in milliseconds, UINT64_MAX if unknown.
    */
   uint64_t end;
 
   /**
    * Text of the cue (not 0-terminated).
    */
   char *text;
 
   /**
    * Number of bytes in @e text.
    */
   size_t len;
 
   /**
    * Position in the stream, to keep the sort stable.
    */
   unsigned int seq;
 
   /**
    * 1 if @e text is an ASS dialogue event.
    */
   int is_ass;
 };
 
 
 /**
  * The cues of a subtitle stream.
  */
 struct CueList
 {
   /**
    * Array of cues.
    */
   struct Cue *cues;
 
   /**
    * Number of valid entries in @e cues.
    */
   unsigned int count;
 
   /**
    * Allocated length of @e cues.
    */
   unsigned int cap;
 
   /**
    * Total number of text bytes in @e cues.
    */
   size_t bytes;
 
   /**
    * Set once we ran into one of the limits.
    */
   int full;
 };
 
 
 /**
  * Append a cue to @a cl.
  *
  * @param[in,out] cl list to extend
  * @param start start in milliseconds
  * @param end end in milliseconds, UINT64_MAX if unknown
  * @param text text of the cue
  * @param is_ass 1 if @a text is an ASS dialogue event
  */
 static void
 cue_add (struct CueList *cl,
          uint64_t start,
          uint64_t end,
          const char *text,
          int is_ass)
 {
   struct Cue *c;
   size_t len;
 
   if ( (NULL == text) ||
        (cl->full) )
     return;
   len = strlen (text);
   if (0 == len)
     return;
   if ( (cl->count >= MAX_CUES) ||
        (cl->bytes + len > EXTRACTOR_MEDIA_MAX_SRT) )
   {
     cl->full = 1;
     return;
   }
   if (cl->count == cl->cap)
   {
     unsigned int ncap = (0 == cl->cap) ? 64 : 2 * cl->cap;
     struct Cue *n;
 
     if (ncap > MAX_CUES)
       ncap = MAX_CUES;
     n = realloc (cl->cues,
                  ncap * sizeof (struct Cue));
     if (NULL == n)
     {
       cl->full = 1;
       return;
     }
     cl->cues = n;
     cl->cap = ncap;
   }
   c = &cl->cues[cl->count];
   if (NULL == (c->text = malloc (len)))
   {
     cl->full = 1;
     return;
   }
   memcpy (c->text,
           text,
           len);
   c->len = len;
   c->start = start;
   c->end = end;
   c->is_ass = is_ass;
   c->seq = cl->count;
   cl->bytes += len;
   cl->count++;
 }
 
 
 /**
  * Order cues by start time, then by position in the stream.
  *
  * @param a first `struct Cue`
  * @param b second `struct Cue`
  * @return -1, 0 or 1 as required by qsort()
  */
 static int
 cue_cmp (const void *a,
          const void *b)
 {
   const struct Cue *ca = a;
   const struct Cue *cb = b;
 
   if (ca->start != cb->start)
     return (ca->start < cb->start) ? -1 : 1;
   if (ca->seq != cb->seq)
     return (ca->seq < cb->seq) ? -1 : 1;
   return 0;
 }
 
 
 /**
  * Release @a cl.
  *
  * @param[in,out] cl list to release
  */
 static void
 cue_clear (struct CueList *cl)
 {
   unsigned int i;
 
   for (i = 0; i < cl->count; i++)
     free (cl->cues[i].text);
   free (cl->cues);
   memset (cl, 0, sizeof (*cl));
 }
 
 
 /**
  * Convert @a ts in @a tb to milliseconds relative to @a origin_ms,
  * clamped to [0, #MAX_CUE_TIME_MS].
  *
  * @param ts time stamp
  * @param tb time base of @a ts
  * @param origin_ms start of the file in milliseconds
  * @return time in milliseconds
  */
 static uint64_t
 to_cue_ms (int64_t ts,
            AVRational tb,
            int64_t origin_ms)
 {
   int64_t ms;
 
   if ( (tb.num <= 0) ||
        (tb.den <= 0) )
     return 0;
   ms = av_rescale_q (ts,
                      tb,
                      (AVRational) { 1, 1000 });
   if ( (ms <= 0) ||
        (ms <= origin_ms) )
     return 0;
   if (origin_ms > 0)
     ms -= origin_ms;
   if ((uint64_t) ms > MAX_CUE_TIME_MS)
     return MAX_CUE_TIME_MS;
   return (uint64_t) ms;
 }
 
 
 /**
  * Decode one subtitle packet and collect its cues.
  *
  * @param ctx open subtitle decoder
  * @param pkt packet of the subtitle stream
  * @param tb time base of the stream
  * @param origin_ms start of the file in milliseconds
  * @param[in,out] cl where to put the cues
  */
 static void
 decode_subtitle_packet (AVCodecContext *ctx,
                         AVPacket *pkt,
                         AVRational tb,
                         int64_t origin_ms,
                         struct CueList *cl)
 {
   AVSubtitle sub;
   int got = 0;
   int64_t pts;
   uint64_t base;
   uint64_t start;
   uint64_t end;
   unsigned int i;
 
   pts = (AV_NOPTS_VALUE != pkt->pts) ? pkt->pts : pkt->dts;
   if (AV_NOPTS_VALUE == pts)
     return;   /* no idea when to show it */
   memset (&sub, 0, sizeof (sub));
   if ( (0 > avcodec_decode_subtitle2 (ctx,
                                       &sub,
                                       &got,
                                       pkt)) ||
        (! got) )
     return;
   base = to_cue_ms (pts,
                     tb,
                     origin_ms);
   start = base + sub.start_display_time;
   if ( (sub.end_display_time > sub.start_display_time) &&
        (UINT32_MAX != sub.end_display_time) )
     end = base + sub.end_display_time;
   else if (pkt->duration > 0)
     end = start + to_cue_ms (pkt->duration,
                              tb,
                              0);
   else
     end = UINT64_MAX;
   for (i = 0; i < sub.num_rects; i++)
   {
     const AVSubtitleRect *r = sub.rects[i];
 
     if (NULL == r)
       continue;
     if (SUBTITLE_ASS == r->type)
       cue_add (cl, start, end, r->ass, 1);
     else if (SUBTITLE_TEXT == r->type)
       cue_add (cl, start, end, r->text, 0);
   }
   avsubtitle_free (&sub);
 }
 
 
 /**
  * Find the first subtitle stream in language @a lang.
  *
  * @param io open I/O state
  * @param lang requested language
  * @return stream index, -1 if there is none
  */
 static int
 find_subtitle_stream (struct EXTRACTOR_FFmpegIO *io,
                       const char *lang)
 {
   unsigned int i;
 
   for (i = 0; i < io->fmt->nb_streams; i++)
   {
     const AVStream *st = io->fmt->streams[i];
     const AVDictionaryEntry *e;
 
     if (AVMEDIA_TYPE_SUBTITLE != st->codecpar->codec_type)
       continue;
     e = av_dict_get (st->metadata,
                      "language",
                      NULL,
                      0);
     if ( (NULL != e) &&
          EXTRACTOR_media_lang_matches_ (lang,
                                         e->value) )
       return (int) i;
   }
   return -1;
 }
 
 
 /**
  * Dump the first subtitle stream in the language requested with
  * "srt:LANG" as a SubRip file.  Must be called on a freshly opened
  * demuxer, as it reads the file from where the demuxer stands.
  *
  * @param io open I/O state
  * @param opt plugin options
  * @param[out] touched set to 1 if we read (and discarded) packets, so
  *             that the caller has to reopen the input
  * @return 0 to continue, 1 if the application asked us to stop
  */
 static int
 dump_srt (struct EXTRACTOR_FFmpegIO *io,
           const struct EXTRACTOR_MediaOptions *opt,
           int *touched)
 {
   const AVCodecDescriptor *desc;
   const AVCodec *codec;
   AVCodecContext *ctx = NULL;
   AVPacket *pkt = NULL;
   AVStream *st;
   struct CueList cl;
   struct EXTRACTOR_MediaSrt srt;
   int64_t origin_ms = 0;
   uint64_t next_start;
   unsigned int i;
   int stream;
   int ret = 0;
 
   *touched = 0;
   memset (&cl, 0, sizeof (cl));
   memset (&srt, 0, sizeof (srt));
   stream = find_subtitle_stream (io,
                                  opt->srt_lang);
   if (0 > stream)
     return 0;
   st = io->fmt->streams[stream];
   desc = avcodec_descriptor_get (st->codecpar->codec_id);
   if ( (NULL == desc) ||
        (0 == (desc->props & AV_CODEC_PROP_TEXT_SUB)) )
     return 0;   /* bitmap subtitles; nothing we could write as text */
   if (NULL == (codec = avcodec_find_decoder (st->codecpar->codec_id)))
     return 0;
   if (NULL == (ctx = avcodec_alloc_context3 (codec)))
     return 0;
   if (0 > avcodec_parameters_to_context (ctx,
                                          st->codecpar))
     goto cleanup;
   ctx->pkt_timebase = st->time_base;
   ctx->thread_count = 1;
   if (0 != avcodec_open2 (ctx, codec, NULL))
     goto cleanup;
   if (NULL == (pkt = av_packet_alloc ()))
     goto cleanup;
   if (AV_NOPTS_VALUE != io->fmt->start_time)
     origin_ms = io->fmt->start_time / 1000;
   /* let the demuxer skip everything else */
   for (i = 0; i < io->fmt->nb_streams; i++)
     if ((int) i != stream)
       io->fmt->streams[i]->discard = AVDISCARD_ALL;
   *touched = 1;
   while ( (! cl.full) &&
           (! EXTRACTOR_media_deadline_expired_ (&io->deadline)) )
   {
     (void) io->ec->seek (io->ec->cls,
                          0,
                          SEEK_CUR); /* keep-alive */
     if (0 != av_read_frame (io->fmt,
                             pkt))
       break;
     if (pkt->stream_index == stream)
       decode_subtitle_packet (ctx,
                               pkt,
                               st->time_base,
                               origin_ms,
                               &cl);
     av_packet_unref (pkt);
   }
   if (0 == cl.count)
     goto cleanup;
   /* containers normally deliver subtitles in order, but nothing forces
      them to; and the ends of open-ended cues depend on the order */
   qsort (cl.cues,
          cl.count,
          sizeof (struct Cue),
          &cue_cmp);
   next_start = UINT64_MAX;
   for (i = cl.count; i > 0; i--)
   {
     struct Cue *c = &cl.cues[i - 1];
 
     if (UINT64_MAX == c->end)
       c->end = (UINT64_MAX != next_start)
                ? next_start
                : c->start + DEFAULT_CUE_MS;
     if (c->end < c->start)
       c->end = c->start;
     if ( (i > 1) &&
          (cl.cues[i - 2].start < c->start) )
       next_start = c->start;
   }
   for (i = 0; i < cl.count; i++)
     EXTRACTOR_media_srt_add_ (&srt,
                               cl.cues[i].start,
                               cl.cues[i].end,
                               cl.cues[i].text,
                               cl.cues[i].len,
                               cl.cues[i].is_ass);
   if (0 != EXTRACTOR_media_srt_emit_ (&srt,
                                       io->ec,
                                       "thumbnailffmpeg"))
     ret = 1;
 cleanup:
   EXTRACTOR_media_srt_clear_ (&srt);
   cue_clear (&cl);
   if (NULL != pkt)
     av_packet_free (&pkt);
   avcodec_free_context (&ctx);
   return ret;
 }
 
 
 /**
  * A contact sheet under construction.
  */
 struct Gallery
 {
   /**
    * The sheet, packed RGB24, initially black.
    */
   uint8_t *sheet;
 
   /**
    * Scratch buffer for one tile, packed RGB24.
    */
   uint8_t *tile;
 
   /**
    * The least flat candidate for the current tile, packed RGB24.
    */
   uint8_t *best;
 
   /**
    * Scaler from decoded frames to tiles.
    */
   struct SwsContext *sws;
 
   /**
    * Width of a tile.
    */
   unsigned int tw;
 
   /**
    * Height of a tile.
    */
   unsigned int th;
 
   /**
    * Number of columns.
    */
   unsigned int cols;
 
   /**
    * Number of rows.
    */
   unsigned int rows;
 };
 
 
 /**
  * How much is going on in a tile: the sum of the variances of its
  * three channels.  Used to pick the least boring of several flat
  * candidates.
  *
  * @param rgb packed RGB24 pixels
  * @param width width in pixels
  * @param height height in pixels
  * @return the score, larger means more detail
  */
 static double
 tile_score (const uint8_t *rgb,
             unsigned int width,
             unsigned int height)
 {
   double sum[3] = { 0, 0, 0 };
   double sq[3] = { 0, 0, 0 };
   double n = 0;
   double score = 0;
   unsigned int x;
   unsigned int y;
   unsigned int c;
 
   for (y = 0; y < height; y += 2)
     for (x = 0; x < width; x += 2)
     {
       const uint8_t *p = &rgb[((size_t) y * width + x) * 3];
 
       for (c = 0; c < 3; c++)
       {
         sum[c] += p[c];
         sq[c] += (double) p[c] * p[c];
       }
       n++;
     }
   if (0 == n)
     return 0;
   for (c = 0; c < 3; c++)
     score += sq[c] / n - (sum[c] / n) * (sum[c] / n);
   return score;
 }
 
 
 /**
  * Seek to @a t_ms into the video, decode the frame there and scale it
  * into the tile buffer of @a g.
  *
  * @param io open I/O state
  * @param ctx open video decoder
  * @param stream index of the video stream
  * @param start_ts start of the stream in its time base
  * @param t_ms position relative to @a start_ts in milliseconds
  * @param frame scratch frame
  * @param[in,out] g gallery whose tile buffer to fill
  * @return 0 on success
  */
 static int
 grab_tile (struct EXTRACTOR_FFmpegIO *io,
            AVCodecContext *ctx,
            int stream,
            int64_t start_ts,
            uint64_t t_ms,
            AVFrame *frame,
            struct Gallery *g)
 {
   AVRational tb = io->fmt->streams[stream]->time_base;
   uint8_t *dst[4] = { g->tile, NULL, NULL, NULL };
   int dst_stride[4] = { (int) g->tw * 3, 0, 0, 0 };
   int64_t ts;
   int ret = -1;
 
   if ( (tb.num <= 0) ||
        (tb.den <= 0) )
     return -1;
   ts = av_rescale_q ((int64_t) t_ms,
                      (AVRational) { 1, 1000 },
                      tb);
   if ( (ts < 0) ||
        ( (start_ts > 0) &&
          (ts > INT64_MAX - start_ts) ) )
     return -1;
   ts += start_ts;
   if (0 > av_seek_frame (io->fmt,
                          stream,
                          ts,
                          AVSEEK_FLAG_BACKWARD))
     return -1;
   avcodec_flush_buffers (ctx);
   if (0 != decode_frame_at (io,
                             ctx,
                             stream,
                             ts * av_q2d (tb),
                             0,
                             1,
                             GALLERY_MAX_FRAMES,
                             frame))
     return -1;
   if ( (frame->width <= 0) ||
        (frame->height <= 0) ||
        (frame->width > EXTRACTOR_MEDIA_MAX_DIMENSION) ||
        (frame->height > EXTRACTOR_MEDIA_MAX_DIMENSION) ||
        (((int64_t) frame->width) * frame->height
         > EXTRACTOR_MEDIA_MAX_PIXELS) ||
        (! sws_isSupportedInput (frame->format)) )
     goto cleanup;
   g->sws = sws_getCachedContext (g->sws,
                                  frame->width,
                                  frame->height,
                                  frame->format,
                                  (int) g->tw,
                                  (int) g->th,
                                  AV_PIX_FMT_RGB24,
                                  SWS_BILINEAR,
                                  NULL,
                                  NULL,
                                  NULL);
   if (NULL == g->sws)
     goto cleanup;
   if ((int) g->th != sws_scale (g->sws,
                                 (const uint8_t *const *) frame->data,
                                 frame->linesize,
                                 0,
                                 frame->height,
                                 dst,
                                 dst_stride))
     goto cleanup;
   ret = 0;
 cleanup:
   av_frame_unref (frame);
   return ret;
 }
 
 
 /**
  * Copy a tile into cell @a i of the sheet.
  *
  * @param[in,out] g gallery to update
  * @param i index of the cell
  * @param tile packed RGB24 pixels of the tile
  */
 static void
 place_tile (struct Gallery *g,
             unsigned int i,
             const uint8_t *tile)
 {
   size_t sheet_stride = (size_t) g->cols * g->tw * 3;
   size_t tile_stride = (size_t) g->tw * 3;
   size_t x0 = (size_t) (i % g->cols) * g->tw * 3;
   size_t y0 = (size_t) (i / g->cols) * g->th;
   unsigned int y;
 
   for (y = 0; y < g->th; y++)
     memcpy (&g->sheet[(y0 + y) * sheet_stride + x0],
             &tile[y * tile_stride],
             tile_stride);
 }
 
 
 /**
  * Encode the finished sheet, degrading quality and then size until it
  * fits into #EXTRACTOR_MEDIA_MAX_GALLERY_BYTES.
  *
  * @param ec extraction context, kept alive between attempts
  * @param g the gallery
  * @param opt plugin options
  * @param[out] output set to the image, to be freed with av_free()
  * @param[out] output_size set to the number of bytes in @a output
  * @return 0 on success
  */
 static int
 encode_gallery (struct EXTRACTOR_ExtractContext *ec,
                 const struct Gallery *g,
                 const struct EXTRACTOR_MediaOptions *opt,
                 uint8_t **output,
                 size_t *output_size)
 {
   static const int qscales[] = { JPEG_QSCALE, 10, 20, 31 };
   AVFrame *src;
   unsigned int w;
   unsigned int h;
   unsigned int shrink;
   int r = -1;
 
   if (NULL == (src = av_frame_alloc ()))
     return -1;
   src->format = AV_PIX_FMT_RGB24;
   src->width = (int) (g->cols * g->tw);
   src->height = (int) (g->rows * g->th);
   src->data[0] = g->sheet;
   src->linesize[0] = (int) (g->cols * g->tw * 3);
   w = g->cols * g->tw;
   h = g->rows * g->th;
   for (shrink = 0; shrink < 4; shrink++)
   {
     (void) ec->seek (ec->cls, 0, SEEK_CUR); /* keep-alive */
     if (opt->png)
     {
       r = encode_image (src, w, h, 1, 0, GALLERY_PNG_LEVEL,
                         EXTRACTOR_MEDIA_MAX_GALLERY_BYTES,
                         output, output_size);
     }
     else
     {
       unsigned int q;
 
       for (q = 0; q < sizeof (qscales) / sizeof (qscales[0]); q++)
       {
         (void) ec->seek (ec->cls, 0, SEEK_CUR); /* keep-alive */
         r = encode_image (src, w, h, 0, qscales[q], 0,
                           EXTRACTOR_MEDIA_MAX_GALLERY_BYTES,
                           output, output_size);
         if (-2 != r)
           break;
       }
     }
     if (-2 != r)
       break;
     /* still too big: halve the sheet (i.e. every tile) */
     w /= 2;
     h /= 2;
     if ( (w < 2) ||
          (h < 2) )
       break;
   }
   /* the frame only borrowed the sheet */
   src->data[0] = NULL;
   av_frame_free (&src);
   return (0 == r) ? 0 : -1;
 }
 
 
 /**
  * Build and emit a contact sheet for "gallery:N".
  *
  * @param io open I/O state
  * @param ctx open video decoder
  * @param stream index of the video stream
  * @param opt plugin options
  * @param[out] stop set to 1 if the application asked us to stop
  * @param[out] touched set to 1 if we moved the demuxer
  * @return 0 if we are done (sheet emitted, or the application asked us
  *         to stop), 1 if the caller should fall back to a single
  *         thumbnail
  */
 static int
 make_gallery (struct EXTRACTOR_FFmpegIO *io,
               AVCodecContext *ctx,
               int stream,
               const struct EXTRACTOR_MediaOptions *opt,
               int *stop,
               int *touched)
 {
   /* where to look instead of a flat frame, in eighths of a slice */
   static const int shifts[] = { 0, 2, -2, 3, -3 };
   const AVStream *st = io->fmt->streams[stream];
   struct Gallery g;
   AVFrame *frame = NULL;
   uint8_t *image = NULL;
   size_t image_size = 0;
   uint64_t duration_ms;
   uint64_t slice;
   int64_t start_ts;
   unsigned int n = opt->gallery;
   unsigned int box = opt->size;
   unsigned int got = 0;
   unsigned int i;
   int ret = 1;
 
   *stop = 0;
   *touched = 0;
   memset (&g, 0, sizeof (g));
   if ( (AV_NOPTS_VALUE == io->fmt->duration) ||
        (io->fmt->duration <= 0) )
     return 1;
   duration_ms = (uint64_t) io->fmt->duration / 1000;
   if (duration_ms > MAX_CUE_TIME_MS)
     duration_ms = MAX_CUE_TIME_MS;
   if ( (0 == duration_ms) ||
        (0 == n) )
     return 1;
   /* shrink the tiles until the sheet is of a size we can handle */
   for (;;)
   {
     uint64_t sw;
     uint64_t sh;
 
     if (! EXTRACTOR_media_thumbnail_size_ (ctx->width,
                                            ctx->height,
                                            box,
                                            &g.tw,
                                            &g.th))
       return 1;
     EXTRACTOR_media_gallery_layout_ (n,
                                      g.tw,
                                      g.th,
                                      &g.cols,
                                      &g.rows);
     sw = (uint64_t) g.cols * g.tw;
     sh = (uint64_t) g.rows * g.th;
     if ( (sw * sh <= GALLERY_MAX_SHEET_PIXELS) &&
          (sw * sh <= EXTRACTOR_MEDIA_MAX_PIXELS) &&
          (sw <= EXTRACTOR_MEDIA_MAX_DIMENSION) &&
          (sh <= EXTRACTOR_MEDIA_MAX_DIMENSION) )
       break;
     if (box <= 8)
       return 1;
     box = box * 3 / 4;
     if (box < 8)
       box = 8;
   }
   g.sheet = calloc ((size_t) g.cols * g.tw * g.rows * g.th, 3);
   g.tile = malloc ((size_t) g.tw * g.th * 3);
   g.best = malloc ((size_t) g.tw * g.th * 3);
   if ( (NULL == g.sheet) ||
        (NULL == g.tile) ||
        (NULL == g.best) ||
        (NULL == (frame = av_frame_alloc ())) )
     goto cleanup;
   if (AV_NOPTS_VALUE != st->start_time)
     start_ts = st->start_time;
   else if (AV_NOPTS_VALUE != io->fmt->start_time)
     start_ts = av_rescale_q (io->fmt->start_time,
                              AV_TIME_BASE_Q,
                              st->time_base);
   else
     start_ts = 0;
   slice = duration_ms / n;
   for (i = 0; i < n; i++)
   {
     uint64_t base = EXTRACTOR_media_gallery_time_ (i,
                                                    n,
                                                    duration_ms);
     uint64_t tried[sizeof (shifts) / sizeof (shifts[0])];
     double best_score = -1.0;
     unsigned int ntried = 0;
     unsigned int a;
     int placed = 0;
 
     for (a = 0; a < sizeof (shifts) / sizeof (shifts[0]); a++)
     {
       int64_t off = (int64_t) (slice / 8) * shifts[a];
       uint64_t t;
       unsigned int k;
       double score;
 
       if (EXTRACTOR_media_deadline_expired_ (&io->deadline))
         break;
       if ( (off < 0) &&
            ((uint64_t) -off > base) )
         t = 0;
       else
         t = base + off;
       if (t >= duration_ms)
         t = duration_ms - 1;
       for (k = 0; k < ntried; k++)
         if (tried[k] == t)
           break;
       if (k < ntried)
         continue;
       tried[ntried++] = t;
       *touched = 1;
       if (0 != grab_tile (io,
                           ctx,
                           stream,
                           start_ts,
                           t,
                           frame,
                           &g))
         continue;
       if (! EXTRACTOR_media_frame_is_flat_ (g.tile,
                                             g.tw,
                                             g.th,
                                             (size_t) g.tw * 3))
       {
         place_tile (&g, i, g.tile);
         placed = 1;
         break;
       }
       score = tile_score (g.tile,
                           g.tw,
                           g.th);
       if (score > best_score)
       {
         best_score = score;
         memcpy (g.best,
                 g.tile,
                 (size_t) g.tw * g.th * 3);
       }
     }
     if ( (! placed) &&
          (best_score >= 0.0) )
     {
       place_tile (&g, i, g.best);
       placed = 1;
     }
     if (placed)
       got++;
     if (EXTRACTOR_media_deadline_expired_ (&io->deadline))
       break;   /* emit what we have; the other cells stay black */
   }
   if (0 == got)
     goto cleanup;
   ret = 0;
   if (0 != encode_gallery (io->ec,
                            &g,
                            opt,
                            &image,
                            &image_size))
     goto cleanup;
   if (0 != io->ec->proc (io->ec->cls,
                          "thumbnailffmpeg",
                          EXTRACTOR_METATYPE_THUMBNAIL,
                          EXTRACTOR_METAFORMAT_BINARY,
                          opt->png ? "image/png" : "image/jpeg",
                          (const char *) image,
                          image_size))
     *stop = 1;
 cleanup:
   if (NULL != image)
     av_free (image);
   if (NULL != frame)
     av_frame_free (&frame);
   sws_freeContext (g.sws);
   free (g.sheet);
   free (g.tile);
   free (g.best);
   return ret;
 }
 
 
 /**
  * Main method for the ffmpeg-thumbnailer plugin.
  *
  * @param ec extraction context
  */
 void
 EXTRACTOR_thumbnailffmpeg_extract_method (struct EXTRACTOR_ExtractContext *ec);
 
 void
 EXTRACTOR_thumbnailffmpeg_extract_method (struct EXTRACTOR_ExtractContext *ec)
 {
   struct EXTRACTOR_MediaOptions opt;
   struct EXTRACTOR_FFmpegIO io;
   AVCodecContext *ctx = NULL;
   AVFrame *frame = NULL;
   uint8_t *thumbnail = NULL;
   size_t thumbnail_size = 0;
   double target;
   int stream;
 
   EXTRACTOR_media_options_parse_ (ec->config,
                                   &opt);
   EXTRACTOR_media_deadline_start_ (&io.deadline,
                                    opt.deadline);
   io.ec = ec;
   if (! EXTRACTOR_media_is_media_ (ec, 0))
     return;
   if (0 != EXTRACTOR_ffmpeg_open_ (&io))
     return;
   /* subtitle metadata does not depend on there being a usable video
      stream (think of a subtitles-only .mks), so do it first */
   if (0 != emit_subtitle_languages (&io))
     goto cleanup;
   if ('\0' != opt.srt_lang[0])
   {
     int touched;
 
     if (0 != dump_srt (&io,
                        &opt,
                        &touched))
       goto cleanup;
     if (touched &&
         (0 != EXTRACTOR_ffmpeg_reopen_ (&io)) )
       return;
   }
   stream = EXTRACTOR_ffmpeg_open_stream_ (&io,
                                           AVMEDIA_TYPE_VIDEO,
                                           &ctx);
   if (0 > stream)
     goto cleanup;
   if ( (ctx->width <= 0) ||
        (ctx->height <= 0) ||
        (ctx->width > EXTRACTOR_MEDIA_MAX_DIMENSION) ||
        (ctx->height > EXTRACTOR_MEDIA_MAX_DIMENSION) ||
        (((int64_t) ctx->width) * ctx->height > EXTRACTOR_MEDIA_MAX_PIXELS) )
     goto cleanup;
   if (0 != opt.gallery)
   {
     int stop;
     int touched;
 
     if (0 == make_gallery (&io,
                            ctx,
                            stream,
                            &opt,
                            &stop,
                            &touched))
       goto cleanup;
     /* no duration, or not a single frame: try for one thumbnail */
     if (touched)
     {
       /* gallery seeks may have left the demuxer and decoder anywhere */
       avcodec_free_context (&ctx);
       if (0 != EXTRACTOR_ffmpeg_reopen_ (&io))
         return;
       stream = EXTRACTOR_ffmpeg_open_stream_ (&io,
                                               AVMEDIA_TYPE_VIDEO,
                                               &ctx);
       if (0 > stream)
         goto cleanup;
     }
   }
   target = EXTRACTOR_ffmpeg_seek_ (&io,
                                    stream,
                                    ctx,
                                    opt.offset);
   if (NULL == (frame = av_frame_alloc ()))
     goto cleanup;
   if (0 != decode_frame_at (&io,
                             ctx,
                             stream,
                             target,
                             1,
                             0,
                             MAX_FRAMES,
                             frame))
     goto cleanup;
   if (0 != encode_thumbnail (frame,
                              &opt,
                              &thumbnail,
                              &thumbnail_size))
     goto cleanup;
   ec->proc (ec->cls,
             "thumbnailffmpeg",
             EXTRACTOR_METATYPE_THUMBNAIL,
             EXTRACTOR_METAFORMAT_BINARY,
             opt.png ? "image/png" : "image/jpeg",
             (const char *) thumbnail,
             thumbnail_size);
 cleanup:
   if (NULL != thumbnail)
     av_free (thumbnail);
   if (NULL != frame)
     av_frame_free (&frame);
   if (NULL != ctx)
     avcodec_free_context (&ctx);
   EXTRACTOR_ffmpeg_close_ (&io);
 }
 
 
 /**
  * Keep the diagnostics of the media framework out of the output of the
  * calling application; a decoder complaining about a broken file is not
  * something the application asked to see.
  *
  * ("force-kill" would be tempting here, to get a fresh process for every
  * file, but the host cannot know about that special without loading the
  * plugin in-process: it only finds out that the child is gone once it has
  * already handed it the next file, which is then silently lost.)
  *
  * @return special options for this plugin
  */
 const char *
 EXTRACTOR_thumbnailffmpeg_options (void);
 
 const char *
 EXTRACTOR_thumbnailffmpeg_options ()
 {
   return "close-stderr";
 }
 
 
 /**
  * Silence FFmpeg's logging.
  */
 void __attribute__ ((constructor))
 thumbnailffmpeg_init (void);
 
 void __attribute__ ((constructor))
 thumbnailffmpeg_init ()
 {
   EXTRACTOR_ffmpeg_silence_ ();
 }
 
 
 /* end of thumbnailffmpeg_extractor.c */