libextractor

thumbnailffmpeg_extractor.c (13049B)

---

 /*
      This file is part of libextractor.
      Copyright (C) 2008, 2012, 2026 Vidyut Samanta and Christian Grothoff
 
      libextractor is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
      by the Free Software Foundation; either version 3, or (at your
      option) any later version.
 
      libextractor is distributed in the hope that it will be useful, but
      WITHOUT ANY WARRANTY; without even the implied warranty of
      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
      General Public License for more details.
 
      You should have received a copy of the GNU General Public License
      along with libextractor; see the file COPYING.  If not, write to the
      Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
      Boston, MA 02110-1301, USA.
  */
 /**
  * @file plugins/thumbnailffmpeg_extractor.c
  * @author Christian Grothoff
  * @brief this extractor produces a binary encoded thumbnail of a frame
  *        taken from a video, using FFmpeg for demuxing, decoding and
  *        scaling
  *
  * Unlike the gtk thumbnailer, which can only re-scale a still image, this
  * plugin seeks about 30s into a video and encodes the frame it finds
  * there as a small JPEG (or PNG).  A previous incarnation of this plugin
  * was removed in 2021 because it had accumulated compatibility code for
  * every libav and FFmpeg API since 2008; this one deliberately requires
  * FFmpeg 5.1 or newer and uses only the send/receive APIs, so that there
  * is nothing to keep in sync with upstream churn.
  */
 #include "platform.h"
 #include "extractor.h"
 #include "mediautil.h"
 #include "mediaffmpeg.h"
 
 #include <libavutil/imgutils.h>
 #include <libswscale/swscale.h>
 
 /**
  * Give up after this many decoded frames, no matter what the timestamps
  * say.  A malformed file can otherwise keep us decoding forever.
  */
 #define MAX_FRAMES 2048
 
 /**
  * Mean absolute deviation below which we consider a frame to be a flat
  * surface (a fade-in, a black leader) and not worth showing.
  */
 #define FLAT_FRAME_MAD 2
 
 /**
  * How far to jump ahead when we hit a flat frame, in seconds.
  */
 #define FLAT_FRAME_SKIP 5
 
 /**
  * How often we are willing to do that.
  */
 #define MAX_FLAT_RETRIES 2
 
 /**
  * JPEG quantizer scale; smaller is better and bigger.
  */
 #define JPEG_QSCALE 5
 
 
 /**
  * Is @a frame a flat surface?  Cheap heuristic over the first plane; we
  * only want to catch the completely uniform black frame of a fade-in.
  *
  * @param frame frame to inspect
  * @return 1 if the frame carries no discernible detail
  */
 static int
 is_flat (const AVFrame *frame)
 {
   const uint8_t *p = frame->data[0];
   int stride = frame->linesize[0];
   uint64_t sum = 0;
   uint64_t dev = 0;
   unsigned int n = 0;
   int x;
   int y;
 
   if ( (NULL == p) ||
        (stride <= 0) ||
        (frame->height <= 0) )
     return 0;
   for (y = 0; y < frame->height; y += 4)
     for (x = 0; x < stride; x += 4)
     {
       sum += p[y * stride + x];
       n++;
     }
   if (0 == n)
     return 0;
   sum /= n;
   for (y = 0; y < frame->height; y += 4)
     for (x = 0; x < stride; x += 4)
     {
       uint8_t v = p[y * stride + x];
 
       dev += (v > sum) ? (v - sum) : (sum - v);
     }
   return (dev / n) < FLAT_FRAME_MAD;
 }
 
 
 /**
  * Scale @a frame down and encode it as JPEG or PNG.
  *
  * @param frame the decoded frame to convert
  * @param opt plugin options (target size and format)
  * @param[out] output set to the encoded image, to be freed with av_free()
  * @param[out] output_size set to the number of bytes in @a output
  * @return 0 on success
  */
 static int
 encode_thumbnail (const AVFrame *frame,
                   const struct EXTRACTOR_MediaOptions *opt,
                   uint8_t **output,
                   size_t *output_size)
 {
   const AVCodec *codec;
   AVCodecContext *ctx = NULL;
   struct SwsContext *sws = NULL;
   AVFrame *dst = NULL;
   AVPacket *pkt = NULL;
   enum AVPixelFormat dst_fmt;
   unsigned int width;
   unsigned int height;
   int ret = -1;
   int r;
 
   if (! EXTRACTOR_media_thumbnail_size_ (frame->width,
                                          frame->height,
                                          opt->size,
                                          &width,
                                          &height))
     return -1;
   if (! sws_isSupportedInput (frame->format))
     return -1;
   if (opt->png)
   {
     dst_fmt = AV_PIX_FMT_RGB24;
     codec = avcodec_find_encoder (AV_CODEC_ID_PNG);
   }
   else
   {
     dst_fmt = AV_PIX_FMT_YUVJ420P;
     codec = avcodec_find_encoder (AV_CODEC_ID_MJPEG);
   }
   if (NULL == codec)
     return -1;
   if (NULL == (sws = sws_getContext (frame->width,
                                      frame->height,
                                      frame->format,
                                      (int) width,
                                      (int) height,
                                      dst_fmt,
                                      SWS_BILINEAR,
                                      NULL,
                                      NULL,
                                      NULL)))
     return -1;
   if (NULL == (dst = av_frame_alloc ()))
     goto cleanup;
   dst->format = dst_fmt;
   dst->width = (int) width;
   dst->height = (int) height;
   dst->pts = 0;
   if (0 != av_frame_get_buffer (dst, 0))
     goto cleanup;
   if (0 > sws_scale (sws,
                      (const uint8_t *const *) frame->data,
                      frame->linesize,
                      0,
                      frame->height,
                      dst->data,
                      dst->linesize))
     goto cleanup;
   if (NULL == (ctx = avcodec_alloc_context3 (codec)))
     goto cleanup;
   ctx->width = (int) width;
   ctx->height = (int) height;
   ctx->pix_fmt = dst_fmt;
   ctx->time_base = (AVRational) { 1, 25 };
   ctx->thread_count = 1;
   if (opt->png)
   {
     ctx->compression_level = 9;
   }
   else
   {
     ctx->color_range = AVCOL_RANGE_JPEG;
     ctx->flags |= AV_CODEC_FLAG_QSCALE;
     ctx->global_quality = FF_QP2LAMBDA * JPEG_QSCALE;
   }
   if (0 != avcodec_open2 (ctx, codec, NULL))
     goto cleanup;
   if (NULL == (pkt = av_packet_alloc ()))
     goto cleanup;
   dst->quality = ctx->global_quality;
   if (0 != avcodec_send_frame (ctx, dst))
     goto cleanup;
   r = avcodec_receive_packet (ctx, pkt);
   if (AVERROR (EAGAIN) == r)
   {
     /* encoder wants to be flushed before it hands anything back */
     if (0 != avcodec_send_frame (ctx, NULL))
       goto cleanup;
     r = avcodec_receive_packet (ctx, pkt);
   }
   if (0 != r)
     goto cleanup;
   if ( (pkt->size <= 0) ||
        (pkt->size > EXTRACTOR_MEDIA_MAX_THUMBNAIL) )
     goto cleanup;
   if (NULL == (*output = av_malloc (pkt->size)))
     goto cleanup;
   memcpy (*output,
           pkt->data,
           pkt->size);
   *output_size = (size_t) pkt->size;
   ret = 0;
 cleanup:
   if (NULL != pkt)
     av_packet_free (&pkt);
   if (NULL != ctx)
     avcodec_free_context (&ctx);
   if (NULL != dst)
     av_frame_free (&dst);
   sws_freeContext (sws);
   return ret;
 }
 
 
 /**
  * Decode the video stream until we reach @a target seconds, and hand back
  * the frame we find there.  If we never get there (short file, broken
  * timestamps), the last frame we did decode is returned instead.
  *
  * @param io open I/O state
  * @param ctx open decoder for @a stream
  * @param stream index of the video stream
  * @param target position we are aiming for, in seconds
  * @param[out] frame frame to fill in
  * @return 0 if @a frame was filled in
  */
 static int
 decode_frame_at (struct EXTRACTOR_FFmpegIO *io,
                  AVCodecContext *ctx,
                  int stream,
                  double target,
                  AVFrame *frame)
 {
   AVPacket *pkt;
   AVFrame *last;
   AVRational tb = io->fmt->streams[stream]->time_base;
   unsigned int frames = 0;
   unsigned int retries = 0;
   int have_last = 0;
   int ret = -1;
 
   if (NULL == (pkt = av_packet_alloc ()))
     return -1;
   if (NULL == (last = av_frame_alloc ()))
   {
     av_packet_free (&pkt);
     return -1;
   }
   while (frames < MAX_FRAMES)
   {
     int r;
 
     if (EXTRACTOR_media_deadline_expired_ (&io->deadline))
       break;
     r = av_read_frame (io->fmt, pkt);
     if (0 != r)
     {
       /* end of file: flush the decoder and take what is left */
       avcodec_send_packet (ctx, NULL);
     }
     else if (pkt->stream_index != stream)
     {
       av_packet_unref (pkt);
       continue;
     }
     else if (0 != avcodec_send_packet (ctx, pkt))
     {
       av_packet_unref (pkt);
       continue;
     }
     av_packet_unref (pkt);
     while (0 == avcodec_receive_frame (ctx, frame))
     {
       double ts;
 
       frames++;
       ts = (AV_NOPTS_VALUE == frame->best_effort_timestamp)
            ? -1.0
            : frame->best_effort_timestamp * av_q2d (tb);
       if ( (ts < target) &&
            (frames < MAX_FRAMES) )
       {
         /* not there yet; hold on to it in case we never arrive */
         av_frame_unref (last);
         if (0 == av_frame_ref (last, frame))
           have_last = 1;
         av_frame_unref (frame);
         continue;
       }
       if ( (retries < MAX_FLAT_RETRIES) &&
            is_flat (frame) )
       {
         /* a black leader or a fade-in; try a bit further in */
         retries++;
         target = ((ts < 0.0) ? target : ts) + FLAT_FRAME_SKIP;
         av_frame_unref (last);
         if (0 == av_frame_ref (last, frame))
           have_last = 1;
         av_frame_unref (frame);
         continue;
       }
       av_frame_unref (last);
       av_frame_free (&last);
       av_packet_free (&pkt);
       return 0;
     }
     if (0 != r)
       break;   /* end of file, and the decoder had nothing left either */
   }
   if (have_last)
   {
     av_frame_unref (frame);
     if (0 == av_frame_ref (frame, last))
       ret = 0;
   }
   av_frame_unref (last);
   av_frame_free (&last);
   av_packet_free (&pkt);
   return ret;
 }
 
 
 /**
  * Main method for the ffmpeg-thumbnailer plugin.
  *
  * @param ec extraction context
  */
 void
 EXTRACTOR_thumbnailffmpeg_extract_method (struct EXTRACTOR_ExtractContext *ec);
 
 void
 EXTRACTOR_thumbnailffmpeg_extract_method (struct EXTRACTOR_ExtractContext *ec)
 {
   struct EXTRACTOR_MediaOptions opt;
   struct EXTRACTOR_FFmpegIO io;
   AVCodecContext *ctx = NULL;
   AVFrame *frame = NULL;
   uint8_t *thumbnail = NULL;
   size_t thumbnail_size = 0;
   double target;
   int stream;
 
   EXTRACTOR_media_options_parse_ (ec->config,
                                   &opt);
   EXTRACTOR_media_deadline_start_ (&io.deadline,
                                    opt.deadline);
   io.ec = ec;
   if (! EXTRACTOR_media_is_media_ (ec, 0))
     return;
   if (0 != EXTRACTOR_ffmpeg_open_ (&io))
     return;
   stream = EXTRACTOR_ffmpeg_open_stream_ (&io,
                                           AVMEDIA_TYPE_VIDEO,
                                           &ctx);
   if (0 > stream)
     goto cleanup;
   if ( (ctx->width <= 0) ||
        (ctx->height <= 0) ||
        (ctx->width > EXTRACTOR_MEDIA_MAX_DIMENSION) ||
        (ctx->height > EXTRACTOR_MEDIA_MAX_DIMENSION) ||
        (((int64_t) ctx->width) * ctx->height > EXTRACTOR_MEDIA_MAX_PIXELS) )
     goto cleanup;
   target = EXTRACTOR_ffmpeg_seek_ (&io,
                                    stream,
                                    ctx,
                                    opt.offset);
   if (NULL == (frame = av_frame_alloc ()))
     goto cleanup;
   if (0 != decode_frame_at (&io,
                             ctx,
                             stream,
                             target,
                             frame))
     goto cleanup;
   if (0 != encode_thumbnail (frame,
                              &opt,
                              &thumbnail,
                              &thumbnail_size))
     goto cleanup;
   ec->proc (ec->cls,
             "thumbnailffmpeg",
             EXTRACTOR_METATYPE_THUMBNAIL,
             EXTRACTOR_METAFORMAT_BINARY,
             opt.png ? "image/png" : "image/jpeg",
             (const char *) thumbnail,
             thumbnail_size);
 cleanup:
   if (NULL != thumbnail)
     av_free (thumbnail);
   if (NULL != frame)
     av_frame_free (&frame);
   if (NULL != ctx)
     avcodec_free_context (&ctx);
   EXTRACTOR_ffmpeg_close_ (&io);
 }
 
 
 /**
  * Keep the diagnostics of the media framework out of the output of the
  * calling application; a decoder complaining about a broken file is not
  * something the application asked to see.
  *
  * ("force-kill" would be tempting here, to get a fresh process for every
  * file, but the host cannot know about that special without loading the
  * plugin in-process: it only finds out that the child is gone once it has
  * already handed it the next file, which is then silently lost.)
  *
  * @return special options for this plugin
  */
 const char *
 EXTRACTOR_thumbnailffmpeg_options (void);
 
 const char *
 EXTRACTOR_thumbnailffmpeg_options ()
 {
   return "close-stderr";
 }
 
 
 /**
  * Silence FFmpeg's logging.
  */
 void __attribute__ ((constructor))
 thumbnailffmpeg_init (void);
 
 void __attribute__ ((constructor))
 thumbnailffmpeg_init ()
 {
   EXTRACTOR_ffmpeg_silence_ ();
 }
 
 
 /* end of thumbnailffmpeg_extractor.c */