libextractor

previewopus_extractor.c (16031B)

---

 /*
      This file is part of libextractor.
      Copyright (C) 2013, 2026 Vidyut Samanta and Christian Grothoff
 
      libextractor is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
      by the Free Software Foundation; either version 3, or (at your
      option) any later version.
 
      libextractor is distributed in the hope that it will be useful, but
      WITHOUT ANY WARRANTY; without even the implied warranty of
      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
      General Public License for more details.
 
      You should have received a copy of the GNU General Public License
      along with libextractor; see the file COPYING.  If not, write to the
      Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
      Boston, MA 02110-1301, USA.
  */
 /**
  * @file plugins/previewopus_extractor.c
  * @author Bruno Cabral
  * @author Christian Grothoff
  * @brief this extractor produces a short audio preview: about 15 seconds
  *        taken from the middle of the track, re-encoded as Opus in an Ogg
  *        container, which is the audio equivalent of a thumbnail
  *
  * Uses FFmpeg for demuxing, decoding and resampling and libopus (through
  * FFmpeg) for encoding.  Requires FFmpeg 5.1 or newer; there is
  * deliberately no compatibility code for older versions, since that is
  * what got the previous incarnation of this plugin deleted in 2021.
  */
 #include "platform.h"
 #include "extractor.h"
 #include "mediautil.h"
 #include "mediaffmpeg.h"
 
 #include <libavutil/opt.h>
 #include <libswresample/swresample.h>
 
 /**
  * Sample rate we encode at.  Opus works internally at 48kHz, so anything
  * else would just mean resampling twice.
  */
 #define OUTPUT_SAMPLE_RATE 48000
 
 /**
  * Length of the fade-in and fade-out, in milliseconds.  Without this a
  * preview cut from the middle of a track starts with a click.
  */
 #define FADE_MS 250
 
 /**
  * Refuse to emit a preview shorter than this many milliseconds.
  */
 #define MIN_PREVIEW_MS 500
 
 /**
  * Give up after this many decoded packets.
  */
 #define MAX_PACKETS 100000
 
 
 /**
  * State of the Opus encoder and its Ogg muxer.
  */
 struct Encoder
 {
   /**
    * Muxer producing the Ogg stream.
    */
   AVFormatContext *oc;
 
   /**
    * The Opus encoder.
    */
   AVCodecContext *ctx;
 
   /**
    * Stream in @e oc.
    */
   AVStream *st;
 };
 
 
 /**
  * Decode the audio stream into 48kHz mono 16 bit PCM, starting at
  * @a target seconds and stopping after @a max_samples samples.
  *
  * @param io open I/O state
  * @param ctx open decoder for @a stream
  * @param stream index of the audio stream
  * @param target position we are aiming for, in seconds
  * @param max_samples size of @a pcm in samples
  * @param[out] pcm where to write the samples
  * @return number of samples written
  */
 static size_t
 decode_pcm (struct EXTRACTOR_FFmpegIO *io,
             AVCodecContext *ctx,
             int stream,
             double target,
             size_t max_samples,
             int16_t *pcm)
 {
   struct SwrContext *swr = NULL;
   AVChannelLayout mono = AV_CHANNEL_LAYOUT_MONO;
   AVPacket *pkt;
   AVFrame *frame;
   AVRational tb = io->fmt->streams[stream]->time_base;
   size_t have = 0;
   unsigned int packets = 0;
 
   if (NULL == (pkt = av_packet_alloc ()))
     return 0;
   if (NULL == (frame = av_frame_alloc ()))
   {
     av_packet_free (&pkt);
     return 0;
   }
   while ( (have < max_samples) &&
           (packets < MAX_PACKETS) )
   {
     int r;
 
     if (EXTRACTOR_media_deadline_expired_ (&io->deadline))
       break;
     packets++;
     r = av_read_frame (io->fmt, pkt);
     if (0 != r)
     {
       avcodec_send_packet (ctx, NULL);
     }
     else if (pkt->stream_index != stream)
     {
       av_packet_unref (pkt);
       continue;
     }
     else if (0 != avcodec_send_packet (ctx, pkt))
     {
       av_packet_unref (pkt);
       continue;
     }
     av_packet_unref (pkt);
     while (0 == avcodec_receive_frame (ctx, frame))
     {
       uint8_t *out[1];
       int room;
       int got;
       double ts;
 
       ts = (AV_NOPTS_VALUE == frame->best_effort_timestamp)
            ? -1.0
            : frame->best_effort_timestamp * av_q2d (tb);
       if ( (ts >= 0.0) &&
            (ts + frame->nb_samples / (double) ((frame->sample_rate > 0)
                                                 ? frame->sample_rate
                                                : OUTPUT_SAMPLE_RATE) < target) )
       {
         /* still before the segment we want */
         av_frame_unref (frame);
         continue;
       }
       if (NULL == swr)
       {
         if (0 != swr_alloc_set_opts2 (&swr,
                                       &mono,
                                       AV_SAMPLE_FMT_S16,
                                       OUTPUT_SAMPLE_RATE,
                                       &frame->ch_layout,
                                       frame->format,
                                       frame->sample_rate,
                                       0,
                                       NULL))
         {
           av_frame_unref (frame);
           goto done;
         }
         if (0 > swr_init (swr))
         {
           av_frame_unref (frame);
           goto done;
         }
       }
       room = (int) (max_samples - have);
       out[0] = (uint8_t *) &pcm[have];
       got = swr_convert (swr,
                          out,
                          room,
                          (const uint8_t **) frame->extended_data,
                          frame->nb_samples);
       av_frame_unref (frame);
       if (got <= 0)
         continue;
       have += (size_t) got;
       if (have >= max_samples)
         goto done;
     }
     if (0 != r)
       break;
   }
 done:
   if (NULL != swr)
     swr_free (&swr);
   av_frame_free (&frame);
   av_packet_free (&pkt);
   return have;
 }
 
 
 /**
  * Set up the Opus encoder and the Ogg muxer, writing into a dynamic
  * memory buffer.
  *
  * @param opt plugin options (bitrate)
  * @param[out] enc encoder state to initialize
  * @return 0 on success
  */
 static int
 encoder_start (const struct EXTRACTOR_MediaOptions *opt,
                struct Encoder *enc)
 {
   const AVCodec *codec;
   AVChannelLayout mono = AV_CHANNEL_LAYOUT_MONO;
   int experimental = 0;
 
   memset (enc, 0, sizeof (*enc));
   codec = avcodec_find_encoder_by_name ("libopus");
   if (NULL == codec)
   {
     /* FFmpeg's own Opus encoder is still marked experimental, but it is
        better than emitting nothing at all */
     codec = avcodec_find_encoder (AV_CODEC_ID_OPUS);
     experimental = 1;
   }
   if (NULL == codec)
     return -1;
   if (0 > avformat_alloc_output_context2 (&enc->oc,
                                           NULL,
                                           "ogg",
                                           NULL))
     return -1;
   if (NULL == enc->oc)
     return -1;
   if (NULL == (enc->ctx = avcodec_alloc_context3 (codec)))
     return -1;
   enc->ctx->sample_rate = OUTPUT_SAMPLE_RATE;
   enc->ctx->bit_rate = opt->bitrate;
   enc->ctx->time_base = (AVRational) { 1, OUTPUT_SAMPLE_RATE };
   enc->ctx->thread_count = 1;
   if (0 != av_channel_layout_copy (&enc->ctx->ch_layout,
                                    &mono))
     return -1;
   if (experimental)
   {
     enc->ctx->sample_fmt = AV_SAMPLE_FMT_FLTP;
     enc->ctx->strict_std_compliance = FF_COMPLIANCE_EXPERIMENTAL;
   }
   else
   {
     enc->ctx->sample_fmt = AV_SAMPLE_FMT_S16;
   }
   /* the Ogg muxer needs the OpusHead/OpusTags headers as extradata */
   enc->ctx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
   /* libopus defaults to unconstrained VBR, where the bitrate we asked for
      is little more than a suggestion -- a pure tone happily comes out at
      twice the target.  We care about the size of the result, so ask for
      the constrained variant.  Not an option on the native encoder, which
      simply ignores it. */
   if (NULL != enc->ctx->priv_data)
     (void) av_opt_set (enc->ctx->priv_data,
                        "vbr",
                        "constrained",
                        0);
   if (0 != avcodec_open2 (enc->ctx, codec, NULL))
     return -1;
   if (NULL == (enc->st = avformat_new_stream (enc->oc, NULL)))
     return -1;
   enc->st->time_base = enc->ctx->time_base;
   if (0 > avcodec_parameters_from_context (enc->st->codecpar,
                                            enc->ctx))
     return -1;
   if (0 != avio_open_dyn_buf (&enc->oc->pb))
     return -1;
   if (0 > avformat_write_header (enc->oc, NULL))
     return -1;
   return 0;
 }
 
 
 /**
  * Hand one packet from the encoder to the muxer.
  *
  * @param enc encoder state
  * @param pkt packet to write, unreferenced on return
  * @return 0 on success
  */
 static int
 encoder_write (struct Encoder *enc,
                AVPacket *pkt)
 {
   av_packet_rescale_ts (pkt,
                         enc->ctx->time_base,
                         enc->st->time_base);
   pkt->stream_index = enc->st->index;
   return (0 > av_interleaved_write_frame (enc->oc, pkt)) ? -1 : 0;
 }
 
 
 /**
  * Encode @a num_samples of 48kHz mono PCM and finish the Ogg stream.
  *
  * @param enc encoder state
  * @param pcm samples to encode
  * @param num_samples number of samples in @a pcm
  * @param[out] output set to the Ogg stream, to be freed with av_free()
  * @param[out] output_size set to the number of bytes in @a output
  * @return 0 on success
  */
 static int
 encoder_finish (struct Encoder *enc,
                 const int16_t *pcm,
                 size_t num_samples,
                 uint8_t **output,
                 size_t *output_size)
 {
   AVFrame *frame = NULL;
   AVPacket *pkt = NULL;
   size_t pos = 0;
   int frame_size;
   int ret = -1;
   int size;
 
   frame_size = (enc->ctx->frame_size > 0)
                ? enc->ctx->frame_size
                : OUTPUT_SAMPLE_RATE / 50;
   if (NULL == (pkt = av_packet_alloc ()))
     goto cleanup;
   if (NULL == (frame = av_frame_alloc ()))
     goto cleanup;
   frame->format = enc->ctx->sample_fmt;
   frame->sample_rate = OUTPUT_SAMPLE_RATE;
   frame->nb_samples = frame_size;
   if (0 != av_channel_layout_copy (&frame->ch_layout,
                                    &enc->ctx->ch_layout))
     goto cleanup;
   if (0 != av_frame_get_buffer (frame, 0))
     goto cleanup;
   while (pos < num_samples)
   {
     size_t n = num_samples - pos;
     int r;
 
     if (n > (size_t) frame_size)
       n = frame_size;
     if (0 != av_frame_make_writable (frame))
       goto cleanup;
     if (AV_SAMPLE_FMT_S16 == enc->ctx->sample_fmt)
     {
       memcpy (frame->data[0],
               &pcm[pos],
               n * sizeof (int16_t));
       if (n < (size_t) frame_size)
         memset (frame->data[0] + n * sizeof (int16_t),
                 0,
                 (frame_size - n) * sizeof (int16_t));
     }
     else
     {
       float *dst = (float *) frame->data[0];
       size_t i;
 
       for (i = 0; i < n; i++)
         dst[i] = pcm[pos + i] / 32768.0f;
       for (i = n; i < (size_t) frame_size; i++)
         dst[i] = 0.0f;
     }
     frame->pts = (int64_t) pos;
     if (0 != avcodec_send_frame (enc->ctx, frame))
       goto cleanup;
     while (0 == (r = avcodec_receive_packet (enc->ctx, pkt)))
       if (0 != encoder_write (enc, pkt))
         goto cleanup;
     if (AVERROR (EAGAIN) != r)
       goto cleanup;
     pos += n;
   }
   if (0 != avcodec_send_frame (enc->ctx, NULL))
     goto cleanup;
   while (0 == avcodec_receive_packet (enc->ctx, pkt))
     if (0 != encoder_write (enc, pkt))
       goto cleanup;
   if (0 > av_write_trailer (enc->oc))
     goto cleanup;
   size = avio_close_dyn_buf (enc->oc->pb,
                              output);
   enc->oc->pb = NULL;
   if (size <= 0)
   {
     if (NULL != *output)
       av_free (*output);
     *output = NULL;
     goto cleanup;
   }
   *output_size = (size_t) size;
   ret = 0;
 cleanup:
   if (NULL != frame)
     av_frame_free (&frame);
   if (NULL != pkt)
     av_packet_free (&pkt);
   return ret;
 }
 
 
 /**
  * Clean up the encoder state.
  *
  * @param enc encoder state to free
  */
 static void
 encoder_cleanup (struct Encoder *enc)
 {
   if (NULL != enc->ctx)
     avcodec_free_context (&enc->ctx);
   if (NULL != enc->oc)
   {
     if (NULL != enc->oc->pb)
     {
       uint8_t *leftover = NULL;
 
       (void) avio_close_dyn_buf (enc->oc->pb,
                                  &leftover);
       enc->oc->pb = NULL;
       if (NULL != leftover)
         av_free (leftover);
     }
     avformat_free_context (enc->oc);
     enc->oc = NULL;
   }
 }
 
 
 /**
  * Main method for the opus-preview plugin.
  *
  * @param ec extraction context
  */
 void
 EXTRACTOR_previewopus_extract_method (struct EXTRACTOR_ExtractContext *ec);
 
 void
 EXTRACTOR_previewopus_extract_method (struct EXTRACTOR_ExtractContext *ec)
 {
   struct EXTRACTOR_MediaOptions opt;
   struct EXTRACTOR_FFmpegIO io;
   struct Encoder enc;
   AVCodecContext *ctx = NULL;
   int16_t *pcm = NULL;
   uint8_t *preview = NULL;
   size_t preview_size = 0;
   size_t max_samples;
   size_t have;
   size_t limit;
   double target;
   int stream;
 
   EXTRACTOR_media_options_parse_ (ec->config,
                                   &opt);
   EXTRACTOR_media_deadline_start_ (&io.deadline,
                                    opt.deadline);
   io.ec = ec;
   memset (&enc, 0, sizeof (enc));
   if (! EXTRACTOR_media_is_media_ (ec, 1))
     return;
   if (0 != EXTRACTOR_ffmpeg_open_ (&io))
     return;
   stream = EXTRACTOR_ffmpeg_open_stream_ (&io,
                                           AVMEDIA_TYPE_AUDIO,
                                           &ctx);
   if (0 > stream)
     goto cleanup;
   target = EXTRACTOR_ffmpeg_seek_ (&io,
                                    stream,
                                    ctx,
                                    opt.offset);
   max_samples = (size_t) opt.length * OUTPUT_SAMPLE_RATE;
   if (NULL == (pcm = malloc (max_samples * sizeof (int16_t))))
     goto cleanup;
   have = decode_pcm (&io,
                      ctx,
                      stream,
                      target,
                      max_samples,
                      pcm);
   if (have < (size_t) MIN_PREVIEW_MS * OUTPUT_SAMPLE_RATE / 1000)
     goto cleanup;   /* too short to be worth anything */
   EXTRACTOR_media_fade_s16_ (pcm,
                              have,
                              FADE_MS * OUTPUT_SAMPLE_RATE / 1000);
   if (0 != encoder_start (&opt,
                           &enc))
     goto cleanup;
   if (0 != encoder_finish (&enc,
                            pcm,
                            have,
                            &preview,
                            &preview_size))
     goto cleanup;
   /* the caller asked for a preview of a given length and bitrate, so let
      that decide the limit, but never emit something absurd */
   limit = (size_t) opt.length * opt.bitrate / 8 + 8192;
   if (limit < EXTRACTOR_MEDIA_MAX_PREVIEW)
     limit = EXTRACTOR_MEDIA_MAX_PREVIEW;
   if (preview_size > limit)
     goto cleanup;
   ec->proc (ec->cls,
             "previewopus",
             EXTRACTOR_METATYPE_AUDIO_PREVIEW,
             EXTRACTOR_METAFORMAT_BINARY,
             "audio/ogg",
             (const char *) preview,
             preview_size);
 cleanup:
   if (NULL != preview)
     av_free (preview);
   encoder_cleanup (&enc);
   free (pcm);
   if (NULL != ctx)
     avcodec_free_context (&ctx);
   EXTRACTOR_ffmpeg_close_ (&io);
 }
 
 
 /**
  * Keep the diagnostics of the media framework out of the output of the
  * calling application; a decoder complaining about a broken file is not
  * something the application asked to see.
  *
  * ("force-kill" would be tempting here, to get a fresh process for every
  * file, but the host cannot know about that special without loading the
  * plugin in-process: it only finds out that the child is gone once it has
  * already handed it the next file, which is then silently lost.)
  *
  * @return special options for this plugin
  */
 const char *
 EXTRACTOR_previewopus_options (void);
 
 const char *
 EXTRACTOR_previewopus_options ()
 {
   return "close-stderr";
 }
 
 
 /**
  * Silence FFmpeg's logging.
  */
 void __attribute__ ((constructor))
 previewopus_init (void);
 
 void __attribute__ ((constructor))
 previewopus_init ()
 {
   EXTRACTOR_ffmpeg_silence_ ();
 }
 
 
 /* end of previewopus_extractor.c */