previewgst_extractor.c (12909B)
1 /* 2 This file is part of libextractor. 3 Copyright (C) 2026 Vidyut Samanta and Christian Grothoff 4 5 libextractor is free software; you can redistribute it and/or modify 6 it under the terms of the GNU General Public License as published 7 by the Free Software Foundation; either version 3, or (at your 8 option) any later version. 9 10 libextractor is distributed in the hope that it will be useful, but 11 WITHOUT ANY WARRANTY; without even the implied warranty of 12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 General Public License for more details. 14 15 You should have received a copy of the GNU General Public License 16 along with libextractor; see the file COPYING. If not, write to the 17 Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, 18 Boston, MA 02110-1301, USA. 19 */ 20 /** 21 * @file plugins/previewgst_extractor.c 22 * @author Christian Grothoff 23 * @brief this extractor produces a short audio preview: about 15 seconds 24 * taken from the middle of the track, re-encoded as Opus in an Ogg 25 * container, which is the audio equivalent of a thumbnail 26 * 27 * This is the GStreamer counterpart of previewopus. The entire job is 28 * done by the pipeline 29 * 30 * playbin(appsrc://) ! audioconvert ! volume ! audioresample 31 * ! opusenc ! oggmux ! appsink 32 * 33 * where the segment boundaries come from a seek with an explicit stop 34 * position, so that the pipeline reaches EOS -- and thus finishes the Ogg 35 * stream -- exactly where we want the preview to end. The volume element 36 * carries a control source that fades the preview in and out; without it 37 * a segment cut from the middle of a track begins with a click. 38 */ 39 #include "platform.h" 40 #include "extractor.h" 41 #include "mediautil.h" 42 #include "mediagst.h" 43 44 #include <gst/app/gstappsink.h> 45 #include <gst/controller/gstinterpolationcontrolsource.h> 46 #include <gst/controller/gstdirectcontrolbinding.h> 47 48 /** 49 * Sample rate we encode at. Opus works internally at 48kHz, so anything 50 * else would just mean resampling twice. 51 */ 52 #define OUTPUT_SAMPLE_RATE 48000 53 54 /** 55 * Length of the fade-in and fade-out, in nanoseconds. 56 */ 57 #define FADE_NS (250 * GST_MSECOND) 58 59 /** 60 * Refuse to emit a preview shorter than this. 61 */ 62 #define MIN_PREVIEW_NS (500 * GST_MSECOND) 63 64 65 /** 66 * Build the audio sink bin that encodes what playbin decodes. 67 * 68 * @param opt plugin options (bitrate) 69 * @param[out] appsink set to the sink at the end of the bin 70 * @param[out] volume set to the element carrying the fade 71 * @return the bin, or NULL if an element is missing 72 */ 73 static GstElement * 74 build_encoder_bin (const struct EXTRACTOR_MediaOptions *opt, 75 GstElement **appsink, 76 GstElement **volume) 77 { 78 GstElement *bin; 79 GstElement *convert; 80 GstElement *vol; 81 GstElement *resample; 82 GstElement *filter; 83 GstElement *encoder; 84 GstElement *muxer; 85 GstElement *sink; 86 GstCaps *caps; 87 GstPad *pad; 88 89 convert = gst_element_factory_make ("audioconvert", NULL); 90 vol = gst_element_factory_make ("volume", NULL); 91 resample = gst_element_factory_make ("audioresample", NULL); 92 filter = gst_element_factory_make ("capsfilter", NULL); 93 encoder = gst_element_factory_make ("opusenc", NULL); 94 muxer = gst_element_factory_make ("oggmux", NULL); 95 sink = gst_element_factory_make ("appsink", NULL); 96 if ( (NULL == convert) || 97 (NULL == vol) || 98 (NULL == resample) || 99 (NULL == filter) || 100 (NULL == encoder) || 101 (NULL == muxer) || 102 (NULL == sink) ) 103 { 104 /* gst-plugins-base is not installed, or not completely */ 105 if (NULL != convert) 106 gst_object_unref (convert); 107 if (NULL != vol) 108 gst_object_unref (vol); 109 if (NULL != resample) 110 gst_object_unref (resample); 111 if (NULL != filter) 112 gst_object_unref (filter); 113 if (NULL != encoder) 114 gst_object_unref (encoder); 115 if (NULL != muxer) 116 gst_object_unref (muxer); 117 if (NULL != sink) 118 gst_object_unref (sink); 119 return NULL; 120 } 121 caps = gst_caps_new_simple ("audio/x-raw", 122 "rate", G_TYPE_INT, OUTPUT_SAMPLE_RATE, 123 "channels", G_TYPE_INT, 1, 124 NULL); 125 g_object_set (filter, 126 "caps", caps, 127 NULL); 128 gst_caps_unref (caps); 129 g_object_set (encoder, 130 "bitrate", (gint) opt->bitrate, 131 NULL); 132 g_object_set (sink, 133 "sync", FALSE, 134 "max-buffers", 0, 135 NULL); 136 bin = gst_bin_new (NULL); 137 gst_bin_add_many (GST_BIN (bin), 138 convert, 139 vol, 140 resample, 141 filter, 142 encoder, 143 muxer, 144 sink, 145 NULL); 146 if (! gst_element_link_many (convert, 147 vol, 148 resample, 149 filter, 150 encoder, 151 muxer, 152 sink, 153 NULL)) 154 { 155 gst_object_unref (bin); 156 return NULL; 157 } 158 if (NULL == (pad = gst_element_get_static_pad (convert, "sink"))) 159 { 160 gst_object_unref (bin); 161 return NULL; 162 } 163 if (! gst_element_add_pad (bin, 164 gst_ghost_pad_new ("sink", pad))) 165 { 166 gst_object_unref (pad); 167 gst_object_unref (bin); 168 return NULL; 169 } 170 gst_object_unref (pad); 171 *appsink = sink; 172 *volume = vol; 173 return bin; 174 } 175 176 177 /** 178 * Make @a volume ramp up at the start of the segment and down at its end. 179 * 180 * @param volume the volume element 181 * @param length length of the segment, in nanoseconds 182 */ 183 static void 184 setup_fade (GstElement *volume, 185 gint64 length) 186 { 187 GstControlSource *cs; 188 GstTimedValueControlSource *tv; 189 gint64 fade = FADE_NS; 190 191 if (length <= 0) 192 return; 193 if (fade * 2 > length) 194 fade = length / 2; 195 if (0 == fade) 196 return; 197 if (NULL == (cs = gst_interpolation_control_source_new ())) 198 return; 199 g_object_set (cs, 200 "mode", GST_INTERPOLATION_MODE_LINEAR, 201 NULL); 202 if (! gst_object_add_control_binding ( 203 GST_OBJECT (volume), 204 gst_direct_control_binding_new_absolute (GST_OBJECT (volume), 205 "volume", 206 cs))) 207 { 208 /* not fatal: the preview is merely less pleasant to listen to */ 209 gst_object_unref (cs); 210 return; 211 } 212 tv = GST_TIMED_VALUE_CONTROL_SOURCE (cs); 213 gst_timed_value_control_source_set (tv, 0, 0.0); 214 gst_timed_value_control_source_set (tv, fade, 1.0); 215 gst_timed_value_control_source_set (tv, length - fade, 1.0); 216 gst_timed_value_control_source_set (tv, length, 0.0); 217 gst_object_unref (cs); 218 } 219 220 221 /** 222 * Collect everything the appsink hands us until it reaches EOS. 223 * 224 * @param sink the appsink at the end of our pipeline 225 * @param deadline when to give up 226 * @param limit maximum number of bytes to collect 227 * @return the Ogg stream, or NULL; free with g_byte_array_unref() 228 */ 229 static GByteArray * 230 collect_output (GstElement *sink, 231 const struct EXTRACTOR_MediaDeadline *deadline, 232 size_t limit) 233 { 234 GByteArray *out; 235 236 out = g_byte_array_new (); 237 while (1) 238 { 239 GstSample *sample; 240 GstBuffer *buffer; 241 GstMapInfo mi; 242 uint64_t remaining; 243 244 remaining = EXTRACTOR_media_deadline_remaining_ (deadline); 245 if (0 == remaining) 246 break; 247 sample = gst_app_sink_try_pull_sample (GST_APP_SINK (sink), 248 remaining * GST_MSECOND); 249 if (NULL == sample) 250 break; /* end of stream, or we ran out of time */ 251 buffer = gst_sample_get_buffer (sample); 252 if ( (NULL != buffer) && 253 (gst_buffer_map (buffer, &mi, GST_MAP_READ)) ) 254 { 255 if (out->len + mi.size <= limit) 256 g_byte_array_append (out, 257 mi.data, 258 mi.size); 259 else 260 { 261 /* the stream would be truncated, and a truncated Ogg file is 262 worse than none at all */ 263 gst_buffer_unmap (buffer, &mi); 264 gst_sample_unref (sample); 265 g_byte_array_unref (out); 266 return NULL; 267 } 268 gst_buffer_unmap (buffer, &mi); 269 } 270 gst_sample_unref (sample); 271 } 272 return out; 273 } 274 275 276 /** 277 * Main method for the gstreamer audio preview plugin. 278 * 279 * @param ec extraction context 280 */ 281 void 282 EXTRACTOR_previewgst_extract_method (struct EXTRACTOR_ExtractContext *ec); 283 284 void 285 EXTRACTOR_previewgst_extract_method (struct EXTRACTOR_ExtractContext *ec) 286 { 287 struct EXTRACTOR_MediaOptions opt; 288 struct EXTRACTOR_MediaDeadline deadline; 289 struct EXTRACTOR_GstIO io; 290 GstElement *pipeline; 291 GstElement *bin; 292 GstElement *sink; 293 GstElement *volume; 294 GByteArray *output = NULL; 295 gint64 duration; 296 gint64 start; 297 gint64 stop; 298 size_t limit; 299 300 EXTRACTOR_media_options_parse_ (ec->config, 301 &opt); 302 EXTRACTOR_media_deadline_start_ (&deadline, 303 opt.deadline); 304 if (! EXTRACTOR_media_is_media_ (ec, 1)) 305 return; 306 if (! gst_is_initialized ()) 307 gst_init (NULL, NULL); 308 if (NULL == (bin = build_encoder_bin (&opt, 309 &sink, 310 &volume))) 311 return; 312 if (NULL == (pipeline = 313 EXTRACTOR_gst_playbin_ (&io, 314 ec, 315 EXTRACTOR_GST_PLAY_FLAG_AUDIO, 316 gst_element_factory_make ("fakesink", 317 NULL), 318 bin))) 319 return; 320 if (GST_STATE_CHANGE_FAILURE == 321 gst_element_set_state (pipeline, 322 GST_STATE_PAUSED)) 323 goto cleanup; 324 if (! EXTRACTOR_gst_wait_async_ (pipeline, 325 EXTRACTOR_media_deadline_remaining_ ( 326 &deadline) * GST_MSECOND)) 327 goto cleanup; 328 start = EXTRACTOR_gst_sample_position_ (pipeline, 329 opt.offset, 330 &duration); 331 stop = start + ((gint64) opt.length) * GST_SECOND; 332 if ( (0 != duration) && 333 (stop > duration) ) 334 stop = duration; 335 if ( (0 != duration) && 336 (stop - start < MIN_PREVIEW_NS) ) 337 goto cleanup; /* too short to be worth anything */ 338 setup_fade (volume, 339 stop - start); 340 if (! gst_element_seek (pipeline, 341 1.0, 342 GST_FORMAT_TIME, 343 GST_SEEK_FLAG_FLUSH 344 | GST_SEEK_FLAG_ACCURATE, 345 GST_SEEK_TYPE_SET, 346 start, 347 GST_SEEK_TYPE_SET, 348 stop)) 349 goto cleanup; 350 if (! EXTRACTOR_gst_wait_async_ (pipeline, 351 EXTRACTOR_media_deadline_remaining_ ( 352 &deadline) * GST_MSECOND)) 353 goto cleanup; 354 if (GST_STATE_CHANGE_FAILURE == 355 gst_element_set_state (pipeline, 356 GST_STATE_PLAYING)) 357 goto cleanup; 358 /* the caller asked for a preview of a given length and bitrate, so let 359 that decide the limit, but never emit something absurd */ 360 limit = (size_t) opt.length * opt.bitrate / 8 + 8192; 361 if (limit < EXTRACTOR_MEDIA_MAX_PREVIEW) 362 limit = EXTRACTOR_MEDIA_MAX_PREVIEW; 363 output = collect_output (sink, 364 &deadline, 365 limit); 366 if (NULL == output) 367 goto cleanup; 368 if (0 == output->len) 369 goto cleanup; 370 ec->proc (ec->cls, 371 "previewgst", 372 EXTRACTOR_METATYPE_AUDIO_PREVIEW, 373 EXTRACTOR_METAFORMAT_BINARY, 374 "audio/ogg", 375 (const char *) output->data, 376 output->len); 377 cleanup: 378 if (NULL != output) 379 g_byte_array_unref (output); 380 gst_element_set_state (pipeline, 381 GST_STATE_NULL); 382 gst_object_unref (pipeline); 383 } 384 385 386 /** 387 * Keep the diagnostics of the media framework out of the output of the 388 * calling application; a decoder complaining about a broken file is not 389 * something the application asked to see. 390 * 391 * ("force-kill" would be tempting here, to get a fresh process for every 392 * file, but the host cannot know about that special without loading the 393 * plugin in-process: it only finds out that the child is gone once it has 394 * already handed it the next file, which is then silently lost.) 395 * 396 * @return special options for this plugin 397 */ 398 const char * 399 EXTRACTOR_previewgst_options (void); 400 401 const char * 402 EXTRACTOR_previewgst_options () 403 { 404 return "close-stderr"; 405 } 406 407 408 /* end of previewgst_extractor.c */