thumbnailgst_extractor.c (42560B)
1 /* 2 This file is part of libextractor. 3 Copyright (C) 2026 Vidyut Samanta and Christian Grothoff 4 5 libextractor is free software; you can redistribute it and/or modify 6 it under the terms of the GNU General Public License as published 7 by the Free Software Foundation; either version 3, or (at your 8 option) any later version. 9 10 libextractor is distributed in the hope that it will be useful, but 11 WITHOUT ANY WARRANTY; without even the implied warranty of 12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 General Public License for more details. 14 15 You should have received a copy of the GNU General Public License 16 along with libextractor; see the file COPYING. If not, write to the 17 Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, 18 Boston, MA 02110-1301, USA. 19 */ 20 /** 21 * @file plugins/thumbnailgst_extractor.c 22 * @author Christian Grothoff 23 * @brief this extractor produces a binary encoded thumbnail of a frame 24 * taken from a video, using GStreamer 25 * 26 * This is the GStreamer counterpart of thumbnailffmpeg: it seeks about 27 * 30s into a video and encodes the frame it finds there as a small JPEG 28 * (or PNG). Which of the two is built depends on what was found at 29 * configure time; if both are installed, both will report a thumbnail and 30 * it is up to the application to disable one of them in its plugin 31 * configuration string. 32 * 33 * The decoders and the image encoder are themselves GStreamer plugins, 34 * resolved at run time, so a missing "jpeg" or "playback" plugin cannot 35 * be detected when libextractor is built. We simply report nothing. 36 * 37 * Two options go beyond a single thumbnail (see mediautil.h): 38 * 39 * - "gallery:N" replaces the thumbnail by a contact sheet of N tiles, 40 * sampled at the midpoints of N equal slices of the video. A tile 41 * that shows nothing (see EXTRACTOR_media_frame_is_flat_()) is 42 * re-sampled a quarter and three eighths of a slice later and 43 * earlier; if all candidates are flat, the least flat one is used. 44 * Each tile is scaled to fit into "size" and converted to RGB by 45 * gst_video_convert_sample(), the sheet is composed in memory and 46 * encoded by another gst_video_convert_sample(). Without a known 47 * duration we fall back to the plain thumbnail. 48 * 49 * - "srt:LANG" additionally dumps the first subtitle track in language 50 * LANG as SubRip. For that the file is run through a second pipeline, 51 * appsrc ! parsebin, which demuxes (but does not decode) all streams. 52 * Text subtitles come out of it as text/x-raw (Pango markup or plain 53 * UTF-8) or as application/x-ass / application/x-ssa events. All 54 * markup is stripped (tags removed, entities decoded): SubRip only 55 * knows a handful of tags anyway, and the FFmpeg plugin, which works 56 * on ASS events, cannot keep them either, so both plugins produce the 57 * same text. The language comes from the stream's tags (sticky TAG 58 * event or the GstStream of the STREAM_START event), matched with 59 * EXTRACTOR_media_lang_matches_(). 60 */ 61 #include "platform.h" 62 #include "extractor.h" 63 #include "mediautil.h" 64 #include "mediagst.h" 65 66 #include <gst/video/video.h> 67 #include <gst/app/gstappsink.h> 68 69 70 /** 71 * Scale and encode the raw video @a sample. 72 * 73 * @param sample raw frame to convert 74 * @param opt plugin options (target size and format) 75 * @param timeout how long the conversion may take, in nanoseconds 76 * @return the encoded image, NULL on error 77 */ 78 static GstSample * 79 encode_thumbnail (GstSample *sample, 80 const struct EXTRACTOR_MediaOptions *opt, 81 GstClockTime timeout) 82 { 83 GstCaps *caps; 84 GstCaps *to_caps; 85 GstStructure *s; 86 GstSample *out; 87 gint width; 88 gint height; 89 unsigned int tw; 90 unsigned int th; 91 92 if (NULL == (caps = gst_sample_get_caps (sample))) 93 return NULL; 94 if (NULL == (s = gst_caps_get_structure (caps, 0))) 95 return NULL; 96 if ( (! gst_structure_get_int (s, "width", &width)) || 97 (! gst_structure_get_int (s, "height", &height)) || 98 (0 >= width) || 99 (0 >= height) ) 100 return NULL; 101 if (! EXTRACTOR_media_thumbnail_size_ ((unsigned int) width, 102 (unsigned int) height, 103 opt->size, 104 &tw, 105 &th)) 106 return NULL; 107 to_caps = gst_caps_new_simple (opt->png ? "image/png" : "image/jpeg", 108 "width", G_TYPE_INT, (gint) tw, 109 "height", G_TYPE_INT, (gint) th, 110 NULL); 111 out = gst_video_convert_sample (sample, 112 to_caps, 113 timeout, 114 NULL); 115 gst_caps_unref (to_caps); 116 return out; 117 } 118 119 120 /** 121 * Sample a single frame about opt->offset seconds into the video and 122 * emit it as a thumbnail. 123 * 124 * @param ec extraction context 125 * @param opt plugin options 126 * @param deadline when we must be done 127 * @return return value of `proc` (0 if nothing was emitted) 128 */ 129 static int 130 make_thumbnail (struct EXTRACTOR_ExtractContext *ec, 131 const struct EXTRACTOR_MediaOptions *opt, 132 const struct EXTRACTOR_MediaDeadline *deadline) 133 { 134 struct EXTRACTOR_GstIO io; 135 GstElement *pipeline; 136 GstElement *sink; 137 GstSample *sample = NULL; 138 GstSample *thumbnail = NULL; 139 GstBuffer *buffer; 140 GstMapInfo mi; 141 GstClockTime timeout; 142 gint64 duration; 143 gint64 target; 144 int ret = 0; 145 146 sink = gst_element_factory_make ("fakesink", NULL); 147 if (NULL != sink) 148 g_object_set (sink, 149 "sync", FALSE, 150 NULL); 151 if (NULL == (pipeline = 152 EXTRACTOR_gst_playbin_ (&io, 153 ec, 154 EXTRACTOR_GST_PLAY_FLAG_VIDEO, 155 sink, 156 gst_element_factory_make ("fakesink", 157 NULL)))) 158 return 0; 159 if (GST_STATE_CHANGE_FAILURE == 160 gst_element_set_state (pipeline, 161 GST_STATE_PAUSED)) 162 goto cleanup; 163 if (! EXTRACTOR_gst_wait_async_ (pipeline, 164 EXTRACTOR_media_deadline_remaining_ ( 165 deadline) * GST_MSECOND)) 166 goto cleanup; 167 target = EXTRACTOR_gst_sample_position_ (pipeline, 168 opt->offset, 169 &duration); 170 if (target >= GST_SECOND) 171 { 172 /* a key frame is good enough for a thumbnail and much cheaper than 173 decoding forward to an exact position */ 174 if (gst_element_seek_simple (pipeline, 175 GST_FORMAT_TIME, 176 GST_SEEK_FLAG_FLUSH 177 | GST_SEEK_FLAG_KEY_UNIT, 178 target)) 179 (void) EXTRACTOR_gst_wait_async_ (pipeline, 180 EXTRACTOR_media_deadline_remaining_ ( 181 deadline) * GST_MSECOND); 182 } 183 g_object_get (sink, 184 "last-sample", 185 &sample, 186 NULL); 187 if (NULL == sample) 188 goto cleanup; 189 timeout = EXTRACTOR_media_deadline_remaining_ (deadline) * GST_MSECOND; 190 if (0 == timeout) 191 goto cleanup; 192 if (NULL == (thumbnail = encode_thumbnail (sample, 193 opt, 194 timeout))) 195 goto cleanup; 196 if (NULL == (buffer = gst_sample_get_buffer (thumbnail))) 197 goto cleanup; 198 if (! gst_buffer_map (buffer, &mi, GST_MAP_READ)) 199 goto cleanup; 200 if ( (0 < mi.size) && 201 (EXTRACTOR_MEDIA_MAX_THUMBNAIL >= mi.size) ) 202 ret = ec->proc (ec->cls, 203 "thumbnailgst", 204 EXTRACTOR_METATYPE_THUMBNAIL, 205 EXTRACTOR_METAFORMAT_BINARY, 206 opt->png ? "image/png" : "image/jpeg", 207 (const char *) mi.data, 208 mi.size); 209 gst_buffer_unmap (buffer, &mi); 210 cleanup: 211 if (NULL != thumbnail) 212 gst_sample_unref (thumbnail); 213 if (NULL != sample) 214 gst_sample_unref (sample); 215 gst_element_set_state (pipeline, 216 GST_STATE_NULL); 217 gst_object_unref (pipeline); 218 return ret; 219 } 220 221 222 /* ************************* gallery ************************* */ 223 224 225 /** 226 * How "flat" a frame is: the share (in 1/1000) of pixels inside the 227 * most populated narrow window of levels, for the channel where that 228 * share is lowest. Frames with a lower value show more; see 229 * EXTRACTOR_media_frame_is_flat_() for the yes/no version. 230 * 231 * @param rgb packed RGB24 pixels 232 * @param width width in pixels 233 * @param height height in pixels 234 * @param stride bytes per row 235 * @return flatness in [0,1000] 236 */ 237 static unsigned int 238 flatness (const uint8_t *rgb, 239 unsigned int width, 240 unsigned int height, 241 size_t stride) 242 { 243 uint32_t hist[3][256]; 244 uint64_t total = 0; 245 unsigned int worst = 1000; 246 unsigned int x; 247 unsigned int y; 248 unsigned int c; 249 250 memset (hist, 0, sizeof (hist)); 251 for (y = 0; y < height; y++) 252 { 253 const uint8_t *row = rgb + (size_t) y * stride; 254 255 for (x = 0; x < width; x++) 256 { 257 hist[0][row[3 * x]]++; 258 hist[1][row[3 * x + 1]]++; 259 hist[2][row[3 * x + 2]]++; 260 total++; 261 } 262 } 263 if (0 == total) 264 return 1000; 265 for (c = 0; c < 3; c++) 266 { 267 uint64_t win = 0; 268 uint64_t best = 0; 269 unsigned int i; 270 271 for (i = 0; i < 256; i++) 272 { 273 win += hist[c][i]; 274 if (i >= 32) 275 win -= hist[c][i - 32]; 276 if (win > best) 277 best = win; 278 } 279 if ((unsigned int) (best * 1000 / total) < worst) 280 worst = (unsigned int) (best * 1000 / total); 281 } 282 return worst; 283 } 284 285 286 /** 287 * Seek the paused @a pipeline to @a ms and return the frame shown there. 288 * 289 * @param pipeline paused, prerolled pipeline 290 * @param io I/O state of the pipeline 291 * @param sink the appsink receiving the video 292 * @param ms position in milliseconds 293 * @param deadline when we must be done 294 * @return the frame, NULL on error or timeout 295 */ 296 static GstSample * 297 sample_at (GstElement *pipeline, 298 struct EXTRACTOR_GstIO *io, 299 GstElement *sink, 300 uint64_t ms, 301 const struct EXTRACTOR_MediaDeadline *deadline) 302 { 303 if (! gst_element_seek_simple (pipeline, 304 GST_FORMAT_TIME, 305 GST_SEEK_FLAG_FLUSH 306 | GST_SEEK_FLAG_ACCURATE, 307 (gint64) (ms * GST_MSECOND))) 308 return NULL; 309 if (! EXTRACTOR_gst_wait_async_alive_ (pipeline, 310 EXTRACTOR_media_deadline_remaining_ ( 311 deadline) * GST_MSECOND, 312 io)) 313 return NULL; 314 return gst_app_sink_try_pull_preroll (GST_APP_SINK (sink), 315 EXTRACTOR_media_deadline_remaining_ ( 316 deadline) * GST_MSECOND); 317 } 318 319 320 /** 321 * Convert @a sample to packed RGB of exactly @a width x @a height. 322 * 323 * @param sample frame to convert 324 * @param width target width 325 * @param height target height 326 * @param deadline when we must be done 327 * @return the converted frame, NULL on error 328 */ 329 static GstSample * 330 to_rgb (GstSample *sample, 331 unsigned int width, 332 unsigned int height, 333 const struct EXTRACTOR_MediaDeadline *deadline) 334 { 335 GstCaps *to_caps; 336 GstSample *out; 337 GstClockTime timeout; 338 339 timeout = EXTRACTOR_media_deadline_remaining_ (deadline) * GST_MSECOND; 340 if (0 == timeout) 341 return NULL; 342 /* no pixel-aspect-ratio here: fixing it would make videoscale add 343 borders to keep the display aspect ratio, and a letterboxed tile 344 would look flat to the check below */ 345 to_caps = gst_caps_new_simple ("video/x-raw", 346 "format", G_TYPE_STRING, "RGB", 347 "width", G_TYPE_INT, (gint) width, 348 "height", G_TYPE_INT, (gint) height, 349 NULL); 350 out = gst_video_convert_sample (sample, 351 to_caps, 352 timeout, 353 NULL); 354 gst_caps_unref (to_caps); 355 return out; 356 } 357 358 359 /** 360 * Map the RGB frame in @a sample, check that it has the expected size. 361 * 362 * @param sample frame from to_rgb() 363 * @param width expected width 364 * @param height expected height 365 * @param[out] frame mapped frame, to be released with 366 * gst_video_frame_unmap() 367 * @return 1 on success 368 */ 369 static int 370 map_rgb (GstSample *sample, 371 unsigned int width, 372 unsigned int height, 373 GstVideoFrame *frame) 374 { 375 GstVideoInfo vi; 376 GstCaps *caps; 377 GstBuffer *buffer; 378 379 if ( (NULL == (caps = gst_sample_get_caps (sample))) || 380 (NULL == (buffer = gst_sample_get_buffer (sample))) || 381 (! gst_video_info_from_caps (&vi, caps)) || 382 (GST_VIDEO_FORMAT_RGB != GST_VIDEO_INFO_FORMAT (&vi)) || 383 (width != (unsigned int) GST_VIDEO_INFO_WIDTH (&vi)) || 384 (height != (unsigned int) GST_VIDEO_INFO_HEIGHT (&vi)) ) 385 return 0; 386 if (! gst_video_frame_map (frame, 387 &vi, 388 buffer, 389 GST_MAP_READ)) 390 return 0; 391 return 1; 392 } 393 394 395 /** 396 * Sample tile @a i of @a n and scale it to @a tw x @a th RGB. If the 397 * frame at the nominal time is flat, try other moments in the same 398 * slice; if they are all flat, return the least flat one. 399 * 400 * @param pipeline paused, prerolled pipeline 401 * @param io I/O state of the pipeline 402 * @param sink the appsink receiving the video 403 * @param i tile index 404 * @param n number of tiles 405 * @param duration_ms length of the video 406 * @param tw tile width 407 * @param th tile height 408 * @param deadline when we must be done 409 * @return the tile (RGB, @a tw x @a th), NULL if nothing could be sampled 410 */ 411 static GstSample * 412 sample_tile (GstElement *pipeline, 413 struct EXTRACTOR_GstIO *io, 414 GstElement *sink, 415 unsigned int i, 416 unsigned int n, 417 uint64_t duration_ms, 418 unsigned int tw, 419 unsigned int th, 420 const struct EXTRACTOR_MediaDeadline *deadline) 421 { 422 uint64_t base = EXTRACTOR_media_gallery_time_ (i, n, duration_ms); 423 int64_t slice = (int64_t) (duration_ms / n); 424 int64_t shifts[5]; 425 uint64_t tried[5]; 426 GstSample *best = NULL; 427 unsigned int best_score = UINT_MAX; 428 unsigned int k; 429 430 shifts[0] = 0; 431 shifts[1] = slice / 4; 432 shifts[2] = -slice / 4; 433 shifts[3] = slice * 3 / 8; 434 shifts[4] = -slice * 3 / 8; 435 for (k = 0; k < 5; k++) 436 { 437 int64_t t = (int64_t) base + shifts[k]; 438 GstSample *frame; 439 GstSample *rgb; 440 GstVideoFrame vf; 441 unsigned int score; 442 int flat; 443 unsigned int j; 444 int dup = 0; 445 446 if (t < 0) 447 t = 0; 448 if ((uint64_t) t >= duration_ms) 449 t = (int64_t) duration_ms - 1; 450 for (j = 0; j < k; j++) 451 if (tried[j] == (uint64_t) t) 452 dup = 1; 453 tried[k] = (uint64_t) t; 454 if (dup) 455 continue; 456 if (EXTRACTOR_media_deadline_expired_ (deadline)) 457 break; 458 EXTRACTOR_gst_keepalive_ (io); 459 if (NULL == (frame = sample_at (pipeline, 460 io, 461 sink, 462 (uint64_t) t, 463 deadline))) 464 continue; 465 rgb = to_rgb (frame, 466 tw, 467 th, 468 deadline); 469 gst_sample_unref (frame); 470 if (NULL == rgb) 471 continue; 472 if (! map_rgb (rgb, 473 tw, 474 th, 475 &vf)) 476 { 477 gst_sample_unref (rgb); 478 continue; 479 } 480 score = flatness (GST_VIDEO_FRAME_PLANE_DATA (&vf, 0), 481 tw, 482 th, 483 GST_VIDEO_FRAME_PLANE_STRIDE (&vf, 0)); 484 flat = EXTRACTOR_media_frame_is_flat_ ( 485 GST_VIDEO_FRAME_PLANE_DATA (&vf, 0), 486 tw, 487 th, 488 GST_VIDEO_FRAME_PLANE_STRIDE (&vf, 0)); 489 gst_video_frame_unmap (&vf); 490 if ( (NULL == best) || 491 (score < best_score) ) 492 { 493 if (NULL != best) 494 gst_sample_unref (best); 495 best = rgb; 496 best_score = score; 497 } 498 else 499 { 500 gst_sample_unref (rgb); 501 } 502 if (! flat) 503 break; 504 } 505 return best; 506 } 507 508 509 /** 510 * Emit a contact sheet of opt->gallery frames. 511 * 512 * @param ec extraction context 513 * @param opt plugin options 514 * @param deadline when we must be done 515 * @param[out] ret set to the return value of `proc` 516 * @return 1 if the gallery was dealt with (emitted, or failed for good), 517 * 0 if the caller should fall back to a single thumbnail 518 */ 519 static int 520 make_gallery (struct EXTRACTOR_ExtractContext *ec, 521 const struct EXTRACTOR_MediaOptions *opt, 522 const struct EXTRACTOR_MediaDeadline *deadline, 523 int *ret) 524 { 525 struct EXTRACTOR_GstIO io; 526 GstElement *pipeline; 527 GstElement *sink; 528 GstCaps *caps; 529 GstSample *first = NULL; 530 GstSample *sheet_sample = NULL; 531 GstSample *encoded = NULL; 532 unsigned int shrink; 533 GstBuffer *sheet = NULL; 534 GstVideoInfo sheet_info; 535 GstStructure *s; 536 GstMapInfo mi; 537 gint64 duration = 0; 538 gint width; 539 gint height; 540 unsigned int tw; 541 unsigned int th; 542 unsigned int cols; 543 unsigned int rows; 544 unsigned int n = opt->gallery; 545 unsigned int i; 546 uint64_t duration_ms; 547 int handled = 0; 548 549 *ret = 0; 550 sink = gst_element_factory_make ("appsink", NULL); 551 if (NULL != sink) 552 { 553 caps = gst_caps_new_empty_simple ("video/x-raw"); 554 g_object_set (sink, 555 "sync", FALSE, 556 "caps", caps, 557 NULL); 558 gst_caps_unref (caps); 559 } 560 if (NULL == (pipeline = 561 EXTRACTOR_gst_playbin_ (&io, 562 ec, 563 EXTRACTOR_GST_PLAY_FLAG_VIDEO, 564 sink, 565 gst_element_factory_make ("fakesink", 566 NULL)))) 567 return 0; 568 if (GST_STATE_CHANGE_FAILURE == 569 gst_element_set_state (pipeline, 570 GST_STATE_PAUSED)) 571 goto cleanup; 572 if (! EXTRACTOR_gst_wait_async_alive_ (pipeline, 573 EXTRACTOR_media_deadline_remaining_ ( 574 deadline) * GST_MSECOND, 575 &io)) 576 goto cleanup; 577 if ( (! gst_element_query_duration (pipeline, 578 GST_FORMAT_TIME, 579 &duration)) || 580 (duration < (gint64) GST_MSECOND) ) 581 goto cleanup; /* unknown length: plain thumbnail */ 582 duration_ms = (uint64_t) duration / GST_MSECOND; 583 if (NULL == (first = gst_app_sink_try_pull_preroll ( 584 GST_APP_SINK (sink), 585 EXTRACTOR_media_deadline_remaining_ (deadline) 586 * GST_MSECOND))) 587 goto cleanup; 588 if ( (NULL == (caps = gst_sample_get_caps (first))) || 589 (NULL == (s = gst_caps_get_structure (caps, 0))) || 590 (! gst_structure_get_int (s, "width", &width)) || 591 (! gst_structure_get_int (s, "height", &height)) || 592 (0 >= width) || 593 (0 >= height) || 594 (! EXTRACTOR_media_thumbnail_size_ ((unsigned int) width, 595 (unsigned int) height, 596 opt->size, 597 &tw, 598 &th)) ) 599 goto cleanup; 600 /* from here on, we produce a gallery or nothing */ 601 handled = 1; 602 EXTRACTOR_media_gallery_layout_ (n, 603 tw, 604 th, 605 &cols, 606 &rows); 607 while ( ((uint64_t) cols * tw) * ((uint64_t) rows * th) 608 > EXTRACTOR_MEDIA_MAX_GALLERY_PIXELS) 609 { 610 tw = (tw * 3) / 4; 611 th = (th * 3) / 4; 612 if ( (0 == tw) || 613 (0 == th) ) 614 goto cleanup; 615 } 616 if (! gst_video_info_set_format (&sheet_info, 617 GST_VIDEO_FORMAT_RGB, 618 cols * tw, 619 rows * th)) 620 goto cleanup; 621 if (NULL == (sheet = gst_buffer_new_allocate (NULL, 622 GST_VIDEO_INFO_SIZE ( 623 &sheet_info), 624 NULL))) 625 goto cleanup; 626 if (! gst_buffer_map (sheet, &mi, GST_MAP_WRITE)) 627 goto cleanup; 628 memset (mi.data, 0, mi.size); 629 for (i = 0; i < n; i++) 630 { 631 GstSample *tile; 632 GstVideoFrame vf; 633 size_t dstride = GST_VIDEO_INFO_PLANE_STRIDE (&sheet_info, 0); 634 uint8_t *dst = mi.data 635 + GST_VIDEO_INFO_PLANE_OFFSET (&sheet_info, 0) 636 + (size_t) (i / cols) * th * dstride 637 + (size_t) (i % cols) * tw * 3; 638 unsigned int y; 639 640 tile = sample_tile (pipeline, 641 &io, 642 sink, 643 i, 644 n, 645 duration_ms, 646 tw, 647 th, 648 deadline); 649 if (EXTRACTOR_media_deadline_expired_ (deadline)) 650 { 651 /* better no gallery than one with holes */ 652 if (NULL != tile) 653 gst_sample_unref (tile); 654 gst_buffer_unmap (sheet, &mi); 655 goto cleanup; 656 } 657 if (NULL == tile) 658 continue; /* leave the tile black */ 659 if (map_rgb (tile, 660 tw, 661 th, 662 &vf)) 663 { 664 const uint8_t *src = GST_VIDEO_FRAME_PLANE_DATA (&vf, 0); 665 size_t sstride = GST_VIDEO_FRAME_PLANE_STRIDE (&vf, 0); 666 667 for (y = 0; y < th; y++) 668 memcpy (dst + (size_t) y * dstride, 669 src + (size_t) y * sstride, 670 (size_t) tw * 3); 671 gst_video_frame_unmap (&vf); 672 } 673 gst_sample_unref (tile); 674 } 675 gst_buffer_unmap (sheet, &mi); 676 /* the decoding pipeline is no longer needed; free its memory before 677 we encode */ 678 gst_element_set_state (pipeline, 679 GST_STATE_NULL); 680 EXTRACTOR_gst_keepalive_ (&io); 681 caps = gst_video_info_to_caps (&sheet_info); 682 sheet_sample = gst_sample_new (sheet, 683 caps, 684 NULL, 685 NULL); 686 gst_caps_unref (caps); 687 /* encode; if the result is too large, halve the sheet and retry */ 688 for (shrink = 0; shrink < 4; shrink++) 689 { 690 GstCaps *to_caps; 691 GstClockTime timeout; 692 GstBuffer *buffer; 693 unsigned int ow = (cols * tw) >> shrink; 694 unsigned int oh = (rows * th) >> shrink; 695 696 if ( (0 == ow) || 697 (0 == oh) ) 698 break; 699 EXTRACTOR_gst_keepalive_ (&io); 700 timeout = EXTRACTOR_media_deadline_remaining_ (deadline) * GST_MSECOND; 701 if (0 == timeout) 702 break; 703 to_caps = gst_caps_new_simple (opt->png ? "image/png" : "image/jpeg", 704 "width", G_TYPE_INT, (gint) ow, 705 "height", G_TYPE_INT, (gint) oh, 706 NULL); 707 encoded = gst_video_convert_sample (sheet_sample, 708 to_caps, 709 timeout, 710 NULL); 711 gst_caps_unref (to_caps); 712 if (NULL == encoded) 713 break; 714 if (NULL == (buffer = gst_sample_get_buffer (encoded))) 715 break; 716 if (! gst_buffer_map (buffer, &mi, GST_MAP_READ)) 717 break; 718 if ( (0 < mi.size) && 719 (EXTRACTOR_MEDIA_MAX_GALLERY_BYTES >= mi.size) ) 720 { 721 *ret = ec->proc (ec->cls, 722 "thumbnailgst", 723 EXTRACTOR_METATYPE_THUMBNAIL, 724 EXTRACTOR_METAFORMAT_BINARY, 725 opt->png ? "image/png" : "image/jpeg", 726 (const char *) mi.data, 727 mi.size); 728 gst_buffer_unmap (buffer, &mi); 729 break; 730 } 731 gst_buffer_unmap (buffer, &mi); 732 gst_sample_unref (encoded); 733 encoded = NULL; 734 } 735 cleanup: 736 if (NULL != encoded) 737 gst_sample_unref (encoded); 738 if (NULL != sheet_sample) 739 gst_sample_unref (sheet_sample); 740 if (NULL != sheet) 741 gst_buffer_unref (sheet); 742 if (NULL != first) 743 gst_sample_unref (first); 744 gst_element_set_state (pipeline, 745 GST_STATE_NULL); 746 gst_object_unref (pipeline); 747 return handled; 748 } 749 750 751 /* ************************* subtitles ************************* */ 752 753 754 /** 755 * Most subtitle tracks we keep apart while we find out their language. 756 */ 757 #define MAX_SUB_TRACKS 16 758 759 /** 760 * Most streams of one file we connect to a sink at all. 761 */ 762 #define MAX_STREAMS 64 763 764 765 /** 766 * How the payload of a subtitle stream is encoded. 767 */ 768 enum SubKind 769 { 770 /** 771 * Plain UTF-8 text (text/x-raw, format=utf8). 772 */ 773 SUB_PLAIN, 774 775 /** 776 * Pango markup (text/x-raw, format=pango-markup). 777 */ 778 SUB_MARKUP, 779 780 /** 781 * ASS/SSA dialogue events (application/x-ass, application/x-ssa). 782 */ 783 SUB_ASS 784 }; 785 786 787 struct SubState; 788 789 790 /** 791 * One subtitle track of the file. 792 */ 793 struct SubTrack 794 { 795 /** 796 * The subtitle dump this track belongs to. 797 */ 798 struct SubState *st; 799 800 /** 801 * Encoding of the payload. 802 */ 803 enum SubKind kind; 804 805 /** 806 * 1 if the language matches, -1 if it does not, 0 if still unknown. 807 */ 808 int match; 809 810 /** 811 * Cues collected so far (only while @e match is not -1). 812 */ 813 struct EXTRACTOR_MediaSrt srt; 814 }; 815 816 817 /** 818 * State of a subtitle dump. 819 */ 820 struct SubState 821 { 822 /** 823 * Protects @e tracks, @e num_tracks and @e num_streams against 824 * concurrent "pad-added" signals. 825 */ 826 GMutex lock; 827 828 /** 829 * The pipeline, to add sinks to. 830 */ 831 GstElement *pipeline; 832 833 /** 834 * Requested language. 835 */ 836 const char *want; 837 838 /** 839 * The subtitle tracks, in the order the demuxer exposed them. 840 */ 841 struct SubTrack tracks[MAX_SUB_TRACKS]; 842 843 /** 844 * Number of entries used in @e tracks. 845 */ 846 unsigned int num_tracks; 847 848 /** 849 * Number of streams connected to a sink. 850 */ 851 unsigned int num_streams; 852 }; 853 854 855 /** 856 * Get the language tag of the stream flowing through @a pad, from the 857 * sticky TAG events or the GstStream of the STREAM_START event. 858 * 859 * @param pad pad to inspect 860 * @return the language (to be freed with g_free()), NULL if unknown 861 */ 862 static gchar * 863 pad_language (GstPad *pad) 864 { 865 GstEvent *ev; 866 gchar *lang = NULL; 867 guint i; 868 869 for (i = 0; 870 (NULL == lang) && 871 (i < 16) && 872 (NULL != (ev = gst_pad_get_sticky_event (pad, 873 GST_EVENT_TAG, 874 i))); 875 i++) 876 { 877 GstTagList *tags = NULL; 878 879 gst_event_parse_tag (ev, 880 &tags); 881 if ( (NULL == tags) || 882 (! gst_tag_list_get_string (tags, 883 GST_TAG_LANGUAGE_CODE, 884 &lang)) ) 885 lang = NULL; 886 gst_event_unref (ev); 887 } 888 if ( (NULL == lang) && 889 (NULL != (ev = gst_pad_get_sticky_event (pad, 890 GST_EVENT_STREAM_START, 891 0))) ) 892 { 893 GstStream *stream = NULL; 894 895 gst_event_parse_stream (ev, 896 &stream); 897 if (NULL != stream) 898 { 899 GstTagList *tags = gst_stream_get_tags (stream); 900 901 if (NULL != tags) 902 { 903 if (! gst_tag_list_get_string (tags, 904 GST_TAG_LANGUAGE_CODE, 905 &lang)) 906 lang = NULL; 907 gst_tag_list_unref (tags); 908 } 909 gst_object_unref (stream); 910 } 911 gst_event_unref (ev); 912 } 913 return lang; 914 } 915 916 917 /** 918 * Encode @a cp as UTF-8. 919 * 920 * @param cp code point, valid 921 * @param[out] out at least 4 bytes 922 * @return number of bytes written 923 */ 924 static size_t 925 put_utf8 (uint32_t cp, 926 char *out) 927 { 928 if (cp < 0x80) 929 { 930 out[0] = (char) cp; 931 return 1; 932 } 933 if (cp < 0x800) 934 { 935 out[0] = (char) (0xC0 | (cp >> 6)); 936 out[1] = (char) (0x80 | (cp & 0x3F)); 937 return 2; 938 } 939 if (cp < 0x10000) 940 { 941 out[0] = (char) (0xE0 | (cp >> 12)); 942 out[1] = (char) (0x80 | ((cp >> 6) & 0x3F)); 943 out[2] = (char) (0x80 | (cp & 0x3F)); 944 return 3; 945 } 946 out[0] = (char) (0xF0 | (cp >> 18)); 947 out[1] = (char) (0x80 | ((cp >> 12) & 0x3F)); 948 out[2] = (char) (0x80 | ((cp >> 6) & 0x3F)); 949 out[3] = (char) (0x80 | (cp & 0x3F)); 950 return 4; 951 } 952 953 954 /** 955 * Decode the XML entity @a name (without '&' and ';'). 956 * 957 * @param name entity name 958 * @param len length of @a name 959 * @param[out] cp the code point it stands for 960 * @return 1 if the entity is known 961 */ 962 static int 963 decode_entity (const char *name, 964 size_t len, 965 uint32_t *cp) 966 { 967 static const struct 968 { 969 const char *name; 970 uint32_t cp; 971 } named[] = { 972 { "amp", '&' }, 973 { "lt", '<' }, 974 { "gt", '>' }, 975 { "quot", '"' }, 976 { "apos", '\'' }, 977 { "nbsp", 0xA0 }, 978 { NULL, 0 } 979 }; 980 uint64_t v = 0; 981 size_t i; 982 983 if ( (len >= 2) && 984 ('#' == name[0]) ) 985 { 986 int hex = ( ('x' == name[1]) || ('X' == name[1]) ); 987 988 i = hex ? 2 : 1; 989 if (i == len) 990 return 0; 991 for (; i < len; i++) 992 { 993 char c = name[i]; 994 unsigned int d; 995 996 if ( (c >= '0') && (c <= '9') ) 997 d = c - '0'; 998 else if (hex && (c >= 'a') && (c <= 'f')) 999 d = c - 'a' + 10; 1000 else if (hex && (c >= 'A') && (c <= 'F')) 1001 d = c - 'A' + 10; 1002 else 1003 return 0; 1004 v = v * (hex ? 16 : 10) + d; 1005 if (v > 0x10FFFF) 1006 return 0; 1007 } 1008 if ( (0 == v) || 1009 ( (v >= 0xD800) && (v <= 0xDFFF) ) ) 1010 return 0; 1011 *cp = (uint32_t) v; 1012 return 1; 1013 } 1014 for (i = 0; NULL != named[i].name; i++) 1015 if ( (strlen (named[i].name) == len) && 1016 (0 == memcmp (named[i].name, name, len)) ) 1017 { 1018 *cp = named[i].cp; 1019 return 1; 1020 } 1021 return 0; 1022 } 1023 1024 1025 /** 1026 * Turn Pango markup into plain text: drop all tags, decode entities. 1027 * The result is never longer than the input. 1028 * 1029 * @param in markup 1030 * @param len number of bytes in @a in 1031 * @param[out] out at least @a len bytes 1032 * @return number of bytes written to @a out 1033 */ 1034 static size_t 1035 strip_markup (const char *in, 1036 size_t len, 1037 char *out) 1038 { 1039 size_t i = 0; 1040 size_t o = 0; 1041 1042 while ( (i < len) && 1043 ('\0' != in[i]) ) 1044 { 1045 if ('<' == in[i]) 1046 { 1047 while ( (i < len) && 1048 ('>' != in[i]) ) 1049 i++; 1050 i++; 1051 continue; 1052 } 1053 if ('&' == in[i]) 1054 { 1055 size_t e = i + 1; 1056 uint32_t cp; 1057 1058 /* the longest entity we know is "" */ 1059 while ( (e < len) && 1060 (e - i < 12) && 1061 (';' != in[e]) ) 1062 e++; 1063 if ( (e < len) && 1064 (';' == in[e]) && 1065 (decode_entity (&in[i + 1], 1066 e - i - 1, 1067 &cp)) ) 1068 { 1069 /* an entity is at least as long as its UTF-8 encoding */ 1070 o += put_utf8 (cp, 1071 &out[o]); 1072 i = e + 1; 1073 continue; 1074 } 1075 } 1076 out[o++] = in[i++]; 1077 } 1078 return o; 1079 } 1080 1081 1082 /** 1083 * Add the subtitle in @a buf to @a t. 1084 * 1085 * @param t track the buffer belongs to 1086 * @param pad pad the buffer arrived at (for its segment) 1087 * @param buf the buffer 1088 */ 1089 static void 1090 add_cue (struct SubTrack *t, 1091 GstPad *pad, 1092 GstBuffer *buf) 1093 { 1094 GstClockTime start = GST_BUFFER_PTS (buf); 1095 GstClockTime dur = GST_BUFFER_DURATION (buf); 1096 GstClockTime end; 1097 GstEvent *ev; 1098 GstMapInfo mi; 1099 1100 if (! GST_CLOCK_TIME_IS_VALID (start)) 1101 return; 1102 if ( (! GST_CLOCK_TIME_IS_VALID (dur)) || 1103 (dur > GST_CLOCK_TIME_NONE - 1 - start) ) 1104 dur = 0; 1105 end = start + dur; 1106 if (NULL != (ev = gst_pad_get_sticky_event (pad, 1107 GST_EVENT_SEGMENT, 1108 0))) 1109 { 1110 const GstSegment *seg; 1111 guint64 st; 1112 1113 gst_event_parse_segment (ev, 1114 &seg); 1115 if ( (GST_FORMAT_TIME == seg->format) && 1116 (GST_CLOCK_TIME_IS_VALID (st = 1117 gst_segment_to_stream_time (seg, 1118 GST_FORMAT_TIME, 1119 start))) ) 1120 { 1121 start = st; 1122 end = st + dur; 1123 } 1124 gst_event_unref (ev); 1125 } 1126 if (! gst_buffer_map (buf, 1127 &mi, 1128 GST_MAP_READ)) 1129 return; 1130 if (SUB_MARKUP == t->kind) 1131 { 1132 char *plain = malloc (mi.size + 1); 1133 1134 if (NULL != plain) 1135 { 1136 size_t plen = strip_markup ((const char *) mi.data, 1137 mi.size, 1138 plain); 1139 1140 EXTRACTOR_media_srt_add_ (&t->srt, 1141 start / GST_MSECOND, 1142 end / GST_MSECOND, 1143 plain, 1144 plen, 1145 0); 1146 free (plain); 1147 } 1148 } 1149 else 1150 { 1151 EXTRACTOR_media_srt_add_ (&t->srt, 1152 start / GST_MSECOND, 1153 end / GST_MSECOND, 1154 (const char *) mi.data, 1155 mi.size, 1156 (SUB_ASS == t->kind) ? 1 : 0); 1157 } 1158 gst_buffer_unmap (buf, 1159 &mi); 1160 } 1161 1162 1163 /** 1164 * Buffer probe on the sink of a subtitle stream: find out the language 1165 * (if still unknown) and collect the cue. 1166 * 1167 * @param pad sink pad of the fakesink 1168 * @param info the buffer 1169 * @param cls the `struct SubTrack` 1170 * @return #GST_PAD_PROBE_OK 1171 */ 1172 static GstPadProbeReturn 1173 sub_probe (GstPad *pad, 1174 GstPadProbeInfo *info, 1175 gpointer cls) 1176 { 1177 struct SubTrack *t = cls; 1178 GstBuffer *buf = GST_PAD_PROBE_INFO_BUFFER (info); 1179 1180 if (NULL == buf) 1181 return GST_PAD_PROBE_OK; 1182 if (0 == t->match) 1183 { 1184 gchar *lang = pad_language (pad); 1185 1186 if (NULL != lang) 1187 { 1188 t->match = EXTRACTOR_media_lang_matches_ (t->st->want, 1189 lang) ? 1 : -1; 1190 g_free (lang); 1191 if (-1 == t->match) 1192 EXTRACTOR_media_srt_clear_ (&t->srt); 1193 } 1194 } 1195 if (-1 != t->match) 1196 add_cue (t, 1197 pad, 1198 buf); 1199 return GST_PAD_PROBE_OK; 1200 } 1201 1202 1203 /** 1204 * parsebin exposed a new stream. Connect it to a fakesink; if it is a 1205 * text subtitle stream, watch the buffers arriving there. 1206 * 1207 * @param bin the parsebin 1208 * @param pad the new pad 1209 * @param cls the `struct SubState` 1210 */ 1211 static void 1212 sub_pad_added (GstElement *bin, 1213 GstPad *pad, 1214 gpointer cls) 1215 { 1216 struct SubState *st = cls; 1217 struct SubTrack *track = NULL; 1218 GstCaps *caps; 1219 GstElement *sink; 1220 GstPad *sinkpad; 1221 enum SubKind kind = SUB_PLAIN; 1222 int is_sub = 0; 1223 1224 (void) bin; 1225 if (GST_PAD_SRC != GST_PAD_DIRECTION (pad)) 1226 return; 1227 caps = gst_pad_get_current_caps (pad); 1228 if (NULL == caps) 1229 caps = gst_pad_query_caps (pad, 1230 NULL); 1231 if ( (NULL != caps) && 1232 (! gst_caps_is_empty (caps)) && 1233 (! gst_caps_is_any (caps)) ) 1234 { 1235 const GstStructure *s = gst_caps_get_structure (caps, 0); 1236 const char *name = gst_structure_get_name (s); 1237 1238 if (0 == strcmp (name, 1239 "text/x-raw")) 1240 { 1241 const char *fmt = gst_structure_get_string (s, 1242 "format"); 1243 1244 is_sub = 1; 1245 kind = ( (NULL != fmt) && 1246 (0 == strcmp (fmt, 1247 "utf8")) ) 1248 ? SUB_PLAIN 1249 : SUB_MARKUP; 1250 } 1251 else if ( (0 == strcmp (name, 1252 "application/x-ass")) || 1253 (0 == strcmp (name, 1254 "application/x-ssa")) ) 1255 { 1256 is_sub = 1; 1257 kind = SUB_ASS; 1258 } 1259 } 1260 if (NULL != caps) 1261 gst_caps_unref (caps); 1262 g_mutex_lock (&st->lock); 1263 if (st->num_streams >= MAX_STREAMS) 1264 { 1265 g_mutex_unlock (&st->lock); 1266 return; 1267 } 1268 st->num_streams++; 1269 if (is_sub && 1270 (st->num_tracks < MAX_SUB_TRACKS) ) 1271 { 1272 track = &st->tracks[st->num_tracks++]; 1273 track->st = st; 1274 track->kind = kind; 1275 } 1276 g_mutex_unlock (&st->lock); 1277 if (NULL == (sink = gst_element_factory_make ("fakesink", 1278 NULL))) 1279 return; 1280 g_object_set (sink, 1281 "sync", FALSE, 1282 "async", FALSE, 1283 NULL); 1284 if (! gst_bin_add (GST_BIN (st->pipeline), 1285 sink)) 1286 { 1287 gst_object_unref (sink); 1288 return; 1289 } 1290 sinkpad = gst_element_get_static_pad (sink, 1291 "sink"); 1292 if (NULL != track) 1293 gst_pad_add_probe (sinkpad, 1294 GST_PAD_PROBE_TYPE_BUFFER, 1295 &sub_probe, 1296 track, 1297 NULL); 1298 (void) gst_element_sync_state_with_parent (sink); 1299 (void) gst_pad_link (pad, 1300 sinkpad); 1301 gst_object_unref (sinkpad); 1302 } 1303 1304 1305 /** 1306 * Dump the first subtitle track in language opt->srt_lang as SRT. 1307 * The whole file is demuxed (not decoded) for this; if we run out of 1308 * time or the file turns out to be broken halfway, the cues found so 1309 * far are emitted. 1310 * 1311 * @param ec extraction context 1312 * @param opt plugin options 1313 * @param deadline when we must be done 1314 * @return return value of `proc` (0 if nothing was emitted) 1315 */ 1316 static int 1317 dump_subtitles (struct EXTRACTOR_ExtractContext *ec, 1318 const struct EXTRACTOR_MediaOptions *opt, 1319 const struct EXTRACTOR_MediaDeadline *deadline) 1320 { 1321 struct EXTRACTOR_GstIO io; 1322 struct SubState st; 1323 GstElement *pipeline = NULL; 1324 GstElement *src; 1325 GstElement *parse; 1326 GstBus *bus; 1327 unsigned int i; 1328 int ret = 0; 1329 1330 memset (&st, 0, sizeof (st)); 1331 g_mutex_init (&st.lock); 1332 st.want = opt->srt_lang; 1333 src = EXTRACTOR_gst_appsrc_ (&io, 1334 ec); 1335 parse = gst_element_factory_make ("parsebin", 1336 NULL); 1337 pipeline = gst_pipeline_new (NULL); 1338 if ( (NULL == src) || 1339 (NULL == parse) || 1340 (NULL == pipeline) ) 1341 { 1342 if (NULL != src) 1343 gst_object_unref (src); 1344 if (NULL != parse) 1345 gst_object_unref (parse); 1346 if (NULL != pipeline) 1347 gst_object_unref (pipeline); 1348 g_mutex_clear (&st.lock); 1349 return 0; 1350 } 1351 gst_bin_add_many (GST_BIN (pipeline), 1352 src, 1353 parse, 1354 NULL); 1355 st.pipeline = pipeline; 1356 g_signal_connect (parse, 1357 "pad-added", 1358 G_CALLBACK (&sub_pad_added), 1359 &st); 1360 if (! gst_element_link (src, 1361 parse)) 1362 goto cleanup; 1363 if (GST_STATE_CHANGE_FAILURE == 1364 gst_element_set_state (pipeline, 1365 GST_STATE_PLAYING)) 1366 goto cleanup; 1367 if (NULL != (bus = gst_element_get_bus (pipeline))) 1368 { 1369 GstClockTime timeout; 1370 GstMessage *msg = NULL; 1371 1372 /* wait in short slices, so that we can tell the host that we are 1373 still alive (see EXTRACTOR_gst_keepalive_()) */ 1374 while ( (NULL == msg) && 1375 (0 != (timeout = EXTRACTOR_media_deadline_remaining_ (deadline) 1376 * GST_MSECOND)) ) 1377 { 1378 if (timeout > 100 * GST_MSECOND) 1379 timeout = 100 * GST_MSECOND; 1380 msg = gst_bus_timed_pop_filtered (bus, 1381 timeout, 1382 GST_MESSAGE_ERROR 1383 | GST_MESSAGE_EOS); 1384 if (NULL == msg) 1385 EXTRACTOR_gst_keepalive_ (&io); 1386 } 1387 if (NULL != msg) 1388 gst_message_unref (msg); 1389 gst_object_unref (bus); 1390 } 1391 /* this joins the streaming threads; afterwards nobody touches st */ 1392 gst_element_set_state (pipeline, 1393 GST_STATE_NULL); 1394 for (i = 0; i < st.num_tracks; i++) 1395 if (1 == st.tracks[i].match) 1396 { 1397 ret = EXTRACTOR_media_srt_emit_ (&st.tracks[i].srt, 1398 ec, 1399 "thumbnailgst"); 1400 break; 1401 } 1402 cleanup: 1403 gst_element_set_state (pipeline, 1404 GST_STATE_NULL); 1405 for (i = 0; i < st.num_tracks; i++) 1406 EXTRACTOR_media_srt_clear_ (&st.tracks[i].srt); 1407 gst_object_unref (pipeline); 1408 g_mutex_clear (&st.lock); 1409 return ret; 1410 } 1411 1412 1413 /** 1414 * Main method for the gstreamer-thumbnailer plugin. 1415 * 1416 * @param ec extraction context 1417 */ 1418 void 1419 EXTRACTOR_thumbnailgst_extract_method (struct EXTRACTOR_ExtractContext *ec); 1420 1421 void 1422 EXTRACTOR_thumbnailgst_extract_method (struct EXTRACTOR_ExtractContext *ec) 1423 { 1424 struct EXTRACTOR_MediaOptions opt; 1425 struct EXTRACTOR_MediaDeadline deadline; 1426 int ret = 0; 1427 1428 EXTRACTOR_media_options_parse_ (ec->config, 1429 &opt); 1430 EXTRACTOR_media_deadline_start_ (&deadline, 1431 opt.deadline); 1432 if (! EXTRACTOR_media_is_media_ (ec, 0)) 1433 return; 1434 if (! gst_is_initialized ()) 1435 gst_init (NULL, NULL); 1436 if ( (0 == opt.gallery) || 1437 (! make_gallery (ec, 1438 &opt, 1439 &deadline, 1440 &ret)) ) 1441 ret = make_thumbnail (ec, 1442 &opt, 1443 &deadline); 1444 if ( (0 == ret) && 1445 ('\0' != opt.srt_lang[0]) ) 1446 (void) dump_subtitles (ec, 1447 &opt, 1448 &deadline); 1449 } 1450 1451 1452 /** 1453 * Keep the diagnostics of the media framework out of the output of the 1454 * calling application; a decoder complaining about a broken file is not 1455 * something the application asked to see. 1456 * 1457 * ("force-kill" would be tempting here, to get a fresh process for every 1458 * file, but the host cannot know about that special without loading the 1459 * plugin in-process: it only finds out that the child is gone once it has 1460 * already handed it the next file, which is then silently lost.) 1461 * 1462 * @return special options for this plugin 1463 */ 1464 const char * 1465 EXTRACTOR_thumbnailgst_options (void); 1466 1467 const char * 1468 EXTRACTOR_thumbnailgst_options () 1469 { 1470 return "close-stderr"; 1471 } 1472 1473 1474 /* end of thumbnailgst_extractor.c */