libextractor

extractor_datasource.c (34409B)

---

 /*
      This file is part of libextractor.
      Copyright (C) 2002, 2003, 2004, 2005, 2006, 2009, 2012 Vidyut Samanta and Christian Grothoff
 
      libextractor is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
      by the Free Software Foundation; either version 3, or (at your
      option) any later version.
 
      libextractor is distributed in the hope that it will be useful, but
      WITHOUT ANY WARRANTY; without even the implied warranty of
      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
      General Public License for more details.
 
      You should have received a copy of the GNU General Public License
      along with libextractor; see the file COPYING.  If not, write to the
      Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
      Boston, MA 02110-1301, USA.
  */
 /**
  * @file main/extractor_datasource.c
  * @brief random access and possibly decompression of data from buffer in memory or file on disk
  * @author Christian Grothoff
  */
 #include "platform.h"
 #include "extractor_logging.h"
 #include "extractor_datasource.h"
 
 #if HAVE_LIBBZ2
 #include <bzlib.h>
 #define MIN_BZ2_HEADER 4
 #ifndef MIN_COMPRESSED_HEADER
 #define MIN_COMPRESSED_HEADER MIN_ZLIB_HEADER
 #endif
 #endif
 
 #if HAVE_ZLIB
 #include <zlib.h>
 #define MIN_ZLIB_HEADER 12
 #ifndef MIN_COMPRESSED_HEADER
 #define MIN_COMPRESSED_HEADER MIN_BZ2_HEADER
 #endif
 #endif
 
 #ifndef MIN_COMPRESSED_HEADER
 #define MIN_COMPRESSED_HEADER -1
 #endif
 
 #ifndef O_LARGEFILE
 #define O_LARGEFILE 0
 #endif
 
 /**
  * Maximum size of an IO buffer.
  */
 #define MAX_READ (4 * 1024 * 1024)
 
 /**
  * Data is read from the source and shoved into decompressor
  * in chunks this big.
  */
 #define COM_CHUNK_SIZE (16 * 1024)
 
 
 /**
  * Enum with the various possible types of compression supported.
  */
 enum ExtractorCompressionType
 {
   /**
    * We cannot tell from the data (header incomplete).
    */
   COMP_TYPE_UNDEFINED = -1,
 
   /**
    * Invalid header (likely uncompressed)
    */
   COMP_TYPE_INVALID = 0,
 
   /**
    * libz / gzip compression.
    */
   COMP_TYPE_ZLIB = 1,
 
   /**
    * bz2 compression
    */
   COMP_TYPE_BZ2 = 2
 };
 
 
 /**
  * Abstraction of the data source (file or a memory buffer)
  * for the decompressor.
  */
 struct BufferedFileDataSource
 {
   /**
    * Pointer to the buffer to read from (may be NULL)
    */
   const void *data;
 
   /**
    * A buffer to read into. For fd != -1: when data != NULL,
    * data is used directly.
    */
   void *buffer;
 
   /**
    * Size of the file (or the data buffer)
    */
   uint64_t fsize;
 
   /**
    * Position of the buffer in the file.
    */
   uint64_t fpos;
 
   /**
    * Position within the buffer.  Our absolute offset in the file
    * is thus 'fpos + buffer_pos'.
    */
   size_t buffer_pos;
 
   /**
    * Number of valid bytes in the buffer (<= buffer_size)
    */
   size_t buffer_bytes;
 
   /**
    * Allocated size of the buffer
    */
   size_t buffer_size;
 
   /**
    * Descriptor of the file to read data from (may be -1)
    */
   int fd;
 
 };
 
 
 /**
  * An object from which uncompressed data can be read
  */
 struct CompressedFileSource
 {
   /**
    * The source of data
    */
   struct BufferedFileDataSource *bfds;
 
   /**
    * Decompression target buffer.
    */
   char result[COM_CHUNK_SIZE];
 
   /**
    * At which offset in 'result' is 'fpos'?
    */
   size_t result_pos;
 
   /**
    * Size of the source (same as bfds->fsize)
    */
   int64_t fsize;
 
   /**
    * Position within the (decompressed) source
    */
   int64_t fpos;
 
   /**
    * Total size of the uncompressed data. Remains -1 until
    * decompression is finished.
    */
   int64_t uncompressed_size;
 
 #if HAVE_LIBBZ2
   /**
    * BZ2 stream object
    */
   bz_stream bstrm;
 #endif
 
 #if HAVE_ZLIB
   /**
    * ZLIB stream object
    */
   z_stream strm;
 
   /**
    * Length of gzip header (may be 0, in that case ZLIB parses the header)
    */
   int gzip_header_length;
 #endif
 
   /**
    * The type of compression used in the source
    */
   enum ExtractorCompressionType compression_type;
 
 };
 
 
 /**
  * Makes bfds seek to 'pos' and read a chunk of bytes there.
  * Changes bfds->fpos, bfds->buffer_bytes and bfds->buffer_pos.
  * Does almost nothing for memory-backed bfds.
  *
  * @param bfds bfds
  * @param pos position
  * @return 0 on success, -1 on error
  */
 static int
 bfds_pick_next_buffer_at (struct BufferedFileDataSource *bfds,
                           uint64_t pos)
 {
   int64_t position;
   ssize_t rd;
 
   if (pos > bfds->fsize)
   {
     LOG ("Invalid seek operation\n");
     return -1;   /* invalid */
   }
   if (NULL == bfds->buffer)
   {
     bfds->buffer_pos = pos;
     return 0;
   }
   position = (int64_t) lseek (bfds->fd, pos, SEEK_SET);
   if (position < 0)
   {
     LOG_STRERROR ("lseek");
     return -1;
   }
   bfds->fpos = position;
   bfds->buffer_pos = 0;
   rd = read (bfds->fd, bfds->buffer, bfds->buffer_size);
   if (rd < 0)
   {
     LOG_STRERROR ("read");
     return -1;
   }
   bfds->buffer_bytes = rd;
   return 0;
 }
 
 
 /**
  * Creates a bfds
  *
  * @param data data buffer to use as a source (NULL if fd != -1)
  * @param fd file descriptor to use as a source (-1 if data != NULL)
  * @param fsize size of the file (or the buffer)
  * @return newly allocated bfds
  */
 static struct BufferedFileDataSource *
 bfds_new (const void *data,
           int fd,
           int64_t fsize)
 {
   struct BufferedFileDataSource *result;
   size_t xtra;
 
   if (fsize > MAX_READ)
     xtra = MAX_READ;
   else
     xtra = (size_t) fsize;
   if ( (-1 == fd) && (NULL == data) )
   {
     LOG ("Invalid arguments\n");
     return NULL;
   }
   if ( (-1 != fd) && (NULL != data) )
     fd = -1; /* don't need fd */
   if (NULL != data)
     xtra = 0;
   if (NULL == (result = malloc (sizeof (struct BufferedFileDataSource) + xtra)))
   {
     LOG_STRERROR ("malloc");
     return NULL;
   }
   memset (result, 0, sizeof (struct BufferedFileDataSource));
   result->data = (NULL != data) ? data : &result[1];
   result->buffer = (NULL != data) ? NULL : &result[1];
   result->buffer_size = (NULL != data) ? fsize : xtra;
   result->buffer_bytes = (NULL != data) ? fsize : 0;
   result->fsize = fsize;
   result->fd = fd;
   bfds_pick_next_buffer_at (result, 0);
   return result;
 }
 
 
 /**
  * Unallocates bfds
  *
  * @param bfds bfds to deallocate
  */
 static void
 bfds_delete (struct BufferedFileDataSource *bfds)
 {
   free (bfds);
 }
 
 
 /**
  * Makes bfds seek to 'pos' in 'whence' mode.
  * Will try to seek within the buffer, will move the buffer location if
  * the seek request falls outside of the buffer range.
  *
  * @param bfds bfds
  * @param pos position to seek to
  * @param whence one of the seek constants (SEEK_CUR, SEEK_SET, SEEK_END)
  * @return new absolute position, -1 on error
  */
 static int64_t
 bfds_seek (struct BufferedFileDataSource *bfds,
            int64_t pos, int whence)
 {
   uint64_t npos;
   size_t nbpos;
 
   switch (whence)
   {
   case SEEK_CUR:
     npos = bfds->fpos + bfds->buffer_pos + pos;
     if (npos > bfds->fsize)
     {
       LOG ("Invalid seek operation to %lld from %llu (max is %llu)\n",
            (long long) pos,
            bfds->fpos + bfds->buffer_pos,
            (unsigned long long) bfds->fsize);
       return -1;
     }
     nbpos = bfds->buffer_pos + pos;
     if ( (NULL == bfds->buffer) ||
          (nbpos < bfds->buffer_bytes) )
     {
       bfds->buffer_pos = nbpos;
       return npos;
     }
     if (0 != bfds_pick_next_buffer_at (bfds,
                                        npos))
     {
       LOG ("seek operation failed\n");
       return -1;
     }
     return npos;
   case SEEK_END:
     if (pos > 0)
     {
       LOG ("Invalid seek operation\n");
       return -1;
     }
     if (bfds->fsize < -pos)
     {
       LOG ("Invalid seek operation\n");
       return -1;
     }
     pos = bfds->fsize + pos;
   /* fall-through! */
   case SEEK_SET:
     if (pos < 0)
     {
       LOG ("Invalid seek operation\n");
       return -1;
     }
     if (pos > bfds->fsize)
     {
       LOG ("Invalid seek operation (%lld > %llu) %d\n",
            (long long) pos,
            (unsigned long long) bfds->fsize,
            SEEK_SET == whence);
       return -1;
     }
     if ( (NULL == bfds->buffer) ||
          ( (bfds->fpos <= pos) &&
            (bfds->fpos + bfds->buffer_bytes > pos) ) )
     {
       bfds->buffer_pos = pos - bfds->fpos;
       return pos;
     }
     if (0 != bfds_pick_next_buffer_at (bfds, pos))
     {
       LOG ("seek operation failed\n");
       return -1;
     }
     ASSERT (pos == bfds->fpos + bfds->buffer_pos);
     return pos;
   }
   return -1;
 }
 
 
 /**
  * Fills 'buf_ptr' with a chunk of data. Will
  * fail if 'count' exceeds buffer size.
  *
  * @param bfds bfds
  * @param buf_ptr location to store data
  * @param count number of bytes to read
  * @return number of bytes (<= count) available at location pointed by buf_ptr,
  *         0 for end of stream, -1 on error
  */
 static ssize_t
 bfds_read (struct BufferedFileDataSource *bfds,
            void *buf_ptr,
            size_t count)
 {
   char *cbuf = buf_ptr;
   uint64_t old_off;
   size_t avail;
   size_t ret;
 
   old_off = bfds->fpos + bfds->buffer_pos;
   if (old_off == bfds->fsize)
     return 0; /* end of stream */
   ret = 0;
   while (count > 0)
   {
     if ( (bfds->buffer_bytes == bfds->buffer_pos) &&
          (0 != bfds_pick_next_buffer_at (bfds,
                                          bfds->fpos + bfds->buffer_bytes)) )
     {
       /* revert to original position, invalidate buffer */
       bfds->fpos = old_off;
       bfds->buffer_bytes = 0;
       bfds->buffer_pos = 0;
       LOG ("read operation failed\n");
       return -1; /* getting more failed */
     }
     avail = bfds->buffer_bytes - bfds->buffer_pos;
     if (avail > count)
       avail = count;
     if (0 == avail)
       break;
     memcpy (&cbuf[ret], bfds->data + bfds->buffer_pos, avail);
     bfds->buffer_pos += avail;
     count -= avail;
     ret += avail;
   }
   return ret;
 }
 
 
 #if HAVE_ZLIB
 
 #define FHCRC 0x02
 #define FEXTRA 0x04
 #define FNAME 0x08
 #define FCOMMENT 0x10
 
 /**
  * Initializes gz-decompression object. Might report metadata about
  * compresse stream, if available. Resets the stream to the beginning.
  *
  * @param cfs cfs to initialize
  * @param proc callback for metadata
  * @param proc_cls callback cls
  * @return 1 on success, 0 to terminate extraction, -1 on error
  */
 static int
 cfs_init_decompressor_zlib (struct CompressedFileSource *cfs,
                             EXTRACTOR_MetaDataProcessor proc, void *proc_cls)
 {
   unsigned int gzip_header_length = 10;
   unsigned char hdata[12];
   ssize_t rsize;
 
   if (0 != bfds_seek (cfs->bfds, 0, SEEK_SET))
   {
     LOG ("Failed to seek to offset 0!\n");
     return -1;
   }
   /* Process gzip header */
   rsize = bfds_read (cfs->bfds, hdata, sizeof (hdata));
   if ( (-1 == rsize) ||
        (sizeof (hdata) > (size_t) rsize) )
     return -1;
   if (0 != (hdata[3] & FEXTRA))
     gzip_header_length += 2 + (hdata[10] & 0xff) + ((hdata[11] & 0xff) * 256);
 
   if (0 != (hdata[3] & FNAME))
   {
     /* FNAME set */
     char fname[1024];
     char *cptr;
     size_t len;
     ssize_t buf_bytes;
 
     if (gzip_header_length > bfds_seek (cfs->bfds, gzip_header_length,
                                         SEEK_SET))
     {
       LOG ("Corrupt gzip, failed to seek to end of header\n");
       return -1;
     }
     buf_bytes = bfds_read (cfs->bfds, fname, sizeof (fname));
     if (buf_bytes <= 0)
     {
       LOG ("Corrupt gzip, failed to read filename\n");
       return -1;
     }
     if (NULL == (cptr = memchr (fname, 0, buf_bytes)))
     {
       LOG ("Corrupt gzip, failed to read filename terminator\n");
       return -1;
     }
     len = cptr - fname;
     if ( (NULL != proc) &&
          (0 != proc (proc_cls, "<zlib>", EXTRACTOR_METATYPE_FILENAME,
                      EXTRACTOR_METAFORMAT_C_STRING, "text/plain",
                      fname,
                      len)) )
       return 0; /* done */
     gzip_header_length += len + 1;
   }
 
   if (0 != (hdata[3] & FCOMMENT))
   {
     /* FCOMMENT set */
     char fcomment[1024];
     char *cptr;
     ssize_t buf_bytes;
     size_t len;
 
     if (gzip_header_length > bfds_seek (cfs->bfds, gzip_header_length,
                                         SEEK_SET))
     {
       LOG ("Corrupt gzip, failed to seek to end of header\n");
       return -1;
     }
     buf_bytes = bfds_read (cfs->bfds, fcomment, sizeof (fcomment));
     if (buf_bytes <= 0)
     {
       LOG ("Corrupt gzip, failed to read comment\n");
       return -1;
     }
     if (NULL == (cptr = memchr (fcomment, 0, buf_bytes)))
     {
       LOG ("Corrupt gzip, failed to read comment terminator\n");
       return -1;
     }
     len = cptr - fcomment;
     if ( (NULL != proc) &&
          (0 != proc (proc_cls, "<zlib>", EXTRACTOR_METATYPE_COMMENT,
                      EXTRACTOR_METAFORMAT_C_STRING, "text/plain",
                      (const char *) fcomment,
                      len)) )
       return 0; /* done */
     gzip_header_length += len + 1;
   }
   if (0 != (hdata[3] & FHCRC))
     gzip_header_length += 2;
   memset (&cfs->strm, 0, sizeof (z_stream));
 
 #ifdef ZLIB_VERNUM
   /* zlib will take care of its header */
   gzip_header_length = 0;
 #endif
   cfs->gzip_header_length = gzip_header_length;
 
   if (cfs->gzip_header_length !=
       bfds_seek (cfs->bfds, cfs->gzip_header_length, SEEK_SET))
   {
     LOG ("Failed to seek to start to initialize gzip decompressor\n");
     return -1;
   }
   cfs->strm.avail_out = COM_CHUNK_SIZE;
   /*
    * note: maybe plain inflateInit(&strm) is adequate,
    * it looks more backward-compatible also ;
    *
    * ZLIB_VERNUM isn't defined by zlib version 1.1.4 ;
    * there might be a better check.
    */
   if (Z_OK !=
       inflateInit2 (&cfs->strm,
 #ifdef ZLIB_VERNUM
                     15 + 32
 #else
                     -MAX_WBITS
 #endif
                     ))
   {
     LOG ("Failed to initialize zlib decompression\n");
     return -1;
   }
   return 1;
 }
 
 
 #endif
 
 
 #if HAVE_LIBBZ2
 /**
  * Initializes bz2-decompression object. Might report metadata about
  * compresse stream, if available. Resets the stream to the beginning.
  *
  * @param cfs cfs to initialize
  * @param proc callback for metadata
  * @param proc_cls callback cls
  * @return 1 on success, -1 on error
  */
 static int
 cfs_init_decompressor_bz2 (struct CompressedFileSource *cfs,
                            EXTRACTOR_MetaDataProcessor proc,
                            void *proc_cls)
 {
   if (0 !=
       bfds_seek (cfs->bfds,
                  0,
                  SEEK_SET))
   {
     LOG ("Failed to seek to start to initialize BZ2 decompressor\n");
     return -1;
   }
   memset (&cfs->bstrm,
           0,
           sizeof (bz_stream));
   if (BZ_OK !=
       BZ2_bzDecompressInit (&cfs->bstrm,
                             0,
                             0))
   {
     LOG ("Failed to initialize BZ2 decompressor\n");
     return -1;
   }
   cfs->bstrm.avail_out = COM_CHUNK_SIZE;
   return 1;
 }
 
 
 #endif
 
 
 /**
  * Initializes decompression object. Might report metadata about
  * compresse stream, if available. Resets the stream to the beginning.
  *
  * @param cfs cfs to initialize
  * @param proc callback for metadata
  * @param proc_cls callback cls
  * @return 1 on success, 0 to terminate extraction, -1 on error
  */
 static int
 cfs_init_decompressor (struct CompressedFileSource *cfs,
                        EXTRACTOR_MetaDataProcessor proc,
                        void *proc_cls)
 {
   cfs->result_pos = 0;
   cfs->fpos = 0;
   switch (cfs->compression_type)
   {
 #if HAVE_ZLIB
   case COMP_TYPE_ZLIB:
     return cfs_init_decompressor_zlib (cfs, proc, proc_cls);
 #endif
 #if HAVE_LIBBZ2
   case COMP_TYPE_BZ2:
     return cfs_init_decompressor_bz2 (cfs, proc, proc_cls);
 #endif
   default:
     LOG ("invalid compression type selected\n");
     return -1;
   }
 }
 
 
 #if HAVE_ZLIB
 /**
  * Deinitializes gz-decompression object.
  *
  * @param cfs cfs to deinitialize
  * @return 1 on success, -1 on error
  */
 static int
 cfs_deinit_decompressor_zlib (struct CompressedFileSource *cfs)
 {
   inflateEnd (&cfs->strm);
   return 1;
 }
 
 
 #endif
 
 
 #if HAVE_LIBBZ2
 /**
  * Deinitializes bz2-decompression object.
  *
  * @param cfs cfs to deinitialize
  * @return 1 on success, -1 on error
  */
 static int
 cfs_deinit_decompressor_bz2 (struct CompressedFileSource *cfs)
 {
   BZ2_bzDecompressEnd (&cfs->bstrm);
   return 1;
 }
 
 
 #endif
 
 
 /**
  * Deinitializes decompression object.
  *
  * @param cfs cfs to deinitialize
  * @return 1 on success, -1 on error
  */
 static int
 cfs_deinit_decompressor (struct CompressedFileSource *cfs)
 {
   switch (cfs->compression_type)
   {
 #if HAVE_ZLIB
   case COMP_TYPE_ZLIB:
     return cfs_deinit_decompressor_zlib (cfs);
 #endif
 #if HAVE_LIBBZ2
   case COMP_TYPE_BZ2:
     return cfs_deinit_decompressor_bz2 (cfs);
 #endif
   default:
     LOG ("invalid compression type selected\n");
     return -1;
   }
 }
 
 
 /**
  * Resets the compression stream to begin uncompressing
  * from the beginning. Used at initialization time, and when
  * seeking backward.
  *
  * @param cfs cfs to reset
  * @return 1 on success, 0 to terminate extraction,
  *        -1 on error
  */
 static int
 cfs_reset_stream (struct CompressedFileSource *cfs)
 {
   if (-1 == cfs_deinit_decompressor (cfs))
     return -1;
   return cfs_init_decompressor (cfs, NULL, NULL);
 }
 
 
 /**
  * Destroy compressed file source.
  *
  * @param cfs source to destroy
  */
 static void
 cfs_destroy (struct CompressedFileSource *cfs)
 {
   cfs_deinit_decompressor (cfs);
   free (cfs);
 }
 
 
 /**
  * Allocates and initializes new cfs object.
  *
  * @param bfds data source to use
  * @param fsize size of the source
  * @param compression_type type of compression used
  * @param proc metadata callback to call with meta data found upon opening
  * @param proc_cls callback cls
  * @return newly allocated cfs on success, NULL on error
  */
 static struct CompressedFileSource *
 cfs_new (struct BufferedFileDataSource *bfds,
          int64_t fsize,
          enum ExtractorCompressionType compression_type,
          EXTRACTOR_MetaDataProcessor proc, void *proc_cls)
 {
   struct CompressedFileSource *cfs;
 
   if (NULL == (cfs = malloc (sizeof (struct CompressedFileSource))))
   {
     LOG_STRERROR ("malloc");
     return NULL;
   }
   memset (cfs, 0, sizeof (struct CompressedFileSource));
   cfs->compression_type = compression_type;
   cfs->bfds = bfds;
   cfs->fsize = fsize;
   cfs->uncompressed_size = -1;
   if (1 != cfs_init_decompressor (cfs,
                                   proc, proc_cls))
   {
     free (cfs);
     return NULL;
   }
   return cfs;
 }
 
 
 #if HAVE_ZLIB
 /**
  * Fills 'data' with new uncompressed data.  Does the actual
  * decompression. Will set uncompressed_size on the end of compressed
  * stream.
  *
  * @param cfds cfs to read from
  * @param data where to copy the data
  * @param size number of bytes available in data
  * @return number of bytes in data. 0 if no more data can be uncompressed, -1 on error
  */
 static ssize_t
 cfs_read_zlib (struct CompressedFileSource *cfs,
                void *data,
                size_t size)
 {
   char *dst = data;
   int ret;
   size_t rc;
   ssize_t in;
   unsigned char buf[COM_CHUNK_SIZE];
 
   if (cfs->fpos == cfs->uncompressed_size)
   {
     /* end of file */
     return 0;
   }
   rc = 0;
   if (COM_CHUNK_SIZE > cfs->strm.avail_out + cfs->result_pos)
   {
     /* got left-over decompressed data from previous round! */
     in = COM_CHUNK_SIZE - (cfs->strm.avail_out + cfs->result_pos);
     if (in > size)
       in = size;
     memcpy (&dst[rc], &cfs->result[cfs->result_pos], in);
     cfs->fpos += in;
     cfs->result_pos += in;
     rc += in;
   }
   ret = Z_OK;
   while ( (rc < size) && (Z_STREAM_END != ret) )
   {
     /* read block from original data source */
     in = bfds_read (cfs->bfds,
                     buf, sizeof (buf));
     if (in < 0)
     {
       LOG ("unexpected EOF\n");
       return -1; /* unexpected EOF */
     }
     if (0 == in)
     {
       cfs->uncompressed_size = cfs->fpos;
       return rc;
     }
     cfs->strm.next_in = buf;
     cfs->strm.avail_in = (uInt) in;
     cfs->strm.next_out = (unsigned char *) cfs->result;
     cfs->strm.avail_out = COM_CHUNK_SIZE;
     cfs->result_pos = 0;
     ret = inflate (&cfs->strm, Z_SYNC_FLUSH);
     if ( (Z_OK != ret) && (Z_STREAM_END != ret) )
     {
       LOG ("unexpected gzip inflate error: %d\n", ret);
       return -1; /* unexpected error */
     }
     /* go backwards by the number of bytes left in the buffer */
     if (-1 == bfds_seek (cfs->bfds, -(int64_t) cfs->strm.avail_in, SEEK_CUR))
     {
       LOG ("seek failed\n");
       return -1;
     }
     /* copy decompressed bytes to target buffer */
     in = COM_CHUNK_SIZE - cfs->strm.avail_out;
     if (in > size - rc)
     {
       if (Z_STREAM_END == ret)
       {
         cfs->uncompressed_size = cfs->fpos + in;
         ret = Z_OK;
       }
       in = size - rc;
     }
     memcpy (&dst[rc], &cfs->result[cfs->result_pos], in);
     cfs->fpos += in;
     cfs->result_pos += in;
     rc += in;
   }
   if (Z_STREAM_END == ret)
   {
     cfs->uncompressed_size = cfs->fpos;
   }
   return rc;
 }
 
 
 #endif
 
 
 #if HAVE_LIBBZ2
 /**
  * Fills 'data' with new uncompressed data.  Does the actual
  * decompression. Will set uncompressed_size on the end of compressed
  * stream.
  *
  * @param cfds cfs to read from
  * @param data where to copy the data
  * @param size number of bytes available in data
  * @return number of bytes in data. 0 if no more data can be uncompressed, -1 on error
  */
 static ssize_t
 cfs_read_bz2 (struct CompressedFileSource *cfs,
               void *data,
               size_t size)
 {
   char *dst = data;
   int ret;
   size_t rc;
   ssize_t in;
   char buf[COM_CHUNK_SIZE];
 
   if (cfs->fpos == cfs->uncompressed_size)
   {
     /* end of file */
     return 0;
   }
   rc = 0;
   if (COM_CHUNK_SIZE > cfs->bstrm.avail_out + cfs->result_pos)
   {
     /* got left-over decompressed data from previous round! */
     in = COM_CHUNK_SIZE - (cfs->bstrm.avail_out + cfs->result_pos);
     if (in > size)
       in = size;
     memcpy (&dst[rc], &cfs->result[cfs->result_pos], in);
     cfs->fpos += in;
     cfs->result_pos += in;
     rc += in;
   }
   ret = BZ_OK;
   while ( (rc < size) && (BZ_STREAM_END != ret) )
   {
     /* read block from original data source */
     in = bfds_read (cfs->bfds,
                     buf, sizeof (buf));
     if (in < 0)
     {
       LOG ("unexpected EOF\n");
       return -1; /* unexpected EOF */
     }
     if (0 == in)
     {
       cfs->uncompressed_size = cfs->fpos;
       return rc;
     }
     cfs->bstrm.next_in = buf;
     cfs->bstrm.avail_in = (unsigned int) in;
     cfs->bstrm.next_out = cfs->result;
     cfs->bstrm.avail_out = COM_CHUNK_SIZE;
     cfs->result_pos = 0;
     ret = BZ2_bzDecompress (&cfs->bstrm);
     if ( (BZ_OK != ret) && (BZ_STREAM_END != ret) )
     {
       LOG ("unexpected bzip2 decompress error: %d\n", ret);
       return -1; /* unexpected error */
     }
     /* go backwards by the number of bytes left in the buffer */
     if (-1 == bfds_seek (cfs->bfds, -(int64_t) cfs->bstrm.avail_in, SEEK_CUR))
     {
       LOG ("seek failed\n");
       return -1;
     }
     /* copy decompressed bytes to target buffer */
     in = COM_CHUNK_SIZE - cfs->bstrm.avail_out;
     if (in > size - rc)
     {
       if (BZ_STREAM_END == ret)
       {
         cfs->uncompressed_size = cfs->fpos + in;
         ret = BZ_OK;
       }
       in = size - rc;
     }
     memcpy (&dst[rc], &cfs->result[cfs->result_pos], in);
     cfs->fpos += in;
     cfs->result_pos += in;
     rc += in;
   }
   if (BZ_STREAM_END == ret)
   {
     cfs->uncompressed_size = cfs->fpos;
   }
   return rc;
 }
 
 
 #endif
 
 
 /**
  * Fills 'data' with new uncompressed data.  Does the actual
  * decompression. Will set uncompressed_size on the end of compressed
  * stream.
  *
  * @param cfds cfs to read from
  * @param data where to copy the data
  * @param size number of bytes available in data
  * @return number of bytes in data. 0 if no more data can be uncompressed, -1 on error
  */
 static ssize_t
 cfs_read (struct CompressedFileSource *cfs,
           void *data,
           size_t size)
 {
   switch (cfs->compression_type)
   {
 #if HAVE_ZLIB
   case COMP_TYPE_ZLIB:
     return cfs_read_zlib (cfs, data, size);
 #endif
 #if HAVE_LIBBZ2
   case COMP_TYPE_BZ2:
     return cfs_read_bz2 (cfs, data, size);
 #endif
   default:
     LOG ("invalid compression type selected\n");
     return -1;
   }
 }
 
 
 /**
  * Moves the buffer to 'position' in uncompressed steam. If position
  * requires seeking backwards beyond the boundaries of the buffer, resets the
  * stream and repeats decompression from the beginning to 'position'.
  *
  * @param cfs cfs to seek on
  * @param position new starting point for the buffer
  * @param whence one of the seek constants (SEEK_CUR, SEEK_SET, SEEK_END)
  * @return new absolute buffer position, -1 on error or EOS
  */
 static int64_t
 cfs_seek (struct CompressedFileSource *cfs,
           int64_t position,
           int whence)
 {
   uint64_t nposition;
   int64_t delta;
 
   switch (whence)
   {
   case SEEK_CUR:
     if (cfs->fpos + position < 0)
     {
       /* underflow */
       LOG ("Invalid seek operation\n");
       return -1;
     }
     if ( (-1 != cfs->uncompressed_size) &&
          (cfs->fpos + position > cfs->uncompressed_size) )
     {
       LOG ("Invalid seek operation\n");
       return -1;
     }
     nposition = cfs->fpos + position;
     break;
   case SEEK_END:
     ASSERT (-1 != cfs->uncompressed_size);
     if (position > 0)
     {
       LOG ("Invalid seek operation\n");
       return -1;
     }
     if (cfs->uncompressed_size < -position)
     {
       LOG ("Invalid seek operation\n");
       return -1;
     }
     nposition = cfs->uncompressed_size + position;
     break;
   case SEEK_SET:
     if (position < 0)
     {
       LOG ("Invalid seek operation\n");
       return -1;
     }
     if ( (-1 != cfs->uncompressed_size) &&
          (cfs->uncompressed_size < position) )
     {
       LOG ("Invalid seek operation\n");
       return -1;
     }
     nposition = (uint64_t) position;
     break;
   default:
     LOG ("Invalid seek operation\n");
     return -1;
   }
   delta = nposition - cfs->fpos;
   if (delta < 0)
   {
     if (cfs->result_pos >= -delta)
     {
       cfs->result_pos += delta;
       cfs->fpos += delta;
       delta = 0;
     }
     else
     {
       if (-1 == cfs_reset_stream (cfs))
       {
         LOG ("Failed to restart compressed stream for seek operation\n");
         return -1;
       }
       delta = nposition;
     }
   }
   while (delta > 0)
   {
     char buf[COM_CHUNK_SIZE];
     size_t max;
     int64_t ret;
 
     max = (sizeof (buf) > delta) ? delta : sizeof (buf);
     ret = cfs_read (cfs, buf, max);
     if (-1 == ret)
     {
       LOG ("Failed to read decompressed stream for seek operation\n");
       return -1;
     }
     if (0 == ret)
     {
       LOG (
         "Reached unexpected end of stream at %llu during seek operation to %llu (%d left)\n",
         (unsigned long long) cfs->fpos,
         (unsigned long long) nposition,
         delta);
       return -1;
     }
     ASSERT (ret <= delta);
     delta -= ret;
   }
   return cfs->fpos;
 }
 
 
 /**
  * Detect if we have compressed data on our hands.
  *
  * @param data pointer to a data buffer or NULL (in case fd is not -1)
  * @param fd a file to read data from, or -1 (if data is not NULL)
  * @param fsize size of data (if data is not NULL) or of file (if fd is not -1)
  * @return -1 to indicate an error, 0 to indicate uncompressed data, or a type (> 0) of compression
  */
 static enum ExtractorCompressionType
 get_compression_type (struct BufferedFileDataSource *bfds)
 {
   unsigned char read_data[3];
 
   if (0 != bfds_seek (bfds, 0, SEEK_SET))
     return COMP_TYPE_INVALID;
   if (sizeof (read_data) !=
       bfds_read (bfds, read_data, sizeof (read_data)))
     return COMP_TYPE_UNDEFINED;
 
 #if HAVE_ZLIB
   if ( (bfds->fsize >= MIN_ZLIB_HEADER) &&
        (read_data[0] == 0x1f) &&
        (read_data[1] == 0x8b) &&
        (read_data[2] == 0x08) )
     return COMP_TYPE_ZLIB;
 #endif
 #if HAVE_LIBBZ2
   if ( (bfds->fsize >= MIN_BZ2_HEADER) &&
        (read_data[0] == 'B') &&
        (read_data[1] == 'Z') &&
        (read_data[2] == 'h'))
     return COMP_TYPE_BZ2;
 #endif
   return COMP_TYPE_INVALID;
 }
 
 
 /**
  * Handle to a datasource we can use for the plugins.
  */
 struct EXTRACTOR_Datasource
 {
 
   /**
    * Underlying buffered data source.
    */
   struct BufferedFileDataSource *bfds;
 
   /**
    * Compressed file source (NULL if not applicable).
    */
   struct CompressedFileSource *cfs;
 
   /**
    * Underlying file descriptor, -1 for none.
    */
   int fd;
 };
 
 
 /**
  * Create a datasource from a file on disk.
  *
  * @param filename name of the file on disk
  * @param proc metadata callback to call with meta data found upon opening
  * @param proc_cls callback cls
  * @return handle to the datasource, NULL on error
  */
 struct EXTRACTOR_Datasource *
 EXTRACTOR_datasource_create_from_file_ (const char *filename,
                                         EXTRACTOR_MetaDataProcessor proc,
                                         void *proc_cls)
 {
   struct BufferedFileDataSource *bfds;
   struct EXTRACTOR_Datasource *ds;
   enum ExtractorCompressionType ct;
   int fd;
   struct stat sb;
   int64_t fsize;
   int winmode = 0;
 #if WINDOWS
   winmode = O_BINARY;
 #endif
 
   if (-1 == (fd = open (filename, O_RDONLY | O_LARGEFILE | winmode)))
   {
     LOG_STRERROR_FILE ("open", filename);
     return NULL;
   }
   if ( (0 != fstat (fd, &sb)) ||
        (S_ISDIR (sb.st_mode)) )
   {
     if (! S_ISDIR (sb.st_mode))
       LOG_STRERROR_FILE ("fstat", filename);
     else
       LOG ("Skipping directory `%s'\n", filename);
     (void) close (fd);
     return NULL;
   }
   fsize = (int64_t) sb.st_size;
   if (0 == fsize)
   {
     (void) close (fd);
     return NULL;
   }
   bfds = bfds_new (NULL, fd, fsize);
   if (NULL == bfds)
   {
     (void) close (fd);
     return NULL;
   }
   if (NULL == (ds = malloc (sizeof (struct EXTRACTOR_Datasource))))
   {
     LOG_STRERROR ("malloc");
     bfds_delete (bfds);
     (void) close (fd);
     return NULL;
   }
   ds->bfds = bfds;
   ds->fd = fd;
   ds->cfs = NULL;
   ct = get_compression_type (bfds);
   if ( (COMP_TYPE_ZLIB == ct) ||
        (COMP_TYPE_BZ2 == ct) )
   {
     ds->cfs = cfs_new (bfds, fsize, ct, proc, proc_cls);
     if (NULL == ds->cfs)
     {
       LOG ("Failed to initialize decompressor\n");
       bfds_delete (bfds);
       free (ds);
       (void) close (fd);
       return NULL;
     }
   }
   return ds;
 }
 
 
 /**
  * Create a datasource from a buffer in memory.
  *
  * @param buf data in memory
  * @param size number of bytes in 'buf'
  * @param proc metadata callback to call with meta data found upon opening
  * @param proc_cls callback cls
  * @return handle to the datasource
  */
 struct EXTRACTOR_Datasource *
 EXTRACTOR_datasource_create_from_buffer_ (const char *buf,
                                           size_t size,
                                           EXTRACTOR_MetaDataProcessor proc,
                                           void *proc_cls)
 {
   struct BufferedFileDataSource *bfds;
   struct EXTRACTOR_Datasource *ds;
   enum ExtractorCompressionType ct;
 
   if (0 == size)
     return NULL;
   if (NULL == (bfds = bfds_new (buf, -1, size)))
   {
     LOG ("Failed to initialize buffer data source\n");
     return NULL;
   }
   if (NULL == (ds = malloc (sizeof (struct EXTRACTOR_Datasource))))
   {
     LOG_STRERROR ("malloc");
     bfds_delete (bfds);
     return NULL;
   }
   ds->bfds = bfds;
   ds->fd = -1;
   ds->cfs = NULL;
   ct = get_compression_type (bfds);
   if ( (COMP_TYPE_ZLIB == ct) ||
        (COMP_TYPE_BZ2 == ct) )
   {
     ds->cfs = cfs_new (bfds, size, ct, proc, proc_cls);
     if (NULL == ds->cfs)
     {
       LOG ("Failed to initialize decompressor\n");
       bfds_delete (bfds);
       free (ds);
       return NULL;
     }
   }
   return ds;
 }
 
 
 /**
  * Destroy a data source.
  *
  * @param ds source to destroy
  */
 void
 EXTRACTOR_datasource_destroy_ (struct EXTRACTOR_Datasource *ds)
 {
   if (NULL != ds->cfs)
     cfs_destroy (ds->cfs);
   bfds_delete (ds->bfds);
   if (-1 != ds->fd)
     (void) close (ds->fd);
   free (ds);
 }
 
 
 /**
  * Make 'size' bytes of data from the data source available at 'data'.
  *
  * @param cls must be a 'struct EXTRACTOR_Datasource'
  * @param data where the data should be copied to
  * @param size maximum number of bytes requested
  * @return number of bytes now available in data (can be smaller than 'size'),
  *         -1 on error
  */
 ssize_t
 EXTRACTOR_datasource_read_ (void *cls,
                             void *data,
                             size_t size)
 {
   struct EXTRACTOR_Datasource *ds = cls;
 
   if (NULL != ds->cfs)
     return cfs_read (ds->cfs, data, size);
   return bfds_read (ds->bfds, data, size);
 }
 
 
 /**
  * Seek in the datasource.  Use 'SEEK_CUR' for whence and 'pos' of 0 to
  * obtain the current position in the file.
  *
  * @param cls must be a 'struct EXTRACTOR_Datasource'
  * @param pos position to seek (see 'man lseek')
  * @param whence how to see (absolute to start, relative, absolute to end)
  * @return new absolute position, UINT64_MAX on error (i.e. desired position
  *         does not exist)
  */
 int64_t
 EXTRACTOR_datasource_seek_ (void *cls,
                             int64_t pos,
                             int whence)
 {
   struct EXTRACTOR_Datasource *ds = cls;
 
   if (NULL != ds->cfs)
   {
     if ( (SEEK_END == whence) &&
          (-1 == ds->cfs->uncompressed_size) )
     {
       /* need to obtain uncompressed size */
       (void) EXTRACTOR_datasource_get_size_ (ds, 1);
       if (-1 == ds->cfs->uncompressed_size)
         return -1;
     }
     return cfs_seek (ds->cfs, pos, whence);
   }
   return bfds_seek (ds->bfds, pos, whence);
 }
 
 
 /**
  * Determine the overall size of the data source (after compression).
  *
  * @param cls must be a 'struct EXTRACTOR_Datasource'
  * @param force force computing the size if it is unavailable
  * @return overall file size, -1 on error or unknown
  */
 int64_t
 EXTRACTOR_datasource_get_size_ (void *cls,
                                 int force)
 {
   struct EXTRACTOR_Datasource *ds = cls;
   char buf[32 * 1024];
   uint64_t pos;
 
   if (NULL != ds->cfs)
   {
     if ( (force) &&
          (-1 == ds->cfs->uncompressed_size) )
     {
       pos = ds->cfs->fpos;
       while ( (-1 == ds->cfs->uncompressed_size) &&
               (-1 != cfs_read (ds->cfs, buf, sizeof (buf))) )
         ;
       if (-1 == cfs_seek (ds->cfs, pos, SEEK_SET))
       {
         LOG (
           "Serious problem, I moved the buffer to determine the file size but could not restore it...\n");
         return -1;
       }
       if (-1 == ds->cfs->uncompressed_size)
         return -1;
     }
     return ds->cfs->uncompressed_size;
   }
   return ds->bfds->fsize;
 }
 
 
 /* end of extractor_datasource.c */