-misc hacking on train

1 files changed, 2 insertions, 574 deletions

diff --git a/src/main/extractor.c b/src/main/extractor.c
index 1f1c561..d5dbd4d 100644
--- a/src/main/extractor.c
+++ b/src/main/extractor.c
@@ -113,35 +113,6 @@
  */
 #define OPMODE_FILE 3
 
-/**
- * Header used for our IPC replies.  A header
- * with all fields being zero is used to indicate
- * the end of the stream.
- */
-struct IpcHeader
-{
-  /**
-   * Type of the meta data.
-   */
-  enum EXTRACTOR_MetaType meta_type;
-
-  /**
-   * Format of the meta data.
-   */
-  enum EXTRACTOR_MetaFormat meta_format;
-
-  /**
-   * Number of bytes of meta data (value)
-   */
-  size_t data_len;
-  
-  /**
-   * Length of the mime type string describing the meta data value's mime type,
-   * including 0-terminator, 0 for mime type of "NULL".
-   */
-  size_t mime_len;
-};
-
 
 /**
  * Writes 'size' bytes from 'buf' to 'fd', returns only when
@@ -173,345 +144,12 @@ write_all (int fd,
 }
 
 
-/**
- * Function called by a plugin in a child process.  Transmits
- * the meta data back to the parent process.
- *
- * @param cls closure, "int*" of the FD for transmission
- * @param plugin_name name of the plugin that produced this value;
- *        special values can be used (i.e. '<zlib>' for zlib being
- *        used in the main libextractor library and yielding
- *        meta data).
- * @param type libextractor-type describing the meta data
- * @param format basic format information about data 
- * @param data_mime_type mime-type of data (not of the original file);
- *        can be NULL (if mime-type is not known)
- * @param data actual meta-data found
- * @param data_len number of bytes in data
- * @return 0 to continue extracting, 1 to abort (transmission error)
- */ 
-static int
-transmit_reply (void *cls,
-                const char *plugin_name,
-                enum EXTRACTOR_MetaType type,
-                enum EXTRACTOR_MetaFormat format,
-                const char *data_mime_type,
-                const char *data,
-                size_t data_len)
-{
-  static const unsigned char meta_byte = MESSAGE_META;
-  int *cpipe_out = cls;
-  struct IpcHeader hdr;
-  size_t mime_len;
-
-  if (NULL == data_mime_type)
-    mime_len = 0;
-  else
-    mime_len = strlen (data_mime_type) + 1;
-  if (mime_len > MAX_MIME_LEN)
-    mime_len = MAX_MIME_LEN;
-  hdr.meta_type = type;
-  hdr.meta_format = format;
-  hdr.data_len = data_len;
-  hdr.mime_len = mime_len;
-  if ( (sizeof (meta_byte) != 
-        write_all (*cpipe_out,
-                   &meta_byte, sizeof (meta_byte))) ||
-       (sizeof (hdr) != 
-        write_all (*cpipe_out, 
-                   &hdr, sizeof (hdr))) ||
-       (mime_len !=
-        write_all (*cpipe_out, 
-                   data_mime_type, mime_len)) ||
-       (data_len !=
-        write_all (*cpipe_out, 
-                   data, data_len)) )
-    return 1;
-  return 0;
-}
-
-
-/**
- * Main loop function for plugins.
- * Reads a message from the plugin input pipe and acts on it.
- * Can be called recursively (once) in OPMODE_DECOMPRESS.
- * plugin->waiting_for_update == 1 indicates the recursive call.
- *
- * @param plugin plugin context
- * @return 0, always
- */ 
-static int
-process_requests (struct EXTRACTOR_PluginList *plugin)
-{
-  int in, out;
-  int read_result1, read_result2, read_result3, read_result4;
-  unsigned char code;
-  char *shm_name = NULL;
-  size_t shm_name_len;
-  int extract_reply;
-  struct IpcHeader hdr;
-  int do_break;
-#ifdef WINDOWS
-  HANDLE map;
-  MEMORY_BASIC_INFORMATION mi;
-#endif
-
-  in = plugin->pipe_in;
-  out = plugin->cpipe_out;
-
-  /* The point of recursing into this function is to request
-   * a seek from LE server and wait for a reply. This snipper
-   * requests a seek.
-   */
-  if (plugin->waiting_for_update == 1)
-  {
-    unsigned char seek_byte = MESSAGE_SEEK;
-    if (write (out, &seek_byte, 1) != 1)
-      return -1;
-    if (write (out, &plugin->seek_request, sizeof (int64_t)) != sizeof (int64_t))
-      return -1;
-  }
-
-  memset (&hdr, 0, sizeof (hdr));
-  do_break = 0;
-  while (!do_break)
-  {
-    read_result1 = read (in, &code, 1);
-    if (read_result1 <= 0)
-      break;
-    switch (code)
-    {
-    case MESSAGE_INIT_STATE:
-      read_result2 = read (in, &plugin->operation_mode, sizeof (uint8_t));
-      read_result3 = read (in, &plugin->fsize, sizeof (int64_t));
-      read_result4 = read (in, &shm_name_len, sizeof (size_t));
-      if ((read_result2 < sizeof (uint8_t)) ||
-          (read_result3 < sizeof (int64_t)) ||
-          (read_result4 < sizeof (size_t)))
-      {
-        do_break = 1;
-        break;
-      }
-      if (plugin->operation_mode != OPMODE_MEMORY &&
-          plugin->operation_mode != OPMODE_DECOMPRESS &&
-          plugin->operation_mode != OPMODE_FILE)
-      {
-        do_break = 1;
-        break;
-      }
-      if ((plugin->operation_mode == OPMODE_MEMORY ||
-          plugin->operation_mode == OPMODE_DECOMPRESS) &&
-          shm_name_len > MAX_SHM_NAME)
-      {
-        do_break = 1;
-        break;
-      }
-      /* Fsize may be -1 only in decompression mode */
-      if (plugin->operation_mode != OPMODE_DECOMPRESS && plugin->fsize <= 0)
-      {
-        do_break = 1;
-        break;
-      }
-      if (shm_name != NULL)
-        free (shm_name);
-      shm_name = malloc (shm_name_len);
-      if (shm_name == NULL)
-      {
-        do_break = 1;
-        break;
-      }
-      read_result2 = read (in, shm_name, shm_name_len);
-      if (read_result2 < shm_name_len)
-      {
-        do_break = 1;
-        break;
-      }
-      shm_name[shm_name_len - 1] = '\0';
-      do_break = init_state_method (plugin, plugin->operation_mode, plugin->fsize, shm_name);
-      /* in OPMODE_MEMORY and OPMODE_FILE we can start extracting right away,
-       * there won't be UPDATED_SHM message, and we don't need it
-       */
-      if (!do_break && (plugin->operation_mode == OPMODE_MEMORY ||
-          plugin->operation_mode == OPMODE_FILE))
-      {
-        extract_reply = plugin->extract_method (plugin, transmit_reply, &out);
-        unsigned char done_byte = MESSAGE_DONE;
-        if (write (out, &done_byte, 1) != 1)
-        {
-          do_break = 1;
-          break;
-        }
-        if ((plugin->specials != NULL) &&
-            (NULL != strstr (plugin->specials, "force-kill")))
-        {
-          /* we're required to die after each file since this
-             plugin only supports a single file at a time */
-#if !WINDOWS
-          fsync (out);
-#else
-          _commit (out);
-#endif
-          _exit (0);
-        }
-      }
-      break;
-    case MESSAGE_DISCARD_STATE:
-      discard_state_method (plugin);
-      break;
-    case MESSAGE_UPDATED_SHM:
-      if (plugin->operation_mode == OPMODE_DECOMPRESS)
-      {
-        read_result2 = read (in, &plugin->fpos, sizeof (int64_t));
-        read_result3 = read (in, &plugin->map_size, sizeof (size_t));
-        read_result4 = read (in, &plugin->fsize, sizeof (int64_t));
-        if ((read_result2 < sizeof (int64_t)) || (read_result3 < sizeof (size_t)) ||
-            plugin->fpos < 0 || (plugin->operation_mode != OPMODE_DECOMPRESS && (plugin->fsize <= 0 || plugin->fpos >= plugin->fsize)))
-        {
-          do_break = 1;
-          break;
-        }
-        /* FIXME: also check mapped region size (lseek for *nix, VirtualQuery for W32) */
-        /* Re-map the shm */
-#if !WINDOWS
-        if ((-1 == plugin->shm_id) ||
-            (NULL == (plugin->shm_ptr = mmap (NULL, plugin->map_size, PROT_READ, MAP_SHARED, plugin->shm_id, 0))) ||
-            (plugin->shm_ptr == (void *) -1))
-        {
-          do_break = 1;
-          break;
-        }
-#else
-        if ((plugin->map_handle == 0) ||
-           (NULL == (plugin->shm_ptr = MapViewOfFile (plugin->map_handle, FILE_MAP_READ, 0, 0, 0))))
-        {
-          do_break = 1;
-          break;
-        }
-#endif
-        if (plugin->waiting_for_update == 1)
-        {
-          /* We were only waiting for this one message */
-          do_break = 1;
-          plugin->waiting_for_update = 2;
-          break;
-        }
-        /* Run extractor on mapped region (recursive call doesn't reach this
-         * point and breaks out earlier.
-         */
-        extract_reply = plugin->extract_method (plugin, transmit_reply, &out);
-        /* Unmap the shm */
-#if !WINDOWS
-        if ((plugin->shm_ptr != NULL) &&
-            (plugin->shm_ptr != (void*) -1) )
-          munmap (plugin->shm_ptr, plugin->map_size);
-#else
-        if (plugin->shm_ptr != NULL)
-          UnmapViewOfFile (plugin->shm_ptr);
-#endif
-        plugin->shm_ptr = NULL;
-        if (extract_reply == 1)
-        {
-          /* Tell LE that we're done */
-          unsigned char done_byte = MESSAGE_DONE;
-          if (write (out, &done_byte, 1) != 1)
-          {
-            do_break = 1;
-            break;
-          }
-          if ((plugin->specials != NULL) &&
-              (NULL != strstr (plugin->specials, "force-kill")))
-          {
-            /* we're required to die after each file since this
-               plugin only supports a single file at a time */
-#if !WINDOWS
-            fsync (out);
-#else
-            _commit (out);
-#endif
-            _exit (0);
-          }
-        }
-        else
-        {
-          /* Tell LE that we're not done, and we need to seek */
-          unsigned char seek_byte = MESSAGE_SEEK;
-          if (write (out, &seek_byte, 1) != 1)
-          {
-            do_break = 1;
-            break;
-          }
-          if (write (out, &plugin->seek_request, sizeof (int64_t)) != sizeof (int64_t))
-          {
-            do_break = 1;
-            break;
-          }
-        }
-      }
-      else
-      {
-        /* This is mostly to safely skip unrelated messages */
-        int64_t t;
-        size_t t2;
-        read_result2 = read (in, &t, sizeof (int64_t));
-        read_result3 = read (in, &t2, sizeof (size_t));
-        read_result4 = read (in, &t, sizeof (int64_t));
-      }
-      break;
-    }
-  }
-  return 0;
-}
-
-
-/**
- * 'main' function of the child process. Loads the plugin,
- * sets up its in and out pipes, then runs the request serving function.
- *
- * @param plugin extractor plugin to use
- * @param in stream to read from
- * @param out stream to write to
- */
-static void
-plugin_main (struct EXTRACTOR_PluginList *plugin, 
-             int in, int out)
-{
-  if (plugin == NULL)
-  {
-    close (in);
-    close (out);
-    return;
-  }
-  if (0 != EXTRACTOR_plugin_load_ (plugin))
-  {
-    close (in);
-    close (out);
-#if DEBUG
-    fprintf (stderr, "Plugin `%s' failed to load!\n", plugin->short_libname);
-#endif
-    return;
-  }  
-  if ((plugin->specials != NULL) &&
-      (NULL != strstr (plugin->specials, "close-stderr")))
-    close (2);
-  if ((plugin->specials != NULL) &&
-      (NULL != strstr (plugin->specials, "close-stdout")))
-    close (1);
-
-  plugin->pipe_in = in;
-  /* Compiler will complain, and it's right. This is a kind of hack...*/
-  plugin->cpipe_out = out;
-  process_requests (plugin);
-
-  close (in);
-  close (out);
-}
-
 
 /**
  * Open a file
  */
 static int 
-file_open(const char *filename, int oflag, ...)
+file_open (const char *filename, int oflag, ...)
 {
   int mode;
   const char *fn;
@@ -794,112 +432,6 @@ pl_pick_next_buffer_at (struct EXTRACTOR_PluginList *plugin,
 }
 
 
-/**
- * Moves current absolute buffer position to @pos in @whence mode.
- * Will move logical position withouth shifting the buffer, if possible.
- * Will not move beyond the end of file.
- *
- * @param plugin plugin context
- * @param pos position to move to
- * @param whence seek mode (SEEK_CUR, SEEK_SET, SEEK_END)
- * @return new absolute position, -1 on error
- */
-static int64_t
-pl_seek (struct EXTRACTOR_PluginList *plugin, int64_t pos, int whence)
-{
-  switch (whence)
-  {
-  case SEEK_CUR:
-    if (plugin->shm_pos + pos < plugin->map_size && plugin->shm_pos + pos >= 0)
-    {
-      plugin->shm_pos += pos;
-      return plugin->fpos + plugin->shm_pos;
-    }
-    if (0 != pl_pick_next_buffer_at (plugin, plugin->fpos + plugin->shm_pos + pos, 1))
-      return -1;
-    plugin->shm_pos += pos;
-    return plugin->fpos + plugin->shm_pos;
-    break;
-  case SEEK_SET:
-    if (pos < 0)
-      return -1;
-    if (pos >= plugin->fpos && pos < plugin->fpos + plugin->map_size)
-    {
-      plugin->shm_pos = pos - plugin->fpos;
-      return pos;
-    }
-    if (0 != pl_pick_next_buffer_at (plugin, pos, 1))
-      return -1;
-    if (pos >= plugin->fpos && pos < plugin->fpos + plugin->map_size)
-    {
-      plugin->shm_pos = pos - plugin->fpos;
-      return pos;
-    }
-    return -1;
-    break;
-  case SEEK_END:
-    while (plugin->fsize == -1)
-    {
-      pl_pick_next_buffer_at (plugin, plugin->fpos + plugin->map_size + pos, 0);
-    }
-    if (plugin->fsize + pos - 1 < plugin->fpos || plugin->fsize + pos - 1 > plugin->fpos + plugin->map_size)
-    {
-      if (0 != pl_pick_next_buffer_at (plugin, plugin->fsize - MAX_READ, 0))
-        return -1;
-    }
-    plugin->shm_pos = plugin->fsize + pos - plugin->fpos;
-    if (plugin->shm_pos < 0)
-      plugin->shm_pos = 0;
-    else if (plugin->shm_pos >= plugin->map_size)
-      plugin->shm_pos = plugin->map_size - 1;
-    return plugin->fpos + plugin->shm_pos - 1;
-    break;
-  }
-  return -1;
-}
-
-
-static int64_t
-pl_get_fsize (struct EXTRACTOR_PluginList *plugin)
-{
-  return plugin->fsize;
-}
-
-
-/**
- * Fills @data with a pointer to the data buffer.
- * Equivalent to read(), except you don't have to allocate and free
- * a buffer, since the data is already in memory.
- * Will move the buffer, if necessary
- *
- * @param plugin plugin context
- * @param data location to store data pointer
- * @param count number of bytes to read
- * @return number of bytes (<= count) avalable in @data, -1 on error
- */
-static int64_t
-pl_read (struct EXTRACTOR_PluginList *plugin, unsigned char **data, size_t count)
-{
-  *data = NULL;
-  if (count > MAX_READ)
-    return -1;
-  if (count > plugin->map_size - plugin->shm_pos)
-  {
-    int64_t actual_count;
-    if (plugin->fpos + plugin->shm_pos != pl_seek (plugin, plugin->fpos + plugin->shm_pos, SEEK_SET))
-      return -1;
-    *data = &plugin->shm_ptr[plugin->shm_pos];
-    actual_count = (count < plugin->map_size - plugin->shm_pos) ? count : (plugin->map_size - plugin->shm_pos);
-    plugin->shm_pos += actual_count;
-    return actual_count;
-  }
-  else
-  {
-    *data = &plugin->shm_ptr[plugin->shm_pos];
-    plugin->shm_pos += count;
-    return count;
-  }
-}
 
 
 /**
@@ -1002,111 +534,7 @@ ask_in_process_plugin (struct EXTRACTOR_PluginList *plugin,
 }
 
 
-/**
- * Receive a reply from plugin (seek request, metadata and done message)
- *
- * @param plugin plugin context
- * @param proc metadata callback
- * @param proc_cls callback cls
- * @return 0 on success, -1 on error
- */
-static int
-receive_reply (struct EXTRACTOR_PluginList *plugin,
-               EXTRACTOR_MetaDataProcessor proc, void *proc_cls)
-{
-  int read_result;
-  unsigned char code;
-  int64_t seek_position;
-  struct IpcHeader hdr;
-  char *mime_type;
-  char *data;
-  int must_read = 1;
-
-  while (must_read)
-    {
-      read_result = plugin_read (plugin, &code, 1);
-      if (read_result < 1)
-        return -1;
-      switch (code)
-        {
-        case MESSAGE_DONE: /* Done */
-          plugin->seek_request = -1;
-          must_read = 0;
-          break;
-        case MESSAGE_SEEK: /* Seek */
-          read_result = plugin_read (plugin, 
-                                     &seek_position, sizeof (int64_t));
-          if (read_result < sizeof (int64_t))
-            return -1;
-          plugin->seek_request = seek_position;
-          must_read = 0;
-          break;
-        case MESSAGE_META: /* Meta */
-          read_result = plugin_read (plugin, 
-                                     &hdr, sizeof (hdr));
-          if (read_result < sizeof (hdr)) 
-            return -1;
-          /* FIXME: check hdr for sanity */
-          if (hdr.data_len > MAX_META_DATA)
-            return -1; /* not allowing more than MAX_META_DATA meta data */
-          if (0 == hdr.mime_len)
-            {
-              mime_type = NULL;
-            }
-          else
-            {
-              if (NULL == (mime_type = malloc (hdr.mime_len)))
-                return -1;
-              read_result = plugin_read (plugin, 
-                                         mime_type, 
-                                         hdr.mime_len);
-              if ( (read_result < hdr.mime_len) ||
-                   ('\0' != mime_type[hdr.mime_len-1]) )
-                {
-                  if (NULL != mime_type)
-                    free (mime_type);
-                  return -1;
-                }
-            }
-          if (0 == hdr.data_len)
-            {
-              data = NULL;
-            }
-          else
-            {
-              if (NULL == (data = malloc (hdr.data_len)))
-                {
-                  if (NULL != mime_type)
-                    free (mime_type);
-                  return -1;
-                }
-              read_result = plugin_read (plugin, 
-                                         data, hdr.data_len);
-              if (read_result < hdr.data_len)
-                {
-                  if (NULL != mime_type)
-                    free (mime_type);
-                  free (data);
-                  return -1;
-                }
-            }
-          read_result = proc (proc_cls, 
-                              plugin->short_libname, 
-                              hdr.meta_type, hdr.meta_format,
-                              mime_type, data, hdr.data_len);
-          if (NULL != mime_type)
-            free (mime_type);
-          if (NULL != data)
-            free (data);
-          if (0 != read_result)
-            return 1;
-          break;
-        default:
-          return -1;
-        }
-    }
-  return 0;
-}
+
 
 
 /**