3 files changed, 1953 insertions, 1204 deletions

diff --git a/src/main/extractor.c b/src/main/extractor.c
index 17ba1d2..a4ccfa1 100644
--- a/src/main/extractor.c
+++ b/src/main/extractor.c
@@ -23,7 +23,7 @@
 #include "extractor.h"
 #include <dirent.h>
 #include <sys/types.h>
-#ifndef WINDOWS
+#if !WINDOWS
 #include <sys/wait.h>
 #include <sys/shm.h>
 #endif
@@ -59,117 +59,53 @@
  */
 #define MAX_MIME_LEN 256
 
+#define MAX_SHM_NAME 255
+
 /**
  * Set to 1 to get failure info,
  * 2 for actual debug info.
  */ 
 #define DEBUG 1
 
+#define MESSAGE_INIT_STATE 0x01
+#define MESSAGE_UPDATED_SHM 0x02
+#define MESSAGE_DONE 0x03
+#define MESSAGE_SEEK 0x04
+#define MESSAGE_META 0x05
+#define MESSAGE_DISCARD_STATE 0x06
 
 /**
- * Stop the child process of this plugin.
+ * Header used for our IPC replies.  A header
+ * with all fields being zero is used to indicate
+ * the end of the stream.
  */
-static void
-stop_process (struct EXTRACTOR_PluginList *plugin)
+struct IpcHeader
 {
-  int status;
-#ifdef WINDOWS
-  HANDLE process;
-#endif
-
-#if DEBUG
-#ifndef WINDOWS
-  if (plugin->cpid == -1)
-#else
-  if (plugin->hProcess == INVALID_HANDLE_VALUE)
-#endif
-    fprintf (stderr,
-             "Plugin `%s' choked on this input\n",
-             plugin->short_libname);
-#endif
-#ifndef WINDOWS
-  if ( (plugin->cpid == -1) ||
-       (plugin->cpid == 0) )
-    return;
-  kill (plugin->cpid, SIGKILL);
-  waitpid (plugin->cpid, &status, 0);
-  plugin->cpid = -1;
-  close (plugin->cpipe_out);
-  fclose (plugin->cpipe_in);
-#else
-  if (plugin->hProcess == INVALID_HANDLE_VALUE ||
-      plugin->hProcess == NULL)
-  return;
-  TerminateProcess (plugin->hProcess, 0);
-  CloseHandle (plugin->hProcess);
-  plugin->hProcess = INVALID_HANDLE_VALUE;
-  close (plugin->cpipe_out);
-  fclose (plugin->cpipe_in);
-#endif
-  plugin->cpipe_out = -1;
-  plugin->cpipe_in = NULL;
-}
-
+  enum EXTRACTOR_MetaType meta_type;
+  enum EXTRACTOR_MetaFormat meta_format;
+  size_t data_len;
+  size_t mime_len;
+};
 
-/**
- * Remove a plugin from a list.
- *
- * @param prev the current list of plugins
- * @param library the name of the plugin to remove
- * @return the reduced list, unchanged if the plugin was not loaded
- */
-struct EXTRACTOR_PluginList *
-EXTRACTOR_plugin_remove(struct EXTRACTOR_PluginList * prev,
-                        const char * library)
+#if !WINDOWS
+int
+plugin_open_shm (struct EXTRACTOR_PluginList *plugin, char *shm_name)
 {
-  struct EXTRACTOR_PluginList *pos;
-  struct EXTRACTOR_PluginList *first;
-
-  pos = prev;
-  first = prev;
-  while ((pos != NULL) && (0 != strcmp (pos->short_libname, library)))
-    {
-      prev = pos;
-      pos = pos->next;
-    }
-  if (pos != NULL)
-    {
-      /* found, close library */
-      if (first == pos)
-        first = pos->next;
-      else
-        prev->next = pos->next;
-      /* found */
-      stop_process (pos);
-      free (pos->short_libname);
-      free (pos->libname);
-      free (pos->plugin_options);
-      if (NULL != pos->libraryHandle) 
-        lt_dlclose (pos->libraryHandle);      
-      free (pos);
-    }
-#if DEBUG
-  else
-    fprintf(stderr,
-            "Unloading plugin `%s' failed!\n",
-            library);
-#endif
-  return first;
+  if (plugin->shm_id != -1)
+    close (plugin->shm_id);
+  plugin->shm_id = shm_open (shm_name, O_RDONLY, 0);
+  return plugin->shm_id;
 }
-
-
-/**
- * Remove all plugins from the given list (destroys the list).
- *
- * @param plugin the list of plugins
- */
-void 
-EXTRACTOR_plugin_remove_all(struct EXTRACTOR_PluginList *plugins)
+#else
+HANDLE
+plugin_open_shm (struct EXTRACTOR_PluginList *plugin, char *shm_name)
 {
-  while (plugins != NULL)
-    plugins = EXTRACTOR_plugin_remove (plugins, plugins->short_libname);
+  if (plugin->map_handle != 0)
+    CloseHandle (plugin->map_handle);
+  plugin->map_handle = OpenFileMapping (FILE_MAP_READ, FALSE, shm_name);
+  return plugin->map_handle;
 }
-
+#endif
 
 static int
 write_all (int fd,
@@ -187,44 +123,9 @@ write_all (int fd,
         return -1;
       off += ret;
     }
-  return 0;
-}
-
-
-static int
-read_all (int fd,
-          void *buf,
-          size_t size)
-{
-  char *data = buf;
-  size_t off = 0;
-  ssize_t ret;
-  
-  while (off < size)
-    {
-      ret = read (fd, &data[off], size - off);
-      if (ret <= 0)
-        return -1;
-      off += ret;
-    }
-  return 0;
+  return size;
 }
 
-
-/**
- * Header used for our IPC replies.  A header
- * with all fields being zero is used to indicate
- * the end of the stream.
- */
-struct IpcHeader
-{
-  enum EXTRACTOR_MetaType type;
-  enum EXTRACTOR_MetaFormat format;
-  size_t data_len;
-  size_t mime_len;
-};
-
-
 /**
  * Function called by a plugin in a child process.  Transmits
  * the meta data back to the parent process.
@@ -254,6 +155,8 @@ transmit_reply (void *cls,
   int *cpipe_out = cls;
   struct IpcHeader hdr;
   size_t mime_len;
+  unsigned char meta_byte = MESSAGE_META;
+  unsigned char zero_byte = 0;
 
   if (data_mime_type == NULL)
     mime_len = 0;
@@ -261,23 +164,19 @@ transmit_reply (void *cls,
     mime_len = strlen (data_mime_type) + 1;
   if (mime_len > MAX_MIME_LEN)
     mime_len = MAX_MIME_LEN;
-  hdr.type = type;
-  hdr.format = format;
+  hdr.meta_type = type;
+  hdr.meta_format = format;
   hdr.data_len = data_len;
   hdr.mime_len = mime_len;
-  if ( (hdr.type == 0) &&
-       (hdr.format == 0) &&
-       (hdr.data_len == 0) &&
-       (hdr.mime_len == 0) )
-    return 0; /* better skip this one, would signal termination... */    
-  if ( (0 != write_all (*cpipe_out, &hdr, sizeof(hdr))) ||
-       (0 != write_all (*cpipe_out, data_mime_type, mime_len)) ||
-       (0 != write_all (*cpipe_out, data, data_len)) )
-    return 1;  
+  if ((1 != write_all (*cpipe_out, &meta_byte, 1)) ||
+      (sizeof(hdr) != write_all (*cpipe_out, &hdr, sizeof(hdr))) ||
+      (mime_len -1 != write_all (*cpipe_out, data_mime_type, mime_len - 1)) ||
+      (1 != write_all (*cpipe_out, &zero_byte, 1)) ||
+      (data_len != write_all (*cpipe_out, data, data_len)))
+    return 1;
   return 0;
 }
 
-
 /**
  * 'main' function of the child process.  Reads shm-filenames from
  * 'in' (line-by-line) and writes meta data blocks to 'out'.  The meta
@@ -288,23 +187,20 @@ transmit_reply (void *cls,
  * @param out stream to write to
  */
 static void
-process_requests (struct EXTRACTOR_PluginList *plugin,
-                  int in,
-                  int out)
+process_requests (struct EXTRACTOR_PluginList *plugin, int in, int out)
 {
-  char hfn[256];
-  char tfn[256];
-  char sze[256];
-  size_t hfn_len;
-  size_t tfn_len;
-  size_t sze_len;
-  char *fn;
-  FILE *fin;
-  void *ptr;
-  int shmid;
+  int read_result1, read_result2, read_result3;
+  unsigned char code;
+  int64_t fsize = -1;
+  int64_t position = 0;
+  void *shm_ptr = NULL;
+  size_t shm_size = 0;
+  char *shm_name = NULL;
+  size_t shm_name_len;
+
+  int extract_reply;
+
   struct IpcHeader hdr;
-  size_t size;
-  int want_tail;
   int do_break;
 #ifdef WINDOWS
   HANDLE map;
@@ -312,165 +208,554 @@ process_requests (struct EXTRACTOR_PluginList *plugin,
 #endif
 
   if (plugin == NULL)
-    {
-      close (in);
-      close (out);
-      return;
-    }
+  {
+    close (in);
+    close (out);
+    return;
+  }
   if (0 != plugin_load (plugin))
-    {
-      close (in);
-      close (out);
+  {
+    close (in);
+    close (out);
 #if DEBUG
-      fprintf (stderr,
-               "Plugin `%s' failed to load!\n",
-               plugin->short_libname);
+    fprintf (stderr, "Plugin `%s' failed to load!\n", plugin->short_libname);
 #endif
-      return;
-    }  
-  want_tail = 0;
-  if ( (plugin->specials != NULL) &&
-       (NULL != strstr (plugin->specials,
-                        "want-tail")) )
-    {
-      want_tail = 1;
-    }
-  if ( (plugin->specials != NULL) &&
-       (NULL != strstr (plugin->specials,
-                        "close-stderr")) )
-    {
-      close (2);
-    }
-  if ( (plugin->specials != NULL) &&
-       (NULL != strstr (plugin->specials,
-                        "close-stdout")) )
-    {
-      close (1);
-    }
+    return;
+  }  
+  if ((plugin->specials != NULL) &&
+      (NULL != strstr (plugin->specials, "close-stderr")))
+    close (2);
+  if ((plugin->specials != NULL) &&
+      (NULL != strstr (plugin->specials, "close-stdout")))
+    close (1);
 
   memset (&hdr, 0, sizeof (hdr));
-  fin = fdopen (in, "r");
-  if (fin == NULL)
-    {
-      close (in);
-      close (out);
-      return;
-    }
-  while (NULL != fgets (hfn, sizeof(hfn), fin))
+  do_break = 0;
+  while (!do_break)
+  {
+    read_result1 = read (in, &code, 1);
+    if (read_result1 <= 0)
+      break;
+    switch (code)
     {
-      hfn_len = strlen (hfn);
-      if (hfn_len <= 1)
-        break;
-      ptr = NULL;
-      hfn[--hfn_len] = '\0'; /* kill newline */
-      if (NULL == fgets (tfn, sizeof(tfn), fin))
-        break;
-      if ('!' != tfn[0])
-        break;
-      tfn_len = strlen (tfn);
-      tfn[--tfn_len] = '\0'; /* kill newline */
-      if ( (want_tail) &&
-           (tfn_len > 1) )
-        {
-          fn = &tfn[1];
-        }
-      else
-        {
-          fn = hfn;     
-        }
-      if (NULL == fgets (sze, sizeof(sze), fin))
-        break;
-      if ('s' != sze[0])
-        break;
-      sze_len = strlen (sze);
-      sze[--sze_len] = '\0'; /* kill newline */
-      size = strtol (&sze[1], NULL, 10);
-      if (size == LONG_MIN || size == LONG_MAX || size == 0)
+    case MESSAGE_INIT_STATE:
+      read_result2 = read (in, &fsize, sizeof (int64_t));
+      read_result3 = read (in, &shm_name_len, sizeof (size_t));
+      if ((read_result2 < sizeof (int64_t)) || (read_result3 < sizeof (size_t)) ||
+          shm_name_len > MAX_SHM_NAME || fsize <= 0)
+      {
+        do_break = 1;
+        break;
+      }
+      if (shm_name != NULL)
+        free (shm_name);
+      shm_name = malloc (shm_name_len);
+      if (shm_name == NULL)
+      {
+        do_break = 1;
         break;
-      do_break = 0;
-#ifndef WINDOWS
-      if ( (-1 != (shmid = shm_open (fn, O_RDONLY, 0))) &&
-           (SIZE_MAX != (size = lseek (shmid, 0, SEEK_END))) &&
-           (NULL != (ptr = mmap (NULL, size, PROT_READ, MAP_SHARED, shmid, 0))) &&
-           (ptr != (void*) -1) )
+      }
+      read_result2 = read (in, shm_name, shm_name_len);
+      if (read_result2 < shm_name_len)
+      {
+        do_break = 1;
+        break;
+      }
+      shm_name[shm_name_len - 1] = '\0';
+#if !WINDOWS
+      if (shm_ptr != NULL)
+        munmap (shm_ptr, shm_size);
+      if (-1 == plugin_open_shm (plugin, shm_name))
+      {
+        do_break = 1;
+        break;
+      }
 #else
-      /* Despite the obvious, this must be READWRITE, not READONLY */
-      map = OpenFileMapping (PAGE_READWRITE, FALSE, fn);
-      ptr = MapViewOfFile (map, FILE_MAP_READ, 0, 0, 0);
-      if (ptr != NULL)
+      if (shm_ptr != NULL)
+        UnmapViewOfFile (shm_ptr);
+      if (INVALID_HANDLE_VALUE == plugin_open_shm (plugin, shm_name))
       {
-        if (0 == VirtualQuery (ptr, &mi, sizeof (mi)) || mi.RegionSize < size)
-        {
-          UnmapViewOfFile (ptr);
-          ptr = NULL;
-        }
+        do_break = 1;
+        break;
+      }
+#endif
+      plugin->fsize = fsize;
+      plugin->init_state_method (plugin);
+      break;
+    case MESSAGE_DISCARD_STATE:
+      plugin->discard_state_method (plugin);
+#if !WINDOWS
+      if (shm_ptr != NULL && shm_size > 0)
+        munmap (shm_ptr, shm_size);
+      if (plugin->shm_id != -1)
+        close (plugin->shm_id);
+      plugin->shm_id = -1;
+      shm_size = 0;
+#else
+      if (shm_ptr != NULL)
+        UnmapViewOfFile (shm_ptr);
+      if (plugin->map_handle != 0)
+        CloseHandle (plugin->map_handle);
+      plugin->map_handle = 0;
+#endif
+      shm_ptr = NULL;
+      break;
+    case MESSAGE_UPDATED_SHM:
+      read_result2 = read (in, &position, sizeof (int64_t));
+      read_result3 = read (in, &shm_size, sizeof (size_t));
+      if ((read_result2 < sizeof (int64_t)) || (read_result3 < sizeof (size_t)) ||
+          position < 0 || fsize <= 0 || position >= fsize)
+      {
+        do_break = 1;
+        break;
+      }
+      /* FIXME: also check mapped region size (lseek for *nix, VirtualQuery for W32) */
+#if !WINDOWS
+      if ((-1 == plugin->shm_id) ||
+          (NULL == (shm_ptr = mmap (NULL, shm_size, PROT_READ, MAP_SHARED, plugin->shm_id, 0))) ||
+          (shm_ptr == (void *) -1))
+      {
+        do_break = 1;
+        break;
+      }
+#else
+      if ((plugin->map_handle == 0) ||
+         (NULL == (shm_ptr = MapViewOfFile (plugin->map_handle, FILE_MAP_READ, 0, 0, 0))))
+      {
+        do_break = 1;
+        break;
       }
-      if (ptr != NULL)
 #endif
-        {
-          if ( ( (plugin->extractMethod != NULL) &&
-                 (0 != plugin->extractMethod (ptr,
-                                              size,
-                                              &transmit_reply,
-                                              &out,
-                                              plugin->plugin_options)) ) ||
-               (0 != write_all (out, &hdr, sizeof(hdr))) )
-            do_break = 1;
-        }
-#ifndef WINDOWS
-      if ( (ptr != NULL) &&
-           (ptr != (void*) -1) )
-        munmap (ptr, size);
-      if (-1 != shmid)
-        close (shmid);
+      plugin->position = position;
+      plugin->shm_ptr = shm_ptr;
+      plugin->map_size = shm_size;
+      /* Now, ideally a plugin would do reads and seeks on a virtual "plugin" object
+       * completely transparently, and the underlying code would return bytes from
+       * the memory map, or would block and wait for a seek to happen.
+       * That, however, requires somewhat different architecture, and even more wrapping
+       * and hand-helding. It's easier to make plugins aware of the fact that they work
+       * with discrete in-memory buffers with expensive seeking, not continuous files.
+       */
+      extract_reply = plugin->extract_method (plugin, transmit_reply, &out);
+#if !WINDOWS
+      if ((shm_ptr != NULL) &&
+          (shm_ptr != (void*) -1) )
+        munmap (shm_ptr, shm_size);
 #else
-      if (ptr != NULL && ptr != (void*) -1)
-        UnmapViewOfFile (ptr);
-      if (map != NULL)
-        CloseHandle (map);
+      if (shm_ptr != NULL)
+        UnmapViewOfFile (shm_ptr);
 #endif
-      if (do_break)
-        break;
-      if ( (plugin->specials != NULL) &&
-           (NULL != strstr (plugin->specials,
-                            "force-kill")) )
-        {
-          /* we're required to die after each file since this
-             plugin only supports a single file at a time */
-          _exit (0);
-        }
+      if (extract_reply == 1)
+      {
+        unsigned char done_byte = MESSAGE_DONE;
+        if (write (out, &done_byte, 1) != 1)
+        {
+          do_break = 1;
+          break;
+        }
+        if ((plugin->specials != NULL) &&
+            (NULL != strstr (plugin->specials, "force-kill")))
+        {
+          /* we're required to die after each file since this
+             plugin only supports a single file at a time */
+#if !WINDOWS
+          fsync (out);
+#else
+          _commit (out);
+#endif
+          _exit (0);
+        }
+      }
+      else
+      {
+        unsigned char seek_byte = MESSAGE_SEEK;
+        if (write (out, &seek_byte, 1) != 1)
+        {
+          do_break = 1;
+          break;
+        }
+        if (write (out, &plugin->seek_request, sizeof (int64_t)) != sizeof (int64_t))
+        {
+          do_break = 1;
+          break;
+        }
+      }
+      break;
     }
-  fclose (fin);
+  }
+  close (in);
   close (out);
 }
 
+#if !WINDOWS
 
-#ifdef WINDOWS
+/**
+ * Start the process for the given plugin.
+ */ 
 static void
-write_plugin_data (int fd, const struct EXTRACTOR_PluginList *plugin)
+start_process (struct EXTRACTOR_PluginList *plugin)
 {
-  size_t i;
-  DWORD len;
-  char *str;
+  int p1[2];
+  int p2[2];
+  pid_t pid;
+  int status;
 
-  i = strlen (plugin->libname) + 1;
-  write (fd, &i, sizeof (size_t));
-  write (fd, plugin->libname, i);
-  i = strlen (plugin->short_libname) + 1;
-  write (fd, &i, sizeof (size_t));
-  write (fd, plugin->short_libname, i);
-  if (plugin->plugin_options != NULL)
+  switch (plugin->flags)
+  {
+  case EXTRACTOR_OPTION_DEFAULT_POLICY:
+    if (-1 != plugin->cpid && 0 != plugin->cpid)
+      return;
+    break;
+  case EXTRACTOR_OPTION_OUT_OF_PROCESS_NO_RESTART:
+    if (0 != plugin->cpid)
+      return;
+    break;
+  case EXTRACTOR_OPTION_IN_PROCESS:
+    return;
+    break;
+  case EXTRACTOR_OPTION_DISABLED:
+    return;
+    break;
+  }
+
+  plugin->cpid = -1;
+  if (0 != pipe (p1))
+  {
+    plugin->flags = EXTRACTOR_OPTION_DISABLED;
+    return;
+  }
+  if (0 != pipe (p2))
+  {
+    close (p1[0]);
+    close (p1[1]);
+    plugin->flags = EXTRACTOR_OPTION_DISABLED;
+    return;
+  }
+  pid = fork ();
+  plugin->cpid = pid;
+  if (pid == -1)
+  {
+    close (p1[0]);
+    close (p1[1]);
+    close (p2[0]);
+    close (p2[1]);
+    plugin->flags = EXTRACTOR_OPTION_DISABLED;
+    return;
+  }
+  if (pid == 0)
+  {
+    close (p1[1]);
+    close (p2[0]);
+    process_requests (plugin, p1[0], p2[1]);
+    _exit (0);
+  }
+  close (p1[0]);
+  close (p2[1]);
+  plugin->cpipe_in = fdopen (p1[1], "w");
+  if (plugin->cpipe_in == NULL)
+  {
+    perror ("fdopen");
+    (void) kill (plugin->cpid, SIGKILL);
+    waitpid (plugin->cpid, &status, 0);
+    close (p1[1]);
+    close (p2[0]);
+    plugin->cpid = -1;
+    plugin->flags = EXTRACTOR_OPTION_DISABLED;
+    return;
+  }
+  plugin->cpipe_out = p2[0];
+}
+
+/**
+ * Stop the child process of this plugin.
+ */
+static void
+stop_process (struct EXTRACTOR_PluginList *plugin)
+{
+  int status;
+
+#if DEBUG
+  if (plugin->cpid == -1)
+    fprintf (stderr,
+             "Plugin `%s' choked on this input\n",
+             plugin->short_libname);
+#endif
+  if ( (plugin->cpid == -1) ||
+       (plugin->cpid == 0) )
+    return;
+  kill (plugin->cpid, SIGKILL);
+  waitpid (plugin->cpid, &status, 0);
+  plugin->cpid = -1;
+  close (plugin->cpipe_out);
+  fclose (plugin->cpipe_in);
+  plugin->cpipe_out = -1;
+  plugin->cpipe_in = NULL;
+
+  if (plugin->flags != EXTRACTOR_OPTION_DEFAULT_POLICY)
+    plugin->flags = EXTRACTOR_OPTION_DISABLED;
+
+  plugin->seek_request = -1;
+}
+
+static int
+write_plugin_data (const struct EXTRACTOR_PluginList *plugin)
+{
+  /* only does anything on Windows */
+  return 0;
+}
+
+#define plugin_print(plug, fmt, ...) fprintf (plug->cpipe_in, fmt, ...)
+#define plugin_write(plug, buf, size) write_all (fileno (plug->cpipe_in), buf, size)
+
+#else /* WINDOWS */
+
+#ifndef PIPE_BUF
+#define PIPE_BUF 512
+#endif
+
+/* Copyright Bob Byrnes  <byrnes <at> curl.com>
+   http://permalink.gmane.org/gmane.os.cygwin.patches/2121
+*/
+/* Create a pipe, and return handles to the read and write ends,
+   just like CreatePipe, but ensure that the write end permits
+   FILE_READ_ATTRIBUTES access, on later versions of win32 where
+   this is supported.  This access is needed by NtQueryInformationFile,
+   which is used to implement select and nonblocking writes.
+   Note that the return value is either NO_ERROR or GetLastError,
+   unlike CreatePipe, which returns a bool for success or failure.  */
+static int
+create_selectable_pipe (PHANDLE read_pipe_ptr, PHANDLE write_pipe_ptr,
+                        LPSECURITY_ATTRIBUTES sa_ptr, DWORD psize,
+                        DWORD dwReadMode, DWORD dwWriteMode)
+{
+  /* Default to error. */
+  *read_pipe_ptr = *write_pipe_ptr = INVALID_HANDLE_VALUE;
+
+  HANDLE read_pipe = INVALID_HANDLE_VALUE, write_pipe = INVALID_HANDLE_VALUE;
+
+  /* Ensure that there is enough pipe buffer space for atomic writes.  */
+  if (psize < PIPE_BUF)
+    psize = PIPE_BUF;
+
+  char pipename[MAX_PATH];
+
+  /* Retry CreateNamedPipe as long as the pipe name is in use.
+   * Retrying will probably never be necessary, but we want
+   * to be as robust as possible.  */
+  while (1)
+  {
+    static volatile LONG pipe_unique_id;
+
+    snprintf (pipename, sizeof pipename, "\\\\.\\pipe\\gnunet-%d-%ld",
+              getpid (), InterlockedIncrement ((LONG *) & pipe_unique_id));
+    /* Use CreateNamedPipe instead of CreatePipe, because the latter
+     * returns a write handle that does not permit FILE_READ_ATTRIBUTES
+     * access, on versions of win32 earlier than WinXP SP2.
+     * CreatePipe also stupidly creates a full duplex pipe, which is
+     * a waste, since only a single direction is actually used.
+     * It's important to only allow a single instance, to ensure that
+     * the pipe was not created earlier by some other process, even if
+     * the pid has been reused.  We avoid FILE_FLAG_FIRST_PIPE_INSTANCE
+     * because that is only available for Win2k SP2 and WinXP.  */
+    read_pipe = CreateNamedPipeA (pipename, PIPE_ACCESS_INBOUND | dwReadMode, PIPE_TYPE_BYTE | PIPE_READMODE_BYTE, 1,   /* max instances */
+                                  psize,        /* output buffer size */
+                                  psize,        /* input buffer size */
+                                  NMPWAIT_USE_DEFAULT_WAIT, sa_ptr);
+
+    if (read_pipe != INVALID_HANDLE_VALUE)
     {
-      i = strlen (plugin->plugin_options) + 1;
-      str = plugin->plugin_options;
+      break;
     }
-  else
+
+    DWORD err = GetLastError ();
+
+    switch (err)
     {
-      i = 0;
+    case ERROR_PIPE_BUSY:
+      /* The pipe is already open with compatible parameters.
+       * Pick a new name and retry.  */
+      continue;
+    case ERROR_ACCESS_DENIED:
+      /* The pipe is already open with incompatible parameters.
+       * Pick a new name and retry.  */
+      continue;
+    case ERROR_CALL_NOT_IMPLEMENTED:
+      /* We are on an older Win9x platform without named pipes.
+       * Return an anonymous pipe as the best approximation.  */
+      if (CreatePipe (read_pipe_ptr, write_pipe_ptr, sa_ptr, psize))
+      {
+        return 0;
+      }
+      err = GetLastError ();
+      return err;
+    default:
+      return err;
     }
-  write (fd, &i, sizeof (size_t));
-  if (i > 0)
-    write (fd, str, i);
+    /* NOTREACHED */
+  }
+
+  /* Open the named pipe for writing.
+   * Be sure to permit FILE_READ_ATTRIBUTES access.  */
+  write_pipe = CreateFileA (pipename, GENERIC_WRITE | FILE_READ_ATTRIBUTES, 0,  /* share mode */
+                            sa_ptr, OPEN_EXISTING, dwWriteMode, /* flags and attributes */
+                            0); /* handle to template file */
+
+  if (write_pipe == INVALID_HANDLE_VALUE)
+  {
+    /* Failure. */
+    DWORD err = GetLastError ();
+
+    CloseHandle (read_pipe);
+    return err;
+  }
+
+  /* Success. */
+  *read_pipe_ptr = read_pipe;
+  *write_pipe_ptr = write_pipe;
+  return 0;
+}
+
+static int
+write_to_pipe (HANDLE h, OVERLAPPED *ov, unsigned char *buf, size_t size, unsigned char **old_buf)
+{
+  DWORD written;
+  BOOL bresult;
+  DWORD err;
+
+  if (WAIT_OBJECT_0 != WaitForSingleObject (ov->hEvent, INFINITE))
+    return -1;
+  
+  ResetEvent (ov->hEvent);
+
+  if (*old_buf != NULL)
+    free (*old_buf);
+
+  *old_buf = malloc (size);
+  if (*old_buf == NULL)
+    return -1;
+  memcpy (*old_buf, buf, size);
+  written = 0;
+  ov->Offset = 0;
+  ov->OffsetHigh = 0;
+  ov->Pointer = 0;
+  ov->Internal = 0;
+  ov->InternalHigh = 0;
+  bresult = WriteFile (h, *old_buf, size, &written, ov);
+
+  if (bresult == TRUE)
+  {
+    SetEvent (ov->hEvent);
+    free (*old_buf);
+    *old_buf = NULL;
+    return written;
+  }
+
+  err = GetLastError ();
+  if (err == ERROR_IO_PENDING)
+    return size;
+  SetEvent (ov->hEvent);
+  *old_buf = NULL;
+  SetLastError (err);
+  return -1;
+}
+
+static int
+print_to_pipe (HANDLE h, OVERLAPPED *ov, unsigned char **buf, const char *fmt, ...)
+{
+  va_list va;
+  va_list vacp;
+  size_t size;
+  char *print_buf;
+  int result;
+
+  va_start (va, fmt);
+  va_copy (vacp, va);
+  size = VSNPRINTF (NULL, 0, fmt, vacp) + 1;
+  va_end (vacp);
+  if (size <= 0)
+  {
+    va_end (va);
+    return size;
+  }
+
+  print_buf = malloc (size);
+  if (print_buf == NULL)
+    return -1;
+  VSNPRINTF (print_buf, size, fmt, va);
+  va_end (va);
+  
+  result = write_to_pipe (h, ov, print_buf, size, buf);
+  free (buf);
+  return result;
+}
+
+#define plugin_print(plug, fmt, ...) print_to_pipe (plug->cpipe_in, &plug->ov_write, &plug->ov_write_buffer, fmt, ...)
+#define plugin_write(plug, buf, size) write_to_pipe (plug->cpipe_in, &plug->ov_write, buf, size, &plug->ov_write_buffer)
+
+static int
+write_plugin_data (struct EXTRACTOR_PluginList *plugin)
+{
+  size_t libname_len, shortname_len, opts_len;
+  DWORD len;
+  char *str;
+  size_t total_len = 0;
+  unsigned char *buf, *ptr;
+
+  switch (plugin->flags)
+  {
+  case EXTRACTOR_OPTION_DEFAULT_POLICY:
+    break;
+  case EXTRACTOR_OPTION_OUT_OF_PROCESS_NO_RESTART:
+    break;
+  case EXTRACTOR_OPTION_IN_PROCESS:
+    return 0;
+    break;
+  case EXTRACTOR_OPTION_DISABLED:
+    return 0;
+    break;
+  }
+
+  libname_len = strlen (plugin->libname) + 1;
+  total_len += sizeof (size_t) + libname_len;
+  shortname_len = strlen (plugin->short_libname) + 1;
+  total_len += sizeof (size_t) + shortname_len;
+  if (plugin->plugin_options != NULL)
+  {
+    opts_len = strlen (plugin->plugin_options) + 1;
+    total_len += opts_len;
+  }
+  else
+  {
+    opts_len = 0;
+  }
+  total_len += sizeof (size_t);
+
+  buf = malloc (total_len);
+  if (buf == NULL)
+    return -1;
+  ptr = buf;
+  memcpy (ptr, &libname_len, sizeof (size_t));
+  ptr += sizeof (size_t);
+  memcpy (ptr, plugin->libname, libname_len);
+  ptr += libname_len;
+  memcpy (ptr, &shortname_len, sizeof (size_t));
+  ptr += sizeof (size_t);
+  memcpy (ptr, plugin->short_libname, shortname_len);
+  ptr += shortname_len;
+  memcpy (ptr, &opts_len, sizeof (size_t));
+  ptr += sizeof (size_t);
+  if (opts_len > 0)
+  {
+    memcpy (ptr, plugin->plugin_options, opts_len);
+    ptr += opts_len;
+  }
+  if (total_len != write_to_pipe (plugin->cpipe_in, &plugin->ov_write, buf, total_len, &plugin->ov_write_buffer))
+  {
+    free (buf);
+    return -1;
+  }
+  free (buf);
+  return 0;
 }
 
 static struct EXTRACTOR_PluginList *
@@ -485,183 +770,112 @@ read_plugin_data (int fd)
   read (fd, &i, sizeof (size_t));
   ret->libname = malloc (i);
   if (ret->libname == NULL)
-    {
-      free (ret);
-      return NULL;
-    }
+  {
+    free (ret);
+    return NULL;
+  }
   read (fd, ret->libname, i);
+  ret->libname[i - 1] = '\0';
 
   read (fd, &i, sizeof (size_t));
   ret->short_libname = malloc (i);
   if (ret->short_libname == NULL)
-    {
-      free (ret->libname);
-      free (ret);
-      return NULL;
-    }
+  {
+    free (ret->libname);
+    free (ret);
+    return NULL;
+  }
   read (fd, ret->short_libname, i);
+  ret->short_libname[i - 1] = '\0';
 
   read (fd, &i, sizeof (size_t));
   if (i == 0)
-    {
-      ret->plugin_options = NULL;
-    }
+  {
+    ret->plugin_options = NULL;
+  }
   else
+  {
+    ret->plugin_options = malloc (i);
+    if (ret->plugin_options == NULL)
     {
-      ret->plugin_options = malloc (i);
-      if (ret->plugin_options == NULL)
-        {
-          free (ret->short_libname);
-          free (ret->libname);
-          free (ret);
-          return NULL;
-        }
-      read (fd, ret->plugin_options, i);
+      free (ret->short_libname);
+      free (ret->libname);
+      free (ret);
+      return NULL;
     }
+    read (fd, ret->plugin_options, i);
+    ret->plugin_options[i - 1] = '\0';
+  }
   return ret;
 }
 
-
-void CALLBACK 
-RundllEntryPoint (HWND hwnd, 
-                  HINSTANCE hinst, 
-                  LPSTR lpszCmdLine, 
-                  int nCmdShow)
-{
-  intptr_t in_h;
-  intptr_t out_h;
-  int in, out;
-
-  sscanf(lpszCmdLine, "%lu %lu", &in_h, &out_h);
-  in = _open_osfhandle (in_h, _O_RDONLY);
-  out = _open_osfhandle (out_h, 0);
-  setmode (in, _O_BINARY);
-  setmode (out, _O_BINARY);
-  process_requests (read_plugin_data (in),
-                    in, out);
-}
-
-void CALLBACK 
-RundllEntryPointA (HWND hwnd, 
-                  HINSTANCE hinst, 
-                  LPSTR lpszCmdLine, 
-                  int nCmdShow)
-{
-  return RundllEntryPoint(hwnd, hinst, lpszCmdLine, nCmdShow);
-}
-#endif
-
-
 /**
  * Start the process for the given plugin.
  */ 
 static void
 start_process (struct EXTRACTOR_PluginList *plugin)
 {
-#if !WINDOWS
-  int p1[2];
-  int p2[2];
-  pid_t pid;
-  int status;
-
-  plugin->cpid = -1;
-  if (0 != pipe (p1))
-    {
-      plugin->flags = EXTRACTOR_OPTION_DISABLED;
-      return;
-    }
-  if (0 != pipe (p2))
-    {
-      close (p1[0]);
-      close (p1[1]);
-      plugin->flags = EXTRACTOR_OPTION_DISABLED;
-      return;
-    }
-  pid = fork ();
-  plugin->cpid = pid;
-  if (pid == -1)
-    {
-      close (p1[0]);
-      close (p1[1]);
-      close (p2[0]);
-      close (p2[1]);
-      plugin->flags = EXTRACTOR_OPTION_DISABLED;
-      return;
-    }
-  if (pid == 0)
-    {
-      close (p1[1]);
-      close (p2[0]);
-      process_requests (plugin, p1[0], p2[1]);
-      _exit (0);
-    }
-  close (p1[0]);
-  close (p2[1]);
-  plugin->cpipe_in = fdopen (p1[1], "w");
-  if (plugin->cpipe_in == NULL)
-    {
-      perror ("fdopen");
-      (void) kill (plugin->cpid, SIGKILL);
-      waitpid (plugin->cpid, &status, 0);
-      close (p1[1]);
-      close (p2[0]);
-      plugin->cpid = -1;
-      plugin->flags = EXTRACTOR_OPTION_DISABLED;
-      return;
-    }
-  plugin->cpipe_out = p2[0];
-#else
-  int p1[2];
-  int p2[2];
+  HANDLE p1[2];
+  HANDLE p2[2];
   STARTUPINFO startup;
   PROCESS_INFORMATION proc;
   char cmd[MAX_PATH + 1];
   char arg1[10], arg2[10];
-  HANDLE p10_os = INVALID_HANDLE_VALUE, p21_os = INVALID_HANDLE_VALUE;
   HANDLE p10_os_inh = INVALID_HANDLE_VALUE, p21_os_inh = INVALID_HANDLE_VALUE;
+  SECURITY_ATTRIBUTES sa;
 
-  plugin->hProcess = NULL;
-  if (0 != _pipe (p1, 0, _O_BINARY | _O_NOINHERIT))
-    {
-      plugin->flags = EXTRACTOR_OPTION_DISABLED;
+  switch (plugin->flags)
+  {
+  case EXTRACTOR_OPTION_DEFAULT_POLICY:
+    if (plugin->hProcess != INVALID_HANDLE_VALUE && plugin->hProcess != 0)
       return;
-    }
-  if (0 != _pipe (p2, 0, _O_BINARY | _O_NOINHERIT))
-    {
-      close (p1[0]);
-      close (p1[1]);
-      plugin->flags = EXTRACTOR_OPTION_DISABLED;
+    break;
+  case EXTRACTOR_OPTION_OUT_OF_PROCESS_NO_RESTART:
+    if (plugin->hProcess != 0)
       return;
-    }
+    break;
+  case EXTRACTOR_OPTION_IN_PROCESS:
+    return;
+    break;
+  case EXTRACTOR_OPTION_DISABLED:
+    return;
+    break;
+  }
 
-  memset (&startup, 0, sizeof (STARTUPINFO));
+  sa.nLength = sizeof (sa);
+  sa.lpSecurityDescriptor = NULL;
+  sa.bInheritHandle = FALSE;
 
-  p10_os = (HANDLE) _get_osfhandle (p1[0]);
-  p21_os = (HANDLE) _get_osfhandle (p2[1]);
+  plugin->hProcess = NULL;
 
-  if (p10_os == INVALID_HANDLE_VALUE || p21_os == INVALID_HANDLE_VALUE)
+  if (0 != create_selectable_pipe (&p1[0], &p1[1], &sa, 1024, FILE_FLAG_OVERLAPPED, FILE_FLAG_OVERLAPPED))
   {
-    close (p1[0]);
-    close (p1[1]);
-    close (p2[0]);
-    close (p2[1]);
     plugin->flags = EXTRACTOR_OPTION_DISABLED;
     return;
   }
+  if (0 != create_selectable_pipe (&p2[0], &p2[1], &sa, 1024, FILE_FLAG_OVERLAPPED, FILE_FLAG_OVERLAPPED))
+  {
+    CloseHandle (p1[0]);
+    CloseHandle (p1[1]);
+    plugin->flags = EXTRACTOR_OPTION_DISABLED;
+    return;
+  }
+
+  memset (&startup, 0, sizeof (STARTUPINFO));
 
-  if (!DuplicateHandle (GetCurrentProcess (), p10_os, GetCurrentProcess (),
+  if (!DuplicateHandle (GetCurrentProcess (), p1[0], GetCurrentProcess (),
       &p10_os_inh, 0, TRUE, DUPLICATE_SAME_ACCESS)
-      || !DuplicateHandle (GetCurrentProcess (), p21_os, GetCurrentProcess (),
+      || !DuplicateHandle (GetCurrentProcess (), p2[1], GetCurrentProcess (),
       &p21_os_inh, 0, TRUE, DUPLICATE_SAME_ACCESS))
   {
     if (p10_os_inh != INVALID_HANDLE_VALUE)
       CloseHandle (p10_os_inh);
     if (p21_os_inh != INVALID_HANDLE_VALUE)
       CloseHandle (p21_os_inh);
-    close (p1[0]);
-    close (p1[1]);
-    close (p2[0]);
-    close (p2[1]);
+    CloseHandle (p1[0]);
+    CloseHandle (p1[1]);
+    CloseHandle (p2[0]);
+    CloseHandle (p2[1]);
     plugin->flags = EXTRACTOR_OPTION_DISABLED;
     return;
   }
@@ -675,186 +889,584 @@ start_process (struct EXTRACTOR_PluginList *plugin)
     CloseHandle (proc.hThread);
   }
   else
-    {
-      close (p1[0]);
-      close (p1[1]);
-      close (p2[0]);
-      close (p2[1]);
-      plugin->flags = EXTRACTOR_OPTION_DISABLED;
-      return;
-    }
-  close (p1[0]);
-  close (p2[1]);
+  {
+    CloseHandle (p1[0]);
+    CloseHandle (p1[1]);
+    CloseHandle (p2[0]);
+    CloseHandle (p2[1]);
+    plugin->flags = EXTRACTOR_OPTION_DISABLED;
+    return;
+  }
+  CloseHandle (p1[0]);
+  CloseHandle (p2[1]);
   CloseHandle (p10_os_inh);
   CloseHandle (p21_os_inh);
 
-  write_plugin_data (p1[1], plugin);
+  plugin->cpipe_in = p1[1];
+  plugin->cpipe_out = p2[0];
 
-  plugin->cpipe_in = fdopen (p1[1], "w");
-  if (plugin->cpipe_in == NULL)
+  memset (&plugin->ov_read, 0, sizeof (OVERLAPPED));
+  memset (&plugin->ov_write, 0, sizeof (OVERLAPPED));
+
+  plugin->ov_write_buffer = NULL;
+
+  plugin->ov_write.hEvent = CreateEvent (NULL, TRUE, TRUE, NULL);
+  plugin->ov_read.hEvent = CreateEvent (NULL, TRUE, TRUE, NULL);
+}
+
+/**
+ * Stop the child process of this plugin.
+ */
+static void
+stop_process (struct EXTRACTOR_PluginList *plugin)
+{
+  int status;
+  HANDLE process;
+
+#if DEBUG
+  if (plugin->hProcess == INVALID_HANDLE_VALUE)
+    fprintf (stderr,
+             "Plugin `%s' choked on this input\n",
+             plugin->short_libname);
+#endif
+  if (plugin->hProcess == INVALID_HANDLE_VALUE ||
+      plugin->hProcess == NULL)
+    return;
+  TerminateProcess (plugin->hProcess, 0);
+  CloseHandle (plugin->hProcess);
+  plugin->hProcess = INVALID_HANDLE_VALUE;
+  CloseHandle (plugin->cpipe_out);
+  CloseHandle (plugin->cpipe_in);
+  plugin->cpipe_out = INVALID_HANDLE_VALUE;
+  plugin->cpipe_in = INVALID_HANDLE_VALUE;
+  CloseHandle (plugin->ov_read.hEvent);
+  CloseHandle (plugin->ov_write.hEvent);
+  if (plugin->ov_write_buffer != NULL)
+  {
+    free (plugin->ov_write_buffer);
+    plugin->ov_write_buffer = NULL;
+  }
+
+  if (plugin->flags != EXTRACTOR_OPTION_DEFAULT_POLICY)
+    plugin->flags = EXTRACTOR_OPTION_DISABLED;
+
+  plugin->seek_request = -1;
+}
+
+#endif /* WINDOWS */
+
+/**
+ * Remove a plugin from a list.
+ *
+ * @param prev the current list of plugins
+ * @param library the name of the plugin to remove
+ * @return the reduced list, unchanged if the plugin was not loaded
+ */
+struct EXTRACTOR_PluginList *
+EXTRACTOR_plugin_remove(struct EXTRACTOR_PluginList * prev,
+                        const char * library)
+{
+  struct EXTRACTOR_PluginList *pos;
+  struct EXTRACTOR_PluginList *first;
+
+  pos = prev;
+  first = prev;
+  while ((pos != NULL) && (0 != strcmp (pos->short_libname, library)))
     {
-      perror ("fdopen");
-      TerminateProcess (plugin->hProcess, 0);
-      WaitForSingleObject (plugin->hProcess, INFINITE);
-      CloseHandle (plugin->hProcess);
-      close (p1[1]);
-      close (p2[0]);
-      plugin->hProcess = INVALID_HANDLE_VALUE;
-      plugin->flags = EXTRACTOR_OPTION_DISABLED;
-      return;
+      prev = pos;
+      pos = pos->next;
     }
-  plugin->cpipe_out = p2[0];
+  if (pos != NULL)
+    {
+      /* found, close library */
+      if (first == pos)
+        first = pos->next;
+      else
+        prev->next = pos->next;
+      /* found */
+      stop_process (pos);
+      free (pos->short_libname);
+      free (pos->libname);
+      free (pos->plugin_options);
+      if (NULL != pos->libraryHandle) 
+        lt_dlclose (pos->libraryHandle);      
+      free (pos);
+    }
+#if DEBUG
+  else
+    fprintf(stderr,
+            "Unloading plugin `%s' failed!\n",
+            library);
 #endif
+  return first;
 }
 
 
 /**
- * Extract meta data using the given plugin, running the
- * actual code of the plugin out-of-process.
+ * Remove all plugins from the given list (destroys the list).
  *
- * @param plugin which plugin to call
- * @param size size of the file mapped by shmfn or tshmfn
- * @param shmfn file name of the shared memory segment
- * @param tshmfn file name of the shared memory segment for the end of the data
- * @param proc function to call on the meta data
+ * @param plugin the list of plugins
+ */
+void 
+EXTRACTOR_plugin_remove_all(struct EXTRACTOR_PluginList *plugins)
+{
+  while (plugins != NULL)
+    plugins = EXTRACTOR_plugin_remove (plugins, plugins->short_libname);
+}
+
+
+
+/**
+ * Open a file
+ */
+static int file_open(const char *filename, int oflag, ...)
+{
+  int mode;
+  const char *fn;
+#ifdef MINGW
+  char szFile[_MAX_PATH + 1];
+  long lRet;
+
+  if ((lRet = plibc_conv_to_win_path(filename, szFile)) != ERROR_SUCCESS)
+  {
+    errno = ENOENT;
+    SetLastError(lRet);
+    return -1;
+  }
+  fn = szFile;
+#else
+  fn = filename;
+#endif
+  mode = 0;
+#ifdef MINGW
+  /* Set binary mode */
+  mode |= O_BINARY;
+#endif
+  return OPEN(fn, oflag, mode);
+}
+
+#ifndef O_LARGEFILE
+#define O_LARGEFILE 0
+#endif
+
+#if HAVE_ZLIB
+#define MIN_ZLIB_HEADER 12
+#endif
+#if HAVE_LIBBZ2
+#define MIN_BZ2_HEADER 4
+#endif
+#if !defined (MIN_COMPRESSED_HEADER) && HAVE_ZLIB
+#define MIN_COMPRESSED_HEADER MIN_ZLIB_HEADER
+#endif
+#if !defined (MIN_COMPRESSED_HEADER) && HAVE_LIBBZ2
+#define MIN_COMPRESSED_HEADER MIN_BZ2_HEADER
+#endif
+#if !defined (MIN_COMPRESSED_HEADER)
+#define MIN_COMPRESSED_HEADER -1
+#endif
+
+#define COMPRESSED_DATA_PROBE_SIZE 3
+
+/**
+ * Try to decompress compressed data
+ *
+ * @param data data to decompress, or NULL (if fd is not -1)
+ * @param fd file to read data from, or -1 (if data is not NULL)
+ * @param fsize size of data (if data is not NULL) or size of fd file (if fd is not -1)
+ * @param compression_type type of compression, as returned by get_compression_type ()
+ * @param buffer a pointer to a buffer pointer, buffer pointer is NEVER a NULL and already has some data (usually - COMPRESSED_DATA_PROBE_SIZE bytes) in it.
+ * @param buffer_size a pointer to buffer size
+ * @param proc callback for metadata
  * @param proc_cls cls for proc
- * @return 0 if proc did not return non-zero
+ * @return 0 on success, anything else on error
  */
 static int
-extract_oop (struct EXTRACTOR_PluginList *plugin,
-             size_t size,
-             const char *shmfn,
-             const char *tshmfn,
-             EXTRACTOR_MetaDataProcessor proc,
-             void *proc_cls)
+try_to_decompress (const unsigned char *data, int fd, int64_t fsize, int compression_type, void **buffer, size_t *buffer_size, EXTRACTOR_MetaDataProcessor proc, void *proc_cls)
 {
-  struct IpcHeader hdr;
-  char mimetype[MAX_MIME_LEN + 1];
-  char *data;
+  unsigned char *new_buffer;
+  ssize_t read_result;
 
-#ifndef WINDOWS
-  if (plugin->cpid == -1)
-#else
-  if (plugin->hProcess == INVALID_HANDLE_VALUE)
+  unsigned char *buf;
+  unsigned char *rbuf;
+  size_t dsize;
+#if HAVE_ZLIB
+  z_stream strm;
+  int ret;
+  size_t pos;
 #endif
-    return 0;
-  if (0 >= fprintf (plugin->cpipe_in, 
-                    "%s\n",
-                    shmfn))
-    {
-      stop_process (plugin);
-#ifndef WINDOWS
-      plugin->cpid = -1;
-#else
-      plugin->hProcess = INVALID_HANDLE_VALUE;
+#if HAVE_LIBBZ2
+  bz_stream bstrm;
+  int bret;
+  size_t bpos;
 #endif
-      if (plugin->flags != EXTRACTOR_OPTION_DEFAULT_POLICY)
-        plugin->flags = EXTRACTOR_OPTION_DISABLED;
-      return 0;
+
+  if (fd != -1)
+  {
+    if (fsize > *buffer_size)
+    {
+      /* Read the rest of the file. Can't de-compress it partially anyway */
+      /* Memory mapping is not useful here, because memory mapping ALSO takes up
+       * memory (even more than a buffer, since it might be aligned), and
+       * because we need to read every byte anyway (lazy on-demand reads into
+       * memory provided by memory mapping won't help).
+       */
+      new_buffer = realloc (*buffer, fsize);
+      if (new_buffer == NULL)
+      {
+        free (*buffer);
+        return -1;
+      }
+      read_result = READ (fd, &new_buffer[*buffer_size], fsize - *buffer_size);
+      if (read_result != fsize - *buffer_size)
+      {
+        free (*buffer);
+        return -1;
+      }
+      *buffer = new_buffer;
+      *buffer_size = fsize;
     }
-  if (0 >= fprintf (plugin->cpipe_in, 
-                    "!%s\n",
-                    (tshmfn != NULL) ? tshmfn : ""))
+    data = (const unsigned char *) new_buffer;
+  }
+
+#if HAVE_ZLIB
+  if (compression_type == 1) 
+  {
+    /* Process gzip header */
+    unsigned int gzip_header_length = 10;
+
+    if (data[3] & 0x4) /* FEXTRA  set */
+      gzip_header_length += 2 + (unsigned) (data[10] & 0xff) +
+        (((unsigned) (data[11] & 0xff)) * 256);
+
+    if (data[3] & 0x8) /* FNAME set */
     {
-      stop_process (plugin);
-#ifndef WINDOWS
-      plugin->cpid = -1;
-#else
-      plugin->hProcess = INVALID_HANDLE_VALUE;
-#endif
-      if (plugin->flags != EXTRACTOR_OPTION_DEFAULT_POLICY)
-        plugin->flags = EXTRACTOR_OPTION_DISABLED;
-      return 0;
+      const unsigned char *cptr = data + gzip_header_length;
+
+      /* stored file name is here */
+      while ((cptr - data) < fsize)
+      {
+        if ('\0' == *cptr)
+        break;
+        cptr++;
+      }
+
+      if (0 != proc (proc_cls, "<zlib>", EXTRACTOR_METATYPE_FILENAME,
+          EXTRACTOR_METAFORMAT_C_STRING, "text/plain",
+          (const char *) (data + gzip_header_length),
+          cptr - (data + gzip_header_length)))
+        return 0; /* done */
+
+      gzip_header_length = (cptr - data) + 1;
     }
-  if (0 >= fprintf (plugin->cpipe_in, 
-                    "s%lu\n",
-                    size))
+
+    if (data[3] & 0x16) /* FCOMMENT set */
     {
-      stop_process (plugin);
-#ifndef WINDOWS
-      plugin->cpid = -1;
-#else
-      plugin->hProcess = INVALID_HANDLE_VALUE;
+      const unsigned char * cptr = data + gzip_header_length;
+
+      /* stored comment is here */
+      while (cptr < data + fsize)
+      {
+        if ('\0' == *cptr)
+          break;
+        cptr ++;
+      }  
+
+      if (0 != proc (proc_cls, "<zlib>", EXTRACTOR_METATYPE_COMMENT,
+          EXTRACTOR_METAFORMAT_C_STRING, "text/plain",
+          (const char *) (data + gzip_header_length),
+          cptr - (data + gzip_header_length)))
+        return 0; /* done */
+
+      gzip_header_length = (cptr - data) + 1;
+    }
+
+    if (data[3] & 0x2) /* FCHRC set */
+      gzip_header_length += 2;
+
+    memset (&strm, 0, sizeof (z_stream));
+
+#ifdef ZLIB_VERNUM
+    gzip_header_length = 0;
 #endif
-      if (plugin->flags != EXTRACTOR_OPTION_DEFAULT_POLICY)
-        plugin->flags = EXTRACTOR_OPTION_DISABLED;
-      return 0;
+
+    if (fsize > gzip_header_length)
+    {
+      strm.next_in = (Bytef *) data + gzip_header_length;
+      strm.avail_in = fsize - gzip_header_length;
     }
-  fflush (plugin->cpipe_in);
-  while (1)
+    else
     {
-      if (0 != read_all (plugin->cpipe_out,
-                         &hdr,
-                         sizeof(hdr)))
-        {
-          stop_process (plugin);
-#ifndef WINDOWS
-      plugin->cpid = -1;
+      strm.next_in = (Bytef *) data;
+      strm.avail_in = 0;
+    }
+    strm.total_in = 0;
+    strm.zalloc = NULL;
+    strm.zfree = NULL;
+    strm.opaque = NULL;
+
+    /*
+     * note: maybe plain inflateInit(&strm) is adequate,
+     * it looks more backward-compatible also ;
+     *
+     * ZLIB_VERNUM isn't defined by zlib version 1.1.4 ;
+     * there might be a better check.
+     */
+    if (Z_OK == inflateInit2 (&strm,
+#ifdef ZLIB_VERNUM
+        15 + 32
 #else
-      plugin->hProcess = INVALID_HANDLE_VALUE;
+        -MAX_WBITS
 #endif
-          if (plugin->flags != EXTRACTOR_OPTION_DEFAULT_POLICY)
-            plugin->flags = EXTRACTOR_OPTION_DISABLED;
-          return 0;
-        }
-      if  ( (hdr.type == 0) &&
-            (hdr.format == 0) &&
-            (hdr.data_len == 0) &&
-            (hdr.mime_len == 0) )
-        break;
-      if (hdr.mime_len > MAX_MIME_LEN)
-        {
-          stop_process (plugin);          
-#ifndef WINDOWS
-      plugin->cpid = -1;
-#else
-      plugin->hProcess = INVALID_HANDLE_VALUE;
+        ))
+    {
+      pos = 0;
+      dsize = 2 * fsize;
+      if ( (dsize > MAX_DECOMPRESS) ||
+           (dsize < fsize) )
+        dsize = MAX_DECOMPRESS;
+      buf = malloc (dsize);
+
+      if (buf != NULL)
+      {
+        strm.next_out = (Bytef *) buf;
+        strm.avail_out = dsize;
+
+        do
+        {
+          ret = inflate (&strm, Z_SYNC_FLUSH);
+          if (ret == Z_OK)
+          {
+            if (dsize == MAX_DECOMPRESS)
+              break;
+
+            pos += strm.total_out;
+            strm.total_out = 0;
+            dsize *= 2;
+
+            if (dsize > MAX_DECOMPRESS)
+              dsize = MAX_DECOMPRESS;
+
+            rbuf = realloc (buf, dsize);
+            if (rbuf == NULL)
+            {
+              free (buf);
+              buf = NULL;
+              break;
+            }
+
+            buf = rbuf;
+            strm.next_out = (Bytef *) &buf[pos];
+            strm.avail_out = dsize - pos;
+          }
+          else if (ret != Z_STREAM_END) 
+          {
+            /* error */
+            free (buf);
+            buf = NULL;
+          }
+        } while ((buf != NULL) && (ret != Z_STREAM_END));
+
+        dsize = pos + strm.total_out;
+        if ((dsize == 0) && (buf != NULL))
+        {
+          free (buf);
+          buf = NULL;
+        }
+      }
+
+      inflateEnd (&strm);
+
+      if (fd != -1)
+        if (*buffer != NULL)
+          free (*buffer);
+
+      if (buf == NULL)
+      {
+        return -1;
+      }
+      else
+      {
+        *buffer = buf;
+        *buffer_size = dsize;
+        return 0;
+      }
+    }
+  }
 #endif
-          if (plugin->flags != EXTRACTOR_OPTION_DEFAULT_POLICY)
-            plugin->flags = EXTRACTOR_OPTION_DISABLED;
-          return 0;
-        }
-      data = malloc (hdr.data_len);
-      if (data == NULL)
-        {
-          stop_process (plugin);
-          return 1;
-        }
-      if ( (0 != (read_all (plugin->cpipe_out,
-                            mimetype,
-                            hdr.mime_len))) ||
-           (0 != (read_all (plugin->cpipe_out,
-                            data,
-                            hdr.data_len))) )
-        {
-          stop_process (plugin);
-#ifndef WINDOWS
-      plugin->cpid = -1;
-#else
-      plugin->hProcess = INVALID_HANDLE_VALUE;
+  
+#if HAVE_LIBBZ2
+  if (compression_type == 2) 
+  {
+    memset(&bstrm, 0, sizeof (bz_stream));
+    bstrm.next_in = (char *) data;
+    bstrm.avail_in = fsize;
+    bstrm.total_in_lo32 = 0;
+    bstrm.total_in_hi32 = 0;
+    bstrm.bzalloc = NULL;
+    bstrm.bzfree = NULL;
+    bstrm.opaque = NULL;
+    if (BZ_OK == BZ2_bzDecompressInit(&bstrm, 0,0)) 
+    {
+      bpos = 0;
+      dsize = 2 * fsize;
+      if ( (dsize > MAX_DECOMPRESS) || (dsize < fsize) )
+        dsize = MAX_DECOMPRESS;
+      buf = malloc (dsize);
+
+      if (buf != NULL) 
+      {
+        bstrm.next_out = (char *) buf;
+        bstrm.avail_out = dsize;
+
+        do
+        {
+          bret = BZ2_bzDecompress (&bstrm);
+          if (bret == Z_OK) 
+          {
+            if (dsize == MAX_DECOMPRESS)
+              break;
+            bpos += bstrm.total_out_lo32;
+            bstrm.total_out_lo32 = 0;
+
+            dsize *= 2;
+            if (dsize > MAX_DECOMPRESS)
+              dsize = MAX_DECOMPRESS;
+
+            rbuf = realloc(buf, dsize);
+            if (rbuf == NULL)
+            {
+              free (buf);
+              buf = NULL;
+              break;
+            }
+
+            buf = rbuf;
+            bstrm.next_out = (char*) &buf[bpos];
+            bstrm.avail_out = dsize - bpos;
+          } 
+          else if (bret != BZ_STREAM_END) 
+          {
+            /* error */
+            free (buf);
+            buf = NULL;
+          }
+        } while ((buf != NULL) && (bret != BZ_STREAM_END));
+
+        dsize = bpos + bstrm.total_out_lo32;
+        if ((dsize == 0) && (buf != NULL))
+        {
+          free (buf);
+          buf = NULL;
+        }
+      }
+
+      BZ2_bzDecompressEnd (&bstrm);
+
+      if (fd != -1)
+        if (*buffer != NULL)
+          free (*buffer);
+
+      if (buf == NULL)
+      {
+        return -1;
+      }
+      else
+      {
+        *buffer = buf;
+        *buffer_size = dsize;
+        return 0;
+      }
+    }
+  }
 #endif
-          free (data);
-          if (plugin->flags != EXTRACTOR_OPTION_DEFAULT_POLICY)
-            plugin->flags = EXTRACTOR_OPTION_DISABLED;
-          return 0;
-        }          
-      mimetype[hdr.mime_len] = '\0';
-      if ( (proc != NULL) &&
-           (0 != proc (proc_cls, 
-                       plugin->short_libname,
-                       hdr.type,
-                       hdr.format,
-                       mimetype,
-                       data,
-                       hdr.data_len)) )
-        proc = NULL;    
-      free (data);
+  return -1;
+}
+
+/**
+ * Detect if we have compressed data on our hands.
+ *
+ * @param data pointer to a data buffer or NULL (in case fd is not -1)
+ * @param fd a file to read data from, or -1 (if data is not NULL)
+ * @param fsize size of data (if data is not NULL) or of file (if fd is not -1)
+ * @param buffer will receive a pointer to the data that this function read
+ * @param buffer_size will receive size of the buffer
+ * @return -1 to indicate an error, 0 to indicate uncompressed data, or a type (> 0) of compression
+ */
+static int
+get_compression_type (const unsigned char *data, int fd, int64_t fsize, void **buffer, size_t *buffer_size)
+{
+  void *read_data = NULL;
+  size_t read_data_size = 0;
+  ssize_t read_result;
+
+  if ((MIN_COMPRESSED_HEADER < 0) || (fsize < MIN_COMPRESSED_HEADER))
+  {
+    *buffer = NULL;
+    return 0;
+  }
+  if (data == NULL)
+  {
+    read_data_size = COMPRESSED_DATA_PROBE_SIZE;
+    read_data = malloc (read_data_size);
+    if (read_data == NULL)
+      return -1;
+    read_result = READ (fd, read_data, read_data_size);
+    if (read_result != read_data_size)
+    {
+      free (read_data);
+      return -1;
     }
-  if (NULL == proc)
+    *buffer = read_data;
+    *buffer_size = read_data_size;
+    data = (const void *) read_data;
+  }
+#if HAVE_ZLIB
+  if ((fsize >= MIN_ZLIB_HEADER) && (data[0] == 0x1f) && (data[1] == 0x8b) && (data[2] == 0x08))
+    return 1;
+#endif
+#if HAVE_LIBBZ2
+  if ((fsize >= MIN_BZ2_HEADER) && (data[0] == 'B') && (data[1] == 'Z') && (data[2] == 'h')) 
+    return 2;
+#endif
+  return 0;
+}
+
+#if WINDOWS
+
+/**
+ * Setup a shared memory segment.
+ *
+ * @param ptr set to the location of the map segment
+ * @param map where to store the map handle
+ * @param fn name of the mapping
+ * @param fn_size size available in fn
+ * @param size number of bytes to allocated for the mapping
+ * @return 0 on success
+ */
+static int
+make_shm_w32 (void **ptr, HANDLE *map, char *fn, size_t fn_size, size_t size)
+{
+  const char *tpath = "Local\\";
+  snprintf (fn, fn_size, "%slibextractor-shm-%u-%u", tpath, getpid(),
+      (unsigned int) RANDOM());
+  *map = CreateFileMapping (INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE, 0, size, fn);
+  *ptr = MapViewOfFile (*map, FILE_MAP_WRITE, 0, 0, size);
+  if (*ptr == NULL)
+  {
+    CloseHandle (*map);
     return 1;
+  }
   return 0;
-}            
+}
+
+static void
+destroy_shm_w32 (void *ptr, HANDLE map)
+{
+  UnmapViewOfFile (ptr);
+  CloseHandle (map);
+}
 
+#else
 
 /**
  * Setup a shared memory segment.
@@ -867,22 +1479,10 @@ extract_oop (struct EXTRACTOR_PluginList *plugin,
  * @return 0 on success
  */
 static int
-make_shm (int is_tail,
-          void **ptr,
-#ifndef WINDOWS
-          int *shmid,
-#else
-          HANDLE *map,
-#endif    
-          char *fn,
-          size_t fn_size,
-          size_t size)
+make_shm_posix (void **ptr, int *shmid, char *fn, size_t fn_size, size_t size)
 {
   const char *tpath;
-#ifdef WINDOWS
-  tpath = "Local\\";
-#elif SOMEBSD
-  const char *tpath;
+#if SOMEBSD
   /* this works on FreeBSD, not sure about others... */
   tpath = getenv ("TMPDIR");
   if (tpath == NULL)
@@ -890,578 +1490,606 @@ make_shm (int is_tail,
 #else
   tpath = "/"; /* Linux */
 #endif 
-  snprintf (fn,
-            fn_size,
-            "%slibextractor-%sshm-%u-%u",
-            tpath,
-            (is_tail) ? "t" : "",
-            getpid(),
-            (unsigned int) RANDOM());
-#ifndef WINDOWS
+  snprintf (fn, fn_size, "%slibextractor-shm-%u-%u", tpath, getpid(),
+      (unsigned int) RANDOM());
   *shmid = shm_open (fn, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR);
   *ptr = NULL;
-  if (-1 == (*shmid))
-    return 1;    
-  if ( (0 != ftruncate (*shmid, size)) ||
-       (NULL == (*ptr = mmap (NULL, size, PROT_WRITE, MAP_SHARED, *shmid, 0))) ||
-       (*ptr == (void*) -1) )
+  if (-1 == *shmid)
+    return 1;
+  if ((0 != ftruncate (*shmid, size)) ||
+      (NULL == (*ptr = mmap (NULL, size, PROT_WRITE, MAP_SHARED, *shmid, 0))) ||
+      (*ptr == (void*) -1) )
+  {
+    close (*shmid);
+    *shmid = -1;
+    shm_unlink (fn);
+    return 1;
+  }
+  return 0;
+}
+
+static void
+destroy_shm_posix (void *ptr, int shm_id, size_t size, char *shm_name)
+{
+  if (NULL != ptr)
+    munmap (ptr, size);
+  if (shm_id != -1)
+    close (shm_id);
+  shm_unlink (shm_name);
+}
+#endif
+
+
+static void
+init_plugin_state (struct EXTRACTOR_PluginList *plugin, char *shm_name, int64_t fsize)
+{
+  int write_result;
+  int init_state_size;
+  unsigned char *init_state;
+  int t;
+  size_t shm_name_len = strlen (shm_name) + 1;
+  init_state_size = 1 + sizeof (size_t) + shm_name_len + sizeof (int64_t);
+  switch (plugin->flags)
+  {
+  case EXTRACTOR_OPTION_DEFAULT_POLICY:
+  case EXTRACTOR_OPTION_OUT_OF_PROCESS_NO_RESTART:
+    init_state = malloc (init_state_size);
+    if (init_state == NULL)
     {
-      close (*shmid);
-      *shmid = -1;
-      shm_unlink (fn);
-      return 1;
+      stop_process (plugin);
+      return;
     }
-  return 0;
-#else
-  *map = CreateFileMapping (INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE, 0, size, fn);
-  *ptr = MapViewOfFile (*map, FILE_MAP_WRITE, 0, 0, size);
-  if (*ptr == NULL)
+    t = 0;
+    init_state[t] = MESSAGE_INIT_STATE;
+    t += 1;
+    memcpy (&init_state[t], &fsize, sizeof (int64_t));
+    t += sizeof (int64_t);
+    memcpy (&init_state[t], &shm_name_len, sizeof (size_t));
+    t += sizeof (size_t);
+    memcpy (&init_state[t], shm_name, shm_name_len);
+    t += shm_name_len;
+    write_result = plugin_write (plugin, init_state, init_state_size);
+    free (init_state);
+    if (write_result < init_state_size)
     {
-      CloseHandle (*map);
-      return 1;
+      stop_process (plugin);
+      return;
     }
-  return 0;
-#endif
+    plugin->seek_request = 0;
+    break;
+  case EXTRACTOR_OPTION_IN_PROCESS:
+    plugin_open_shm (plugin, shm_name);
+    plugin->fsize = fsize;
+    plugin->init_state_method (plugin);
+    plugin->seek_request = 0;
+    return;
+    break;
+  case EXTRACTOR_OPTION_DISABLED:
+    return;
+    break;
+  }
+}
+
+static void
+discard_plugin_state (struct EXTRACTOR_PluginList *plugin)
+{
+  int write_result;
+  unsigned char discard_state = MESSAGE_DISCARD_STATE;
+  switch (plugin->flags)
+  {
+  case EXTRACTOR_OPTION_DEFAULT_POLICY:
+  case EXTRACTOR_OPTION_OUT_OF_PROCESS_NO_RESTART:
+    /* This is somewhat clumsy, but it's the only stop-indicating
+     * non-W32/POSIX-specific field i could think of...
+     */
+    if (plugin->cpipe_out != -1)
+    {
+      write_result = plugin_write (plugin, &discard_state, 1);
+      if (write_result < 1)
+      {
+        stop_process (plugin);
+        return;
+      }
+    }
+    break;
+  case EXTRACTOR_OPTION_IN_PROCESS:
+    plugin->discard_state_method (plugin);
+    return;
+    break;
+  case EXTRACTOR_OPTION_DISABLED:
+    return;
+    break;
+  }
 }
 
+static int
+give_shm_to_plugin (struct EXTRACTOR_PluginList *plugin, int64_t position, size_t map_size)
+{
+  int write_result;
+  int updated_shm_size = 1 + sizeof (int64_t) + sizeof (size_t);
+  unsigned char updated_shm[updated_shm_size];
+  int t = 0;
+  updated_shm[t] = MESSAGE_UPDATED_SHM;
+  t += 1;
+  memcpy (&updated_shm[t], &position, sizeof (int64_t));
+  t += sizeof (int64_t);
+  memcpy (&updated_shm[t], &map_size, sizeof (size_t));
+  t += sizeof (size_t);
+  switch (plugin->flags)
+  {
+  case EXTRACTOR_OPTION_DEFAULT_POLICY:
+  case EXTRACTOR_OPTION_OUT_OF_PROCESS_NO_RESTART:
+    if (plugin->seek_request < 0)
+      return 0;
+    write_result = plugin_write (plugin, updated_shm, updated_shm_size);
+    if (write_result < updated_shm_size)
+    {
+      stop_process (plugin);
+      return 0;
+    }
+    return 1;
+  case EXTRACTOR_OPTION_IN_PROCESS:
+    plugin->position = position;
+    plugin->map_size = map_size;
+    return 0;
+  case EXTRACTOR_OPTION_DISABLED:
+    return 0;
+  default:
+    return 1;
+  }
+}
 
-/**
- * Extract keywords using the given set of plugins.
- *
- * @param plugins the list of plugins to use
- * @param data data to process, never NULL
- * @param size number of bytes in data, ignored if data is NULL
- * @param tdata end of file data, or NULL
- * @param tsize number of bytes in tdata
- * @param proc function to call for each meta data item found
- * @param proc_cls cls argument to proc
- */
 static void
-extract (struct EXTRACTOR_PluginList *plugins,
-         const char * data,
-         size_t size,
-         const char * tdata,
-         size_t tsize,
-         EXTRACTOR_MetaDataProcessor proc,
-         void *proc_cls) 
+ask_in_process_plugin (struct EXTRACTOR_PluginList *plugin, int64_t position, void *shm_ptr, EXTRACTOR_MetaDataProcessor proc, void *proc_cls)
 {
-  struct EXTRACTOR_PluginList *ppos;
-  enum EXTRACTOR_Options flags;
-  void *ptr;
-  void *tptr;
-  char fn[255];
-  char tfn[255];
-  int want_shm;
-  int want_tail;
-#ifndef WINDOWS
-  int shmid;
-  int tshmid;
-#else
-  HANDLE map;
-  HANDLE tmap;
-#endif
+  int extract_reply;
+  switch (plugin->flags)
+  {
+  case EXTRACTOR_OPTION_DEFAULT_POLICY:
+  case EXTRACTOR_OPTION_OUT_OF_PROCESS_NO_RESTART:
+    return;
+  case EXTRACTOR_OPTION_IN_PROCESS:
+    if (plugin->seek_request >= 0)
+    {
+      plugin->shm_ptr = shm_ptr;
+      extract_reply = plugin->extract_method (plugin, proc, proc_cls);
+      if (extract_reply == 1)
+        plugin->seek_request = -1;
+    }
+    break;
+  case EXTRACTOR_OPTION_DISABLED:
+    return;
+    break;
+  }
+}
 
-  want_shm = 0;
-  ppos = plugins;
-  while (NULL != ppos)
-    {      
-      switch (ppos->flags)
-        {
-        case EXTRACTOR_OPTION_DEFAULT_POLICY:
-#ifndef WINDOWS
-          if ( (0 == ppos->cpid) ||
-               (-1 == ppos->cpid) )
-#else
-          if (ppos->hProcess == NULL || ppos->hProcess == INVALID_HANDLE_VALUE)
-#endif
-            start_process (ppos);
-          want_shm = 1;
-          break;
-        case EXTRACTOR_OPTION_OUT_OF_PROCESS_NO_RESTART:
-#ifndef WINDOWS
-          if (0 == ppos->cpid)
+#if !WINDOWS
+int
+plugin_read (struct EXTRACTOR_PluginList *plugin, unsigned char *buf, size_t size)
+{
+  ssize_t read_result;
+  size_t read_count = 0;
+  while (read_count < size)
+  {
+    read_result = read (plugin->cpipe_out, &buf[read_count], size - read_count);
+    if (read_result <= 0)
+      return read_result;
+    read_count += read_result;
+  }
+  return read_count;
+}
 #else
-          if (ppos->hProcess == NULL)
+int 
+plugin_read (struct EXTRACTOR_PluginList *plugin, unsigned char *buf, size_t size)
+{
+  DWORD bytes_read;
+  BOOL bresult;
+  size_t read_count = 0;
+  while (read_count < size)
+  {
+    bresult = ReadFile (plugin->cpipe_out, &buf[read_count], size - read_count, &bytes_read, NULL);
+    if (!bresult)
+      return -1;
+    read_count += bytes_read;
+  }
+  return read_count;
+}
 #endif
-            start_process (ppos);
-          want_shm = 1;
-          break;
-        case EXTRACTOR_OPTION_IN_PROCESS:
-          break;
-        case EXTRACTOR_OPTION_DISABLED:
-          break;
-        }      
-      ppos = ppos->next;
+
+static int
+receive_reply (struct EXTRACTOR_PluginList *plugin, EXTRACTOR_MetaDataProcessor proc, void *proc_cls)
+{
+  int read_result;
+  unsigned char code;
+  int must_read = 1;
+
+  int64_t seek_position;
+  struct IpcHeader hdr;
+  char *mime_type;
+  char *data;
+
+  while (must_read)
+  {
+    read_result = plugin_read (plugin, &code, 1);
+    if (read_result < 1)
+      return -1;
+    switch (code)
+    {
+    case MESSAGE_DONE: /* Done */
+      plugin->seek_request = -1;
+      must_read = 0;
+      break;
+    case MESSAGE_SEEK: /* Seek */
+      read_result = plugin_read (plugin, (unsigned char *) &seek_position, sizeof (int64_t));
+      if (read_result < sizeof (int64_t))
+        return -1;
+      plugin->seek_request = seek_position;
+      must_read = 0;
+      break;
+    case MESSAGE_META: /* Meta */
+      read_result = plugin_read (plugin, (unsigned char *) &hdr, sizeof (hdr));
+      if (read_result < sizeof (hdr)) /* FIXME: check hdr for sanity */
+        return -1;
+      mime_type = malloc (hdr.mime_len + 1);
+      if (mime_type == NULL)
+        return -1;
+      read_result = plugin_read (plugin, (unsigned char *) mime_type, hdr.mime_len);
+      if (read_result < hdr.mime_len)
+        return -1;
+      mime_type[hdr.mime_len] = '\0';
+      data = malloc (hdr.data_len);
+      if (data == NULL)
+      {
+        free (mime_type);
+        return -1;
+      }
+      read_result = plugin_read (plugin, (unsigned char *) data, hdr.data_len);
+      if (read_result < hdr.data_len)
+      {
+        free (mime_type);
+        free (data);
+        return -1;
+      }
+      read_result = proc (proc_cls, plugin->short_libname, hdr.meta_type, hdr.meta_format, mime_type, data, hdr.data_len);
+      free (mime_type);
+      free (data);
+      if (read_result != 0)
+        return 1;
+      break;
+    default:
+      return -1;
     }
-  ptr = NULL;
-  tptr = NULL;
-  if (want_shm)
+  }
+  return 0;
+}
+
+#if !WINDOWS
+static int
+wait_for_reply (struct EXTRACTOR_PluginList *plugins, EXTRACTOR_MetaDataProcessor proc, void *proc_cls)
+{
+  int ready;
+  int result;
+  struct timeval tv;
+  fd_set to_check;
+  int highest = 0;
+  int read_result;
+  struct EXTRACTOR_PluginList *ppos;
+
+  FD_ZERO (&to_check);
+
+  for (ppos = plugins; NULL != ppos; ppos = ppos->next)
+  {
+    switch (ppos->flags)
     {
-      if (size > MAX_READ)
-        size = MAX_READ;
-      if (0 == make_shm (0, 
-                         &ptr,
-#ifndef WINDOWS
-                         &shmid,
-#else
-                         &map,
-#endif
-                         fn, sizeof(fn), size))
-        {
-          memcpy (ptr, data, size);      
-          if ( (tdata != NULL) &&
-               (0 == make_shm (1,
-                               &tptr,
-#ifndef WINDOWS
-                               &tshmid,
-#else
-                               &tmap,
-#endif
-                               tfn, sizeof(tfn), tsize)) )
-            {
-              memcpy (tptr, tdata, tsize);      
-            }
-          else
-            {
-              tptr = NULL;
-            }
-        }
-      else
-        {
-          want_shm = 0;
-        }           
+    case EXTRACTOR_OPTION_DEFAULT_POLICY:
+    case EXTRACTOR_OPTION_OUT_OF_PROCESS_NO_RESTART:
+      if (ppos->seek_request == -1)
+        continue;
+      FD_SET (ppos->cpipe_out, &to_check);
+      if (highest < ppos->cpipe_out)
+        highest = ppos->cpipe_out;
+      break;
+    case EXTRACTOR_OPTION_IN_PROCESS:
+      break;
+    case EXTRACTOR_OPTION_DISABLED:
+      break;
     }
-  ppos = plugins;
-  while (NULL != ppos)
+  }
+
+  tv.tv_sec = 10;
+  tv.tv_usec = 0;
+  ready = select (highest + 1, &to_check, NULL, NULL, &tv);
+  if (ready <= 0)
+    /* an error or timeout -> something's wrong or all plugins hung up */
+    return -1;
+
+  result = 0;
+  for (ppos = plugins; NULL != ppos; ppos = ppos->next)
+  {
+    switch (ppos->flags)
     {
-      flags = ppos->flags;
-      if (! want_shm)
-        flags = EXTRACTOR_OPTION_IN_PROCESS;
-      switch (flags)
-        {
-        case EXTRACTOR_OPTION_DEFAULT_POLICY:
-          if (0 != extract_oop (ppos, (tptr != NULL) ? tsize : size, fn, 
-                                (tptr != NULL) ? tfn : NULL,
-                                proc, proc_cls))
-            {
-              ppos = NULL;
-              break;
-            }
-#ifndef WINDOWS
-          if (ppos->cpid == -1)
+    case EXTRACTOR_OPTION_DEFAULT_POLICY:
+    case EXTRACTOR_OPTION_OUT_OF_PROCESS_NO_RESTART:
+      if (ppos->seek_request == -1)
+        continue;
+      if (FD_ISSET (ppos->cpipe_out, &to_check))
+      {
+        read_result = receive_reply (ppos, proc, proc_cls);
+        if (read_result < 0)
+        {
+          stop_process (ppos);
+        }
+        result += 1;
+      }
+      break;
+    case EXTRACTOR_OPTION_IN_PROCESS:
+      break;
+    case EXTRACTOR_OPTION_DISABLED:
+      break;
+    }
+  }
+  return result;
+}
 #else
-      if (ppos->hProcess == INVALID_HANDLE_VALUE)
-#endif
-            {
-              start_process (ppos);
-              if (0 != extract_oop (ppos, (tptr != NULL) ? tsize : size, fn, 
-                                    (tptr != NULL) ? tfn : NULL,
-                                    proc, proc_cls))
-                {
-                  ppos = NULL;
-                  break;
-                }
-            }
-          break;
-        case EXTRACTOR_OPTION_OUT_OF_PROCESS_NO_RESTART:
-          if (0 != extract_oop (ppos, (tptr != NULL) ? tsize : size, fn, 
-                                (tptr != NULL) ? tfn : NULL,
-                                proc, proc_cls))
-            {
-              ppos = NULL;
-              break;
-            }
-          break;
-        case EXTRACTOR_OPTION_IN_PROCESS:                 
-          want_tail = ( (ppos->specials != NULL) &&
-                        (NULL != strstr (ppos->specials,
-                                         "want-tail")));
-          if (NULL == ppos->extractMethod) 
-            plugin_load (ppos);     
-          if ( ( (ppos->specials == NULL) ||
-                 (NULL == strstr (ppos->specials,
-                                  "oop-only")) ) )
-            {
-              if (want_tail)
-                {
-                  if ( (NULL != ppos->extractMethod) &&
-                       (tdata != NULL) &&
-                       (0 != ppos->extractMethod (tdata, 
-                                                  tsize, 
-                                                  proc, 
-                                                  proc_cls,
-                                                  ppos->plugin_options)) )
-                    {
-                      ppos = NULL;
-                      break;
-                    }
-                }
-              else
-                {
-                  if ( (NULL != ppos->extractMethod) &&
-                       (0 != ppos->extractMethod (data, 
-                                                  size, 
-                                                  proc, 
-                                                  proc_cls,
-                                                  ppos->plugin_options)) )
-                    {
-                      ppos = NULL;
-                      break;
-                    }
-                }
-            }
-          break;
-        case EXTRACTOR_OPTION_DISABLED:
-          break;
-        }      
-      if (ppos == NULL)
-        break;
-      ppos = ppos->next;
+static int
+wait_for_reply (struct EXTRACTOR_PluginList *plugins, EXTRACTOR_MetaDataProcessor proc, void *proc_cls)
+{
+  int result;
+  DWORD ms;
+  DWORD first_ready;
+  DWORD dwresult;
+  DWORD bytes_read;
+  BOOL bresult;
+  int i;
+  HANDLE events[MAXIMUM_WAIT_OBJECTS];
+  
+
+  struct EXTRACTOR_PluginList *ppos;
+
+  i = 0;
+  for (ppos = plugins; NULL != ppos; ppos = ppos->next)
+  {
+    if (i == MAXIMUM_WAIT_OBJECTS)
+      return -1;
+    if (ppos->seek_request == -1)
+      continue;
+    switch (ppos->flags)
+    {
+    case EXTRACTOR_OPTION_DEFAULT_POLICY:
+    case EXTRACTOR_OPTION_OUT_OF_PROCESS_NO_RESTART:
+      if (WaitForSingleObject (ppos->ov_read.hEvent, 0) == WAIT_OBJECT_0)
+      {
+        ResetEvent (ppos->ov_read.hEvent);
+        bresult = ReadFile (ppos->cpipe_out, &i, 0, &bytes_read, &ppos->ov_read);
+        if (bresult == TRUE)
+        {
+          SetEvent (ppos->ov_read.hEvent);
+        }
+        else
+        {
+          DWORD err = GetLastError ();
+          if (err != ERROR_IO_PENDING)
+            SetEvent (ppos->ov_read.hEvent);
+        }
+      }
+      events[i] = ppos->ov_read.hEvent;
+      i++;
+      break;
+    case EXTRACTOR_OPTION_IN_PROCESS:
+      break;
+    case EXTRACTOR_OPTION_DISABLED:
+      break;
     }
-  if (want_shm)
+  }
+
+  ms = 10000;
+  first_ready = WaitForMultipleObjects (i, events, FALSE, ms);
+  if (first_ready == WAIT_TIMEOUT || first_ready == WAIT_FAILED)
+    /* an error or timeout -> something's wrong or all plugins hung up */
+    return -1;
+
+  i = 0;
+  result = 0;
+  for (ppos = plugins; NULL != ppos; ppos = ppos->next)
+  {
+    int read_result;
+    switch (ppos->flags)
     {
-#ifndef WINDOWS
-      if (NULL != ptr)
-        munmap (ptr, size);
-      if (shmid != -1)
-        close (shmid);
-      shm_unlink (fn);
-      if (NULL != tptr)
-        {
-          munmap (tptr, tsize);
-          shm_unlink (tfn);
-          if (tshmid != -1)
-            close (tshmid);
-        }
-#else
-      UnmapViewOfFile (ptr);
-      CloseHandle (map);
-      if (tptr != NULL)
-        {
-          UnmapViewOfFile (tptr);
-          CloseHandle (tmap);
-        }
+    case EXTRACTOR_OPTION_DEFAULT_POLICY:
+    case EXTRACTOR_OPTION_OUT_OF_PROCESS_NO_RESTART:
+      if (ppos->seek_request == -1)
+        continue;
+      if (i < first_ready)
+      {
+        i += 1;
+        continue;
+      }
+      dwresult = WaitForSingleObject (ppos->ov_read.hEvent, 0);
+      read_result = 0;
+      if (dwresult == WAIT_OBJECT_0)
+      {
+        read_result = receive_reply (ppos, proc, proc_cls);
+        result += 1;
+      }
+      if (dwresult == WAIT_FAILED || read_result < 0)
+      {
+        stop_process (ppos);
+        if (dwresult == WAIT_FAILED)
+          result += 1;
+      }
+      i++;
+      break;
+    case EXTRACTOR_OPTION_IN_PROCESS:
+      break;
+    case EXTRACTOR_OPTION_DISABLED:
+      break;
+    }
+  }
+  return result;
+}
+
 #endif
+
+static int64_t
+seek_to_new_position (struct EXTRACTOR_PluginList *plugins, int fd, int64_t fsize, int64_t current_position)
+{
+  int64_t min_pos = fsize;
+  struct EXTRACTOR_PluginList *ppos;
+  for (ppos = plugins; NULL != ppos; ppos = ppos->next)
+  {
+    switch (ppos->flags)
+    {
+    case EXTRACTOR_OPTION_DEFAULT_POLICY:
+    case EXTRACTOR_OPTION_OUT_OF_PROCESS_NO_RESTART:
+    case EXTRACTOR_OPTION_IN_PROCESS:
+    if (ppos->seek_request > 0 && ppos->seek_request >= current_position &&
+        ppos->seek_request <= min_pos)
+      min_pos = ppos->seek_request;
+      break;
+    case EXTRACTOR_OPTION_DISABLED:
+      break;
     }
+  }
+  if (min_pos >= fsize)
+    return -1;
+#if WINDOWS
+  _lseeki64 (fd, min_pos, SEEK_SET);
+#elif !HAVE_SEEK64
+  lseek64 (fd, min_pos, SEEK_SET);
+#else
+  if (min_pos >= INT_MAX)
+    return -1;
+  lseek (fd, (ssize_t) min_pos, SEEK_SET);
+#endif
+  return min_pos;
 }
 
+static void
+load_in_process_plugin (struct EXTRACTOR_PluginList *plugin)
+{
+  switch (plugin->flags)
+  {
+  case EXTRACTOR_OPTION_DEFAULT_POLICY:
+  case EXTRACTOR_OPTION_OUT_OF_PROCESS_NO_RESTART:
+  case EXTRACTOR_OPTION_DISABLED:
+    break;
+  case EXTRACTOR_OPTION_IN_PROCESS:
+    plugin_load (plugin);
+    break;
+  }
+}
 
 /**
- * If the given data is compressed using gzip or bzip2, decompress
- * it.  Run 'extract' on the decompressed contents (or the original
- * contents if they were not compressed).
+ * Extract keywords using the given set of plugins.
  *
  * @param plugins the list of plugins to use
- * @param data data to process, never NULL
- * @param size number of bytes in data
- * @param tdata end of file data, or NULL
- * @param tsize number of bytes in tdata
+ * @param data data to process, or NULL if fds is not -1
+ * @param fd file to read data from, or -1 if data is not NULL
+ * @param fsize size of data or size of file
+ * @param buffer a buffer with data alteady read from the file (if fd != -1)
+ * @param buffer_size size of buffer
  * @param proc function to call for each meta data item found
  * @param proc_cls cls argument to proc
  */
 static void
-decompress_and_extract (struct EXTRACTOR_PluginList *plugins,
-                        const unsigned char * data,
-                        size_t size,
-                        const char * tdata,
-                        size_t tsize,
-                        EXTRACTOR_MetaDataProcessor proc,
-                        void *proc_cls) {
-  unsigned char * buf;
-  unsigned char * rbuf;
-  size_t dsize;
-#if HAVE_ZLIB
-  z_stream strm;
-  int ret;
-  size_t pos;
+do_extract (struct EXTRACTOR_PluginList *plugins, const char *data, int fd, int64_t fsize, void *buffer, size_t buffer_size, EXTRACTOR_MetaDataProcessor proc, void *proc_cls)
+{
+  int shm_result;
+  unsigned char *shm_ptr;
+#if !WINDOWS
+  int shm_id;
+#else
+  HANDLE map_handle;
 #endif
-#if HAVE_LIBBZ2
-  bz_stream bstrm;
-  int bret;
-  size_t bpos;
+  char shm_name[MAX_SHM_NAME + 1];
+
+  struct EXTRACTOR_PluginList *ppos;
+
+  int64_t position = 0;
+  size_t map_size;
+  ssize_t read_result;
+  int kill_plugins = 0;
+
+  map_size = (fd == -1) ? fsize : MAX_READ;
+
+  /* Make a shared memory object. Even if we're running in-process. Simpler that way */
+#if !WINDOWS
+  shm_result = make_shm_posix ((void **) &shm_ptr, &shm_id, shm_name, MAX_SHM_NAME,
+      map_size);
+#else  
+  shm_result = make_shm_w32 ((void **) &shm_ptr, &map_handle, shm_name, MAX_SHM_NAME,
+      map_size);
 #endif
+  if (shm_result != 0)
+    return;
 
-  buf = NULL;
-  dsize = 0;
-#if HAVE_ZLIB
-  /* try gzip decompression first */
-  if ( (size >= 12) &&
-       (data[0] == 0x1f) &&
-       (data[1] == 0x8b) &&
-       (data[2] == 0x08) ) 
+  /* This three-loops-instead-of-one construction is intended to increase parallelism */
+  for (ppos = plugins; NULL != ppos; ppos = ppos->next)
+    start_process (ppos);
+
+  for (ppos = plugins; NULL != ppos; ppos = ppos->next)
+    load_in_process_plugin (ppos);
+
+  for (ppos = plugins; NULL != ppos; ppos = ppos->next)
+    write_plugin_data (ppos);
+
+  for (ppos = plugins; NULL != ppos; ppos = ppos->next)
+    init_plugin_state (ppos, shm_name, fsize);
+
+  while (1)
+  {
+    int plugins_not_ready = 0;
+    if (fd != -1)
     {
-      /* Process gzip header */
-      unsigned int gzip_header_length = 10;
-      
-      if (data[3] & 0x4) /* FEXTRA  set */
-        gzip_header_length += 2 + (unsigned) (data[10] & 0xff)
-          + (((unsigned) (data[11] & 0xff)) * 256);
-      
-      if (data[3] & 0x8) /* FNAME set */
-        {
-          const unsigned char * cptr = data + gzip_header_length;
-          /* stored file name is here */
-          while (cptr < data + size)
-            {
-              if ('\0' == *cptr)
-                break;        
-              cptr++;
-            }
-          if (0 != proc (proc_cls,
-                         "<zlib>",
-                         EXTRACTOR_METATYPE_FILENAME,
-                         EXTRACTOR_METAFORMAT_C_STRING,
-                         "text/plain",
-                         (const char*) (data + gzip_header_length),
-                         cptr - (data + gzip_header_length)))
-            return; /* done */    
-          gzip_header_length = (cptr - data) + 1;
-        }
-      if (data[3] & 0x16) /* FCOMMENT set */
-        {
-          const unsigned char * cptr = data + gzip_header_length;
-          /* stored comment is here */    
-          while (cptr < data + size)
-            {
-              if('\0' == *cptr)
-                break;
-              cptr ++;
-            }   
-          if (0 != proc (proc_cls,
-                         "<zlib>",
-                         EXTRACTOR_METATYPE_COMMENT,
-                         EXTRACTOR_METAFORMAT_C_STRING,
-                         "text/plain",
-                         (const char*) (data + gzip_header_length),
-                         cptr - (data + gzip_header_length)))
-            return; /* done */
-          gzip_header_length = (cptr - data) + 1;
-        }
-      if(data[3] & 0x2) /* FCHRC set */
-        gzip_header_length += 2;
-      memset(&strm,
-             0,
-             sizeof(z_stream));
-#ifdef ZLIB_VERNUM
-      gzip_header_length = 0;
-#endif
-      if (size > gzip_header_length) 
-        {
-          strm.next_in = (Bytef*) data + gzip_header_length;
-          strm.avail_in = size - gzip_header_length;
-        }
+      /* fill the share buffer with data from the file */
+      if (buffer_size > 0)
+        memcpy (shm_ptr, buffer, buffer_size);
+      read_result = READ (fd, &shm_ptr[buffer_size], MAX_READ - buffer_size);
+      if (read_result <= 0)
+        break;
       else
-        {
-          strm.next_in = (Bytef*) data;
-          strm.avail_in = 0;
-        }
-      strm.total_in = 0;
-      strm.zalloc = NULL;
-      strm.zfree = NULL;
-      strm.opaque = NULL;
-      
-      /*
-       * note: maybe plain inflateInit(&strm) is adequate,
-       * it looks more backward-compatible also ;
-       *
-       * ZLIB_VERNUM isn't defined by zlib version 1.1.4 ;
-       * there might be a better check.
-       */
-      if (Z_OK == inflateInit2(&strm,
-#ifdef ZLIB_VERNUM
-                               15 + 32
-#else
-                               -MAX_WBITS
-#endif
-                               )) {
-        dsize = 2 * size;
-        if (dsize > MAX_DECOMPRESS)
-          dsize = MAX_DECOMPRESS;
-        buf = malloc(dsize);
-        pos = 0;
-        if (buf == NULL) 
-          {
-            inflateEnd(&strm);
-          } 
-        else 
-          {
-            strm.next_out = (Bytef*) buf;
-            strm.avail_out = dsize;
-            do
-              {
-                ret = inflate(&strm,
-                              Z_SYNC_FLUSH);
-                if (ret == Z_OK) 
-                  {
-                    if (dsize == MAX_DECOMPRESS)
-                      break;
-                    pos += strm.total_out;
-                    strm.total_out = 0;
-                    dsize *= 2;
-                    if (dsize > MAX_DECOMPRESS)
-                      dsize = MAX_DECOMPRESS;
-                    rbuf = realloc(buf, dsize);
-                    if (rbuf == NULL)
-                      {
-                        free (buf);
-                        buf = NULL;
-                        break;
-                      }
-                    buf = rbuf;
-                    strm.next_out = (Bytef*) &buf[pos];
-                    strm.avail_out = dsize - pos;
-                  }
-                else if (ret != Z_STREAM_END) 
-                  {
-                    /* error */
-                    free(buf);
-                    buf = NULL;
-                  }
-              } while ( (buf != NULL) &&                
-                        (ret != Z_STREAM_END) );
-            dsize = pos + strm.total_out;
-            inflateEnd(&strm);
-            if ( (dsize == 0) &&
-                 (buf != NULL) )
-              {
-                free(buf);
-                buf = NULL;
-              }
-          }
-      }
+        map_size = read_result + buffer_size;
+      if (buffer_size > 0)
+         buffer_size = 0;
     }
-#endif
-  
-#if HAVE_LIBBZ2
-  if ( (size >= 4) &&
-       (data[0] == 'B') &&
-       (data[1] == 'Z') &&
-       (data[2] == 'h') ) 
+    for (ppos = plugins; NULL != ppos; ppos = ppos->next)
+      plugins_not_ready += give_shm_to_plugin (ppos, position, map_size);
+    for (ppos = plugins; NULL != ppos; ppos = ppos->next)
+      ask_in_process_plugin (ppos, position, shm_ptr, proc, proc_cls);
+    while (plugins_not_ready > 0 && !kill_plugins)
     {
-      /* now try bz2 decompression */
-      memset(&bstrm,
-             0,
-             sizeof(bz_stream));
-      bstrm.next_in = (char*) data;
-      bstrm.avail_in = size;
-      bstrm.total_in_lo32 = 0;
-      bstrm.total_in_hi32 = 0;
-      bstrm.bzalloc = NULL;
-      bstrm.bzfree = NULL;
-      bstrm.opaque = NULL;
-      if ( (buf == NULL) &&
-           (BZ_OK == BZ2_bzDecompressInit(&bstrm,
-                                          0,
-                                          0)) ) 
-        {
-          dsize = 2 * size;
-          if (dsize > MAX_DECOMPRESS)
-            dsize = MAX_DECOMPRESS;
-          buf = malloc(dsize);
-          bpos = 0;
-          if (buf == NULL) 
-            {
-              BZ2_bzDecompressEnd(&bstrm);
-            }
-          else 
-            {
-              bstrm.next_out = (char*) buf;
-              bstrm.avail_out = dsize;
-              do {
-                bret = BZ2_bzDecompress(&bstrm);
-                if (bret == Z_OK) 
-                  {
-                    if (dsize == MAX_DECOMPRESS)
-                      break;
-                    bpos += bstrm.total_out_lo32;
-                    bstrm.total_out_lo32 = 0;
-                    dsize *= 2;
-                    if (dsize > MAX_DECOMPRESS)
-                      dsize = MAX_DECOMPRESS;
-                    rbuf = realloc(buf, dsize);
-                    if (rbuf == NULL)
-                      {
-                        free (buf);
-                        buf = NULL;
-                        break;
-                      }
-                    buf = rbuf;
-                    bstrm.next_out = (char*) &buf[bpos];
-                    bstrm.avail_out = dsize - bpos;
-                  } 
-                else if (bret != BZ_STREAM_END) 
-                  {
-                    /* error */
-                    free(buf);
-                    buf = NULL;
-                  }
-              } while ( (buf != NULL) &&
-                        (bret != BZ_STREAM_END) );
-              dsize = bpos + bstrm.total_out_lo32;
-              BZ2_bzDecompressEnd(&bstrm);
-              if ( (dsize == 0) &&
-                   (buf != NULL) )
-                {
-                  free(buf);
-                  buf = NULL;
-                }
-            }
-        }
+      int ready = wait_for_reply (plugins, proc, proc_cls);
+      if (ready <= 0)
+        kill_plugins = 1;
+      plugins_not_ready -= ready;
     }
-#endif  
-  if (buf != NULL) 
+    if (kill_plugins)
+      break;
+    if (fd != -1)
     {
-      data = buf;
-      size = dsize;
+      position += map_size;
+      position = seek_to_new_position (plugins, fd, fsize, position);
+      if (position < 0)
+        break;
     }
-  extract (plugins,
-           (const char*) data,
-           size,
-           tdata, 
-           tsize,
-           proc,
-           proc_cls);
-  if (buf != NULL)
-    free(buf);
-  errno = 0; /* kill transient errors */
-}
-
+    else
+      break;
+  }
 
-/**
- * Open a file
- */
-static int file_open(const char *filename, int oflag, ...)
-{
-  int mode;
-  const char *fn;
-#ifdef MINGW
-  char szFile[_MAX_PATH + 1];
-  long lRet;
+  if (kill_plugins)
+    for (ppos = plugins; NULL != ppos; ppos = ppos->next)
+      stop_process (ppos);
+  for (ppos = plugins; NULL != ppos; ppos = ppos->next)
+    discard_plugin_state (ppos);
 
-  if ((lRet = plibc_conv_to_win_path(filename, szFile)) != ERROR_SUCCESS)
-  {
-    errno = ENOENT;
-    SetLastError(lRet);
-    return -1;
-  }
-  fn = szFile;
+#if WINDOWS
+  destroy_shm_w32 (shm_ptr, map_handle);
 #else
-  fn = filename;
-#endif
-  mode = 0;
-#ifdef MINGW
-  /* Set binary mode */
-  mode |= O_BINARY;
+  destroy_shm_posix (shm_ptr, shm_id, (fd == -1) ? fsize : MAX_READ, shm_name);
 #endif
-  return OPEN(fn, oflag, mode);
 }
 
 
-#ifndef O_LARGEFILE
-#define O_LARGEFILE 0
-#endif
-
-
 /**
  * Extract keywords from a file using the given set of plugins.
  * If needed, opens the file and loads its data (via mmap).  Then
@@ -1478,93 +2106,152 @@ static int file_open(const char *filename, int oflag, ...)
  */
 void
 EXTRACTOR_extract (struct EXTRACTOR_PluginList *plugins,
-                   const char *filename,
-                   const void *data,
-                   size_t size,
-                   EXTRACTOR_MetaDataProcessor proc,
-                   void *proc_cls)
+       const char *filename,
+       const void *data,
+       size_t size,
+       EXTRACTOR_MetaDataProcessor proc,
+       void *proc_cls)
 {
-  int fd;
-  void * buffer;
-  void * tbuffer;
-  struct stat fstatbuf;
-  size_t fsize;
-  size_t tsize;
-  int eno;
-  off_t offset;
-  long pg;
-#ifdef WINDOWS
-  SYSTEM_INFO sys;
-#endif
-
-  fd = -1;
-  buffer = NULL;
-  if ( (data == NULL) &&
-       (filename != NULL) &&
-       (0 == STAT(filename, &fstatbuf)) &&
-       (!S_ISDIR(fstatbuf.st_mode)) &&
-       (-1 != (fd = file_open (filename,
-                               O_RDONLY | O_LARGEFILE))) )
-    {      
-      fsize = (fstatbuf.st_size > 0xFFFFFFFF) ? 0xFFFFFFFF : fstatbuf.st_size;
-      if (fsize == 0) 
-        {
-          close(fd);
-          return;
-        }
-      if (fsize > MAX_READ)
-        fsize = MAX_READ;
-      buffer = MMAP(NULL, fsize, PROT_READ, MAP_PRIVATE, fd, 0);
-      if ( (buffer == NULL) || (buffer == (void *) -1) ) 
-        {
-          eno = errno;
-          close(fd);
-          errno = eno;
-          return;
-        }
+  int fd = -1;
+  struct stat64 fstatbuf;
+  int64_t fsize = 0;
+  int memory_only = 1;
+  int compression_type = -1;
+  void *buffer = NULL;
+  size_t buffer_size;
+  int decompression_result;
+
+  /* If data is not given, then we need to read it from the file. Try opening it */
+  if ((data == NULL) &&
+      (filename != NULL) &&
+      (0 == STAT64(filename, &fstatbuf)) &&
+      (!S_ISDIR(fstatbuf.st_mode)) &&
+      (-1 != (fd = file_open (filename,
+             O_RDONLY | O_LARGEFILE))))
+  {
+    /* Empty files are of no interest */
+    fsize = fstatbuf.st_size;
+    if (fsize == 0) 
+    {
+       close(fd);
+       return;
     }
-  if ( (buffer == NULL) &&
-       (data == NULL) )
+    /* File is too big -> can't read it into memory */
+    if (fsize > MAX_READ)
+      memory_only = 0;
+  }
+
+  /* Data is not given, and we've failed to open the file with data -> exit */
+  if ((fsize == 0) && (data == NULL))
+    return;
+  /* fsize is now size of the data OR size of the file */
+  if (data != NULL)
+    fsize = size;
+
+  errno = 0;
+  /* Peek at first few bytes of the file (or of the data), and see if it's compressed.
+   * If data is NULL, buffer is allocated by the function and holds the first few bytes
+   * of the file, buffer_size is set too.
+   */
+  compression_type = get_compression_type (data, fd, fsize, &buffer, &buffer_size);
+  if (compression_type < 0)
+  {
+    /* errno is set by get_compression_type () */
+    if (fd != -1)
+      close (fd);
     return;
-  /* for footer extraction */
-  tsize = 0;
-  tbuffer = NULL;
-  if ( (data == NULL) &&
-       (fstatbuf.st_size > fsize) &&
-       (fstatbuf.st_size > MAX_READ) )
+  }
+  if (compression_type > 0)
+  {
+    /* Don't assume that MAX_DECOMPRESS < MAX_READ */
+    if ((fsize > MAX_DECOMPRESS) || (fsize > MAX_READ))
+    {
+      /* File or data is to big to be decompressed in-memory (the only kind of decompression we do) */
+      errno = EFBIG;
+      if (fd != -1)
+        close (fd);
+      if (buffer != NULL)
+        free (buffer);
+      return;
+    }
+    /* Decompress data (or file contents + what we've read so far. Either way it writes a new
+     * pointer to buffer, sets buffer_size, and frees the old buffer (if it wasn't NULL).
+     * In case of failure it cleans up the buffer after itself.
+     * Will also report compression-related metadata to the caller.
+     */
+    decompression_result = try_to_decompress (data, fd, fsize, compression_type, &buffer, &buffer_size, proc, proc_cls);
+    if (decompression_result != 0)
+    {
+      /* Buffer is taken care of already */
+      close (fd);
+      errno = EILSEQ;
+      return;
+    }
+    else
     {
-      pg = SYSCONF (_SC_PAGE_SIZE);
-      if ( (pg > 0) &&
-           (pg < MAX_READ) )
-        {
-          offset = (1 + (fstatbuf.st_size - MAX_READ) / pg) * pg;
-          if (offset < fstatbuf.st_size)
-            {
-              tsize = fstatbuf.st_size - offset;
-              tbuffer = MMAP (NULL, tsize, PROT_READ, MAP_PRIVATE, fd, offset);
-              if ( (tbuffer == NULL) || (tbuffer == (void *) -1) ) 
-                {
-                  tsize = 0;
-                  tbuffer = NULL;
-                }
-            }
-        }
+      close (fd);
+      fd = -1;
     }
-  decompress_and_extract (plugins,
-                          buffer != NULL ? buffer : data,
-                          buffer != NULL ? fsize : size,
-                          tbuffer,
-                          tsize,
-                          proc,
-                          proc_cls);
+  }
+
+  /* Now we either have a non-NULL data of fsize bytes
+   * OR a valid fd to read from and a small buffer of buffer_size bytes
+   * OR an invalid fd and a big buffer of buffer_size bytes
+   * Simplify this situation a bit:
+   */
+  if ((data == NULL) && (fd == -1) && (buffer_size > 0))
+  {
+    data = (const void *) buffer;
+    fsize = buffer_size;
+  }
+
+  /* Now we either have a non-NULL data of fsize bytes
+   * OR a valid fd to read from and a small buffer of buffer_size bytes
+   * and we might need to free the buffer later in either case
+   */
+
+  /* do_extract () might set errno itself, but from our point of view everything is OK */
+  errno = 0;
+
+  do_extract (plugins, data, fd, fsize, buffer, buffer_size, proc, proc_cls);
+
   if (buffer != NULL)
-    MUNMAP (buffer, fsize);
-  if (tbuffer != NULL)
-    MUNMAP (tbuffer, tsize);
+    free (buffer);
   if (-1 != fd)
-    close(fd);  
+    close(fd);
 }
 
+
+#if WINDOWS
+void CALLBACK 
+RundllEntryPoint (HWND hwnd, 
+                  HINSTANCE hinst, 
+                  LPSTR lpszCmdLine, 
+                  int nCmdShow)
+{
+  intptr_t in_h;
+  intptr_t out_h;
+  int in, out;
+
+  sscanf(lpszCmdLine, "%lu %lu", &in_h, &out_h);
+  in = _open_osfhandle (in_h, _O_RDONLY);
+  out = _open_osfhandle (out_h, 0);
+  setmode (in, _O_BINARY);
+  setmode (out, _O_BINARY);
+  process_requests (read_plugin_data (in),
+                    in, out);
+}
+
+void CALLBACK 
+RundllEntryPointA (HWND hwnd, 
+                  HINSTANCE hinst, 
+                  LPSTR lpszCmdLine, 
+                  int nCmdShow)
+{
+  return RundllEntryPoint(hwnd, hinst, lpszCmdLine, nCmdShow);
+}
+#endif
+
 /**
  * Initialize gettext and libltdl (and W32 if needed).
  */
@@ -1579,12 +2266,12 @@ void __attribute__ ((constructor)) EXTRACTOR_ltdl_init() {
   if (err > 0) {
 #if DEBUG
     fprintf(stderr,
-            _("Initialization of plugin mechanism failed: %s!\n"),
-            lt_dlerror());
+      _("Initialization of plugin mechanism failed: %s!\n"),
+      lt_dlerror());
 #endif
     return;
   }
-#ifdef MINGW
+#if WINDOWS
   plibc_init("GNU", PACKAGE);
 #endif
 }
@@ -1594,12 +2281,10 @@ void __attribute__ ((constructor)) EXTRACTOR_ltdl_init() {
  * Deinit.
  */
 void __attribute__ ((destructor)) EXTRACTOR_ltdl_fini() {
-#ifdef MINGW
+#if WINDOWS
   plibc_shutdown();
 #endif
   lt_dlexit ();
 }
 
-
-
 /* end of extractor.c */
diff --git a/src/main/extractor_plugins.c b/src/main/extractor_plugins.c
index 026e86b..f5c38f0 100644
--- a/src/main/extractor_plugins.c
+++ b/src/main/extractor_plugins.c
@@ -204,15 +204,24 @@ plugin_load (struct EXTRACTOR_PluginList *plugin)
       plugin->flags = EXTRACTOR_OPTION_DISABLED;

       return -1;

     }

-  plugin->extractMethod = get_symbol_with_prefix (plugin->libraryHandle,

-                                                  "_EXTRACTOR_%s_extract",

+  plugin->extract_method = get_symbol_with_prefix (plugin->libraryHandle,

+                                                  "_EXTRACTOR_%s_extract_method",

                                                   plugin->libname,

                                                   &plugin->specials);

-  if (plugin->extractMethod == NULL) 

+  plugin->init_state_method = get_symbol_with_prefix (plugin->libraryHandle,

+                                                  "_EXTRACTOR_%s_init_state_method",

+                                                  plugin->libname,

+                                                  &plugin->specials);

+  plugin->discard_state_method = get_symbol_with_prefix (plugin->libraryHandle,

+                                                  "_EXTRACTOR_%s_discard_state_method",

+                                                  plugin->libname,

+                                                  &plugin->specials);

+  if (plugin->extract_method == NULL || plugin->init_state_method == NULL ||

+      plugin->discard_state_method == NULL) 

     {

 #if DEBUG

       fprintf (stderr,

-               "Resolving `extract' method of plugin `%s' failed: %s\n",

+               "Resolving `extract', 'init_state' or 'discard_state' method(s) of plugin `%s' failed: %s\n",

                plugin->short_libname,

                lt_dlerror ());

 #endif

@@ -243,8 +252,15 @@ EXTRACTOR_plugin_add (struct EXTRACTOR_PluginList * prev,
                       enum EXTRACTOR_Options flags)

 {

   struct EXTRACTOR_PluginList *result;

+  struct EXTRACTOR_PluginList *i;

   char *libname;

 

+  for (i = prev; i != NULL; i = i->next)

+  {

+    if (strcmp (i->short_libname, library) == 0)

+      return prev;

+  }

+

   libname = find_plugin (library);

   if (libname == NULL)

     {

diff --git a/src/main/extractor_plugins.h b/src/main/extractor_plugins.h
index ea0eabb..bea5c2b 100644
--- a/src/main/extractor_plugins.h
+++ b/src/main/extractor_plugins.h
@@ -64,7 +64,9 @@ struct EXTRACTOR_PluginList
   /**

    * Pointer to the function used for meta data extraction.

    */

-  EXTRACTOR_ExtractMethod extractMethod;

+  EXTRACTOR_extract_method extract_method;

+  EXTRACTOR_init_state_method init_state_method;

+  EXTRACTOR_discard_state_method discard_state_method;

 

   /**

    * Options for the plugin.

@@ -84,26 +86,72 @@ struct EXTRACTOR_PluginList
   enum EXTRACTOR_Options flags;

 

   /**

-   * Process ID of the child process for this plugin. 0 for 

-   * none.

+   * Process ID of the child process for this plugin. 0 for none.

    */

-#ifndef WINDOWS

+#if !WINDOWS

   int cpid;

 #else

   HANDLE hProcess;

 #endif

 

   /**

-   * Pipe used to send information about shared memory segments to

-   * the child process.  NULL if not initialized.

+   * Pipe used to communicate information to the plugin child process.

+   * NULL if not initialized.

    */

+#if !WINDOWS

   FILE *cpipe_in;

+#else

+  HANDLE cpipe_in;

+#endif

+

+  /**

+   * A position this plugin wants us to seek to. -1 if it's finished.

+   * Starts at 0;

+   */

+  int64_t seek_request;

+

+#if !WINDOWS

+  int shm_id;

+#else

+  HANDLE map_handle;

+#endif

+

+  void *state;

+

+  int64_t fsize;

+

+  int64_t position;

+

+  unsigned char *shm_ptr;

+

+  size_t map_size;

 

   /**

    * Pipe used to read information about extracted meta data from

-   * the child process.  -1 if not initialized.

+   * the plugin child process.  -1 if not initialized.

    */

+#if !WINDOWS

   int cpipe_out;

+#else

+  HANDLE cpipe_out;

+#endif

+

+#if WINDOWS

+  /**

+   * A structure for overlapped reads on W32.

+   */

+  OVERLAPPED ov_read;

+

+  /**

+   * A structure for overlapped writes on W32.

+   */

+  OVERLAPPED ov_write;

+

+  /**

+   * A write buffer for overlapped writes on W32

+   */

+  unsigned char *ov_write_buffer;

+#endif

 };

 

 /**