24 files changed, 1831 insertions, 0 deletions

diff --git a/contrib/scripts/coverage.sh b/contrib/scripts/coverage.sh
new file mode 100755
index 000000000..dd6a6ab53
--- /dev/null
+++ b/contrib/scripts/coverage.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+# make sure configure was run with coverage enabled...
+lcov --directory . --zerocounters
+make check
+rm `find * -name "test_*.gc??"` `find * -name "perf_*.gc??"`
+for n in `find * -name "*.gc??" | grep libs`
+do
+  cd `dirname $n`
+  mv `basename $n` ..
+  cd -
+done
+lcov --directory . --capture --output-file app.info
+mkdir -p doc/coverage
+genhtml -o doc/coverage app.info
diff --git a/contrib/scripts/debug b/contrib/scripts/debug
new file mode 100755
index 000000000..3de2c9a14
--- /dev/null
+++ b/contrib/scripts/debug
@@ -0,0 +1,37 @@
+#!/bin/bash
+# /proc/sys/kernel/core_pattern should be core.%p.%E
+
+COREPID=$1
+
+COREFILES=`ls -1 *core.$COREPID* 2>/dev/null | wc -l`
+COREFILE=`ls -1 *core.$COREPID* 2>/dev/null | head -n 1`
+
+if [ $COREFILES -gt 1 ]; then
+    echo "Multiple files, using $COREFILE"
+fi
+
+
+if [ $COREFILES -eq 0 ]; then
+    SERVICENAME=$1
+    COREFILES=`ls -1 core.*.*$SERVICENAME 2>/dev/null | wc -l`
+    COREFILE=`ls -1 core.*.*$SERVICENAME 2>/dev/null | head -n 1`
+
+    if [ $COREFILES -gt 1 ]; then
+        echo "Multiple files, using $COREFILE"
+    fi
+fi
+
+if [ $COREFILES -eq 0 ]; then
+    echo "Core file for $1 not found"
+    exit 1
+fi
+
+echo "Using $COREFILE"
+
+EXECPATH=${COREFILE#*!}
+EXECPATH=`echo $EXECPATH | sed -e 's/!/\//g'`
+echo $EXECPATH
+echo ""
+echo ""
+
+gdb --core $COREFILE /$EXECPATH
diff --git a/contrib/scripts/find_typedefs.py b/contrib/scripts/find_typedefs.py
new file mode 100644
index 000000000..68f5c2782
--- /dev/null
+++ b/contrib/scripts/find_typedefs.py
@@ -0,0 +1,99 @@
+from __future__ import print_function
+import os
+import re
+import sys
+
+
+debug = False
+
+
+def get_td_from_function_signature(line, file, num):
+    left_paren = line.find('(')
+    if left_paren > 0:
+        left_paren += 1
+        li = line[left_paren:]
+        right_paren = line.find(')')
+        if right_paren > 0 and right_paren > left_paren and line[right_paren:].find('(') >= 0:
+            fname = line[:right_paren]
+            fname = fname.lstrip(' ').lstrip('*').lstrip(' ').rstrip(' ')
+            if len(fname) > 0:
+                if debug:
+                    print("from {0}:{1}".format(file, num))
+                print("-T {0}".format(fname))
+
+
+def get_td_from_simple_type(line, file, num):
+    line = line.rstrip(' ').rstrip('\t').rstrip(' ').rstrip('\t')
+    right_space = line.rfind(' ')
+    right_tab = line.rfind('\t')
+    sep = right_tab if right_tab > right_space else right_space
+    sep += 1
+    tname = line[sep:]
+    tname = tname.lstrip('*')
+    if len(tname) > 0:
+        if debug:
+            print("from {0}:{1}".format(file, num))
+        print("-T {0}".format(tname))
+
+
+def find_typedefs(file):
+    with open(file, 'rb') as f:
+        td = False
+        td_struct = False
+        td_level = 0
+        td_line = []
+        data = f.read()
+        for i, l in enumerate(data.splitlines(False)):
+            # Don't try to be too smart: only count lines that begin with 'typedef '
+            l = l.rstrip(' ').rstrip('\t')
+            if len(l) == 0:
+                continue
+            if not td:
+                if l[:8] != 'typedef ':
+                    continue
+                else:
+                    td = True
+                    if l[8:].lstrip(' ').lstrip('\t')[:6] == 'struct':
+                        td_struct = True
+            if td_struct:
+                leftcbrace = l.find('{')
+                if leftcbrace >= 0:
+                    if td_level == 0:
+                        td_line.append(l[:leftcbrace])
+                    l = l[leftcbrace + 1:]
+                    td_level += 1
+                rightcbrace = l.rfind('}')
+                if rightcbrace >= 0:
+                    td_level -= 1
+                    if td_level == 0:
+                        td_line.append(l[rightcbrace + 1:])
+            else:
+                td_line.append(l)
+            if len(l) > 0 and l[-1] == ';' and(not td_struct or td_level == 0):
+                td_line = ' '.join(td_line)
+                td_line = td_line[:-1]
+                if len(td_line) > 0:
+                    if td_line[-1] == ')':
+                        get_td_from_function_signature(td_line, file, i)
+                    else:
+                        get_td_from_simple_type(td_line, file, i)
+                td_line = []
+                td = False
+                td_struct = False
+                td_level = 0
+
+
+def scan_dir(d):
+    for dirpath, dirs, files in os.walk(d):
+        for f in files:
+            if re.match(r'(?!lt_).+\.(c|cc|h)$', f):
+                file = os.path.join(dirpath, f)
+                find_typedefs(file)
+
+
+if __name__ == '__main__':
+    if len(sys.argv[1:]) == 0:
+        arg = os.getcwd()
+    else:
+        arg = sys.argv[1]
+        scan_dir(arg)
diff --git a/contrib/scripts/gdb-iterate-dll.py b/contrib/scripts/gdb-iterate-dll.py
new file mode 100644
index 000000000..79d46aa96
--- /dev/null
+++ b/contrib/scripts/gdb-iterate-dll.py
@@ -0,0 +1,40 @@
+from gdb import *
+
+
+def search_dll(head, field, match, pfield):
+    """
+    Search in a DLL by iterates over it.
+
+    head: name of the symbol denoting the head of the DLL
+    field: the field that should be search for match
+    match: the mathing value for field
+    pfield: the field whose value is to be printed for matched elements; None to
+      print all fields of the matched elemented
+    """
+
+    (symbol, _) = lookup_symbol(head)
+    if symbol is None:
+        print("Can't find symbol: " + head)
+        return
+    symbol_val = symbol.value()
+    while symbol_val:
+        symbol_val_def = symbol_val.dereference()
+        field_val = symbol_val_def[field]
+        if field_val.type.code == gdb.TYPE_CODE_INT:
+            val = int(field_val)
+            res = (match == val)
+        elif (field_val.type.code == gdb.TYPE_CODE_STRING) or (field_val.type.code == gdb.TYPE_CODE_ARRAY):
+            val = str(field_val)
+            res = (match == val)
+        elif (field_val.type.code == gdb.TYPE_CODE_TYPEDEF):
+            val = str(field_val)
+            res = match in val
+        else:
+            continue
+
+        if res:
+            if pfield is None:
+                print(symbol_val_def)
+            else:
+                print(symbol_val_def[pfield])
+        symbol_val = symbol_val_def["next"]
diff --git a/contrib/scripts/generate-monkey-db.sh b/contrib/scripts/generate-monkey-db.sh
new file mode 100755
index 000000000..2afe55501
--- /dev/null
+++ b/contrib/scripts/generate-monkey-db.sh
@@ -0,0 +1,17 @@
+#!/bin/sh
+
+BASEPATH="$(dirname $0)"
+OLDDIR="${pwd}"
+GN_HOME="/usr/local/bin"
+
+export CC="cparser"
+export CFLAGS="-m32 --seaspider"
+
+cd $BASEPATH/.. && ./configure --prefix=$GN_HOME --with-extractor=$GN_HOME --with-microhttpd=$GN_HOME --with-libgcrypt=$GN_HOME && make && seaspider
+if test "$?" -ne 0
+then
+        echo "FAIL: building GNUnet"
+        exit 1
+fi
+
+cd $OLDDIR
diff --git a/contrib/scripts/gnunet-chk.py.in b/contrib/scripts/gnunet-chk.py.in
new file mode 100755
index 000000000..c976b2143
--- /dev/null
+++ b/contrib/scripts/gnunet-chk.py.in
@@ -0,0 +1,383 @@
+#!@PYTHON@
+# This file is part of GNUnet.
+# (C) 2013, 2018 Christian Grothoff (and other contributing authors)
+#
+# GNUnet is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published
+# by the Free Software Foundation; either version 3, or (at your
+# option) any later version.
+#
+# GNUnet is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with GNUnet; see the file COPYING.  If not, write to the
+# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+# Boston, MA 02110-1301, USA.
+#
+# File:    gnunet-chk.py
+# Brief:   Computes GNUNET style Content Hash Key for a given file
+# Author:  Sree Harsha Totakura
+
+from hashlib import sha512
+import logging
+import os
+import getopt
+import sys
+from Crypto.Cipher import AES
+from functools import reduce
+
+
+# Defaults
+DBLOCK_SIZE = (32 * 1024)   # Data block size
+
+# Pick a multiple of 2 here to achive 8-byte alignment!  We also
+# probably want DBlocks to have (roughly) the same size as IBlocks.
+# With SHA-512, the optimal value is 32768 byte / 128 byte = 256 (128
+# byte = 2 * 512 bits).  DO NOT CHANGE!
+CHK_PER_INODE = 256
+
+CHK_HASH_SIZE = 64              # SHA-512 hash = 512 bits = 64 bytes
+
+CHK_QUERY_SIZE = CHK_HASH_SIZE  # Again a SHA-512 hash
+
+GNUNET_FS_URI_PREFIX = "gnunet://fs/"  # FS CHK URI prefix
+
+GNUNET_FS_URI_CHK_INFIX = "chk/"  # FS CHK URI infix
+
+
+def encode_data_to_string(data):
+    """Returns an ASCII encoding of the given data block like
+    GNUNET_STRINGS_data_to_string() function.
+
+    data: A bytearray representing the block of data which has to be encoded
+    """
+    echart = "0123456789ABCDEFGHIJKLMNOPQRSTUV"
+    assert (None != data)
+    assert (bytearray == type(data))
+    size = len(data)
+    assert (0 != size)
+    vbit = 0
+    wpos = 0
+    rpos = 0
+    bits = 0
+    out = ""
+    while (rpos < size) or (vbit > 0):
+        if (rpos < size) and (vbit < 5):
+            bits = (bits << 8) | data[rpos]  # eat 8 more bits
+            rpos += 1
+            vbit += 8
+        if (vbit < 5):
+            bits <<= (5 - vbit)  # zero-padding
+            assert (vbit == ((size * 8) % 5))
+            vbit = 5
+        out += echart[(bits >> (vbit - 5)) & 31]
+        wpos += 1
+        vbit -= 5
+    assert (0 == vbit)
+    return out
+
+
+def sha512_hash(data):
+    """ Returns the sha512 hash of the given data.
+
+    data: string to hash
+    """
+    hash_obj = sha512()
+    hash_obj.update(data)
+    return hash_obj.digest()
+
+
+class AESKey:
+    """Class for AES Keys. Contains the main key and the initialization
+    vector. """
+
+    key = None                  # The actual AES key
+    iv = None                   # The initialization vector
+    cipher = None               # The cipher object
+    KEY_SIZE = 32               # AES 256-bit key = 32 bytes
+    IV_SIZE = AES.block_size    # Initialization vector size (= AES block size)
+
+    def __init__(self, passphrase):
+        """Creates a new AES key.
+
+        passphrase: string containing the passphrase to get the AES key and
+        initialization vector
+        """
+        passphrase = bytearray(passphrase)
+        self.key = bytearray(self.KEY_SIZE)
+        self.iv = bytearray(self.IV_SIZE)
+        if (len(passphrase) > self.KEY_SIZE):
+            self.key = passphrase[:self.KEY_SIZE]
+            passphrase = passphrase[self.KEY_SIZE:]
+            if (len(passphrase) > self.IV_SIZE):
+                self.iv = passphrase[:self.IV_SIZE]
+            else:
+                self.iv[0:len(passphrase)] = passphrase
+        else:
+            self.key[0:len(passphrase)] = passphrase
+        self.key = str(self.key)
+        self.iv = str(self.iv)
+        assert (len(self.key) == self.KEY_SIZE)
+        assert (len(self.iv) == self.IV_SIZE)
+
+
+def setup_aes_cipher_(aes_key):
+    """Initializes the AES object with settings similar to those in GNUnet.
+
+    aes_key: the AESKey object
+    Returns the newly initialized AES object
+    """
+    return AES.new(aes_key.key, AES.MODE_CFB, aes_key.iv, segment_size=128)
+
+
+def aes_pad_(data):
+    """Adds padding to the data such that the size of the data is a multiple of
+    16 bytes
+
+    data: the data string
+    Returns a tuple:(pad_len, data). pad_len denotes the number of bytes added
+    as padding; data is the new data string with padded bytes at the end
+    """
+    pad_len = len(data) % 16
+    if (0 != pad_len):
+        pad_len = 16 - pad_len
+        pad_bytes = bytearray(15)
+        data += str(pad_bytes[:pad_len])
+    return (pad_len, data)
+
+
+def aes_encrypt(aes_key, data):
+    """Encrypts the given data using AES.
+
+    aes_key: the AESKey object to use for AES encryption
+    data: the data string to encrypt
+    """
+    (pad_len, data) = aes_pad_(data)
+    cipher = setup_aes_cipher_(aes_key)
+    enc_data = cipher.encrypt(data)
+    if (0 != pad_len):
+        enc_data = enc_data[:-pad_len]
+    return enc_data
+
+
+def aes_decrypt(aes_key, data):
+    """Decrypts the given data using AES
+
+    aes_key: the AESKey object to use for AES decryption
+    data: the data string to decrypt
+    """
+    (pad_len, data) = aes_pad_(data)
+    cipher = setup_aes_cipher_(aes_key)
+    ptext = cipher.decrypt(data)
+    if (0 != pad_len):
+        ptext = ptext[:-pad_len]
+    return ptext
+
+
+class Chk:
+    """Class for the content hash key."""
+    key = None
+    query = None
+    fsize = None
+
+    def __init__(self, key, query):
+        assert (len(key) == CHK_HASH_SIZE)
+        assert (len(query) == CHK_QUERY_SIZE)
+        self.key = key
+        self.query = query
+
+    def setSize(self, size):
+        self.fsize = size
+
+    def uri(self):
+        sizestr = repr(self.fsize)
+        if isinstance(self.fsize, int):
+            sizestr = sizestr[:-1]
+        return GNUNET_FS_URI_PREFIX + GNUNET_FS_URI_CHK_INFIX + \
+            encode_data_to_string(bytearray(self.key)) + "." + \
+            encode_data_to_string(bytearray(self.query)) + "." + \
+            sizestr
+
+
+def compute_depth_(size):
+    """Computes the depth of the hash tree.
+
+    size: the size of the file whose tree's depth has to be computed
+    Returns the depth of the tree. Always > 0.
+    """
+    depth = 1
+    fl = DBLOCK_SIZE
+    while (fl < size):
+        depth += 1
+        if ((fl * CHK_PER_INODE) < fl):
+            return depth
+        fl = fl * CHK_PER_INODE
+    return depth
+
+
+def compute_tree_size_(depth):
+    """Calculate how many bytes of payload a block tree of the given depth MAY
+     correspond to at most (this function ignores the fact that some blocks will
+     only be present partially due to the total file size cutting some blocks
+     off at the end).
+
+     depth: depth of the block.  depth==0 is a DBLOCK.
+     Returns the number of bytes of payload a subtree of this depth may
+     correspond to.
+     """
+    rsize = DBLOCK_SIZE
+    for cnt in range(0, depth):
+        rsize *= CHK_PER_INODE
+    return rsize
+
+
+def compute_chk_offset_(depth, end_offset):
+    """Compute the offset of the CHK for the current block in the IBlock
+    above
+
+    depth: depth of the IBlock in the tree (aka overall number of tree levels
+             minus depth); 0 == DBLOCK
+    end_offset: current offset in the overall file, at the *beginning* of the
+                  block for DBLOCK (depth == 0), otherwise at the *end* of the
+                  block (exclusive)
+    Returns the offset in the list of CHKs in the above IBlock
+    """
+    bds = compute_tree_size_(depth)
+    if (depth > 0):
+        end_offset -= 1
+    ret = end_offset / bds
+    return ret % CHK_PER_INODE
+
+
+def compute_iblock_size_(depth, offset):
+    """Compute the size of the current IBLOCK.  The encoder is triggering the
+    calculation of the size of an IBLOCK at the *end* (hence end_offset) of its
+    construction.  The IBLOCK maybe a full or a partial IBLOCK, and this
+    function is to calculate how long it should be.
+
+    depth: depth of the IBlock in the tree, 0 would be a DBLOCK, must be > 0
+             (this function is for IBLOCKs only!)
+    offset: current offset in the payload (!) of the overall file, must be > 0
+              (since this function is called at the end of a block).
+    Returns the number of elements to be in the corresponding IBlock
+    """
+    assert (depth > 0)
+    assert (offset > 0)
+    bds = compute_tree_size_(depth)
+    mod = offset % bds
+    if mod is 0:
+        ret = CHK_PER_INODE
+    else:
+        bds /= CHK_PER_INODE
+        ret = mod / bds
+        if (mod % bds) is not 0:
+            ret += 1
+    return ret
+
+
+def compute_rootchk(readin, size):
+    """Returns the content hash key after generating the hash tree for the given
+    input stream.
+
+    readin: the stream where to read data from
+    size: the size of data to be read
+    """
+    depth = compute_depth_(size)
+    current_depth = 0
+    chks = [None] * (depth * CHK_PER_INODE)  # list buffer
+    read_offset = 0
+    logging.debug("Begining to calculate tree hash with depth: " + repr(depth))
+    while True:
+        if (depth == current_depth):
+            off = CHK_PER_INODE * (depth - 1)
+            assert (chks[off] is not None)
+            logging.debug("Encoding done, reading CHK `" + chks[off].query + \
+                          "' from " + repr(off) + "\n")
+            uri_chk = chks[off]
+            assert (size == read_offset)
+            uri_chk.setSize(size)
+            return uri_chk
+        if (0 == current_depth):
+            pt_size = min(DBLOCK_SIZE, size - read_offset)
+            try:
+                pt_block = readin.read(pt_size)
+            except IOError:
+                logging.warning("Error reading input file stream")
+                return None
+        else:
+            pt_elements = compute_iblock_size_(current_depth, read_offset)
+            pt_block = ""
+            pt_block = \
+                reduce((lambda ba, chk:
+                        ba + (chk.key + chk.query)),
+                       chks[(current_depth - 1) * CHK_PER_INODE:][:pt_elements],
+                       pt_block)
+            pt_size = pt_elements * (CHK_HASH_SIZE + CHK_QUERY_SIZE)
+        assert (len(pt_block) == pt_size)
+        assert (pt_size <= DBLOCK_SIZE)
+        off = compute_chk_offset_(current_depth, read_offset)
+        logging.debug("Encoding data at offset " + repr(read_offset) + \
+                      " and depth " + repr(current_depth) + " with block " \
+                      "size " + repr(pt_size) + " and target CHK offset " + \
+                      repr(current_depth * CHK_PER_INODE))
+        pt_hash = sha512_hash(pt_block)
+        pt_aes_key = AESKey(pt_hash)
+        pt_enc = aes_encrypt(pt_aes_key, pt_block)
+        pt_enc_hash = sha512_hash(pt_enc)
+        chk = Chk(pt_hash, pt_enc_hash)
+        chks[(current_depth * CHK_PER_INODE) + off] = chk
+        if (0 == current_depth):
+            read_offset += pt_size
+            if (read_offset == size) or \
+                    (0 == (read_offset % (CHK_PER_INODE * DBLOCK_SIZE))):
+                current_depth += 1
+        else:
+            if (CHK_PER_INODE == off) or (read_offset == size):
+                current_depth += 1
+            else:
+                current_depth = 0
+
+
+def chkuri_from_path(path):
+    """Returns the CHK URI of the file at the given path.
+
+    path: the path of the file whose CHK has to be calculated
+    """
+    size = os.path.getsize(path)
+    readin = open(path, "rb")
+    chk = compute_rootchk(readin, size)
+    readin.close()
+    return chk.uri()
+
+
+def usage():
+    """Prints help about using this script."""
+    print("""
+Usage: gnunet-chk.py [options] file
+Prints the Content Hash Key of given file in GNUNET-style URI.
+
+Options:
+    -h, --help                : prints this message
+""")
+
+
+if '__main__' == __name__:
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], "h", ["help"])
+    except getopt.GetoptError as err:
+        print(err)
+        print("Exception occured")
+        usage()
+        sys.exit(2)
+    for option, value in opts:
+        if option in("-h", "--help"):
+            usage()
+            sys.exit(0)
+    if len(args) != 1:
+        print("Incorrect number of arguments passed")
+        usage()
+        sys.exit(1)
+    print(chkuri_from_path(args[0]))
diff --git a/contrib/scripts/gnunet-logread/gnunet-logread b/contrib/scripts/gnunet-logread/gnunet-logread
new file mode 100755
index 000000000..9b1c65401
--- /dev/null
+++ b/contrib/scripts/gnunet-logread/gnunet-logread
@@ -0,0 +1,198 @@
+#!@PERL@
+# helper tool to make gnunet logs more readable
+# try 'gnunet-logread -h' for usage
+
+use strict;
+use warnings;
+my $DEFAULT_SOCKET = '/tmp/gnunet-logread-ipc.sock';
+
+print STDERR <<X if -t STDIN and $#ARGV == -1;
+*** For a usage message, try '$0 -h'.
+*** For documentation, try 'perldoc $0'.
+*** Listening for GNUNET_log events on STDIN. Type CTRL-D to terminate.
+
+X
+
+use Getopt::Std;
+my (%opts, $name, $ipc, $msg_level, $msg_regex);
+getopts ('i:x:n:s:L:m:fhq', \%opts);
+
+use Pod::Usage qw( pod2usage );
+die pod2usage if $opts{h};
+
+use POSIX qw(mkfifo);
+
+use Term::ANSIColor qw(:constants :pushpop);
+$Term::ANSIColor::AUTOLOCAL = 1;
+
+my %levels = ( NONE => 0, ERROR => 1, WARNING => 2, INFO => 4, DEBUG => 8 );
+
+# Message type numbers to names
+my %msgtypes;
+my $prefix = $ENV{GNUNET_PREFIX} || '/usr';
+my $filename = "$prefix/include/gnunet/gnunet_protocols.h";
+$ipc = $opts{s} || $DEFAULT_SOCKET;
+
+if (open HEADER, $filename)
+{
+    while (<HEADER>)
+    {
+        $msgtypes{$2} = $1 if /^\s*#define\s+GNUNET_MESSAGE_TYPE_(\w+)\s+(\d+)/i;
+    }
+    close HEADER;
+} else {
+    warn <<X;
+Could not read $filename for message codes:
+        $!.
+Please provide a \$GNUNET_PREFIX environment variable to replace "/usr".
+Try also '$0 -h' for help.
+
+X
+}
+
+die "You can't read and write the socket at the same time"
+  if exists $opts{f} and exists $opts{n};
+
+if ((exists $opts{n} or exists $opts{f}) and not -r $ipc) {
+    undef $!;
+    die "Could not mkfifo $ipc: $!" unless mkfifo $ipc, 0600;
+    system('chgrp', 'gnunet', $ipc);
+    die "Could not chgrp $ipc to 'gnunet': $!" if $!;
+    chmod 0660, $ipc;
+    die "Could not chmod $ipc to allow gnunet group writes: $!" if $!;
+}
+
+if (exists $opts{n}) {
+    $name = $opts{n};
+    $msg_level = $opts{L} && exists $levels{$opts{L}} ? $levels{$opts{L}} : 0;
+    $msg_regex = $opts{m};
+    print STDERR "RE: /$msg_regex/\n" if defined $msg_regex;
+    open O, '>', $ipc or die "Cannot write to $ipc: $!";
+}
+
+if (exists $opts{f}) {
+    open(I, $ipc) or die "Cannot read from $ipc: $!";
+    &perform while <I>;
+    close I;
+} else {
+    &perform while <>;
+}
+fileno O and close O;
+exit;
+
+
+sub perform {
+    if (fileno O) {
+        my ($time, $type, $size, $from, $to, $level, $msg);
+        if (($time, $type, $size, $from, $to) =
+            /^([A-Z][a-z]{2}\ .[0-9]\ [0-9:]{8}(?:-[0-9]{6})?)\ util-client-.*\b
+             (?: Received | Transmitting )\ message \b.*?\b
+             type \s+ (\d+) \b.*?\b
+             size \s+ (\d+) \b.*?\b
+             (?: from \s+ (\S+)
+               | to   \s+ (\S+) ) /x)
+        {
+            $from ||= $name;
+            $to ||= $name;
+            my ($time, $type, $size, $from, $to) = ($1, $2, $3,
+                                                $4 || $name, $5 || $name);
+            my $msg = exists $msgtypes{$type} ? $msgtypes{$type} : $type;
+            my $ofh = select O;
+            print O "$time\t$from -> $to\t$msg ($size)\n";
+            $| = 1;
+            select $ofh;
+        }
+        if (($time, $level, $msg) =
+            /^([A-Z][a-z]{2}\ .[0-9]\ [0-9:]{8}(?:-[0-9]{6})?)
+              \s+\S+\s+(\S+)\s+(.+)/x
+            and (exists $levels{$level}
+                 && $levels{$level} <= $msg_level
+                 && (!defined $msg_regex || $msg =~ /$msg_regex/i)))
+        {
+            print O "$time\t$name\t$level: $msg\n";
+        }
+    }
+    return if $opts{x} and /$opts{x}/io;
+    return if $opts{i} and not /$opts{i}/io;
+
+    # Timestamp (e.g. Nov 01 19:36:11-384136)
+    s/^([A-Z][a-z]{2} .[0-9] [0-9:]{8}(?:-[0-9]{6})?)/YELLOW $1/e;
+
+    # Log levels
+    s/\b(ERROR  )\b/RED $1/ex;
+    s/\b(WARNING)\b/YELLOW $1/ex;
+    s/\b(INFO   )\b/GREEN $1/ex;
+    s/\b(DEBUG  )\b/BRIGHT_BLACK $1/ex;
+
+    # Service names
+    # TODO: might read the list from $GNUNET_PREFIX/libexec/gnunet/
+    s/\b(multicast|psyc|psycstore|social)\b/BLUE $1/gex;
+
+    # Add message type names
+    s/(\s+type\s+)(\d+)/
+      $1 . BRIGHT_CYAN (exists $msgtypes{$2} ? $msgtypes{$2} : 'UNKNOWN') .
+      CYAN " ($2)"/gei;
+
+    # logread-ipc output
+    s/(\s+)([A-Z_]+)( \(\d+\))$/$1 . BRIGHT_CYAN $2 . CYAN $3/e;
+
+    print;
+}
+
+__END__
+
+=pod
+
+=head1 NAME
+
+gnunet-logread - a GNUnet log analyzer, colorizer and aggregator
+
+=head1 SYNOPSIS
+
+        <gnunet-service> |& $0 [<options>]
+    or
+        $0 [<options>] [<logfile>]
+
+ Options:
+    -f                          Follow input from IPC FIFO socket.
+
+   Regular screen output options:
+    -i <regex>                  Include only messages that match <regex>.
+    -x <regex>                  Exclude all messages that match <regex>.
+    -q                          Quiet: Do not show usage advice to new users.
+
+   Options to forward messages to the IPC FIFO socket:
+    -n <component_name>         Name of the component we are forwarding messages for.
+    -s </path/to/ipc.sock>      Default = $DEFAULT_SOCKET
+    -L <LOGLEVEL>               Minimum level of messages to forward:
+                                Log levels: NONE, ERROR, WARNING, INFO, DEBUG.
+    -m <regex>                  Only forward messages matching a regular expression.
+
+ See 'perldoc gnunet-logread' for a longer explanation.
+
+=head1 MOTIVATION
+
+GNUnet debug logs are a tedious read, but given a complex system that we
+cannot run all parts of in a debugger all the time, some gathering and
+structuring of events and message passing is useful.
+
+At first, this tool simply makes logs easier to read. Both if viewed in
+real-time or taken from disk. Then it also allows to extract all message
+passing events from it and forward them to a special process that aggregates
+all message passing events and therefore helps you make sense of all the
+inter-process communication (IPC) happening between the various pieces of
+the GNUnet system beast.
+
+That master process is simply an extra gnunet-logread that you run in a
+separate window and adorn it with the '-f' flag. The submitting processes
+instead need to be given a '-n' flag. That is because from the GNUnet logs
+it isn't clear which process events belong to. For example you may be
+having events taking place in the 'util' subsystem of gnunet-psyc-service
+just as much as in the 'util' subsystem of gnunet-multicast-service. In
+order to make sense of them it is necessary to manually add that info. This
+could be remedied by extending the semantics of the GNUNET_log facility
+instead, but that is still subject to further consideration.
+
+=head1 AUTHORS
+
+tg & lynX
diff --git a/contrib/scripts/gnunet-logread/gnunet-logread-ipc b/contrib/scripts/gnunet-logread/gnunet-logread-ipc
new file mode 100755
index 000000000..72f9f47df
--- /dev/null
+++ b/contrib/scripts/gnunet-logread/gnunet-logread-ipc
@@ -0,0 +1,10 @@
+#!/bin/sh
+#
+# Usage: gnunet-logread-ipc | gnunet-logread
+#
+# ... obsoleted by gnunet-logread's new -f option that does the same thing
+
+# FIXME: Replace /tmp with our use of $TMPDIR and similar.
+ipc=${1:-/tmp/gnunet-logread-ipc.sock}
+test -e "$ipc" || mkfifo "$ipc"
+cat "$ipc"
diff --git a/contrib/scripts/gnunet-logread/gnunet-logread-ipc-sdedit b/contrib/scripts/gnunet-logread/gnunet-logread-ipc-sdedit
new file mode 100755
index 000000000..f8b7dc735
--- /dev/null
+++ b/contrib/scripts/gnunet-logread/gnunet-logread-ipc-sdedit
@@ -0,0 +1,60 @@
+#!@PERL@
+
+# 1. Start sdedit and enable 'RT diagram server' in 'Global preferences'.
+#
+# 2. Start this tool (see defaults below):
+#    gnunet-logread-ipc-sdedit -n buffer-name -i /path/to/ipc.sock -h <sdedit-host> -p <sdedit-port>
+#
+# 3. Start a gnunet-logread instance for each component with the -n <component_name> option
+
+use strict;
+use warnings;
+
+use Getopt::Std;
+use IO::Socket::INET;
+use POSIX qw(mkfifo);
+
+my %opts;
+getopts ('i:n:h:p:', \%opts);
+
+my $ipc  = $opts{i} || '/tmp/gnunet-logread-ipc.sock';
+my $name = $opts{n} || 'gnunet';
+my $host = $opts{h} || 'localhost';
+my $port = $opts{p} || 16001;
+my %svcs = map { $_ => 1 } @ARGV;
+
+my $sdedit = IO::Socket::INET->new(PeerAddr => $host,
+                                   PeerPort => $port,
+                                   Proto => 'tcp')
+    or die "Cannot connect to $host:$port: $!\n";
+
+print $sdedit "$name\n";
+print $sdedit "_t:time[e]\n";
+print $sdedit "$_:$_\[ap\] \"$_\"\n" for @ARGV;
+print $sdedit "_e:ext[e]\n";
+print $sdedit "\n";
+
+mkfifo $ipc, 0600 or die "$ipc: $!\n" unless -e $ipc;
+open IPC, '<', $ipc or die "$ipc: $!\n";
+$| = 1;
+while (<IPC>)
+{
+    print;
+    my ($time, $from, $to, $msg, $svc);
+    if (my ($time, $from, $to, $msg) =
+        /^([A-Z][a-z]{2}\ .[0-9]\ [0-9:]{8}(?:-[0-9]{6})?)\s+
+         (\S+)\s+ -> \s+(\S+)\s+ (\S+\s+ \(\d+\))/x)
+    {
+        $from = '_e' unless exists $svcs{$from};
+        $to = '_e' unless exists $svcs{$to};
+        print $sdedit "*0 _t\n$time\n*0\n", "$from:$to.$msg\n"
+    }
+    elsif (($time, $svc, $msg) =
+           /^([A-Z][a-z]{2}\ .[0-9]\ [0-9:]{8}(?:-[0-9]{6})?)\s+
+             (\S+)\s+(.+)/x)
+    {
+        print $sdedit "*0 _t\n$time\n*0\n", "*0 $svc\n$msg\n*0\n"
+    }
+}
+
+close IPC;
diff --git a/contrib/scripts/gnunet-suidfix b/contrib/scripts/gnunet-suidfix
new file mode 100755
index 000000000..992378966
--- /dev/null
+++ b/contrib/scripts/gnunet-suidfix
@@ -0,0 +1,27 @@
+#!/bin/sh
+#
+# "suidfix" is german and it means something like immediate suicide.
+
+# taken from dangole's lede config.. thx!
+suid_root_helpers="exit nat-server nat-client transport-bluetooth transport-wlan vpn"
+libexec="${GNUNET_PREFIX}/lib/gnunet/libexec"
+
+chmodown_execbin() {
+        if [ -x $1 ]; then
+                if [ "$3" ]; then
+                        chown $3 $1 2>/dev/null && chmod $2 $1
+                else
+                        chmod $2 $1
+                fi
+                ls -l $1
+        else
+                echo "Missing: $1"
+        fi
+}
+
+for helper in $suid_root_helpers; do
+        chmodown_execbin ${libexec}/gnunet-helper-$helper u+s
+done
+chmodown_execbin ${libexec}/gnunet-helper-dns 4750 root:gnunetdns
+chmodown_execbin ${libexec}/gnunet-service-dns 2750 gnunet:gnunetdns
+
diff --git a/contrib/scripts/gnunet_janitor.py.in b/contrib/scripts/gnunet_janitor.py.in
new file mode 100644
index 000000000..74fc70886
--- /dev/null
+++ b/contrib/scripts/gnunet_janitor.py.in
@@ -0,0 +1,78 @@
+#!@PYTHON@
+#    This file is part of GNUnet.
+#    (C) 2011 Christian Grothoff (and other contributing authors)
+#
+#    GNUnet is free software; you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published
+#    by the Free Software Foundation; either version 2, or (at your
+#    option) any later version.
+#
+#    GNUnet is distributed in the hope that it will be useful, but
+#    WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#    General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with GNUnet; see the file COPYING.  If not, write to the
+#    Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+#    Boston, MA 02110-1301, USA.
+#
+# Finds any gnunet processes still running in the system and kills them
+#
+# gnunet janitor can be used by invoking `make' like this:
+# TESTS_ENVIRONMENT='${top_srcdir}/contrib/gnunet_janitor.py &&' make check
+
+from __future__ import print_function
+import os
+import re
+import subprocess
+import sys
+import shutil
+import time
+import signal
+import terminate
+
+if os.name == 'nt':
+  from win32com.client import GetObject
+  WMI = GetObject('winmgmts:')
+
+def get_process_list ():
+  result = []
+  if os.name == 'nt':
+    processes = WMI.InstancesOf('Win32_Process')
+    for p in processes:
+      result.append ((p.Properties_('ProcessId').Value, re.sub (r'(.+)\.exe', r'\1', p.Properties_('Name').Value)))
+  else:
+    pids = [pid for pid in os.listdir('/proc') if pid.isdigit ()]
+    for pid in pids:
+      with open (os.path.join ('/proc', pid, 'cmdline'), 'rb') as p:
+        cmdline = p.read ().split ('\x00')
+        if len (cmdline) > 0:
+          result.append ((pid, cmdline[0]))
+  return result
+
+def main ():
+  procs = get_process_list ()
+  gnunet_procs = []
+  for p in procs:
+    if re.match (r'gnunet-.+', p[1]):
+      gnunet_procs.append (p)
+  for p in gnunet_procs:
+    if re.match (r'gnunet-service-arm', p[1]):
+      print ("killing arm process {0:5} {1}".format (p[0], p[1]))
+      try:
+        terminate.safe_terminate_process_by_pid (int (p[0]), 1)
+      except OSError as e:
+        print ("failed: {0}".format (e))
+        pass
+  for p in gnunet_procs:
+    if not re.match (r'gnunet-service-arm', p[1]):
+      print ("killing non-arm process {0:5} {1}".format (p[0], p[1]))
+      try:
+        terminate.safe_terminate_process_by_pid (int (p[0]), 1)
+      except OSError as e:
+        print ("failed: {0}".format (e))
+        pass
+
+if __name__ == '__main__':
+  sys.exit (main ())
diff --git a/contrib/scripts/gnunet_pyexpect.py.in b/contrib/scripts/gnunet_pyexpect.py.in
new file mode 100644
index 000000000..cfeb06d8d
--- /dev/null
+++ b/contrib/scripts/gnunet_pyexpect.py.in
@@ -0,0 +1,83 @@
+#!@PYTHON@
+#    This file is part of GNUnet.
+#    (C) 2010 Christian Grothoff (and other contributing authors)
+#
+#    GNUnet is free software; you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published
+#    by the Free Software Foundation; either version 2, or (at your
+#    option) any later version.
+#
+#    GNUnet is distributed in the hope that it will be useful, but
+#    WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#    General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with GNUnet; see the file COPYING.  If not, write to the
+#    Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+#    Boston, MA 02110-1301, USA.
+#
+# Testcase for gnunet-peerinfo
+from __future__ import print_function
+import os
+import re
+import subprocess
+import sys
+import shutil
+import time
+
+class pexpect (object):
+  def __init__ (self):
+    super (pexpect, self).__init__ ()
+
+  def spawn (self, stdin, arglist, *pargs, **kwargs):
+    env = kwargs.pop ('env', None)
+    if env is None:
+      env = os.environ.copy ()
+    # This messes up some testcases, disable log redirection
+    env.pop ('GNUNET_FORCE_LOGFILE', None)
+    self.proc = subprocess.Popen (arglist, *pargs, env=env, **kwargs)
+    if self.proc is None:
+      print ("Failed to spawn a process {0}".format (arglist))
+      sys.exit (1)
+    if stdin is not None:
+      self.stdo, self.stde = self.proc.communicate (stdin)
+    else:
+      self.stdo, self.stde = self.proc.communicate ()
+    return self.proc
+
+  def expect (self, s, r, flags=0):
+    stream = self.stdo if s == 'stdout' else self.stde
+    if isinstance (r, str):
+      if r == "EOF":
+        if len (stream) == 0:
+          return True
+        else:
+          print ("Failed to find `{1}' in {0}, which is `{2}' ({3})".format (s, r, stream, len (stream)))
+          sys.exit (2)
+      raise ValueError ("Argument `r' should be an instance of re.RegexObject or a special string, but is `{0}'".format (r))
+    m = r.search (stream.decode(), flags)
+    if not m:
+      print ("Failed to find `{1}' in {0}, which is is `{2}'".format (s, r.pattern, stream))
+      sys.exit (2)
+    stream = stream[m.end ():]
+    if s == 'stdout':
+      self.stdo = stream
+    else:
+      self.stde = stream
+    return m
+
+  def read (self, s, size=-1):
+    stream = self.stdo if s == 'stdout' else self.stde
+    result = ""
+    if size < 0:
+      result = stream
+      new_stream = ""
+    else:
+      result = stream[0:size]
+      new_stream = stream[size:]
+    if s == 'stdout':
+      self.stdo = new_stream
+    else:
+      self.stde = new_stream
+    return result
diff --git a/contrib/scripts/process_log.sh b/contrib/scripts/process_log.sh
new file mode 100755
index 000000000..c25c515c2
--- /dev/null
+++ b/contrib/scripts/process_log.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# Usage: service should print "STARTING SERVICE (srvc) for peer [PEER]" where:
+# - "srvc" is the service name (in lowercase, as in the log output).
+#   It cannot contain parenthesis in its name.
+# - "PEER" is the peer ID. Should be 4 alfanumeric characters
+
+grep "STARTING SERVICE " log > __tmp_peers
+
+SED_EXPR=""
+while read -r line; do
+    SRVC=`echo "$line" | sed -e 's/.*(\([^)]*\)).*/\1/'`
+    PEER=`echo "$line" | sed -e 's/.*\[\(....\)\].*/\1/'`
+    PID=`echo "$line" | sed -e "s/.*$SRVC-\([0-9]*\).*/\1/"`
+    echo "$SRVC $PID => $PEER"
+    
+    SED_EXPR="${SED_EXPR}s/$SRVC-\([a-z2]*\)-$PID/$SRVC \1 $PEER/;"
+    SED_EXPR="${SED_EXPR}s/$SRVC-$PID/$SRVC XXX $PEER/;"
+    SED_EXPR="${SED_EXPR}s/$SRVC-api-[0-9]/$SRVC-api-                                            /;"
+done < __tmp_peers
+rm __tmp_peers
+
+sed -e "$SED_EXPR" log > .log
+echo "$0 sed regex: $SED_EXPR" >> .log
+
+SIZE=`stat -c%s .log`
+
+if [[ "`ps aux | grep "kwrite .lo[g]"`" = "" && "$SIZE" < 10000000 ]]; then
+    kwrite .log --geometry 960x1140-960 &
+fi
diff --git a/contrib/scripts/pydiffer.py.in b/contrib/scripts/pydiffer.py.in
new file mode 100644
index 000000000..10145371c
--- /dev/null
+++ b/contrib/scripts/pydiffer.py.in
@@ -0,0 +1,44 @@
+#!@PYTHON@
+import os
+import sys
+import difflib
+import filecmp
+
+
+def getdiff(old, new):
+    diff = []
+    with open(old) as a:
+        with open(new) as b:
+            for l in difflib.unified_diff(a.read().splitlines(), b.read().splitlines()):
+                diff.append(l)
+    return diff
+
+
+def dc_getdiff(dc, old, new):
+    diff = []
+    for f in dc.left_only:
+        diff.append("Only in {}: {}".format(old, f))
+    for f in dc.right_only:
+        diff.append("Only in {}: {}".format(new, f))
+    for f in dc.diff_files:
+        r = getdiff(os.path.join(old, f), os.path.join(new, f))
+        diff.extend(r)
+    for dn, dc in dc.subdirs.items():
+        r = dc_getdiff(dc, os.path.join(old, dn), os.path.join(new, dn))
+        diff.extend(r)
+    return diff
+
+
+def dcdiff(old, new):
+    dc = filecmp.dircmp(old, new)
+    diff = dc_getdiff(dc, old, new)
+    return diff
+
+
+def main():
+    for l in dcdiff(sys.argv[1], sys.argv[2]):
+        print(l)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/contrib/scripts/pydmesg b/contrib/scripts/pydmesg
new file mode 100755
index 000000000..d60e08fe3
--- /dev/null
+++ b/contrib/scripts/pydmesg
@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+# coding=utf8
+
+# Copyright (C) 2010 Saúl ibarra Corretgé <saghul@gmail.com>
+#
+
+"""
+pydmesg: dmesg with human-readable timestamps
+"""
+
+from __future__ import with_statement
+
+import re
+import subprocess
+import sys
+
+from datetime import datetime, timedelta
+
+
+_datetime_format = "%Y-%m-%d %H:%M:%S"
+_dmesg_line_regex = re.compile("^\[(?P<time>\d+\.\d+)\](?P<line>.*)$")
+
+def exec_process(cmdline, silent, input=None, **kwargs):
+    """Execute a subprocess and returns the returncode, stdout buffer and stderr buffer.
+       Optionally prints stdout and stderr while running."""
+    try:
+        sub = subprocess.Popen(cmdline, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs)
+        stdout, stderr = sub.communicate(input=input)
+        returncode = sub.returncode
+        if not silent:
+            sys.stdout.write(stdout)
+            sys.stderr.write(stderr)
+    except OSError,e:
+        if e.errno == 2:
+            raise RuntimeError('"%s" is not present on this system' % cmdline[0])
+        else:
+            raise
+    if returncode != 0:
+        raise RuntimeError('Got return value %d while executing "%s", stderr output was:\n%s' % (returncode, " ".join(cmdline), stderr.rstrip("\n")))
+    return stdout
+
+def human_dmesg():
+    now = datetime.now()
+    uptime_diff = None
+    try:
+        with open('/proc/uptime') as f:
+            uptime_diff = f.read().strip().split()[0]
+    except IndexError:
+        return
+    else:
+        try:
+            uptime = now - timedelta(seconds=int(uptime_diff.split('.')[0]), microseconds=int(uptime_diff.split('.')[1]))
+        except IndexError:
+            return
+
+    dmesg_data = exec_process(['dmesg'], True)
+    for line in dmesg_data.split('\n'):
+        if not line:
+            continue
+        match = _dmesg_line_regex.match(line)
+        if match:
+            try:
+                seconds = int(match.groupdict().get('time', '').split('.')[0])
+                nanoseconds = int(match.groupdict().get('time', '').split('.')[1])
+                microseconds = int(round(nanoseconds * 0.001))
+                line = match.groupdict().get('line', '')
+                t = uptime + timedelta(seconds=seconds, microseconds=microseconds)
+            except IndexError:
+                pass
+            else:
+                print "[%s]%s" % (t.strftime(_datetime_format), line)
+
+
+if __name__ == '__main__':
+    human_dmesg()
diff --git a/contrib/scripts/regression.sh b/contrib/scripts/regression.sh
new file mode 100755
index 000000000..1f799797a
--- /dev/null
+++ b/contrib/scripts/regression.sh
@@ -0,0 +1,54 @@
+#!/bin/sh
+cd
+cd gnunet
+svn up > /dev/null
+H=`hostname`
+echo "================START===================" >> regression.$H
+RUN=`date +%0y%0m%0d%0k%0M`
+uname -a >> regression.$H
+date >> regression.$H
+echo "Run $RUN" >> regression.$H
+svn up >> regression.$H
+export GNUNET_PREFIX=$HOME
+export PATH=$HOME/bin:$PATH
+./bootstrap >> regression.$H.$RUN  2>&1
+./configure --prefix=$HOME --with-extractor=$HOME --with-microhttpd=$HOME  >> regression.$H.$RUN  2>&1
+if [ $? != 0 ]
+then
+  echo configure failed >> regression.$H
+  exit
+fi
+KEEP=0
+make clean >> regression.$H.$RUN 2>&1
+make install >> regression.$H.$RUN 2>&1
+cd src
+for n in `ls --hide "Makefile*"`
+do
+  cd $n
+  if [ -f Makefile ]
+  then
+    make check >> ../regression.$H.$RUN.$n  2>&1
+    if [ $? != 0 ]
+    then
+      echo Tests for $n failed >> ../regression.$H
+      echo "--------- Details for $n -----------" >> ../regression.$H
+      cat regression.$H.$RUN.$n >> ../regression.$H
+      echo "----- End of Details for $n --------" >> ../regression.$H
+      KEEP=1
+    else
+      echo Tests for $n succeeded >> ../regression.$H
+    fi
+  fi
+  cd ..
+done
+echo "================END====================" >> regression.$H
+
+if [ $KEEP == 0]
+then
+  rm regression.$H.$RUN*
+  rm regression.$H
+else
+  svn add regression.$H > /dev/null
+  svn commit -m "Regression in tests on $H" regression.$H
+fi
+
diff --git a/contrib/scripts/removetrailingwhitespace.py.in b/contrib/scripts/removetrailingwhitespace.py.in
new file mode 100755
index 000000000..5824fb591
--- /dev/null
+++ b/contrib/scripts/removetrailingwhitespace.py.in
@@ -0,0 +1,15 @@
+#!@PYTHON@
+
+import sys
+import re
+
+
+for fileName in sys.argv[1:]:
+    f = open(fileName, 'r+')
+    fileString = f.read()
+
+    fileString = re.sub(r'[ ]+\n', r'\n', fileString)
+    fileString = re.sub(r'\r', r'', fileString)
+    f.seek(0)
+    f.write(fileString)
+    f.truncate(len(fileString))
diff --git a/contrib/scripts/report.sh b/contrib/scripts/report.sh
new file mode 100755
index 000000000..7fbb2f46c
--- /dev/null
+++ b/contrib/scripts/report.sh
@@ -0,0 +1,252 @@
+#!/bin/sh
+
+TEST=`type type|grep not`
+if test -n "$TEST"; then
+  WHICH=which
+else
+  WHICH=type
+fi
+
+echo "Please submit the following information with your bug report: "
+echo "--------------------------------------------------------------"
+OS=`uname -s 2>/dev/null`
+echo "OS             : $OS"
+REL=`uname -r 2>/dev/null`
+echo "OS RELEASE     : $REL"
+HW=`uname -m 2>/dev/null`
+echo "HARDWARE       : $HW"
+
+TEST=`$WHICH gcc 2>/dev/null`
+if test -n "$TEST"; then
+  VERS=`gcc --version 2>/dev/null | head -n 1`
+  echo "gcc            : $VERS"
+else
+  echo "gcc            : Not Found";
+fi
+
+TEST=`$WHICH gmake 2>/dev/null`
+if test -n "$TEST" ; then
+        gmake --version 2>/dev/null |\
+                awk -F, '{print $1}' |\
+                awk '/GNU Make/{print "GNU gmake      :",$NF}'
+else
+  TEST=`make --version 2>/dev/null`
+  if test -n "$TEST"; then
+                make --version 2>/dev/null |\
+                        awk -F, '{print $1}' |\
+                        awk '/GNU Make/{print "make           :",$NF}'
+  else
+                echo "make           : Not Found"
+  fi
+fi
+
+TEST=`$WHICH autoconf 2>/dev/null`
+if test -n "$TEST"; then
+  autoconf --version |\
+    head -n 1 |\
+    awk '{\
+        if (length($4) == 0) {\
+                print "autoconf       : "$3\
+        } else {\
+                print "autoconf       : "$4\
+        }}'
+else
+  echo "autoconf       : Not Found"
+fi
+
+TEST=`$WHICH automake 2>/dev/null`
+if test -n "$TEST"; then
+  automake --version 2>/dev/null |\
+    head -n 1 |\
+    awk '{print "automake       : "$4}'
+else
+  echo "automake       : Not Found"
+fi
+
+TEST=`$WHICH libtool 2>/dev/null`
+if test -n "$TEST"; then
+  libtool --version 2>/dev/null |\
+    head -n 1 |\
+    awk '{print "libtool        : "$4}'
+else
+  echo "libtool        : Not Found"
+fi
+
+TEST=`$WHICH extract 2>/dev/null`
+if test -n "$TEST"; then
+  extract -v 2>/dev/null |\
+    head -n 1 |\
+    awk '{print "libextractor   : "$2}'
+else
+  echo "libextractor   : Not Found"
+fi
+
+if test -x gnunetd; then
+  gnunetd -v | sed -e "s/v//" 2>/dev/null |\
+    awk '{print "GNUnet 0.8     : "$2 (may conflict!)}'
+else
+  echo "GNUnet 0.8     : Not Found (good)"
+fi
+
+TEST=`$WHICH gnunet-arm 2>/dev/null`
+if test -n "$TEST"; then
+  gnunet-arm -v | sed -e "s/v//" 2>/dev/null |\
+    awk '{print "GNUnet 0.9     : "$2}'
+else
+  echo "GNUnet 0.9     : Not Found"
+fi
+
+TEST=`$WHICH libgcrypt-config 2> /dev/null`
+if test -n "$TEST"; then
+  libgcrypt-config --version 2> /dev/null | \
+    awk '{print "libgcrypt      : "$1}'
+else
+  echo "libgcrypt      : Not Found"
+fi
+
+TEST=`$WHICH mysql_config 2> /dev/null`
+if test -n "$TEST"; then
+  mysql_config --version 2> /dev/null | \
+    awk '{print "mysql          : "$1}'
+else
+  echo "mysql          : Not Found"
+fi
+
+TEST=`$WHICH pkg-config 2> /dev/null`
+if test -n "$TEST"; then
+  pkg-config --version 2> /dev/null | \
+    awk '{print "pkg-config     : "$1}'
+else
+  echo "pkg-config     : Not Found"
+fi
+
+TEST=`$WHICH pkg-config 2> /dev/null`
+if test -n "$TEST"; then
+  pkg-config --modversion glib-2.0 2> /dev/null | \
+    awk '{print "glib2          : "$1}'
+else
+  echo "glib2          : Not Found"
+fi
+
+TEST=`$WHICH pkg-config 2> /dev/null`
+if test -n "$TEST"; then
+  pkg-config --modversion gtk+-2.0 2> /dev/null | \
+    awk '{print "gtk2+          : "$1}'
+else
+  echo "gtk2+          : Not Found"
+fi
+
+TEST=`$WHICH dpkg 2> /dev/null`
+if test -n "$TEST"; then
+  LINES=`dpkg -s libgmp-dev | grep Version | wc -l 2> /dev/null`
+  if test "$LINES" = "1"
+  then
+    VERSION=`dpkg -s libgmp-dev | grep Version | awk '{print $2}'`
+    echo "GMP            : libgmp-dev-$VERSION.deb"
+  else
+    echo "GMP            : dpkg: libgmp-dev not installed"
+  fi
+else
+  TEST=`$WHICH rpm 2> /dev/null`
+  if test -n "$TEST"; then
+    rpm -q gmp | sed -e "s/gmp-//" 2> /dev/null | \
+      awk '{print "GMP            : "$1.rpm}'
+  else
+    echo "GMP            : Test not available"
+  fi
+fi
+
+TEST=`$WHICH dpkg 2> /dev/null`
+if test -n "$TEST"; then
+  LINES=`dpkg -s libunistring-dev | grep Version | wc -l`
+  if test "$LINES" = "1"
+  then
+    VERSION=`dpkg -s libunistring-dev | grep Version | awk '{print $2}'`
+    echo "libunistring   : libunistring3-dev-$VERSION.deb"
+  else
+    echo "libunistring   : dpkg: libunistring3-dev not installed"
+  fi
+else
+  TEST=`$WHICH rpm 2> /dev/null`
+  if test -n "$TEST"; then
+    rpm -q unistring | sed -e "s/unistring-//" 2> /dev/null | \
+      awk '{print "libunistring   : "$1.rpm}'
+  else
+    echo "libunistring   : Test not available"
+  fi
+fi
+
+TEST=`$WHICH gettext 2> /dev/null`
+if test -n "$TEST"; then
+  gettext --version | head -n1 2> /dev/null | \
+    awk '{print "GNU gettext    : "$4}'
+else
+  echo "GNU gettext    : Not found"
+fi
+
+
+TEST=`$WHICH curl-config 2> /dev/null`
+if test -n "$TEST"; then
+  curl-config --version | head -n1 2> /dev/null | \
+    awk '{print "libcurl        : "$2}'
+else
+  echo "libcurl        : Not found"
+fi
+
+TEST=`$WHICH gnurl-config 2> /dev/null`
+if test -n "$TEST"; then
+  gnurl-config --version | head -n1 2> /dev/null | \
+    awk '{print "libgnurl       : "$2}'
+else
+  echo "libgnurl       : Not found"
+fi
+
+echo -n "libmicrohttpd  : "
+TMPFILE=`mktemp /tmp/mhd-version-testXXXXXX`
+cat - >$TMPFILE.c <<EOF
+#include <microhttpd.h>
+#include <stdio.h>
+int main()
+{
+  fprintf (stdout, "%X\n", MHD_VERSION);
+  return 0;
+}
+EOF
+
+gcc -o $TMPFILE $TMPFILE.c 2> /dev/null && $TMPFILE || echo "Not found"
+rm -f $TMPFILE $TMPFILE.bin
+
+
+echo -n "GNU GLPK       : "
+TMPFILE=`mktemp /tmp/glpk-version-testXXXXXX`
+cat - >$TMPFILE.c <<EOF
+#include <glpk.h>
+#include <stdio.h>
+int main()
+{
+  fprintf (stdout, "%u.%u\n", GLP_MAJOR_VERSION, GLP_MINOR_VERSION);
+  return 0;
+}
+EOF
+
+gcc -o $TMPFILE $TMPFILE.c 2> /dev/null && $TMPFILE || echo "Not found"
+rm -f $TMPFILE $TMPFILE.bin
+
+
+echo -n "GNUtls         : "
+TMPFILE=`mktemp /tmp/gnutls-version-testXXXXXX`
+cat - >$TMPFILE.c <<EOF
+#include <gnutls/gnutls.h>
+#include <stdio.h>
+int main()
+{
+  fprintf (stdout, "%s\n", GNUTLS_VERSION);
+  return 0;
+}
+EOF
+
+gcc -o $TMPFILE $TMPFILE.c 2> /dev/null && $TMPFILE || echo "Not found"
+rm -f $TMPFILE $TMPFILE.bin
+
+
+echo "--------------------------------------------------------------"
diff --git a/contrib/scripts/revisionary.sh b/contrib/scripts/revisionary.sh
new file mode 100755
index 000000000..5778cf148
--- /dev/null
+++ b/contrib/scripts/revisionary.sh
@@ -0,0 +1,98 @@
+#!/usr/local/bin/bash
+
+STARTREVISION=14033
+ENDREVISION=15268
+CURRENTREVISION=$STARTREVISION 
+HOME_DIR='/home/gnunet/FreeBSD7-AMD64-wachs/freebsd7-amd64-wachs/build'
+
+
+CMD_UPDATE="svn up --force --accept theirs-full -r"
+CMD_CLEANUP="killall -s 31 -r gnunet-*; make distclean;"
+CMD_PREPARATION="./bootstrap; ./configure --prefix=/tmp/gnb --with-extractor=/usr/local"
+CMD_BUILD="make all"
+CMD_TEST="cd src/transport ; make test_transport_api_tcp; ./test_transport_api_tcp"
+
+#LOGGING=""
+LOGGING=" 1> /dev/null 2> errors.txt"
+LOGFILE="log.txt"
+
+function output ()
+{
+  eval echo $1
+  eval echo $1 >> $LOGFILE
+}
+
+
+while [ $CURRENTREVISION -le $ENDREVISION ]; do
+  output 'Testing revision $CURRENTREVISION'
+# updating
+  output ' -> updating '
+  eval cd $HOME_DIR
+  CMD="$CMD_UPDATE $CURRENTREVISION $LOGGING"
+  eval $CMD
+  result=$?
+  if [ $result -eq 0 ]; then
+    output "    updating OK"
+  else
+    output "    updating FAILED"
+    (( CURRENTREVISION++ )) 
+    continue
+  fi
+
+# clean up
+  output " -> cleanup "
+  CMD="$CMD_CLEANUP $LOGGING"
+  eval $CMD
+  result=$?
+  if [ $result -eq 0 ]; then
+    output "    cleanup OK"
+  else
+    output "    cleanup FAILED"
+    (( CURRENTREVISION++ ))     
+    continue
+  fi
+# preparing 
+  output " -> preparation "
+  CMD="$CMD_PREPARATION $LOGGING"
+  #echo $CMD
+  eval $CMD
+  result=$?
+  if [ $result -eq 0 ]; then
+    output "    preparation OK"
+  else
+    output "    preparation FAILED"
+    (( CURRENTREVISION++ )) 
+    continue
+  fi
+# building
+  output  " -> building "
+  CMD="$CMD_BUILD $LOGGING"
+  #echo $CMD
+  eval $CMD
+  result=$?
+  if [ $result -eq 0 ]; then
+    output "    building OK"
+  else
+    output "    building FAILED"
+    (( CURRENTREVISION++ )) 
+    continue
+  fi
+# testing
+  output " -> testing "
+  CMD="$CMD_TEST $LOGGING"
+  #echo $CMD
+  eval $CMD
+  result=$?
+  testresult=result
+  if [ $result -eq 0 ]; then
+    output "    testing OK"
+  else
+    output "    testing FAILED"
+    output 'Revision $CURRENTREVISION FAILED'
+  fi
+  (( CURRENTREVISION++ ))
+done
+
+exit
+
+
diff --git a/contrib/scripts/terminate.py.in b/contrib/scripts/terminate.py.in
new file mode 100644
index 000000000..4a6719f38
--- /dev/null
+++ b/contrib/scripts/terminate.py.in
@@ -0,0 +1,64 @@
+#!@PYTHON@
+#    This file is part of GNUnet.
+#    (C) 2011 Christian Grothoff (and other contributing authors)
+#
+#    GNUnet is free software; you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published
+#    by the Free Software Foundation; either version 2, or (at your
+#    option) any later version.
+#
+#    GNUnet is distributed in the hope that it will be useful, but
+#    WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#    General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with GNUnet; see the file COPYING.  If not, write to the
+#    Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+#    Boston, MA 02110-1301, USA.
+#
+# Utility module that implements safe process termination for W32.
+# For other platforms it's equivalent to Popen.kill ()
+# Requires pywin32 on W32.
+
+import sys
+import os
+import subprocess
+if os.name == 'nt':
+  import win32api
+  import win32process
+
+class dummyobj (object):
+  pass
+
+def safe_terminate_process_by_pid (pid, code):
+  if os.name == 'nt':
+    p = dummyobj ()
+    p._handle = win32api.OpenProcess (2 | 1024 | 8 | 32 | 16, 0, pid)
+    result = safe_terminate_process (p, code)
+    win32api.CloseHandle (p._handle)
+    return result
+  else:
+    return os.kill (int (pid), SIGKILL)
+
+def safe_terminate_process (proc, code):
+  if os.name == 'nt':
+    cp = win32api.GetCurrentProcess ()
+    result = False
+    dupproc = win32api.DuplicateHandle (cp, proc._handle, cp, 2 | 1024 | 8 | 32 | 16, 0, 0)
+    try:
+      exitcode = win32process.GetExitCodeProcess (dupproc)
+      if exitcode == 0x103:
+        kernel32 = win32api.GetModuleHandle ("kernel32")
+        exitprocess = win32api.GetProcAddress (kernel32, "ExitProcess")
+        th, tid = win32process.CreateRemoteThread (dupproc, None, 0, exitprocess, code, 0)
+        win32api.CloseHandle (th)
+        result = True
+      else:
+        result = True
+    # except failed to get exit code? failed to get module handle?
+    finally:
+      win32api.CloseHandle (dupproc)
+    return result
+  else:
+    return proc.kill ()
diff --git a/contrib/scripts/testbed_cleanup.sh b/contrib/scripts/testbed_cleanup.sh
new file mode 100755
index 000000000..57413fba0
--- /dev/null
+++ b/contrib/scripts/testbed_cleanup.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+###################################################################################
+# Script to clean a previous run of testbed which has crashed. This scripts kills #
+# the peers and cleans the temporary files created for those peers                #
+#                                                                                 #
+# Author: Sree Harsha Totakura                                                    #
+###################################################################################
+
+for host in `cut -d : -f 1 < infiniband_cluster.hosts | cut -d @ -f 2`
+do
+    echo "ssh --> $host"
+    ssh $host 'pkill -SIGKILL gnunet; rm -rf /tmp/gnunet-pipe*; rm -rf /tmp/testbed*'
+done
diff --git a/contrib/scripts/texinfo-hacks.el b/contrib/scripts/texinfo-hacks.el
new file mode 100644
index 000000000..bfb5c98fa
--- /dev/null
+++ b/contrib/scripts/texinfo-hacks.el
@@ -0,0 +1,18 @@
+;;;; hacks.el --- a few functions to help me work on the manual
+;;;; Jim Blandy <jimb@red-bean.com> --- October 1998
+;;;; -- imported from https://git.savannah.gnu.org/cgit/guile.git/tree/doc/hacks.el
+;;;; This code should be covered by the same license as GNU Guile (GPL3).
+
+(defun jh-exemplify-region (start end)
+  (interactive "r")
+  (save-excursion
+    (save-restriction
+      (narrow-to-region start end)
+
+      ;; Texinfo doesn't handle tabs well.
+      (untabify (point-min) (point-max))
+
+      ;; Quote any characters special to texinfo.
+      (goto-char (point-min))
+      (while (re-search-forward "[{}@]" nil t)
+        (replace-match "@\\&")))))
diff --git a/contrib/scripts/visualize_stats.sh b/contrib/scripts/visualize_stats.sh
new file mode 100755
index 000000000..aaa5e657b
--- /dev/null
+++ b/contrib/scripts/visualize_stats.sh
@@ -0,0 +1,86 @@
+#!/bin/bash
+#
+# This script polls gnunet-stats repeatedly to create statistics plots. 
+# Use 'collect' to collect statistics and 'plot' to plot whats been
+# collected. All plots will be written to $STATDIR as separate .png files.
+#
+# WARNING: calling 'collect' will delete all files in $STATDIR.
+#
+# Requires: gnuplot
+#
+# Note: gnuplot syntax has changed across versions. This
+# script perhaps will not produce color images with older gnuplots.
+# The script should work atleast with gnuplot 3.8k patchlevel 1.
+#
+
+SLEEP=120
+GNUNET=$HOME/
+STATDIR=$GNUNET/stats
+IMAGEVIEWER='display'
+TMP=/tmp/.gnuplot_error
+
+##########################################################################
+
+mkdir -p $STATDIR
+
+case "$1" in
+  collect)
+    rm -f $STATDIR/*
+  
+    STARTTIME=`date +%s`
+    IFS=":"
+    
+    while true; do
+        NOW=`date +%s`
+        RELAT=$[$NOW-$STARTTIME]
+        gnunet-statistics | while read KEY VALUE; do
+                KEY=`echo $KEY | tr / .`
+                # Collect stats of previous round
+                if [ -e "$STATDIR/$KEY.dat" ]; then
+                        PREV=`tail --lines=1 "$STATDIR/$KEY.dat" | sed -e "s/.* //g"`
+                else
+                        PREV=$VALUE
+                fi
+
+                # Write new stats
+                echo $RELAT $VALUE >>"$STATDIR/$KEY.dat"
+                echo $RELAT $PREV $VALUE >>"$STATDIR/$KEY.diff"
+        
+        done
+        sleep $SLEEP
+    done
+  ;;
+  plot)
+        # Plot incremental
+        ls -1 $STATDIR/*.dat | while read FILENAME; do
+                BASENAME=`basename "$FILENAME" | sed -e "s/ *\..*//g"`
+                echo "set terminal png;set output '$FILENAME.png';set title '$BASENAME - incr';plot '$FILENAME' using (\$1/60):(\$2) title '' with lines;" | nice gnuplot 2> $TMP
+         EC=`cat $TMP | grep "empty" | grep "Warning" | wc -l`
+         if test $EC -ge 1
+         then
+           rm "$FILENAME.png"
+         fi
+        done
+        
+        # Plot diff
+        ls -1 $STATDIR/*.diff | while read FILENAME; do
+                BASENAME=`basename "$FILENAME" | sed -e "s/ *\..*//g"`
+                echo "set terminal png;set output '$FILENAME.png';set title '$BASENAME - diff';plot '$FILENAME' using (\$1/60):(\$3-\$2) title '' with lines;" | nice gnuplot 2> $TMP
+         EC=`cat $TMP | grep "empty" | grep "Warning" | wc -l`
+         if test $EC -ge 1 
+         then
+          rm "$FILENAME.png"
+         fi
+
+        done
+  ;;
+  view)
+        $IMAGEVIEWER $STATDIR/*.png
+  ;;
+  *)
+     echo $"Usage: $0 {collect|plot|view}"
+     exit 1
+    
+esac
+
+  
diff --git a/contrib/scripts/zonewalk-to-types.sh b/contrib/scripts/zonewalk-to-types.sh
new file mode 100755
index 000000000..c453702e6
--- /dev/null
+++ b/contrib/scripts/zonewalk-to-types.sh
@@ -0,0 +1,35 @@
+#!/bin/sh
+# This script is in the public domain.
+# Converts the output of gnunet-zonewalk (DNS resolutions)
+# into a proper input for gnunet-gns-benchmark.
+
+NUM_CLIENTS=3
+# How many different groups of names should we
+# create?  1/N will be in the 'shared' group.
+
+# FILE ($1) contains results from DNS lookup; strip
+# everything but the hostnames, remove duplicates
+# and then randomize the order.
+cat $1 | awk '{print $1}' | sort | uniq | shuf > $1.tmp
+TOTAL=`cat $1.tmp | wc -l`
+GROUP_SIZE=`expr $TOTAL / \( $NUM_TYPES + 1 \)`
+
+# First group (0) is to be shared among all clients
+for i in `seq 1 $NUM_CLIENTS`
+do
+  cat $1.tmp | head -n $GROUP_SIZE | awk "{print 0 \" \" \$1}" > $1.$i.tmp
+done
+
+# Second group (1) is unique per client
+OFF=0
+for i in `seq 1 $NUM_CLIENTS`
+do
+  END=`expr $OFF + $GROUP_SIZE`
+  cat $1.tmp | head -n $END | tail -n $GROUP_SIZE | awk "{print 1 \" \" \$1}" >> $1.$i.tmp
+# Shuffle again, so we mix the different request categories in terms of
+# when we issue the queries.
+  cat $1.$i.tmp | shuf > $1.$i
+  OFF="$END"
+  rm $1.$i.tmp
+done
+rm $1.tmp