summaryrefslogtreecommitdiff
path: root/contrib/scripts
diff options
context:
space:
mode:
authorNils Gillmann <ng0@n0.is>2018-05-19 14:43:13 +0000
committerNils Gillmann <ng0@n0.is>2018-05-19 14:43:13 +0000
commit6ab60d4920bb3199aee8cd872b930e9e3e808ba7 (patch)
tree3d761dbf8793a1d2422fbd14667255c7e0292ea4 /contrib/scripts
parentc2f27dfe8218545c327ab49d225a49910347c5c6 (diff)
Restructure contrib folder.
contrib/pogen.sh -> bin/pogen.sh bootstrap: Use new pogen location and execute it. contrib/openvpn-tap32: Move to contrib/3rdparty/Windows/openvpn-tap32. contrib/gnunet-logo*: Move to contrib/branding/logo/ Delete old patches in contrib, predating git. Move buildbot data to contrib/ci/buildbot, move docker data to contrib/ci/docker. Create contrib/conf and populate it with config files found in contrib and bin. Move gns related data to contrib/gns. delete contrib/repeat.sh Move contrib/log.php into contrib/web/log.php. Create folder contrib/scripts and use it for most scripts in contrib. Remove trailing whitespace in doc/Makefile.am Signed-off-by: Nils Gillmann <ng0@n0.is>
Diffstat (limited to 'contrib/scripts')
-rwxr-xr-xcontrib/scripts/coverage.sh14
-rwxr-xr-xcontrib/scripts/debug37
-rw-r--r--contrib/scripts/find_typedefs.py99
-rw-r--r--contrib/scripts/gdb-iterate-dll.py40
-rwxr-xr-xcontrib/scripts/generate-monkey-db.sh17
-rwxr-xr-xcontrib/scripts/gnunet-chk.py.in383
-rwxr-xr-xcontrib/scripts/gnunet-logread/gnunet-logread198
-rwxr-xr-xcontrib/scripts/gnunet-logread/gnunet-logread-ipc10
-rwxr-xr-xcontrib/scripts/gnunet-logread/gnunet-logread-ipc-sdedit60
-rwxr-xr-xcontrib/scripts/gnunet-suidfix27
-rw-r--r--contrib/scripts/gnunet_janitor.py.in78
-rw-r--r--contrib/scripts/gnunet_pyexpect.py.in83
-rwxr-xr-xcontrib/scripts/process_log.sh30
-rw-r--r--contrib/scripts/pydiffer.py.in44
-rwxr-xr-xcontrib/scripts/pydmesg75
-rwxr-xr-xcontrib/scripts/regression.sh54
-rwxr-xr-xcontrib/scripts/removetrailingwhitespace.py.in15
-rwxr-xr-xcontrib/scripts/report.sh252
-rwxr-xr-xcontrib/scripts/revisionary.sh98
-rw-r--r--contrib/scripts/terminate.py.in64
-rwxr-xr-xcontrib/scripts/testbed_cleanup.sh14
-rw-r--r--contrib/scripts/texinfo-hacks.el18
-rwxr-xr-xcontrib/scripts/visualize_stats.sh86
-rwxr-xr-xcontrib/scripts/zonewalk-to-types.sh35
24 files changed, 1831 insertions, 0 deletions
diff --git a/contrib/scripts/coverage.sh b/contrib/scripts/coverage.sh
new file mode 100755
index 000000000..dd6a6ab53
--- /dev/null
+++ b/contrib/scripts/coverage.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+# make sure configure was run with coverage enabled...
+lcov --directory . --zerocounters
+make check
+rm `find * -name "test_*.gc??"` `find * -name "perf_*.gc??"`
+for n in `find * -name "*.gc??" | grep libs`
+do
+ cd `dirname $n`
+ mv `basename $n` ..
+ cd -
+done
+lcov --directory . --capture --output-file app.info
+mkdir -p doc/coverage
+genhtml -o doc/coverage app.info
diff --git a/contrib/scripts/debug b/contrib/scripts/debug
new file mode 100755
index 000000000..3de2c9a14
--- /dev/null
+++ b/contrib/scripts/debug
@@ -0,0 +1,37 @@
+#!/bin/bash
+# /proc/sys/kernel/core_pattern should be core.%p.%E
+
+COREPID=$1
+
+COREFILES=`ls -1 *core.$COREPID* 2>/dev/null | wc -l`
+COREFILE=`ls -1 *core.$COREPID* 2>/dev/null | head -n 1`
+
+if [ $COREFILES -gt 1 ]; then
+ echo "Multiple files, using $COREFILE"
+fi
+
+
+if [ $COREFILES -eq 0 ]; then
+ SERVICENAME=$1
+ COREFILES=`ls -1 core.*.*$SERVICENAME 2>/dev/null | wc -l`
+ COREFILE=`ls -1 core.*.*$SERVICENAME 2>/dev/null | head -n 1`
+
+ if [ $COREFILES -gt 1 ]; then
+ echo "Multiple files, using $COREFILE"
+ fi
+fi
+
+if [ $COREFILES -eq 0 ]; then
+ echo "Core file for $1 not found"
+ exit 1
+fi
+
+echo "Using $COREFILE"
+
+EXECPATH=${COREFILE#*!}
+EXECPATH=`echo $EXECPATH | sed -e 's/!/\//g'`
+echo $EXECPATH
+echo ""
+echo ""
+
+gdb --core $COREFILE /$EXECPATH
diff --git a/contrib/scripts/find_typedefs.py b/contrib/scripts/find_typedefs.py
new file mode 100644
index 000000000..68f5c2782
--- /dev/null
+++ b/contrib/scripts/find_typedefs.py
@@ -0,0 +1,99 @@
+from __future__ import print_function
+import os
+import re
+import sys
+
+
+debug = False
+
+
+def get_td_from_function_signature(line, file, num):
+ left_paren = line.find('(')
+ if left_paren > 0:
+ left_paren += 1
+ li = line[left_paren:]
+ right_paren = line.find(')')
+ if right_paren > 0 and right_paren > left_paren and line[right_paren:].find('(') >= 0:
+ fname = line[:right_paren]
+ fname = fname.lstrip(' ').lstrip('*').lstrip(' ').rstrip(' ')
+ if len(fname) > 0:
+ if debug:
+ print("from {0}:{1}".format(file, num))
+ print("-T {0}".format(fname))
+
+
+def get_td_from_simple_type(line, file, num):
+ line = line.rstrip(' ').rstrip('\t').rstrip(' ').rstrip('\t')
+ right_space = line.rfind(' ')
+ right_tab = line.rfind('\t')
+ sep = right_tab if right_tab > right_space else right_space
+ sep += 1
+ tname = line[sep:]
+ tname = tname.lstrip('*')
+ if len(tname) > 0:
+ if debug:
+ print("from {0}:{1}".format(file, num))
+ print("-T {0}".format(tname))
+
+
+def find_typedefs(file):
+ with open(file, 'rb') as f:
+ td = False
+ td_struct = False
+ td_level = 0
+ td_line = []
+ data = f.read()
+ for i, l in enumerate(data.splitlines(False)):
+ # Don't try to be too smart: only count lines that begin with 'typedef '
+ l = l.rstrip(' ').rstrip('\t')
+ if len(l) == 0:
+ continue
+ if not td:
+ if l[:8] != 'typedef ':
+ continue
+ else:
+ td = True
+ if l[8:].lstrip(' ').lstrip('\t')[:6] == 'struct':
+ td_struct = True
+ if td_struct:
+ leftcbrace = l.find('{')
+ if leftcbrace >= 0:
+ if td_level == 0:
+ td_line.append(l[:leftcbrace])
+ l = l[leftcbrace + 1:]
+ td_level += 1
+ rightcbrace = l.rfind('}')
+ if rightcbrace >= 0:
+ td_level -= 1
+ if td_level == 0:
+ td_line.append(l[rightcbrace + 1:])
+ else:
+ td_line.append(l)
+ if len(l) > 0 and l[-1] == ';' and(not td_struct or td_level == 0):
+ td_line = ' '.join(td_line)
+ td_line = td_line[:-1]
+ if len(td_line) > 0:
+ if td_line[-1] == ')':
+ get_td_from_function_signature(td_line, file, i)
+ else:
+ get_td_from_simple_type(td_line, file, i)
+ td_line = []
+ td = False
+ td_struct = False
+ td_level = 0
+
+
+def scan_dir(d):
+ for dirpath, dirs, files in os.walk(d):
+ for f in files:
+ if re.match(r'(?!lt_).+\.(c|cc|h)$', f):
+ file = os.path.join(dirpath, f)
+ find_typedefs(file)
+
+
+if __name__ == '__main__':
+ if len(sys.argv[1:]) == 0:
+ arg = os.getcwd()
+ else:
+ arg = sys.argv[1]
+ scan_dir(arg)
diff --git a/contrib/scripts/gdb-iterate-dll.py b/contrib/scripts/gdb-iterate-dll.py
new file mode 100644
index 000000000..79d46aa96
--- /dev/null
+++ b/contrib/scripts/gdb-iterate-dll.py
@@ -0,0 +1,40 @@
+from gdb import *
+
+
+def search_dll(head, field, match, pfield):
+ """
+ Search in a DLL by iterates over it.
+
+ head: name of the symbol denoting the head of the DLL
+ field: the field that should be search for match
+ match: the mathing value for field
+ pfield: the field whose value is to be printed for matched elements; None to
+ print all fields of the matched elemented
+ """
+
+ (symbol, _) = lookup_symbol(head)
+ if symbol is None:
+ print("Can't find symbol: " + head)
+ return
+ symbol_val = symbol.value()
+ while symbol_val:
+ symbol_val_def = symbol_val.dereference()
+ field_val = symbol_val_def[field]
+ if field_val.type.code == gdb.TYPE_CODE_INT:
+ val = int(field_val)
+ res = (match == val)
+ elif (field_val.type.code == gdb.TYPE_CODE_STRING) or (field_val.type.code == gdb.TYPE_CODE_ARRAY):
+ val = str(field_val)
+ res = (match == val)
+ elif (field_val.type.code == gdb.TYPE_CODE_TYPEDEF):
+ val = str(field_val)
+ res = match in val
+ else:
+ continue
+
+ if res:
+ if pfield is None:
+ print(symbol_val_def)
+ else:
+ print(symbol_val_def[pfield])
+ symbol_val = symbol_val_def["next"]
diff --git a/contrib/scripts/generate-monkey-db.sh b/contrib/scripts/generate-monkey-db.sh
new file mode 100755
index 000000000..2afe55501
--- /dev/null
+++ b/contrib/scripts/generate-monkey-db.sh
@@ -0,0 +1,17 @@
+#!/bin/sh
+
+BASEPATH="$(dirname $0)"
+OLDDIR="${pwd}"
+GN_HOME="/usr/local/bin"
+
+export CC="cparser"
+export CFLAGS="-m32 --seaspider"
+
+cd $BASEPATH/.. && ./configure --prefix=$GN_HOME --with-extractor=$GN_HOME --with-microhttpd=$GN_HOME --with-libgcrypt=$GN_HOME && make && seaspider
+if test "$?" -ne 0
+then
+ echo "FAIL: building GNUnet"
+ exit 1
+fi
+
+cd $OLDDIR
diff --git a/contrib/scripts/gnunet-chk.py.in b/contrib/scripts/gnunet-chk.py.in
new file mode 100755
index 000000000..c976b2143
--- /dev/null
+++ b/contrib/scripts/gnunet-chk.py.in
@@ -0,0 +1,383 @@
+#!@PYTHON@
+# This file is part of GNUnet.
+# (C) 2013, 2018 Christian Grothoff (and other contributing authors)
+#
+# GNUnet is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published
+# by the Free Software Foundation; either version 3, or (at your
+# option) any later version.
+#
+# GNUnet is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with GNUnet; see the file COPYING. If not, write to the
+# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+# Boston, MA 02110-1301, USA.
+#
+# File: gnunet-chk.py
+# Brief: Computes GNUNET style Content Hash Key for a given file
+# Author: Sree Harsha Totakura
+
+from hashlib import sha512
+import logging
+import os
+import getopt
+import sys
+from Crypto.Cipher import AES
+from functools import reduce
+
+
+# Defaults
+DBLOCK_SIZE = (32 * 1024) # Data block size
+
+# Pick a multiple of 2 here to achive 8-byte alignment! We also
+# probably want DBlocks to have (roughly) the same size as IBlocks.
+# With SHA-512, the optimal value is 32768 byte / 128 byte = 256 (128
+# byte = 2 * 512 bits). DO NOT CHANGE!
+CHK_PER_INODE = 256
+
+CHK_HASH_SIZE = 64 # SHA-512 hash = 512 bits = 64 bytes
+
+CHK_QUERY_SIZE = CHK_HASH_SIZE # Again a SHA-512 hash
+
+GNUNET_FS_URI_PREFIX = "gnunet://fs/" # FS CHK URI prefix
+
+GNUNET_FS_URI_CHK_INFIX = "chk/" # FS CHK URI infix
+
+
+def encode_data_to_string(data):
+ """Returns an ASCII encoding of the given data block like
+ GNUNET_STRINGS_data_to_string() function.
+
+ data: A bytearray representing the block of data which has to be encoded
+ """
+ echart = "0123456789ABCDEFGHIJKLMNOPQRSTUV"
+ assert (None != data)
+ assert (bytearray == type(data))
+ size = len(data)
+ assert (0 != size)
+ vbit = 0
+ wpos = 0
+ rpos = 0
+ bits = 0
+ out = ""
+ while (rpos < size) or (vbit > 0):
+ if (rpos < size) and (vbit < 5):
+ bits = (bits << 8) | data[rpos] # eat 8 more bits
+ rpos += 1
+ vbit += 8
+ if (vbit < 5):
+ bits <<= (5 - vbit) # zero-padding
+ assert (vbit == ((size * 8) % 5))
+ vbit = 5
+ out += echart[(bits >> (vbit - 5)) & 31]
+ wpos += 1
+ vbit -= 5
+ assert (0 == vbit)
+ return out
+
+
+def sha512_hash(data):
+ """ Returns the sha512 hash of the given data.
+
+ data: string to hash
+ """
+ hash_obj = sha512()
+ hash_obj.update(data)
+ return hash_obj.digest()
+
+
+class AESKey:
+ """Class for AES Keys. Contains the main key and the initialization
+ vector. """
+
+ key = None # The actual AES key
+ iv = None # The initialization vector
+ cipher = None # The cipher object
+ KEY_SIZE = 32 # AES 256-bit key = 32 bytes
+ IV_SIZE = AES.block_size # Initialization vector size (= AES block size)
+
+ def __init__(self, passphrase):
+ """Creates a new AES key.
+
+ passphrase: string containing the passphrase to get the AES key and
+ initialization vector
+ """
+ passphrase = bytearray(passphrase)
+ self.key = bytearray(self.KEY_SIZE)
+ self.iv = bytearray(self.IV_SIZE)
+ if (len(passphrase) > self.KEY_SIZE):
+ self.key = passphrase[:self.KEY_SIZE]
+ passphrase = passphrase[self.KEY_SIZE:]
+ if (len(passphrase) > self.IV_SIZE):
+ self.iv = passphrase[:self.IV_SIZE]
+ else:
+ self.iv[0:len(passphrase)] = passphrase
+ else:
+ self.key[0:len(passphrase)] = passphrase
+ self.key = str(self.key)
+ self.iv = str(self.iv)
+ assert (len(self.key) == self.KEY_SIZE)
+ assert (len(self.iv) == self.IV_SIZE)
+
+
+def setup_aes_cipher_(aes_key):
+ """Initializes the AES object with settings similar to those in GNUnet.
+
+ aes_key: the AESKey object
+ Returns the newly initialized AES object
+ """
+ return AES.new(aes_key.key, AES.MODE_CFB, aes_key.iv, segment_size=128)
+
+
+def aes_pad_(data):
+ """Adds padding to the data such that the size of the data is a multiple of
+ 16 bytes
+
+ data: the data string
+ Returns a tuple:(pad_len, data). pad_len denotes the number of bytes added
+ as padding; data is the new data string with padded bytes at the end
+ """
+ pad_len = len(data) % 16
+ if (0 != pad_len):
+ pad_len = 16 - pad_len
+ pad_bytes = bytearray(15)
+ data += str(pad_bytes[:pad_len])
+ return (pad_len, data)
+
+
+def aes_encrypt(aes_key, data):
+ """Encrypts the given data using AES.
+
+ aes_key: the AESKey object to use for AES encryption
+ data: the data string to encrypt
+ """
+ (pad_len, data) = aes_pad_(data)
+ cipher = setup_aes_cipher_(aes_key)
+ enc_data = cipher.encrypt(data)
+ if (0 != pad_len):
+ enc_data = enc_data[:-pad_len]
+ return enc_data
+
+
+def aes_decrypt(aes_key, data):
+ """Decrypts the given data using AES
+
+ aes_key: the AESKey object to use for AES decryption
+ data: the data string to decrypt
+ """
+ (pad_len, data) = aes_pad_(data)
+ cipher = setup_aes_cipher_(aes_key)
+ ptext = cipher.decrypt(data)
+ if (0 != pad_len):
+ ptext = ptext[:-pad_len]
+ return ptext
+
+
+class Chk:
+ """Class for the content hash key."""
+ key = None
+ query = None
+ fsize = None
+
+ def __init__(self, key, query):
+ assert (len(key) == CHK_HASH_SIZE)
+ assert (len(query) == CHK_QUERY_SIZE)
+ self.key = key
+ self.query = query
+
+ def setSize(self, size):
+ self.fsize = size
+
+ def uri(self):
+ sizestr = repr(self.fsize)
+ if isinstance(self.fsize, int):
+ sizestr = sizestr[:-1]
+ return GNUNET_FS_URI_PREFIX + GNUNET_FS_URI_CHK_INFIX + \
+ encode_data_to_string(bytearray(self.key)) + "." + \
+ encode_data_to_string(bytearray(self.query)) + "." + \
+ sizestr
+
+
+def compute_depth_(size):
+ """Computes the depth of the hash tree.
+
+ size: the size of the file whose tree's depth has to be computed
+ Returns the depth of the tree. Always > 0.
+ """
+ depth = 1
+ fl = DBLOCK_SIZE
+ while (fl < size):
+ depth += 1
+ if ((fl * CHK_PER_INODE) < fl):
+ return depth
+ fl = fl * CHK_PER_INODE
+ return depth
+
+
+def compute_tree_size_(depth):
+ """Calculate how many bytes of payload a block tree of the given depth MAY
+ correspond to at most (this function ignores the fact that some blocks will
+ only be present partially due to the total file size cutting some blocks
+ off at the end).
+
+ depth: depth of the block. depth==0 is a DBLOCK.
+ Returns the number of bytes of payload a subtree of this depth may
+ correspond to.
+ """
+ rsize = DBLOCK_SIZE
+ for cnt in range(0, depth):
+ rsize *= CHK_PER_INODE
+ return rsize
+
+
+def compute_chk_offset_(depth, end_offset):
+ """Compute the offset of the CHK for the current block in the IBlock
+ above
+
+ depth: depth of the IBlock in the tree (aka overall number of tree levels
+ minus depth); 0 == DBLOCK
+ end_offset: current offset in the overall file, at the *beginning* of the
+ block for DBLOCK (depth == 0), otherwise at the *end* of the
+ block (exclusive)
+ Returns the offset in the list of CHKs in the above IBlock
+ """
+ bds = compute_tree_size_(depth)
+ if (depth > 0):
+ end_offset -= 1
+ ret = end_offset / bds
+ return ret % CHK_PER_INODE
+
+
+def compute_iblock_size_(depth, offset):
+ """Compute the size of the current IBLOCK. The encoder is triggering the
+ calculation of the size of an IBLOCK at the *end* (hence end_offset) of its
+ construction. The IBLOCK maybe a full or a partial IBLOCK, and this
+ function is to calculate how long it should be.
+
+ depth: depth of the IBlock in the tree, 0 would be a DBLOCK, must be > 0
+ (this function is for IBLOCKs only!)
+ offset: current offset in the payload (!) of the overall file, must be > 0
+ (since this function is called at the end of a block).
+ Returns the number of elements to be in the corresponding IBlock
+ """
+ assert (depth > 0)
+ assert (offset > 0)
+ bds = compute_tree_size_(depth)
+ mod = offset % bds
+ if mod is 0:
+ ret = CHK_PER_INODE
+ else:
+ bds /= CHK_PER_INODE
+ ret = mod / bds
+ if (mod % bds) is not 0:
+ ret += 1
+ return ret
+
+
+def compute_rootchk(readin, size):
+ """Returns the content hash key after generating the hash tree for the given
+ input stream.
+
+ readin: the stream where to read data from
+ size: the size of data to be read
+ """
+ depth = compute_depth_(size)
+ current_depth = 0
+ chks = [None] * (depth * CHK_PER_INODE) # list buffer
+ read_offset = 0
+ logging.debug("Begining to calculate tree hash with depth: " + repr(depth))
+ while True:
+ if (depth == current_depth):
+ off = CHK_PER_INODE * (depth - 1)
+ assert (chks[off] is not None)
+ logging.debug("Encoding done, reading CHK `" + chks[off].query + \
+ "' from " + repr(off) + "\n")
+ uri_chk = chks[off]
+ assert (size == read_offset)
+ uri_chk.setSize(size)
+ return uri_chk
+ if (0 == current_depth):
+ pt_size = min(DBLOCK_SIZE, size - read_offset)
+ try:
+ pt_block = readin.read(pt_size)
+ except IOError:
+ logging.warning("Error reading input file stream")
+ return None
+ else:
+ pt_elements = compute_iblock_size_(current_depth, read_offset)
+ pt_block = ""
+ pt_block = \
+ reduce((lambda ba, chk:
+ ba + (chk.key + chk.query)),
+ chks[(current_depth - 1) * CHK_PER_INODE:][:pt_elements],
+ pt_block)
+ pt_size = pt_elements * (CHK_HASH_SIZE + CHK_QUERY_SIZE)
+ assert (len(pt_block) == pt_size)
+ assert (pt_size <= DBLOCK_SIZE)
+ off = compute_chk_offset_(current_depth, read_offset)
+ logging.debug("Encoding data at offset " + repr(read_offset) + \
+ " and depth " + repr(current_depth) + " with block " \
+ "size " + repr(pt_size) + " and target CHK offset " + \
+ repr(current_depth * CHK_PER_INODE))
+ pt_hash = sha512_hash(pt_block)
+ pt_aes_key = AESKey(pt_hash)
+ pt_enc = aes_encrypt(pt_aes_key, pt_block)
+ pt_enc_hash = sha512_hash(pt_enc)
+ chk = Chk(pt_hash, pt_enc_hash)
+ chks[(current_depth * CHK_PER_INODE) + off] = chk
+ if (0 == current_depth):
+ read_offset += pt_size
+ if (read_offset == size) or \
+ (0 == (read_offset % (CHK_PER_INODE * DBLOCK_SIZE))):
+ current_depth += 1
+ else:
+ if (CHK_PER_INODE == off) or (read_offset == size):
+ current_depth += 1
+ else:
+ current_depth = 0
+
+
+def chkuri_from_path(path):
+ """Returns the CHK URI of the file at the given path.
+
+ path: the path of the file whose CHK has to be calculated
+ """
+ size = os.path.getsize(path)
+ readin = open(path, "rb")
+ chk = compute_rootchk(readin, size)
+ readin.close()
+ return chk.uri()
+
+
+def usage():
+ """Prints help about using this script."""
+ print("""
+Usage: gnunet-chk.py [options] file
+Prints the Content Hash Key of given file in GNUNET-style URI.
+
+Options:
+ -h, --help : prints this message
+""")
+
+
+if '__main__' == __name__:
+ try:
+ opts, args = getopt.getopt(sys.argv[1:], "h", ["help"])
+ except getopt.GetoptError as err:
+ print(err)
+ print("Exception occured")
+ usage()
+ sys.exit(2)
+ for option, value in opts:
+ if option in("-h", "--help"):
+ usage()
+ sys.exit(0)
+ if len(args) != 1:
+ print("Incorrect number of arguments passed")
+ usage()
+ sys.exit(1)
+ print(chkuri_from_path(args[0]))
diff --git a/contrib/scripts/gnunet-logread/gnunet-logread b/contrib/scripts/gnunet-logread/gnunet-logread
new file mode 100755
index 000000000..9b1c65401
--- /dev/null
+++ b/contrib/scripts/gnunet-logread/gnunet-logread
@@ -0,0 +1,198 @@
+#!@PERL@
+# helper tool to make gnunet logs more readable
+# try 'gnunet-logread -h' for usage
+
+use strict;
+use warnings;
+my $DEFAULT_SOCKET = '/tmp/gnunet-logread-ipc.sock';
+
+print STDERR <<X if -t STDIN and $#ARGV == -1;
+*** For a usage message, try '$0 -h'.
+*** For documentation, try 'perldoc $0'.
+*** Listening for GNUNET_log events on STDIN. Type CTRL-D to terminate.
+
+X
+
+use Getopt::Std;
+my (%opts, $name, $ipc, $msg_level, $msg_regex);
+getopts ('i:x:n:s:L:m:fhq', \%opts);
+
+use Pod::Usage qw( pod2usage );
+die pod2usage if $opts{h};
+
+use POSIX qw(mkfifo);
+
+use Term::ANSIColor qw(:constants :pushpop);
+$Term::ANSIColor::AUTOLOCAL = 1;
+
+my %levels = ( NONE => 0, ERROR => 1, WARNING => 2, INFO => 4, DEBUG => 8 );
+
+# Message type numbers to names
+my %msgtypes;
+my $prefix = $ENV{GNUNET_PREFIX} || '/usr';
+my $filename = "$prefix/include/gnunet/gnunet_protocols.h";
+$ipc = $opts{s} || $DEFAULT_SOCKET;
+
+if (open HEADER, $filename)
+{
+ while (<HEADER>)
+ {
+ $msgtypes{$2} = $1 if /^\s*#define\s+GNUNET_MESSAGE_TYPE_(\w+)\s+(\d+)/i;
+ }
+ close HEADER;
+} else {
+ warn <<X;
+Could not read $filename for message codes:
+ $!.
+Please provide a \$GNUNET_PREFIX environment variable to replace "/usr".
+Try also '$0 -h' for help.
+
+X
+}
+
+die "You can't read and write the socket at the same time"
+ if exists $opts{f} and exists $opts{n};
+
+if ((exists $opts{n} or exists $opts{f}) and not -r $ipc) {
+ undef $!;
+ die "Could not mkfifo $ipc: $!" unless mkfifo $ipc, 0600;
+ system('chgrp', 'gnunet', $ipc);
+ die "Could not chgrp $ipc to 'gnunet': $!" if $!;
+ chmod 0660, $ipc;
+ die "Could not chmod $ipc to allow gnunet group writes: $!" if $!;
+}
+
+if (exists $opts{n}) {
+ $name = $opts{n};
+ $msg_level = $opts{L} && exists $levels{$opts{L}} ? $levels{$opts{L}} : 0;
+ $msg_regex = $opts{m};
+ print STDERR "RE: /$msg_regex/\n" if defined $msg_regex;
+ open O, '>', $ipc or die "Cannot write to $ipc: $!";
+}
+
+if (exists $opts{f}) {
+ open(I, $ipc) or die "Cannot read from $ipc: $!";
+ &perform while <I>;
+ close I;
+} else {
+ &perform while <>;
+}
+fileno O and close O;
+exit;
+
+
+sub perform {
+ if (fileno O) {
+ my ($time, $type, $size, $from, $to, $level, $msg);
+ if (($time, $type, $size, $from, $to) =
+ /^([A-Z][a-z]{2}\ .[0-9]\ [0-9:]{8}(?:-[0-9]{6})?)\ util-client-.*\b
+ (?: Received | Transmitting )\ message \b.*?\b
+ type \s+ (\d+) \b.*?\b
+ size \s+ (\d+) \b.*?\b
+ (?: from \s+ (\S+)
+ | to \s+ (\S+) ) /x)
+ {
+ $from ||= $name;
+ $to ||= $name;
+ my ($time, $type, $size, $from, $to) = ($1, $2, $3,
+ $4 || $name, $5 || $name);
+ my $msg = exists $msgtypes{$type} ? $msgtypes{$type} : $type;
+ my $ofh = select O;
+ print O "$time\t$from -> $to\t$msg ($size)\n";
+ $| = 1;
+ select $ofh;
+ }
+ if (($time, $level, $msg) =
+ /^([A-Z][a-z]{2}\ .[0-9]\ [0-9:]{8}(?:-[0-9]{6})?)
+ \s+\S+\s+(\S+)\s+(.+)/x
+ and (exists $levels{$level}
+ && $levels{$level} <= $msg_level
+ && (!defined $msg_regex || $msg =~ /$msg_regex/i)))
+ {
+ print O "$time\t$name\t$level: $msg\n";
+ }
+ }
+ return if $opts{x} and /$opts{x}/io;
+ return if $opts{i} and not /$opts{i}/io;
+
+ # Timestamp (e.g. Nov 01 19:36:11-384136)
+ s/^([A-Z][a-z]{2} .[0-9] [0-9:]{8}(?:-[0-9]{6})?)/YELLOW $1/e;
+
+ # Log levels
+ s/\b(ERROR )\b/RED $1/ex;
+ s/\b(WARNING)\b/YELLOW $1/ex;
+ s/\b(INFO )\b/GREEN $1/ex;
+ s/\b(DEBUG )\b/BRIGHT_BLACK $1/ex;
+
+ # Service names
+ # TODO: might read the list from $GNUNET_PREFIX/libexec/gnunet/
+ s/\b(multicast|psyc|psycstore|social)\b/BLUE $1/gex;
+
+ # Add message type names
+ s/(\s+type\s+)(\d+)/
+ $1 . BRIGHT_CYAN (exists $msgtypes{$2} ? $msgtypes{$2} : 'UNKNOWN') .
+ CYAN " ($2)"/gei;
+
+ # logread-ipc output
+ s/(\s+)([A-Z_]+)( \(\d+\))$/$1 . BRIGHT_CYAN $2 . CYAN $3/e;
+
+ print;
+}
+
+__END__
+
+=pod
+
+=head1 NAME
+
+gnunet-logread - a GNUnet log analyzer, colorizer and aggregator
+
+=head1 SYNOPSIS
+
+ <gnunet-service> |& $0 [<options>]
+ or
+ $0 [<options>] [<logfile>]
+
+ Options:
+ -f Follow input from IPC FIFO socket.
+
+ Regular screen output options:
+ -i <regex> Include only messages that match <regex>.
+ -x <regex> Exclude all messages that match <regex>.
+ -q Quiet: Do not show usage advice to new users.
+
+ Options to forward messages to the IPC FIFO socket:
+ -n <component_name> Name of the component we are forwarding messages for.
+ -s </path/to/ipc.sock> Default = $DEFAULT_SOCKET
+ -L <LOGLEVEL> Minimum level of messages to forward:
+ Log levels: NONE, ERROR, WARNING, INFO, DEBUG.
+ -m <regex> Only forward messages matching a regular expression.
+
+ See 'perldoc gnunet-logread' for a longer explanation.
+
+=head1 MOTIVATION
+
+GNUnet debug logs are a tedious read, but given a complex system that we
+cannot run all parts of in a debugger all the time, some gathering and
+structuring of events and message passing is useful.
+
+At first, this tool simply makes logs easier to read. Both if viewed in
+real-time or taken from disk. Then it also allows to extract all message
+passing events from it and forward them to a special process that aggregates
+all message passing events and therefore helps you make sense of all the
+inter-process communication (IPC) happening between the various pieces of
+the GNUnet system beast.
+
+That master process is simply an extra gnunet-logread that you run in a
+separate window and adorn it with the '-f' flag. The submitting processes
+instead need to be given a '-n' flag. That is because from the GNUnet logs
+it isn't clear which process events belong to. For example you may be
+having events taking place in the 'util' subsystem of gnunet-psyc-service
+just as much as in the 'util' subsystem of gnunet-multicast-service. In
+order to make sense of them it is necessary to manually add that info. This
+could be remedied by extending the semantics of the GNUNET_log facility
+instead, but that is still subject to further consideration.
+
+=head1 AUTHORS
+
+tg & lynX
diff --git a/contrib/scripts/gnunet-logread/gnunet-logread-ipc b/contrib/scripts/gnunet-logread/gnunet-logread-ipc
new file mode 100755
index 000000000..72f9f47df
--- /dev/null
+++ b/contrib/scripts/gnunet-logread/gnunet-logread-ipc
@@ -0,0 +1,10 @@
+#!/bin/sh
+#
+# Usage: gnunet-logread-ipc | gnunet-logread
+#
+# ... obsoleted by gnunet-logread's new -f option that does the same thing
+
+# FIXME: Replace /tmp with our use of $TMPDIR and similar.
+ipc=${1:-/tmp/gnunet-logread-ipc.sock}
+test -e "$ipc" || mkfifo "$ipc"
+cat "$ipc"
diff --git a/contrib/scripts/gnunet-logread/gnunet-logread-ipc-sdedit b/contrib/scripts/gnunet-logread/gnunet-logread-ipc-sdedit
new file mode 100755
index 000000000..f8b7dc735
--- /dev/null
+++ b/contrib/scripts/gnunet-logread/gnunet-logread-ipc-sdedit
@@ -0,0 +1,60 @@
+#!@PERL@
+
+# 1. Start sdedit and enable 'RT diagram server' in 'Global preferences'.
+#
+# 2. Start this tool (see defaults below):
+# gnunet-logread-ipc-sdedit -n buffer-name -i /path/to/ipc.sock -h <sdedit-host> -p <sdedit-port>
+#
+# 3. Start a gnunet-logread instance for each component with the -n <component_name> option
+
+use strict;
+use warnings;
+
+use Getopt::Std;
+use IO::Socket::INET;
+use POSIX qw(mkfifo);
+
+my %opts;
+getopts ('i:n:h:p:', \%opts);
+
+my $ipc = $opts{i} || '/tmp/gnunet-logread-ipc.sock';
+my $name = $opts{n} || 'gnunet';
+my $host = $opts{h} || 'localhost';
+my $port = $opts{p} || 16001;
+my %svcs = map { $_ => 1 } @ARGV;
+
+my $sdedit = IO::Socket::INET->new(PeerAddr => $host,
+ PeerPort => $port,
+ Proto => 'tcp')
+ or die "Cannot connect to $host:$port: $!\n";
+
+print $sdedit "$name\n";
+print $sdedit "_t:time[e]\n";
+print $sdedit "$_:$_\[ap\] \"$_\"\n" for @ARGV;
+print $sdedit "_e:ext[e]\n";
+print $sdedit "\n";
+
+mkfifo $ipc, 0600 or die "$ipc: $!\n" unless -e $ipc;
+open IPC, '<', $ipc or die "$ipc: $!\n";
+$| = 1;
+while (<IPC>)
+{
+ print;
+ my ($time, $from, $to, $msg, $svc);
+ if (my ($time, $from, $to, $msg) =
+ /^([A-Z][a-z]{2}\ .[0-9]\ [0-9:]{8}(?:-[0-9]{6})?)\s+
+ (\S+)\s+ -> \s+(\S+)\s+ (\S+\s+ \(\d+\))/x)
+ {
+ $from = '_e' unless exists $svcs{$from};
+ $to = '_e' unless exists $svcs{$to};
+ print $sdedit "*0 _t\n$time\n*0\n", "$from:$to.$msg\n"
+ }
+ elsif (($time, $svc, $msg) =
+ /^([A-Z][a-z]{2}\ .[0-9]\ [0-9:]{8}(?:-[0-9]{6})?)\s+
+ (\S+)\s+(.+)/x)
+ {
+ print $sdedit "*0 _t\n$time\n*0\n", "*0 $svc\n$msg\n*0\n"
+ }
+}
+
+close IPC;
diff --git a/contrib/scripts/gnunet-suidfix b/contrib/scripts/gnunet-suidfix
new file mode 100755
index 000000000..992378966
--- /dev/null
+++ b/contrib/scripts/gnunet-suidfix
@@ -0,0 +1,27 @@
+#!/bin/sh
+#
+# "suidfix" is german and it means something like immediate suicide.
+
+# taken from dangole's lede config.. thx!
+suid_root_helpers="exit nat-server nat-client transport-bluetooth transport-wlan vpn"
+libexec="${GNUNET_PREFIX}/lib/gnunet/libexec"
+
+chmodown_execbin() {
+ if [ -x $1 ]; then
+ if [ "$3" ]; then
+ chown $3 $1 2>/dev/null && chmod $2 $1
+ else
+ chmod $2 $1
+ fi
+ ls -l $1
+ else
+ echo "Missing: $1"
+ fi
+}
+
+for helper in $suid_root_helpers; do
+ chmodown_execbin ${libexec}/gnunet-helper-$helper u+s
+done
+chmodown_execbin ${libexec}/gnunet-helper-dns 4750 root:gnunetdns
+chmodown_execbin ${libexec}/gnunet-service-dns 2750 gnunet:gnunetdns
+
diff --git a/contrib/scripts/gnunet_janitor.py.in b/contrib/scripts/gnunet_janitor.py.in
new file mode 100644
index 000000000..74fc70886
--- /dev/null
+++ b/contrib/scripts/gnunet_janitor.py.in
@@ -0,0 +1,78 @@
+#!@PYTHON@
+# This file is part of GNUnet.
+# (C) 2011 Christian Grothoff (and other contributing authors)
+#
+# GNUnet is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published
+# by the Free Software Foundation; either version 2, or (at your
+# option) any later version.
+#
+# GNUnet is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with GNUnet; see the file COPYING. If not, write to the
+# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+# Boston, MA 02110-1301, USA.
+#
+# Finds any gnunet processes still running in the system and kills them
+#
+# gnunet janitor can be used by invoking `make' like this:
+# TESTS_ENVIRONMENT='${top_srcdir}/contrib/gnunet_janitor.py &&' make check
+
+from __future__ import print_function
+import os
+import re
+import subprocess
+import sys
+import shutil
+import time
+import signal
+import terminate
+
+if os.name == 'nt':
+ from win32com.client import GetObject
+ WMI = GetObject('winmgmts:')
+
+def get_process_list ():
+ result = []
+ if os.name == 'nt':
+ processes = WMI.InstancesOf('Win32_Process')
+ for p in processes:
+ result.append ((p.Properties_('ProcessId').Value, re.sub (r'(.+)\.exe', r'\1', p.Properties_('Name').Value)))
+ else:
+ pids = [pid for pid in os.listdir('/proc') if pid.isdigit ()]
+ for pid in pids:
+ with open (os.path.join ('/proc', pid, 'cmdline'), 'rb') as p:
+ cmdline = p.read ().split ('\x00')
+ if len (cmdline) > 0:
+ result.append ((pid, cmdline[0]))
+ return result
+
+def main ():
+ procs = get_process_list ()
+ gnunet_procs = []
+ for p in procs:
+ if re.match (r'gnunet-.+', p[1]):
+ gnunet_procs.append (p)
+ for p in gnunet_procs:
+ if re.match (r'gnunet-service-arm', p[1]):
+ print ("killing arm process {0:5} {1}".format (p[0], p[1]))
+ try:
+ terminate.safe_terminate_process_by_pid (int (p[0]), 1)
+ except OSError as e:
+ print ("failed: {0}".format (e))
+ pass
+ for p in gnunet_procs:
+ if not re.match (r'gnunet-service-arm', p[1]):
+ print ("killing non-arm process {0:5} {1}".format (p[0], p[1]))
+ try:
+ terminate.safe_terminate_process_by_pid (int (p[0]), 1)
+ except OSError as e:
+ print ("failed: {0}".format (e))
+ pass
+
+if __name__ == '__main__':
+ sys.exit (main ())
diff --git a/contrib/scripts/gnunet_pyexpect.py.in b/contrib/scripts/gnunet_pyexpect.py.in
new file mode 100644
index 000000000..cfeb06d8d
--- /dev/null
+++ b/contrib/scripts/gnunet_pyexpect.py.in
@@ -0,0 +1,83 @@
+#!@PYTHON@
+# This file is part of GNUnet.
+# (C) 2010 Christian Grothoff (and other contributing authors)
+#
+# GNUnet is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published
+# by the Free Software Foundation; either version 2, or (at your
+# option) any later version.
+#
+# GNUnet is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with GNUnet; see the file COPYING. If not, write to the
+# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+# Boston, MA 02110-1301, USA.
+#
+# Testcase for gnunet-peerinfo
+from __future__ import print_function
+import os
+import re
+import subprocess
+import sys
+import shutil
+import time
+
+class pexpect (object):
+ def __init__ (self):
+ super (pexpect, self).__init__ ()
+
+ def spawn (self, stdin, arglist, *pargs, **kwargs):
+ env = kwargs.pop ('env', None)
+ if env is None:
+ env = os.environ.copy ()
+ # This messes up some testcases, disable log redirection
+ env.pop ('GNUNET_FORCE_LOGFILE', None)
+ self.proc = subprocess.Popen (arglist, *pargs, env=env, **kwargs)
+ if self.proc is None:
+ print ("Failed to spawn a process {0}".format (arglist))
+ sys.exit (1)
+ if stdin is not None:
+ self.stdo, self.stde = self.proc.communicate (stdin)
+ else:
+ self.stdo, self.stde = self.proc.communicate ()
+ return self.proc
+
+ def expect (self, s, r, flags=0):
+ stream = self.stdo if s == 'stdout' else self.stde
+ if isinstance (r, str):
+ if r == "EOF":
+ if len (stream) == 0:
+ return True
+ else:
+ print ("Failed to find `{1}' in {0}, which is `{2}' ({3})".format (s, r, stream, len (stream)))
+ sys.exit (2)
+ raise ValueError ("Argument `r' should be an instance of re.RegexObject or a special string, but is `{0}'".format (r))
+ m = r.search (stream.decode(), flags)
+ if not m:
+ print ("Failed to find `{1}' in {0}, which is is `{2}'".format (s, r.pattern, stream))
+ sys.exit (2)
+ stream = stream[m.end ():]
+ if s == 'stdout':
+ self.stdo = stream
+ else:
+ self.stde = stream
+ return m
+
+ def read (self, s, size=-1):
+ stream = self.stdo if s == 'stdout' else self.stde
+ result = ""
+ if size < 0:
+ result = stream
+ new_stream = ""
+ else:
+ result = stream[0:size]
+ new_stream = stream[size:]
+ if s == 'stdout':
+ self.stdo = new_stream
+ else:
+ self.stde = new_stream
+ return result
diff --git a/contrib/scripts/process_log.sh b/contrib/scripts/process_log.sh
new file mode 100755
index 000000000..c25c515c2
--- /dev/null
+++ b/contrib/scripts/process_log.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# Usage: service should print "STARTING SERVICE (srvc) for peer [PEER]" where:
+# - "srvc" is the service name (in lowercase, as in the log output).
+# It cannot contain parenthesis in its name.
+# - "PEER" is the peer ID. Should be 4 alfanumeric characters
+
+grep "STARTING SERVICE " log > __tmp_peers
+
+SED_EXPR=""
+while read -r line; do
+ SRVC=`echo "$line" | sed -e 's/.*(\([^)]*\)).*/\1/'`
+ PEER=`echo "$line" | sed -e 's/.*\[\(....\)\].*/\1/'`
+ PID=`echo "$line" | sed -e "s/.*$SRVC-\([0-9]*\).*/\1/"`
+ echo "$SRVC $PID => $PEER"
+
+ SED_EXPR="${SED_EXPR}s/$SRVC-\([a-z2]*\)-$PID/$SRVC \1 $PEER/;"
+ SED_EXPR="${SED_EXPR}s/$SRVC-$PID/$SRVC XXX $PEER/;"
+ SED_EXPR="${SED_EXPR}s/$SRVC-api-[0-9]/$SRVC-api- /;"
+done < __tmp_peers
+rm __tmp_peers
+
+sed -e "$SED_EXPR" log > .log
+echo "$0 sed regex: $SED_EXPR" >> .log
+
+SIZE=`stat -c%s .log`
+
+if [[ "`ps aux | grep "kwrite .lo[g]"`" = "" && "$SIZE" < 10000000 ]]; then
+ kwrite .log --geometry 960x1140-960 &
+fi
diff --git a/contrib/scripts/pydiffer.py.in b/contrib/scripts/pydiffer.py.in
new file mode 100644
index 000000000..10145371c
--- /dev/null
+++ b/contrib/scripts/pydiffer.py.in
@@ -0,0 +1,44 @@
+#!@PYTHON@
+import os
+import sys
+import difflib
+import filecmp
+
+
+def getdiff(old, new):
+ diff = []
+ with open(old) as a:
+ with open(new) as b:
+ for l in difflib.unified_diff(a.read().splitlines(), b.read().splitlines()):
+ diff.append(l)
+ return diff
+
+
+def dc_getdiff(dc, old, new):
+ diff = []
+ for f in dc.left_only:
+ diff.append("Only in {}: {}".format(old, f))
+ for f in dc.right_only:
+ diff.append("Only in {}: {}".format(new, f))
+ for f in dc.diff_files:
+ r = getdiff(os.path.join(old, f), os.path.join(new, f))
+ diff.extend(r)
+ for dn, dc in dc.subdirs.items():
+ r = dc_getdiff(dc, os.path.join(old, dn), os.path.join(new, dn))
+ diff.extend(r)
+ return diff
+
+
+def dcdiff(old, new):
+ dc = filecmp.dircmp(old, new)
+ diff = dc_getdiff(dc, old, new)
+ return diff
+
+
+def main():
+ for l in dcdiff(sys.argv[1], sys.argv[2]):
+ print(l)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/contrib/scripts/pydmesg b/contrib/scripts/pydmesg
new file mode 100755
index 000000000..d60e08fe3
--- /dev/null
+++ b/contrib/scripts/pydmesg
@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+# coding=utf8
+
+# Copyright (C) 2010 Saúl ibarra Corretgé <saghul@gmail.com>
+#
+
+"""
+pydmesg: dmesg with human-readable timestamps
+"""
+
+from __future__ import with_statement
+
+import re
+import subprocess
+import sys
+
+from datetime import datetime, timedelta
+
+
+_datetime_format = "%Y-%m-%d %H:%M:%S"
+_dmesg_line_regex = re.compile("^\[(?P<time>\d+\.\d+)\](?P<line>.*)$")
+
+def exec_process(cmdline, silent, input=None, **kwargs):
+ """Execute a subprocess and returns the returncode, stdout buffer and stderr buffer.
+ Optionally prints stdout and stderr while running."""
+ try:
+ sub = subprocess.Popen(cmdline, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs)
+ stdout, stderr = sub.communicate(input=input)
+ returncode = sub.returncode
+ if not silent:
+ sys.stdout.write(stdout)
+ sys.stderr.write(stderr)
+ except OSError,e:
+ if e.errno == 2:
+ raise RuntimeError('"%s" is not present on this system' % cmdline[0])
+ else:
+ raise
+ if returncode != 0:
+ raise RuntimeError('Got return value %d while executing "%s", stderr output was:\n%s' % (returncode, " ".join(cmdline), stderr.rstrip("\n")))
+ return stdout
+
+def human_dmesg():
+ now = datetime.now()
+ uptime_diff = None
+ try:
+ with open('/proc/uptime') as f:
+ uptime_diff = f.read().strip().split()[0]
+ except IndexError:
+ return
+ else:
+ try:
+ uptime = now - timedelta(seconds=int(uptime_diff.split('.')[0]), microseconds=int(uptime_diff.split('.')[1]))
+ except IndexError:
+ return
+
+ dmesg_data = exec_process(['dmesg'], True)
+ for line in dmesg_data.split('\n'):
+ if not line:
+ continue
+ match = _dmesg_line_regex.match(line)
+ if match:
+ try:
+ seconds = int(match.groupdict().get('time', '').split('.')[0])
+ nanoseconds = int(match.groupdict().get('time', '').split('.')[1])
+ microseconds = int(round(nanoseconds * 0.001))
+ line = match.groupdict().get('line', '')
+ t = uptime + timedelta(seconds=seconds, microseconds=microseconds)
+ except IndexError:
+ pass
+ else:
+ print "[%s]%s" % (t.strftime(_datetime_format), line)
+
+
+if __name__ == '__main__':
+ human_dmesg()
diff --git a/contrib/scripts/regression.sh b/contrib/scripts/regression.sh
new file mode 100755
index 000000000..1f799797a
--- /dev/null
+++ b/contrib/scripts/regression.sh
@@ -0,0 +1,54 @@
+#!/bin/sh
+cd
+cd gnunet
+svn up > /dev/null
+H=`hostname`
+echo "================START===================" >> regression.$H
+RUN=`date +%0y%0m%0d%0k%0M`
+uname -a >> regression.$H
+date >> regression.$H
+echo "Run $RUN" >> regression.$H
+svn up >> regression.$H
+export GNUNET_PREFIX=$HOME
+export PATH=$HOME/bin:$PATH
+./bootstrap >> regression.$H.$RUN 2>&1
+./configure --prefix=$HOME --with-extractor=$HOME --with-microhttpd=$HOME >> regression.$H.$RUN 2>&1
+if [ $? != 0 ]
+then
+ echo configure failed >> regression.$H
+ exit
+fi
+KEEP=0
+make clean >> regression.$H.$RUN 2>&1
+make install >> regression.$H.$RUN 2>&1
+cd src
+for n in `ls --hide "Makefile*"`
+do
+ cd $n
+ if [ -f Makefile ]
+ then
+ make check >> ../regression.$H.$RUN.$n 2>&1
+ if [ $? != 0 ]
+ then
+ echo Tests for $n failed >> ../regression.$H
+ echo "--------- Details for $n -----------" >> ../regression.$H
+ cat regression.$H.$RUN.$n >> ../regression.$H
+ echo "----- End of Details for $n --------" >> ../regression.$H
+ KEEP=1
+ else
+ echo Tests for $n succeeded >> ../regression.$H
+ fi
+ fi
+ cd ..
+done
+echo "================END====================" >> regression.$H
+
+if [ $KEEP == 0]
+then
+ rm regression.$H.$RUN*
+ rm regression.$H
+else
+ svn add regression.$H > /dev/null
+ svn commit -m "Regression in tests on $H" regression.$H
+fi
+
diff --git a/contrib/scripts/removetrailingwhitespace.py.in b/contrib/scripts/removetrailingwhitespace.py.in
new file mode 100755
index 000000000..5824fb591
--- /dev/null
+++ b/contrib/scripts/removetrailingwhitespace.py.in
@@ -0,0 +1,15 @@
+#!@PYTHON@
+
+import sys
+import re
+
+
+for fileName in sys.argv[1:]:
+ f = open(fileName, 'r+')
+ fileString = f.read()
+
+ fileString = re.sub(r'[ ]+\n', r'\n', fileString)
+ fileString = re.sub(r'\r', r'', fileString)
+ f.seek(0)
+ f.write(fileString)
+ f.truncate(len(fileString))
diff --git a/contrib/scripts/report.sh b/contrib/scripts/report.sh
new file mode 100755
index 000000000..7fbb2f46c
--- /dev/null
+++ b/contrib/scripts/report.sh
@@ -0,0 +1,252 @@
+#!/bin/sh
+
+TEST=`type type|grep not`
+if test -n "$TEST"; then
+ WHICH=which
+else
+ WHICH=type
+fi
+
+echo "Please submit the following information with your bug report: "
+echo "--------------------------------------------------------------"
+OS=`uname -s 2>/dev/null`
+echo "OS : $OS"
+REL=`uname -r 2>/dev/null`
+echo "OS RELEASE : $REL"
+HW=`uname -m 2>/dev/null`
+echo "HARDWARE : $HW"
+
+TEST=`$WHICH gcc 2>/dev/null`
+if test -n "$TEST"; then
+ VERS=`gcc --version 2>/dev/null | head -n 1`
+ echo "gcc : $VERS"
+else
+ echo "gcc : Not Found";
+fi
+
+TEST=`$WHICH gmake 2>/dev/null`
+if test -n "$TEST" ; then
+ gmake --version 2>/dev/null |\
+ awk -F, '{print $1}' |\
+ awk '/GNU Make/{print "GNU gmake :",$NF}'
+else
+ TEST=`make --version 2>/dev/null`
+ if test -n "$TEST"; then
+ make --version 2>/dev/null |\
+ awk -F, '{print $1}' |\
+ awk '/GNU Make/{print "make :",$NF}'
+ else
+ echo "make : Not Found"
+ fi
+fi
+
+TEST=`$WHICH autoconf 2>/dev/null`
+if test -n "$TEST"; then
+ autoconf --version |\
+ head -n 1 |\
+ awk '{\
+ if (length($4) == 0) {\
+ print "autoconf : "$3\
+ } else {\
+ print "autoconf : "$4\
+ }}'
+else
+ echo "autoconf : Not Found"
+fi
+
+TEST=`$WHICH automake 2>/dev/null`
+if test -n "$TEST"; then
+ automake --version 2>/dev/null |\
+ head -n 1 |\
+ awk '{print "automake : "$4}'
+else
+ echo "automake : Not Found"
+fi
+
+TEST=`$WHICH libtool 2>/dev/null`
+if test -n "$TEST"; then
+ libtool --version 2>/dev/null |\
+ head -n 1 |\
+ awk '{print "libtool : "$4}'
+else
+ echo "libtool : Not Found"
+fi
+
+TEST=`$WHICH extract 2>/dev/null`
+if test -n "$TEST"; then
+ extract -v 2>/dev/null |\
+ head -n 1 |\
+ awk '{print "libextractor : "$2}'
+else
+ echo "libextractor : Not Found"
+fi
+
+if test -x gnunetd; then
+ gnunetd -v | sed -e "s/v//" 2>/dev/null |\
+ awk '{print "GNUnet 0.8 : "$2 (may conflict!)}'
+else
+ echo "GNUnet 0.8 : Not Found (good)"
+fi
+
+TEST=`$WHICH gnunet-arm 2>/dev/null`
+if test -n "$TEST"; then
+ gnunet-arm -v | sed -e "s/v//" 2>/dev/null |\
+ awk '{print "GNUnet 0.9 : "$2}'
+else
+ echo "GNUnet 0.9 : Not Found"
+fi
+
+TEST=`$WHICH libgcrypt-config 2> /dev/null`
+if test -n "$TEST"; then
+ libgcrypt-config --version 2> /dev/null | \
+ awk '{print "libgcrypt : "$1}'
+else
+ echo "libgcrypt : Not Found"
+fi
+
+TEST=`$WHICH mysql_config 2> /dev/null`
+if test -n "$TEST"; then
+ mysql_config --version 2> /dev/null | \
+ awk '{print "mysql : "$1}'
+else
+ echo "mysql : Not Found"
+fi
+
+TEST=`$WHICH pkg-config 2> /dev/null`
+if test -n "$TEST"; then
+ pkg-config --version 2> /dev/null | \
+ awk '{print "pkg-config : "$1}'
+else
+ echo "pkg-config : Not Found"
+fi
+
+TEST=`$WHICH pkg-config 2> /dev/null`
+if test -n "$TEST"; then
+ pkg-config --modversion glib-2.0 2> /dev/null | \
+ awk '{print "glib2 : "$1}'
+else
+ echo "glib2 : Not Found"
+fi
+
+TEST=`$WHICH pkg-config 2> /dev/null`
+if test -n "$TEST"; then
+ pkg-config --modversion gtk+-2.0 2> /dev/null | \
+ awk '{print "gtk2+ : "$1}'
+else
+ echo "gtk2+ : Not Found"
+fi
+
+TEST=`$WHICH dpkg 2> /dev/null`
+if test -n "$TEST"; then
+ LINES=`dpkg -s libgmp-dev | grep Version | wc -l 2> /dev/null`
+ if test "$LINES" = "1"
+ then
+ VERSION=`dpkg -s libgmp-dev | grep Version | awk '{print $2}'`
+ echo "GMP : libgmp-dev-$VERSION.deb"
+ else
+ echo "GMP : dpkg: libgmp-dev not installed"
+ fi
+else
+ TEST=`$WHICH rpm 2> /dev/null`
+ if test -n "$TEST"; then
+ rpm -q gmp | sed -e "s/gmp-//" 2> /dev/null | \
+ awk '{print "GMP : "$1.rpm}'
+ else
+ echo "GMP : Test not available"
+ fi
+fi
+
+TEST=`$WHICH dpkg 2> /dev/null`
+if test -n "$TEST"; then
+ LINES=`dpkg -s libunistring-dev | grep Version | wc -l`
+ if test "$LINES" = "1"
+ then
+ VERSION=`dpkg -s libunistring-dev | grep Version | awk '{print $2}'`
+ echo "libunistring : libunistring3-dev-$VERSION.deb"
+ else
+ echo "libunistring : dpkg: libunistring3-dev not installed"
+ fi
+else
+ TEST=`$WHICH rpm 2> /dev/null`
+ if test -n "$TEST"; then
+ rpm -q unistring | sed -e "s/unistring-//" 2> /dev/null | \
+ awk '{print "libunistring : "$1.rpm}'
+ else
+ echo "libunistring : Test not available"
+ fi
+fi
+
+TEST=`$WHICH gettext 2> /dev/null`
+if test -n "$TEST"; then
+ gettext --version | head -n1 2> /dev/null | \
+ awk '{print "GNU gettext : "$4}'
+else
+ echo "GNU gettext : Not found"
+fi
+
+
+TEST=`$WHICH curl-config 2> /dev/null`
+if test -n "$TEST"; then
+ curl-config --version | head -n1 2> /dev/null | \
+ awk '{print "libcurl : "$2}'
+else
+ echo "libcurl : Not found"
+fi
+
+TEST=`$WHICH gnurl-config 2> /dev/null`
+if test -n "$TEST"; then
+ gnurl-config --version | head -n1 2> /dev/null | \
+ awk '{print "libgnurl : "$2}'
+else
+ echo "libgnurl : Not found"
+fi
+
+echo -n "libmicrohttpd : "
+TMPFILE=`mktemp /tmp/mhd-version-testXXXXXX`
+cat - >$TMPFILE.c <<EOF
+#include <microhttpd.h>
+#include <stdio.h>
+int main()
+{
+ fprintf (stdout, "%X\n", MHD_VERSION);
+ return 0;
+}
+EOF
+
+gcc -o $TMPFILE $TMPFILE.c 2> /dev/null && $TMPFILE || echo "Not found"
+rm -f $TMPFILE $TMPFILE.bin
+
+
+echo -n "GNU GLPK : "
+TMPFILE=`mktemp /tmp/glpk-version-testXXXXXX`
+cat - >$TMPFILE.c <<EOF
+#include <glpk.h>
+#include <stdio.h>
+int main()
+{
+ fprintf (stdout, "%u.%u\n", GLP_MAJOR_VERSION, GLP_MINOR_VERSION);
+ return 0;
+}
+EOF
+
+gcc -o $TMPFILE $TMPFILE.c 2> /dev/null && $TMPFILE || echo "Not found"
+rm -f $TMPFILE $TMPFILE.bin
+
+
+echo -n "GNUtls : "
+TMPFILE=`mktemp /tmp/gnutls-version-testXXXXXX`
+cat - >$TMPFILE.c <<EOF
+#include <gnutls/gnutls.h>
+#include <stdio.h>
+int main()
+{
+ fprintf (stdout, "%s\n", GNUTLS_VERSION);
+ return 0;
+}
+EOF
+
+gcc -o $TMPFILE $TMPFILE.c 2> /dev/null && $TMPFILE || echo "Not found"
+rm -f $TMPFILE $TMPFILE.bin
+
+
+echo "--------------------------------------------------------------"
diff --git a/contrib/scripts/revisionary.sh b/contrib/scripts/revisionary.sh
new file mode 100755
index 000000000..5778cf148
--- /dev/null
+++ b/contrib/scripts/revisionary.sh
@@ -0,0 +1,98 @@
+#!/usr/local/bin/bash
+
+STARTREVISION=14033
+ENDREVISION=15268
+CURRENTREVISION=$STARTREVISION
+HOME_DIR='/home/gnunet/FreeBSD7-AMD64-wachs/freebsd7-amd64-wachs/build'
+
+
+CMD_UPDATE="svn up --force --accept theirs-full -r"
+CMD_CLEANUP="killall -s 31 -r gnunet-*; make distclean;"
+CMD_PREPARATION="./bootstrap; ./configure --prefix=/tmp/gnb --with-extractor=/usr/local"
+CMD_BUILD="make all"
+CMD_TEST="cd src/transport ; make test_transport_api_tcp; ./test_transport_api_tcp"
+
+#LOGGING=""
+LOGGING=" 1> /dev/null 2> errors.txt"
+LOGFILE="log.txt"
+
+function output ()
+{
+ eval echo $1
+ eval echo $1 >> $LOGFILE
+}
+
+
+while [ $CURRENTREVISION -le $ENDREVISION ]; do
+ output 'Testing revision $CURRENTREVISION'
+# updating
+ output ' -> updating '
+ eval cd $HOME_DIR
+ CMD="$CMD_UPDATE $CURRENTREVISION $LOGGING"
+ eval $CMD
+ result=$?
+ if [ $result -eq 0 ]; then
+ output " updating OK"
+ else
+ output " updating FAILED"
+ (( CURRENTREVISION++ ))
+ continue
+ fi
+
+# clean up
+ output " -> cleanup "
+ CMD="$CMD_CLEANUP $LOGGING"
+ eval $CMD
+ result=$?
+ if [ $result -eq 0 ]; then
+ output " cleanup OK"
+ else
+ output " cleanup FAILED"
+ (( CURRENTREVISION++ ))
+ continue
+ fi
+# preparing
+ output " -> preparation "
+ CMD="$CMD_PREPARATION $LOGGING"
+ #echo $CMD
+ eval $CMD
+ result=$?
+ if [ $result -eq 0 ]; then
+ output " preparation OK"
+ else
+ output " preparation FAILED"
+ (( CURRENTREVISION++ ))
+ continue
+ fi
+# building
+ output " -> building "
+ CMD="$CMD_BUILD $LOGGING"
+ #echo $CMD
+ eval $CMD
+ result=$?
+ if [ $result -eq 0 ]; then
+ output " building OK"
+ else
+ output " building FAILED"
+ (( CURRENTREVISION++ ))
+ continue
+ fi
+# testing
+ output " -> testing "
+ CMD="$CMD_TEST $LOGGING"
+ #echo $CMD
+ eval $CMD
+ result=$?
+ testresult=result
+ if [ $result -eq 0 ]; then
+ output " testing OK"
+ else
+ output " testing FAILED"
+ output 'Revision $CURRENTREVISION FAILED'
+ fi
+ (( CURRENTREVISION++ ))
+done
+
+exit
+
+
diff --git a/contrib/scripts/terminate.py.in b/contrib/scripts/terminate.py.in
new file mode 100644
index 000000000..4a6719f38
--- /dev/null
+++ b/contrib/scripts/terminate.py.in
@@ -0,0 +1,64 @@
+#!@PYTHON@
+# This file is part of GNUnet.
+# (C) 2011 Christian Grothoff (and other contributing authors)
+#
+# GNUnet is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published
+# by the Free Software Foundation; either version 2, or (at your
+# option) any later version.
+#
+# GNUnet is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with GNUnet; see the file COPYING. If not, write to the
+# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+# Boston, MA 02110-1301, USA.
+#
+# Utility module that implements safe process termination for W32.
+# For other platforms it's equivalent to Popen.kill ()
+# Requires pywin32 on W32.
+
+import sys
+import os
+import subprocess
+if os.name == 'nt':
+ import win32api
+ import win32process
+
+class dummyobj (object):
+ pass
+
+def safe_terminate_process_by_pid (pid, code):
+ if os.name == 'nt':
+ p = dummyobj ()
+ p._handle = win32api.OpenProcess (2 | 1024 | 8 | 32 | 16, 0, pid)
+ result = safe_terminate_process (p, code)
+ win32api.CloseHandle (p._handle)
+ return result
+ else:
+ return os.kill (int (pid), SIGKILL)
+
+def safe_terminate_process (proc, code):
+ if os.name == 'nt':
+ cp = win32api.GetCurrentProcess ()
+ result = False
+ dupproc = win32api.DuplicateHandle (cp, proc._handle, cp, 2 | 1024 | 8 | 32 | 16, 0, 0)
+ try:
+ exitcode = win32process.GetExitCodeProcess (dupproc)
+ if exitcode == 0x103:
+ kernel32 = win32api.GetModuleHandle ("kernel32")
+ exitprocess = win32api.GetProcAddress (kernel32, "ExitProcess")
+ th, tid = win32process.CreateRemoteThread (dupproc, None, 0, exitprocess, code, 0)
+ win32api.CloseHandle (th)
+ result = True
+ else:
+ result = True
+ # except failed to get exit code? failed to get module handle?
+ finally:
+ win32api.CloseHandle (dupproc)
+ return result
+ else:
+ return proc.kill ()
diff --git a/contrib/scripts/testbed_cleanup.sh b/contrib/scripts/testbed_cleanup.sh
new file mode 100755
index 000000000..57413fba0
--- /dev/null
+++ b/contrib/scripts/testbed_cleanup.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+###################################################################################
+# Script to clean a previous run of testbed which has crashed. This scripts kills #
+# the peers and cleans the temporary files created for those peers #
+# #
+# Author: Sree Harsha Totakura #
+###################################################################################
+
+for host in `cut -d : -f 1 < infiniband_cluster.hosts | cut -d @ -f 2`
+do
+ echo "ssh --> $host"
+ ssh $host 'pkill -SIGKILL gnunet; rm -rf /tmp/gnunet-pipe*; rm -rf /tmp/testbed*'
+done
diff --git a/contrib/scripts/texinfo-hacks.el b/contrib/scripts/texinfo-hacks.el
new file mode 100644
index 000000000..bfb5c98fa
--- /dev/null
+++ b/contrib/scripts/texinfo-hacks.el
@@ -0,0 +1,18 @@
+;;;; hacks.el --- a few functions to help me work on the manual
+;;;; Jim Blandy <jimb@red-bean.com> --- October 1998
+;;;; -- imported from https://git.savannah.gnu.org/cgit/guile.git/tree/doc/hacks.el
+;;;; This code should be covered by the same license as GNU Guile (GPL3).
+
+(defun jh-exemplify-region (start end)
+ (interactive "r")
+ (save-excursion
+ (save-restriction
+ (narrow-to-region start end)
+
+ ;; Texinfo doesn't handle tabs well.
+ (untabify (point-min) (point-max))
+
+ ;; Quote any characters special to texinfo.
+ (goto-char (point-min))
+ (while (re-search-forward "[{}@]" nil t)
+ (replace-match "@\\&")))))
diff --git a/contrib/scripts/visualize_stats.sh b/contrib/scripts/visualize_stats.sh
new file mode 100755
index 000000000..aaa5e657b
--- /dev/null
+++ b/contrib/scripts/visualize_stats.sh
@@ -0,0 +1,86 @@
+#!/bin/bash
+#
+# This script polls gnunet-stats repeatedly to create statistics plots.
+# Use 'collect' to collect statistics and 'plot' to plot whats been
+# collected. All plots will be written to $STATDIR as separate .png files.
+#
+# WARNING: calling 'collect' will delete all files in $STATDIR.
+#
+# Requires: gnuplot
+#
+# Note: gnuplot syntax has changed across versions. This
+# script perhaps will not produce color images with older gnuplots.
+# The script should work atleast with gnuplot 3.8k patchlevel 1.
+#
+
+SLEEP=120
+GNUNET=$HOME/
+STATDIR=$GNUNET/stats
+IMAGEVIEWER='display'
+TMP=/tmp/.gnuplot_error
+
+##########################################################################
+
+mkdir -p $STATDIR
+
+case "$1" in
+ collect)
+ rm -f $STATDIR/*
+
+ STARTTIME=`date +%s`
+ IFS=":"
+
+ while true; do
+ NOW=`date +%s`
+ RELAT=$[$NOW-$STARTTIME]
+ gnunet-statistics | while read KEY VALUE; do
+ KEY=`echo $KEY | tr / .`
+ # Collect stats of previous round
+ if [ -e "$STATDIR/$KEY.dat" ]; then
+ PREV=`tail --lines=1 "$STATDIR/$KEY.dat" | sed -e "s/.* //g"`
+ else
+ PREV=$VALUE
+ fi
+
+ # Write new stats
+ echo $RELAT $VALUE >>"$STATDIR/$KEY.dat"
+ echo $RELAT $PREV $VALUE >>"$STATDIR/$KEY.diff"
+
+ done
+ sleep $SLEEP
+ done
+ ;;
+ plot)
+ # Plot incremental
+ ls -1 $STATDIR/*.dat | while read FILENAME; do
+ BASENAME=`basename "$FILENAME" | sed -e "s/ *\..*//g"`
+ echo "set terminal png;set output '$FILENAME.png';set title '$BASENAME - incr';plot '$FILENAME' using (\$1/60):(\$2) title '' with lines;" | nice gnuplot 2> $TMP
+ EC=`cat $TMP | grep "empty" | grep "Warning" | wc -l`
+ if test $EC -ge 1
+ then
+ rm "$FILENAME.png"
+ fi
+ done
+
+ # Plot diff
+ ls -1 $STATDIR/*.diff | while read FILENAME; do
+ BASENAME=`basename "$FILENAME" | sed -e "s/ *\..*//g"`
+ echo "set terminal png;set output '$FILENAME.png';set title '$BASENAME - diff';plot '$FILENAME' using (\$1/60):(\$3-\$2) title '' with lines;" | nice gnuplot 2> $TMP
+ EC=`cat $TMP | grep "empty" | grep "Warning" | wc -l`
+ if test $EC -ge 1
+ then
+ rm "$FILENAME.png"
+ fi
+
+ done
+ ;;
+ view)
+ $IMAGEVIEWER $STATDIR/*.png
+ ;;
+ *)
+ echo $"Usage: $0 {collect|plot|view}"
+ exit 1
+
+esac
+
+
diff --git a/contrib/scripts/zonewalk-to-types.sh b/contrib/scripts/zonewalk-to-types.sh
new file mode 100755
index 000000000..c453702e6
--- /dev/null
+++ b/contrib/scripts/zonewalk-to-types.sh
@@ -0,0 +1,35 @@
+#!/bin/sh
+# This script is in the public domain.
+# Converts the output of gnunet-zonewalk (DNS resolutions)
+# into a proper input for gnunet-gns-benchmark.
+
+NUM_CLIENTS=3
+# How many different groups of names should we
+# create? 1/N will be in the 'shared' group.
+
+# FILE ($1) contains results from DNS lookup; strip
+# everything but the hostnames, remove duplicates
+# and then randomize the order.
+cat $1 | awk '{print $1}' | sort | uniq | shuf > $1.tmp
+TOTAL=`cat $1.tmp | wc -l`
+GROUP_SIZE=`expr $TOTAL / \( $NUM_TYPES + 1 \)`
+
+# First group (0) is to be shared among all clients
+for i in `seq 1 $NUM_CLIENTS`
+do
+ cat $1.tmp | head -n $GROUP_SIZE | awk "{print 0 \" \" \$1}" > $1.$i.tmp
+done
+
+# Second group (1) is unique per client
+OFF=0
+for i in `seq 1 $NUM_CLIENTS`
+do
+ END=`expr $OFF + $GROUP_SIZE`
+ cat $1.tmp | head -n $END | tail -n $GROUP_SIZE | awk "{print 1 \" \" \$1}" >> $1.$i.tmp
+# Shuffle again, so we mix the different request categories in terms of
+# when we issue the queries.
+ cat $1.$i.tmp | shuf > $1.$i
+ OFF="$END"
+ rm $1.$i.tmp
+done
+rm $1.tmp