aboutsummaryrefslogtreecommitdiff
path: root/doc
diff options
context:
space:
mode:
authorng0 <contact.ng0@cryptolab.net>2017-08-17 11:19:05 +0000
committerng0 <ng0@infotropique.org>2017-08-17 11:23:46 +0000
commitd587ba0915db6854959c4dfc2730b34d9577a395 (patch)
treefe8002fe2ba07bba65f50e1bdb12ea4f69f02eeb /doc
parent6bcc73a1cbb1d4a609884762eab1b6de761ad1d9 (diff)
downloadgnunet-d587ba0915db6854959c4dfc2730b34d9577a395.tar.gz
gnunet-d587ba0915db6854959c4dfc2730b34d9577a395.zip
doc: Merge 'gnunet-texinfo' repository into 'doc' folder of gnunet.
* doc/chapters: New directory, the chapters of doc/gnunet.texi. * doc/images: New directory with images for the documentation. * doc/Makefile: Build the documentation. * doc/.gitignore: ignore build artifacts. * doc/gnunet.texi, doc/fdl-1.3.texi, doc/gpl-3.0.texi: New files. Signed-off-by: ng0 <ng0@infotropique.org>
Diffstat (limited to 'doc')
-rw-r--r--doc/.gitignore11
-rw-r--r--doc/Makefile.am45
-rw-r--r--doc/chapters/developer.texi7486
-rw-r--r--doc/chapters/installation.texi4205
-rw-r--r--doc/chapters/philosophy.texi335
-rw-r--r--doc/chapters/user.texi1788
-rw-r--r--doc/fdl-1.3.texi505
-rw-r--r--doc/gnunet.texi177
-rw-r--r--doc/gpl-3.0.texi717
-rw-r--r--doc/images/daemon_lego_block.pngbin0 -> 7636 bytes
-rw-r--r--doc/images/gnunet-0-10-peerinfo.pngbin0 -> 80127 bytes
-rw-r--r--doc/images/gnunet-fs-gtk-0-10-star-tab.pngbin0 -> 63464 bytes
-rw-r--r--doc/images/gnunet-gtk-0-10-download-area.pngbin0 -> 7634 bytes
-rw-r--r--doc/images/gnunet-gtk-0-10-fs-menu.pngbin0 -> 8614 bytes
-rw-r--r--doc/images/gnunet-gtk-0-10-fs-publish-editing.pngbin0 -> 55507 bytes
-rw-r--r--doc/images/gnunet-gtk-0-10-fs-publish-select.pngbin0 -> 43448 bytes
-rw-r--r--doc/images/gnunet-gtk-0-10-fs-publish-with-file.pngbin0 -> 27371 bytes
-rw-r--r--doc/images/gnunet-gtk-0-10-fs-publish-with-file_0.pngbin0 -> 27371 bytes
-rw-r--r--doc/images/gnunet-gtk-0-10-fs-publish.pngbin0 -> 26496 bytes
-rw-r--r--doc/images/gnunet-gtk-0-10-fs-published.pngbin0 -> 59635 bytes
-rw-r--r--doc/images/gnunet-gtk-0-10-fs-search.pngbin0 -> 72151 bytes
-rw-r--r--doc/images/gnunet-gtk-0-10-fs.pngbin0 -> 55706 bytes
-rw-r--r--doc/images/gnunet-gtk-0-10-gns-a-done.pngbin0 -> 30880 bytes
-rw-r--r--doc/images/gnunet-gtk-0-10-gns-a.pngbin0 -> 29895 bytes
-rw-r--r--doc/images/gnunet-gtk-0-10-gns.pngbin0 -> 63783 bytes
-rw-r--r--doc/images/gnunet-gtk-0-10-identity.pngbin0 -> 62404 bytes
-rw-r--r--doc/images/gnunet-gtk-0-10-search-selected.pngbin0 -> 104599 bytes
-rw-r--r--doc/images/gnunet-gtk-0-10-traffic.pngbin0 -> 68515 bytes
-rw-r--r--doc/images/gnunet-gtk-0-10.pngbin0 -> 72897 bytes
-rw-r--r--doc/images/gnunet-namestore-gtk-phone.pngbin0 -> 32631 bytes
-rw-r--r--doc/images/gnunet-namestore-gtk-vpn.pngbin0 -> 35836 bytes
-rw-r--r--doc/images/gnunet-setup-exit.pngbin0 -> 30062 bytes
-rw-r--r--doc/images/iceweasel-preferences.pngbin0 -> 57047 bytes
-rw-r--r--doc/images/iceweasel-proxy.pngbin0 -> 38773 bytes
-rw-r--r--doc/images/service_lego_block.pngbin0 -> 15157 bytes
-rw-r--r--doc/images/service_stack.pngbin0 -> 18862 bytes
36 files changed, 15269 insertions, 0 deletions
diff --git a/doc/.gitignore b/doc/.gitignore
index 7b5069d48..daa5ccaa0 100644
--- a/doc/.gitignore
+++ b/doc/.gitignore
@@ -1,2 +1,13 @@
1gnunet-c-tutorial.aux 1gnunet-c-tutorial.aux
2gnunet-c-tutorial.out 2gnunet-c-tutorial.out
3*.log
4*.aux
5*.pdf
6*.toc
7*.cp
8*.cps
9*.html
10*~
11*.info
12\#*\#
13version.texi
diff --git a/doc/Makefile.am b/doc/Makefile.am
index d05ea4855..4c61f2e80 100644
--- a/doc/Makefile.am
+++ b/doc/Makefile.am
@@ -2,3 +2,48 @@
2SUBDIRS = man doxygen 2SUBDIRS = man doxygen
3 3
4docdir = $(datadir)/doc/gnunet/ 4docdir = $(datadir)/doc/gnunet/
5
6gnunet_doc_images = \
7 images/gnunet-gtk-0-10-gns-a-done.png \
8 images/gnunet-gtk-0-10-gns-a.png \
9 images/daemon_lego_block.png \
10 images/gnunet-gtk-0-10-gns.png \
11 images/gnunet-0-10-peerinfo.png \
12 images/gnunet-gtk-0-10-identity.png \
13 images/gnunet-fs-gtk-0-10-star-tab.png \
14 images/gnunet-gtk-0-10.png \
15 images/gnunet-gtk-0-10-download-area.png \
16 images/gnunet-gtk-0-10-search-selected.png \
17 images/gnunet-gtk-0-10-fs-menu.png \
18 images/gnunet-gtk-0-10-traffic.png \
19 images/gnunet-gtk-0-10-fs.png \
20 images/gnunet-namestore-gtk-phone.png \
21 images/gnunet-gtk-0-10-fs-publish-editing.png \
22 images/gnunet-namestore-gtk-vpn.png \
23 images/gnunet-gtk-0-10-fs-published.png \
24 images/gnunet-setup-exit.png \
25 images/gnunet-gtk-0-10-fs-publish.png \
26 images/iceweasel-preferences.png \
27 images/gnunet-gtk-0-10-fs-publish-select.png \
28 images/iceweasel-proxy.png \
29 images/gnunet-gtk-0-10-fs-publish-with-file_0.png \
30 images/service_lego_block.png \
31 images/gnunet-gtk-0-10-fs-publish-with-file.png \
32 images/service_stack.png \
33 images/gnunet-gtk-0-10-fs-search.png
34
35info_TEXINFOS = \
36 gnunet.texi \
37 gnunet-c-tutorial.tex
38
39gnunet_TEXINFOS = \
40 chapters/developer.texi \
41 chapters/installation.texi \
42 chapters/philosophy.texi \
43 chapters/user.texi \
44 fdl-1.3.texi \
45 gpl-3.0.texi
46
47EXTRA_DIST += \
48 $(gnunet_TEXINFOS) \
49 $(gnunet_doc_images)
diff --git a/doc/chapters/developer.texi b/doc/chapters/developer.texi
new file mode 100644
index 000000000..ce6b16087
--- /dev/null
+++ b/doc/chapters/developer.texi
@@ -0,0 +1,7486 @@
1@c ***************************************************************************
2@node GNUnet Developer Handbook
3@chapter GNUnet Developer Handbook
4
5This book is intended to be an introduction for programmers that want to
6extend the GNUnet framework. GNUnet is more than a simple peer-to-peer
7application. For developers, GNUnet is:
8
9@itemize @bullet
10@item Free software under the GNU General Public License, with a community
11that believes in the GNU philosophy
12@item
13A set of standards, including coding conventions and architectural rules
14@item
15A set of layered protocols, both specifying the communication between peers as
16well as the communication between components of a single peer.
17@item
18A set of libraries with well-defined APIs suitable for writing extensions
19@end itemize
20
21In particular, the architecture specifies that a peer consists of many
22processes communicating via protocols. Processes can be written in almost
23any language. C and Java APIs exist for accessing existing services and for
24writing extensions. It is possible to write extensions in other languages by
25implementing the necessary IPC protocols.
26
27GNUnet can be extended and improved along many possible dimensions, and anyone
28interested in free software and freedom-enhancing networking is welcome to
29join the effort. This developer handbook attempts to provide an initial
30introduction to some of the key design choices and central components of the
31system. This manual is far from complete, and we welcome informed
32contributions, be it in the form of new chapters or insightful comments.
33
34However, the website is experiencing a constant onslaught of sophisticated
35link-spam entered manually by exploited workers solving puzzles and
36customizing text. To limit this commercial defacement, we are strictly
37moderating comments and have disallowed "normal" users from posting new
38content. However, this is really only intended to keep the spam at bay. If
39you are a real user or aspiring developer, please drop us a note (IRC, e-mail,
40contact form) with your user profile ID number included. We will then relax
41these restrictions on your account. We're sorry for this inconvenience;
42however, few people would want to read this site if 99% of it was
43advertisements for bogus websites.
44
45
46
47@c ***************************************************************************
48
49
50
51
52
53
54
55
56@menu
57* Developer Introduction::
58* Code overview::
59* System Architecture::
60* Subsystem stability::
61* Naming conventions and coding style guide::
62* Build-system::
63* Developing extensions for GNUnet using the gnunet-ext template::
64* Writing testcases::
65* GNUnet's TESTING library::
66* Performance regression analysis with Gauger::
67* GNUnet's TESTBED Subsystem::
68* libgnunetutil::
69* The Automatic Restart Manager (ARM)::
70* GNUnet's TRANSPORT Subsystem::
71* NAT library::
72* Distance-Vector plugin::
73* SMTP plugin::
74* Bluetooth plugin::
75* WLAN plugin::
76* The ATS Subsystem::
77* GNUnet's CORE Subsystem::
78* GNUnet's CADET subsystem::
79* GNUnet's NSE subsystem::
80* GNUnet's HOSTLIST subsystem::
81* GNUnet's IDENTITY subsystem::
82* GNUnet's NAMESTORE Subsystem::
83* GNUnet's PEERINFO subsystem::
84* GNUnet's PEERSTORE subsystem::
85* GNUnet's SET Subsystem::
86* GNUnet's STATISTICS subsystem::
87* GNUnet's Distributed Hash Table (DHT)::
88* The GNU Name System (GNS)::
89* The GNS Namecache::
90* The REVOCATION Subsystem::
91* GNUnet's File-sharing (FS) Subsystem::
92* GNUnet's REGEX Subsystem::
93@end menu
94
95@node Developer Introduction
96@section Developer Introduction
97
98This developer handbook is intended as first introduction to GNUnet for new
99developers that want to extend the GNUnet framework. After the introduction,
100each of the GNUnet subsystems (directories in the src/ tree) is (supposed to
101be) covered in its own chapter. In addition to this documentation, GNUnet
102developers should be aware of the services available on the GNUnet server to
103them.
104
105New developers can have a look a the GNUnet tutorials for C and java available
106in the src/ directory of the repository or under the following links:
107
108@itemize @bullet
109@item GNUnet C tutorial
110@item GNUnet Java tutorial
111@end itemize
112
113In addition to this book, the GNUnet server contains various resources for
114GNUnet developers. They are all conveniently reachable via the "Developer"
115entry in the navigation menu. Some additional tools (such as static analysis
116reports) require a special developer access to perform certain operations. If
117you feel you need access, you should contact
118@uref{http://grothoff.org/christian/, Christian Grothoff}, GNUnet's maintainer.
119
120The public subsystems on the GNUnet server that help developers are:
121
122@itemize @bullet
123@item The Version control system keeps our code and enables distributed
124development. Only developers with write access can commit code, everyone else
125is encouraged to submit patches to the
126@uref{http://mail.gnu.org/mailman/listinfo/gnunet-developers, developer
127mailinglist}.
128@item The GNUnet bugtracking system is used to track feature requests, open bug
129reports and their resolutions. Anyone can report bugs, only developers can
130claim to have fixed them.
131@item A buildbot is used to check GNUnet builds automatically on a range of
132platforms. Builds are triggered automatically after 30 minutes of no changes to
133Git.
134@item The current quality of our automated test suite is assessed using Code
135coverage analysis. This analysis is run daily; however the webpage is only
136updated if all automated tests pass at that time. Testcases that improve our
137code coverage are always welcome.
138@item We try to automatically find bugs using a static analysis scan. This scan
139is run daily; however the webpage is only updated if all automated tests pass
140at the time. Note that not everything that is flagged by the analysis is a bug,
141sometimes even good code can be marked as possibly problematic. Nevertheless,
142developers are encouraged to at least be aware of all issues in their code that
143are listed.
144@item We use Gauger for automatic performance regression visualization. Details
145on how to use Gauger are here.
146@item We use @uref{http://junit.org/, junit} to automatically test gnunet-java.
147Automatically generated, current reports on the test suite are here.
148@item We use Cobertura to generate test coverage reports for gnunet-java.
149Current reports on test coverage are here.
150@end itemize
151
152
153
154@c ***************************************************************************
155@menu
156* Project overview::
157@end menu
158
159@node Project overview
160@subsection Project overview
161
162The GNUnet project consists at this point of several sub-projects. This section
163is supposed to give an initial overview about the various sub-projects. Note
164that this description also lists projects that are far from complete, including
165even those that have literally not a single line of code in them yet.
166
167GNUnet sub-projects in order of likely relevance are currently:
168
169@table @asis
170
171@item svn/gnunet Core of the P2P framework, including file-sharing, VPN and
172chat applications; this is what the developer handbook covers mostly
173@item svn/gnunet-gtk/ Gtk+-based user interfaces, including gnunet-fs-gtk
174(file-sharing), gnunet-statistics-gtk (statistics over time),
175gnunet-peerinfo-gtk (information about current connections and known peers),
176gnunet-chat-gtk (chat GUI) and gnunet-setup (setup tool for "everything")
177@item svn/gnunet-fuse/ Mounting directories shared via GNUnet's file-sharing on Linux
178@item svn/gnunet-update/ Installation and update tool
179@item svn/gnunet-ext/
180Template for starting 'external' GNUnet projects
181@item svn/gnunet-java/ Java
182APIs for writing GNUnet services and applications
183@item svn/gnunet-www/ Code
184and media helping drive the GNUnet website
185@item svn/eclectic/ Code to run
186GNUnet nodes on testbeds for research, development, testing and evaluation
187@item svn/gnunet-qt/ qt-based GNUnet GUI (dead?)
188@item svn/gnunet-cocoa/
189cocoa-based GNUnet GUI (dead?)
190
191@end table
192
193We are also working on various supporting libraries and tools:
194
195@table @asis
196@item svn/Extractor/ GNU libextractor (meta data extraction)
197@item svn/libmicrohttpd/ GNU libmicrohttpd (embedded HTTP(S) server library)
198@item svn/gauger/ Tool for performance regression analysis
199@item svn/monkey/ Tool for automated debugging of distributed systems
200@item svn/libmwmodem/ Library for accessing satellite connection quality reports
201@end table
202
203Finally, there are various external projects (see links for a list of those
204that have a public website) which build on top of the GNUnet framework.
205
206@c ***************************************************************************
207@node Code overview
208@section Code overview
209
210This section gives a brief overview of the GNUnet source code. Specifically, we
211sketch the function of each of the subdirectories in the @code{gnunet/src/}
212directory. The order given is roughly bottom-up (in terms of the layers of the
213system).
214@table @asis
215
216@item util/ --- libgnunetutil Library with general utility functions, all
217GNUnet binaries link against this library. Anything from memory allocation and
218data structures to cryptography and inter-process communication. The goal is to
219provide an OS-independent interface and more 'secure' or convenient
220implementations of commonly used primitives. The API is spread over more than a
221dozen headers, developers should study those closely to avoid duplicating
222existing functions.
223@item hello/ --- libgnunethello HELLO messages are used to
224describe under which addresses a peer can be reached (for example, protocol,
225IP, port). This library manages parsing and generating of HELLO messages.
226@item block/ --- libgnunetblock The DHT and other components of GNUnet store
227information in units called 'blocks'. Each block has a type and the type
228defines a particular format and how that binary format is to be linked to a
229hash code (the key for the DHT and for databases). The block library is a
230wapper around block plugins which provide the necessary functions for each
231block type.
232@item statistics/ The statistics service enables associating
233values (of type uint64_t) with a componenet name and a string. The main uses is
234debugging (counting events), performance tracking and user entertainment (what
235did my peer do today?).
236@item arm/ The automatic-restart-manager (ARM) service
237is the GNUnet master service. Its role is to start gnunet-services, to re-start
238them when they crashed and finally to shut down the system when requested.
239@item peerinfo/ The peerinfo service keeps track of which peers are known to
240the local peer and also tracks the validated addresses for each peer (in the
241form of a HELLO message) for each of those peers. The peer is not necessarily
242connected to all peers known to the peerinfo service. Peerinfo provides
243persistent storage for peer identities --- peers are not forgotten just because
244of a system restart.
245@item datacache/ --- libgnunetdatacache The datacache
246library provides (temporary) block storage for the DHT. Existing plugins can
247store blocks in Sqlite, Postgres or MySQL databases. All data stored in the
248cache is lost when the peer is stopped or restarted (datacache uses temporary
249tables).
250@item datastore/ The datastore service stores file-sharing blocks in
251databases for extended periods of time. In contrast to the datacache, data is
252not lost when peers restart. However, quota restrictions may still cause old,
253expired or low-priority data to be eventually discarded. Existing plugins can
254store blocks in Sqlite, Postgres or MySQL databases.
255@item template/ Template
256for writing a new service. Does nothing.
257@item ats/ The automatic transport
258selection (ATS) service is responsible for deciding which address (i.e. which
259transport plugin) should be used for communication with other peers, and at
260what bandwidth.
261@item nat/ --- libgnunetnat Library that provides basic
262functions for NAT traversal. The library supports NAT traversal with manual
263hole-punching by the user, UPnP and ICMP-based autonomous NAT traversal. The
264library also includes an API for testing if the current configuration works and
265the @code{gnunet-nat-server} which provides an external service to test the
266local configuration.
267@item fragmentation/ --- libgnunetfragmentation Some
268transports (UDP and WLAN, mostly) have restrictions on the maximum transfer
269unit (MTU) for packets. The fragmentation library can be used to break larger
270packets into chunks of at most 1k and transmit the resulting fragments
271reliabily (with acknowledgement, retransmission, timeouts, etc.).
272@item transport/ The transport service is responsible for managing the basic P2P
273communication. It uses plugins to support P2P communication over TCP, UDP,
274HTTP, HTTPS and other protocols.The transport service validates peer addresses,
275enforces bandwidth restrictions, limits the total number of connections and
276enforces connectivity restrictions (i.e. friends-only).
277@item peerinfo-tool/
278This directory contains the gnunet-peerinfo binary which can be used to inspect
279the peers and HELLOs known to the peerinfo service.
280@item core/ The core
281service is responsible for establishing encrypted, authenticated connections
282with other peers, encrypting and decrypting messages and forwarding messages to
283higher-level services that are interested in them.
284@item testing/ ---
285libgnunettesting The testing library allows starting (and stopping) peers for
286writing testcases.@
287It also supports automatic generation of configurations for
288peers ensuring that the ports and paths are disjoint. libgnunettesting is also
289the foundation for the testbed service
290@item testbed/ The testbed service is
291used for creating small or large scale deployments of GNUnet peers for
292evaluation of protocols. It facilitates peer depolyments on multiple hosts (for
293example, in a cluster) and establishing varous network topologies (both
294underlay and overlay).
295@item nse/ The network size estimation (NSE) service
296implements a protocol for (securely) estimating the current size of the P2P
297network.
298@item dht/ The distributed hash table (DHT) service provides a
299distributed implementation of a hash table to store blocks under hash keys in
300the P2P network.
301@item hostlist/ The hostlist service allows learning about
302other peers in the network by downloading HELLO messages from an HTTP server,
303can be configured to run such an HTTP server and also implements a P2P protocol
304to advertise and automatically learn about other peers that offer a public
305hostlist server.
306@item topology/ The topology service is responsible for
307maintaining the mesh topology. It tries to maintain connections to friends
308(depending on the configuration) and also tries to ensure that the peer has a
309decent number of active connections at all times. If necessary, new connections
310are added. All peers should run the topology service, otherwise they may end up
311not being connected to any other peer (unless some other service ensures that
312core establishes the required connections). The topology service also tells the
313transport service which connections are permitted (for friend-to-friend
314networking)
315@item fs/ The file-sharing (FS) service implements GNUnet's
316file-sharing application. Both anonymous file-sharing (using gap) and
317non-anonymous file-sharing (using dht) are supported.
318@item cadet/ The CADET
319service provides a general-purpose routing abstraction to create end-to-end
320encrypted tunnels in mesh networks. We wrote a paper documenting key aspects of
321the design.
322@item tun/ --- libgnunettun Library for building IPv4, IPv6
323packets and creating checksums for UDP, TCP and ICMP packets. The header
324defines C structs for common Internet packet formats and in particular structs
325for interacting with TUN (virtual network) interfaces.
326@item mysql/ ---
327libgnunetmysql Library for creating and executing prepared MySQL statements and
328to manage the connection to the MySQL database. Essentially a lightweight
329wrapper for the interaction between GNUnet components and libmysqlclient.
330@item dns/ Service that allows intercepting and modifying DNS requests of the
331local machine. Currently used for IPv4-IPv6 protocol translation (DNS-ALG) as
332implemented by "pt/" and for the GNUnet naming system. The service can also be
333configured to offer an exit service for DNS traffic.
334@item vpn/ The virtual
335public network (VPN) service provides a virtual tunnel interface (VTUN) for IP
336routing over GNUnet. Needs some other peers to run an "exit" service to work.
337Can be activated using the "gnunet-vpn" tool or integrated with DNS using the
338"pt" daemon.
339@item exit/ Daemon to allow traffic from the VPN to exit this
340peer to the Internet or to specific IP-based services of the local peer.
341Currently, an exit service can only be restricted to IPv4 or IPv6, not to
342specific ports and or IP address ranges. If this is not acceptable, additional
343firewall rules must be added manually. exit currently only works for normal
344UDP, TCP and ICMP traffic; DNS queries need to leave the system via a DNS
345service.
346@item pt/ protocol translation daemon. This daemon enables 4-to-6,
3476-to-4, 4-over-6 or 6-over-4 transitions for the local system. It essentially
348uses "DNS" to intercept DNS replies and then maps results to those offered by
349the VPN, which then sends them using mesh to some daemon offering an
350appropriate exit service.
351@item identity/ Management of egos (alter egos) of a
352user; identities are essentially named ECC private keys and used for zones in
353the GNU name system and for namespaces in file-sharing, but might find other
354uses later
355@item revocation/ Key revocation service, can be used to revoke the
356private key of an identity if it has been compromised
357@item namecache/ Cache
358for resolution results for the GNU name system; data is encrypted and can be
359shared among users, loss of the data should ideally only result in a
360performance degradation (persistence not required)
361@item namestore/ Database
362for the GNU name system with per-user private information, persistence required
363@item gns/ GNU name system, a GNU approach to DNS and PKI.
364@item dv/ A plugin
365for distance-vector (DV)-based routing. DV consists of a service and a
366transport plugin to provide peers with the illusion of a direct P2P connection
367for connections that use multiple (typically up to 3) hops in the actual
368underlay network.
369@item regex/ Service for the (distributed) evaluation of
370regular expressions.
371@item scalarproduct/ The scalar product service offers an
372API to perform a secure multiparty computation which calculates a scalar
373product between two peers without exposing the private input vectors of the
374peers to each other.
375@item consensus/ The consensus service will allow a set
376of peers to agree on a set of values via a distributed set union computation.
377@item rest/ The rest API allows access to GNUnet services using RESTful
378interaction. The services provide plugins that can exposed by the rest server.
379@item experimentation/ The experimentation daemon coordinates distributed
380experimentation to evaluate transport and ats properties
381@end table
382
383@c ***************************************************************************
384@node System Architecture
385@section System Architecture
386
387GNUnet developers like legos. The blocks are indestructible, can be stacked
388together to construct complex buildings and it is generally easy to swap one
389block for a different one that has the same shape. GNUnet's architecture is
390based on legos:
391
392
393
394This chapter documents the GNUnet lego system, also known as GNUnet's system
395architecture.
396
397The most common GNUnet component is a service. Services offer an API (or
398several, depending on what you count as "an API") which is implemented as a
399library. The library communicates with the main process of the service using a
400service-specific network protocol. The main process of the service typically
401doesn't fully provide everything that is needed --- it has holes to be filled
402by APIs to other services.
403
404A special kind of component in GNUnet are user interfaces and daemons. Like
405services, they have holes to be filled by APIs of other services. Unlike
406services, daemons do not implement their own network protocol and they have no
407API:
408
409The GNUnet system provides a range of services, daemons and user interfaces,
410which are then combined into a layered GNUnet instance (also known as a peer).
411
412Note that while it is generally possible to swap one service for another
413compatible service, there is often only one implementation. However, during
414development we often have a "new" version of a service in parallel with an
415"old" version. While the "new" version is not working, developers working on
416other parts of the service can continue their development by simply using the
417"old" service. Alternative design ideas can also be easily investigated by
418swapping out individual components. This is typically achieved by simply
419changing the name of the "BINARY" in the respective configuration section.
420
421Key properties of GNUnet services are that they must be separate processes and
422that they must protect themselves by applying tight error checking against the
423network protocol they implement (thereby achieving a certain degree of
424robustness).
425
426On the other hand, the APIs are implemented to tolerate failures of the
427service, isolating their host process from errors by the service. If the
428service process crashes, other services and daemons around it should not also
429fail, but instead wait for the service process to be restarted by ARM.
430
431
432@c ***************************************************************************
433@node Subsystem stability
434@section Subsystem stability
435
436This page documents the current stability of the various GNUnet subsystems.
437Stability here describes the expected degree of compatibility with future
438versions of GNUnet. For each subsystem we distinguish between compatibility on
439the P2P network level (communication protocol between peers), the IPC level
440(communication between the service and the service library) and the API level
441(stability of the API). P2P compatibility is relevant in terms of which
442applications are likely going to be able to communicate with future versions of
443the network. IPC communication is relevant for the implementation of language
444bindings that re-implement the IPC messages. Finally, API compatibility is
445relevant to developers that hope to be able to avoid changes to applications
446build on top of the APIs of the framework.
447
448The following table summarizes our current view of the stability of the
449respective protocols or APIs:
450
451@multitable @columnfractions .20 .20 .20 .20
452@headitem Subsystem @tab P2P @tab IPC @tab C API
453@item util @tab n/a @tab n/a @tab stable
454@item arm @tab n/a @tab stable @tab stable
455@item ats @tab n/a @tab unstable @tab testing
456@item block @tab n/a @tab n/a @tab stable
457@item cadet @tab testing @tab testing @tab testing
458@item consensus @tab experimental @tab experimental @tab experimental
459@item core @tab stable @tab stable @tab stable
460@item datacache @tab n/a @tab n/a @tab stable
461@item datastore @tab n/a @tab stable @tab stable
462@item dht @tab stable @tab stable @tab stable
463@item dns @tab stable @tab stable @tab stable
464@item dv @tab testing @tab testing @tab n/a
465@item exit @tab testing @tab n/a @tab n/a
466@item fragmentation @tab stable @tab n/a @tab stable
467@item fs @tab stable @tab stable @tab stable
468@item gns @tab stable @tab stable @tab stable
469@item hello @tab n/a @tab n/a @tab testing
470@item hostlist @tab stable @tab stable @tab n/a
471@item identity @tab stable @tab stable @tab n/a
472@item multicast @tab experimental @tab experimental @tab experimental
473@item mysql @tab stable @tab n/a @tab stable
474@item namestore @tab n/a @tab stable @tab stable
475@item nat @tab n/a @tab n/a @tab stable
476@item nse @tab stable @tab stable @tab stable
477@item peerinfo @tab n/a @tab stable @tab stable
478@item psyc @tab experimental @tab experimental @tab experimental
479@item pt @tab n/a @tab n/a @tab n/a
480@item regex @tab stable @tab stable @tab stable
481@item revocation @tab stable @tab stable @tab stable
482@item social @tab experimental @tab experimental @tab experimental
483@item statistics @tab n/a @tab stable @tab stable
484@item testbed @tab n/a @tab testing @tab testing
485@item testing @tab n/a @tab n/a @tab testing
486@item topology @tab n/a @tab n/a @tab n/a
487@item transport @tab stable @tab stable @tab stable
488@item tun @tab n/a @tab n/a @tab stable
489@item vpn @tab testing @tab n/a @tab n/a
490@end multitable
491
492Here is a rough explanation of the values:
493
494@table @samp
495@item stable
496No incompatible changes are planned at this time; for IPC/APIs, if
497there are incompatible changes, they will be minor and might only require
498minimal changes to existing code; for P2P, changes will be avoided if at all
499possible for the 0.10.x-series
500
501@item testing
502No incompatible changes are
503planned at this time, but the code is still known to be in flux; so while we
504have no concrete plans, our expectation is that there will still be minor
505modifications; for P2P, changes will likely be extensions that should not break
506existing code
507
508@item unstable
509Changes are planned and will happen; however, they
510will not be totally radical and the result should still resemble what is there
511now; nevertheless, anticipated changes will break protocol/API compatibility
512
513@item experimental
514Changes are planned and the result may look nothing like
515what the API/protocol looks like today
516
517@item unknown
518Someone should think about where this subsystem headed
519
520@item n/a
521This subsystem does not have an API/IPC-protocol/P2P-protocol
522@end table
523
524@c ***************************************************************************
525@node Naming conventions and coding style guide
526@section Naming conventions and coding style guide
527
528Here you can find some rules to help you write code for GNUnet.
529
530
531
532@c ***************************************************************************
533@menu
534* Naming conventions::
535* Coding style::
536@end menu
537
538@node Naming conventions
539@subsection Naming conventions
540
541
542@c ***************************************************************************
543@menu
544* include files::
545* binaries::
546* logging::
547* configuration::
548* exported symbols::
549* private (library-internal) symbols (including structs and macros)::
550* testcases::
551* performance tests::
552* src/ directories::
553@end menu
554
555@node include files
556@subsubsection include files
557
558@itemize @bullet
559@item _lib: library without need for a process
560@item _service: library that needs a service process
561@item _plugin: plugin definition
562@item _protocol: structs used in network protocol
563@item exceptions:
564@itemize @bullet
565@item gnunet_config.h --- generated
566@item platform.h --- first included
567@item plibc.h --- external library
568@item gnunet_common.h --- fundamental routines
569@item gnunet_directories.h --- generated
570@item gettext.h --- external library
571@end itemize
572@end itemize
573
574@c ***************************************************************************
575@node binaries
576@subsubsection binaries
577
578@itemize @bullet
579@item gnunet-service-xxx: service process (has listen socket)
580@item gnunet-daemon-xxx: daemon process (no listen socket)
581@item gnunet-helper-xxx[-yyy]: SUID helper for module xxx
582@item gnunet-yyy: command-line tool for end-users
583@item libgnunet_plugin_xxx_yyy.so: plugin for API xxx
584@item libgnunetxxx.so: library for API xxx
585@end itemize
586
587@c ***************************************************************************
588@node logging
589@subsubsection logging
590
591@itemize @bullet
592@item services and daemons use their directory name in GNUNET_log_setup (i.e.
593'core') and log using plain 'GNUNET_log'.
594@item command-line tools use their full name in GNUNET_log_setup (i.e.
595'gnunet-publish') and log using plain 'GNUNET_log'.
596@item service access libraries log using 'GNUNET_log_from' and use
597'DIRNAME-api' for the component (i.e. 'core-api')
598@item pure libraries (without associated service) use 'GNUNET_log_from' with
599the component set to their library name (without lib or '.so'), which should
600also be their directory name (i.e. 'nat')
601@item plugins should use 'GNUNET_log_from' with the directory name and the
602plugin name combined to produce the component name (i.e. 'transport-tcp').
603@item logging should be unified per-file by defining a LOG macro with the
604appropriate arguments, along these lines:@ #define LOG(kind,...)
605GNUNET_log_from (kind, "example-api",__VA_ARGS__)
606@end itemize
607
608@c ***************************************************************************
609@node configuration
610@subsubsection configuration
611
612@itemize @bullet
613@item paths (that are substituted in all filenames) are in PATHS (have as few
614as possible)
615@item all options for a particular module (src/MODULE) are under [MODULE]
616@item options for a plugin of a module are under [MODULE-PLUGINNAME]
617@end itemize
618
619@c ***************************************************************************
620@node exported symbols
621@subsubsection exported symbols
622
623@itemize @bullet
624@item must start with "GNUNET_modulename_" and be defined in "modulename.c"
625@item exceptions: those defined in gnunet_common.h
626@end itemize
627
628@c ***************************************************************************
629@node private (library-internal) symbols (including structs and macros)
630@subsubsection private (library-internal) symbols (including structs and macros)
631
632@itemize @bullet
633@item must NOT start with any prefix
634@item must not be exported in a way that linkers could use them or@ other
635libraries might see them via headers; they must be either@ declared/defined in
636C source files or in headers that are in@ the respective directory under
637src/modulename/ and NEVER be@ declared in src/include/.
638@end itemize
639
640@node testcases
641@subsubsection testcases
642
643@itemize @bullet
644@item must be called "test_module-under-test_case-description.c"
645@item "case-description" maybe omitted if there is only one test
646@end itemize
647
648@c ***************************************************************************
649@node performance tests
650@subsubsection performance tests
651
652@itemize @bullet
653@item must be called "perf_module-under-test_case-description.c"
654@item "case-description" maybe omitted if there is only one performance test
655@item Must only be run if HAVE_BENCHMARKS is satisfied
656@end itemize
657
658@c ***************************************************************************
659@node src/ directories
660@subsubsection src/ directories
661
662@itemize @bullet
663@item gnunet-NAME: end-user applications (i.e., gnunet-search, gnunet-arm)
664@item gnunet-service-NAME: service processes with accessor library (i.e.,
665gnunet-service-arm)
666@item libgnunetNAME: accessor library (_service.h-header) or standalone library
667(_lib.h-header)
668@item gnunet-daemon-NAME: daemon process without accessor library (i.e.,
669gnunet-daemon-hostlist) and no GNUnet management port
670@item libgnunet_plugin_DIR_NAME: loadable plugins (i.e.,
671libgnunet_plugin_transport_tcp)
672@end itemize
673
674@c ***************************************************************************
675@node Coding style
676@subsection Coding style
677
678@itemize @bullet
679@item GNU guidelines generally apply
680@item Indentation is done with spaces, two per level, no tabs
681@item C99 struct initialization is fine
682@item declare only one variable per line, so@
683
684@example
685int i; int j;
686@end example
687
688instead of
689
690@example
691int i,j;
692@end example
693
694This helps keep diffs small and forces developers to think precisely about the
695type of every variable. Note that @code{char *} is different from @code{const
696char*} and @code{int} is different from @code{unsigned int} or @code{uint32_t}.
697Each variable type should be chosen with care.
698
699@item While @code{goto} should generally be avoided, having a @code{goto} to
700the end of a function to a block of clean up statements (free, close, etc.) can
701be acceptable.
702
703@item Conditions should be written with constants on the left (to avoid
704accidental assignment) and with the 'true' target being either the 'error' case
705or the significantly simpler continuation. For example:@
706
707@example
708if (0 != stat ("filename," &sbuf)) @{ error(); @} else @{
709 /* handle normal case here */
710@}
711@end example
712
713
714instead of
715@example
716if (stat ("filename," &sbuf) == 0) @{
717 /* handle normal case here */
718@} else @{ error(); @}
719@end example
720
721
722If possible, the error clause should be terminated with a 'return' (or 'goto'
723to some cleanup routine) and in this case, the 'else' clause should be omitted:
724@example
725if (0 != stat ("filename," &sbuf)) @{ error(); return; @}
726/* handle normal case here */
727@end example
728
729
730This serves to avoid deep nesting. The 'constants on the left' rule applies to
731all constants (including. @code{GNUNET_SCHEDULER_NO_TASK}), NULL, and enums).
732With the two above rules (constants on left, errors in 'true' branch), there is
733only one way to write most branches correctly.
734
735@item Combined assignments and tests are allowed if they do not hinder code
736clarity. For example, one can write:@
737
738@example
739if (NULL == (value = lookup_function())) @{ error(); return; @}
740@end example
741
742
743@item Use @code{break} and @code{continue} wherever possible to avoid deep(er)
744nesting. Thus, we would write:@
745
746@example
747next = head; while (NULL != (pos = next)) @{ next = pos->next; if (!
748should_free (pos)) continue; GNUNET_CONTAINER_DLL_remove (head, tail, pos);
749GNUNET_free (pos); @}
750@end example
751
752
753instead of
754@example
755next = head; while (NULL != (pos = next)) @{ next =
756pos->next; if (should_free (pos)) @{
757 /* unnecessary nesting! */
758 GNUNET_CONTAINER_DLL_remove (head, tail, pos); GNUNET_free (pos); @} @}
759@end example
760
761
762@item We primarily use @code{for} and @code{while} loops. A @code{while} loop
763is used if the method for advancing in the loop is not a straightforward
764increment operation. In particular, we use:@
765
766@example
767next = head;
768while (NULL != (pos = next))
769@{
770 next = pos->next;
771 if (! should_free (pos))
772 continue;
773 GNUNET_CONTAINER_DLL_remove (head, tail, pos);
774 GNUNET_free (pos);
775@}
776@end example
777
778
779to free entries in a list (as the iteration changes the structure of the list
780due to the free; the equivalent @code{for} loop does no longer follow the
781simple @code{for} paradigm of @code{for(INIT;TEST;INC)}). However, for loops
782that do follow the simple @code{for} paradigm we do use @code{for}, even if it
783involves linked lists:
784@example
785/* simple iteration over a linked list */
786for (pos = head; NULL != pos; pos = pos->next)
787@{
788 use (pos);
789@}
790@end example
791
792
793@item The first argument to all higher-order functions in GNUnet must be
794declared to be of type @code{void *} and is reserved for a closure. We do not
795use inner functions, as trampolines would conflict with setups that use
796non-executable stacks.@ The first statement in a higher-order function, which
797unusually should be part of the variable declarations, should assign the
798@code{cls} argument to the precise expected type. For example:
799@example
800int callback (void *cls, char *args) @{
801 struct Foo *foo = cls; int other_variables;
802
803 /* rest of function */
804@}
805@end example
806
807
808@item It is good practice to write complex @code{if} expressions instead of
809using deeply nested @code{if} statements. However, except for addition and
810multiplication, all operators should use parens. This is fine:@
811
812@example
813if ( (1 == foo) || ((0 == bar) && (x != y)) )
814 return x;
815@end example
816
817
818However, this is not:
819@example
820if (1 == foo)
821 return x;
822if (0 == bar && x != y)
823 return x;
824@end example
825
826
827Note that splitting the @code{if} statement above is debateable as the
828@code{return x} is a very trivial statement. However, once the logic after the
829branch becomes more complicated (and is still identical), the "or" formulation
830should be used for sure.
831
832@item There should be two empty lines between the end of the function and the
833comments describing the following function. There should be a single empty line
834after the initial variable declarations of a function. If a function has no
835local variables, there should be no initial empty line. If a long function
836consists of several complex steps, those steps might be separated by an empty
837line (possibly followed by a comment describing the following step). The code
838should not contain empty lines in arbitrary places; if in doubt, it is likely
839better to NOT have an empty line (this way, more code will fit on the screen).
840@end itemize
841
842@c ***************************************************************************
843@node Build-system
844@section Build-system
845
846If you have code that is likely not to compile or build rules you might want to
847not trigger for most developers, use "if HAVE_EXPERIMENTAL" in your
848Makefile.am. Then it is OK to (temporarily) add non-compiling (or
849known-to-not-port) code.
850
851If you want to compile all testcases but NOT run them, run configure with the@
852@code{--enable-test-suppression} option.
853
854If you want to run all testcases, including those that take a while, run
855configure with the@ @code{--enable-expensive-testcases} option.
856
857If you want to compile and run benchmarks, run configure with the@
858@code{--enable-benchmarks} option.
859
860If you want to obtain code coverage results, run configure with the@
861@code{--enable-coverage} option and run the coverage.sh script in contrib/.
862
863@c ***************************************************************************
864@node Developing extensions for GNUnet using the gnunet-ext template
865@section Developing extensions for GNUnet using the gnunet-ext template
866
867
868For developers who want to write extensions for GNUnet we provide the
869gnunet-ext template to provide an easy to use skeleton.
870
871gnunet-ext contains the build environment and template files for the
872development of GNUnet services, command line tools, APIs and tests.
873
874First of all you have to obtain gnunet-ext from SVN:
875
876@code{svn co https://gnunet.org/svn/gnunet-ext}
877
878The next step is to bootstrap and configure it. For configure you have to
879provide the path containing GNUnet with @code{--with-gnunet=/path/to/gnunet}
880and the prefix where you want the install the extension using
881@code{--prefix=/path/to/install}@ @code{@ ./bootstrap@ ./configure
882--prefix=/path/to/install --with-gnunet=/path/to/gnunet@ }
883
884When your GNUnet installation is not included in the default linker search
885path, you have to add @code{/path/to/gnunet} to the file @code{/etc/ld.so.conf}
886and run @code{ldconfig} or your add it to the environmental variable
887@code{LD_LIBRARY_PATH} by using
888
889@code{export LD_LIBRARY_PATH=/path/to/gnunet/lib}
890
891@c ***************************************************************************
892@node Writing testcases
893@section Writing testcases
894
895Ideally, any non-trivial GNUnet code should be covered by automated testcases.
896Testcases should reside in the same place as the code that is being tested. The
897name of source files implementing tests should begin with "test_" followed by
898the name of the file that contains the code that is being tested.
899
900Testcases in GNUnet should be integrated with the autotools build system. This
901way, developers and anyone building binary packages will be able to run all
902testcases simply by running @code{make check}. The final testcases shipped with
903the distribution should output at most some brief progress information and not
904display debug messages by default. The success or failure of a testcase must be
905indicated by returning zero (success) or non-zero (failure) from the main
906method of the testcase. The integration with the autotools is relatively
907straightforward and only requires modifications to the @code{Makefile.am} in
908the directory containing the testcase. For a testcase testing the code in
909@code{foo.c} the @code{Makefile.am} would contain the following lines:
910@example
911check_PROGRAMS = test_foo TESTS = $(check_PROGRAMS) test_foo_SOURCES =
912test_foo.c test_foo_LDADD = $(top_builddir)/src/util/libgnunetutil.la
913@end example
914
915Naturally, other libraries used by the testcase may be specified in the
916@code{LDADD} directive as necessary.
917
918Often testcases depend on additional input files, such as a configuration file.
919These support files have to be listed using the EXTRA_DIST directive in order
920to ensure that they are included in the distribution. Example:
921@example
922EXTRA_DIST = test_foo_data.conf
923@end example
924
925
926Executing @code{make check} will run all testcases in the current directory and
927all subdirectories. Testcases can be compiled individually by running
928@code{make test_foo} and then invoked directly using @code{./test_foo}. Note
929that due to the use of plugins in GNUnet, it is typically necessary to run
930@code{make install} before running any testcases. Thus the canonical command
931@code{make check install} has to be changed to @code{make install check} for
932GNUnet.
933
934@c ***************************************************************************
935@node GNUnet's TESTING library
936@section GNUnet's TESTING library
937
938The TESTING library is used for writing testcases which involve starting a
939single or multiple peers. While peers can also be started by testcases using
940the ARM subsystem, using TESTING library provides an elegant way to do this.
941The configurations of the peers are auto-generated from a given template to
942have non-conflicting port numbers ensuring that peers' services do not run into
943bind errors. This is achieved by testing ports' availability by binding a
944listening socket to them before allocating them to services in the generated
945configurations.
946
947An another advantage while using TESTING is that it shortens the testcase
948startup time as the hostkeys for peers are copied from a pre-computed set of
949hostkeys instead of generating them at peer startup which may take a
950considerable amount of time when starting multiple peers or on an embedded
951processor.
952
953TESTING also allows for certain services to be shared among peers. This feature
954is invaluable when testing with multiple peers as it helps to reduce the number
955of services run per each peer and hence the total number of processes run per
956testcase.
957
958TESTING library only handles creating, starting and stopping peers. Features
959useful for testcases such as connecting peers in a topology are not available
960in TESTING but are available in the TESTBED subsystem. Furthermore, TESTING
961only creates peers on the localhost, however by using TESTBED testcases can
962benefit from creating peers across multiple hosts.
963
964@menu
965* API::
966* Finer control over peer stop::
967* Helper functions::
968* Testing with multiple processes::
969@end menu
970
971@c ***************************************************************************
972@node API
973@subsection API
974
975TESTING abstracts a group of peers as a TESTING system. All peers in a system
976have common hostname and no two services of these peers have a same port or a
977UNIX domain socket path.
978
979TESTING system can be created with the function
980@code{GNUNET_TESTING_system_create()} which returns a handle to the system.
981This function takes a directory path which is used for generating the
982configurations of peers, an IP address from which connections to the peers'
983services should be allowed, the hostname to be used in peers' configuration,
984and an array of shared service specifications of type @code{struct
985GNUNET_TESTING_SharedService}.
986
987The shared service specification must specify the name of the service to share,
988the configuration pertaining to that shared service and the maximum number of
989peers that are allowed to share a single instance of the shared service.
990
991TESTING system created with @code{GNUNET_TESTING_system_create()} chooses ports
992from the default range 12000 - 56000 while auto-generating configurations for
993peers. This range can be customised with the function
994@code{GNUNET_TESTING_system_create_with_portrange()}. This function is similar
995to @code{GNUNET_TESTING_system_create()} except that it take 2 additional
996parameters --- the start and end of the port range to use.
997
998A TESTING system is destroyed with the funciton
999@code{GNUNET_TESTING_system_destory()}. This function takes the handle of the
1000system and a flag to remove the files created in the directory used to generate
1001configurations.
1002
1003A peer is created with the function @code{GNUNET_TESTING_peer_configure()}.
1004This functions takes the system handle, a configuration template from which the
1005configuration for the peer is auto-generated and the index from where the
1006hostkey for the peer has to be copied from. When successfull, this function
1007returs a handle to the peer which can be used to start and stop it and to
1008obtain the identity of the peer. If unsuccessful, a NULL pointer is returned
1009with an error message. This function handles the generated configuration to
1010have non-conflicting ports and paths.
1011
1012Peers can be started and stopped by calling the functions
1013@code{GNUNET_TESTING_peer_start()} and @code{GNUNET_TESTING_peer_stop()}
1014respectively. A peer can be destroyed by calling the function
1015@code{GNUNET_TESTING_peer_destroy}. When a peer is destroyed, the ports and
1016paths in allocated in its configuration are reclaimed for usage in new
1017peers.
1018
1019@c ***************************************************************************
1020@node Finer control over peer stop
1021@subsection Finer control over peer stop
1022
1023Using @code{GNUNET_TESTING_peer_stop()} is normally fine for testcases.
1024However, calling this function for each peer is inefficient when trying to
1025shutdown multiple peers as this function sends the termination signal to the
1026given peer process and waits for it to terminate. It would be faster in this
1027case to send the termination signals to the peers first and then wait on them.
1028This is accomplished by the functions @code{GNUNET_TESTING_peer_kill()} which
1029sends a termination signal to the peer, and the function
1030@code{GNUNET_TESTING_peer_wait()} which waits on the peer.
1031
1032Further finer control can be achieved by choosing to stop a peer asynchronously
1033with the function @code{GNUNET_TESTING_peer_stop_async()}. This function takes
1034a callback parameter and a closure for it in addition to the handle to the peer
1035to stop. The callback function is called with the given closure when the peer
1036is stopped. Using this function eliminates blocking while waiting for the peer
1037to terminate.
1038
1039An asynchronous peer stop can be cancelled by calling the function
1040@code{GNUNET_TESTING_peer_stop_async_cancel()}. Note that calling this function
1041does not prevent the peer from terminating if the termination signal has
1042already been sent to it. It does, however, cancels the callback to be called
1043when the peer is stopped.
1044
1045@c ***************************************************************************
1046@node Helper functions
1047@subsection Helper functions
1048
1049Most of the testcases can benefit from an abstraction which configures a peer
1050and starts it. This is provided by the function
1051@code{GNUNET_TESTING_peer_run()}. This function takes the testing directory
1052pathname, a configuration template, a callback and its closure. This function
1053creates a peer in the given testing directory by using the configuration
1054template, starts the peer and calls the given callback with the given closure.
1055
1056The function @code{GNUNET_TESTING_peer_run()} starts the ARM service of the
1057peer which starts the rest of the configured services. A similar function
1058@code{GNUNET_TESTING_service_run} can be used to just start a single service of
1059a peer. In this case, the peer's ARM service is not started; instead, only the
1060given service is run.
1061
1062@c ***************************************************************************
1063@node Testing with multiple processes
1064@subsection Testing with multiple processes
1065
1066When testing GNUnet, the splitting of the code into a services and clients
1067often complicates testing. The solution to this is to have the testcase fork
1068@code{gnunet-service-arm}, ask it to start the required server and daemon
1069processes and then execute appropriate client actions (to test the client APIs
1070or the core module or both). If necessary, multiple ARM services can be forked
1071using different ports (!) to simulate a network. However, most of the time only
1072one ARM process is needed. Note that on exit, the testcase should shutdown ARM
1073with a @code{TERM} signal (to give it the chance to cleanly stop its child
1074processes).
1075
1076The following code illustrates spawning and killing an ARM process from a
1077testcase:
1078@example
1079static void run (void *cls, char *const *args, const char
1080*cfgfile, const struct GNUNET_CONFIGURATION_Handle *cfg) @{ struct
1081GNUNET_OS_Process *arm_pid; arm_pid = GNUNET_OS_start_process (NULL, NULL,
1082"gnunet-service-arm", "gnunet-service-arm", "-c", cfgname, NULL);
1083 /* do real test work here */
1084 if (0 != GNUNET_OS_process_kill (arm_pid, SIGTERM)) GNUNET_log_strerror
1085 (GNUNET_ERROR_TYPE_WARNING, "kill"); GNUNET_assert (GNUNET_OK ==
1086 GNUNET_OS_process_wait (arm_pid)); GNUNET_OS_process_close (arm_pid); @}
1087
1088GNUNET_PROGRAM_run (argc, argv, "NAME-OF-TEST", "nohelp", options, &run, cls);
1089@end example
1090
1091
1092An alternative way that works well to test plugins is to implement a
1093mock-version of the environment that the plugin expects and then to simply load
1094the plugin directly.
1095
1096@c ***************************************************************************
1097@node Performance regression analysis with Gauger
1098@section Performance regression analysis with Gauger
1099
1100To help avoid performance regressions, GNUnet uses Gauger. Gauger is a simple
1101logging tool that allows remote hosts to send performance data to a central
1102server, where this data can be analyzed and visualized. Gauger shows graphs of
1103the repository revisions and the performace data recorded for each revision, so
1104sudden performance peaks or drops can be identified and linked to a specific
1105revision number.
1106
1107In the case of GNUnet, the buildbots log the performance data obtained during
1108the tests after each build. The data can be accesed on GNUnet's Gauger page.
1109
1110The menu on the left allows to select either the results of just one build bot
1111(under "Hosts") or review the data from all hosts for a given test result
1112(under "Metrics"). In case of very different absolute value of the results, for
1113instance arm vs. amd64 machines, the option "Normalize" on a metric view can
1114help to get an idea about the performance evolution across all hosts.
1115
1116Using Gauger in GNUnet and having the performance of a module tracked over time
1117is very easy. First of course, the testcase must generate some consistent
1118metric, which makes sense to have logged. Highly volatile or random dependant
1119metrics probably are not ideal candidates for meaningful regression detection.
1120
1121To start logging any value, just include @code{gauger.h} in your testcase code.
1122Then, use the macro @code{GAUGER()} to make the buildbots log whatever value is
1123of interest for you to @code{gnunet.org}'s Gauger server. No setup is necessary
1124as most buildbots have already everything in place and new metrics are created
1125on demand. To delete a metric, you need to contact a member of the GNUnet
1126development team (a file will need to be removed manually from the respective
1127directory).
1128
1129The code in the test should look like this:
1130@example
1131[other includes]
1132#include <gauger.h>
1133
1134int main (int argc, char *argv[]) @{
1135
1136 [run test, generate data] GAUGER("YOUR_MODULE", "METRIC_NAME", (float)value,
1137 "UNIT"); @}
1138@end example
1139
1140
1141Where:
1142@table @asis
1143
1144@item @strong{YOUR_MODULE} is a category in the gauger page and should be the
1145name of the module or subsystem like "Core" or "DHT"
1146@item @strong{METRIC} is
1147the name of the metric being collected and should be concise and descriptive,
1148like "PUT operations in sqlite-datastore".
1149@item @strong{value} is the value
1150of the metric that is logged for this run.
1151@item @strong{UNIT} is the unit in
1152which the value is measured, for instance "kb/s" or "kb of RAM/node".
1153@end table
1154
1155If you wish to use Gauger for your own project, you can grab a copy of the
1156latest stable release or check out Gauger's Subversion repository.
1157
1158@c ***************************************************************************
1159@node GNUnet's TESTBED Subsystem
1160@section GNUnet's TESTBED Subsystem
1161
1162The TESTBED subsystem facilitates testing and measuring of multi-peer
1163deployments on a single host or over multiple hosts.
1164
1165The architecture of the testbed module is divided into the following:
1166@itemize @bullet
1167
1168@item Testbed API: An API which is used by the testing driver programs. It
1169provides with functions for creating, destroying, starting, stopping peers,
1170etc.
1171
1172@item Testbed service (controller): A service which is started through the
1173Testbed API. This service handles operations to create, destroy, start, stop
1174peers, connect them, modify their configurations.
1175
1176@item Testbed helper: When a controller has to be started on a host, the
1177testbed API starts the testbed helper on that host which in turn starts the
1178controller. The testbed helper receives a configuration for the controller
1179through its stdin and changes it to ensure the controller doesn't run into any
1180port conflict on that host.
1181@end itemize
1182
1183
1184The testbed service (controller) is different from the other GNUnet services in
1185that it is not started by ARM and is not supposed to be run as a daemon. It is
1186started by the testbed API through a testbed helper. In a typical scenario
1187involving multiple hosts, a controller is started on each host. Controllers
1188take up the actual task of creating peers, starting and stopping them on the
1189hosts they run.
1190
1191While running deployments on a single localhost the testbed API starts the
1192testbed helper directly as a child process. When running deployments on remote
1193hosts the testbed API starts Testbed Helpers on each remote host through remote
1194shell. By default testbed API uses SSH as a remote shell. This can be changed
1195by setting the environmental variable GNUNET_TESTBED_RSH_CMD to the required
1196remote shell program. This variable can also contain parameters which are to be
1197passed to the remote shell program. For e.g:@ @code{@ export
1198GNUNET_TESTBED_RSH_CMD="ssh -o BatchMode=yes -o
1199NoHostAuthenticationForLocalhost=yes %h"@ }@ Substitutions are allowed int the
1200above command string also allows for substitions. through placemarks which
1201begin with a `%'. At present the following substitutions are supported
1202@itemize @bullet
1203@item
1204%h: hostname
1205@item
1206%u: username
1207@item
1208%p: port
1209@end itemize
1210
1211Note that the substitution placemark is replaced only when the corresponding
1212field is available and only once. Specifying @code{%u@@%h} doesn't work either.
1213If you want to user username substitutions for SSH use the argument @code{-l}
1214before the username substitution. Ex: @code{ssh -l %u -p %p %h}
1215
1216The testbed API and the helper communicate through the helpers stdin and
1217stdout. As the helper is started through a remote shell on remote hosts any
1218output messages from the remote shell interfere with the communication and
1219results in a failure while starting the helper. For this reason, it is
1220suggested to use flags to make the remote shells produce no output messages and
1221to have password-less logins. The default remote shell, SSH, the default
1222options are "-o BatchMode=yes -o NoHostBasedAuthenticationForLocalhost=yes".
1223Password-less logins should be ensured by using SSH keys.
1224
1225Since the testbed API executes the remote shell as a non-interactive shell,
1226certain scripts like .bashrc, .profiler may not be executed. If this is the
1227case testbed API can be forced to execute an interactive shell by setting up
1228the environmental variable `GNUNET_TESTBED_RSH_CMD_SUFFIX' to a shell program.
1229An example could be:@ @code{@ export GNUNET_TESTBED_RSH_CMD_SUFFIX="sh -lc"@ }@
1230The testbed API will then execute the remote shell program as: @code{
1231$GNUNET_TESTBED_RSH_CMD -p $port $dest $GNUNET_TESTBED_RSH_CMD_SUFFIX
1232gnunet-helper-testbed }
1233
1234On some systems, problems may arise while starting testbed helpers if GNUnet is
1235installed into a custom location since the helper may not be found in the
1236standard path. This can be addressed by setting the variable
1237`HELPER_BINARY_PATH' to the path of the testbed helper. Testbed API will then
1238use this path to start helper binaries both locally and remotely.
1239
1240Testbed API can accessed by including "gnunet_testbed_service.h" file and
1241linking with -lgnunettestbed.
1242
1243
1244
1245@c ***************************************************************************
1246@menu
1247* Supported Topologies::
1248* Hosts file format::
1249* Topology file format::
1250* Testbed Barriers::
1251* Automatic large-scale deployment of GNUnet in the PlanetLab testbed::
1252* TESTBED Caveats::
1253@end menu
1254
1255@node Supported Topologies
1256@subsection Supported Topologies
1257
1258While testing multi-peer deployments, it is often needed that the peers are
1259connected in some topology. This requirement is addressed by the function
1260@code{GNUNET_TESTBED_overlay_connect()} which connects any given two peers in
1261the testbed.
1262
1263The API also provides a helper function
1264@code{GNUNET_TESTBED_overlay_configure_topology()} to connect a given set of
1265peers in any of the following supported topologies:
1266@itemize @bullet
1267
1268@item @code{GNUNET_TESTBED_TOPOLOGY_CLIQUE}: All peers are connected with each
1269other
1270
1271@item @code{GNUNET_TESTBED_TOPOLOGY_LINE}: Peers are connected to form a line
1272
1273@item @code{GNUNET_TESTBED_TOPOLOGY_RING}: Peers are connected to form a ring
1274topology
1275
1276@item @code{GNUNET_TESTBED_TOPOLOGY_2D_TORUS}: Peers are connected to form a 2
1277dimensional torus topology. The number of peers may not be a perfect square, in
1278that case the resulting torus may not have the uniform poloidal and toroidal
1279lengths
1280
1281@item @code{GNUNET_TESTBED_TOPOLOGY_ERDOS_RENYI}: Topology is generated to form
1282a random graph. The number of links to be present should be given
1283
1284@item @code{GNUNET_TESTBED_TOPOLOGY_SMALL_WORLD}: Peers are connected to form a
12852D Torus with some random links among them. The number of random links are to
1286be given
1287
1288@item @code{GNUNET_TESTBED_TOPOLOGY_SMALL_WORLD_RING}: Peers are connected to
1289form a ring with some random links among them. The number of random links are
1290to be given
1291
1292@item @code{GNUNET_TESTBED_TOPOLOGY_SCALE_FREE}: Connects peers in a topology
1293where peer connectivity follows power law - new peers are connected with high
1294probabililty to well connected peers. See Emergence of Scaling in Random
1295Networks. Science 286, 509-512, 1999.
1296
1297@item @code{GNUNET_TESTBED_TOPOLOGY_FROM_FILE}: The topology information is
1298loaded from a file. The path to the file has to be given. See Topology file
1299format for the format of this file.
1300
1301@item @code{GNUNET_TESTBED_TOPOLOGY_NONE}: No topology
1302@end itemize
1303
1304
1305The above supported topologies can be specified respectively by setting the
1306variable @code{OVERLAY_TOPOLOGY} to the following values in the configuration
1307passed to Testbed API functions @code{GNUNET_TESTBED_test_run()} and
1308@code{GNUNET_TESTBED_run()}:
1309@itemize @bullet
1310@item @code{CLIQUE}
1311@item @code{RING}
1312@item @code{LINE}
1313@item @code{2D_TORUS}
1314@item @code{RANDOM}
1315@item @code{SMALL_WORLD}
1316@item @code{SMALL_WORLD_RING}
1317@item @code{SCALE_FREE}
1318@item @code{FROM_FILE}
1319@item @code{NONE}
1320@end itemize
1321
1322
1323Topologies @code{RANDOM}, @code{SMALL_WORLD} and @code{SMALL_WORLD_RING}
1324require the option @code{OVERLAY_RANDOM_LINKS} to be set to the number of
1325random links to be generated in the configuration. The option will be ignored
1326for the rest of the topologies.
1327
1328Toplogy @code{SCALE_FREE} requires the options @code{SCALE_FREE_TOPOLOGY_CAP}
1329to be set to the maximum number of peers which can connect to a peer and
1330@code{SCALE_FREE_TOPOLOGY_M} to be set to how many peers a peer should be
1331atleast connected to.
1332
1333Similarly, the topology @code{FROM_FILE} requires the option
1334@code{OVERLAY_TOPOLOGY_FILE} to contain the path of the file containing the
1335topology information. This option is ignored for the rest of the topologies.
1336See Topology file format for the format of this file.
1337
1338@c ***************************************************************************
1339@node Hosts file format
1340@subsection Hosts file format
1341
1342The testbed API offers the function GNUNET_TESTBED_hosts_load_from_file() to
1343load from a given file details about the hosts which testbed can use for
1344deploying peers. This function is useful to keep the data about hosts separate
1345instead of hard coding them in code.
1346
1347Another helper function from testbed API, GNUNET_TESTBED_run() also takes a
1348hosts file name as its parameter. It uses the above function to populate the
1349hosts data structures and start controllers to deploy peers.
1350
1351These functions require the hosts file to be of the following format:
1352@itemize @bullet
1353@item Each line is interpreted to have details about a host
1354@item Host details should include the username to use for logging into the
1355host, the hostname of the host and the port number to use for the remote shell
1356program. All thee values should be given.
1357@item These details should be given in the following format:
1358@code{<username>@@<hostname>:<port>}
1359@end itemize
1360
1361Note that having canonical hostnames may cause problems while resolving the IP
1362addresses (See this bug). Hence it is advised to provide the hosts' IP
1363numerical addresses as hostnames whenever possible.
1364
1365@c ***************************************************************************
1366@node Topology file format
1367@subsection Topology file format
1368
1369A topology file describes how peers are to be connected. It should adhere to
1370the following format for testbed to parse it correctly.
1371
1372Each line should begin with the target peer id. This should be followed by a
1373colon(`:') and origin peer ids seperated by `|'. All spaces except for newline
1374characters are ignored. The API will then try to connect each origin peer to
1375the target peer.
1376
1377For example, the following file will result in 5 overlay connections: [2->1],
1378[3->1],[4->3], [0->3], [2->0]@ @code{@ 1:2|3@ 3:4| 0@ 0: 2@ }
1379
1380@c ***************************************************************************
1381@node Testbed Barriers
1382@subsection Testbed Barriers
1383
1384The testbed subsystem's barriers API facilitates coordination among the peers
1385run by the testbed and the experiment driver. The concept is similar to the
1386barrier synchronisation mechanism found in parallel programming or
1387multi-threading paradigms - a peer waits at a barrier upon reaching it until
1388the barrier is reached by a predefined number of peers. This predefined number
1389of peers required to cross a barrier is also called quorum. We say a peer has
1390reached a barrier if the peer is waiting for the barrier to be crossed.
1391Similarly a barrier is said to be reached if the required quorum of peers reach
1392the barrier. A barrier which is reached is deemed as crossed after all the
1393peers waiting on it are notified.
1394
1395The barriers API provides the following functions:
1396@itemize @bullet
1397@item @strong{@code{GNUNET_TESTBED_barrier_init()}:} function to initialse a
1398barrier in the experiment
1399@item @strong{@code{GNUNET_TESTBED_barrier_cancel()}:} function to cancel a
1400barrier which has been initialised before
1401@item @strong{@code{GNUNET_TESTBED_barrier_wait()}:} function to signal barrier
1402service that the caller has reached a barrier and is waiting for it to be
1403crossed
1404@item @strong{@code{GNUNET_TESTBED_barrier_wait_cancel()}:} function to stop
1405waiting for a barrier to be crossed
1406@end itemize
1407
1408
1409Among the above functions, the first two, namely
1410@code{GNUNET_TESTBED_barrier_init()} and @code{GNUNET_TESTBED_barrier_cancel()}
1411are used by experiment drivers. All barriers should be initialised by the
1412experiment driver by calling @code{GNUNET_TESTBED_barrier_init()}. This
1413function takes a name to identify the barrier, the quorum required for the
1414barrier to be crossed and a notification callback for notifying the experiment
1415driver when the barrier is crossed. @code{GNUNET_TESTBED_barrier_cancel()}
1416cancels an initialised barrier and frees the resources allocated for it. This
1417function can be called upon a initialised barrier before it is crossed.
1418
1419The remaining two functions @code{GNUNET_TESTBED_barrier_wait()} and
1420@code{GNUNET_TESTBED_barrier_wait_cancel()} are used in the peer's processes.
1421@code{GNUNET_TESTBED_barrier_wait()} connects to the local barrier service
1422running on the same host the peer is running on and registers that the caller
1423has reached the barrier and is waiting for the barrier to be crossed. Note that
1424this function can only be used by peers which are started by testbed as this
1425function tries to access the local barrier service which is part of the testbed
1426controller service. Calling @code{GNUNET_TESTBED_barrier_wait()} on an
1427uninitialised barrier results in failure.
1428@code{GNUNET_TESTBED_barrier_wait_cancel()} cancels the notification registered
1429by @code{GNUNET_TESTBED_barrier_wait()}.
1430
1431
1432@c ***************************************************************************
1433@menu
1434* Implementation::
1435@end menu
1436
1437@node Implementation
1438@subsubsection Implementation
1439
1440Since barriers involve coordination between experiment driver and peers, the
1441barrier service in the testbed controller is split into two components. The
1442first component responds to the message generated by the barrier API used by
1443the experiment driver (functions @code{GNUNET_TESTBED_barrier_init()} and
1444@code{GNUNET_TESTBED_barrier_cancel()}) and the second component to the
1445messages generated by barrier API used by peers (functions
1446@code{GNUNET_TESTBED_barrier_wait()} and
1447@code{GNUNET_TESTBED_barrier_wait_cancel()}).
1448
1449Calling @code{GNUNET_TESTBED_barrier_init()} sends a
1450@code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_INIT} message to the master
1451controller. The master controller then registers a barrier and calls
1452@code{GNUNET_TESTBED_barrier_init()} for each its subcontrollers. In this way
1453barrier initialisation is propagated to the controller hierarchy. While
1454propagating initialisation, any errors at a subcontroller such as timeout
1455during further propagation are reported up the hierarchy back to the experiment
1456driver.
1457
1458Similar to @code{GNUNET_TESTBED_barrier_init()},
1459@code{GNUNET_TESTBED_barrier_cancel()} propagates
1460@code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_CANCEL} message which causes
1461controllers to remove an initialised barrier.
1462
1463The second component is implemented as a separate service in the binary
1464`gnunet-service-testbed' which already has the testbed controller service.
1465Although this deviates from the gnunet process architecture of having one
1466service per binary, it is needed in this case as this component needs access to
1467barrier data created by the first component. This component responds to
1468@code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_WAIT} messages from local peers when
1469they call @code{GNUNET_TESTBED_barrier_wait()}. Upon receiving
1470@code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_WAIT} message, the service checks if
1471the requested barrier has been initialised before and if it was not
1472initialised, an error status is sent through
1473@code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_STATUS} message to the local peer and
1474the connection from the peer is terminated. If the barrier is initialised
1475before, the barrier's counter for reached peers is incremented and a
1476notification is registered to notify the peer when the barrier is reached. The
1477connection from the peer is left open.
1478
1479When enough peers required to attain the quorum send
1480@code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_WAIT} messages, the controller sends
1481a @code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_STATUS} message to its parent
1482informing that the barrier is crossed. If the controller has started further
1483subcontrollers, it delays this message until it receives a similar notification
1484from each of those subcontrollers. Finally, the barriers API at the experiment
1485driver receives the @code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_STATUS} when the
1486barrier is reached at all the controllers.
1487
1488The barriers API at the experiment driver responds to the
1489@code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_STATUS} message by echoing it back to
1490the master controller and notifying the experiment controller through the
1491notification callback that a barrier has been crossed. The echoed
1492@code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_STATUS} message is propagated by the
1493master controller to the controller hierarchy. This propagation triggers the
1494notifications registered by peers at each of the controllers in the hierarchy.
1495Note the difference between this downward propagation of the
1496@code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_STATUS} message from its upward
1497propagation --- the upward propagation is needed for ensuring that the barrier
1498is reached by all the controllers and the downward propagation is for
1499triggering that the barrier is crossed.
1500
1501@c ***************************************************************************
1502@node Automatic large-scale deployment of GNUnet in the PlanetLab testbed
1503@subsection Automatic large-scale deployment of GNUnet in the PlanetLab testbed
1504
1505PlanetLab is as a testbed for computer networking and distributed systems
1506research. It was established in 2002 and as of June 2010 was composed of 1090
1507nodes at 507 sites worldwide.
1508
1509To automate the GNUnet we created a set of automation tools to simplify the
1510large-scale deployment. We provide you a set of scripts you can use to deploy
1511GNUnet on a set of nodes and manage your installation.
1512
1513Please also check @uref{https://gnunet.org/installation-fedora8-svn} and@
1514@uref{https://gnunet.org/installation-fedora12-svn} to find detailled
1515instructions how to install GNUnet on a PlanetLab node.
1516
1517
1518@c ***************************************************************************
1519@menu
1520* PlanetLab Automation for Fedora8 nodes::
1521* Install buildslave on PlanetLab nodes running fedora core 8::
1522* Setup a new PlanetLab testbed using GPLMT::
1523* Why do i get an ssh error when using the regex profiler?::
1524@end menu
1525
1526@node PlanetLab Automation for Fedora8 nodes
1527@subsubsection PlanetLab Automation for Fedora8 nodes
1528
1529@c ***************************************************************************
1530@node Install buildslave on PlanetLab nodes running fedora core 8
1531@subsubsection Install buildslave on PlanetLab nodes running fedora core 8
1532@c ** Actually this is a subsubsubsection, but must be fixed differently
1533@c ** as subsubsection is the lowest.
1534
1535Since most of the PlanetLab nodes are running the very old fedora core 8 image,
1536installing the buildslave software is quite some pain. For our PlanetLab
1537testbed we figured out how to install the buildslave software best.
1538
1539Install Distribute for python:@ @code{@ curl
1540http://python-distribute.org/distribute_setup.py | sudo python@ }
1541
1542Install Distribute for zope.interface <= 3.8.0 (4.0 and 4.0.1 will not work):@
1543@code{@ wget
1544http://pypi.python.org/packages/source/z/zope.interface/zope.interface-3.8.0.tar.gz@
1545tar zvfz zope.interface-3.8.0.tar.gz@ cd zope.interface-3.8.0@ sudo python
1546setup.py install@ }
1547
1548Install the buildslave software (0.8.6 was the latest version):@ @code{@ wget
1549http://buildbot.googlecode.com/files/buildbot-slave-0.8.6p1.tar.gz@ tar xvfz
1550buildbot-slave-0.8.6p1.tar.gz@ cd buildslave-0.8.6p1@ sudo python setup.py
1551install@ }
1552
1553The setup will download the matching twisted package and install it.@ It will
1554also try to install the latest version of zope.interface which will fail to
1555install. Buildslave will work anyway since version 3.8.0 was installed before!
1556
1557@c ***************************************************************************
1558@node Setup a new PlanetLab testbed using GPLMT
1559@subsubsection Setup a new PlanetLab testbed using GPLMT
1560
1561@itemize @bullet
1562@item Get a new slice and assign nodes
1563Ask your PlanetLab PI to give you a new slice and assign the nodes you need
1564@item Install a buildmaster
1565You can stick to the buildbot documentation:@
1566@uref{http://buildbot.net/buildbot/docs/current/manual/installation.html}
1567@item Install the buildslave software on all nodes
1568To install the buildslave on all nodes assigned to your slice you can use the
1569tasklist @code{install_buildslave_fc8.xml} provided with GPLMT:
1570
1571@code{@ ./gplmt.py -c contrib/tumple_gnunet.conf -t
1572contrib/tasklists/install_buildslave_fc8.xml -a -p <planetlab password>@ }
1573
1574@item Create the buildmaster configuration and the slave setup commands
1575
1576The master and the and the slaves have need to have credentials and the master
1577has to have all nodes configured. This can be done with the
1578@code{create_buildbot_configuration.py} script in the @code{scripts} directory
1579
1580This scripts takes a list of nodes retrieved directly from PlanetLab or read
1581from a file and a configuration template and creates:@
1582 - a tasklist which can be executed with gplmt to setup the slaves@
1583 - a master.cfg file containing a PlanetLab nodes
1584
1585A configuration template is included in the <contrib>, most important is that
1586the script replaces the following tags in the template:
1587
1588%GPLMT_BUILDER_DEFINITION :@ GPLMT_BUILDER_SUMMARY@ GPLMT_SLAVES@
1589%GPLMT_SCHEDULER_BUILDERS
1590
1591Create configuration for all nodes assigned to a slice:@ @code{@
1592./create_buildbot_configuration.py -u <planetlab username> -p <planetlab
1593password> -s <slice> -m <buildmaster+port> -t <template>@ }@ Create
1594configuration for some nodes in a file:@ @code{@
1595./create_buildbot_configuration.p -f <node_file> -m <buildmaster+port> -t
1596<template>@ }
1597
1598@item Copy the @code{master.cfg} to the buildmaster and start it
1599Use @code{buildbot start <basedir>} to start the server
1600@item Setup the buildslaves
1601@end itemize
1602
1603@c ***************************************************************************
1604@node Why do i get an ssh error when using the regex profiler?
1605@subsubsection Why do i get an ssh error when using the regex profiler?
1606
1607Why do i get an ssh error "Permission denied (publickey,password)." when using
1608the regex profiler although passwordless ssh to localhost works using publickey
1609and ssh-agent?
1610
1611You have to generate a public/private-key pair with no password:@
1612@code{ssh-keygen -t rsa -b 4096 -f ~/.ssh/id_localhost}@
1613and then add the following to your ~/.ssh/config file:
1614
1615@code{Host 127.0.0.1@ IdentityFile ~/.ssh/id_localhost}
1616
1617now make sure your hostsfile looks like@
1618
1619[USERNAME]@@127.0.0.1:22@
1620[USERNAME]@@127.0.0.1:22
1621
1622You can test your setup by running `ssh 127.0.0.1` in a terminal and then in
1623the opened session run it again. If you were not asked for a password on either
1624login, then you should be good to go.
1625
1626@c ***************************************************************************
1627@node TESTBED Caveats
1628@subsection TESTBED Caveats
1629
1630This section documents a few caveats when using the GNUnet testbed
1631subsystem.
1632
1633
1634@c ***************************************************************************
1635@menu
1636* CORE must be started::
1637* ATS must want the connections::
1638@end menu
1639
1640@node CORE must be started
1641@subsubsection CORE must be started
1642
1643A simple issue is #3993: Your configuration MUST somehow ensure that for each
1644peer the CORE service is started when the peer is setup, otherwise TESTBED may
1645fail to connect peers when the topology is initialized, as TESTBED will start
1646some CORE services but not necessarily all (but it relies on all of them
1647running). The easiest way is to set 'FORCESTART = YES' in the '[core]' section
1648of the configuration file. Alternatively, having any service that directly or
1649indirectly depends on CORE being started with FORCESTART will also do. This
1650issue largely arises if users try to over-optimize by not starting any services
1651with FORCESTART.
1652
1653@c ***************************************************************************
1654@node ATS must want the connections
1655@subsubsection ATS must want the connections
1656
1657When TESTBED sets up connections, it only offers the respective HELLO
1658information to the TRANSPORT service. It is then up to the ATS service to
1659@strong{decide} to use the connection. The ATS service will typically eagerly
1660establish any connection if the number of total connections is low (relative to
1661bandwidth). Details may further depend on the specific ATS backend that was
1662configured. If ATS decides to NOT establish a connection (even though TESTBED
1663provided the required information), then that connection will count as failed
1664for TESTBED. Note that you can configure TESTBED to tolerate a certain number
1665of connection failures (see '-e' option of gnunet-testbed-profiler). This issue
1666largely arises for dense overlay topologies, especially if you try to create
1667cliques with more than 20 peers.
1668
1669@c ***************************************************************************
1670@node libgnunetutil
1671@section libgnunetutil
1672
1673libgnunetutil is the fundamental library that all GNUnet code builds upon.
1674Ideally, this library should contain most of the platform dependent code
1675(except for user interfaces and really special needs that only few applications
1676have). It is also supposed to offer basic services that most if not all GNUnet
1677binaries require. The code of libgnunetutil is in the src/util/ directory. The
1678public interface to the library is in the gnunet_util.h header. The functions
1679provided by libgnunetutil fall roughly into the following categories (in
1680roughly the order of importance for new developers):
1681@itemize @bullet
1682@item logging (common_logging.c)
1683@item memory allocation (common_allocation.c)
1684@item endianess conversion (common_endian.c)
1685@item internationalization (common_gettext.c)
1686@item String manipulation (string.c)
1687@item file access (disk.c)
1688@item buffered disk IO (bio.c)
1689@item time manipulation (time.c)
1690@item configuration parsing (configuration.c)
1691@item command-line handling (getopt*.c)
1692@item cryptography (crypto_*.c)
1693@item data structures (container_*.c)
1694@item CPS-style scheduling (scheduler.c)
1695@item Program initialization (program.c)
1696@item Networking (network.c, client.c, server*.c, service.c)
1697@item message queueing (mq.c)
1698@item bandwidth calculations (bandwidth.c)
1699@item Other OS-related (os*.c, plugin.c, signal.c)
1700@item Pseudonym management (pseudonym.c)
1701@end itemize
1702
1703It should be noted that only developers that fully understand this entire API
1704will be able to write good GNUnet code.
1705
1706Ideally, porting GNUnet should only require porting the gnunetutil library.
1707More testcases for the gnunetutil APIs are therefore a great way to make
1708porting of GNUnet easier.
1709
1710@menu
1711* Logging::
1712* Interprocess communication API (IPC)::
1713* Cryptography API::
1714* Message Queue API::
1715* Service API::
1716* Optimizing Memory Consumption of GNUnet's (Multi-) Hash Maps::
1717* The CONTAINER_MDLL API::
1718@end menu
1719
1720@c ***************************************************************************
1721@node Logging
1722@subsection Logging
1723
1724GNUnet is able to log its activity, mostly for the purposes of debugging the
1725program at various levels.
1726
1727@code{gnunet_common.h} defines several @strong{log levels}:
1728@table @asis
1729
1730@item ERROR for errors (really problematic situations, often leading to
1731crashes)
1732@item WARNING for warnings (troubling situations that might have
1733negative consequences, although not fatal)
1734@item INFO for various information.
1735Used somewhat rarely, as GNUnet statistics is used to hold and display most of
1736the information that users might find interesting.
1737@item DEBUG for debugging.
1738Does not produce much output on normal builds, but when extra logging is
1739enabled at compile time, a staggering amount of data is outputted under this
1740log level.
1741@end table
1742
1743
1744Normal builds of GNUnet (configured with @code{--enable-logging[=yes]}) are
1745supposed to log nothing under DEBUG level. The @code{--enable-logging=verbose}
1746configure option can be used to create a build with all logging enabled.
1747However, such build will produce large amounts of log data, which is
1748inconvenient when one tries to hunt down a specific problem.
1749
1750To mitigate this problem, GNUnet provides facilities to apply a filter to
1751reduce the logs:
1752@table @asis
1753
1754@item Logging by default When no log levels are configured in any other way
1755(see below), GNUnet will default to the WARNING log level. This mostly applies
1756to GNUnet command line utilities, services and daemons; tests will always set
1757log level to WARNING or, if @code{--enable-logging=verbose} was passed to
1758configure, to DEBUG. The default level is suggested for normal operation.
1759@item The -L option Most GNUnet executables accept an "-L loglevel" or
1760"--log=loglevel" option. If used, it makes the process set a global log level
1761to "loglevel". Thus it is possible to run some processes with -L DEBUG, for
1762example, and others with -L ERROR to enable specific settings to diagnose
1763problems with a particular process.
1764@item Configuration files. Because GNUnet
1765service and deamon processes are usually launched by gnunet-arm, it is not
1766possible to pass different custom command line options directly to every one of
1767them. The options passed to @code{gnunet-arm} only affect gnunet-arm and not
1768the rest of GNUnet. However, one can specify a configuration key "OPTIONS" in
1769the section that corresponds to a service or a daemon, and put a value of "-L
1770loglevel" there. This will make the respective service or daemon set its log
1771level to "loglevel" (as the value of OPTIONS will be passed as a command-line
1772argument).
1773
1774To specify the same log level for all services without creating separate
1775"OPTIONS" entries in the configuration for each one, the user can specify a
1776config key "GLOBAL_POSTFIX" in the [arm] section of the configuration file. The
1777value of GLOBAL_POSTFIX will be appended to all command lines used by the ARM
1778service to run other services. It can contain any option valid for all GNUnet
1779commands, thus in particular the "-L loglevel" option. The ARM service itself
1780is, however, unaffected by GLOBAL_POSTFIX; to set log level for it, one has to
1781specify "OPTIONS" key in the [arm] section.
1782@item Environment variables.
1783Setting global per-process log levels with "-L loglevel" does not offer
1784sufficient log filtering granularity, as one service will call interface
1785libraries and supporting libraries of other GNUnet services, potentially
1786producing lots of debug log messages from these libraries. Also, changing the
1787config file is not always convenient (especially when running the GNUnet test
1788suite).@ To fix that, and to allow GNUnet to use different log filtering at
1789runtime without re-compiling the whole source tree, the log calls were changed
1790to be configurable at run time. To configure them one has to define environment
1791variables "GNUNET_FORCE_LOGFILE", "GNUNET_LOG" and/or "GNUNET_FORCE_LOG":
1792@itemize @bullet
1793
1794@item "GNUNET_LOG" only affects the logging when no global log level is
1795configured by any other means (that is, the process does not explicitly set its
1796own log level, there are no "-L loglevel" options on command line or in
1797configuration files), and can be used to override the default WARNING log
1798level.
1799
1800@item "GNUNET_FORCE_LOG" will completely override any other log configuration
1801options given.
1802
1803@item "GNUNET_FORCE_LOGFILE" will completely override the location of the file
1804to log messages to. It should contain a relative or absolute file name. Setting
1805GNUNET_FORCE_LOGFILE is equivalent to passing "--log-file=logfile" or "-l
1806logfile" option (see below). It supports "[]" format in file names, but not
1807"@{@}" (see below).
1808@end itemize
1809
1810
1811Because environment variables are inherited by child processes when they are
1812launched, starting or re-starting the ARM service with these variables will
1813propagate them to all other services.
1814
1815"GNUNET_LOG" and "GNUNET_FORCE_LOG" variables must contain a specially
1816formatted @strong{logging definition} string, which looks like this:@ @code{@
1817[component];[file];[function];[from_line[-to_line]];loglevel@emph{[/component...]}@
1818}@ That is, a logging definition consists of definition entries, separated by
1819slashes ('/'). If only one entry is present, there is no need to add a slash
1820to its end (although it is not forbidden either).@ All definition fields
1821(component, file, function, lines and loglevel) are mandatory, but (except for
1822the loglevel) they can be empty. An empty field means "match anything". Note
1823that even if fields are empty, the semicolon (';') separators must be
1824present.@ The loglevel field is mandatory, and must contain one of the log
1825level names (ERROR, WARNING, INFO or DEBUG).@ The lines field might contain
1826one non-negative number, in which case it matches only one line, or a range
1827"from_line-to_line", in which case it matches any line in the interval
1828[from_line;to_line] (that is, including both start and end line).@ GNUnet
1829mostly defaults component name to the name of the service that is implemented
1830in a process ('transport', 'core', 'peerinfo', etc), but logging calls can
1831specify custom component names using @code{GNUNET_log_from}.@ File name and
1832function name are provided by the compiler (__FILE__ and __FUNCTION__
1833built-ins).
1834
1835Component, file and function fields are interpreted as non-extended regular
1836expressions (GNU libc regex functions are used). Matching is case-sensitive, ^
1837and $ will match the beginning and the end of the text. If a field is empty,
1838its contents are automatically replaced with a ".*" regular expression, which
1839matches anything. Matching is done in the default way, which means that the
1840expression matches as long as it's contained anywhere in the string. Thus
1841"GNUNET_" will match both "GNUNET_foo" and "BAR_GNUNET_BAZ". Use '^' and/or '$'
1842to make sure that the expression matches at the start and/or at the end of the
1843string.@ The semicolon (';') can't be escaped, and GNUnet will not use it in
1844component names (it can't be used in function names and file names anyway).@
1845
1846@end table
1847
1848
1849Every logging call in GNUnet code will be (at run time) matched against the
1850log definitions passed to the process. If a log definition fields are matching
1851the call arguments, then the call log level is compared the the log level of
1852that definition. If the call log level is less or equal to the definition log
1853level, the call is allowed to proceed. Otherwise the logging call is
1854forbidden, and nothing is logged. If no definitions matched at all, GNUnet
1855will use the global log level or (if a global log level is not specified) will
1856default to WARNING (that is, it will allow the call to proceed, if its level
1857is less or equal to the global log level or to WARNING).
1858
1859That is, definitions are evaluated from left to right, and the first matching
1860definition is used to allow or deny the logging call. Thus it is advised to
1861place narrow definitions at the beginning of the logdef string, and generic
1862definitions - at the end.
1863
1864Whether a call is allowed or not is only decided the first time this particular
1865call is made. The evaluation result is then cached, so that any attempts to
1866make the same call later will be allowed or disallowed right away. Because of
1867that runtime log level evaluation should not significantly affect the process
1868performance.@ Log definition parsing is only done once, at the first call to
1869GNUNET_log_setup () made by the process (which is usually done soon after it
1870starts).
1871
1872At the moment of writing there is no way to specify logging definitions from
1873configuration files, only via environment variables.
1874
1875At the moment GNUnet will stop processing a log definition when it encounters
1876an error in definition formatting or an error in regular expression syntax, and
1877will not report the failure in any way.
1878
1879
1880@c ***************************************************************************
1881@menu
1882* Examples::
1883* Log files::
1884* Updated behavior of GNUNET_log::
1885@end menu
1886
1887@node Examples
1888@subsubsection Examples
1889
1890@table @asis
1891
1892@item @code{GNUNET_FORCE_LOG=";;;;DEBUG" gnunet-arm -s} Start GNUnet process
1893tree, running all processes with DEBUG level (one should be careful with it, as
1894log files will grow at alarming rate!)
1895@item @code{GNUNET_FORCE_LOG="core;;;;DEBUG" gnunet-arm -s} Start GNUnet process
1896tree, running the core service under DEBUG level (everything else will use
1897configured or default level).
1898@item @code{GNUNET_FORCE_LOG=";gnunet-service-transport_validation.c;;;DEBUG" gnunet-arm -s}
1899Start GNUnet process tree, allowing any logging calls from
1900gnunet-service-transport_validation.c (everything else will use configured or
1901default level).
1902@item @code{GNUNET_FORCE_LOG="fs;gnunet-service-fs_push.c;;;DEBUG" gnunet-arm -s}
1903Start GNUnet process tree, allowing any logging calls from
1904gnunet-gnunet-service-fs_push.c (everything else will use configured or default
1905level).
1906@item @code{GNUNET_FORCE_LOG=";;GNUNET_NETWORK_socket_select;;DEBUG" gnunet-arm -s}
1907Start GNUnet process tree, allowing any logging calls from the
1908GNUNET_NETWORK_socket_select function (everything else will use configured or
1909default level).
1910@item @code{GNUNET_FORCE_LOG="transport.*;;.*send.*;;DEBUG/;;;;WARNING" gnunet-arm -s}
1911Start GNUnet process tree, allowing any logging calls from the components
1912that have "transport" in their names, and are made from function that have
1913"send" in their names. Everything else will be allowed to be logged only if it
1914has WARNING level.
1915@end table
1916
1917
1918On Windows, one can use batch files to run GNUnet processes with special
1919environment variables, without affecting the whole system. Such batch file will
1920look like this:@ @code{@ set GNUNET_FORCE_LOG=;;do_transmit;;DEBUG@ gnunet-arm
1921-s@ }@ (note the absence of double quotes in the environment variable
1922definition, as opposed to earlier examples, which use the shell).@ Another
1923limitation, on Windows, GNUNET_FORCE_LOGFILE @strong{MUST} be set in order to
1924GNUNET_FORCE_LOG to work.
1925
1926
1927@c ***************************************************************************
1928@node Log files
1929@subsubsection Log files
1930
1931GNUnet can be told to log everything into a file instead of stderr (which is
1932the default) using the "--log-file=logfile" or "-l logfile" option. This option
1933can also be passed via command line, or from the "OPTION" and "GLOBAL_POSTFIX"
1934configuration keys (see above). The file name passed with this option is
1935subject to GNUnet filename expansion. If specified in "GLOBAL_POSTFIX", it is
1936also subject to ARM service filename expansion, in particular, it may contain
1937"@{@}" (left and right curly brace) sequence, which will be replaced by ARM
1938with the name of the service. This is used to keep logs from more than one
1939service separate, while only specifying one template containing "@{@}" in
1940GLOBAL_POSTFIX.
1941
1942As part of a secondary file name expansion, the first occurrence of "[]"
1943sequence ("left square brace" followed by "right square brace") in the file
1944name will be replaced with a process identifier or the process when it
1945initializes its logging subsystem. As a result, all processes will log into
1946different files. This is convenient for isolating messages of a particular
1947process, and prevents I/O races when multiple processes try to write into the
1948file at the same time. This expansion is done independently of "@{@}"
1949expansion that ARM service does (see above).
1950
1951The log file name that is specified via "-l" can contain format characters
1952from the 'strftime' function family. For example, "%Y" will be replaced with
1953the current year. Using "basename-%Y-%m-%d.log" would include the current
1954year, month and day in the log file. If a GNUnet process runs for long enough
1955to need more than one log file, it will eventually clean up old log files.
1956Currently, only the last three log files (plus the current log file) are
1957preserved. So once the fifth log file goes into use (so after 4 days if you
1958use "%Y-%m-%d" as above), the first log file will be automatically deleted.
1959Note that if your log file name only contains "%Y", then log files would be
1960kept for 4 years and the logs from the first year would be deleted once year 5
1961begins. If you do not use any date-related string format codes, logs would
1962never be automatically deleted by GNUnet.
1963
1964
1965@c ***************************************************************************
1966
1967@node Updated behavior of GNUNET_log
1968@subsubsection Updated behavior of GNUNET_log
1969
1970It's currently quite common to see constructions like this all over the code:
1971@example
1972#if MESH_DEBUG GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "MESH: client
1973disconnected\n"); #endif
1974@end example
1975
1976The reason for the #if is not to avoid displaying the message when disabled
1977(GNUNET_ERROR_TYPE takes care of that), but to avoid the compiler including it
1978in the binary at all, when compiling GNUnet for platforms with restricted
1979storage space / memory (MIPS routers, ARM plug computers / dev boards, etc).
1980
1981This presents several problems: the code gets ugly, hard to write and it is
1982very easy to forget to include the #if guards, creating non-consistent code. A
1983new change in GNUNET_log aims to solve these problems.
1984
1985@strong{This change requires to @code{./configure} with at least
1986@code{--enable-logging=verbose} to see debug messages.}
1987
1988Here is an example of code with dense debug statements:
1989@example
1990switch (restrict_topology) @{
1991case GNUNET_TESTING_TOPOLOGY_CLIQUE: #if VERBOSE_TESTING
1992GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, _("Blacklisting all but clique
1993topology\n")); #endif unblacklisted_connections = create_clique (pg,
1994&remove_connections, BLACKLIST, GNUNET_NO); break; case
1995GNUNET_TESTING_TOPOLOGY_SMALL_WORLD_RING: #if VERBOSE_TESTING GNUNET_log
1996(GNUNET_ERROR_TYPE_DEBUG, _("Blacklisting all but small world (ring)
1997topology\n")); #endif unblacklisted_connections = create_small_world_ring (pg,
1998&remove_connections, BLACKLIST); break;
1999@end example
2000
2001
2002Pretty hard to follow, huh?
2003
2004From now on, it is not necessary to include the #if / #endif statements to
2005acheive the same behavior. The GNUNET_log and GNUNET_log_from macros take care
2006of it for you, depending on the configure option:
2007@itemize @bullet
2008@item If @code{--enable-logging} is set to @code{no}, the binary will contain
2009no log messages at all.
2010@item If @code{--enable-logging} is set to @code{yes}, the binary will contain
2011no DEBUG messages, and therefore running with -L DEBUG will have no effect.
2012Other messages (ERROR, WARNING, INFO, etc) will be included.
2013@item If @code{--enable-logging} is set to @code{verbose}, or
2014@code{veryverbose} the binary will contain DEBUG messages (still, it will be
2015neccessary to run with -L DEBUG or set the DEBUG config option to show them).
2016@end itemize
2017
2018
2019If you are a developer:
2020@itemize @bullet
2021@item please make sure that you @code{./configure
2022--enable-logging=@{verbose,veryverbose@}}, so you can see DEBUG messages.
2023@item please remove the @code{#if} statements around @code{GNUNET_log
2024(GNUNET_ERROR_TYPE_DEBUG, ...)} lines, to improve the readibility of your code.
2025@end itemize
2026
2027Since now activating DEBUG automatically makes it VERBOSE and activates
2028@strong{all} debug messages by default, you probably want to use the
2029https://gnunet.org/logging functionality to filter only relevant messages. A
2030suitable configuration could be:@ @code{$ export
2031GNUNET_FORCE_LOG="^YOUR_SUBSYSTEM$;;;;DEBUG/;;;;WARNING"}@ Which will behave
2032almost like enabling DEBUG in that subsytem before the change. Of course you
2033can adapt it to your particular needs, this is only a quick example.
2034
2035@c ***************************************************************************
2036@node Interprocess communication API (IPC)
2037@subsection Interprocess communication API (IPC)
2038
2039In GNUnet a variety of new message types might be defined and used in
2040interprocess communication, in this tutorial we use the @code{struct
2041AddressLookupMessage} as a example to introduce how to construct our own
2042message type in GNUnet and how to implement the message communication between
2043service and client.@ (Here, a client uses the @code{struct
2044AddressLookupMessage} as a request to ask the server to return the address of
2045any other peer connecting to the service.)
2046
2047
2048@c ***************************************************************************
2049@menu
2050* Define new message types::
2051* Define message struct::
2052* Client: Establish connection::
2053* Client: Initialize request message::
2054* Client: Send request and receive response::
2055* Server: Startup service::
2056* Server: Add new handles for specified messages::
2057* Server: Process request message::
2058* Server: Response to client::
2059* Server: Notification of clients::
2060* Conversion between Network Byte Order (Big Endian) and Host Byte Order::
2061@end menu
2062
2063@node Define new message types
2064@subsubsection Define new message types
2065
2066First of all, you should define the new message type in
2067@code{gnunet_protocols.h}:
2068@example
2069 // Request to look addresses of peers in server.
2070#define GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_LOOKUP 29
2071 // Response to the address lookup request.
2072#define GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_REPLY 30
2073@end example
2074
2075@c ***************************************************************************
2076@node Define message struct
2077@subsubsection Define message struct
2078
2079After the type definition, the specified message structure should also be
2080described in the header file, e.g. transport.h in our case.
2081@example
2082GNUNET_NETWORK_STRUCT_BEGIN
2083
2084struct AddressLookupMessage @{ struct GNUNET_MessageHeader header; int32_t
2085numeric_only GNUNET_PACKED; struct GNUNET_TIME_AbsoluteNBO timeout; uint32_t
2086addrlen GNUNET_PACKED;
2087 /* followed by 'addrlen' bytes of the actual address, then
2088 followed by the 0-terminated name of the transport */ @};
2089 GNUNET_NETWORK_STRUCT_END
2090@end example
2091
2092
2093Please note @code{GNUNET_NETWORK_STRUCT_BEGIN} and @code{GNUNET_PACKED} which
2094both ensure correct alignment when sending structs over the network
2095
2096@menu
2097@end menu
2098
2099@c ***************************************************************************
2100@node Client: Establish connection
2101@subsubsection Client: Establish connection
2102@c %**end of header
2103
2104
2105At first, on the client side, the underlying API is employed to create a new
2106connection to a service, in our example the transport service would be
2107connected.
2108@example
2109struct GNUNET_CLIENT_Connection *client; client =
2110GNUNET_CLIENT_connect ("transport", cfg);
2111@end example
2112
2113@c ***************************************************************************
2114@node Client: Initialize request message
2115@subsubsection Client: Initialize request message
2116@c %**end of header
2117
2118When the connection is ready, we initialize the message. In this step, all the
2119fields of the message should be properly initialized, namely the size, type,
2120and some extra user-defined data, such as timeout, name of transport, address
2121and name of transport.
2122@example
2123struct AddressLookupMessage *msg; size_t len =
2124sizeof (struct AddressLookupMessage) + addressLen + strlen (nameTrans) + 1;
2125msg->header->size = htons (len); msg->header->type = htons
2126(GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_LOOKUP); msg->timeout =
2127GNUNET_TIME_absolute_hton (abs_timeout); msg->addrlen = htonl (addressLen);
2128char *addrbuf = (char *) &msg[1]; memcpy (addrbuf, address, addressLen); char
2129*tbuf = &addrbuf[addressLen]; memcpy (tbuf, nameTrans, strlen (nameTrans) + 1);
2130@end example
2131
2132Note that, here the functions @code{htonl}, @code{htons} and
2133@code{GNUNET_TIME_absolute_hton} are applied to convert little endian into big
2134endian, about the usage of the big/small edian order and the corresponding
2135conversion function please refer to Introduction of Big Endian and Little
2136Endian.
2137
2138@c ***************************************************************************
2139@node Client: Send request and receive response
2140@subsubsection Client: Send request and receive response
2141@c %**end of header
2142
2143FIXME: This is very outdated, see the tutorial for the
2144current API!
2145
2146Next, the client would send the constructed message as a request to the service
2147and wait for the response from the service. To accomplish this goal, there are
2148a number of API calls that can be used. In this example,
2149@code{GNUNET_CLIENT_transmit_and_get_response} is chosen as the most
2150appropriate function to use.
2151@example
2152GNUNET_CLIENT_transmit_and_get_response
2153(client, msg->header, timeout, GNUNET_YES, &address_response_processor,
2154arp_ctx);
2155@end example
2156
2157the argument @code{address_response_processor} is a function with
2158@code{GNUNET_CLIENT_MessageHandler} type, which is used to process the reply
2159message from the service.
2160
2161@node Server: Startup service
2162@subsubsection Server: Startup service
2163
2164After receiving the request message, we run a standard GNUnet service startup
2165sequence using @code{GNUNET_SERVICE_run}, as follows,
2166@example
2167int main(int
2168argc, char**argv) @{ GNUNET_SERVICE_run(argc, argv, "transport"
2169GNUNET_SERVICE_OPTION_NONE, &run, NULL)); @}
2170@end example
2171
2172@c ***************************************************************************
2173@node Server: Add new handles for specified messages
2174@subsubsection Server: Add new handles for specified messages
2175@c %**end of header
2176
2177in the function above the argument @code{run} is used to initiate transport
2178service,and defined like this:
2179@example
2180static void run (void *cls, struct
2181GNUNET_SERVER_Handle *serv, const struct GNUNET_CONFIGURATION_Handle *cfg) @{
2182GNUNET_SERVER_add_handlers (serv, handlers); @}
2183@end example
2184
2185
2186Here, @code{GNUNET_SERVER_add_handlers} must be called in the run function to
2187add new handlers in the service. The parameter @code{handlers} is a list of
2188@code{struct GNUNET_SERVER_MessageHandler} to tell the service which function
2189should be called when a particular type of message is received, and should be
2190defined in this way:
2191@example
2192static struct GNUNET_SERVER_MessageHandler
2193handlers[] = @{ @{&handle_start, NULL, GNUNET_MESSAGE_TYPE_TRANSPORT_START,
21940@}, @{&handle_send, NULL, GNUNET_MESSAGE_TYPE_TRANSPORT_SEND, 0@},
2195@{&handle_try_connect, NULL, GNUNET_MESSAGE_TYPE_TRANSPORT_TRY_CONNECT, sizeof
2196(struct TryConnectMessage)@}, @{&handle_address_lookup, NULL,
2197GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_LOOKUP, 0@}, @{NULL, NULL, 0, 0@} @};
2198@end example
2199
2200
2201As shown, the first member of the struct in the first area is a callback
2202function, which is called to process the specified message types, given as the
2203third member. The second parameter is the closure for the callback function,
2204which is set to @code{NULL} in most cases, and the last parameter is the
2205expected size of the message of this type, usually we set it to 0 to accept
2206variable size, for special cases the exact size of the specified message also
2207can be set. In addition, the terminator sign depicted as @code{@{NULL, NULL, 0,
22080@}} is set in the last aera.
2209
2210@c ***************************************************************************
2211@node Server: Process request message
2212@subsubsection Server: Process request message
2213@c %**end of header
2214
2215After the initialization of transport service, the request message would be
2216processed. Before handling the main message data, the validity of this message
2217should be checked out, e.g., to check whether the size of message is correct.
2218@example
2219size = ntohs (message->size); if (size < sizeof (struct
2220AddressLookupMessage)) @{ GNUNET_break_op (0); GNUNET_SERVER_receive_done
2221(client, GNUNET_SYSERR); return; @}
2222@end example
2223
2224
2225Note that, opposite to the construction method of the request message in the
2226client, in the server the function @code{nothl} and @code{ntohs} should be
2227employed during the extraction of the data from the message, so that the data
2228in big endian order can be converted back into little endian order. See more in
2229detail please refer to Introduction of Big Endian and Little Endian.
2230
2231Moreover in this example, the name of the transport stored in the message is a
22320-terminated string, so we should also check whether the name of the transport
2233in the received message is 0-terminated:
2234@example
2235nameTransport = (const char *)
2236&address[addressLen]; if (nameTransport[size - sizeof (struct
2237AddressLookupMessage)
2238 - addressLen - 1] != '\0') @{ GNUNET_break_op
2239 (0); GNUNET_SERVER_receive_done (client,
2240 GNUNET_SYSERR); return; @}
2241@end example
2242
2243Here, @code{GNUNET_SERVER_receive_done} should be called to tell the service
2244that the request is done and can receive the next message. The argument
2245@code{GNUNET_SYSERR} here indicates that the service didn't understand the
2246request message, and the processing of this request would be terminated.
2247
2248In comparison to the aforementioned situation, when the argument is equal to
2249@code{GNUNET_OK}, the service would continue to process the requst message.
2250
2251@c ***************************************************************************
2252@node Server: Response to client
2253@subsubsection Server: Response to client
2254@c %**end of header
2255
2256Once the processing of current request is done, the server should give the
2257response to the client. A new @code{struct AddressLookupMessage} would be
2258produced by the server in a similar way as the client did and sent to the
2259client, but here the type should be
2260@code{GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_REPLY} rather than
2261@code{GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_LOOKUP} in client.
2262@example
2263struct
2264AddressLookupMessage *msg; size_t len = sizeof (struct AddressLookupMessage) +
2265addressLen + strlen (nameTrans) + 1; msg->header->size = htons (len);
2266msg->header->type = htons (GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_REPLY);
2267
2268// ...
2269
2270struct GNUNET_SERVER_TransmitContext *tc; tc =
2271GNUNET_SERVER_transmit_context_create (client);
2272GNUNET_SERVER_transmit_context_append_data (tc, NULL, 0,
2273GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_REPLY);
2274GNUNET_SERVER_transmit_context_run (tc, rtimeout);
2275@end example
2276
2277
2278Note that, there are also a number of other APIs provided to the service to
2279send the message.
2280
2281@c ***************************************************************************
2282@node Server: Notification of clients
2283@subsubsection Server: Notification of clients
2284@c %**end of header
2285
2286Often a service needs to (repeatedly) transmit notifications to a client or a
2287group of clients. In these cases, the client typically has once registered for
2288a set of events and then needs to receive a message whenever such an event
2289happens (until the client disconnects). The use of a notification context can
2290help manage message queues to clients and handle disconnects. Notification
2291contexts can be used to send individualized messages to a particular client or
2292to broadcast messages to a group of clients. An individualized notification
2293might look like this:
2294@example
2295 GNUNET_SERVER_notification_context_unicast(nc,
2296 client, msg, GNUNET_YES);
2297@end example
2298
2299
2300Note that after processing the original registration message for notifications,
2301the server code still typically needs to call@
2302@code{GNUNET_SERVER_receive_done} so that the client can transmit further
2303messages to the server.
2304
2305@c ***************************************************************************
2306@node Conversion between Network Byte Order (Big Endian) and Host Byte Order
2307@subsubsection Conversion between Network Byte Order (Big Endian) and Host Byte Order
2308@c %** subsub? it's a referenced page on the ipc document.
2309@c %**end of header
2310
2311Here we can simply comprehend big endian and little endian as Network Byte
2312Order and Host Byte Order respectively. What is the difference between both
2313two?
2314
2315Usually in our host computer we store the data byte as Host Byte Order, for
2316example, we store a integer in the RAM which might occupies 4 Byte, as Host
2317Byte Order the higher Byte would be stored at the lower address of RAM, and
2318the lower Byte would be stored at the higher address of RAM. However, contrast
2319to this, Network Byte Order just take the totally opposite way to store the
2320data, says, it will store the lower Byte at the lower address, and the higher
2321Byte will stay at higher address.
2322
2323For the current communication of network, we normally exchange the information
2324by surveying the data package, every two host wants to communicate with each
2325other must send and receive data package through network. In order to maintain
2326the identity of data through the transmission in the network, the order of the
2327Byte storage must changed before sending and after receiving the data.
2328
2329There ten convenient functions to realize the conversion of Byte Order in
2330GNUnet, as following:
2331@table @asis
2332
2333@item uint16_t htons(uint16_t hostshort) Convert host byte order to net byte
2334order with short int
2335@item uint32_t htonl(uint32_t hostlong) Convert host byte
2336order to net byte order with long int
2337@item uint16_t ntohs(uint16_t netshort)
2338Convert net byte order to host byte order with short int
2339@item uint32_t
2340ntohl(uint32_t netlong) Convert net byte order to host byte order with long int
2341@item unsigned long long GNUNET_ntohll (unsigned long long netlonglong) Convert
2342net byte order to host byte order with long long int
2343@item unsigned long long
2344GNUNET_htonll (unsigned long long hostlonglong) Convert host byte order to net
2345byte order with long long int
2346@item struct GNUNET_TIME_RelativeNBO
2347GNUNET_TIME_relative_hton (struct GNUNET_TIME_Relative a) Convert relative time
2348to network byte order.
2349@item struct GNUNET_TIME_Relative
2350GNUNET_TIME_relative_ntoh (struct GNUNET_TIME_RelativeNBO a) Convert relative
2351time from network byte order.
2352@item struct GNUNET_TIME_AbsoluteNBO
2353GNUNET_TIME_absolute_hton (struct GNUNET_TIME_Absolute a) Convert relative time
2354to network byte order.
2355@item struct GNUNET_TIME_Absolute
2356GNUNET_TIME_absolute_ntoh (struct GNUNET_TIME_AbsoluteNBO a) Convert relative
2357time from network byte order.
2358@end table
2359
2360@c ***************************************************************************
2361
2362@node Cryptography API
2363@subsection Cryptography API
2364@c %**end of header
2365
2366The gnunetutil APIs provides the cryptographic primitives used in GNUnet.
2367GNUnet uses 2048 bit RSA keys for the session key exchange and for signing
2368messages by peers and most other public-key operations. Most researchers in
2369cryptography consider 2048 bit RSA keys as secure and practically unbreakable
2370for a long time. The API provides functions to create a fresh key pair, read a
2371private key from a file (or create a new file if the file does not exist),
2372encrypt, decrypt, sign, verify and extraction of the public key into a format
2373suitable for network transmission.
2374
2375For the encryption of files and the actual data exchanged between peers GNUnet
2376uses 256-bit AES encryption. Fresh, session keys are negotiated for every new
2377connection.@ Again, there is no published technique to break this cipher in any
2378realistic amount of time. The API provides functions for generation of keys,
2379validation of keys (important for checking that decryptions using RSA
2380succeeded), encryption and decryption.
2381
2382GNUnet uses SHA-512 for computing one-way hash codes. The API provides
2383functions to compute a hash over a block in memory or over a file on disk.
2384
2385The crypto API also provides functions for randomizing a block of memory,
2386obtaining a single random number and for generating a permuation of the numbers
23870 to n-1. Random number generation distinguishes between WEAK and STRONG random
2388number quality; WEAK random numbers are pseudo-random whereas STRONG random
2389numbers use entropy gathered from the operating system.
2390
2391Finally, the crypto API provides a means to deterministically generate a
23921024-bit RSA key from a hash code. These functions should most likely not be
2393used by most applications; most importantly,@
2394GNUNET_CRYPTO_rsa_key_create_from_hash does not create an RSA-key that should
2395be considered secure for traditional applications of RSA.
2396
2397@c ***************************************************************************
2398@node Message Queue API
2399@subsection Message Queue API
2400@c %**end of header
2401
2402@strong{ Introduction }@ Often, applications need to queue messages that are to
2403be sent to other GNUnet peers, clients or services. As all of GNUnet's
2404message-based communication APIs, by design, do not allow messages to be
2405queued, it is common to implement custom message queues manually when they are
2406needed. However, writing very similar code in multiple places is tedious and
2407leads to code duplication.
2408
2409MQ (for Message Queue) is an API that provides the functionality to implement
2410and use message queues. We intend to eventually replace all of the custom
2411message queue implementations in GNUnet with MQ.
2412
2413@strong{ Basic Concepts }@ The two most important entities in MQ are queues and
2414envelopes.
2415
2416Every queue is backed by a specific implementation (e.g. for mesh, stream,
2417connection, server client, etc.) that will actually deliver the queued
2418messages. For convenience,@ some queues also allow to specify a list of message
2419handlers. The message queue will then also wait for incoming messages and
2420dispatch them appropriately.
2421
2422An envelope holds the the memory for a message, as well as metadata (Where is
2423the envelope queued? What should happen after it has been sent?). Any envelope
2424can only be queued in one message queue.
2425
2426@strong{ Creating Queues }@ The following is a list of currently available
2427message queues. Note that to avoid layering issues, message queues for higher
2428level APIs are not part of @code{libgnunetutil}, but@ the respective API itself
2429provides the queue implementation.
2430@table @asis
2431
2432@item @code{GNUNET_MQ_queue_for_connection_client} Transmits queued messages
2433over a @code{GNUNET_CLIENT_Connection}@ handle. Also supports receiving with
2434message handlers.@
2435
2436@item @code{GNUNET_MQ_queue_for_server_client} Transmits queued messages over a
2437@code{GNUNET_SERVER_Client}@ handle. Does not support incoming message
2438handlers.@
2439
2440@item @code{GNUNET_MESH_mq_create} Transmits queued messages over a
2441@code{GNUNET_MESH_Tunnel}@ handle. Does not support incoming message handlers.@
2442
2443@item @code{GNUNET_MQ_queue_for_callbacks} This is the most general
2444implementation. Instead of delivering and receiving messages with one of
2445GNUnet's communication APIs, implementation callbacks are called. Refer to
2446"Implementing Queues" for a more detailed explanation.
2447@end table
2448
2449
2450@strong{ Allocating Envelopes }@ A GNUnet message (as defined by the
2451GNUNET_MessageHeader) has three parts: The size, the type, and the body.
2452
2453MQ provides macros to allocate an envelope containing a message conveniently,@
2454automatically setting the size and type fields of the message.
2455
2456Consider the following simple message, with the body consisting of a single
2457number value.@ @code{}
2458@example
2459struct NumberMessage @{
2460 /** Type: GNUNET_MESSAGE_TYPE_EXAMPLE_1 */
2461 struct GNUNET_MessageHeader header; uint32_t number GNUNET_PACKED; @};
2462@end example
2463
2464An envelope containing an instance of the NumberMessage can be constructed like
2465this:
2466@example
2467struct GNUNET_MQ_Envelope *ev; struct NumberMessage *msg; ev =
2468GNUNET_MQ_msg (msg, GNUNET_MESSAGE_TYPE_EXAMPLE_1); msg->number = htonl (42);
2469@end example
2470
2471
2472In the above code, @code{GNUNET_MQ_msg} is a macro. The return value is the
2473newly allocated envelope. The first argument must be a pointer to some
2474@code{struct} containing a @code{struct GNUNET_MessageHeader header} field,
2475while the second argument is the desired message type, in host byte order.
2476
2477The @code{msg} pointer now points to an allocated message, where the message
2478type and the message size are already set. The message's size is inferred from
2479the type of the @code{msg} pointer: It will be set to 'sizeof(*msg)', properly
2480converted to network byte order.
2481
2482If the message body's size is dynamic, the the macro @code{GNUNET_MQ_msg_extra}
2483can be used to allocate an envelope whose message has additional space
2484allocated after the @code{msg} structure.
2485
2486If no structure has been defined for the message,
2487@code{GNUNET_MQ_msg_header_extra} can be used to allocate additional space
2488after the message header. The first argument then must be a pointer to a
2489@code{GNUNET_MessageHeader}.
2490
2491@strong{Envelope Properties}@ A few functions in MQ allow to set additional
2492properties on envelopes:
2493@table @asis
2494
2495@item @code{GNUNET_MQ_notify_sent} Allows to specify a function that will be
2496called once the envelope's message@ has been sent irrevocably. An envelope can
2497be canceled precisely up to the@ point where the notify sent callback has been
2498called.
2499@item @code{GNUNET_MQ_disable_corking} No corking will be used when
2500sending the message. Not every@ queue supports this flag, per default,
2501envelopes are sent with corking.@
2502
2503@end table
2504
2505
2506@strong{Sending Envelopes}@ Once an envelope has been constructed, it can be
2507queued for sending with @code{GNUNET_MQ_send}.
2508
2509Note that in order to avoid memory leaks, an envelope must either be sent (the
2510queue will free it) or destroyed explicitly with @code{GNUNET_MQ_discard}.
2511
2512@strong{Canceling Envelopes}@ An envelope queued with @code{GNUNET_MQ_send} can
2513be canceled with @code{GNUNET_MQ_cancel}. Note that after the notify sent
2514callback has been called, canceling a message results in undefined behavior.
2515Thus it is unsafe to cancel an envelope that does not have a notify sent
2516callback. When canceling an envelope, it is not necessary@ to call
2517@code{GNUNET_MQ_discard}, and the envelope can't be sent again.
2518
2519@strong{ Implementing Queues }@ @code{TODO}
2520
2521@c ***************************************************************************
2522@node Service API
2523@subsection Service API
2524@c %**end of header
2525
2526Most GNUnet code lives in the form of services. Services are processes that
2527offer an API for other components of the system to build on. Those other
2528components can be command-line tools for users, graphical user interfaces or
2529other services. Services provide their API using an IPC protocol. For this,
2530each service must listen on either a TCP port or a UNIX domain socket; for
2531this, the service implementation uses the server API. This use of server is
2532exposed directly to the users of the service API. Thus, when using the service
2533API, one is usually also often using large parts of the server API. The service
2534API provides various convenience functions, such as parsing command-line
2535arguments and the configuration file, which are not found in the server API.
2536The dual to the service/server API is the client API, which can be used to
2537access services.
2538
2539The most common way to start a service is to use the GNUNET_SERVICE_run
2540function from the program's main function. GNUNET_SERVICE_run will then parse
2541the command line and configuration files and, based on the options found there,
2542start the server. It will then give back control to the main program, passing
2543the server and the configuration to the GNUNET_SERVICE_Main callback.
2544GNUNET_SERVICE_run will also take care of starting the scheduler loop. If this
2545is inappropriate (for example, because the scheduler loop is already running),
2546GNUNET_SERVICE_start and related functions provide an alternative to
2547GNUNET_SERVICE_run.
2548
2549When starting a service, the service_name option is used to determine which
2550sections in the configuration file should be used to configure the service. A
2551typical value here is the name of the src/ sub-directory, for example
2552"statistics". The same string would also be given to GNUNET_CLIENT_connect to
2553access the service.
2554
2555Once a service has been initialized, the program should use the@
2556GNUNET_SERVICE_Main callback to register message handlers using
2557GNUNET_SERVER_add_handlers. The service will already have registered a handler
2558for the "TEST" message.
2559
2560The option bitfield (enum GNUNET_SERVICE_Options) determines how a service
2561should behave during shutdown. There are three key strategies:
2562@table @asis
2563
2564@item instant (GNUNET_SERVICE_OPTION_NONE) Upon receiving the shutdown signal
2565from the scheduler, the service immediately terminates the server, closing all
2566existing connections with clients.
2567@item manual
2568(GNUNET_SERVICE_OPTION_MANUAL_SHUTDOWN) The service does nothing by itself
2569during shutdown. The main program will need to take the appropriate action by
2570calling GNUNET_SERVER_destroy or GNUNET_SERVICE_stop (depending on how the
2571service was initialized) to terminate the service. This method is used by
2572gnunet-service-arm and rather uncommon.
2573@item soft
2574(GNUNET_SERVICE_OPTION_SOFT_SHUTDOWN) Upon receiving the shutdown signal from
2575the scheduler, the service immediately tells the server to stop listening for
2576incoming clients. Requests from normal existing clients are still processed and
2577the server/service terminates once all normal clients have disconnected.
2578Clients that are not expected to ever disconnect (such as clients that monitor
2579performance values) can be marked as 'monitor' clients using
2580GNUNET_SERVER_client_mark_monitor. Those clients will continue to be processed
2581until all 'normal' clients have disconnected. Then, the server will terminate,
2582closing the monitor connections. This mode is for example used by 'statistics',
2583allowing existing 'normal' clients to set (possibly persistent) statistic
2584values before terminating.
2585@end table
2586
2587@c ***************************************************************************
2588@node Optimizing Memory Consumption of GNUnet's (Multi-) Hash Maps
2589@subsection Optimizing Memory Consumption of GNUnet's (Multi-) Hash Maps
2590@c %**end of header
2591
2592A commonly used data structure in GNUnet is a (multi-)hash map. It is most
2593often used to map a peer identity to some data structure, but also to map
2594arbitrary keys to values (for example to track requests in the distributed hash
2595table or in file-sharing). As it is commonly used, the DHT is actually
2596sometimes responsible for a large share of GNUnet's overall memory consumption
2597(for some processes, 30% is not uncommon). The following text documents some
2598API quirks (and their implications for applications) that were recently
2599introduced to minimize the footprint of the hash map.
2600
2601
2602@c ***************************************************************************
2603@menu
2604* Analysis::
2605* Solution::
2606* Migration::
2607* Conclusion::
2608* Availability::
2609@end menu
2610
2611@node Analysis
2612@subsubsection Analysis
2613@c %**end of header
2614
2615The main reason for the "excessive" memory consumption by the hash map is that
2616GNUnet uses 512-bit cryptographic hash codes --- and the (multi-)hash map also
2617uses the same 512-bit 'struct GNUNET_HashCode'. As a result, storing just the
2618keys requires 64 bytes of memory for each key. As some applications like to
2619keep a large number of entries in the hash map (after all, that's what maps
2620are good for), 64 bytes per hash is significant: keeping a pointer to the
2621value and having a linked list for collisions consume between 8 and 16 bytes,
2622and 'malloc' may add about the same overhead per allocation, putting us in the
262316 to 32 byte per entry ballpark. Adding a 64-byte key then triples the
2624overall memory requirement for the hash map.
2625
2626To make things "worse", most of the time storing the key in the hash map is
2627not required: it is typically already in memory elsewhere! In most cases, the
2628values stored in the hash map are some application-specific struct that _also_
2629contains the hash. Here is a simplified example:
2630@example
2631struct MyValue @{
2632struct GNUNET_HashCode key; unsigned int my_data; @};
2633
2634// ...
2635val = GNUNET_malloc (sizeof (struct MyValue)); val->key = key; val->my_data =
263642; GNUNET_CONTAINER_multihashmap_put (map, &key, val, ...);
2637@end example
2638
2639
2640This is a common pattern as later the entries might need to be removed, and at
2641that time it is convenient to have the key immediately at hand:
2642@example
2643GNUNET_CONTAINER_multihashmap_remove (map, &val->key, val);
2644@end example
2645
2646
2647Note that here we end up with two times 64 bytes for the key, plus maybe 64
2648bytes total for the rest of the 'struct MyValue' and the map entry in the hash
2649map. The resulting redundant storage of the key increases overall memory
2650consumption per entry from the "optimal" 128 bytes to 192 bytes. This is not
2651just an extreme example: overheads in practice are actually sometimes close to
2652those highlighted in this example. This is especially true for maps with a
2653significant number of entries, as there we tend to really try to keep the
2654entries small.
2655@c ***************************************************************************
2656@node Solution
2657@subsubsection Solution
2658@c %**end of header
2659
2660The solution that has now been implemented is to @strong{optionally} allow the
2661hash map to not make a (deep) copy of the hash but instead have a pointer to
2662the hash/key in the entry. This reduces the memory consumption for the key
2663from 64 bytes to 4 to 8 bytes. However, it can also only work if the key is
2664actually stored in the entry (which is the case most of the time) and if the
2665entry does not modify the key (which in all of the code I'm aware of has been
2666always the case if there key is stored in the entry). Finally, when the client
2667stores an entry in the hash map, it @strong{must} provide a pointer to the key
2668within the entry, not just a pointer to a transient location of the key. If
2669the client code does not meet these requirements, the result is a dangling
2670pointer and undefined behavior of the (multi-)hash map API.
2671@c ***************************************************************************
2672@node Migration
2673@subsubsection Migration
2674@c %**end of header
2675
2676To use the new feature, first check that the values contain the respective key
2677(and never modify it). Then, all calls to
2678@code{GNUNET_CONTAINER_multihashmap_put} on the respective map must be audited
2679and most likely changed to pass a pointer into the value's struct. For the
2680initial example, the new code would look like this:
2681@example
2682struct MyValue @{
2683struct GNUNET_HashCode key; unsigned int my_data; @};
2684
2685// ...
2686val = GNUNET_malloc (sizeof (struct MyValue)); val->key = key; val->my_data =
268742; GNUNET_CONTAINER_multihashmap_put (map, &val->key, val, ...);
2688@end example
2689
2690
2691Note that @code{&val} was changed to @code{&val->key} in the argument to the
2692@code{put} call. This is critical as often @code{key} is on the stack or in
2693some other transient data structure and thus having the hash map keep a pointer
2694to @code{key} would not work. Only the key inside of @code{val} has the same
2695lifetime as the entry in the map (this must of course be checked as well).
2696Naturally, @code{val->key} must be intiialized before the @code{put} call. Once
2697all @code{put} calls have been converted and double-checked, you can change the
2698call to create the hash map from
2699@example
2700map =
2701GNUNET_CONTAINER_multihashmap_create (SIZE, GNUNET_NO);
2702@end example
2703
2704to
2705
2706@example
2707map = GNUNET_CONTAINER_multihashmap_create (SIZE, GNUNET_YES);
2708@end example
2709
2710If everything was done correctly, you now use about 60 bytes less memory per
2711entry in @code{map}. However, if now (or in the future) any call to @code{put}
2712does not ensure that the given key is valid until the entry is removed from the
2713map, undefined behavior is likely to be observed.
2714@c ***************************************************************************
2715@node Conclusion
2716@subsubsection Conclusion
2717@c %**end of header
2718
2719The new optimization can is often applicable and can result in a reduction in
2720memory consumption of up to 30% in practice. However, it makes the code less
2721robust as additional invariants are imposed on the multi hash map client. Thus
2722applications should refrain from enabling the new mode unless the resulting
2723performance increase is deemed significant enough. In particular, it should
2724generally not be used in new code (wait at least until benchmarks exist).
2725@c ***************************************************************************
2726@node Availability
2727@subsubsection Availability
2728@c %**end of header
2729
2730The new multi hash map code was committed in SVN 24319 (will be in GNUnet
27310.9.4). Various subsystems (transport, core, dht, file-sharing) were
2732previously audited and modified to take advantage of the new capability. In
2733particular, memory consumption of the file-sharing service is expected to drop
2734by 20-30% due to this change.
2735
2736@c ***************************************************************************
2737@node The CONTAINER_MDLL API
2738@subsection The CONTAINER_MDLL API
2739@c %**end of header
2740
2741This text documents the GNUNET_CONTAINER_MDLL API. The GNUNET_CONTAINER_MDLL
2742API is similar to the GNUNET_CONTAINER_DLL API in that it provides operations
2743for the construction and manipulation of doubly-linked lists. The key
2744difference to the (simpler) DLL-API is that the MDLL-version allows a single
2745element (instance of a "struct") to be in multiple linked lists at the same
2746time.
2747
2748Like the DLL API, the MDLL API stores (most of) the data structures for the
2749doubly-linked list with the respective elements; only the 'head' and 'tail'
2750pointers are stored "elsewhere" --- and the application needs to provide the
2751locations of head and tail to each of the calls in the MDLL API. The key
2752difference for the MDLL API is that the "next" and "previous" pointers in the
2753struct can no longer be simply called "next" and "prev" --- after all, the
2754element may be in multiple doubly-linked lists, so we cannot just have one
2755"next" and one "prev" pointer!
2756
2757The solution is to have multiple fields that must have a name of the format
2758"next_XX" and "prev_XX" where "XX" is the name of one of the doubly-linked
2759lists. Here is a simple example:
2760@example
2761struct MyMultiListElement @{ struct
2762MyMultiListElement *next_ALIST; struct MyMultiListElement *prev_ALIST; struct
2763MyMultiListElement *next_BLIST; struct MyMultiListElement *prev_BLIST; void
2764*data; @};
2765@end example
2766
2767
2768Note that by convention, we use all-uppercase letters for the list names. In
2769addition, the program needs to have a location for the head and tail pointers
2770for both lists, for example:
2771@example
2772static struct MyMultiListElement
2773*head_ALIST; static struct MyMultiListElement *tail_ALIST; static struct
2774MyMultiListElement *head_BLIST; static struct MyMultiListElement *tail_BLIST;
2775@end example
2776
2777
2778Using the MDLL-macros, we can now insert an element into the ALIST:
2779@example
2780GNUNET_CONTAINER_MDLL_insert (ALIST, head_ALIST, tail_ALIST, element);
2781@end example
2782
2783
2784Passing "ALIST" as the first argument to MDLL specifies which of the next/prev
2785fields in the 'struct MyMultiListElement' should be used. The extra "ALIST"
2786argument and the "_ALIST" in the names of the next/prev-members are the only
2787differences between the MDDL and DLL-API. Like the DLL-API, the MDLL-API offers
2788functions for inserting (at head, at tail, after a given element) and removing
2789elements from the list. Iterating over the list should be done by directly
2790accessing the "next_XX" and/or "prev_XX" members.
2791
2792@c ***************************************************************************
2793@node The Automatic Restart Manager (ARM)
2794@section The Automatic Restart Manager (ARM)
2795@c %**end of header
2796
2797GNUnet's Automated Restart Manager (ARM) is the GNUnet service responsible for
2798system initialization and service babysitting. ARM starts and halts services,
2799detects configuration changes and restarts services impacted by the changes as
2800needed. It's also responsible for restarting services in case of crashes and is
2801planned to incorporate automatic debugging for diagnosing service crashes
2802providing developers insights about crash reasons. The purpose of this document
2803is to give GNUnet developer an idea about how ARM works and how to interact
2804with it.
2805
2806@menu
2807* Basic functionality::
2808* Key configuration options::
2809* Availability2::
2810* Reliability::
2811@end menu
2812
2813@c ***************************************************************************
2814@node Basic functionality
2815@subsection Basic functionality
2816@c %**end of header
2817
2818@itemize @bullet
2819@item ARM source code can be found under "src/arm".@ Service processes are
2820managed by the functions in "gnunet-service-arm.c" which is controlled with
2821"gnunet-arm.c" (main function in that file is ARM's entry point).
2822
2823@item The functions responsible for communicating with ARM , starting and
2824stopping services -including ARM service itself- are provided by the ARM API
2825"arm_api.c".@ Function: GNUNET_ARM_connect() returns to the caller an ARM
2826handle after setting it to the caller's context (configuration and scheduler in
2827use). This handle can be used afterwards by the caller to communicate with ARM.
2828Functions GNUNET_ARM_start_service() and GNUNET_ARM_stop_service() are used for
2829starting and stopping services respectively.
2830
2831@item A typical example of using these basic ARM services can be found in file
2832test_arm_api.c. The test case connects to ARM, starts it, then uses it to start
2833a service "resolver", stops the "resolver" then stops "ARM".
2834@end itemize
2835
2836@c ***************************************************************************
2837@node Key configuration options
2838@subsection Key configuration options
2839@c %**end of header
2840
2841Configurations for ARM and services should be available in a .conf file (As an
2842example, see test_arm_api_data.conf). When running ARM, the configuration file
2843to use should be passed to the command:@ @code{@ $ gnunet-arm -s -c
2844configuration_to_use.conf@ }@ If no configuration is passed, the default
2845configuration file will be used (see GNUNET_PREFIX/share/gnunet/defaults.conf
2846which is created from contrib/defaults.conf).@ Each of the services is having a
2847section starting by the service name between square brackets, for example:
2848"[arm]". The following options configure how ARM configures or interacts with
2849the various services:
2850
2851@table @asis
2852
2853@item PORT Port number on which the service is listening for incoming TCP
2854connections. ARM will start the services should it notice a request at this
2855port.
2856
2857@item HOSTNAME Specifies on which host the service is deployed. Note
2858that ARM can only start services that are running on the local system (but will
2859not check that the hostname matches the local machine name). This option is
2860used by the @code{gnunet_client_lib.h} implementation to determine which system
2861to connect to. The default is "localhost".
2862
2863@item BINARY The name of the service binary file.
2864
2865@item OPTIONS To be passed to the service.
2866
2867@item PREFIX A command to pre-pend to the actual command, for example, running
2868a service with "valgrind" or "gdb"
2869
2870@item DEBUG Run in debug mode (much verbosity).
2871
2872@item AUTOSTART ARM will listen to UNIX domain socket and/or TCP port of the
2873service and start the service on-demand.
2874
2875@item FORCESTART ARM will always
2876start this service when the peer is started.
2877
2878@item ACCEPT_FROM IPv4 addresses the service accepts connections from.
2879
2880@item ACCEPT_FROM6 IPv6 addresses the service accepts connections from.
2881
2882@end table
2883
2884
2885Options that impact the operation of ARM overall are in the "[arm]" section.
2886ARM is a normal service and has (except for AUTOSTART) all of the options that
2887other services do. In addition, ARM has the following options:
2888@table @asis
2889
2890@item GLOBAL_PREFIX Command to be pre-pended to all services that are going to
2891run.@
2892
2893@item GLOBAL_POSTFIX Global option that will be supplied to all the services
2894that are going to run.@
2895
2896@end table
2897
2898@c ***************************************************************************
2899@node Availability2
2900@subsection Availability2
2901@c %**end of header
2902
2903As mentioned before, one of the features provided by ARM is starting services
2904on demand. Consider the example of one service "client" that wants to connect
2905to another service a "server". The "client" will ask ARM to run the "server".
2906ARM starts the "server". The "server" starts listening to incoming connections.
2907The "client" will establish a connection with the "server". And then, they will
2908start to communicate together.@ One problem with that scheme is that it's
2909slow!@ The "client" service wants to communicate with the "server" service at
2910once and is not willing wait for it to be started and listening to incoming
2911connections before serving its request.@ One solution for that problem will be
2912that ARM starts all services as default services. That solution will solve the
2913problem, yet, it's not quite practical, for some services that are going to be
2914started can never be used or are going to be used after a relatively long
2915time.@ The approach followed by ARM to solve this problem is as follows:
2916@itemize @bullet
2917
2918
2919@item For each service having a PORT field in the configuration file and that
2920is not one of the default services ( a service that accepts incoming
2921connections from clients), ARM creates listening sockets for all addresses
2922associated with that service.
2923
2924@item The "client" will immediately establish a connection with the "server".
2925
2926@item ARM --- pretending to be the "server" --- will listen on the respective
2927port and notice the incoming connection from the "client" (but not accept it),
2928instead
2929
2930@item Once there is an incoming connection, ARM will start the "server",
2931passing on the listen sockets (now, the service is started and can do its
2932work).
2933
2934@item Other client services now can directly connect directly to the "server".
2935@end itemize
2936
2937@c ***************************************************************************
2938@node Reliability
2939@subsection Reliability
2940
2941One of the features provided by ARM, is the automatic restart of crashed
2942services.@ ARM needs to know which of the running services died. Function
2943"gnunet-service-arm.c/maint_child_death()" is responsible for that. The
2944function is scheduled to run upon receiving a SIGCHLD signal. The function,
2945then, iterates ARM's list of services running and monitors which service has
2946died (crashed). For all crashing services, ARM restarts them.@ Now, considering
2947the case of a service having a serious problem causing it to crash each time
2948it's started by ARM. If ARM keeps blindly restarting such a service, we are
2949going to have the pattern: start-crash-restart-crash-restart-crash and so
2950forth!! Which is of course not practical.@ For that reason, ARM schedules the
2951service to be restarted after waiting for some delay that grows exponentially
2952with each crash/restart of that service.@ To clarify the idea, considering the
2953following example:
2954@itemize @bullet
2955
2956
2957@item Service S crashed.
2958
2959@item ARM receives the SIGCHLD and inspects its list of services to find the
2960dead one(s).
2961
2962@item ARM finds S dead and schedules it for restarting after "backoff" time
2963which is initially set to 1ms. ARM will double the backoff time correspondent
2964to S (now backoff(S) = 2ms)
2965
2966@item Because there is a severe problem with S, it crashed again.
2967
2968@item Again ARM receives the SIGCHLD and detects that it's S again that's
2969crashed. ARM schedules it for restarting but after its new backoff time (which
2970became 2ms), and doubles its backoff time (now backoff(S) = 4).
2971
2972@item and so on, until backoff(S) reaches a certain threshold
2973(EXPONENTIAL_BACKOFF_THRESHOLD is set to half an hour), after reaching it,
2974backoff(S) will remain half an hour, hence ARM won't be busy for a lot of time
2975trying to restart a problematic service.
2976@end itemize
2977
2978@c ***************************************************************************
2979@node GNUnet's TRANSPORT Subsystem
2980@section GNUnet's TRANSPORT Subsystem
2981@c %**end of header
2982
2983This chapter documents how the GNUnet transport subsystem works. The GNUnet
2984transport subsystem consists of three main components: the transport API (the
2985interface used by the rest of the system to access the transport service), the
2986transport service itself (most of the interesting functions, such as choosing
2987transports, happens here) and the transport plugins. A transport plugin is a
2988concrete implementation for how two GNUnet peers communicate; many plugins
2989exist, for example for communication via TCP, UDP, HTTP, HTTPS and others.
2990Finally, the transport subsystem uses supporting code, especially the NAT/UPnP
2991library to help with tasks such as NAT traversal.
2992
2993Key tasks of the transport service include:
2994@itemize @bullet
2995
2996
2997@item Create our HELLO message, notify clients and neighbours if our HELLO
2998changes (using NAT library as necessary)
2999
3000@item Validate HELLOs from other peers (send PING), allow other peers to
3001validate our HELLO's addresses (send PONG)
3002
3003@item Upon request, establish connections to other peers (using address
3004selection from ATS subsystem) and maintain them (again using PINGs and PONGs)
3005as long as desired
3006
3007@item Accept incoming connections, give ATS service the opportunity to switch
3008communication channels
3009
3010@item Notify clients about peers that have connected to us or that have been
3011disconnected from us
3012
3013@item If a (stateful) connection goes down unexpectedly (without explicit
3014DISCONNECT), quickly attempt to recover (without notifying clients) but do
3015notify clients quickly if reconnecting fails
3016
3017@item Send (payload) messages arriving from clients to other peers via
3018transport plugins and receive messages from other peers, forwarding those to
3019clients
3020
3021@item Enforce inbound traffic limits (using flow-control if it is applicable);
3022outbound traffic limits are enforced by CORE, not by us (!)
3023
3024@item Enforce restrictions on P2P connection as specified by the blacklist
3025configuration and blacklisting clients
3026@end itemize
3027
3028
3029Note that the term "clients" in the list above really refers to the GNUnet-CORE
3030service, as CORE is typically the only client of the transport service.
3031
3032@menu
3033* Address validation protocol::
3034@end menu
3035
3036@node Address validation protocol
3037@subsection Address validation protocol
3038@c %**end of header
3039
3040This section documents how the GNUnet transport service validates connections
3041with other peers. It is a high-level description of the protocol necessary to
3042understand the details of the implementation. It should be noted that when we
3043talk about PING and PONG messages in this section, we refer to transport-level
3044PING and PONG messages, which are different from core-level PING and PONG
3045messages (both in implementation and function).
3046
3047The goal of transport-level address validation is to minimize the chances of a
3048successful man-in-the-middle attack against GNUnet peers on the transport
3049level. Such an attack would not allow the adversary to decrypt the P2P
3050transmissions, but a successful attacker could at least measure traffic volumes
3051and latencies (raising the adversaries capablities by those of a global passive
3052adversary in the worst case). The scenarios we are concerned about is an
3053attacker, Mallory, giving a HELLO to Alice that claims to be for Bob, but
3054contains Mallory's IP address instead of Bobs (for some transport). Mallory
3055would then forward the traffic to Bob (by initiating a connection to Bob and
3056claiming to be Alice). As a further complication, the scheme has to work even
3057if say Alice is behind a NAT without traversal support and hence has no address
3058of her own (and thus Alice must always initiate the connection to Bob).
3059
3060An additional constraint is that HELLO messages do not contain a cryptographic
3061signature since other peers must be able to edit (i.e. remove) addresses from
3062the HELLO at any time (this was not true in GNUnet 0.8.x). A basic
3063@strong{assumption} is that each peer knows the set of possible network
3064addresses that it @strong{might} be reachable under (so for example, the
3065external IP address of the NAT plus the LAN address(es) with the respective
3066ports).
3067
3068The solution is the following. If Alice wants to validate that a given address
3069for Bob is valid (i.e. is actually established @strong{directly} with the
3070intended target), it sends a PING message over that connection to Bob. Note
3071that in this case, Alice initiated the connection so only she knows which
3072address was used for sure (Alice maybe behind NAT, so whatever address Bob
3073sees may not be an address Alice knows she has). Bob checks that the address
3074given in the PING is actually one of his addresses (does not belong to
3075Mallory), and if it is, sends back a PONG (with a signature that says that Bob
3076owns/uses the address from the PING). Alice checks the signature and is happy
3077if it is valid and the address in the PONG is the address she used. This is
3078similar to the 0.8.x protocol where the HELLO contained a signature from Bob
3079for each address used by Bob. Here, the purpose code for the signature is
3080@code{GNUNET_SIGNATURE_PURPOSE_TRANSPORT_PONG_OWN}. After this, Alice will
3081remember Bob's address and consider the address valid for a while (12h in the
3082current implementation). Note that after this exchange, Alice only considers
3083Bob's address to be valid, the connection itself is not considered
3084'established'. In particular, Alice may have many addresses for Bob that she
3085considers valid.
3086
3087The PONG message is protected with a nonce/challenge against replay attacks
3088and uses an expiration time for the signature (but those are almost
3089implementation details).
3090
3091@node NAT library
3092@section NAT library
3093@c %**end of header
3094
3095The goal of the GNUnet NAT library is to provide a general-purpose API for NAT
3096traversal @strong{without} third-party support. So protocols that involve
3097contacting a third peer to help establish a connection between two peers are
3098outside of the scope of this API. That does not mean that GNUnet doesn't
3099support involving a third peer (we can do this with the distance-vector
3100transport or using application-level protocols), it just means that the NAT API
3101is not concerned with this possibility. The API is written so that it will work
3102for IPv6-NAT in the future as well as current IPv4-NAT. Furthermore, the NAT
3103API is always used, even for peers that are not behind NAT --- in that case,
3104the mapping provided is simply the identity.
3105
3106NAT traversal is initiated by calling @code{GNUNET_NAT_register}. Given a set
3107of addresses that the peer has locally bound to (TCP or UDP), the NAT library
3108will return (via callback) a (possibly longer) list of addresses the peer
3109@strong{might} be reachable under. Internally, depending on the configuration,
3110the NAT library will try to punch a hole (using UPnP) or just "know" that the
3111NAT was manually punched and generate the respective external IP address (the
3112one that should be globally visible) based on the given information.
3113
3114The NAT library also supports ICMP-based NAT traversal. Here, the other peer
3115can request connection-reversal by this peer (in this special case, the peer is
3116even allowed to configure a port number of zero). If the NAT library detects a
3117connection-reversal request, it returns the respective target address to the
3118client as well. It should be noted that connection-reversal is currently only
3119intended for TCP, so other plugins @strong{must} pass @code{NULL} for the
3120reversal callback. Naturally, the NAT library also supports requesting
3121connection reversal from a remote peer (@code{GNUNET_NAT_run_client}).
3122
3123Once initialized, the NAT handle can be used to test if a given address is
3124possibly a valid address for this peer (@code{GNUNET_NAT_test_address}). This
3125is used for validating our addresses when generating PONGs.
3126
3127Finally, the NAT library contains an API to test if our NAT configuration is
3128correct. Using @code{GNUNET_NAT_test_start} @strong{before} binding to the
3129respective port, the NAT library can be used to test if the configuration
3130works. The test function act as a local client, initialize the NAT traversal
3131and then contact a @code{gnunet-nat-server} (running by default on
3132@code{gnunet.org}) and ask for a connection to be established. This way, it is
3133easy to test if the current NAT configuration is valid.
3134
3135@node Distance-Vector plugin
3136@section Distance-Vector plugin
3137@c %**end of header
3138
3139The Distance Vector (DV) transport is a transport mechanism that allows peers
3140to act as relays for each other, thereby connecting peers that would otherwise
3141be unable to connect. This gives a larger connection set to applications that
3142may work better with more peers to choose from (for example, File Sharing
3143and/or DHT).
3144
3145The Distance Vector transport essentially has two functions. The first is
3146"gossiping" connection information about more distant peers to directly
3147connected peers. The second is taking messages intended for non-directly
3148connected peers and encapsulating them in a DV wrapper that contains the
3149required information for routing the message through forwarding peers. Via
3150gossiping, optimal routes through the known DV neighborhood are discovered and
3151utilized and the message encapsulation provides some benefits in addition to
3152simply getting the message from the correct source to the proper destination.
3153
3154The gossiping function of DV provides an up to date routing table of peers that
3155are available up to some number of hops. We call this a fisheye view of the
3156network (like a fish, nearby objects are known while more distant ones
3157unknown). Gossip messages are sent only to directly connected peers, but they
3158are sent about other knowns peers within the "fisheye distance". Whenever two
3159peers connect, they immediately gossip to each other about their appropriate
3160other neighbors. They also gossip about the newly connected peer to previously
3161connected neighbors. In order to keep the routing tables up to date, disconnect
3162notifications are propogated as gossip as well (because disconnects may not be
3163sent/received, timeouts are also used remove stagnant routing table entries).
3164
3165Routing of messages via DV is straightforward. When the DV transport is
3166notified of a message destined for a non-direct neighbor, the appropriate
3167forwarding peer is selected, and the base message is encapsulated in a DV
3168message which contains information about the initial peer and the intended
3169recipient. At each forwarding hop, the initial peer is validated (the
3170forwarding peer ensures that it has the initial peer in its neighborhood,
3171otherwise the message is dropped). Next the base message is re-encapsulated in
3172a new DV message for the next hop in the forwarding chain (or delivered to the
3173current peer, if it has arrived at the destination).
3174
3175Assume a three peer network with peers Alice, Bob and Carol. Assume that Alice
3176<-> Bob and Bob <-> Carol are direct (e.g. over TCP or UDP transports)
3177connections, but that Alice cannot directly connect to Carol. This may be the
3178case due to NAT or firewall restrictions, or perhaps based on one of the peers
3179respective configurations. If the Distance Vector transport is enabled on all
3180three peers, it will automatically discover (from the gossip protocol) that
3181Alice and Carol can connect via Bob and provide a "virtual" Alice <-> Carol
3182connection. Routing between Alice and Carol happens as follows; Alice creates a
3183message destined for Carol and notifies the DV transport about it. The DV
3184transport at Alice looks up Carol in the routing table and finds that the
3185message must be sent through Bob for Carol. The message is encapsulated setting
3186Alice as the initiator and Carol as the destination and sent to Bob. Bob
3187receives the messages, verifies both Alice and Carol are known to Bob, and
3188re-wraps the message in a new DV message for Carol. The DV transport at Carol
3189receives this message, unwraps the original message, and delivers it to Carol
3190as though it came directly from Alice.
3191
3192@node SMTP plugin
3193@section SMTP plugin
3194@c %**end of header
3195
3196This page describes the new SMTP transport plugin for GNUnet as it exists in
3197the 0.7.x and 0.8.x branch. SMTP support is currently not available in GNUnet
31980.9.x. This page also describes the transport layer abstraction (as it existed
3199in 0.7.x and 0.8.x) in more detail and gives some benchmarking results. The
3200performance results presented are quite old and maybe outdated at this point.
3201@itemize @bullet
3202@item Why use SMTP for a peer-to-peer transport?
3203@item SMTPHow does it work?
3204@item How do I configure my peer?
3205@item How do I test if it works?
3206@item How fast is it?
3207@item Is there any additional documentation?
3208@end itemize
3209
3210
3211@menu
3212* Why use SMTP for a peer-to-peer transport?::
3213* How does it work?::
3214* How do I configure my peer?::
3215* How do I test if it works?::
3216* How fast is it?::
3217@end menu
3218
3219@node Why use SMTP for a peer-to-peer transport?
3220@subsection Why use SMTP for a peer-to-peer transport?
3221@c %**end of header
3222
3223There are many reasons why one would not want to use SMTP:
3224@itemize @bullet
3225@item SMTP is using more bandwidth than TCP, UDP or HTTP
3226@item SMTP has a much higher latency.
3227@item SMTP requires significantly more computation (encoding and decoding time)
3228for the peers.
3229@item SMTP is significantly more complicated to configure.
3230@item SMTP may be abused by tricking GNUnet into sending mail to@
3231non-participating third parties.
3232@end itemize
3233
3234So why would anybody want to use SMTP?
3235@itemize @bullet
3236@item SMTP can be used to contact peers behind NAT boxes (in virtual private
3237networks).
3238@item SMTP can be used to circumvent policies that limit or prohibit
3239peer-to-peer traffic by masking as "legitimate" traffic.
3240@item SMTP uses E-mail addresses which are independent of a specific IP, which
3241can be useful to address peers that use dynamic IP addresses.
3242@item SMTP can be used to initiate a connection (e.g. initial address exchange)
3243and peers can then negotiate the use of a more efficient protocol (e.g. TCP)
3244for the actual communication.
3245@end itemize
3246
3247In summary, SMTP can for example be used to send a message to a peer behind a
3248NAT box that has a dynamic IP to tell the peer to establish a TCP connection
3249to a peer outside of the private network. Even an extraordinary overhead for
3250this first message would be irrelevant in this type of situation.
3251
3252@node How does it work?
3253@subsection How does it work?
3254@c %**end of header
3255
3256When a GNUnet peer needs to send a message to another GNUnet peer that has
3257advertised (only) an SMTP transport address, GNUnet base64-encodes the message
3258and sends it in an E-mail to the advertised address. The advertisement
3259contains a filter which is placed in the E-mail header, such that the
3260receiving host can filter the tagged E-mails and forward it to the GNUnet peer
3261process. The filter can be specified individually by each peer and be changed
3262over time. This makes it impossible to censor GNUnet E-mail messages by
3263searching for a generic filter.
3264
3265@node How do I configure my peer?
3266@subsection How do I configure my peer?
3267@c %**end of header
3268
3269First, you need to configure @code{procmail} to filter your inbound E-mail for
3270GNUnet traffic. The GNUnet messages must be delivered into a pipe, for example
3271@code{/tmp/gnunet.smtp}. You also need to define a filter that is used by
3272procmail to detect GNUnet messages. You are free to choose whichever filter
3273you like, but you should make sure that it does not occur in your other
3274E-mail. In our example, we will use @code{X-mailer: GNUnet}. The
3275@code{~/.procmailrc} configuration file then looks like this:
3276@example
3277:0:
3278* ^X-mailer: GNUnet
3279/tmp/gnunet.smtp
3280# where do you want your other e-mail delivered to (default: /var/spool/mail/)
3281:0: /var/spool/mail/
3282@end example
3283
3284After adding this file, first make sure that your regular E-mail still works
3285(e.g. by sending an E-mail to yourself). Then edit the GNUnet configuration.
3286In the section @code{SMTP} you need to specify your E-mail address under
3287@code{EMAIL}, your mail server (for outgoing mail) under @code{SERVER}, the
3288filter (X-mailer: GNUnet in the example) under @code{FILTER} and the name of
3289the pipe under @code{PIPE}.@ The completed section could then look like this:
3290@example
3291EMAIL = me@@mail.gnu.org MTU = 65000 SERVER = mail.gnu.org:25 FILTER =
3292"X-mailer: GNUnet" PIPE = /tmp/gnunet.smtp
3293@end example
3294
3295Finally, you need to add @code{smtp} to the list of @code{TRANSPORTS} in the
3296@code{GNUNETD} section. GNUnet peers will use the E-mail address that you
3297specified to contact your peer until the advertisement times out. Thus, if you
3298are not sure if everything works properly or if you are not planning to be
3299online for a long time, you may want to configure this timeout to be short,
3300e.g. just one hour. For this, set @code{HELLOEXPIRES} to @code{1} in the
3301@code{GNUNETD} section.
3302
3303This should be it, but you may probably want to test it first.@
3304@node How do I test if it works?
3305@subsection How do I test if it works?
3306@c %**end of header
3307
3308Any transport can be subjected to some rudimentary tests using the
3309@code{gnunet-transport-check} tool. The tool sends a message to the local node
3310via the transport and checks that a valid message is received. While this test
3311does not involve other peers and can not check if firewalls or other network
3312obstacles prohibit proper operation, this is a great testcase for the SMTP
3313transport since it tests pretty much nearly all of the functionality.
3314
3315@code{gnunet-transport-check} should only be used without running
3316@code{gnunetd} at the same time. By default, @code{gnunet-transport-check}
3317tests all transports that are specified in the configuration file. But you can
3318specifically test SMTP by giving the option @code{--transport=smtp}.
3319
3320Note that this test always checks if a transport can receive and send. While
3321you can configure most transports to only receive or only send messages, this
3322test will only work if you have configured the transport to send and receive
3323messages.
3324
3325@node How fast is it?
3326@subsection How fast is it?
3327@c %**end of header
3328
3329We have measured the performance of the UDP, TCP and SMTP transport layer
3330directly and when used from an application using the GNUnet core. Measureing
3331just the transport layer gives the better view of the actual overhead of the
3332protocol, whereas evaluating the transport from the application puts the
3333overhead into perspective from a practical point of view.
3334
3335The loopback measurements of the SMTP transport were performed on three
3336different machines spanning a range of modern SMTP configurations. We used a
3337PIII-800 running RedHat 7.3 with the Purdue Computer Science configuration
3338which includes filters for spam. We also used a Xenon 2 GHZ with a vanilla
3339RedHat 8.0 sendmail configuration. Furthermore, we used qmail on a PIII-1000
3340running Sorcerer GNU Linux (SGL). The numbers for UDP and TCP are provided
3341using the SGL configuration. The qmail benchmark uses qmail's internal
3342filtering whereas the sendmail benchmarks relies on procmail to filter and
3343deliver the mail. We used the transport layer to send a message of b bytes
3344(excluding transport protocol headers) directly to the local machine. This
3345way, network latency and packet loss on the wire have no impact on the
3346timings. n messages were sent sequentially over the transport layer, sending
3347message i+1 after the i-th message was received. All messages were sent over
3348the same connection and the time to establish the connection was not taken
3349into account since this overhead is miniscule in practice --- as long as a
3350connection is used for a significant number of messages.
3351
3352@multitable @columnfractions .20 .15 .15 .15 .15 .15
3353@headitem Transport @tab UDP @tab TCP @tab SMTP (Purdue sendmail) @tab SMTP (RH 8.0) @tab SMTP (SGL qmail)
3354@item 11 bytes @tab 31 ms @tab 55 ms @tab 781 s @tab 77 s @tab 24 s
3355@item 407 bytes @tab 37 ms @tab 62 ms @tab 789 s @tab 78 s @tab 25 s
3356@item 1,221 bytes @tab 46 ms @tab 73 ms @tab 804 s @tab 78 s @tab 25 s
3357@end multitable
3358
3359The benchmarks show that UDP and TCP are, as expected, both significantly
3360faster compared with any of the SMTP services. Among the SMTP implementations,
3361there can be significant differences depending on the SMTP configuration.
3362Filtering with an external tool like procmail that needs to re-parse its
3363configuration for each mail can be very expensive. Applying spam filters can
3364also significantly impact the performance of the underlying SMTP
3365implementation. The microbenchmark shows that SMTP can be a viable solution
3366for initiating peer-to-peer sessions: a couple of seconds to connect to a peer
3367are probably not even going to be noticed by users. The next benchmark
3368measures the possible throughput for a transport. Throughput can be measured
3369by sending multiple messages in parallel and measuring packet loss. Note that
3370not only UDP but also the TCP transport can actually loose messages since the
3371TCP implementation drops messages if the @code{write} to the socket would
3372block. While the SMTP protocol never drops messages itself, it is often so
3373slow that only a fraction of the messages can be sent and received in the
3374given time-bounds. For this benchmark we report the message loss after
3375allowing t time for sending m messages. If messages were not sent (or
3376received) after an overall timeout of t, they were considered lost. The
3377benchmark was performed using two Xeon 2 GHZ machines running RedHat 8.0 with
3378sendmail. The machines were connected with a direct 100 MBit ethernet
3379connection.@ Figures udp1200, tcp1200 and smtp-MTUs show that the throughput
3380for messages of size 1,200 octects is 2,343 kbps, 3,310 kbps and 6 kbps for
3381UDP, TCP and SMTP respectively. The high per-message overhead of SMTP can be
3382improved by increasing the MTU, for example, an MTU of 12,000 octets improves
3383the throughput to 13 kbps as figure smtp-MTUs shows. Our research paper) has
3384some more details on the benchmarking results.
3385
3386@node Bluetooth plugin
3387@section Bluetooth plugin
3388@c %**end of header
3389
3390This page describes the new Bluetooth transport plugin for GNUnet. The plugin
3391is still in the testing stage so don't expect it to work perfectly. If you
3392have any questions or problems just post them here or ask on the IRC channel.
3393@itemize @bullet
3394@item What do I need to use the Bluetooth plugin transport?
3395@item BluetoothHow does it work?
3396@item What possible errors should I be aware of?
3397@item How do I configure my peer?
3398@item How can I test it?
3399@end itemize
3400
3401
3402
3403@menu
3404* What do I need to use the Bluetooth plugin transport?::
3405* How does it work2?::
3406* What possible errors should I be aware of?::
3407* How do I configure my peer2?::
3408* How can I test it?::
3409* The implementation of the Bluetooth transport plugin::
3410@end menu
3411
3412@node What do I need to use the Bluetooth plugin transport?
3413@subsection What do I need to use the Bluetooth plugin transport?
3414@c %**end of header
3415
3416If you are a Linux user and you want to use the Bluetooth transport plugin you
3417should install the BlueZ development libraries (if they aren't already
3418installed). For instructions about how to install the libraries you should
3419check out the BlueZ site (@uref{http://www.bluez.org/, http://www.bluez.org}).
3420If you don't know if you have the necesarry libraries, don't worry, just run
3421the GNUnet configure script and you will be able to see a notification at the
3422end which will warn you if you don't have the necessary libraries.
3423
3424If you are a Windows user you should have installed the
3425@emph{MinGW}/@emph{MSys2} with the latest updates (especially the
3426@emph{ws2bth} header). If this is your first build of GNUnet on Windows you
3427should check out the SBuild repository. It will semi-automatically assembles a
3428@emph{MinGW}/@emph{MSys2} installation with a lot of extra packages which are
3429needed for the GNUnet build. So this will ease your work!@ Finally you just
3430have to be sure that you have the correct drivers for your Bluetooth device
3431installed and that your device is on and in a discoverable mode. The Windows
3432Bluetooth Stack supports only the RFCOMM protocol so we cannot turn on your
3433device programatically!
3434
3435@node How does it work2?
3436@subsection How does it work2?
3437@c %**end of header
3438
3439The Bluetooth transport plugin uses virtually the same code as the WLAN plugin
3440and only the helper binary is different. The helper takes a single argument,
3441which represents the interface name and is specified in the configuration
3442file. Here are the basic steps that are followed by the helper binary used on
3443Linux:
3444
3445@itemize @bullet
3446@item it verifies if the name corresponds to a Bluetooth interface name
3447@item it verifies if the iterface is up (if it is not, it tries to bring it up)
3448@item it tries to enable the page and inquiry scan in order to make the device
3449discoverable and to accept incoming connection requests
3450@emph{The above operations require root access so you should start the
3451transport plugin with root privileges.}
3452@item it finds an available port number and registers a SDP service which will
3453be used to find out on which port number is the server listening on and switch
3454the socket in listening mode
3455@item it sends a HELLO message with its address
3456@item finally it forwards traffic from the reading sockets to the STDOUT and
3457from the STDIN to the writing socket
3458@end itemize
3459
3460Once in a while the device will make an inquiry scan to discover the nearby
3461devices and it will send them randomly HELLO messages for peer discovery.
3462
3463@node What possible errors should I be aware of?
3464@subsection What possible errors should I be aware of?
3465@c %**end of header
3466
3467@emph{This section is dedicated for Linux users}
3468
3469Well there are many ways in which things could go wrong but I will try to
3470present some tools that you could use to debug and some scenarios.
3471@itemize @bullet
3472
3473@item @code{bluetoothd -n -d} : use this command to enable logging in the
3474foreground and to print the logging messages
3475
3476@item @code{hciconfig}: can be used to configure the Bluetooth devices. If you
3477run it without any arguments it will print information about the state of the
3478interfaces. So if you receive an error that the device couldn't be brought up
3479you should try to bring it manually and to see if it works (use @code{hciconfig
3480-a hciX up}). If you can't and the Bluetooth address has the form
348100:00:00:00:00:00 it means that there is something wrong with the D-Bus daemon
3482or with the Bluetooth daemon. Use @code{bluetoothd} tool to see the logs
3483
3484@item @code{sdptool} can be used to control and interogate SDP servers. If you
3485encounter problems regarding the SDP server (like the SDP server is down) you
3486should check out if the D-Bus daemon is running correctly and to see if the
3487Bluetooth daemon started correctly(use @code{bluetoothd} tool). Also, sometimes
3488the SDP service could work but somehow the device couldn't register his
3489service. Use @code{sdptool browse [dev-address]} to see if the service is
3490registered. There should be a service with the name of the interface and GNUnet
3491as provider.
3492
3493@item @code{hcitool} : another useful tool which can be used to configure the
3494device and to send some particular commands to it.
3495
3496@item @code{hcidump} : could be used for low level debugging
3497@end itemize
3498
3499@node How do I configure my peer2?
3500@subsection How do I configure my peer2?
3501@c %**end of header
3502
3503On Linux, you just have to be sure that the interface name corresponds to the
3504one that you want to use. Use the @code{hciconfig} tool to check that. By
3505default it is set to hci0 but you can change it.
3506
3507A basic configuration looks like this:
3508@example
3509[transport-bluetooth]
3510# Name of the interface (typically hciX)
3511INTERFACE = hci0
3512# Real hardware, no testing
3513TESTMODE = 0 TESTING_IGNORE_KEYS = ACCEPT_FROM;
3514@end example
3515
3516
3517In order to use the Bluetooth transport plugin when the transport service is
3518started, you must add the plugin name to the default transport service plugins
3519list. For example:
3520@example
3521[transport] ... PLUGINS = dns bluetooth ...
3522@end example
3523
3524If you want to use only the Bluetooth plugin set @emph{PLUGINS = bluetooth}
3525
3526On Windows, you cannot specify which device to use. The only thing that you
3527should do is to add @emph{bluetooth} on the plugins list of the transport
3528service.
3529
3530@node How can I test it?
3531@subsection How can I test it?
3532@c %**end of header
3533
3534If you have two Bluetooth devices on the same machine which use Linux you
3535must:
3536@itemize @bullet
3537
3538@item create two different file configuration (one which will use the first
3539interface (@emph{hci0}) and the other which will use the second interface
3540(@emph{hci1})). Let's name them @emph{peer1.conf} and @emph{peer2.conf}.
3541
3542@item run @emph{gnunet-peerinfo -c peerX.conf -s} in order to generate the
3543peers private keys. The @strong{X} must be replace with 1 or 2.
3544
3545@item run @emph{gnunet-arm -c peerX.conf -s -i=transport} in order to start the
3546transport service. (Make sure that you have "bluetooth" on the transport
3547plugins list if the Bluetooth transport service doesn't start.)
3548
3549@item run @emph{gnunet-peerinfo -c peer1.conf -s} to get the first peer's ID.
3550If you already know your peer ID (you saved it from the first command), this
3551can be skipped.
3552
3553@item run @emph{gnunet-transport -c peer2.conf -p=PEER1_ID -s} to start sending
3554data for benchmarking to the other peer.
3555@end itemize
3556
3557
3558This scenario will try to connect the second peer to the first one and then
3559start sending data for benchmarking.
3560
3561On Windows you cannot test the plugin functionality using two Bluetooth devices
3562from the same machine because after you install the drivers there will occur
3563some conflicts between the Bluetooth stacks. (At least that is what happend on
3564my machine : I wasn't able to use the Bluesoleil stack and the WINDCOMM one in
3565the same time).
3566
3567If you have two different machines and your configuration files are good you
3568can use the same scenario presented on the begining of this section.
3569
3570Another way to test the plugin functionality is to create your own application
3571which will use the GNUnet framework with the Bluetooth transport service.
3572
3573@node The implementation of the Bluetooth transport plugin
3574@subsection The implementation of the Bluetooth transport plugin
3575@c %**end of header
3576
3577This page describes the implementation of the Bluetooth transport plugin.
3578
3579First I want to remind you that the Bluetooth transport plugin uses virtually
3580the same code as the WLAN plugin and only the helper binary is different. Also
3581the scope of the helper binary from the Bluetooth transport plugin is the same
3582as the one used for the wlan transport plugin: it acceses the interface and
3583then it forwards traffic in both directions between the Bluetooth interface
3584and stdin/stdout of the process involved.
3585
3586The Bluetooth plugin transport could be used both on Linux and Windows
3587platforms.
3588
3589@itemize @bullet
3590@item Linux functionality
3591@item Windows functionality
3592@item Pending Features
3593@end itemize
3594
3595
3596
3597@menu
3598* Linux functionality::
3599* THE INITIALIZATION::
3600* THE LOOP::
3601* Details about the broadcast implementation::
3602* Windows functionality::
3603* Pending features::
3604@end menu
3605
3606@node Linux functionality
3607@subsubsection Linux functionality
3608@c %**end of header
3609
3610In order to implement the plugin functionality on Linux I used the BlueZ
3611stack. For the communication with the other devices I used the RFCOMM
3612protocol. Also I used the HCI protocol to gain some control over the device.
3613The helper binary takes a single argument (the name of the Bluetooth
3614interface) and is separated in two stages:
3615
3616@c %** 'THE INITIALIZATION' should be in bigger letters or stand out, not
3617@c %** starting a new section?
3618@node THE INITIALIZATION
3619@subsubsection THE INITIALIZATION
3620
3621@itemize @bullet
3622@item first, it checks if we have root privilegies (@emph{Remember that we need
3623to have root privilegies in order to be able to bring the interface up if it is
3624down or to change its state.}).
3625
3626@item second, it verifies if the interface with the given name exists.
3627
3628@strong{If the interface with that name exists and it is a Bluetooth
3629interface:}
3630
3631@item it creates a RFCOMM socket which will be used for listening and call the
3632@emph{open_device} method
3633
3634On the @emph{open_device} method:
3635@itemize @bullet
3636@item creates a HCI socket used to send control events to the the device
3637@item searches for the device ID using the interface name
3638@item saves the device MAC address
3639@item checks if the interface is down and tries to bring it UP
3640@item checks if the interface is in discoverable mode and tries to make it
3641discoverable
3642@item closes the HCI socket and binds the RFCOMM one
3643@item switches the RFCOMM socket in listening mode
3644@item registers the SDP service (the service will be used by the other devices
3645to get the port on which this device is listening on)
3646@end itemize
3647
3648@item drops the root privilegies
3649
3650@strong{If the interface is not a Bluetooth interface the helper exits with a
3651suitable error}
3652@end itemize
3653
3654@c %** Same as for @node entry above
3655@node THE LOOP
3656@subsubsection THE LOOP
3657
3658The helper binary uses a list where it saves all the connected neighbour
3659devices (@emph{neighbours.devices}) and two buffers (@emph{write_pout} and
3660@emph{write_std}). The first message which is send is a control message with
3661the device's MAC address in order to announce the peer presence to the
3662neighbours. Here are a short description of what happens in the main loop:
3663
3664@itemize @bullet
3665@item Every time when it receives something from the STDIN it processes the
3666data and saves the message in the first buffer (@emph{write_pout}). When it has
3667something in the buffer, it gets the destination address from the buffer,
3668searches the destination address in the list (if there is no connection with
3669that device, it creates a new one and saves it to the list) and sends the
3670message.
3671@item Every time when it receives something on the listening socket it accepts
3672the connection and saves the socket on a list with the reading sockets.
3673@item Every time when it receives something from a reading socket it parses the
3674message, verifies the CRC and saves it in the @emph{write_std} buffer in order
3675to be sent later to the STDOUT.
3676@end itemize
3677
3678So in the main loop we use the select function to wait until one of the file
3679descriptor saved in one of the two file descriptors sets used is ready to use.
3680The first set (@emph{rfds}) represents the reading set and it could contain the
3681list with the reading sockets, the STDIN file descriptor or the listening
3682socket. The second set (@emph{wfds}) is the writing set and it could contain
3683the sending socket or the STDOUT file descriptor. After the select function
3684returns, we check which file descriptor is ready to use and we do what is
3685supposed to do on that kind of event. @emph{For example:} if it is the
3686listening socket then we accept a new connection and save the socket in the
3687reading list; if it is the STDOUT file descriptor, then we write to STDOUT the
3688message from the @emph{write_std} buffer.
3689
3690To find out on which port a device is listening on we connect to the local SDP
3691server and searche the registered service for that device.
3692
3693@emph{You should be aware of the fact that if the device fails to connect to
3694another one when trying to send a message it will attempt one more time. If it
3695fails again, then it skips the message.}
3696@emph{Also you should know that the
3697transport Bluetooth plugin has support for @strong{broadcast messages}.}
3698
3699@node Details about the broadcast implementation
3700@subsubsection Details about the broadcast implementation
3701@c %**end of header
3702
3703First I want to point out that the broadcast functionality for the CONTROL
3704messages is not implemented in a conventional way. Since the inquiry scan time
3705is too big and it will take some time to send a message to all the
3706discoverable devices I decided to tackle the problem in a different way. Here
3707is how I did it:
3708
3709@itemize @bullet
3710@item If it is the first time when I have to broadcast a message I make an
3711inquiry scan and save all the devices' addresses to a vector.
3712@item After the inquiry scan ends I take the first address from the list and I
3713try to connect to it. If it fails, I try to connect to the next one. If it
3714succeeds, I save the socket to a list and send the message to the device.
3715@item When I have to broadcast another message, first I search on the list for
3716a new device which I'm not connected to. If there is no new device on the list
3717I go to the beginning of the list and send the message to the old devices.
3718After 5 cycles I make a new inquiry scan to check out if there are new
3719discoverable devices and save them to the list. If there are no new
3720discoverable devices I reset the cycling counter and go again through the old
3721list and send messages to the devices saved in it.
3722@end itemize
3723
3724@strong{Therefore}:
3725
3726@itemize @bullet
3727@item every time when I have a broadcast message I look up on the list for a
3728new device and send the message to it
3729@item if I reached the end of the list for 5 times and I'm connected to all the
3730devices from the list I make a new inquiry scan. @emph{The number of the list's
3731cycles after an inquiry scan could be increased by redefining the MAX_LOOPS
3732variable}
3733@item when there are no new devices I send messages to the old ones.
3734@end itemize
3735
3736Doing so, the broadcast control messages will reach the devices but with delay.
3737
3738@emph{NOTICE:} When I have to send a message to a certain device first I check
3739on the broadcast list to see if we are connected to that device. If not we try
3740to connect to it and in case of success we save the address and the socket on
3741the list. If we are already connected to that device we simply use the socket.
3742
3743@node Windows functionality
3744@subsubsection Windows functionality
3745@c %**end of header
3746
3747For Windows I decided to use the Microsoft Bluetooth stack which has the
3748advantage of coming standard from Windows XP SP2. The main disadvantage is
3749that it only supports the RFCOMM protocol so we will not be able to have a low
3750level control over the Bluetooth device. Therefore it is the user
3751responsability to check if the device is up and in the discoverable mode. Also
3752there are no tools which could be used for debugging in order to read the data
3753coming from and going to a Bluetooth device, which obviously hindered my work.
3754Another thing that slowed down the implementation of the plugin (besides that
3755I wasn't too accomodated with the win32 API) was that there were some bugs on
3756MinGW regarding the Bluetooth. Now they are solved but you should keep in mind
3757that you should have the latest updates (especially the @emph{ws2bth} header).
3758
3759Besides the fact that it uses the Windows Sockets, the Windows implemenation
3760follows the same principles as the Linux one:
3761
3762@itemize @bullet
3763@item
3764It has a initalization part where it initializes the Windows Sockets, creates a
3765RFCOMM socket which will be binded and switched to the listening mode and
3766registers a SDP service.
3767In the Microsoft Bluetooth API there are two ways to work with the SDP:
3768@itemize @bullet
3769@item an easy way which works with very simple service records
3770@item a hard way which is useful when you need to update or to delete the
3771record
3772@end itemize
3773@end itemize
3774
3775Since I only needed the SDP service to find out on which port the device is
3776listening on and that did not change, I decided to use the easy way. In order
3777to register the service I used the @emph{WSASetService} function and I
3778generated the @emph{Universally Unique Identifier} with the @emph{guidgen.exe}
3779Windows's tool.
3780
3781In the loop section the only difference from the Linux implementation is that
3782I used the GNUNET_NETWORK library for functions like @emph{accept},
3783@emph{bind}, @emph{connect} or @emph{select}. I decided to use the
3784GNUNET_NETWORK library because I also needed to interact with the STDIN and
3785STDOUT handles and on Windows the select function is only defined for sockets,
3786and it will not work for arbitrary file handles.
3787
3788Another difference between Linux and Windows implementation is that in Linux,
3789the Bluetooth address is represented in 48 bits while in Windows is
3790represented in 64 bits. Therefore I had to do some changes on
3791@emph{plugin_transport_wlan} header.
3792
3793Also, currently on Windows the Bluetooth plugin doesn't have support for
3794broadcast messages. When it receives a broadcast message it will skip it.
3795
3796@node Pending features
3797@subsubsection Pending features
3798@c %**end of header
3799
3800@itemize @bullet
3801@item Implement the broadcast functionality on Windows @emph{(currently working
3802on)}
3803@item Implement a testcase for the helper :@ @emph{@ The testcase consists of a
3804program which emaluates the plugin and uses the helper. It will simulate
3805connections, disconnections and data transfers.@ }
3806@end itemize
3807
3808If you have a new idea about a feature of the plugin or suggestions about how
3809I could improve the implementation you are welcome to comment or to contact
3810me.
3811
3812@node WLAN plugin
3813@section WLAN plugin
3814@c %**end of header
3815
3816This section documents how the wlan transport plugin works. Parts which are not
3817implemented yet or could be better implemented are described at the end.
3818
3819@node The ATS Subsystem
3820@section The ATS Subsystem
3821@c %**end of header
3822
3823ATS stands for "automatic transport selection", and the function of ATS in
3824GNUnet is to decide on which address (and thus transport plugin) should be used
3825for two peers to communicate, and what bandwidth limits should be imposed on
3826such an individual connection. To help ATS make an informed decision,
3827higher-level services inform the ATS service about their requirements and the
3828quality of the service rendered. The ATS service also interacts with the
3829transport service to be appraised of working addresses and to communicate its
3830resource allocation decisions. Finally, the ATS service's operation can be
3831observed using a monitoring API.
3832
3833The main logic of the ATS service only collects the available addresses, their
3834performance characteristics and the applications requirements, but does not
3835make the actual allocation decision. This last critical step is left to an ATS
3836plugin, as we have implemented (currently three) different allocation
3837strategies which differ significantly in their performance and maturity, and it
3838is still unclear if any particular plugin is generally superior.
3839
3840@node GNUnet's CORE Subsystem
3841@section GNUnet's CORE Subsystem
3842@c %**end of header
3843
3844The CORE subsystem in GNUnet is responsible for securing link-layer
3845communications between nodes in the GNUnet overlay network. CORE builds on the
3846TRANSPORT subsystem which provides for the actual, insecure, unreliable
3847link-layer communication (for example, via UDP or WLAN), and then adds
3848fundamental security to the connections:
3849
3850@itemize @bullet
3851@item confidentiality with so-called perfect forward secrecy; we use
3852@uref{http://en.wikipedia.org/wiki/Elliptic_curve_Diffie%E2%80%93Hellman,
3853ECDHE} powered by @uref{http://cr.yp.to/ecdh.html, Curve25519} for the key
3854exchange and then use symmetric encryption, encrypting with both
3855@uref{http://en.wikipedia.org/wiki/Rijndael, AES-256} and
3856@uref{http://en.wikipedia.org/wiki/Twofish, Twofish}
3857@item @uref{http://en.wikipedia.org/wiki/Authentication, authentication} is
3858achieved by signing the ephemeral keys using @uref{http://ed25519.cr.yp.to/,
3859Ed25519}, a deterministic variant of @uref{http://en.wikipedia.org/wiki/ECDSA,
3860ECDSA}
3861@item integrity protection (using @uref{http://en.wikipedia.org/wiki/SHA-2,
3862SHA-512} to do @uref{http://en.wikipedia.org/wiki/Authenticated_encryption,
3863encrypt-then-MAC)}
3864@item @uref{http://en.wikipedia.org/wiki/Replay_attack, replay} protection
3865(using nonces, timestamps, challenge-response, message counters and ephemeral
3866keys)
3867@item liveness (keep-alive messages, timeout)
3868@end itemize
3869
3870@menu
3871* Limitations::
3872* When is a peer "connected"?::
3873* libgnunetcore::
3874* The CORE Client-Service Protocol::
3875* The CORE Peer-to-Peer Protocol::
3876@end menu
3877
3878@node Limitations
3879@subsection Limitations
3880@c %**end of header
3881
3882CORE does not perform @uref{http://en.wikipedia.org/wiki/Routing, routing};
3883using CORE it is only possible to communicate with peers that happen to
3884already be "directly" connected with each other. CORE also does not have an
3885API to allow applications to establish such "direct" connections --- for this,
3886applications can ask TRANSPORT, but TRANSPORT might not be able to establish a
3887"direct" connection. The TOPOLOGY subsystem is responsible for trying to keep
3888a few "direct" connections open at all times. Applications that need to talk
3889to particular peers should use the CADET subsystem, as it can establish
3890arbitrary "indirect" connections.
3891
3892Because CORE does not perform routing, CORE must only be used directly by
3893applications that either perform their own routing logic (such as anonymous
3894file-sharing) or that do not require routing, for example because they are
3895based on flooding the network. CORE communication is unreliable and delivery
3896is possibly out-of-order. Applications that require reliable communication
3897should use the CADET service. Each application can only queue one message per
3898target peer with the CORE service at any time; messages cannot be larger than
3899approximately 63 kilobytes. If messages are small, CORE may group multiple
3900messages (possibly from different applications) prior to encryption. If
3901permitted by the application (using the @uref{http://baus.net/on-tcp_cork/,
3902cork} option), CORE may delay transmissions to facilitate grouping of multiple
3903small messages. If cork is not enabled, CORE will transmit the message as soon
3904as TRANSPORT allows it (TRANSPORT is responsible for limiting bandwidth and
3905congestion control). CORE does not allow flow control; applications are
3906expected to process messages at line-speed. If flow control is needed,
3907applications should use the CADET service.
3908
3909@node When is a peer "connected"?
3910@subsection When is a peer "connected"?
3911@c %**end of header
3912
3913In addition to the security features mentioned above, CORE also provides one
3914additional key feature to applications using it, and that is a limited form of
3915protocol-compatibility checking. CORE distinguishes between TRANSPORT-level
3916connections (which enable communication with other peers) and
3917application-level connections. Applications using the CORE API will
3918(typically) learn about application-level connections from CORE, and not about
3919TRANSPORT-level connections. When a typical application uses CORE, it will
3920specify a set of message types (from @code{gnunet_protocols.h}) that it
3921understands. CORE will then notify the application about connections it has
3922with other peers if and only if those applications registered an intersecting
3923set of message types with their CORE service. Thus, it is quite possible that
3924CORE only exposes a subset of the established direct connections to a
3925particular application --- and different applications running above CORE might
3926see different sets of connections at the same time.
3927
3928A special case are applications that do not register a handler for any message
3929type. CORE assumes that these applications merely want to monitor connections
3930(or "all" messages via other callbacks) and will notify those applications
3931about all connections. This is used, for example, by the @code{gnunet-core}
3932command-line tool to display the active connections. Note that it is also
3933possible that the TRANSPORT service has more active connections than the CORE
3934service, as the CORE service first has to perform a key exchange with
3935connecting peers before exchanging information about supported message types
3936and notifying applications about the new connection.
3937
3938@node libgnunetcore
3939@subsection libgnunetcore
3940@c %**end of header
3941
3942The CORE API (defined in @code{gnunet_core_service.h}) is the basic messaging
3943API used by P2P applications built using GNUnet. It provides applications the
3944ability to send and receive encrypted messages to the peer's "directly"
3945connected neighbours.
3946
3947As CORE connections are generally "direct" connections,@ applications must not
3948assume that they can connect to arbitrary peers this way, as "direct"
3949connections may not always be possible. Applications using CORE are notified
3950about which peers are connected. Creating new "direct" connections must be
3951done using the TRANSPORT API.
3952
3953The CORE API provides unreliable, out-of-order delivery. While the
3954implementation tries to ensure timely, in-order delivery, both message losses
3955and reordering are not detected and must be tolerated by the application. Most
3956important, the core will NOT perform retransmission if messages could not be
3957delivered.
3958
3959Note that CORE allows applications to queue one message per connected peer.
3960The rate at which each connection operates is influenced by the preferences
3961expressed by local application as well as restrictions imposed by the other
3962peer. Local applications can express their preferences for particular
3963connections using the "performance" API of the ATS service.
3964
3965Applications that require more sophisticated transmission capabilities such as
3966TCP-like behavior, or if you intend to send messages to arbitrary remote
3967peers, should use the CADET API.
3968
3969The typical use of the CORE API is to connect to the CORE service using
3970@code{GNUNET_CORE_connect}, process events from the CORE service (such as
3971peers connecting, peers disconnecting and incoming messages) and send messages
3972to connected peers using @code{GNUNET_CORE_notify_transmit_ready}. Note that
3973applications must cancel pending transmission requests if they receive a
3974disconnect event for a peer that had a transmission pending; furthermore,
3975queueing more than one transmission request per peer per application using the
3976service is not permitted.
3977
3978The CORE API also allows applications to monitor all communications of the
3979peer prior to encryption (for outgoing messages) or after decryption (for
3980incoming messages). This can be useful for debugging, diagnostics or to
3981establish the presence of cover traffic (for anonymity). As monitoring
3982applications are often not interested in the payload, the monitoring callbacks
3983can be configured to only provide the message headers (including the message
3984type and size) instead of copying the full data stream to the monitoring
3985client.
3986
3987The init callback of the @code{GNUNET_CORE_connect} function is called with
3988the hash of the public key of the peer. This public key is used to identify
3989the peer globally in the GNUnet network. Applications are encouraged to check
3990that the provided hash matches the hash that they are using (as theoretically
3991the application may be using a different configuration file with a different
3992private key, which would result in hard to find bugs).
3993
3994As with most service APIs, the CORE API isolates applications from crashes of
3995the CORE service. If the CORE service crashes, the application will see
3996disconnect events for all existing connections. Once the connections are
3997re-established, the applications will be receive matching connect events.
3998
3999@node The CORE Client-Service Protocol
4000@subsection The CORE Client-Service Protocol
4001@c %**end of header
4002
4003This section describes the protocol between an application using the CORE
4004service (the client) and the CORE service process itself.
4005
4006
4007@menu
4008* Setup2::
4009* Notifications::
4010* Sending::
4011@end menu
4012
4013@node Setup2
4014@subsubsection Setup2
4015@c %**end of header
4016
4017When a client connects to the CORE service, it first sends a
4018@code{InitMessage} which specifies options for the connection and a set of
4019message type values which are supported by the application. The options
4020bitmask specifies which events the client would like to be notified about. The
4021options include:
4022
4023@table @asis
4024@item GNUNET_CORE_OPTION_NOTHING No notifications
4025@item GNUNET_CORE_OPTION_STATUS_CHANGE Peers connecting and disconnecting
4026@item GNUNET_CORE_OPTION_FULL_INBOUND All inbound messages (after decryption) with
4027full payload
4028@item GNUNET_CORE_OPTION_HDR_INBOUND Just the @code{MessageHeader}
4029of all inbound messages
4030@item GNUNET_CORE_OPTION_FULL_OUTBOUND All outbound
4031messages (prior to encryption) with full payload
4032@item GNUNET_CORE_OPTION_HDR_OUTBOUND Just the @code{MessageHeader} of all outbound
4033messages
4034@end table
4035
4036Typical applications will only monitor for connection status changes.
4037
4038The CORE service responds to the @code{InitMessage} with an
4039@code{InitReplyMessage} which contains the peer's identity. Afterwards, both
4040CORE and the client can send messages.
4041
4042@node Notifications
4043@subsubsection Notifications
4044@c %**end of header
4045
4046The CORE will send @code{ConnectNotifyMessage}s and
4047@code{DisconnectNotifyMessage}s whenever peers connect or disconnect from the
4048CORE (assuming their type maps overlap with the message types registered by
4049the client). When the CORE receives a message that matches the set of message
4050types specified during the @code{InitMessage} (or if monitoring is enabled in
4051for inbound messages in the options), it sends a @code{NotifyTrafficMessage}
4052with the peer identity of the sender and the decrypted payload. The same
4053message format (except with @code{GNUNET_MESSAGE_TYPE_CORE_NOTIFY_OUTBOUND}
4054for the message type) is used to notify clients monitoring outbound messages;
4055here, the peer identity given is that of the receiver.
4056
4057@node Sending
4058@subsubsection Sending
4059@c %**end of header
4060
4061When a client wants to transmit a message, it first requests a transmission
4062slot by sending a @code{SendMessageRequest} which specifies the priority,
4063deadline and size of the message. Note that these values may be ignored by
4064CORE. When CORE is ready for the message, it answers with a
4065@code{SendMessageReady} response. The client can then transmit the payload
4066with a @code{SendMessage} message. Note that the actual message size in the
4067@code{SendMessage} is allowed to be smaller than the size in the original
4068request. A client may at any time send a fresh @code{SendMessageRequest},
4069which then superceeds the previous @code{SendMessageRequest}, which is then no
4070longer valid. The client can tell which @code{SendMessageRequest} the CORE
4071service's @code{SendMessageReady} message is for as all of these messages
4072contain a "unique" request ID (based on a counter incremented by the client
4073for each request).
4074
4075@node The CORE Peer-to-Peer Protocol
4076@subsection The CORE Peer-to-Peer Protocol
4077@c %**end of header
4078
4079
4080@menu
4081* Creating the EphemeralKeyMessage::
4082* Establishing a connection::
4083* Encryption and Decryption::
4084* Type maps::
4085@end menu
4086
4087@node Creating the EphemeralKeyMessage
4088@subsubsection Creating the EphemeralKeyMessage
4089@c %**end of header
4090
4091When the CORE service starts, each peer creates a fresh ephemeral (ECC)
4092public-private key pair and signs the corresponding @code{EphemeralKeyMessage}
4093with its long-term key (which we usually call the peer's identity; the hash of
4094the public long term key is what results in a @code{struct
4095GNUNET_PeerIdentity} in all GNUnet APIs. The ephemeral key is ONLY used for an
4096@uref{http://en.wikipedia.org/wiki/Elliptic_curve_Diffie%E2%80%93Hellman,
4097ECDHE} exchange by the CORE service to establish symmetric session keys. A
4098peer will use the same @code{EphemeralKeyMessage} for all peers for
4099@code{REKEY_FREQUENCY}, which is usually 12 hours. After that time, it will
4100create a fresh ephemeral key (forgetting the old one) and broadcast the new
4101@code{EphemeralKeyMessage} to all connected peers, resulting in fresh
4102symmetric session keys. Note that peers independently decide on when to
4103discard ephemeral keys; it is not a protocol violation to discard keys more
4104often. Ephemeral keys are also never stored to disk; restarting a peer will
4105thus always create a fresh ephemeral key. The use of ephemeral keys is what
4106provides @uref{http://en.wikipedia.org/wiki/Forward_secrecy, forward secrecy}.
4107
4108Just before transmission, the @code{EphemeralKeyMessage} is patched to reflect
4109the current sender_status, which specifies the current state of the connection
4110from the point of view of the sender. The possible values are:
4111
4112@table @asis
4113@item KX_STATE_DOWN Initial value, never used on the network
4114@item KX_STATE_KEY_SENT We sent our ephemeral key, do not know the key of the other
4115peer
4116@item KX_STATE_KEY_RECEIVED This peer has received a valid ephemeral key
4117of the other peer, but we are waiting for the other peer to confirm it's
4118authenticity (ability to decode) via challenge-response.
4119@item KX_STATE_UP The
4120connection is fully up from the point of view of the sender (now performing
4121keep-alives)
4122@item KX_STATE_REKEY_SENT The sender has initiated a rekeying
4123operation; the other peer has so far failed to confirm a working connection
4124using the new ephemeral key
4125@end table
4126
4127@node Establishing a connection
4128@subsubsection Establishing a connection
4129@c %**end of header
4130
4131Peers begin their interaction by sending a @code{EphemeralKeyMessage} to the
4132other peer once the TRANSPORT service notifies the CORE service about the
4133connection. A peer receiving an @code{EphemeralKeyMessage} with a status
4134indicating that the sender does not have the receiver's ephemeral key, the
4135receiver's @code{EphemeralKeyMessage} is sent in response.@ Additionally, if
4136the receiver has not yet confirmed the authenticity of the sender, it also
4137sends an (encrypted)@code{PingMessage} with a challenge (and the identity of
4138the target) to the other peer. Peers receiving a @code{PingMessage} respond
4139with an (encrypted) @code{PongMessage} which includes the challenge. Peers
4140receiving a @code{PongMessage} check the challenge, and if it matches set the
4141connection to @code{KX_STATE_UP}.
4142
4143@node Encryption and Decryption
4144@subsubsection Encryption and Decryption
4145@c %**end of header
4146
4147All functions related to the key exchange and encryption/decryption of
4148messages can be found in @code{gnunet-service-core_kx.c} (except for the
4149cryptographic primitives, which are in @code{util/crypto*.c}).@ Given the key
4150material from ECDHE, a
4151@uref{http://en.wikipedia.org/wiki/Key_derivation_function, Key derivation
4152function} is used to derive two pairs of encryption and decryption keys for
4153AES-256 and TwoFish, as well as initialization vectors and authentication keys
4154(for @uref{http://en.wikipedia.org/wiki/HMAC, HMAC}). The HMAC is computed
4155over the encrypted payload. Encrypted messages include an iv_seed and the HMAC
4156in the header.
4157
4158Each encrypted message in the CORE service includes a sequence number and a
4159timestamp in the encrypted payload. The CORE service remembers the largest
4160observed sequence number and a bit-mask which represents which of the previous
416132 sequence numbers were already used. Messages with sequence numbers lower
4162than the largest observed sequence number minus 32 are discarded. Messages
4163with a timestamp that is less than @code{REKEY_TOLERANCE} off (5 minutes) are
4164also discarded. This of course means that system clocks need to be reasonably
4165synchronized for peers to be able to communicate. Additionally, as the
4166ephemeral key changes every 12h, a peer would not even be able to decrypt
4167messages older than 12h.
4168
4169@node Type maps
4170@subsubsection Type maps
4171@c %**end of header
4172
4173Once an encrypted connection has been established, peers begin to exchange
4174type maps. Type maps are used to allow the CORE service to determine which
4175(encrypted) connections should be shown to which applications. A type map is
4176an array of 65536 bits representing the different types of messages understood
4177by applications using the CORE service. Each CORE service maintains this map,
4178simply by setting the respective bit for each message type supported by any of
4179the applications using the CORE service. Note that bits for message types
4180embedded in higher-level protocols (such as MESH) will not be included in
4181these type maps.
4182
4183Typically, the type map of a peer will be sparse. Thus, the CORE service
4184attempts to compress its type map using @code{gzip}-style compression
4185("deflate") prior to transmission. However, if the compression fails to
4186compact the map, the map may also be transmitted without compression
4187(resulting in @code{GNUNET_MESSAGE_TYPE_CORE_COMPRESSED_TYPE_MAP} or
4188@code{GNUNET_MESSAGE_TYPE_CORE_BINARY_TYPE_MAP} messages respectively). Upon
4189receiving a type map, the respective CORE service notifies applications about
4190the connection to the other peer if they support any message type indicated in
4191the type map (or no message type at all). If the CORE service experience a
4192connect or disconnect event from an application, it updates its type map
4193(setting or unsetting the respective bits) and notifies its neighbours about
4194the change. The CORE services of the neighbours then in turn generate connect
4195and disconnect events for the peer that sent the type map for their respective
4196applications. As CORE messages may be lost, the CORE service confirms
4197receiving a type map by sending back a
4198@code{GNUNET_MESSAGE_TYPE_CORE_CONFIRM_TYPE_MAP}. If such a confirmation (with
4199the correct hash of the type map) is not received, the sender will retransmit
4200the type map (with exponential back-off).
4201
4202@node GNUnet's CADET subsystem
4203@section GNUnet's CADET subsystem
4204
4205The CADET subsystem in GNUnet is responsible for secure end-to-end
4206communications between nodes in the GNUnet overlay network. CADET builds on the
4207CORE subsystem which provides for the link-layer communication and then adds
4208routing, forwarding and additional security to the connections. CADET offers
4209the same cryptographic services as CORE, but on an end-to-end level. This is
4210done so peers retransmitting traffic on behalf of other peers cannot access the
4211payload data.
4212
4213@itemize @bullet
4214@item CADET provides confidentiality with so-called perfect forward secrecy; we
4215use ECDHE powered by Curve25519 for the key exchange and then use symmetric
4216encryption, encrypting with both AES-256 and Twofish
4217@item authentication is achieved by signing the ephemeral keys using Ed25519, a
4218deterministic variant of ECDSA
4219@item integrity protection (using SHA-512 to do encrypt-then-MAC, although only
4220256 bits are sent to reduce overhead)
4221@item replay protection (using nonces, timestamps, challenge-response, message
4222counters and ephemeral keys)
4223@item liveness (keep-alive messages, timeout)
4224@end itemize
4225
4226Additional to the CORE-like security benefits, CADET offers other properties
4227that make it a more universal service than CORE.
4228
4229@itemize @bullet
4230@item CADET can establish channels to arbitrary peers in GNUnet. If a peer is
4231not immediately reachable, CADET will find a path through the network and ask
4232other peers to retransmit the traffic on its behalf.
4233@item CADET offers (optional) reliability mechanisms. In a reliable channel
4234traffic is guaranteed to arrive complete, unchanged and in-order.
4235@item CADET takes care of flow and congestion control mechanisms, not allowing
4236the sender to send more traffic than the receiver or the network are able to
4237process.
4238@end itemize
4239
4240@menu
4241* libgnunetcadet::
4242@end menu
4243
4244@node libgnunetcadet
4245@subsection libgnunetcadet
4246
4247
4248The CADET API (defined in gnunet_cadet_service.h) is the messaging API used by
4249P2P applications built using GNUnet. It provides applications the ability to
4250send and receive encrypted messages to any peer participating in GNUnet. The
4251API is heavily base on the CORE API.
4252
4253CADET delivers messages to other peers in "channels". A channel is a permanent
4254connection defined by a destination peer (identified by its public key) and a
4255port number. Internally, CADET tunnels all channels towards a destiantion peer
4256using one session key and relays the data on multiple "connections",
4257independent from the channels.
4258
4259Each channel has optional paramenters, the most important being the reliability
4260flag. Should a message get lost on TRANSPORT/CORE level, if a channel is
4261created with as reliable, CADET will retransmit the lost message and deliver it
4262in order to the destination application.
4263
4264To communicate with other peers using CADET, it is necessary to first connect
4265to the service using @code{GNUNET_CADET_connect}. This function takes several
4266parameters in form of callbacks, to allow the client to react to various
4267events, like incoming channels or channels that terminate, as well as specify a
4268list of ports the client wishes to listen to (at the moment it is not possible
4269to start listening on further ports once connected, but nothing prevents a
4270client to connect several times to CADET, even do one connection per listening
4271port). The function returns a handle which has to be used for any further
4272interaction with the service.
4273
4274To connect to a remote peer a client has to call the
4275@code{GNUNET_CADET_channel_create} function. The most important parameters
4276given are the remote peer's identity (it public key) and a port, which
4277specifies which application on the remote peer to connect to, similar to
4278TCP/UDP ports. CADET will then find the peer in the GNUnet network and
4279establish the proper low-level connections and do the necessary key exchanges
4280to assure and authenticated, secure and verified communication. Similar to
4281@code{GNUNET_CADET_connect},@code{GNUNET_CADET_create_channel} returns a handle
4282to interact with the created channel.
4283
4284For every message the client wants to send to the remote application,
4285@code{GNUNET_CADET_notify_transmit_ready} must be called, indicating the
4286channel on which the message should be sent and the size of the message (but
4287not the message itself!). Once CADET is ready to send the message, the provided
4288callback will fire, and the message contents are provided to this callback.
4289
4290Please note the CADET does not provide an explicit notification of when a
4291channel is connected. In loosely connected networks, like big wireless mesh
4292networks, this can take several seconds, even minutes in the worst case. To be
4293alerted when a channel is online, a client can call
4294@code{GNUNET_CADET_notify_transmit_ready} immediately after
4295@code{GNUNET_CADET_create_channel}. When the callback is activated, it means
4296that the channel is online. The callback can give 0 bytes to CADET if no
4297message is to be sent, this is ok.
4298
4299If a transmission was requested but before the callback fires it is no longer
4300needed, it can be cancelled with
4301@code{GNUNET_CADET_notify_transmit_ready_cancel}, which uses the handle given
4302back by @code{GNUNET_CADET_notify_transmit_ready}. As in the case of CORE, only
4303one message can be requested at a time: a client must not call
4304@code{GNUNET_CADET_notify_transmit_ready} again until the callback is called or
4305the request is cancelled.
4306
4307When a channel is no longer needed, a client can call
4308@code{GNUNET_CADET_channel_destroy} to get rid of it. Note that CADET will try
4309to transmit all pending traffic before notifying the remote peer of the
4310destruction of the channel, including retransmitting lost messages if the
4311channel was reliable.
4312
4313Incoming channels, channels being closed by the remote peer, and traffic on any
4314incoming or outgoing channels are given to the client when CADET executes the
4315callbacks given to it at the time of @code{GNUNET_CADET_connect}.
4316
4317Finally, when an application no longer wants to use CADET, it should call
4318@code{GNUNET_CADET_disconnect}, but first all channels and pending
4319transmissions must be closed (otherwise CADET will complain).
4320
4321@node GNUnet's NSE subsystem
4322@section GNUnet's NSE subsystem
4323
4324
4325NSE stands for Network Size Estimation. The NSE subsystem provides other
4326subsystems and users with a rough estimate of the number of peers currently
4327participating in the GNUnet overlay. The computed value is not a precise number
4328as producing a precise number in a decentralized, efficient and secure way is
4329impossible. While NSE's estimate is inherently imprecise, NSE also gives the
4330expected range. For a peer that has been running in a stable network for a
4331while, the real network size will typically (99.7% of the time) be in the range
4332of [2/3 estimate, 3/2 estimate]. We will now give an overview of the algorithm
4333used to calcualte the estimate; all of the details can be found in this
4334technical report.
4335
4336@menu
4337* Motivation::
4338* Principle::
4339* libgnunetnse::
4340* The NSE Client-Service Protocol::
4341* The NSE Peer-to-Peer Protocol::
4342@end menu
4343
4344@node Motivation
4345@subsection Motivation
4346
4347
4348Some subsytems, like DHT, need to know the size of the GNUnet network to
4349optimize some parameters of their own protocol. The decentralized nature of
4350GNUnet makes efficient and securely counting the exact number of peers
4351infeasable. Although there are several decentralized algorithms to count the
4352number of peers in a system, so far there is none to do so securely. Other
4353protocols may allow any malicious peer to manipulate the final result or to
4354take advantage of the system to perform DoS (Denial of Service) attacks against
4355the network. GNUnet's NSE protocol avoids these drawbacks.
4356
4357
4358
4359@menu
4360* Security::
4361@end menu
4362
4363@node Security
4364@subsubsection Security
4365
4366
4367The NSE subsystem is designed to be resilient against these attacks. It uses
4368@uref{http://en.wikipedia.org/wiki/Proof-of-work_system, proofs of work} to
4369prevent one peer from impersonating a large number of participants, which would
4370otherwise allow an adversary to artifically inflate the estimate. The DoS
4371protection comes from the time-based nature of the protocol: the estimates are
4372calculated periodically and out-of-time traffic is either ignored or stored for
4373later retransmission by benign peers. In particular, peers cannot trigger
4374global network communication at will.
4375
4376@node Principle
4377@subsection Principle
4378
4379
4380The algorithm calculates the estimate by finding the globally closest peer ID
4381to a random, time-based value.
4382
4383The idea is that the closer the ID is to the random value, the more "densely
4384packed" the ID space is, and therefore, more peers are in the network.
4385
4386
4387
4388@menu
4389* Example::
4390* Algorithm::
4391* Target value::
4392* Timing::
4393* Controlled Flooding::
4394* Calculating the estimate::
4395@end menu
4396
4397@node Example
4398@subsubsection Example
4399
4400
4401Suppose all peers have IDs between 0 and 100 (our ID space), and the random
4402value is 42. If the closest peer has the ID 70 we can imagine that the average
4403"distance" between peers is around 30 and therefore the are around 3 peers in
4404the whole ID space. On the other hand, if the closest peer has the ID 44, we
4405can imagine that the space is rather packed with peers, maybe as much as 50 of
4406them. Naturally, we could have been rather unlucky, and there is only one peer
4407and happens to have the ID 44. Thus, the current estimate is calculated as the
4408average over multiple rounds, and not just a single sample.
4409
4410@node Algorithm
4411@subsubsection Algorithm
4412
4413
4414Given that example, one can imagine that the job of the subsystem is to
4415efficiently communicate the ID of the closest peer to the target value to all
4416the other peers, who will calculate the estimate from it.
4417
4418@node Target value
4419@subsubsection Target value
4420
4421@c %**end of header
4422
4423The target value itself is generated by hashing the current time, rounded down
4424to an agreed value. If the rounding amount is 1h (default) and the time is
442512:34:56, the time to hash would be 12:00:00. The process is repeated each
4426rouning amount (in this example would be every hour). Every repetition is
4427called a round.
4428
4429@node Timing
4430@subsubsection Timing
4431@c %**end of header
4432
4433The NSE subsystem has some timing control to avoid everybody broadcasting its
4434ID all at one. Once each peer has the target random value, it compares its own
4435ID to the target and calculates the hypothetical size of the network if that
4436peer were to be the closest. Then it compares the hypothetical size with the
4437estimate from the previous rounds. For each value there is an assiciated point
4438in the period, let's call it "broadcast time". If its own hypothetical estimate
4439is the same as the previous global estimate, its "broadcast time" will be in
4440the middle of the round. If its bigger it will be earlier and if its smaler
4441(the most likely case) it will be later. This ensures that the peers closests
4442to the target value start broadcasting their ID the first.
4443
4444@node Controlled Flooding
4445@subsubsection Controlled Flooding
4446
4447@c %**end of header
4448
4449When a peer receives a value, first it verifies that it is closer than the
4450closest value it had so far, otherwise it answers the incoming message with a
4451message containing the better value. Then it checks a proof of work that must
4452be included in the incoming message, to ensure that the other peer's ID is not
4453made up (otherwise a malicious peer could claim to have an ID of exactly the
4454target value every round). Once validated, it compares the brodcast time of the
4455received value with the current time and if it's not too early, sends the
4456received value to its neighbors. Otherwise it stores the value until the
4457correct broadcast time comes. This prevents unnecessary traffic of sub-optimal
4458values, since a better value can come before the broadcast time, rendering the
4459previous one obsolete and saving the traffic that would have been used to
4460broadcast it to the neighbors.
4461
4462@node Calculating the estimate
4463@subsubsection Calculating the estimate
4464
4465@c %**end of header
4466
4467Once the closest ID has been spread across the network each peer gets the exact
4468distance betweed this ID and the target value of the round and calculates the
4469estimate with a mathematical formula described in the tech report. The estimate
4470generated with this method for a single round is not very precise. Remember the
4471case of the example, where the only peer is the ID 44 and we happen to generate
4472the target value 42, thinking there are 50 peers in the network. Therefore, the
4473NSE subsystem remembers the last 64 estimates and calculates an average over
4474them, giving a result of which usually has one bit of uncertainty (the real
4475size could be half of the estimate or twice as much). Note that the actual
4476network size is calculated in powers of two of the raw input, thus one bit of
4477uncertainty means a factor of two in the size estimate.
4478
4479@node libgnunetnse
4480@subsection libgnunetnse
4481
4482@c %**end of header
4483
4484The NSE subsystem has the simplest API of all services, with only two calls:
4485@code{GNUNET_NSE_connect} and @code{GNUNET_NSE_disconnect}.
4486
4487The connect call gets a callback function as a parameter and this function is
4488called each time the network agrees on an estimate. This usually is once per
4489round, with some exceptions: if the closest peer has a late local clock and
4490starts spreading his ID after everyone else agreed on a value, the callback
4491might be activated twice in a round, the second value being always bigger than
4492the first. The default round time is set to 1 hour.
4493
4494The disconnect call disconnects from the NSE subsystem and the callback is no
4495longer called with new estimates.
4496
4497
4498
4499@menu
4500* Results::
4501* Examples2::
4502@end menu
4503
4504@node Results
4505@subsubsection Results
4506
4507@c %**end of header
4508
4509The callback provides two values: the average and the
4510@uref{http://en.wikipedia.org/wiki/Standard_deviation, standard deviation} of
4511the last 64 rounds. The values provided by the callback function are
4512logarithmic, this means that the real estimate numbers can be obtained by
4513calculating 2 to the power of the given value (2average). From a statistics
4514point of view this means that:
4515
4516@itemize @bullet
4517@item 68% of the time the real size is included in the interval
4518[(2average-stddev), 2]
4519@item 95% of the time the real size is included in the interval
4520[(2average-2*stddev, 2^average+2*stddev]
4521@item 99.7% of the time the real size is included in the interval
4522[(2average-3*stddev, 2average+3*stddev]
4523@end itemize
4524
4525The expected standard variation for 64 rounds in a network of stable size is
45260.2. Thus, we can say that normally:
4527
4528@itemize @bullet
4529@item 68% of the time the real size is in the range [-13%, +15%]
4530@item 95% of the time the real size is in the range [-24%, +32%]
4531@item 99.7% of the time the real size is in the range [-34%, +52%]
4532@end itemize
4533
4534As said in the introduction, we can be quite sure that usually the real size is
4535between one third and three times the estimate. This can of course vary with
4536network conditions. Thus, applications may want to also consider the provided
4537standard deviation value, not only the average (in particular, if the standard
4538veriation is very high, the average maybe meaningless: the network size is
4539changing rapidly).
4540
4541@node Examples2
4542@subsubsection Examples2
4543
4544@c %**end of header
4545
4546Let's close with a couple examples.
4547
4548@table @asis
4549
4550@item Average: 10, std dev: 1 Here the estimate would be 2^10 = 1024 peers.@
4551The range in which we can be 95% sure is: [2^8, 2^12] = [256, 4096]. We can be
4552very (>99.7%) sure that the network is not a hundred peers and absolutely sure
4553that it is not a million peers, but somewhere around a thousand.
4554
4555@item Average 22, std dev: 0.2 Here the estimate would be 2^22 = 4 Million peers.@
4556The range in which we can be 99.7% sure is: [2^21.4, 2^22.6] = [2.8M, 6.3M].
4557We can be sure that the network size is around four million, with absolutely
4558way of it being 1 million.
4559
4560@end table
4561
4562To put this in perspective, if someone remembers the LHC Higgs boson results,
4563were announced with "5 sigma" and "6 sigma" certainties. In this case a 5 sigma
4564minimum would be 2 million and a 6 sigma minimum, 1.8 million.
4565
4566@node The NSE Client-Service Protocol
4567@subsection The NSE Client-Service Protocol
4568
4569@c %**end of header
4570
4571As with the API, the client-service protocol is very simple, only has 2
4572different messages, defined in @code{src/nse/nse.h}:
4573
4574@itemize @bullet
4575@item @code{GNUNET_MESSAGE_TYPE_NSE_START}@ This message has no parameters and
4576is sent from the client to the service upon connection.
4577@item @code{GNUNET_MESSAGE_TYPE_NSE_ESTIMATE}@ This message is sent from the
4578service to the client for every new estimate and upon connection. Contains a
4579timestamp for the estimate, the average and the standard deviation for the
4580respective round.
4581@end itemize
4582
4583When the @code{GNUNET_NSE_disconnect} API call is executed, the client simply
4584disconnects from the service, with no message involved.
4585
4586@node The NSE Peer-to-Peer Protocol
4587@subsection The NSE Peer-to-Peer Protocol
4588
4589@c %**end of header
4590
4591The NSE subsystem only has one message in the P2P protocol, the
4592@code{GNUNET_MESSAGE_TYPE_NSE_P2P_FLOOD} message.
4593
4594This message key contents are the timestamp to identify the round (differences
4595in system clocks may cause some peers to send messages way too early or way too
4596late, so the timestamp allows other peers to identify such messages easily),
4597the @uref{http://en.wikipedia.org/wiki/Proof-of-work_system, proof of work}
4598used to make it difficult to mount a
4599@uref{http://en.wikipedia.org/wiki/Sybil_attack, Sybil attack}, and the public
4600key, which is used to verify the signature on the message.
4601
4602Every peer stores a message for the previous, current and next round. The
4603messages for the previous and current round are given to peers that connect to
4604us. The message for the next round is simply stored until our system clock
4605advances to the next round. The message for the current round is what we are
4606flooding the network with right now. At the beginning of each round the peer
4607does the following:
4608
4609@itemize @bullet
4610@item calculates his own distance to the target value
4611@item creates, signs and stores the message for the current round (unless it
4612has a better message in the "next round" slot which came early in the previous
4613round)
4614@item calculates, based on the stored round message (own or received) when to
4615stard flooding it to its neighbors
4616@end itemize
4617
4618Upon receiving a message the peer checks the validity of the message (round,
4619proof of work, signature). The next action depends on the contents of the
4620incoming message:
4621
4622@itemize @bullet
4623@item if the message is worse than the current stored message, the peer sends
4624the current message back immediately, to stop the other peer from spreading
4625suboptimal results
4626@item if the message is better than the current stored message, the peer stores
4627the new message and calculates the new target time to start spreading it to its
4628neighbors (excluding the one the message came from)
4629@item if the message is for the previous round, it is compared to the message
4630stored in the "previous round slot", which may then be updated
4631@item if the message is for the next round, it is compared to the message
4632stored in the "next round slot", which again may then be updated
4633@end itemize
4634
4635Finally, when it comes to send the stored message for the current round to the
4636neighbors there is a random delay added for each neighbor, to avoid traffic
4637spikes and minimize cross-messages.
4638
4639@node GNUnet's HOSTLIST subsystem
4640@section GNUnet's HOSTLIST subsystem
4641
4642@c %**end of header
4643
4644Peers in the GNUnet overlay network need address information so that they can
4645connect with other peers. GNUnet uses so called HELLO messages to store and
4646exchange peer addresses. GNUnet provides several methods for peers to obtain
4647this information:
4648
4649@itemize @bullet
4650@item out-of-band exchange of HELLO messages (manually, using for example
4651gnunet-peerinfo)
4652@item HELLO messages shipped with GNUnet (automatic with distribution)
4653@item UDP neighbor discovery in LAN (IPv4 broadcast, IPv6 multicast)
4654@item topology gossiping (learning from other peers we already connected to),
4655and
4656@item the HOSTLIST daemon covered in this section, which is particularly
4657relevant for bootstrapping new peers.
4658@end itemize
4659
4660New peers have no existing connections (and thus cannot learn from gossip among
4661peers), may not have other peers in their LAN and might be started with an
4662outdated set of HELLO messages from the distribution. In this case, getting new
4663peers to connect to the network requires either manual effort or the use of a
4664HOSTLIST to obtain HELLOs.
4665
4666@menu
4667* HELLOs::
4668* Overview for the HOSTLIST subsystem::
4669* Interacting with the HOSTLIST daemon::
4670* Hostlist security address validation::
4671* The HOSTLIST daemon::
4672* The HOSTLIST server::
4673* The HOSTLIST client::
4674* Usage::
4675@end menu
4676
4677@node HELLOs
4678@subsection HELLOs
4679
4680@c %**end of header
4681
4682The basic information peers require to connect to other peers are contained in
4683so called HELLO messages you can think of as a business card. Besides the
4684identity of the peer (based on the cryptographic public key) a HELLO message
4685may contain address information that specifies ways to contact a peer. By
4686obtaining HELLO messages, a peer can learn how to contact other peers.
4687
4688@node Overview for the HOSTLIST subsystem
4689@subsection Overview for the HOSTLIST subsystem
4690
4691@c %**end of header
4692
4693The HOSTLIST subsystem provides a way to distribute and obtain contact
4694information to connect to other peers using a simple HTTP GET request. It's
4695implementation is split in three parts, the main file for the daemon itself
4696(gnunet-daemon-hostlist.c), the HTTP client used to download peer information
4697(hostlist-client.c) and the server component used to provide this information
4698to other peers (hostlist-server.c). The server is basically a small HTTP web
4699server (based on GNU libmicrohttpd) which provides a list of HELLOs known to
4700the local peer for download. The client component is basically a HTTP client
4701(based on libcurl) which can download hostlists from one or more websites. The
4702hostlist format is a binary blob containing a sequence of HELLO messages. Note
4703that any HTTP server can theoretically serve a hostlist, the build-in hostlist
4704server makes it simply convenient to offer this service.
4705
4706
4707@menu
4708* Features::
4709* Limitations2::
4710@end menu
4711
4712@node Features
4713@subsubsection Features
4714
4715@c %**end of header
4716
4717The HOSTLIST daemon can:
4718
4719@itemize @bullet
4720@item provide HELLO messages with validated addresses obtained from PEERINFO to
4721download for other peers
4722@item download HELLO messages and forward these message to the TRANSPORT
4723subsystem for validation
4724@item advertises the URL of this peer's hostlist address to other peers via
4725gossip
4726@item automatically learn about hostlist servers from the gossip of other peers
4727@end itemize
4728
4729@node Limitations2
4730@subsubsection Limitations2
4731
4732@c %**end of header
4733
4734The HOSTLIST daemon does not:
4735
4736@itemize @bullet
4737@item verify the cryptographic information in the HELLO messages
4738@item verify the address information in the HELLO messages
4739@end itemize
4740
4741@node Interacting with the HOSTLIST daemon
4742@subsection Interacting with the HOSTLIST daemon
4743
4744@c %**end of header
4745
4746The HOSTLIST subsystem is currently implemented as a daemon, so there is no
4747need for the user to interact with it and therefore there is no command line
4748tool and no API to communicate with the daemon. In the future, we can envision
4749changing this to allow users to manually trigger the download of a hostlist.
4750
4751Since there is no command line interface to interact with HOSTLIST, the only
4752way to interact with the hostlist is to use STATISTICS to obtain or modify
4753information about the status of HOSTLIST:
4754@example
4755$ gnunet-statistics -s hostlist
4756@end example
4757
4758In particular, HOSTLIST includes a @strong{persistent} value in statistics that
4759specifies when the hostlist server might be queried next. As this value is
4760exponentially increasing during runtime, developers may want to reset or
4761manually adjust it. Note that HOSTLIST (but not STATISTICS) needs to be
4762shutdown if changes to this value are to have any effect on the daemon (as
4763HOSTLIST does not monitor STATISTICS for changes to the download
4764frequency).
4765
4766@node Hostlist security address validation
4767@subsection Hostlist security address validation
4768
4769@c %**end of header
4770
4771Since information obtained from other parties cannot be trusted without
4772validation, we have to distinguish between @emph{validated} and @emph{not
4773validated} addresses. Before using (and so trusting) information from other
4774parties, this information has to be double-checked (validated). Address
4775validation is not done by HOSTLIST but by the TRANSPORT service.
4776
4777The HOSTLIST component is functionally located between the PEERINFO and the
4778TRANSPORT subsystem. When acting as a server, the daemon obtains valid
4779(@emph{validated}) peer information (HELLO messages) from the PEERINFO service
4780and provides it to other peers. When acting as a client, it contacts the
4781HOSTLIST servers specified in the configuration, downloads the (unvalidated)
4782list of HELLO messages and forwards these information to the TRANSPORT server
4783to validate the addresses.
4784
4785@node The HOSTLIST daemon
4786@subsection The HOSTLIST daemon
4787
4788@c %**end of header
4789
4790The hostlist daemon is the main component of the HOSTLIST subsystem. It is
4791started by the ARM service and (if configured) starts the HOSTLIST client and
4792server components.
4793
4794If the daemon provides a hostlist itself it can advertise it's own hostlist to
4795other peers. To do so it sends a GNUNET_MESSAGE_TYPE_HOSTLIST_ADVERTISEMENT
4796message to other peers when they connect to this peer on the CORE level. This
4797hostlist advertisement message contains the URL to access the HOSTLIST HTTP
4798server of the sender. The daemon may also subscribe to this type of message
4799from CORE service, and then forward these kind of message to the HOSTLIST
4800client. The client then uses all available URLs to download peer information
4801when necessary.
4802
4803When starting, the HOSTLIST daemon first connects to the CORE subsystem and if
4804hostlist learning is enabled, registers a CORE handler to receive this kind of
4805messages. Next it starts (if configured) the client and server. It passes
4806pointers to CORE connect and disconnect and receive handlers where the client
4807and server store their functions, so the daemon can notify them about CORE
4808events.
4809
4810To clean up on shutdown, the daemon has a cleaning task, shutting down all
4811subsystems and disconnecting from CORE.
4812
4813@node The HOSTLIST server
4814@subsection The HOSTLIST server
4815
4816@c %**end of header
4817
4818The server provides a way for other peers to obtain HELLOs. Basically it is a
4819small web server other peers can connect to and download a list of HELLOs using
4820standard HTTP; it may also advertise the URL of the hostlist to other peers
4821connecting on CORE level.
4822
4823
4824@menu
4825* The HTTP Server::
4826* Advertising the URL::
4827@end menu
4828
4829@node The HTTP Server
4830@subsubsection The HTTP Server
4831
4832@c %**end of header
4833
4834During startup, the server starts a web server listening on the port specified
4835with the HTTPPORT value (default 8080). In addition it connects to the PEERINFO
4836service to obtain peer information. The HOSTLIST server uses the
4837GNUNET_PEERINFO_iterate function to request HELLO information for all peers and
4838adds their information to a new hostlist if they are suitable (expired
4839addresses and HELLOs without addresses are both not suitable) and the maximum
4840size for a hostlist is not exceeded (MAX_BYTES_PER_HOSTLISTS = 500000). When
4841PEERINFO finishes (with a last NULL callback), the server destroys the previous
4842hostlist response available for download on the web server and replaces it with
4843the updated hostlist. The hostlist format is basically a sequence of HELLO
4844messages (as obtained from PEERINFO) without any special tokenization. Since
4845each HELLO message contains a size field, the response can easily be split into
4846separate HELLO messages by the client.
4847
4848A HOSTLIST client connecting to the HOSTLIST server will receive the hostlist
4849as a HTTP response and the the server will terminate the connection with the
4850result code HTTP 200 OK. The connection will be closed immediately if no
4851hostlist is available.
4852
4853@node Advertising the URL
4854@subsubsection Advertising the URL
4855
4856@c %**end of header
4857
4858The server also advertises the URL to download the hostlist to other peers if
4859hostlist advertisement is enabled. When a new peer connects and has hostlist
4860learning enabled, the server sends a GNUNET_MESSAGE_TYPE_HOSTLIST_ADVERTISEMENT
4861message to this peer using the CORE service.
4862
4863@node The HOSTLIST client
4864@subsection The HOSTLIST client
4865
4866@c %**end of header
4867
4868The client provides the functionality to download the list of HELLOs from a set
4869of URLs. It performs a standard HTTP request to the URLs configured and learned
4870from advertisement messages received from other peers. When a HELLO is
4871downloaded, the HOSTLIST client forwards the HELLO to the TRANSPORT service for
4872validation.
4873
4874The client supports two modes of operation: download of HELLOs (bootstrapping)
4875and learning of URLs.
4876
4877
4878@menu
4879* Bootstrapping::
4880* Learning::
4881@end menu
4882
4883@node Bootstrapping
4884@subsubsection Bootstrapping
4885
4886@c %**end of header
4887
4888For bootstrapping, it schedules a task to download the hostlist from the set of
4889known URLs. The downloads are only performed if the number of current
4890connections is smaller than a minimum number of connections (at the moment 4).
4891The interval between downloads increases exponentially; however, the
4892exponential growth is limited if it becomes longer than an hour. At that point,
4893the frequency growth is capped at (#number of connections * 1h).
4894
4895Once the decision has been taken to download HELLOs, the daemon chooses a
4896random URL from the list of known URLs. URLs can be configured in the
4897configuration or be learned from advertisement messages. The client uses a HTTP
4898client library (libcurl) to initiate the download using the libcurl multi
4899interface. Libcurl passes the data to the callback_download function which
4900stores the data in a buffer if space is available and the maximum size for a
4901hostlist download is not exceeded (MAX_BYTES_PER_HOSTLISTS = 500000). When a
4902full HELLO was downloaded, the HOSTLIST client offers this HELLO message to the
4903TRANSPORT service for validation. When the download is finished or failed,
4904statistical information about the quality of this URL is updated.
4905
4906@node Learning
4907@subsubsection Learning
4908
4909@c %**end of header
4910
4911The client also manages hostlist advertisements from other peers. The HOSTLIST
4912daemon forwards GNUNET_MESSAGE_TYPE_HOSTLIST_ADVERTISEMENT messages to the
4913client subsystem, which extracts the URL from the message. Next, a test of the
4914newly obtained URL is performed by triggering a download from the new URL. If
4915the URL works correctly, it is added to the list of working URLs.
4916
4917The size of the list of URLs is restricted, so if an additional server is added
4918and the list is full, the URL with the worst quality ranking (determined
4919through successful downloads and number of HELLOs e.g.) is discarded. During
4920shutdown the list of URLs is saved to a file for persistance and loaded on
4921startup. URLs from the configuration file are never discarded.
4922
4923@node Usage
4924@subsection Usage
4925
4926@c %**end of header
4927
4928To start HOSTLIST by default, it has to be added to the DEFAULTSERVICES section
4929for the ARM services. This is done in the default configuration.
4930
4931For more information on how to configure the HOSTLIST subsystem see the
4932installation handbook:@ Configuring the hostlist to bootstrap@ Configuring your
4933peer to provide a hostlist
4934
4935@node GNUnet's IDENTITY subsystem
4936@section GNUnet's IDENTITY subsystem
4937
4938@c %**end of header
4939
4940Identities of "users" in GNUnet are called egos. Egos can be used as pseudonyms
4941(fake names) or be tied to an organization (for example, GNU) or even the
4942actual identity of a human. GNUnet users are expected to have many egos. They
4943might have one tied to their real identity, some for organizations they manage,
4944and more for different domains where they want to operate under a pseudonym.
4945
4946The IDENTITY service allows users to manage their egos. The identity service
4947manages the private keys egos of the local user; it does not manage identities
4948of other users (public keys). Public keys for other users need names to become
4949manageable. GNUnet uses the GNU Name System (GNS) to give names to other users
4950and manage their public keys securely. This chapter is about the IDENTITY
4951service, which is about the management of private keys.
4952
4953On the network, an ego corresponds to an ECDSA key (over Curve25519, using RFC
49546979, as required by GNS). Thus, users can perform actions under a particular
4955ego by using (signing with) a particular private key. Other users can then
4956confirm that the action was really performed by that ego by checking the
4957signature against the respective public key.
4958
4959The IDENTITY service allows users to associate a human-readable name with each
4960ego. This way, users can use names that will remind them of the purpose of a
4961particular ego. The IDENTITY service will store the respective private keys and
4962allows applications to access key information by name. Users can change the
4963name that is locally (!) associated with an ego. Egos can also be deleted,
4964which means that the private key will be removed and it thus will not be
4965possible to perform actions with that ego in the future.
4966
4967Additionally, the IDENTITY subsystem can associate service functions with egos.
4968For example, GNS requires the ego that should be used for the shorten zone. GNS
4969will ask IDENTITY for an ego for the "gns-short" service. The IDENTITY service
4970has a mapping of such service strings to the name of the ego that the user
4971wants to use for this service, for example "my-short-zone-ego".
4972
4973Finally, the IDENTITY API provides access to a special ego, the anonymous ego.
4974The anonymous ego is special in that its private key is not really private, but
4975fixed and known to everyone. Thus, anyone can perform actions as anonymous.
4976This can be useful as with this trick, code does not have to contain a special
4977case to distinguish between anonymous and pseudonymous egos.
4978
4979@menu
4980* libgnunetidentity::
4981* The IDENTITY Client-Service Protocol::
4982@end menu
4983
4984@node libgnunetidentity
4985@subsection libgnunetidentity
4986@c %**end of header
4987
4988
4989@menu
4990* Connecting to the service::
4991* Operations on Egos::
4992* The anonymous Ego::
4993* Convenience API to lookup a single ego::
4994* Associating egos with service functions::
4995@end menu
4996
4997@node Connecting to the service
4998@subsubsection Connecting to the service
4999
5000@c %**end of header
5001
5002First, typical clients connect to the identity service using
5003@code{GNUNET_IDENTITY_connect}. This function takes a callback as a parameter.
5004If the given callback parameter is non-null, it will be invoked to notify the
5005application about the current state of the identities in the system.
5006
5007@itemize @bullet
5008@item First, it will be invoked on all known egos at the time of the
5009connection. For each ego, a handle to the ego and the user's name for the ego
5010will be passed to the callback. Furthermore, a @code{void **} context argument
5011will be provided which gives the client the opportunity to associate some state
5012with the ego.
5013@item Second, the callback will be invoked with NULL for the ego, the name and
5014the context. This signals that the (initial) iteration over all egos has
5015completed.
5016@item Then, the callback will be invoked whenever something changes about an
5017ego. If an ego is renamed, the callback is invoked with the ego handle of the
5018ego that was renamed, and the new name. If an ego is deleted, the callback is
5019invoked with the ego handle and a name of NULL. In the deletion case, the
5020application should also release resources stored in the context.
5021@item When the application destroys the connection to the identity service
5022using @code{GNUNET_IDENTITY_disconnect}, the callback is again invoked with the
5023ego and a name of NULL (equivalent to deletion of the egos). This should again
5024be used to clean up the per-ego context.
5025@end itemize
5026
5027The ego handle passed to the callback remains valid until the callback is
5028invoked with a name of NULL, so it is safe to store a reference to the ego's
5029handle.
5030
5031@node Operations on Egos
5032@subsubsection Operations on Egos
5033
5034@c %**end of header
5035
5036Given an ego handle, the main operations are to get its associated private key
5037using @code{GNUNET_IDENTITY_ego_get_private_key} or its associated public key
5038using @code{GNUNET_IDENTITY_ego_get_public_key}.
5039
5040The other operations on egos are pretty straightforward. Using
5041@code{GNUNET_IDENTITY_create}, an application can request the creation of an
5042ego by specifying the desired name. The operation will fail if that name is
5043already in use. Using @code{GNUNET_IDENTITY_rename} the name of an existing ego
5044can be changed. Finally, egos can be deleted using
5045@code{GNUNET_IDENTITY_delete}. All of these operations will trigger updates to
5046the callback given to the @code{GNUNET_IDENTITY_connect} function of all
5047applications that are connected with the identity service at the time.
5048@code{GNUNET_IDENTITY_cancel} can be used to cancel the operations before the
5049respective continuations would be called. It is not guaranteed that the
5050operation will not be completed anyway, only the continuation will no longer be
5051called.
5052
5053@node The anonymous Ego
5054@subsubsection The anonymous Ego
5055
5056@c %**end of header
5057
5058A special way to obtain an ego handle is to call
5059@code{GNUNET_IDENTITY_ego_get_anonymous}, which returns an ego for the
5060"anonymous" user --- anyone knows and can get the private key for this user, so
5061it is suitable for operations that are supposed to be anonymous but require
5062signatures (for example, to avoid a special path in the code). The anonymous
5063ego is always valid and accessing it does not require a connection to the
5064identity service.
5065
5066@node Convenience API to lookup a single ego
5067@subsubsection Convenience API to lookup a single ego
5068
5069
5070As applications commonly simply have to lookup a single ego, there is a
5071convenience API to do just that. Use @code{GNUNET_IDENTITY_ego_lookup} to
5072lookup a single ego by name. Note that this is the user's name for the ego, not
5073the service function. The resulting ego will be returned via a callback and
5074will only be valid during that callback. The operation can be cancelled via
5075@code{GNUNET_IDENTITY_ego_lookup_cancel} (cancellation is only legal before the
5076callback is invoked).
5077
5078@node Associating egos with service functions
5079@subsubsection Associating egos with service functions
5080
5081
5082The @code{GNUNET_IDENTITY_set} function is used to associate a particular ego
5083with a service function. The name used by the service and the ego are given as
5084arguments. Afterwards, the service can use its name to lookup the associated
5085ego using @code{GNUNET_IDENTITY_get}.
5086
5087@node The IDENTITY Client-Service Protocol
5088@subsection The IDENTITY Client-Service Protocol
5089
5090@c %**end of header
5091
5092A client connecting to the identity service first sends a message with type
5093@code{GNUNET_MESSAGE_TYPE_IDENTITY_START} to the service. After that, the
5094client will receive information about changes to the egos by receiving messages
5095of type @code{GNUNET_MESSAGE_TYPE_IDENTITY_UPDATE}. Those messages contain the
5096private key of the ego and the user's name of the ego (or zero bytes for the
5097name to indicate that the ego was deleted). A special bit @code{end_of_list} is
5098used to indicate the end of the initial iteration over the identity service's
5099egos.
5100
5101The client can trigger changes to the egos by sending CREATE, RENAME or DELETE
5102messages. The CREATE message contains the private key and the desired name. The
5103RENAME message contains the old name and the new name. The DELETE message only
5104needs to include the name of the ego to delete. The service responds to each of
5105these messages with a RESULT_CODE message which indicates success or error of
5106the operation, and possibly a human-readable error message.
5107
5108Finally, the client can bind the name of a service function to an ego by
5109sending a SET_DEFAULT message with the name of the service function and the
5110private key of the ego. Such bindings can then be resolved using a GET_DEFAULT
5111message, which includes the name of the service function. The identity service
5112will respond to a GET_DEFAULT request with a SET_DEFAULT message containing the
5113respective information, or with a RESULT_CODE to indicate an error.
5114
5115@node GNUnet's NAMESTORE Subsystem
5116@section GNUnet's NAMESTORE Subsystem
5117
5118@c %**end of header
5119
5120The NAMESTORE subsystem provides persistent storage for local GNS zone
5121information. All local GNS zone information are managed by NAMESTORE. It
5122provides both the functionality to administer local GNS information (e.g.
5123delete and add records) as well as to retrieve GNS information (e.g to list
5124name information in a client). NAMESTORE does only manage the persistent
5125storage of zone information belonging to the user running the service: GNS
5126information from other users obtained from the DHT are stored by the NAMECACHE
5127subsystem.
5128
5129NAMESTORE uses a plugin-based database backend to store GNS information with
5130good performance. Here sqlite, MySQL and PostgreSQL are supported database
5131backends. NAMESTORE clients interact with the IDENTITY subsystem to obtain
5132cryptographic information about zones based on egos as described with the
5133IDENTITY subsystem., but internally NAMESTORE refers to zones using the ECDSA
5134private key. In addition, it collaborates with the NAMECACHE subsystem and
5135stores zone information when local information are modified in the GNS cache to
5136increase look-up performance for local information.
5137
5138NAMESTORE provides functionality to look-up and store records, to iterate over
5139a specific or all zones and to monitor zones for changes. NAMESTORE
5140functionality can be accessed using the NAMESTORE api or the NAMESTORE command
5141line tool.
5142
5143@menu
5144* libgnunetnamestore::
5145@end menu
5146
5147@node libgnunetnamestore
5148@subsection libgnunetnamestore
5149
5150@c %**end of header
5151
5152To interact with NAMESTORE clients first connect to the NAMESTORE service using
5153the @code{GNUNET_NAMESTORE_connect} passing a configuration handle. As a result
5154they obtain a NAMESTORE handle, they can use for operations, or NULL is
5155returned if the connection failed.
5156
5157To disconnect from NAMESTORE, clients use @code{GNUNET_NAMESTORE_disconnect}
5158and specify the handle to disconnect.
5159
5160NAMESTORE internally uses the ECDSA private key to refer to zones. These
5161private keys can be obtained from the IDENTITY subsytem. Here @emph{egos@emph{
5162can be used to refer to zones or the default ego assigned to the GNS subsystem
5163can be used to obtained the master zone's private key.}}
5164
5165
5166@menu
5167* Editing Zone Information::
5168* Iterating Zone Information::
5169* Monitoring Zone Information::
5170@end menu
5171
5172@node Editing Zone Information
5173@subsubsection Editing Zone Information
5174
5175@c %**end of header
5176
5177NAMESTORE provides functions to lookup records stored under a label in a zone
5178and to store records under a label in a zone.
5179
5180To store (and delete) records, the client uses the
5181@code{GNUNET_NAMESTORE_records_store} function and has to provide namestore
5182handle to use, the private key of the zone, the label to store the records
5183under, the records and number of records plus an callback function. After the
5184operation is performed NAMESTORE will call the provided callback function with
5185the result GNUNET_SYSERR on failure (including timeout/queue drop/failure to
5186validate), GNUNET_NO if content was already there or not found GNUNET_YES (or
5187other positive value) on success plus an additional error message.
5188
5189Records are deleted by using the store command with 0 records to store. It is
5190important to note, that records are not merged when records exist with the
5191label. So a client has first to retrieve records, merge with existing records
5192and then store the result.
5193
5194To perform a lookup operation, the client uses the
5195@code{GNUNET_NAMESTORE_records_store} function. Here he has to pass the
5196namestore handle, the private key of the zone and the label. He also has to
5197provide a callback function which will be called with the result of the lookup
5198operation: the zone for the records, the label, and the records including the
5199number of records included.
5200
5201A special operation is used to set the preferred nickname for a zone. This
5202nickname is stored with the zone and is automatically merged with all labels
5203and records stored in a zone. Here the client uses the
5204@code{GNUNET_NAMESTORE_set_nick} function and passes the private key of the
5205zone, the nickname as string plus a the callback with the result of the
5206operation.
5207
5208@node Iterating Zone Information
5209@subsubsection Iterating Zone Information
5210
5211@c %**end of header
5212
5213A client can iterate over all information in a zone or all zones managed by
5214NAMESTORE. Here a client uses the @code{GNUNET_NAMESTORE_zone_iteration_start}
5215function and passes the namestore handle, the zone to iterate over and a
5216callback function to call with the result. If the client wants to iterate over
5217all the, he passes NULL for the zone. A @code{GNUNET_NAMESTORE_ZoneIterator}
5218handle is returned to be used to continue iteration.
5219
5220NAMESTORE calls the callback for every result and expects the client to call@
5221@code{GNUNET_NAMESTORE_zone_iterator_next} to continue to iterate or
5222@code{GNUNET_NAMESTORE_zone_iterator_stop} to interrupt the iteration. When
5223NAMESTORE reached the last item it will call the callback with a NULL value to
5224indicate.
5225
5226@node Monitoring Zone Information
5227@subsubsection Monitoring Zone Information
5228
5229@c %**end of header
5230
5231Clients can also monitor zones to be notified about changes. Here the clients
5232uses the @code{GNUNET_NAMESTORE_zone_monitor_start} function and passes the
5233private key of the zone and and a callback function to call with updates for a
5234zone. The client can specify to obtain zone information first by iterating over
5235the zone and specify a synchronization callback to be called when the client
5236and the namestore are synced.
5237
5238On an update, NAMESTORE will call the callback with the private key of the
5239zone, the label and the records and their number.
5240
5241To stop monitoring, the client call @code{GNUNET_NAMESTORE_zone_monitor_stop}
5242and passes the handle obtained from the function to start the monitoring.
5243
5244@node GNUnet's PEERINFO subsystem
5245@section GNUnet's PEERINFO subsystem
5246
5247@c %**end of header
5248
5249The PEERINFO subsystem is used to store verified (validated) information about
5250known peers in a persistent way. It obtains these addresses for example from
5251TRANSPORT service which is in charge of address validation. Validation means
5252that the information in the HELLO message are checked by connecting to the
5253addresses and performing a cryptographic handshake to authenticate the peer
5254instance stating to be reachable with these addresses. Peerinfo does not
5255validate the HELLO messages itself but only stores them and gives them to
5256interested clients.
5257
5258As future work, we think about moving from storing just HELLO messages to
5259providing a generic persistent per-peer information store. More and more
5260subsystems tend to need to store per-peer information in persistent way. To not
5261duplicate this functionality we plan to provide a PEERSTORE service providing
5262this functionality
5263
5264@menu
5265* Features2::
5266* Limitations3::
5267* DeveloperPeer Information::
5268* Startup::
5269* Managing Information::
5270* Obtaining Information::
5271* The PEERINFO Client-Service Protocol::
5272* libgnunetpeerinfo::
5273@end menu
5274
5275@node Features2
5276@subsection Features2
5277
5278@c %**end of header
5279
5280@itemize @bullet
5281@item Persistent storage
5282@item Client notification mechanism on update
5283@item Periodic clean up for expired information
5284@item Differentiation between public and friend-only HELLO
5285@end itemize
5286
5287@node Limitations3
5288@subsection Limitations3
5289
5290
5291@itemize @bullet
5292@item Does not perform HELLO validation
5293@end itemize
5294
5295@node DeveloperPeer Information
5296@subsection DeveloperPeer Information
5297
5298@c %**end of header
5299
5300The PEERINFO subsystem stores these information in the form of HELLO messages
5301you can think of as business cards. These HELLO messages contain the public key
5302of a peer and the addresses a peer can be reached under. The addresses include
5303an expiration date describing how long they are valid. This information is
5304updated regularly by the TRANSPORT service by revalidating the address. If an
5305address is expired and not renewed, it can be removed from the HELLO message.
5306
5307Some peer do not want to have their HELLO messages distributed to other peers ,
5308especially when GNUnet's friend-to-friend modus is enabled. To prevent this
5309undesired distribution. PEERINFO distinguishes between @emph{public} and
5310@emph{friend-only} HELLO messages. Public HELLO messages can be freely
5311distributed to other (possibly unknown) peers (for example using the hostlist,
5312gossiping, broadcasting), whereas friend-only HELLO messages may not be
5313distributed to other peers. Friend-only HELLO messages have an additional flag
5314@code{friend_only} set internally. For public HELLO message this flag is not
5315set. PEERINFO does and cannot not check if a client is allowed to obtain a
5316specific HELLO type.
5317
5318The HELLO messages can be managed using the GNUnet HELLO library. Other GNUnet
5319systems can obtain these information from PEERINFO and use it for their
5320purposes. Clients are for example the HOSTLIST component providing these
5321information to other peers in form of a hostlist or the TRANSPORT subsystem
5322using these information to maintain connections to other peers.
5323
5324@node Startup
5325@subsection Startup
5326
5327@c %**end of header
5328
5329During startup the PEERINFO services loads persistent HELLOs from disk. First
5330PEERINFO parses the directory configured in the HOSTS value of the
5331@code{PEERINFO} configuration section to store PEERINFO information.@ For all
5332files found in this directory valid HELLO messages are extracted. In addition
5333it loads HELLO messages shipped with the GNUnet distribution. These HELLOs are
5334used to simplify network bootstrapping by providing valid peer information with
5335the distribution. The use of these HELLOs can be prevented by setting the
5336@code{USE_INCLUDED_HELLOS} in the @code{PEERINFO} configuration section to
5337@code{NO}. Files containing invalid information are removed.
5338
5339@node Managing Information
5340@subsection Managing Information
5341
5342@c %**end of header
5343
5344The PEERINFO services stores information about known PEERS and a single HELLO
5345message for every peer. A peer does not need to have a HELLO if no information
5346are available. HELLO information from different sources, for example a HELLO
5347obtained from a remote HOSTLIST and a second HELLO stored on disk, are combined
5348and merged into one single HELLO message per peer which will be given to
5349clients. During this merge process the HELLO is immediately written to disk to
5350ensure persistence.
5351
5352PEERINFO in addition periodically scans the directory where information are
5353stored for empty HELLO messages with expired TRANSPORT addresses.@ This
5354periodic task scans all files in the directory and recreates the HELLO messages
5355it finds. Expired TRANSPORT addresses are removed from the HELLO and if the
5356HELLO does not contain any valid addresses, it is discarded and removed from
5357disk.
5358
5359@node Obtaining Information
5360@subsection Obtaining Information
5361
5362@c %**end of header
5363
5364When a client requests information from PEERINFO, PEERINFO performs a lookup
5365for the respective peer or all peers if desired and transmits this information
5366to the client. The client can specify if friend-only HELLOs have to be included
5367or not and PEERINFO filters the respective HELLO messages before transmitting
5368information.
5369
5370To notify clients about changes to PEERINFO information, PEERINFO maintains a
5371list of clients interested in this notifications. Such a notification occurs if
5372a HELLO for a peer was updated (due to a merge for example) or a new peer was
5373added.
5374
5375@node The PEERINFO Client-Service Protocol
5376@subsection The PEERINFO Client-Service Protocol
5377
5378@c %**end of header
5379
5380To connect and disconnect to and from the PEERINFO Service PEERINFO utilizes
5381the util client/server infrastructure, so no special messages types are used
5382here.
5383
5384To add information for a peer, the plain HELLO message is transmitted to the
5385service without any wrapping. Alle information required are stored within the
5386HELLO message. The PEERINFO service provides a message handler accepting and
5387processing these HELLO messages.
5388
5389When obtaining PEERINFO information using the iterate functionality specific
5390messages are used. To obtain information for all peers, a @code{struct
5391ListAllPeersMessage} with message type
5392@code{GNUNET_MESSAGE_TYPE_PEERINFO_GET_ALL} and a flag include_friend_only to
5393indicate if friend-only HELLO messages should be included are transmitted. If
5394information for a specific peer is required a @code{struct ListAllPeersMessage}
5395with @code{GNUNET_MESSAGE_TYPE_PEERINFO_GET} containing the peer identity is
5396used.
5397
5398For both variants the PEERINFO service replies for each HELLO message he wants
5399to transmit with a @code{struct ListAllPeersMessage} with type
5400@code{GNUNET_MESSAGE_TYPE_PEERINFO_INFO} containing the plain HELLO. The final
5401message is @code{struct GNUNET_MessageHeader} with type
5402@code{GNUNET_MESSAGE_TYPE_PEERINFO_INFO}. If the client receives this message,
5403he can proceed with the next request if any is pending
5404
5405@node libgnunetpeerinfo
5406@subsection libgnunetpeerinfo
5407
5408@c %**end of header
5409
5410The PEERINFO API consists mainly of three different functionalities:
5411maintaining a connection to the service, adding new information and retrieving
5412information form the PEERINFO service.
5413
5414
5415@menu
5416* Connecting to the Service::
5417* Adding Information::
5418* Obtaining Information2::
5419@end menu
5420
5421@node Connecting to the Service
5422@subsubsection Connecting to the Service
5423
5424@c %**end of header
5425
5426To connect to the PEERINFO service the function @code{GNUNET_PEERINFO_connect}
5427is used, taking a configuration handle as an argument, and to disconnect from
5428PEERINFO the function @code{GNUNET_PEERINFO_disconnect}, taking the PEERINFO
5429handle returned from the connect function has to be called.
5430
5431@node Adding Information
5432@subsubsection Adding Information
5433
5434@c %**end of header
5435
5436@code{GNUNET_PEERINFO_add_peer} adds a new peer to the PEERINFO subsystem
5437storage. This function takes the PEERINFO handle as an argument, the HELLO
5438message to store and a continuation with a closure to be called with the result
5439of the operation. The @code{GNUNET_PEERINFO_add_peer} returns a handle to this
5440operation allowing to cancel the operation with the respective cancel function
5441@code{GNUNET_PEERINFO_add_peer_cancel}. To retrieve information from PEERINFO
5442you can iterate over all information stored with PEERINFO or you can tell
5443PEERINFO to notify if new peer information are available.
5444
5445@node Obtaining Information2
5446@subsubsection Obtaining Information2
5447
5448@c %**end of header
5449
5450To iterate over information in PEERINFO you use @code{GNUNET_PEERINFO_iterate}.
5451This function expects the PEERINFO handle, a flag if HELLO messages intended
5452for friend only mode should be included, a timeout how long the operation
5453should take and a callback with a callback closure to be called for the
5454results. If you want to obtain information for a specific peer, you can specify
5455the peer identity, if this identity is NULL, information for all peers are
5456returned. The function returns a handle to allow to cancel the operation using
5457@code{GNUNET_PEERINFO_iterate_cancel}.
5458
5459To get notified when peer information changes, you can use
5460@code{GNUNET_PEERINFO_notify}. This function expects a configuration handle and
5461a flag if friend-only HELLO messages should be included. The PEERINFO service
5462will notify you about every change and the callback function will be called to
5463notify you about changes. The function returns a handle to cancel notifications
5464with @code{GNUNET_PEERINFO_notify_cancel}.
5465
5466
5467@node GNUnet's PEERSTORE subsystem
5468@section GNUnet's PEERSTORE subsystem
5469
5470@c %**end of header
5471
5472GNUnet's PEERSTORE subsystem offers persistent per-peer storage for other
5473GNUnet subsystems. GNUnet subsystems can use PEERSTORE to persistently store
5474and retrieve arbitrary data. Each data record stored with PEERSTORE contains
5475the following fields:
5476
5477@itemize @bullet
5478@item subsystem: Name of the subsystem responsible for the record.
5479@item peerid: Identity of the peer this record is related to.
5480@item key: a key string identifying the record.
5481@item value: binary record value.
5482@item expiry: record expiry date.
5483@end itemize
5484
5485@menu
5486* Functionality::
5487* Architecture::
5488* libgnunetpeerstore::
5489@end menu
5490
5491@node Functionality
5492@subsection Functionality
5493
5494@c %**end of header
5495
5496Subsystems can store any type of value under a (subsystem, peerid, key)
5497combination. A "replace" flag set during store operations forces the PEERSTORE
5498to replace any old values stored under the same (subsystem, peerid, key)
5499combination with the new value. Additionally, an expiry date is set after which
5500the record is *possibly* deleted by PEERSTORE.
5501
5502Subsystems can iterate over all values stored under any of the following
5503combination of fields:
5504
5505@itemize @bullet
5506@item (subsystem)
5507@item (subsystem, peerid)
5508@item (subsystem, key)
5509@item (subsystem, peerid, key)
5510@end itemize
5511
5512Subsystems can also request to be notified about any new values stored under a
5513(subsystem, peerid, key) combination by sending a "watch" request to
5514PEERSTORE.
5515
5516@node Architecture
5517@subsection Architecture
5518
5519@c %**end of header
5520
5521PEERSTORE implements the following components:
5522
5523@itemize @bullet
5524@item PEERSTORE service: Handles store, iterate and watch operations.
5525@item PEERSTORE API: API to be used by other subsystems to communicate and
5526issue commands to the PEERSTORE service.
5527@item PEERSTORE plugins: Handles the persistent storage. At the moment, only an
5528"sqlite" plugin is implemented.
5529@end itemize
5530
5531@node libgnunetpeerstore
5532@subsection libgnunetpeerstore
5533
5534@c %**end of header
5535
5536libgnunetpeerstore is the library containing the PEERSTORE API. Subsystems
5537wishing to communicate with the PEERSTORE service use this API to open a
5538connection to PEERSTORE. This is done by calling
5539@code{GNUNET_PEERSTORE_connect} which returns a handle to the newly created
5540connection. This handle has to be used with any further calls to the API.
5541
5542To store a new record, the function @code{GNUNET_PEERSTORE_store} is to be used
5543which requires the record fields and a continuation function that will be
5544called by the API after the STORE request is sent to the PEERSTORE service.
5545Note that calling the continuation function does not mean that the record is
5546successfully stored, only that the STORE request has been successfully sent to
5547the PEERSTORE service. @code{GNUNET_PEERSTORE_store_cancel} can be called to
5548cancel the STORE request only before the continuation function has been called.
5549
5550To iterate over stored records, the function @code{GNUNET_PEERSTORE_iterate} is
5551to be used. @emph{peerid} and @emph{key} can be set to NULL. An iterator
5552callback function will be called with each matching record found and a NULL
5553record at the end to signal the end of result set.
5554@code{GNUNET_PEERSTORE_iterate_cancel} can be used to cancel the ITERATE
5555request before the iterator callback is called with a NULL record.
5556
5557To be notified with new values stored under a (subsystem, peerid, key)
5558combination, the function @code{GNUNET_PEERSTORE_watch} is to be used. This
5559will register the watcher with the PEERSTORE service, any new records matching
5560the given combination will trigger the callback function passed to
5561@code{GNUNET_PEERSTORE_watch}. This continues until
5562@code{GNUNET_PEERSTORE_watch_cancel} is called or the connection to the service
5563is destroyed.
5564
5565After the connection is no longer needed, the function
5566@code{GNUNET_PEERSTORE_disconnect} can be called to disconnect from the
5567PEERSTORE service. Any pending ITERATE or WATCH requests will be destroyed. If
5568the @code{sync_first} flag is set to @code{GNUNET_YES}, the API will delay the
5569disconnection until all pending STORE requests are sent to the PEERSTORE
5570service, otherwise, the pending STORE requests will be destroyed as well.
5571
5572@node GNUnet's SET Subsystem
5573@section GNUnet's SET Subsystem
5574
5575@c %**end of header
5576
5577The SET service implements efficient set operations between two peers over a
5578mesh tunnel. Currently, set union and set intersection are the only supported
5579operations. Elements of a set consist of an @emph{element type} and arbitrary
5580binary @emph{data}. The size of an element's data is limited to around 62
5581KB.
5582
5583@menu
5584* Local Sets::
5585* Set Modifications::
5586* Set Operations::
5587* Result Elements::
5588* libgnunetset::
5589* The SET Client-Service Protocol::
5590* The SET Intersection Peer-to-Peer Protocol::
5591* The SET Union Peer-to-Peer Protocol::
5592@end menu
5593
5594@node Local Sets
5595@subsection Local Sets
5596
5597@c %**end of header
5598
5599Sets created by a local client can be modified and reused for multiple
5600operations. As each set operation requires potentially expensive special
5601auxilliary data to be computed for each element of a set, a set can only
5602participate in one type of set operation (i.e. union or intersection). The type
5603of a set is determined upon its creation. If a the elements of a set are needed
5604for an operation of a different type, all of the set's element must be copied
5605to a new set of appropriate type.
5606
5607@node Set Modifications
5608@subsection Set Modifications
5609
5610@c %**end of header
5611
5612Even when set operations are active, one can add to and remove elements from a
5613set. However, these changes will only be visible to operations that have been
5614created after the changes have taken place. That is, every set operation only
5615sees a snapshot of the set from the time the operation was started. This
5616mechanism is @emph{not} implemented by copying the whole set, but by attaching
5617@emph{generation information} to each element and operation.
5618
5619@node Set Operations
5620@subsection Set Operations
5621
5622@c %**end of header
5623
5624Set operations can be started in two ways: Either by accepting an operation
5625request from a remote peer, or by requesting a set operation from a remote
5626peer. Set operations are uniquely identified by the involved @emph{peers}, an
5627@emph{application id} and the @emph{operation type}.
5628
5629The client is notified of incoming set operations by @emph{set listeners}. A
5630set listener listens for incoming operations of a specific operation type and
5631application id. Once notified of an incoming set request, the client can
5632accept the set request (providing a local set for the operation) or reject
5633it.
5634
5635@node Result Elements
5636@subsection Result Elements
5637
5638@c %**end of header
5639
5640The SET service has three @emph{result modes} that determine how an operation's
5641result set is delivered to the client:
5642
5643@itemize @bullet
5644@item @strong{Full Result Set.} All elements of set resulting from the set
5645operation are returned to the client.
5646@item @strong{Added Elements.} Only elements that result from the operation and
5647are not already in the local peer's set are returned. Note that for some
5648operations (like set intersection) this result mode will never return any
5649elements. This can be useful if only the remove peer is actually interested in
5650the result of the set operation.
5651@item @strong{Removed Elements.} Only elements that are in the local peer's
5652initial set but not in the operation's result set are returned. Note that for
5653some operations (like set union) this result mode will never return any
5654elements. This can be useful if only the remove peer is actually interested in
5655the result of the set operation.
5656@end itemize
5657
5658@node libgnunetset
5659@subsection libgnunetset
5660
5661@c %**end of header
5662
5663@menu
5664* Sets::
5665* Listeners::
5666* Operations::
5667* Supplying a Set::
5668* The Result Callback::
5669@end menu
5670
5671@node Sets
5672@subsubsection Sets
5673
5674@c %**end of header
5675
5676New sets are created with @code{GNUNET_SET_create}. Both the local peer's
5677configuration (as each set has its own client connection) and the operation
5678type must be specified. The set exists until either the client calls
5679@code{GNUNET_SET_destroy} or the client's connection to the service is
5680disrupted. In the latter case, the client is notified by the return value of
5681functions dealing with sets. This return value must always be checked.
5682
5683Elements are added and removed with @code{GNUNET_SET_add_element} and
5684@code{GNUNET_SET_remove_element}.
5685
5686@node Listeners
5687@subsubsection Listeners
5688
5689@c %**end of header
5690
5691Listeners are created with @code{GNUNET_SET_listen}. Each time time a remote
5692peer suggests a set operation with an application id and operation type
5693matching a listener, the listener's callack is invoked. The client then must
5694synchronously call either @code{GNUNET_SET_accept} or @code{GNUNET_SET_reject}.
5695Note that the operation will not be started until the client calls
5696@code{GNUNET_SET_commit} (see Section "Supplying a Set").
5697
5698@node Operations
5699@subsubsection Operations
5700
5701@c %**end of header
5702
5703Operations to be initiated by the local peer are created with
5704@code{GNUNET_SET_prepare}. Note that the operation will not be started until
5705the client calls @code{GNUNET_SET_commit} (see Section "Supplying a
5706Set").
5707
5708@node Supplying a Set
5709@subsubsection Supplying a Set
5710
5711@c %**end of header
5712
5713To create symmetry between the two ways of starting a set operation (accepting
5714and nitiating it), the operation handles returned by @code{GNUNET_SET_accept}
5715and @code{GNUNET_SET_prepare} do not yet have a set to operate on, thus they
5716can not do any work yet.
5717
5718The client must call @code{GNUNET_SET_commit} to specify a set to use for an
5719operation. @code{GNUNET_SET_commit} may only be called once per set
5720operation.
5721
5722@node The Result Callback
5723@subsubsection The Result Callback
5724
5725@c %**end of header
5726
5727Clients must specify both a result mode and a result callback with
5728@code{GNUNET_SET_accept} and @code{GNUNET_SET_prepare}. The result callback
5729with a status indicating either that an element was received, or the operation
5730failed or succeeded. The interpretation of the received element depends on the
5731result mode. The callback needs to know which result mode it is used in, as the
5732arguments do not indicate if an element is part of the full result set, or if
5733it is in the difference between the original set and the final set.
5734
5735@node The SET Client-Service Protocol
5736@subsection The SET Client-Service Protocol
5737
5738@c %**end of header
5739
5740@menu
5741* Creating Sets::
5742* Listeners2::
5743* Initiating Operations::
5744* Modifying Sets::
5745* Results and Operation Status::
5746* Iterating Sets::
5747@end menu
5748
5749@node Creating Sets
5750@subsubsection Creating Sets
5751
5752@c %**end of header
5753
5754For each set of a client, there exists a client connection to the service. Sets
5755are created by sending the @code{GNUNET_SERVICE_SET_CREATE} message over a new
5756client connection. Multiple operations for one set are multiplexed over one
5757client connection, using a request id supplied by the client.
5758
5759@node Listeners2
5760@subsubsection Listeners2
5761
5762@c %**end of header
5763
5764Each listener also requires a seperate client connection. By sending the
5765@code{GNUNET_SERVICE_SET_LISTEN} message, the client notifies the service of
5766the application id and operation type it is interested in. A client rejects an
5767incoming request by sending @code{GNUNET_SERVICE_SET_REJECT} on the listener's
5768client connection. In contrast, when accepting an incoming request, a a
5769@code{GNUNET_SERVICE_SET_ACCEPT} message must be sent over the@ set that is
5770supplied for the set operation.
5771
5772@node Initiating Operations
5773@subsubsection Initiating Operations
5774
5775@c %**end of header
5776
5777Operations with remote peers are initiated by sending a
5778@code{GNUNET_SERVICE_SET_EVALUATE} message to the service. The@ client
5779connection that this message is sent by determines the set to use.
5780
5781@node Modifying Sets
5782@subsubsection Modifying Sets
5783
5784@c %**end of header
5785
5786Sets are modified with the @code{GNUNET_SERVICE_SET_ADD} and
5787@code{GNUNET_SERVICE_SET_REMOVE} messages.
5788
5789
5790@c %@menu
5791@c %* Results and Operation Status::
5792@c %* Iterating Sets::
5793@c %@end menu
5794
5795@node Results and Operation Status
5796@subsubsection Results and Operation Status
5797@c %**end of header
5798
5799The service notifies the client of result elements and success/failure of a set
5800operation with the @code{GNUNET_SERVICE_SET_RESULT} message.
5801
5802@node Iterating Sets
5803@subsubsection Iterating Sets
5804
5805@c %**end of header
5806
5807All elements of a set can be requested by sending
5808@code{GNUNET_SERVICE_SET_ITER_REQUEST}. The server responds with
5809@code{GNUNET_SERVICE_SET_ITER_ELEMENT} and eventually terminates the iteration
5810with @code{GNUNET_SERVICE_SET_ITER_DONE}. After each received element, the
5811client@ must send @code{GNUNET_SERVICE_SET_ITER_ACK}. Note that only one set
5812iteration may be active for a set at any given time.
5813
5814@node The SET Intersection Peer-to-Peer Protocol
5815@subsection The SET Intersection Peer-to-Peer Protocol
5816
5817@c %**end of header
5818
5819The intersection protocol operates over CADET and starts with a
5820GNUNET_MESSAGE_TYPE_SET_P2P_OPERATION_REQUEST being sent by the peer initiating
5821the operation to the peer listening for inbound requests. It includes the
5822number of elements of the initiating peer, which is used to decide which side
5823will send a Bloom filter first.
5824
5825The listening peer checks if the operation type and application identifier are
5826acceptable for its current state. If not, it responds with a
5827GNUNET_MESSAGE_TYPE_SET_RESULT and a status of GNUNET_SET_STATUS_FAILURE (and
5828terminates the CADET channel).
5829
5830If the application accepts the request, the listener sends back a@
5831GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_ELEMENT_INFO if it has more elements
5832in the set than the client. Otherwise, it immediately starts with the Bloom
5833filter exchange. If the initiator receives a
5834GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_ELEMENT_INFO response, it beings the
5835Bloom filter exchange, unless the set size is indicated to be zero, in which
5836case the intersection is considered finished after just the initial
5837handshake.
5838
5839
5840@menu
5841* The Bloom filter exchange::
5842* Salt::
5843@end menu
5844
5845@node The Bloom filter exchange
5846@subsubsection The Bloom filter exchange
5847
5848@c %**end of header
5849
5850In this phase, each peer transmits a Bloom filter over the remaining keys of
5851the local set to the other peer using a
5852GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_BF message. This message additionally
5853includes the number of elements left in the sender's set, as well as the XOR
5854over all of the keys in that set.
5855
5856The number of bits 'k' set per element in the Bloom filter is calculated based
5857on the relative size of the two sets. Furthermore, the size of the Bloom filter
5858is calculated based on 'k' and the number of elements in the set to maximize
5859the amount of data filtered per byte transmitted on the wire (while avoiding an
5860excessively high number of iterations).
5861
5862The receiver of the message removes all elements from its local set that do not
5863pass the Bloom filter test. It then checks if the set size of the sender and
5864the XOR over the keys match what is left of his own set. If they do, he sends
5865a@ GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_DONE back to indicate that the
5866latest set is the final result. Otherwise, the receiver starts another Bloom
5867fitler exchange, except this time as the sender.
5868
5869@node Salt
5870@subsubsection Salt
5871
5872@c %**end of header
5873
5874Bloomfilter operations are probablistic: With some non-zero probability the
5875test may incorrectly say an element is in the set, even though it is not.
5876
5877To mitigate this problem, the intersection protocol iterates exchanging Bloom
5878filters using a different random 32-bit salt in each iteration (the salt is
5879also included in the message). With different salts, set operations may fail
5880for different elements. Merging the results from the executions, the
5881probability of failure drops to zero.
5882
5883The iterations terminate once both peers have established that they have sets
5884of the same size, and where the XOR over all keys computes the same 512-bit
5885value (leaving a failure probability of 2-511).
5886
5887@node The SET Union Peer-to-Peer Protocol
5888@subsection The SET Union Peer-to-Peer Protocol
5889
5890@c %**end of header
5891
5892The SET union protocol is based on Eppstein's efficient set reconciliation
5893without prior context. You should read this paper first if you want to
5894understand the protocol.
5895
5896The union protocol operates over CADET and starts with a
5897GNUNET_MESSAGE_TYPE_SET_P2P_OPERATION_REQUEST being sent by the peer initiating
5898the operation to the peer listening for inbound requests. It includes the
5899number of elements of the initiating peer, which is currently not used.
5900
5901The listening peer checks if the operation type and application identifier are
5902acceptable for its current state. If not, it responds with a
5903GNUNET_MESSAGE_TYPE_SET_RESULT and a status of GNUNET_SET_STATUS_FAILURE (and
5904terminates the CADET channel).
5905
5906If the application accepts the request, it sends back a strata estimator using
5907a message of type GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SE. The initiator evaluates
5908the strata estimator and initiates the exchange of invertible Bloom filters,
5909sending a GNUNET_MESSAGE_TYPE_SET_UNION_P2P_IBF.
5910
5911During the IBF exchange, if the receiver cannot invert the Bloom filter or
5912detects a cycle, it sends a larger IBF in response (up to a defined maximum
5913limit; if that limit is reached, the operation fails). Elements decoded while
5914processing the IBF are transmitted to the other peer using
5915GNUNET_MESSAGE_TYPE_SET_P2P_ELEMENTS, or requested from the other peer using
5916GNUNET_MESSAGE_TYPE_SET_P2P_ELEMENT_REQUESTS messages, depending on the sign
5917observed during decoding of the IBF. Peers respond to a
5918GNUNET_MESSAGE_TYPE_SET_P2P_ELEMENT_REQUESTS message with the respective
5919element in a GNUNET_MESSAGE_TYPE_SET_P2P_ELEMENTS message. If the IBF fully
5920decodes, the peer responds with a GNUNET_MESSAGE_TYPE_SET_UNION_P2P_DONE
5921message instead of another GNUNET_MESSAGE_TYPE_SET_UNION_P2P_IBF.
5922
5923All Bloom filter operations use a salt to mingle keys before hasing them into
5924buckets, such that future iterations have a fresh chance of succeeding if they
5925failed due to collisions before.
5926
5927@node GNUnet's STATISTICS subsystem
5928@section GNUnet's STATISTICS subsystem
5929
5930@c %**end of header
5931
5932In GNUnet, the STATISTICS subsystem offers a central place for all subsystems
5933to publish unsigned 64-bit integer run-time statistics. Keeping this
5934information centrally means that there is a unified way for the user to obtain
5935data on all subsystems, and individual subsystems do not have to always include
5936a custom data export method for performance metrics and other statistics. For
5937example, the TRANSPORT system uses STATISTICS to update information about the
5938number of directly connected peers and the bandwidth that has been consumed by
5939the various plugins. This information is valuable for diagnosing connectivity
5940and performance issues.
5941
5942Following the GNUnet service architecture, the STATISTICS subsystem is divided
5943into an API which is exposed through the header
5944@strong{gnunet_statistics_service.h} and the STATISTICS service
5945@strong{gnunet-service-statistics}. The @strong{gnunet-statistics} command-line
5946tool can be used to obtain (and change) information about the values stored by
5947the STATISTICS service. The STATISTICS service does not communicate with other
5948peers.
5949
5950Data is stored in the STATISTICS service in the form of tuples
5951@strong{(subsystem, name, value, persistence)}. The subsystem determines to
5952which other GNUnet's subsystem the data belongs. name is the name through which
5953value is associated. It uniquely identifies the record from among other records
5954belonging to the same subsystem. In some parts of the code, the pair
5955@strong{(subsystem, name)} is called a @strong{statistic} as it identifies the
5956values stored in the STATISTCS service.The persistence flag determines if the
5957record has to be preserved across service restarts. A record is said to be
5958persistent if this flag is set for it; if not, the record is treated as a
5959non-persistent record and it is lost after service restart. Persistent records
5960are written to and read from the file @strong{statistics.data} before shutdown
5961and upon startup. The file is located in the HOME directory of the peer.
5962
5963An anomaly of the STATISTICS service is that it does not terminate immediately
5964upon receiving a shutdown signal if it has any clients connected to it. It
5965waits for all the clients that are not monitors to close their connections
5966before terminating itself. This is to prevent the loss of data during peer
5967shutdown --- delaying the STATISTICS service shutdown helps other services to
5968store important data to STATISTICS during shutdown.
5969
5970@menu
5971* libgnunetstatistics::
5972* The STATISTICS Client-Service Protocol::
5973@end menu
5974
5975@node libgnunetstatistics
5976@subsection libgnunetstatistics
5977
5978@c %**end of header
5979
5980@strong{libgnunetstatistics} is the library containing the API for the
5981STATISTICS subsystem. Any process requiring to use STATISTICS should use this
5982API by to open a connection to the STATISTICS service. This is done by calling
5983the function @code{GNUNET_STATISTICS_create()}. This function takes the
5984subsystem's name which is trying to use STATISTICS and a configuration. All
5985values written to STATISTICS with this connection will be placed in the section
5986corresponding to the given subsystem's name. The connection to STATISTICS can
5987be destroyed with the function GNUNET_STATISTICS_destroy(). This function
5988allows for the connection to be destroyed immediately or upon transferring all
5989pending write requests to the service.
5990
5991Note: STATISTICS subsystem can be disabled by setting @code{DISABLE = YES}
5992under the @code{[STATISTICS]} section in the configuration. With such a
5993configuration all calls to @code{GNUNET_STATISTICS_create()} return @code{NULL}
5994as the STATISTICS subsystem is unavailable and no other functions from the API
5995can be used.
5996
5997
5998@menu
5999* Statistics retrieval::
6000* Setting statistics and updating them::
6001* Watches::
6002@end menu
6003
6004@node Statistics retrieval
6005@subsubsection Statistics retrieval
6006
6007@c %**end of header
6008
6009Once a connection to the statistics service is obtained, information about any
6010other system which uses statistics can be retrieved with the function
6011GNUNET_STATISTICS_get(). This function takes the connection handle, the name of
6012the subsystem whose information we are interested in (a @code{NULL} value will
6013retrieve information of all available subsystems using STATISTICS), the name of
6014the statistic we are interested in (a @code{NULL} value will retrieve all
6015available statistics), a continuation callback which is called when all of
6016requested information is retrieved, an iterator callback which is called for
6017each parameter in the retrieved information and a closure for the
6018aforementioned callbacks. The library then invokes the iterator callback for
6019each value matching the request.
6020
6021Call to @code{GNUNET_STATISTICS_get()} is asynchronous and can be canceled with
6022the function @code{GNUNET_STATISTICS_get_cancel()}. This is helpful when
6023retrieving statistics takes too long and especially when we want to shutdown
6024and cleanup everything.
6025
6026@node Setting statistics and updating them
6027@subsubsection Setting statistics and updating them
6028
6029@c %**end of header
6030
6031So far we have seen how to retrieve statistics, here we will learn how we can
6032set statistics and update them so that other subsystems can retrieve them.
6033
6034A new statistic can be set using the function @code{GNUNET_STATISTICS_set()}.
6035This function takes the name of the statistic and its value and a flag to make
6036the statistic persistent. The value of the statistic should be of the type
6037@code{uint64_t}. The function does not take the name of the subsystem; it is
6038determined from the previous @code{GNUNET_STATISTICS_create()} invocation. If
6039the given statistic is already present, its value is overwritten.
6040
6041An existing statistics can be updated, i.e its value can be increased or
6042decreased by an amount with the function @code{GNUNET_STATISTICS_update()}. The
6043parameters to this function are similar to @code{GNUNET_STATISTICS_set()},
6044except that it takes the amount to be changed as a type @code{int64_t} instead
6045of the value.
6046
6047The library will combine multiple set or update operations into one message if
6048the client performs requests at a rate that is faster than the available IPC
6049with the STATISTICS service. Thus, the client does not have to worry about
6050sending requests too quickly.
6051
6052@node Watches
6053@subsubsection Watches
6054
6055@c %**end of header
6056
6057As interesting feature of STATISTICS lies in serving notifications whenever a
6058statistic of our interest is modified. This is achieved by registering a watch
6059through the function @code{GNUNET_STATISTICS_watch()}. The parameters of this
6060function are similar to those of @code{GNUNET_STATISTICS_get()}. Changes to the
6061respective statistic's value will then cause the given iterator callback to be
6062called. Note: A watch can only be registered for a specific statistic. Hence
6063the subsystem name and the parameter name cannot be @code{NULL} in a call to
6064@code{GNUNET_STATISTICS_watch()}.
6065
6066A registered watch will keep notifying any value changes until
6067@code{GNUNET_STATISTICS_watch_cancel()} is called with the same parameters that
6068are used for registering the watch.
6069
6070@node The STATISTICS Client-Service Protocol
6071@subsection The STATISTICS Client-Service Protocol
6072@c %**end of header
6073
6074
6075@menu
6076* Statistics retrieval2::
6077* Setting and updating statistics::
6078* Watching for updates::
6079@end menu
6080
6081@node Statistics retrieval2
6082@subsubsection Statistics retrieval2
6083
6084@c %**end of header
6085
6086To retrieve statistics, the client transmits a message of type
6087@code{GNUNET_MESSAGE_TYPE_STATISTICS_GET} containing the given subsystem name
6088and statistic parameter to the STATISTICS service. The service responds with a
6089message of type @code{GNUNET_MESSAGE_TYPE_STATISTICS_VALUE} for each of the
6090statistics parameters that match the client request for the client. The end of
6091information retrieved is signaled by the service by sending a message of type
6092@code{GNUNET_MESSAGE_TYPE_STATISTICS_END}.
6093
6094@node Setting and updating statistics
6095@subsubsection Setting and updating statistics
6096
6097@c %**end of header
6098
6099The subsystem name, parameter name, its value and the persistence flag are
6100communicated to the service through the message
6101@code{GNUNET_MESSAGE_TYPE_STATISTICS_SET}.
6102
6103When the service receives a message of type
6104@code{GNUNET_MESSAGE_TYPE_STATISTICS_SET}, it retrieves the subsystem name and
6105checks for a statistic parameter with matching the name given in the message.
6106If a statistic parameter is found, the value is overwritten by the new value
6107from the message; if not found then a new statistic parameter is created with
6108the given name and value.
6109
6110In addition to just setting an absolute value, it is possible to perform a
6111relative update by sending a message of type
6112@code{GNUNET_MESSAGE_TYPE_STATISTICS_SET} with an update flag
6113(@code{GNUNET_STATISTICS_SETFLAG_RELATIVE}) signifying that the value in the
6114message should be treated as an update value.
6115
6116@node Watching for updates
6117@subsubsection Watching for updates
6118
6119@c %**end of header
6120
6121The function registers the watch at the service by sending a message of type
6122@code{GNUNET_MESSAGE_TYPE_STATISTICS_WATCH}. The service then sends
6123notifications through messages of type
6124@code{GNUNET_MESSAGE_TYPE_STATISTICS_WATCH_VALUE} whenever the statistic
6125parameter's value is changed.
6126
6127@node GNUnet's Distributed Hash Table (DHT)
6128@section GNUnet's Distributed Hash Table (DHT)
6129
6130@c %**end of header
6131
6132GNUnet includes a generic distributed hash table that can be used by developers
6133building P2P applications in the framework. This section documents high-level
6134features and how developers are expected to use the DHT. We have a research
6135paper detailing how the DHT works. Also, Nate's thesis includes a detailed
6136description and performance analysis (in chapter 6).
6137
6138Key features of GNUnet's DHT include:
6139
6140@itemize @bullet
6141@item stores key-value pairs with values up to (approximately) 63k in size
6142@item works with many underlay network topologies (small-world, random graph),
6143underlay does not need to be a full mesh / clique
6144@item support for extended queries (more than just a simple 'key'), filtering
6145duplicate replies within the network (bloomfilter) and content validation (for
6146details, please read the subsection on the block library)
6147@item can (optionally) return paths taken by the PUT and GET operations to the
6148application
6149@item provides content replication to handle churn
6150@end itemize
6151
6152GNUnet's DHT is randomized and unreliable. Unreliable means that there is no
6153strict guarantee that a value stored in the DHT is always found --- values are
6154only found with high probability. While this is somewhat true in all P2P DHTs,
6155GNUnet developers should be particularly wary of this fact (this will help you
6156write secure, fault-tolerant code). Thus, when writing any application using
6157the DHT, you should always consider the possibility that a value stored in the
6158DHT by you or some other peer might simply not be returned, or returned with a
6159significant delay. Your application logic must be written to tolerate this
6160(naturally, some loss of performance or quality of service is expected in this
6161case).
6162
6163@menu
6164* Block library and plugins::
6165* libgnunetdht::
6166* The DHT Client-Service Protocol::
6167* The DHT Peer-to-Peer Protocol::
6168@end menu
6169
6170@node Block library and plugins
6171@subsection Block library and plugins
6172
6173@c %**end of header
6174
6175@menu
6176* What is a Block?::
6177* The API of libgnunetblock::
6178* Queries::
6179* Sample Code::
6180* Conclusion2::
6181@end menu
6182
6183@node What is a Block?
6184@subsubsection What is a Block?
6185
6186@c %**end of header
6187
6188Blocks are small (< 63k) pieces of data stored under a key (struct
6189GNUNET_HashCode). Blocks have a type (enum GNUNET_BlockType) which defines
6190their data format. Blocks are used in GNUnet as units of static data exchanged
6191between peers and stored (or cached) locally. Uses of blocks include
6192file-sharing (the files are broken up into blocks), the VPN (DNS information is
6193stored in blocks) and the DHT (all information in the DHT and meta-information
6194for the maintenance of the DHT are both stored using blocks). The block
6195subsystem provides a few common functions that must be available for any type
6196of block.
6197
6198@node The API of libgnunetblock
6199@subsubsection The API of libgnunetblock
6200
6201@c %**end of header
6202
6203The block library requires for each (family of) block type(s) a block plugin
6204(implementing gnunet_block_plugin.h) that provides basic functions that are
6205needed by the DHT (and possibly other subsystems) to manage the block. These
6206block plugins are typically implemented within their respective subsystems.@
6207The main block library is then used to locate, load and query the appropriate
6208block plugin. Which plugin is appropriate is determined by the block type
6209(which is just a 32-bit integer). Block plugins contain code that specifies
6210which block types are supported by a given plugin. The block library loads all
6211block plugins that are installed at the local peer and forwards the application
6212request to the respective plugin.
6213
6214The central functions of the block APIs (plugin and main library) are to allow
6215the mapping of blocks to their respective key (if possible) and the ability to
6216check that a block is well-formed and matches a given request (again, if
6217possible). This way, GNUnet can avoid storing invalid blocks, storing blocks
6218under the wrong key and forwarding blocks in response to a query that they do
6219not answer.
6220
6221One key function of block plugins is that it allows GNUnet to detect duplicate
6222replies (via the Bloom filter). All plugins MUST support detecting duplicate
6223replies (by adding the current response to the Bloom filter and rejecting it if
6224it is encountered again). If a plugin fails to do this, responses may loop in
6225the network.
6226
6227@node Queries
6228@subsubsection Queries
6229@c %**end of header
6230
6231The query format for any block in GNUnet consists of four main components.
6232First, the type of the desired block must be specified. Second, the query must
6233contain a hash code. The hash code is used for lookups in hash tables and
6234databases and must not be unique for the block (however, if possible a unique
6235hash should be used as this would be best for performance). Third, an optional
6236Bloom filter can be specified to exclude known results; replies that hash to
6237the bits set in the Bloom filter are considered invalid. False-positives can be
6238eliminated by sending the same query again with a different Bloom filter
6239mutator value, which parameterizes the hash function that is used. Finally, an
6240optional application-specific "eXtended query" (xquery) can be specified to
6241further constrain the results. It is entirely up to the type-specific plugin to
6242determine whether or not a given block matches a query (type, hash, Bloom
6243filter, and xquery). Naturally, not all xquery's are valid and some types of
6244blocks may not support Bloom filters either, so the plugin also needs to check
6245if the query is valid in the first place.
6246
6247Depending on the results from the plugin, the DHT will then discard the
6248(invalid) query, forward the query, discard the (invalid) reply, cache the
6249(valid) reply, and/or forward the (valid and non-duplicate) reply.
6250
6251@node Sample Code
6252@subsubsection Sample Code
6253
6254@c %**end of header
6255
6256The source code in @strong{plugin_block_test.c} is a good starting point for
6257new block plugins --- it does the minimal work by implementing a plugin that
6258performs no validation at all. The respective @strong{Makefile.am} shows how to
6259build and install a block plugin.
6260
6261@node Conclusion2
6262@subsubsection Conclusion2
6263
6264@c %**end of header
6265
6266In conclusion, GNUnet subsystems that want to use the DHT need to define a
6267block format and write a plugin to match queries and replies. For testing, the
6268"GNUNET_BLOCK_TYPE_TEST" block type can be used; it accepts any query as valid
6269and any reply as matching any query. This type is also used for the DHT command
6270line tools. However, it should NOT be used for normal applications due to the
6271lack of error checking that results from this primitive implementation.
6272
6273@node libgnunetdht
6274@subsection libgnunetdht
6275
6276@c %**end of header
6277
6278The DHT API itself is pretty simple and offers the usual GET and PUT functions
6279that work as expected. The specified block type refers to the block library
6280which allows the DHT to run application-specific logic for data stored in the
6281network.
6282
6283
6284@menu
6285* GET::
6286* PUT::
6287* MONITOR::
6288* DHT Routing Options::
6289@end menu
6290
6291@node GET
6292@subsubsection GET
6293
6294@c %**end of header
6295
6296When using GET, the main consideration for developers (other than the block
6297library) should be that after issuing a GET, the DHT will continuously cause
6298(small amounts of) network traffic until the operation is explicitly canceled.
6299So GET does not simply send out a single network request once; instead, the
6300DHT will continue to search for data. This is needed to achieve good success
6301rates and also handles the case where the respective PUT operation happens
6302after the GET operation was started. Developers should not cancel an existing
6303GET operation and then explicitly re-start it to trigger a new round of
6304network requests; this is simply inefficient, especially as the internal
6305automated version can be more efficient, for example by filtering results in
6306the network that have already been returned.
6307
6308If an application that performs a GET request has a set of replies that it
6309already knows and would like to filter, it can call@
6310@code{GNUNET_DHT_get_filter_known_results} with an array of hashes over the
6311respective blocks to tell the DHT that these results are not desired (any
6312more). This way, the DHT will filter the respective blocks using the block
6313library in the network, which may result in a significant reduction in
6314bandwidth consumption.
6315
6316@node PUT
6317@subsubsection PUT
6318
6319@c %**end of header
6320
6321In contrast to GET operations, developers @strong{must} manually re-run PUT
6322operations periodically (if they intend the content to continue to be
6323available). Content stored in the DHT expires or might be lost due to churn.
6324Furthermore, GNUnet's DHT typically requires multiple rounds of PUT operations
6325before a key-value pair is consistently available to all peers (the DHT
6326randomizes paths and thus storage locations, and only after multiple rounds of
6327PUTs there will be a sufficient number of replicas in large DHTs). An explicit
6328PUT operation using the DHT API will only cause network traffic once, so in
6329order to ensure basic availability and resistance to churn (and adversaries),
6330PUTs must be repeated. While the exact frequency depends on the application, a
6331rule of thumb is that there should be at least a dozen PUT operations within
6332the content lifetime. Content in the DHT typically expires after one day, so
6333DHT PUT operations should be repeated at least every 1-2 hours.
6334
6335@node MONITOR
6336@subsubsection MONITOR
6337
6338@c %**end of header
6339
6340The DHT API also allows applications to monitor messages crossing the local
6341DHT service. The types of messages used by the DHT are GET, PUT and RESULT
6342messages. Using the monitoring API, applications can choose to monitor these
6343requests, possibly limiting themselves to requests for a particular block
6344type.
6345
6346The monitoring API is not only usefu only for diagnostics, it can also be used
6347to trigger application operations based on PUT operations. For example, an
6348application may use PUTs to distribute work requests to other peers. The
6349workers would then monitor for PUTs that give them work, instead of looking
6350for work using GET operations. This can be beneficial, especially if the
6351workers have no good way to guess the keys under which work would be stored.
6352Naturally, additional protocols might be needed to ensure that the desired
6353number of workers will process the distributed workload.
6354
6355@node DHT Routing Options
6356@subsubsection DHT Routing Options
6357
6358@c %**end of header
6359
6360There are two important options for GET and PUT requests:
6361
6362@table @asis
6363@item GNUNET_DHT_RO_DEMULITPLEX_EVERYWHERE This option means that all peers
6364should process the request, even if their peer ID is not closest to the key.
6365For a PUT request, this means that all peers that a request traverses may make
6366a copy of the data. Similarly for a GET request, all peers will check their
6367local database for a result. Setting this option can thus significantly improve
6368caching and reduce bandwidth consumption --- at the expense of a larger DHT
6369database. If in doubt, we recommend that this option should be used.
6370@item GNUNET_DHT_RO_RECORD_ROUTE This option instructs the DHT to record the path
6371that a GET or a PUT request is taking through the overlay network. The
6372resulting paths are then returned to the application with the respective
6373result. This allows the receiver of a result to construct a path to the
6374originator of the data, which might then be used for routing. Naturally,
6375setting this option requires additional bandwidth and disk space, so
6376applications should only set this if the paths are needed by the application
6377logic.
6378@item GNUNET_DHT_RO_FIND_PEER This option is an internal option used by
6379the DHT's peer discovery mechanism and should not be used by applications.
6380@item GNUNET_DHT_RO_BART This option is currently not implemented. It may in
6381the future offer performance improvements for clique topologies.
6382@end table
6383
6384@node The DHT Client-Service Protocol
6385@subsection The DHT Client-Service Protocol
6386
6387@c %**end of header
6388
6389@menu
6390* PUTting data into the DHT::
6391* GETting data from the DHT::
6392* Monitoring the DHT::
6393@end menu
6394
6395@node PUTting data into the DHT
6396@subsubsection PUTting data into the DHT
6397
6398@c %**end of header
6399
6400To store (PUT) data into the DHT, the client sends a@ @code{struct
6401GNUNET_DHT_ClientPutMessage} to the service. This message specifies the block
6402type, routing options, the desired replication level, the expiration time, key,
6403value and a 64-bit unique ID for the operation. The service responds with a@
6404@code{struct GNUNET_DHT_ClientPutConfirmationMessage} with the same 64-bit
6405unique ID. Note that the service sends the confirmation as soon as it has
6406locally processed the PUT request. The PUT may still be propagating through the
6407network at this time.
6408
6409In the future, we may want to change this to provide (limited) feedback to the
6410client, for example if we detect that the PUT operation had no effect because
6411the same key-value pair was already stored in the DHT. However, changing this
6412would also require additional state and messages in the P2P
6413interaction.
6414
6415@node GETting data from the DHT
6416@subsubsection GETting data from the DHT
6417
6418@c %**end of header
6419
6420To retrieve (GET) data from the DHT, the client sends a@ @code{struct
6421GNUNET_DHT_ClientGetMessage} to the service. The message specifies routing
6422options, a replication level (for replicating the GET, not the content), the
6423desired block type, the key, the (optional) extended query and unique 64-bit
6424request ID.
6425
6426Additionally, the client may send any number of@ @code{struct
6427GNUNET_DHT_ClientGetResultSeenMessage}s to notify the service about results
6428that the client is already aware of. These messages consist of the key, the
6429unique 64-bit ID of the request, and an arbitrary number of hash codes over the
6430blocks that the client is already aware of. As messages are restricted to 64k,
6431a client that already knows more than about a thousand blocks may need to send
6432several of these messages. Naturally, the client should transmit these messages
6433as quickly as possible after the original GET request such that the DHT can
6434filter those results in the network early on. Naturally, as these messages are
6435send after the original request, it is conceivalbe that the DHT service may
6436return blocks that match those already known to the client anyway.
6437
6438In response to a GET request, the service will send @code{struct
6439GNUNET_DHT_ClientResultMessage}s to the client. These messages contain the
6440block type, expiration, key, unique ID of the request and of course the value
6441(a block). Depending on the options set for the respective operations, the
6442replies may also contain the path the GET and/or the PUT took through the
6443network.
6444
6445A client can stop receiving replies either by disconnecting or by sending a
6446@code{struct GNUNET_DHT_ClientGetStopMessage} which must contain the key and
6447the 64-bit unique ID of the original request. Using an explicit "stop" message
6448is more common as this allows a client to run many concurrent GET operations
6449over the same connection with the DHT service --- and to stop them
6450individually.
6451
6452@node Monitoring the DHT
6453@subsubsection Monitoring the DHT
6454
6455@c %**end of header
6456
6457To begin monitoring, the client sends a @code{struct
6458GNUNET_DHT_MonitorStartStop} message to the DHT service. In this message, flags
6459can be set to enable (or disable) monitoring of GET, PUT and RESULT messages
6460that pass through a peer. The message can also restrict monitoring to a
6461particular block type or a particular key. Once monitoring is enabled, the DHT
6462service will notify the client about any matching event using @code{struct
6463GNUNET_DHT_MonitorGetMessage}s for GET events, @code{struct
6464GNUNET_DHT_MonitorPutMessage} for PUT events and@ @code{struct
6465GNUNET_DHT_MonitorGetRespMessage} for RESULTs. Each of these messages contains
6466all of the information about the event.
6467
6468@node The DHT Peer-to-Peer Protocol
6469@subsection The DHT Peer-to-Peer Protocol
6470@c %**end of header
6471
6472
6473@menu
6474* Routing GETs or PUTs::
6475* PUTting data into the DHT2::
6476* GETting data from the DHT2::
6477@end menu
6478
6479@node Routing GETs or PUTs
6480@subsubsection Routing GETs or PUTs
6481
6482@c %**end of header
6483
6484When routing GETs or PUTs, the DHT service selects a suitable subset of
6485neighbours for forwarding. The exact number of neighbours can be zero or more
6486and depends on the hop counter of the query (initially zero) in relation to the
6487(log of) the network size estimate, the desired replication level and the
6488peer's connectivity. Depending on the hop counter and our network size
6489estimate, the selection of the peers maybe randomized or by proximity to the
6490key. Furthermore, requests include a set of peers that a request has already
6491traversed; those peers are also excluded from the selection.
6492
6493@node PUTting data into the DHT2
6494@subsubsection PUTting data into the DHT2
6495
6496@c %**end of header
6497
6498To PUT data into the DHT, the service sends a @code{struct PeerPutMessage} of
6499type @code{GNUNET_MESSAGE_TYPE_DHT_P2P_PUT} to the respective neighbour. In
6500addition to the usual information about the content (type, routing options,
6501desired replication level for the content, expiration time, key and value), the
6502message contains a fixed-size Bloom filter with information about which peers
6503(may) have already seen this request. This Bloom filter is used to ensure that
6504DHT messages never loop back to a peer that has already processed the request.
6505Additionally, the message includes the current hop counter and, depending on
6506the routing options, the message may include the full path that the message has
6507taken so far. The Bloom filter should already contain the identity of the
6508previous hop; however, the path should not include the identity of the previous
6509hop and the receiver should append the identity of the sender to the path, not
6510its own identity (this is done to reduce bandwidth).
6511
6512@node GETting data from the DHT2
6513@subsubsection GETting data from the DHT2
6514
6515@c %**end of header
6516
6517A peer can search the DHT by sending @code{struct PeerGetMessage}s of type
6518@code{GNUNET_MESSAGE_TYPE_DHT_P2P_GET} to other peers. In addition to the usual
6519information about the request (type, routing options, desired replication level
6520for the request, the key and the extended query), a GET request also again
6521contains a hop counter, a Bloom filter over the peers that have processed the
6522request already and depending on the routing options the full path traversed by
6523the GET. Finally, a GET request includes a variable-size second Bloom filter
6524and a so-called Bloom filter mutator value which together indicate which
6525replies the sender has already seen. During the lookup, each block that matches
6526they block type, key and extended query is additionally subjected to a test
6527against this Bloom filter. The block plugin is expected to take the hash of the
6528block and combine it with the mutator value and check if the result is not yet
6529in the Bloom filter. The originator of the query will from time to time modify
6530the mutator to (eventually) allow false-positives filtered by the Bloom filter
6531to be returned.
6532
6533Peers that receive a GET request perform a local lookup (depending on their
6534proximity to the key and the query options) and forward the request to other
6535peers. They then remember the request (including the Bloom filter for blocking
6536duplicate results) and when they obtain a matching, non-filtered response a
6537@code{struct PeerResultMessage} of type@
6538@code{GNUNET_MESSAGE_TYPE_DHT_P2P_RESULT} is forwarded to the previous hop.
6539Whenver a result is forwarded, the block plugin is used to update the Bloom
6540filter accordingly, to ensure that the same result is never forwarded more than
6541once. The DHT service may also cache forwarded results locally if the
6542"CACHE_RESULTS" option is set to "YES" in the configuration.
6543
6544@node The GNU Name System (GNS)
6545@section The GNU Name System (GNS)
6546
6547@c %**end of header
6548
6549The GNU Name System (GNS) is a decentralized database that enables users to
6550securely resolve names to values. Names can be used to identify other users
6551(for example, in social networking), or network services (for example, VPN
6552services running at a peer in GNUnet, or purely IP-based services on the
6553Internet). Users interact with GNS by typing in a hostname that ends in ".gnu"
6554or ".zkey".
6555
6556Videos giving an overview of most of the GNS and the motivations behind it is
6557available here and here. The remainder of this chapter targets developers that
6558are familiar with high level concepts of GNS as presented in these talks.
6559
6560GNS-aware applications should use the GNS resolver to obtain the respective
6561records that are stored under that name in GNS. Each record consists of a type,
6562value, expiration time and flags.
6563
6564The type specifies the format of the value. Types below 65536 correspond to DNS
6565record types, larger values are used for GNS-specific records. Applications can
6566define new GNS record types by reserving a number and implementing a plugin
6567(which mostly needs to convert the binary value representation to a
6568human-readable text format and vice-versa). The expiration time specifies how
6569long the record is to be valid. The GNS API ensures that applications are only
6570given non-expired values. The flags are typically irrelevant for applications,
6571as GNS uses them internally to control visibility and validity of records.
6572
6573Records are stored along with a signature. The signature is generated using the
6574private key of the authoritative zone. This allows any GNS resolver to verify
6575the correctness of a name-value mapping.
6576
6577Internally, GNS uses the NAMECACHE to cache information obtained from other
6578users, the NAMESTORE to store information specific to the local users, and the
6579DHT to exchange data between users. A plugin API is used to enable applications
6580to define new GNS record types.
6581
6582@menu
6583* libgnunetgns::
6584* libgnunetgnsrecord::
6585* GNS plugins::
6586* The GNS Client-Service Protocol::
6587* Hijacking the DNS-Traffic using gnunet-service-dns::
6588* Serving DNS lookups via GNS on W32::
6589@end menu
6590
6591@node libgnunetgns
6592@subsection libgnunetgns
6593
6594@c %**end of header
6595
6596The GNS API itself is extremely simple. Clients first connec to the GNS service
6597using @code{GNUNET_GNS_connect}. They can then perform lookups using
6598@code{GNUNET_GNS_lookup} or cancel pending lookups using
6599@code{GNUNET_GNS_lookup_cancel}. Once finished, clients disconnect using
6600@code{GNUNET_GNS_disconnect}.
6601
6602
6603@menu
6604* Looking up records::
6605* Accessing the records::
6606* Creating records::
6607* Future work::
6608@end menu
6609
6610@node Looking up records
6611@subsubsection Looking up records
6612
6613@c %**end of header
6614
6615@code{GNUNET_GNS_lookup} takes a number of arguments:
6616
6617@table @asis
6618@item handle This is simply the GNS connection handle from
6619@code{GNUNET_GNS_connect}.
6620@item name The client needs to specify the name to
6621be resolved. This can be any valid DNS or GNS hostname.
6622@item zone The client
6623needs to specify the public key of the GNS zone against which the resolution
6624should be done (the ".gnu" zone). Note that a key must be provided, even if the
6625name ends in ".zkey". This should typically be the public key of the
6626master-zone of the user.
6627@item type This is the desired GNS or DNS record type
6628to look for. While all records for the given name will be returned, this can be
6629important if the client wants to resolve record types that themselves delegate
6630resolution, such as CNAME, PKEY or GNS2DNS. Resolving a record of any of these
6631types will only work if the respective record type is specified in the request,
6632as the GNS resolver will otherwise follow the delegation and return the records
6633from the respective destination, instead of the delegating record.
6634@item only_cached This argument should typically be set to @code{GNUNET_NO}. Setting
6635it to @code{GNUNET_YES} disables resolution via the overlay network.
6636@item shorten_zone_key If GNS encounters new names during resolution, their
6637respective zones can automatically be learned and added to the "shorten zone".
6638If this is desired, clients must pass the private key of the shorten zone. If
6639NULL is passed, shortening is disabled.
6640@item proc This argument identifies
6641the function to call with the result. It is given proc_cls, the number of
6642records found (possilby zero) and the array of the records as arguments. proc
6643will only be called once. After proc,> has been called, the lookup must no
6644longer be cancelled.
6645@item proc_cls The closure for proc.
6646@end table
6647
6648@node Accessing the records
6649@subsubsection Accessing the records
6650
6651@c %**end of header
6652
6653The @code{libgnunetgnsrecord} library provides an API to manipulate the GNS
6654record array that is given to proc. In particular, it offers functions such as
6655converting record values to human-readable strings (and back). However, most
6656@code{libgnunetgnsrecord} functions are not interesting to GNS client
6657applications.
6658
6659For DNS records, the @code{libgnunetdnsparser} library provides functions for
6660parsing (and serializing) common types of DNS records.
6661
6662@node Creating records
6663@subsubsection Creating records
6664
6665@c %**end of header
6666
6667Creating GNS records is typically done by building the respective record
6668information (possibly with the help of @code{libgnunetgnsrecord} and
6669@code{libgnunetdnsparser}) and then using the @code{libgnunetnamestore} to
6670publish the information. The GNS API is not involved in this
6671operation.
6672
6673@node Future work
6674@subsubsection Future work
6675
6676@c %**end of header
6677
6678In the future, we want to expand @code{libgnunetgns} to allow applications to
6679observe shortening operations performed during GNS resolution, for example so
6680that users can receive visual feedback when this happens.
6681
6682@node libgnunetgnsrecord
6683@subsection libgnunetgnsrecord
6684
6685@c %**end of header
6686
6687The @code{libgnunetgnsrecord} library is used to manipulate GNS records (in
6688plaintext or in their encrypted format). Applications mostly interact with
6689@code{libgnunetgnsrecord} by using the functions to convert GNS record values
6690to strings or vice-versa, or to lookup a GNS record type number by name (or
6691vice-versa). The library also provides various other functions that are mostly
6692used internally within GNS, such as converting keys to names, checking for
6693expiration, encrypting GNS records to GNS blocks, verifying GNS block
6694signatures and decrypting GNS records from GNS blocks.
6695
6696We will now discuss the four commonly used functions of the API.@
6697@code{libgnunetgnsrecord} does not perform these operations itself, but instead
6698uses plugins to perform the operation. GNUnet includes plugins to support
6699common DNS record types as well as standard GNS record types.
6700
6701
6702@menu
6703* Value handling::
6704* Type handling::
6705@end menu
6706
6707@node Value handling
6708@subsubsection Value handling
6709
6710@c %**end of header
6711
6712@code{GNUNET_GNSRECORD_value_to_string} can be used to convert the (binary)
6713representation of a GNS record value to a human readable, 0-terminated UTF-8
6714string. NULL is returned if the specified record type is not supported by any
6715available plugin.
6716
6717@code{GNUNET_GNSRECORD_string_to_value} can be used to try to convert a human
6718readable string to the respective (binary) representation of a GNS record
6719value.
6720
6721@node Type handling
6722@subsubsection Type handling
6723
6724@c %**end of header
6725
6726@code{GNUNET_GNSRECORD_typename_to_number} can be used to obtain the numeric
6727value associated with a given typename. For example, given the typename "A"
6728(for DNS A reocrds), the function will return the number 1. A list of common
6729DNS record types is
6730@uref{http://en.wikipedia.org/wiki/List_of_DNS_record_types, here. Note that
6731not all DNS record types are supported by GNUnet GNSRECORD plugins at this
6732time.}
6733
6734@code{GNUNET_GNSRECORD_number_to_typename} can be used to obtain the typename
6735associated with a given numeric value. For example, given the type number 1,
6736the function will return the typename "A".
6737
6738@node GNS plugins
6739@subsection GNS plugins
6740
6741@c %**end of header
6742
6743Adding a new GNS record type typically involves writing (or extending) a
6744GNSRECORD plugin. The plugin needs to implement the
6745@code{gnunet_gnsrecord_plugin.h} API which provides basic functions that are
6746needed by GNSRECORD to convert typenames and values of the respective record
6747type to strings (and back). These gnsrecord plugins are typically implemented
6748within their respective subsystems. Examples for such plugins can be found in
6749the GNSRECORD, GNS and CONVERSATION subsystems.
6750
6751The @code{libgnunetgnsrecord} library is then used to locate, load and query
6752the appropriate gnsrecord plugin. Which plugin is appropriate is determined by
6753the record type (which is just a 32-bit integer). The @code{libgnunetgnsrecord}
6754library loads all block plugins that are installed at the local peer and
6755forwards the application request to the plugins. If the record type is not
6756supported by the plugin, it should simply return an error code.
6757
6758The central functions of the block APIs (plugin and main library) are the same
6759four functions for converting between values and strings, and typenames and
6760numbers documented in the previous subsection.
6761
6762@node The GNS Client-Service Protocol
6763@subsection The GNS Client-Service Protocol
6764
6765@c %**end of header
6766
6767The GNS client-service protocol consists of two simple messages, the
6768@code{LOOKUP} message and the @code{LOOKUP_RESULT}. Each @code{LOOKUP} message
6769contains a unique 32-bit identifier, which will be included in the
6770corresponding response. Thus, clients can send many lookup requests in parallel
6771and receive responses out-of-order. A @code{LOOKUP} request also includes the
6772public key of the GNS zone, the desired record type and fields specifying
6773whether shortening is enabled or networking is disabled. Finally, the
6774@code{LOOKUP} message includes the name to be resolved.
6775
6776The response includes the number of records and the records themselves in the
6777format created by @code{GNUNET_GNSRECORD_records_serialize}. They can thus be
6778deserialized using @code{GNUNET_GNSRECORD_records_deserialize}.
6779
6780@node Hijacking the DNS-Traffic using gnunet-service-dns
6781@subsection Hijacking the DNS-Traffic using gnunet-service-dns
6782
6783@c %**end of header
6784
6785This section documents how the gnunet-service-dns (and the gnunet-helper-dns)
6786intercepts DNS queries from the local system.@ This is merely one method for
6787how we can obtain GNS queries. It is also possible to change @code{resolv.conf}
6788to point to a machine running @code{gnunet-dns2gns} or to modify libc's name
6789system switch (NSS) configuration to include a GNS resolution plugin. The
6790method described in this chaper is more of a last-ditch catch-all approach.
6791
6792@code{gnunet-service-dns} enables intercepting DNS traffic using policy based
6793routing. We MARK every outgoing DNS-packet if it was not sent by our
6794application. Using a second routing table in the Linux kernel these marked
6795packets are then routed through our virtual network interface and can thus be
6796captured unchanged.
6797
6798Our application then reads the query and decides how to handle it: A query to
6799an address ending in ".gnu" or ".zkey" is hijacked by @code{gnunet-service-gns}
6800and resolved internally using GNS. In the future, a reverse query for an
6801address of the configured virtual network could be answered with records kept
6802about previous forward queries. Queries that are not hijacked by some
6803application using the DNS service will be sent to the original recipient. The
6804answer to the query will always be sent back through the virtual interface with
6805the original nameserver as source address.
6806
6807
6808@menu
6809* Network Setup Details::
6810@end menu
6811
6812@node Network Setup Details
6813@subsubsection Network Setup Details
6814
6815@c %**end of header
6816
6817The DNS interceptor adds the following rules to the Linux kernel:
6818@example
6819iptables -t mangle -I OUTPUT 1 -p udp --sport $LOCALPORT --dport 53 -j
6820ACCEPT iptables -t mangle -I OUTPUT 2 -p udp --dport 53 -j MARK --set-mark 3 ip
6821rule add fwmark 3 table2 ip route add default via $VIRTUALDNS table2
6822@end example
6823
6824Line 1 makes sure that all packets coming from a port our application opened
6825beforehand (@code{$LOCALPORT}) will be routed normally. Line 2 marks every
6826other packet to a DNS-Server with mark 3 (chosen arbitrarily). The third line
6827adds a routing policy based on this mark 3 via the routing table.
6828
6829@node Serving DNS lookups via GNS on W32
6830@subsection Serving DNS lookups via GNS on W32
6831
6832@c %**end of header
6833
6834This section documents how the libw32nsp (and gnunet-gns-helper-service-w32) do
6835DNS resolutions of DNS queries on the local system. This only applies to GNUnet
6836running on W32.
6837
6838W32 has a concept of "Namespaces" and "Namespace providers". These are used to
6839present various name systems to applications in a generic way. Namespaces
6840include DNS, mDNS, NLA and others. For each namespace any number of providers
6841could be registered, and they are queried in an order of priority (which is
6842adjustable).
6843
6844Applications can resolve names by using WSALookupService*() family of
6845functions.
6846
6847However, these are WSA-only facilities. Common BSD socket functions for
6848namespace resolutions are gethostbyname and getaddrinfo (among others). These
6849functions are implemented internally (by default - by mswsock, which also
6850implements the default DNS provider) as wrappers around WSALookupService*()
6851functions (see "Sample Code for a Service Provider" on MSDN).
6852
6853On W32 GNUnet builds a libw32nsp - a namespace provider, which can then be
6854installed into the system by using w32nsp-install (and uninstalled by
6855w32nsp-uninstall), as described in "Installation Handbook".
6856
6857libw32nsp is very simple and has almost no dependencies. As a response to
6858NSPLookupServiceBegin(), it only checks that the provider GUID passed to it by
6859the caller matches GNUnet DNS Provider GUID, checks that name being resolved
6860ends in ".gnu" or ".zkey", then connects to gnunet-gns-helper-service-w32 at
6861127.0.0.1:5353 (hardcoded) and sends the name resolution request there,
6862returning the connected socket to the caller.
6863
6864When the caller invokes NSPLookupServiceNext(), libw32nsp reads a completely
6865formed reply from that socket, unmarshalls it, then gives it back to the
6866caller.
6867
6868At the moment gnunet-gns-helper-service-w32 is implemented to ever give only
6869one reply, and subsequent calls to NSPLookupServiceNext() will fail with
6870WSA_NODATA (first call to NSPLookupServiceNext() might also fail if GNS failed
6871to find the name, or there was an error connecting to it).
6872
6873gnunet-gns-helper-service-w32 does most of the processing:
6874
6875@itemize @bullet
6876@item Maintains a connection to GNS.
6877@item Reads GNS config and loads appropriate keys.
6878@item Checks service GUID and decides on the type of record to look up,
6879refusing to make a lookup outright when unsupported service GUID is passed.
6880@item Launches the lookup
6881@end itemize
6882
6883When lookup result arrives, gnunet-gns-helper-service-w32 forms a complete
6884reply (including filling a WSAQUERYSETW structure and, possibly, a binary blob
6885with a hostent structure for gethostbyname() client), marshalls it, and sends
6886it back to libw32nsp. If no records were found, it sends an empty header.
6887
6888This works for most normal applications that use gethostbyname() or
6889getaddrinfo() to resolve names, but fails to do anything with applications that
6890use alternative means of resolving names (such as sending queries to a DNS
6891server directly by themselves). This includes some of well known utilities,
6892like "ping" and "nslookup".
6893
6894@node The GNS Namecache
6895@section The GNS Namecache
6896
6897@c %**end of header
6898
6899The NAMECACHE subsystem is responsible for caching (encrypted) resolution
6900results of the GNU Name System (GNS). GNS makes zone information available to
6901other users via the DHT. However, as accessing the DHT for every lookup is
6902expensive (and as the DHT's local cache is lost whenever the peer is
6903restarted), GNS uses the NAMECACHE as a more persistent cache for DHT lookups.
6904Thus, instead of always looking up every name in the DHT, GNS first checks if
6905the result is already available locally in the NAMECACHE. Only if there is no
6906result in the NAMECACHE, GNS queries the DHT. The NAMECACHE stores data in the
6907same (encrypted) format as the DHT. It thus makes no sense to iterate over all
6908items in the NAMECACHE --- the NAMECACHE does not have a way to provide the
6909keys required to decrypt the entries.
6910
6911Blocks in the NAMECACHE share the same expiration mechanism as blocks in the
6912DHT --- the block expires wheneever any of the records in the (encrypted) block
6913expires. The expiration time of the block is the only information stored in
6914plaintext. The NAMECACHE service internally performs all of the required work
6915to expire blocks, clients do not have to worry about this. Also, given that
6916NAMECACHE stores only GNS blocks that local users requested, there is no
6917configuration option to limit the size of the NAMECACHE. It is assumed to be
6918always small enough (a few MB) to fit on the drive.
6919
6920The NAMECACHE supports the use of different database backends via a plugin API.
6921
6922@menu
6923* libgnunetnamecache::
6924* The NAMECACHE Client-Service Protocol::
6925* The NAMECACHE Plugin API::
6926@end menu
6927
6928@node libgnunetnamecache
6929@subsection libgnunetnamecache
6930
6931@c %**end of header
6932
6933The NAMECACHE API consists of five simple functions. First, there is
6934@code{GNUNET_NAMECACHE_connect} to connect to the NAMECACHE service. This
6935returns the handle required for all other operations on the NAMECACHE. Using
6936@code{GNUNET_NAMECACHE_block_cache} clients can insert a block into the cache.
6937@code{GNUNET_NAMECACHE_lookup_block} can be used to lookup blocks that were
6938stored in the NAMECACHE. Both operations can be cancelled using
6939@code{GNUNET_NAMECACHE_cancel}. Note that cancelling a
6940@code{GNUNET_NAMECACHE_block_cache} operation can result in the block being
6941stored in the NAMECACHE --- or not. Cancellation primarily ensures that the
6942continuation function with the result of the operation will no longer be
6943invoked. Finally, @code{GNUNET_NAMECACHE_disconnect} closes the connection to
6944the NAMECACHE.
6945
6946The maximum size of a block that can be stored in the NAMECACHE is
6947@code{GNUNET_NAMECACHE_MAX_VALUE_SIZE}, which is defined to be 63 kB.
6948
6949@node The NAMECACHE Client-Service Protocol
6950@subsection The NAMECACHE Client-Service Protocol
6951
6952@c %**end of header
6953
6954All messages in the NAMECACHE IPC protocol start with the @code{struct
6955GNUNET_NAMECACHE_Header} which adds a request ID (32-bit integer) to the
6956standard message header. The request ID is used to match requests with the
6957respective responses from the NAMECACHE, as they are allowed to happen
6958out-of-order.
6959
6960
6961@menu
6962* Lookup::
6963* Store::
6964@end menu
6965
6966@node Lookup
6967@subsubsection Lookup
6968
6969@c %**end of header
6970
6971The @code{struct LookupBlockMessage} is used to lookup a block stored in the
6972cache. It contains the query hash. The NAMECACHE always responds with a
6973@code{struct LookupBlockResponseMessage}. If the NAMECACHE has no response, it
6974sets the expiration time in the response to zero. Otherwise, the response is
6975expected to contain the expiration time, the ECDSA signature, the derived key
6976and the (variable-size) encrypted data of the block.
6977
6978@node Store
6979@subsubsection Store
6980
6981@c %**end of header
6982
6983The @code{struct BlockCacheMessage} is used to cache a block in the NAMECACHE.
6984It has the same structure as the @code{struct LookupBlockResponseMessage}. The
6985service responds with a @code{struct BlockCacheResponseMessage} which contains
6986the result of the operation (success or failure). In the future, we might want
6987to make it possible to provide an error message as well.
6988
6989@node The NAMECACHE Plugin API
6990@subsection The NAMECACHE Plugin API
6991@c %**end of header
6992
6993The NAMECACHE plugin API consists of two functions, @code{cache_block} to store
6994a block in the database, and @code{lookup_block} to lookup a block in the
6995database.
6996
6997
6998@menu
6999* Lookup2::
7000* Store2::
7001@end menu
7002
7003@node Lookup2
7004@subsubsection Lookup2
7005
7006@c %**end of header
7007
7008The @code{lookup_block} function is expected to return at most one block to the
7009iterator, and return @code{GNUNET_NO} if there were no non-expired results. If
7010there are multiple non-expired results in the cache, the lookup is supposed to
7011return the result with the largest expiration time.
7012
7013@node Store2
7014@subsubsection Store2
7015
7016@c %**end of header
7017
7018The @code{cache_block} function is expected to try to store the block in the
7019database, and return @code{GNUNET_SYSERR} if this was not possible for any
7020reason. Furthermore, @code{cache_block} is expected to implicitly perform cache
7021maintenance and purge blocks from the cache that have expired. Note that
7022@code{cache_block} might encounter the case where the database already has
7023another block stored under the same key. In this case, the plugin must ensure
7024that the block with the larger expiration time is preserved. Obviously, this
7025can done either by simply adding new blocks and selecting for the most recent
7026expiration time during lookup, or by checking which block is more recent during
7027the store operation.
7028
7029@node The REVOCATION Subsystem
7030@section The REVOCATION Subsystem
7031@c %**end of header
7032
7033The REVOCATION subsystem is responsible for key revocation of Egos. If a user
7034learns that his private key has been compromised or has lost it, he can use the
7035REVOCATION system to inform all of the other users that this private key is no
7036longer valid. The subsystem thus includes ways to query for the validity of
7037keys and to propagate revocation messages.
7038
7039@menu
7040* Dissemination::
7041* Revocation Message Design Requirements::
7042* libgnunetrevocation::
7043* The REVOCATION Client-Service Protocol::
7044* The REVOCATION Peer-to-Peer Protocol::
7045@end menu
7046
7047@node Dissemination
7048@subsection Dissemination
7049
7050@c %**end of header
7051
7052When a revocation is performed, the revocation is first of all disseminated by
7053flooding the overlay network. The goal is to reach every peer, so that when a
7054peer needs to check if a key has been revoked, this will be purely a local
7055operation where the peer looks at his local revocation list. Flooding the
7056network is also the most robust form of key revocation --- an adversary would
7057have to control a separator of the overlay graph to restrict the propagation of
7058the revocation message. Flooding is also very easy to implement --- peers that
7059receive a revocation message for a key that they have never seen before simply
7060pass the message to all of their neighbours.
7061
7062Flooding can only distribute the revocation message to peers that are online.
7063In order to notify peers that join the network later, the revocation service
7064performs efficient set reconciliation over the sets of known revocation
7065messages whenever two peers (that both support REVOCATION dissemination)
7066connect. The SET service is used to perform this operation
7067efficiently.
7068
7069@node Revocation Message Design Requirements
7070@subsection Revocation Message Design Requirements
7071
7072@c %**end of header
7073
7074However, flooding is also quite costly, creating O(|E|) messages on a network
7075with |E| edges. Thus, revocation messages are required to contain a
7076proof-of-work, the result of an expensive computation (which, however, is cheap
7077to verify). Only peers that have expended the CPU time necessary to provide
7078this proof will be able to flood the network with the revocation message. This
7079ensures that an attacker cannot simply flood the network with millions of
7080revocation messages. The proof-of-work required by GNUnet is set to take days
7081on a typical PC to compute; if the ability to quickly revoke a key is needed,
7082users have the option to pre-compute revocation messages to store off-line and
7083use instantly after their key has expired.
7084
7085Revocation messages must also be signed by the private key that is being
7086revoked. Thus, they can only be created while the private key is in the
7087possession of the respective user. This is another reason to create a
7088revocation message ahead of time and store it in a secure location.
7089
7090@node libgnunetrevocation
7091@subsection libgnunetrevocation
7092
7093@c %**end of header
7094
7095The REVOCATION API consists of two parts, to query and to issue
7096revocations.
7097
7098
7099@menu
7100* Querying for revoked keys::
7101* Preparing revocations::
7102* Issuing revocations::
7103@end menu
7104
7105@node Querying for revoked keys
7106@subsubsection Querying for revoked keys
7107
7108@c %**end of header
7109
7110@code{GNUNET_REVOCATION_query} is used to check if a given ECDSA public key has
7111been revoked. The given callback will be invoked with the result of the check.
7112The query can be cancelled using @code{GNUNET_REVOCATION_query_cancel} on the
7113return value.
7114
7115@node Preparing revocations
7116@subsubsection Preparing revocations
7117
7118@c %**end of header
7119
7120It is often desirable to create a revocation record ahead-of-time and store it
7121in an off-line location to be used later in an emergency. This is particularly
7122true for GNUnet revocations, where performing the revocation operation itself
7123is computationally expensive and thus is likely to take some time. Thus, if
7124users want the ability to perform revocations quickly in an emergency, they
7125must pre-compute the revocation message. The revocation API enables this with
7126two functions that are used to compute the revocation message, but not trigger
7127the actual revocation operation.
7128
7129@code{GNUNET_REVOCATION_check_pow} should be used to calculate the
7130proof-of-work required in the revocation message. This function takes the
7131public key, the required number of bits for the proof of work (which in GNUnet
7132is a network-wide constant) and finally a proof-of-work number as arguments.
7133The function then checks if the given proof-of-work number is a valid proof of
7134work for the given public key. Clients preparing a revocation are expected to
7135call this function repeatedly (typically with a monotonically increasing
7136sequence of numbers of the proof-of-work number) until a given number satisfies
7137the check. That number should then be saved for later use in the revocation
7138operation.
7139
7140@code{GNUNET_REVOCATION_sign_revocation} is used to generate the signature that
7141is required in a revocation message. It takes the private key that (possibly in
7142the future) is to be revoked and returns the signature. The signature can again
7143be saved to disk for later use, which will then allow performing a revocation
7144even without access to the private key.
7145
7146@node Issuing revocations
7147@subsubsection Issuing revocations
7148
7149
7150Given a ECDSA public key, the signature from @code{GNUNET_REVOCATION_sign} and
7151the proof-of-work, @code{GNUNET_REVOCATION_revoke} can be used to perform the
7152actual revocation. The given callback is called upon completion of the
7153operation. @code{GNUNET_REVOCATION_revoke_cancel} can be used to stop the
7154library from calling the continuation; however, in that case it is undefined
7155whether or not the revocation operation will be executed.
7156
7157@node The REVOCATION Client-Service Protocol
7158@subsection The REVOCATION Client-Service Protocol
7159
7160
7161The REVOCATION protocol consists of four simple messages.
7162
7163A @code{QueryMessage} containing a public ECDSA key is used to check if a
7164particular key has been revoked. The service responds with a
7165@code{QueryResponseMessage} which simply contains a bit that says if the given
7166public key is still valid, or if it has been revoked.
7167
7168The second possible interaction is for a client to revoke a key by passing a
7169@code{RevokeMessage} to the service. The @code{RevokeMessage} contains the
7170ECDSA public key to be revoked, a signature by the corresponding private key
7171and the proof-of-work, The service responds with a
7172@code{RevocationResponseMessage} which can be used to indicate that the
7173@code{RevokeMessage} was invalid (i.e. proof of work incorrect), or otherwise
7174indicates that the revocation has been processed successfully.
7175
7176@node The REVOCATION Peer-to-Peer Protocol
7177@subsection The REVOCATION Peer-to-Peer Protocol
7178
7179@c %**end of header
7180
7181Revocation uses two disjoint ways to spread revocation information among peers.
7182First of all, P2P gossip exchanged via CORE-level neighbours is used to quickly
7183spread revocations to all connected peers. Second, whenever two peers (that
7184both support revocations) connect, the SET service is used to compute the union
7185of the respective revocation sets.
7186
7187In both cases, the exchanged messages are @code{RevokeMessage}s which contain
7188the public key that is being revoked, a matching ECDSA signature, and a
7189proof-of-work. Whenever a peer learns about a new revocation this way, it first
7190validates the signature and the proof-of-work, then stores it to disk
7191(typically to a file $GNUNET_DATA_HOME/revocation.dat) and finally spreads the
7192information to all directly connected neighbours.
7193
7194For computing the union using the SET service, the peer with the smaller hashed
7195peer identity will connect (as a "client" in the two-party set protocol) to the
7196other peer after one second (to reduce traffic spikes on connect) and initiate
7197the computation of the set union. All revocation services use a common hash to
7198identify the SET operation over revocation sets.
7199
7200The current implementation accepts revocation set union operations from all
7201peers at any time; however, well-behaved peers should only initiate this
7202operation once after establishing a connection to a peer with a larger hashed
7203peer identity.
7204
7205@node GNUnet's File-sharing (FS) Subsystem
7206@section GNUnet's File-sharing (FS) Subsystem
7207
7208@c %**end of header
7209
7210This chapter describes the details of how the file-sharing service works. As
7211with all services, it is split into an API (libgnunetfs), the service process
7212(gnunet-service-fs) and user interface(s). The file-sharing service uses the
7213datastore service to store blocks and the DHT (and indirectly datacache) for
7214lookups for non-anonymous file-sharing.@ Furthermore, the file-sharing service
7215uses the block library (and the block fs plugin) for validation of DHT
7216operations.
7217
7218In contrast to many other services, libgnunetfs is rather complex since the
7219client library includes a large number of high-level abstractions; this is
7220necessary since the Fs service itself largely only operates on the block level.
7221The FS library is responsible for providing a file-based abstraction to
7222applications, including directories, meta data, keyword search, verification,
7223and so on.
7224
7225The method used by GNUnet to break large files into blocks and to use keyword
7226search is called the "Encoding for Censorship Resistant Sharing" (ECRS). ECRS
7227is largely implemented in the fs library; block validation is also reflected in
7228the block FS plugin and the FS service. ECRS on-demand encoding is implemented
7229in the FS service.
7230
7231NOTE: The documentation in this chapter is quite incomplete.
7232
7233@menu
7234* Encoding for Censorship-Resistant Sharing (ECRS)::
7235* File-sharing persistence directory structure::
7236@end menu
7237
7238@node Encoding for Censorship-Resistant Sharing (ECRS)
7239@subsection Encoding for Censorship-Resistant Sharing (ECRS)
7240
7241@c %**end of header
7242
7243When GNUnet shares files, it uses a content encoding that is called ECRS, the
7244Encoding for Censorship-Resistant Sharing. Most of ECRS is described in the
7245(so far unpublished) research paper attached to this page. ECRS obsoletes the
7246previous ESED and ESED II encodings which were used in GNUnet before version
72470.7.0.@ @ The rest of this page assumes that the reader is familiar with the
7248attached paper. What follows is a description of some minor extensions that
7249GNUnet makes over what is described in the paper. The reason why these
7250extensions are not in the paper is that we felt that they were obvious or
7251trivial extensions to the original scheme and thus did not warrant space in
7252the research report.
7253
7254
7255@menu
7256* Namespace Advertisements::
7257* KSBlocks::
7258@end menu
7259
7260@node Namespace Advertisements
7261@subsubsection Namespace Advertisements
7262
7263@c %**end of header
7264
7265An @code{SBlock} with identifier ′all zeros′ is a signed
7266advertisement for a namespace. This special @code{SBlock} contains metadata
7267describing the content of the namespace. Instead of the name of the identifier
7268for a potential update, it contains the identifier for the root of the
7269namespace. The URI should always be empty. The @code{SBlock} is signed with
7270the content provder′s RSA private key (just like any other SBlock). Peers
7271can search for @code{SBlock}s in order to find out more about a namespace.
7272
7273@node KSBlocks
7274@subsubsection KSBlocks
7275
7276@c %**end of header
7277
7278GNUnet implements @code{KSBlocks} which are @code{KBlocks} that, instead of
7279encrypting a CHK and metadata, encrypt an @code{SBlock} instead. In other
7280words, @code{KSBlocks} enable GNUnet to find @code{SBlocks} using the global
7281keyword search. Usually the encrypted @code{SBlock} is a namespace
7282advertisement. The rationale behind @code{KSBlock}s and @code{SBlock}s is to
7283enable peers to discover namespaces via keyword searches, and, to associate
7284useful information with namespaces. When GNUnet finds @code{KSBlocks} during a
7285normal keyword search, it adds the information to an internal list of
7286discovered namespaces. Users looking for interesting namespaces can then
7287inspect this list, reducing the need for out-of-band discovery of namespaces.
7288Naturally, namespaces (or more specifically, namespace advertisements) can
7289also be referenced from directories, but @code{KSBlock}s should make it easier
7290to advertise namespaces for the owner of the pseudonym since they eliminate
7291the need to first create a directory.
7292
7293Collections are also advertised using @code{KSBlock}s.
7294
7295@table @asis
7296@item Attachment Size
7297@item ecrs.pdf 270.68 KB
7298@item https://gnunet.org/sites/default/files/ecrs.pdf
7299@end table
7300
7301@node File-sharing persistence directory structure
7302@subsection File-sharing persistence directory structure
7303
7304@c %**end of header
7305
7306This section documents how the file-sharing library implements persistence of
7307file-sharing operations and specifically the resulting directory structure.
7308This code is only active if the @code{GNUNET_FS_FLAGS_PERSISTENCE} flag was set
7309when calling @code{GNUNET_FS_start}. In this case, the file-sharing library
7310will try hard to ensure that all major operations (searching, downloading,
7311publishing, unindexing) are persistent, that is, can live longer than the
7312process itself. More specifically, an operation is supposed to live until it is
7313explicitly stopped.
7314
7315If @code{GNUNET_FS_stop} is called before an operation has been stopped, a
7316@code{SUSPEND} event is generated and then when the process calls
7317@code{GNUNET_FS_start} next time, a @code{RESUME} event is generated.
7318Additionally, even if an application crashes (segfault, SIGKILL, system crash)
7319and hence @code{GNUNET_FS_stop} is never called and no @code{SUSPEND} events
7320are generated, operations are still resumed (with @code{RESUME} events). This
7321is implemented by constantly writing the current state of the file-sharing
7322operations to disk. Specifically, the current state is always written to disk
7323whenever anything significant changes (the exception are block-wise progress in
7324publishing and unindexing, since those operations would be slowed down
7325significantly and can be resumed cheaply even without detailed accounting).
7326Note that@ if the process crashes (or is killed) during a serialization
7327operation, FS does not guarantee that this specific operation is recoverable
7328(no strict transactional semantics, again for performance reasons). However,
7329all other unrelated operations should resume nicely.
7330
7331Since we need to serialize the state continuously and want to recover as much
7332as possible even after crashing during a serialization operation, we do not use
7333one large file for serialization. Instead, several directories are used for the
7334various operations. When @code{GNUNET_FS_start} executes, the master
7335directories are scanned for files describing operations to resume. Sometimes,
7336these operations can refer to related operations in child directories which may
7337also be resumed at this point. Note that corrupted files are cleaned up
7338automatically. However, dangling files in child directories (those that are not
7339referenced by files from the master directories) are not automatically removed.
7340
7341Persistence data is kept in a directory that begins with the "STATE_DIR" prefix
7342from the configuration file (by default, "$SERVICEHOME/persistence/") followed
7343by the name of the client as given to @code{GNUNET_FS_start} (for example,
7344"gnunet-gtk") followed by the actual name of the master or child directory.
7345
7346The names for the master directories follow the names of the operations:
7347
7348@itemize @bullet
7349@item "search"
7350@item "download"
7351@item "publish"
7352@item "unindex"
7353@end itemize
7354
7355Each of the master directories contains names (chosen at random) for each
7356active top-level (master) operation. Note that a download that is associated
7357with a search result is not a top-level operation.
7358
7359In contrast to the master directories, the child directories are only consulted
7360when another operation refers to them. For each search, a subdirectory (named
7361after the master search synchronization file) contains the search results.
7362Search results can have an associated download, which is then stored in the
7363general "download-child" directory. Downloads can be recursive, in which case
7364children are stored in subdirectories mirroring the structure of the recursive
7365download (either starting in the master "download" directory or in the
7366"download-child" directory depending on how the download was initiated). For
7367publishing operations, the "publish-file" directory contains information about
7368the individual files and directories that are part of the publication. However,
7369this directory structure is flat and does not mirror the structure of the
7370publishing operation. Note that unindex operations cannot have associated child
7371operations.
7372
7373@node GNUnet's REGEX Subsystem
7374@section GNUnet's REGEX Subsystem
7375
7376@c %**end of header
7377
7378Using the REGEX subsystem, you can discover peers that offer a particular
7379service using regular expressions. The peers that offer a service specify it
7380using a regular expressions. Peers that want to patronize a service search
7381using a string. The REGEX subsystem will then use the DHT to return a set of
7382matching offerers to the patrons.
7383
7384For the technical details, we have "Max's defense talk and Max's Master's
7385thesis. An additional publication is under preparation and available to team
7386members (in Git).
7387
7388@menu
7389* How to run the regex profiler::
7390@end menu
7391
7392@node How to run the regex profiler
7393@subsection How to run the regex profiler
7394
7395@c %**end of header
7396
7397The gnunet-regex-profiler can be used to profile the usage of mesh/regex for a
7398given set of regular expressions and strings. Mesh/regex allows you to announce
7399your peer ID under a certain regex and search for peers matching a particular
7400regex using a string. See https://gnunet.org/szengel2012ms for a full
7401introduction.
7402
7403First of all, the regex profiler uses GNUnet testbed, thus all the implications
7404for testbed also apply to the regex profiler (for example you need
7405password-less ssh login to the machines listed in your hosts file).
7406
7407@strong{Configuration}
7408
7409Moreover, an appropriate configuration file is needed. Generally you can refer
7410to SVN HEAD: contrib/regex_profiler_infiniband.conf for an example
7411configuration. In the following paragraph the important details are
7412highlighted.
7413
7414Announcing of the regular expressions is done by the
7415gnunet-daemon-regexprofiler, therefore you have to make sure it is started, by
7416adding it to the AUTOSTART set of ARM:@
7417@code{
7418[regexprofiler]@
7419AUTOSTART = YES@
7420}
7421
7422Furthermore you have to specify the location of the binary:
7423@example
7424[regexprofiler]
7425# Location of the gnunet-daemon-regexprofiler binary.
7426BINARY = /home/szengel/gnunet/src/mesh/.libs/gnunet-daemon-regexprofiler
7427# Regex prefix that will be applied to all regular expressions and
7428# search string.
7429REGEX_PREFIX = "GNVPN-0001-PAD"
7430@end example
7431
7432When running the profiler with a large scale deployment, you probably want to
7433reduce the workload of each peer. Use the following options to do this.@
7434@example
7435[dht]@
7436# Force network size estimation@
7437FORCE_NSE = 1
7438
7439[dhtcache]
7440DATABASE = heap@
7441# Disable RC-file for Bloom filter? (for benchmarking with limited IO
7442# availability)@
7443DISABLE_BF_RC = YES@
7444# Disable Bloom filter entirely@
7445DISABLE_BF = YES
7446
7447[nse]@
7448# Minimize proof-of-work CPU consumption by NSE@
7449WORKBITS = 1
7450@end example
7451
7452
7453@strong{Options}
7454
7455To finally run the profiler some options and the input data need to be
7456specified on the command line.
7457@code{@ gnunet-regex-profiler -c config-file -d
7458log-file -n num-links -p@ path-compression-length -s search-delay -t
7459matching-timeout -a num-search-strings hosts-file policy-dir
7460search-strings-file@ }
7461
7462@code{config-file} the configuration file created earlier.@ @code{log-file}
7463file where to write statistics output.@ @code{num-links} number of random links
7464between started peers.@ @code{path-compression-length} maximum path compression
7465length in the DFA.@ @code{search-delay} time to wait between peers finished
7466linking and@ starting to match strings.@ @code{matching-timeout} timeout after
7467witch to cancel the searching.@ @code{num-search-strings} number of strings in
7468the search-strings-file.
7469
7470The @code{hosts-file} should contain a list of hosts for the testbed, one per
7471line in the following format. @code{user@@host_ip:port}.
7472
7473The @code{policy-dir} is a folder containing text files containing one or more
7474regular expressions. A peer is started for each file in that folder and the
7475regular expressions in the corresponding file are announced by this peer.
7476
7477The @code{search-strings-file} is a text file containing search strings, one in
7478each line.
7479
7480You can create regular expressions and search strings for every AS in the@
7481Internet using the attached scripts. You need one of the
7482@uref{http://data.caida.org/datasets/routing/routeviews-prefix2as/, CAIDA
7483routeviews prefix2as} data files for this. Run @code{create_regex.py <filename>
7484<output path>} to create the regular expressions and @code{create_strings.py
7485<input path> <outfile>} to create a search strings file from the previously
7486created regular expressions.
diff --git a/doc/chapters/installation.texi b/doc/chapters/installation.texi
new file mode 100644
index 000000000..ea949cdc0
--- /dev/null
+++ b/doc/chapters/installation.texi
@@ -0,0 +1,4205 @@
1@node GNUnet Installation Handbook
2@chapter GNUnet Installation Handbook
3
4This handbook describes how to install (build setup, compilation) and setup
5(configuration, start) GNUnet 0.10.x. After following these instructions you
6should be able to install and then start user-interfaces to interact with the
7network.
8
9This manual is far from complete, and we welcome informed contributions, be it
10in the form of new chapters or insightful comments.
11
12
13
14@menu
15* Dependencies::
16* Generic installation instructions::
17* Build instructions for Ubuntu 12.04 using Git::
18* Build Instructions for Microsoft Windows Platforms::
19* Build instructions for Debian 7.5::
20* Installing GNUnet from Git on Ubuntu 14.4::
21* Build instructions for Debian 8::
22* Outdated build instructions for previous revisions::
23* Portable GNUnet::
24* The grapical configuration interface::
25* How to start and stop a GNUnet peer::
26@end menu
27
28@node Dependencies
29@section Dependencies
30@c %**end of header
31
32This document lists the various known dependencies for GNUnet 0.10.x.
33Suggestions for missing dependencies or wrong version numbers are welcome.
34
35
36
37@menu
38* External dependencies::
39* Fixing libgnurl build issues::
40* Internal dependencies::
41@end menu
42
43@node External dependencies
44@subsection External dependencies
45@c %**end of header
46
47These packages must be installed before a typical GNUnet installation
48can be performed:
49
50@table @asis
51@item GNU libmicrohttpd
520.9.30 or higher
53@item GNU libextractor
541.0 or higher
55@item GNU libtool
562.2 or higher
57@item GNU libunistring
580.9.1.1 or higher
59@item GNU libidn
601.0.0 or higher
61@item @uref{ftp://ftp.gnupg.org/gcrypt/libgcrypt/, GNU libgcrypt}
62@uref{ftp://ftp.gnupg.org/gcrypt/libgcrypt/libgcrypt-1.6.0.tar.bz2, 1.6.0} or
63higher
64@item GnuTLS
65@uref{ftp://ftp.gnutls.org/gcrypt/gnutls/v3.2/gnutls-3.2.7.tar.xz, 3.2.7} or
66higher, compile with libunbound for DANE support; GnuTLS also requires GNU
67nettle 2.7 (update: GnuTLS 3.2.7 appears NOT to work against GNU nettle
68> 2.7, due to some API updatings done by nettle. Thus it should be compiled
69against nettle 2.7 and, in case you get some error on the reference to
70`rpl_strerror' being undefined, follow the instructions on@
71@uref{http://lists.gnupg.org/pipermail/gnutls-devel/2013-November/006588.html, this}
72post (and the link inside it)).
73@item libgnurl
747.34.0 or higher (available from https://gnunet.org/gnurl), should be compiled
75after @code{GnuTLS}
76@item libglpk
774.45 or higher
78@item @uref{http://www.openssl.org/, OpenSSL} (binary)
791.0 or higher
80@item TeX Live
812012 or higher, optional (for gnunet-bcd)
82@item libpulse
832.0 or higher, optional (for gnunet-conversation)
84@item libopus
851.0.1 or higher, optional (for gnunet-conversation)
86@item libogg
871.3.0 or higher, optional (for gnunet-conversation)
88@item certool (binary)
89optional for convenient installation of the GNS proxy
90(available as part of Debian's libnss3-tools)
91@item python-zbar
920.10 or higher, optional (for gnunet-qr)
93@item libsqlite
943.8.0 or higher (note that the code will compile and often work with lower
95version numbers, but you may get subtle bugs with respect to quota management
96in certain rare cases); alternatively, MySQL or Postgres can also be installed,
97but those databases will require more complex configurations (not recommended
98for first-time users)
99@item zlib
100any version we tested worked
101@item Gtk+
1023.0 or higher, optional (for gnunet-gtk)
103@item libgladeui
104must match Gtk+ version, optional (for gnunet-gtk)
105@item libqrencode
1063.0 or higher, optional (for gnunet-namestore-gtk)
107@end table
108
109
110@node Fixing libgnurl build issues
111@subsection Fixing libgnurl build issues
112
113If you have to compile libgnurl from source since the version included in your
114distribution is to old you perhaps get an error message while running the
115@code{configure} script:
116
117@code{@
118 $ configure@
119 ...@
120 checking for 64-bit curl_off_t data type... unknown@
121 checking for 32-bit curl_off_t data type... unknown@
122 checking for 16-bit curl_off_t data type... unknown@
123 configure: error: cannot find data type for curl_off_t.@
124}
125
126If you have to compile libgnurl from source since the version included in your
127distribution is to old, you perhaps get an error message while running the
128@code{configure} script:
129
130@code{@
131 $ configure@
132 ...@
133 checking for 64-bit curl_off_t data type... unknown@
134 checking for 32-bit curl_off_t data type... unknown@
135 checking for 16-bit curl_off_t data type... unknown@
136 configure: error: cannot find data type for curl_off_t.@
137}
138
139Solution:
140
141Before running the configure script, set:
142
143@code{CFLAGS="-I. -I$BUILD_ROOT/include" }
144
145
146
147@node Internal dependencies
148@subsection Internal dependencies
149
150This section tries to give an overview of what processes a typical GNUnet peer
151running a particular application would consist of. All of the processes listed
152here should be automatically started by @code{gnunet-arm -s}. The list is given
153as a rough first guide to users for failure diagnostics. Ideally, end-users
154should never have to worry about these internal dependencies.
155
156In terms of internal dependencies, a minimum file-sharing system consists of
157the following GNUnet processes (in order of dependency):
158
159@itemize @bullet
160@item
161gnunet-service-arm
162@item
163gnunet-service-resolver (required by all)
164@item
165gnunet-service-statistics (required by all)
166@item
167gnunet-service-peerinfo
168@item
169gnunet-service-transport (requires peerinfo)
170@item
171gnunet-service-core (requires transport)
172@item
173gnunet-daemon-hostlist (requires core)
174@item
175gnunet-daemon-topology (requires hostlist, peerinfo)
176@item
177gnunet-service-datastore
178@item
179gnunet-service-dht (requires core)
180@item
181gnunet-service-identity
182@item
183gnunet-service-fs (requires identity, mesh, dht, datastore, core)
184@end itemize
185
186
187A minimum VPN system consists of the following GNUnet processes (in order of
188dependency):
189
190@itemize @bullet
191@item
192gnunet-service-arm
193
194@item
195gnunet-service-resolver (required by all)
196
197@item
198gnunet-service-statistics (required by all)
199
200@item
201gnunet-service-peerinfo
202
203@item
204gnunet-service-transport (requires peerinfo)
205
206@item
207gnunet-service-core (requires transport)
208
209@item
210gnunet-daemon-hostlist (requires core)
211
212@item
213gnunet-service-dht (requires core)
214
215@item
216gnunet-service-mesh (requires dht, core)
217
218@item
219gnunet-service-dns (requires dht)
220
221@item
222gnunet-service-regex (requires dht)
223
224@item
225gnunet-service-vpn (requires regex, dns, mesh, dht)
226@end itemize
227
228
229A minimum GNS system consists of the following GNUnet processes (in order of
230dependency):
231@itemize @bullet
232
233@item
234gnunet-service-arm
235
236@item
237gnunet-service-resolver (required by all)
238
239@item
240gnunet-service-statistics (required by all)
241
242@item
243gnunet-service-peerinfo
244
245@item
246gnunet-service-transport (requires peerinfo)
247
248@item
249gnunet-service-core (requires transport)
250
251@item
252gnunet-daemon-hostlist (requires core)
253
254@item
255gnunet-service-dht (requires core)
256
257@item
258gnunet-service-mesh (requires dht, core)
259
260@item
261gnunet-service-dns (requires dht)
262
263@item
264gnunet-service-regex (requires dht)
265
266@item
267gnunet-service-vpn (requires regex, dns, mesh, dht)
268
269@item
270gnunet-service-identity
271
272@item
273gnunet-service-namestore (requires identity)
274
275@item
276gnunet-service-gns (requires vpn, dns, dht, namestore, identity)
277@end itemize
278
279@node Generic installation instructions
280@section Generic installation instructions
281
282First, in addition to the GNUnet sources you must download the latest version
283of various dependencies. Most distributions do not include sufficiently recent
284versions of these dependencies. Thus, a typically installation on a "modern"
285GNU/Linux distribution requires you to install the following
286dependencies (ideally in this order):
287
288@itemize @bullet
289
290@item
291libgpgerror and libgcrypt
292
293@item
294libnettle and libunbound (possibly from distribution), GnuTLS
295
296@item
297libgnurl (read the README)
298
299@item
300GNU libmicrohttpd
301
302@item
303GNU libextractor (make sure to first install the various mandatory and optional
304dependencies including development headers from your distribution)
305@end itemize
306
307Other dependencies that you should strongly consider to install is a
308database (MySQL, sqlite or Postgres). The following instructions will assume
309that you installed at least sqlite. For most distributions you should be able
310to find pre-build packages for the database. Again, make sure to install the
311client libraries and the respective development headers (if they are
312packaged separately) as well.
313
314You can find specific, detailed instructions for installing of the dependencies
315(and possibly the rest of the GNUnet installation) in the platform-specific
316descriptions, which are linked from the bottom of this page. Please consult
317them now. If your distribution is not listed, please study the instructions for
318Debian stable carefully as you try to install the dependencies for your own
319distribution. Contributing additional instructions for further platforms is
320always appreciated.
321
322Before proceeding further, please double-check the dependency list. Note that
323in addition to satisfying the dependencies, you might have to make sure that
324development headers for the various libraries are also installed. There maybe
325files for other distributions, or you might be able to find equivalent packages
326for your distribution.
327
328While it is possible to build and install GNUnet without having root access,
329we will assume that you have full control over your system in these
330instructions. First, you should create a system user "gnunet" and an additional
331group "gnunetdns". On Debian and Ubuntu GNU/Linux, type:@
332@code{@
333 # adduser --system --home /var/lib/gnunet --group --disabled-password gnunet@
334 # addgroup --system gnunetdns@
335}@
336 On other Unixes, this should have the same effect:@
337@code{@
338 # useradd --system --groups gnunet --home-dir /var/lib/gnunet@
339 # addgroup --system gnunetdns@
340}@
341 Now compile and install GNUnet using:@
342@code{@
343 $ tar xvf gnunet-0.10.?.tar.gz@
344 $ cd gnunet-0.10.?@
345 $ ./configure --with-sudo=sudo --with-nssdir=/lib@
346 $ make@
347 $ sudo make install@
348}@
349
350If you want to be able to enable DEBUG-level log messages, add
351@code{--enable-logging=verbose} to the end of the ./configure command.
352DEBUG-level log messages are in English-only and should only be useful for
353developers (or for filing really detailed bug reports).
354
355Finally, you probably want to compile gnunet-gtk, which includes gnunet-setup
356(graphical tool for configuration) and gnunet-fs-gtk (graphical tool for
357file-sharing):@
358
359@code{@
360 $ tar xvf gnunet-gtk-0.10.?.tar.gz@
361 $ cd gnunet-gtk-0.10.?@
362 $ ./configure --with-gnunet=/usr/local/@
363 $ make@
364 $ sudo make install@
365 $ cd ..@
366 $ sudo ldconfig # just to be safe@
367}@
368 Now, edit @code{/etc/gnunet.conf} to contain the following:@
369@code{@
370 [arm]@
371 SYSTEM_ONLY = YES@
372 USER_ONLY = NO@
373}@
374You may need to update your ld.so cache to include files installed in
375@file{/usr/local/lib}:@
376
377@code{@
378 # ldconfig@
379}@
380
381Then, switch from user root to user gnunet to start the peer:@
382
383@code{@
384 # su -s /bin/sh - gnunet@
385 $ gnunet-arm -c /etc/gnunet.conf -s@
386}@
387
388You may also want to add the last line in the gnunet users @file{crontab}
389prefixed with @code{@@reboot} so that it is executed whenever the system is
390booted:@
391
392@code{@
393 @@reboot /usr/local/bin/gnunet-arm -c /etc/gnunet.conf -s@
394}@
395
396This will only start the system-wide GNUnet services. Type exit to get back
397your root shell. Now, you need to configure the per-user part. For each
398$USER on the system, run:@
399
400@code{@
401 # adduser $USER gnunet@
402}@
403
404to allow them to access the system-wide GNUnet services. Then, each user should
405create a configuration file "~/.config/gnunet.conf" with the lines:@
406
407@code{@
408 [arm]@
409 SYSTEM_ONLY = NO@
410 USER_ONLY = YES@
411 DEFAULTSERVICES = gns@
412}@
413
414and start the per-user services using@
415
416@code{@
417 $ gnunet-arm -c ~/.config/gnunet.conf -s@
418}@
419
420Again, adding a @file{crontab} entry to autostart the peer is advised:@
421@code{@
422@@reboot /usr/local/bin/gnunet-arm -c $HOME/.config/gnunet.conf -s@
423}@
424
425Note that some GNUnet services (such as SOCKS5 proxies) may need a system-wide
426TCP port for each user. For those services, systems with more than one user may
427require each user to specify a different port number in their personal
428configuration file.
429
430Finally, the user should perform the basic initial setup for the GNU Name
431System. This is done by running two commands:@
432
433@example
434$ gnunet-gns-import.sh@
435$ gnunet-gns-proxy-setup-ca@
436@end example
437
438The first generates the default zones, wheras the second setups the GNS
439Certificate Authority with the user's browser. Now, to actiave GNS in the
440normal DNS resolution process, you need to edit your @file{/etc/nsswitch.conf}
441where you should find a line like this:
442@example
443hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4
444@end example
445
446
447The exact details may differ a bit, which is fine. Add the text
448"gns [NOTFOUND=return]" after "files":
449@example
450hosts: files gns [NOTFOUND=return] mdns4_minimal [NOTFOUND=return] dns mdns4
451@end example
452
453
454You might want to make sure that @file{/lib/libnss_gns.so.2} exists on your
455system, it should have been created during the installation.
456
457
458
459@node Build instructions for Ubuntu 12.04 using Git
460@section Build instructions for Ubuntu 12.04 using Git
461
462
463@menu
464* Install the required build tools::
465* Install libgcrypt 1.6 and libgpg-error::
466* Install gnutls with DANE support::
467* Install libgnurl::
468* Install libmicrohttpd from Git::
469* Install libextractor from Git::
470* Install GNUnet dependencies::
471* Build GNUnet::
472* Install the GNUnet-gtk user interface from Git::
473@end menu
474
475@node Install the required build tools
476@subsection Install the required build tools
477
478First, make sure Git is installed on your system:@
479
480$ sudo apt-get install git@
481
482Install the essential buildtools:@
483
484$ sudo apt-get install automake autopoint autoconf libtool
485
486@node Install libgcrypt 1.6 and libgpg-error
487@subsection Install libgcrypt 1.6 and libgpg-error
488
489$ wget ftp://ftp.gnupg.org/gcrypt/libgpg-error/libgpg-error-1.12.tar.bz2@
490$ tar xf libgpg-error-1.12.tar.bz2@
491$ cd libgpg-error-1.12@
492$ ./configure@
493$ sudo make install@
494$ cd ..@
495
496@node Install gnutls with DANE support
497@subsection Install gnutls with DANE support
498
499$ wget http://www.lysator.liu.se/~nisse/archive/nettle-2.7.1.tar.gz@
500$ tar xf nettle-2.7.1.tar.gz@
501$ cd nettle-2.7.1@
502$ ./configure@
503$ sudo make install@
504$ cd ..
505
506$ wget https://www.nlnetlabs.nl/downloads/ldns/ldns-1.6.16.tar.gz@
507$ tar xf ldns-1.6.16.tar.gz@
508$ cd ldns-1.6.16@
509$ ./configure@
510$ sudo make install@
511$ cd ..
512
513$ wget https://unbound.net/downloads/unbound-1.4.21.tar.gz@
514$ tar xf unbound-1.4.21.tar.gz@
515$ cd unbound-1.4.21@
516$ ./configure@
517$ sudo make install@
518$ cd ..
519
520$ wget ftp://ftp.gnutls.org/gcrypt/gnutls/v3.1/gnutls-3.1.17.tar.xz@
521$ tar xf gnutls-3.1.17.tar.xz@
522$ cd gnutls-3.1.17@
523$ ./configure@
524$ sudo make install@
525$ cd ..
526
527$ wget ftp://ftp.gnupg.org/gcrypt/libgcrypt/libgcrypt-1.6.0.tar.bz2@
528$ tar xf libgcrypt-1.6.0.tar.bz2@
529$ cd libgcrypt-1.6.0@
530$ ./configure@
531$ sudo make install@
532$ cd ..@
533
534@node Install libgnurl
535@subsection Install libgnurl
536
537$ wget https://gnunet.org/sites/default/files/gnurl-7.34.0.tar.bz2@
538$ tar xf gnurl-7.34.0.tar.bz2@
539$ cd gnurl-7.34.0@
540$ ./configure --enable-ipv6 --with-gnutls --without-libssh2 \
541 --without-libmetalink --without-winidn --without-librtmp \
542 --without-nghttp2 --without-nss --without-cyassl \
543 --without-polarssl --without-ssl --without-winssl \
544 --without-darwinssl --disable-sspi --disable-ntlm-wb \
545 --disable-ldap --disable-rtsp --disable-dict --disable-telnet \
546 --disable-tftp --disable-pop3 --disable-imap --disable-smtp \
547 --disable-gopher --disable-file --disable-ftp@
548$ sudo make install@
549$ cd ..@
550
551@node Install libmicrohttpd from Git
552@subsection Install libmicrohttpd from Git
553
554$ git clone https://gnunet.org/git/libmicrohttpd@
555$ cd libmicrohttpd/@
556$ ./bootstrap@
557$ ./configure@
558$ sudo make install@
559$ cd ..@
560
561@node Install libextractor from Git
562@subsection Install libextractor from Git
563
564Install libextractor dependencies:@
565
566$ sudo apt-get install zlib1g-dev libgsf-1-dev libmpeg2-4-dev libpoppler-dev \
567 libvorbis-dev libexiv2-dev libjpeg-dev libtiff-dev libgif-dev libvorbis-dev \
568 libflac-dev libsmf-dev g++@
569
570Build libextractor:@
571
572$ git clone https://gnunet.org/git/libextractor@
573$ cd libextractor@
574$ ./bootstrap@
575$ ./configure@
576$ sudo make install@
577$ cd ..@
578
579@node Install GNUnet dependencies
580@subsection Install GNUnet dependencies
581
582$ sudo apt-get install libidn11-dev libunistring-dev libglpk-dev \
583 libpulse-dev libbluetooth-dev libsqlite-dev@
584
585Install libopus@
586
587$ wget http://downloads.xiph.org/releases/opus/opus-1.1.tar.gz@
588$ tar xf opus-1.1.tar.gz@
589$ cd opus-1.1/@
590$ ./configure@
591$ sudo make install@
592
593Choose one or more database backends@
594@itemize @bullet
595
596@item
597SQLite3 @code{$ sudo apt-get install libsqlite3-dev}
598
599@item
600MySQL @code{$ sudo apt-get install libmysqlclient-dev}
601
602@item
603PostgreSQL @code{$ sudo apt-get install libpq-dev postgresql}
604
605@end itemize
606
607
608
609@node Build GNUnet
610@subsection Build GNUnet
611
612
613
614@menu
615* Configuring the installation path::
616* Configuring the system::
617* Installing components requiring sudo permission::
618* Build::
619@end menu
620
621@node Configuring the installation path
622@subsubsection Configuring the installation path
623
624You can specify the location of the GNUnet installation by setting the prefix
625when calling the configure script:@code{ --prefix=DIRECTORY}
626
627@code{@
628 $ export PATH=$PATH:DIRECTORY/bin@
629}
630
631@node Configuring the system
632@subsubsection Configuring the system
633
634Please make sure NOW that you have created a user and group 'gnunet'@
635and additionally a group 'gnunetdns':@
636@code{@
637 $ sudo addgroup gnunet@
638 $ sudo addgroup gnunetdns@
639 $ sudo adduser gnunet@
640}
641
642Each GNUnet user should be added to the 'gnunet' group (may@
643require fresh login to come into effect):
644@code{@
645 $ sudo useradd -G gnunet@
646}
647
648@node Installing components requiring sudo permission
649@subsubsection Installing components requiring sudo permission
650
651Some components, like the nss plugin required for GNS, may require root
652permissions. To allow these few components to be installed use:@
653@code{@
654 $ ./configure --with-sudo}
655
656@node Build
657@subsubsection Build
658
659
660@code{@
661 $ git clone https://gnunet.org/git/gnunet/@
662 $ cd gnunet/@
663 $ ./bootstrap@
664}
665Use the required configure call including the optional installation prefix
666PREFIX or the sudo permissions@
667@code{$ ./configure [ --with-sudo | --with-prefix=PREFIX ]}@
668@code{$ make; sudo make install}
669
670After installing it, you need to create an empty configuration file:@
671@code{mkdir ~/.gnunet; touch ~/.gnunet/gnunet.conf}
672
673And finally you can start GNUnet with@
674@code{$ gnunet-arm -s}
675
676@node Install the GNUnet-gtk user interface from Git
677@subsection Install the GNUnet-gtk user interface from Git
678
679
680Install depencies:@
681@code{$ sudo apt-get install libgtk-3-dev libunique-3.0-dev libgladeui-dev libqrencode-dev}
682
683To build GNUnet (with an optional prefix)and execute:@
684@code{@
685 $ git clone https://gnunet.org/git/gnunet-gtk/@
686 $ cd gnunet-gtk/@
687 $ ./bootstrap@
688 $ ./configure [--prefix=PREFIX] --with-gnunet=DIRECTORY@
689 $ make; sudo make install@
690}
691
692@node Build Instructions for Microsoft Windows Platforms
693@section Build Instructions for Microsoft Windows Platforms
694
695
696
697@menu
698* Introduction to building on MS Windows::
699* Requirements::
700* Dependencies & Initial Setup::
701* GNUnet Installation::
702* Adjusting Windows for running and testing GNUnet::
703* Building the GNUnet Installer::
704* Using GNUnet with Netbeans on Windows::
705@end menu
706
707@node Introduction to building on MS Windows
708@subsection Introduction to building on MS Windows
709
710
711This document is a guide to building GNUnet and its dependencies on Windows
712platforms. GNUnet development is mostly done under Linux and especially SVN
713checkouts may not build out of the box. We regret any inconvenience, and if you
714have problems, please report them.
715
716@node Requirements
717@subsection Requirements
718
719The Howto is based upon a @strong{Windows Server 2008 32bit@strong{
720Installation, @strong{sbuild} and thus a @uref{http://www.mingw.org/wiki/MSYS,
721MSYS+MinGW} (W32-GCC-Compiler-Suite + Unix-like Userland) installation. sbuild
722is a convenient set of scripts which creates a working msys/mingw installation
723and installs most dependencies required for GNUnet. }}
724
725As of the point of the creation of this Howto, GNUnet @strong{requires} a
726Windows @strong{Server} 2003 or newer for full feature support. Windows Vista
727and later will also work, but
728@strong{non-server version can not run a VPN-Exit-Node} as the NAT features
729have been removed as of Windows Vista.
730
731@node Dependencies & Initial Setup
732@subsection Dependencies & Initial Setup
733
734
735@itemize @bullet
736
737@item
738Install a fresh version of @strong{Python 2.x}, even if you are using a x64-OS,
739install a 32-bit version for use with sbuild. Python 3.0 currently is
740incompatible.
741
742@item
743Install your favorite @uref{http://code.google.com/p/tortoisegit/, GIT} &
744@uref{http://tortoisesvn.net/, SVN}-clients.
745
746@item
747You will also need some archive-manager like @uref{http://www.7-zip.org/, 7zip}.
748
749@item
750Pull a copy of sbuild to a directory of your choice, which will be used in the
751remainder of this guide. For now, we will use @file{c:\gnunet\sbuild\}
752
753@item
754in @file{sbuild\src\mingw\mingw32-buildall.sh}, comment out the packages
755@strong{gnunet-svn} and @strong{gnunet-gtk-svn}, as we don't want sbuild to
756compile/install those for us.
757
758@item
759Follow LRN's sbuild installation instructions.-
760@end itemize
761
762Please note that sbuild may (or will most likely) fail during installation,
763thus you really HAVE to @strong{check the logfiles} created during the
764installation process. Certain packages may fail to build initially due to
765missing dependencies, thus you may have to
766@strong{substitute those with binary-versions initially}. Later on once
767dependencies are satisfied you can re-build the newer package versions.
768
769@strong{It is normal that you may have to repeat this step multiple times and
770there is no uniform way to fix all compile-time issues, as the build-process
771of many of the dependencies installed are rather unstable on win32 and certain
772releases may not even compile at all.}
773
774Most dependencies for GNUnet have been set up by sbuild, thus we now should add
775the @file{bin/} directories in your new msys and mingw installations to PATH.
776You will want to create a backup of your finished msys-environment by now.
777
778@node GNUnet Installation
779@subsection GNUnet Installation
780
781First, we need to launch our msys-shell, you can do this via
782
783@file{C:\gnunet\sbuild\msys\msys.bat}
784
785You might wish to take a look at this file and adjust some login-parameters to
786your msys environment.
787
788Also, sbuild added two pointpoints to your msys-environment, though those
789might remain invisible:
790
791@itemize @bullet
792
793@item
794/mingw, which will mount your mingw-directory from sbuild/mingw and the other one is
795
796@item
797/src which contains all the installation sources sbuild just compiled.
798@end itemize
799
800Check out the current gnunet-sources (svn-head) from the gnunet-repository,
801we will do this in your home directory:
802
803@code{svn checkout https://gnunet.org/svn/gnunet/ ~/gnunet}
804
805Now, we will first need to bootstrap the checked out installation and then
806configure it accordingly.
807
808@example
809cd ~/gnunet@
810./bootstrap@
811STRIP=true CPPFLAGS="-DUSE_IPV6=1 -DW32_VEH" CFLAGS="$CFLAGS -g -O2" ./configure --prefix=/ --docdir=/share/doc/gnunet --with-libiconv-prefix=/mingw --with-libintl-prefix=/mingw --with-libcurl=/mingw --with-extractor=/mingw --with-sqlite=/mingw --with-microhttpd=/mingw --with-plibc=/mingw --enable-benchmarks --enable-expensivetests --enable-experimental --with-qrencode=/mingw --enable-silent-rules --enable-experimental 2>&1 | tee -a ./configure.log
812@end example
813
814The parameters above will configure for a reasonable gnunet installation to the
815your msys-root directory. Depending on which features your would like to build
816or you may need to specify additional dependencies. Sbuild installed most libs
817into the /mingw subdirectory, so remember to prefix library locations with
818this path.
819
820Like on a unixoid system, you might want to use your home directory as prefix
821for your own gnunet installation for development, without tainting the
822buildenvironment. Just change the "prefix" parameter to point towards
823~/ in this case.
824
825Now it's time to compile gnunet as usual. Though this will take some time, so
826you may fetch yourself a coffee or some Mate now...
827
828@example
829make@
830make install
831@end example
832
833@node Adjusting Windows for running and testing GNUnet
834@subsection Adjusting Windows for running and testing GNUnet
835
836Assuming the build succeeded and you
837@strong{added the bin directory of your gnunet to PATH}, you can now use your
838gnunet-installation as usual. Remember that UAC or the windows firewall may
839popup initially, blocking further execution of gnunet until you acknowledge
840them (duh!).
841
842You will also have to take the usual steps to get p2p software running properly
843(port forwarding, ...), and gnunet will require administrative permissions as
844it may even install a device-driver (in case you are using gnunet-vpn and/or
845gnunet-exit).
846
847@node Building the GNUnet Installer
848@subsection Building the GNUnet Installer
849
850The GNUnet installer is made with @uref{http://nsis.sourceforge.net/, NSIS}@
851The installer script is located in @file{contrib\win} in the GNUnet source tree.
852
853@node Using GNUnet with Netbeans on Windows
854@subsection Using GNUnet with Netbeans on Windows
855
856TODO
857
858@node Build instructions for Debian 7.5
859@section Build instructions for Debian 7.5
860
861
862These are the installation instructions for Debian 7.5. They were tested using
863a minimal, fresh Debian 7.5 AMD64 installation without non-free software
864(no contrib or non-free). By "minimal", we mean that during installation, we
865did not select any desktop environment, servers or system utilities during the
866"tasksel" step. Note that the packages and the dependencies that we will
867install during this chapter take about 1.5 GB of disk space. Combined with
868GNUnet and space for objects during compilation, you should not even attempt
869this unless you have about 2.5 GB free after the minimal Debian installation.
870Using these instructions to build a VM image is likely to require a minimum of
8714-5 GB for the VM (as you will likely also want a desktop manager).
872
873GNUnet's security model assumes that your @file{/home} directory is encrypted.
874Thus, if possible, you should encrypt your home partition
875(or per-user home directory).
876
877Naturally, the exact details of the starting state for your installation
878should not matter much. For example, if you selected any of those installation
879groups you might simply already have some of the necessary packages installed.
880We did this for testing, as this way we are less likely to forget to mention a
881required package. Note that we will not install a desktop environment, but of
882course you will need to install one to use GNUnet's graphical user interfaces.
883Thus, it is suggested that you simply install the desktop environment of your
884choice before beginning with the instructions.
885
886
887
888@menu
889* Update::
890* Stable? Hah!::
891* Update again::
892* Installing packages::
893* Installing dependencies from source::
894* Installing GNUnet from source::
895* But wait there is more!::
896@end menu
897
898@node Update
899@subsection Update
900
901After any installation, you should begin by running
902
903@example
904# apt-get update@
905# apt-get upgrade@
906@end example
907
908to ensure that all of your packages are up-to-date. Note that the "#" is used
909to indicate that you need to type in this command as "root"
910(or prefix with "sudo"), whereas "$" is used to indicate typing in a command
911as a normal user.
912
913@node Stable? Hah!
914@subsection Stable? Hah!
915
916Yes, we said we start with a Debian 7.5 "stable" system. However, to reduce the
917amount of compilation by hand, we will begin by allowing the installation of
918packages from the testing and unstable distributions as well. We will stick to
919"stable" packages where possible, but some packages will be taken from the
920other distributions. Start by modifying @file{/etc/apt/sources.list} to contain
921the following (possibly adjusted to point to your mirror of choice):
922@example
923# These were there before:
924deb http://ftp.de.debian.org/debian/ wheezy main
925deb-src http://ftp.de.debian.org/debian/ wheezy main
926deb http://security.debian.org/ wheezy/updates main
927deb-src http://security.debian.org/ wheezy/updates main
928deb http://ftp.de.debian.org/debian/ wheezy-updates main
929deb-src http://ftp.de.debian.org/debian/ wheezy-updates main
930
931# Add these lines (feel free to adjust the mirror):
932deb http://ftp.de.debian.org/debian/ testing main
933deb http://ftp.de.debian.org/debian/ unstable main
934@end example
935
936The next step is to create/edit your @file{/etc/apt/preferences} file to look
937like this:
938
939@example
940Package: *
941Pin: release a=stable,n=wheezy
942Pin-Priority: 700
943
944Package: *
945Pin: release o=Debian,a=testing
946Pin-Priority: 650
947
948Package: *
949Pin: release o=Debian,a=unstable
950Pin-Priority: 600
951@end example
952
953You can read more about Apt Preferences here and here. Note that other pinnings
954are likely to also work for GNUnet, the key thing is that you need some
955packages from unstable (as shown below). However, as unstable is unlikely to
956be comprehensive (missing packages) or might be problematic (crashing packages),
957you probably want others from stable and/or testing.
958
959@node Update again
960@subsection Update again
961
962Now, run again@
963
964@example
965# apt-get update@
966# apt-get upgrade@
967@end example
968
969to ensure that all your new distribution indices are downloaded, and that your
970pinning is correct: the upgrade step should cause no changes at all.
971
972@node Installing packages
973@subsection Installing packages
974
975We begin by installing a few Debian packages from stable:@
976
977@example
978# apt-get install gcc make python-zbar libltdl-dev libsqlite3-dev \
979 libunistring-dev libopus-dev libpulse-dev openssl libglpk-dev \
980 texlive libidn11-dev libmysqlclient-dev libpq-dev libarchive-dev \
981 libbz2-dev libexiv2-dev libflac-dev libgif-dev libglib2.0-dev \
982 libgtk-3-dev libmagic-dev libjpeg8-dev libmpeg2-4-dev libmp4v2-dev \
983 librpm-dev libsmf-dev libtidy-dev libtiff5-dev libvorbis-dev \
984 libogg-dev zlib1g-dev g++ gettext libgsf-1-dev libunbound-dev \
985 libqrencode-dev libgladeui-dev nasm texlive-latex-extra \
986 libunique-3.0-dev gawk miniupnpc libfuse-dev libbluetooth-dev
987@end example
988
989After that, we install a few more packages from unstable:@
990
991@example
992# apt-get install -t unstable nettle-dev libgstreamer1.0-dev \
993 gstreamer1.0-plugins-base gstreamer1.0-plugins-good \
994 libgstreamer-plugins-base1.0-dev
995@end example
996
997@node Installing dependencies from source
998@subsection Installing dependencies from source
999
1000Next, we need to install a few dependencies from source. You might want to do
1001this as a "normal" user and only run the @code{make install} steps as root
1002(hence the @code{sudo} in the commands below). Also, you do this from any
1003directory. We begin by downloading all dependencies, then extracting the
1004sources, and finally compiling and installing the libraries:@
1005
1006@example
1007 $ wget https://libav.org/releases/libav-9.10.tar.xz@
1008 $ wget http://ftp.gnu.org/gnu/libextractor/libextractor-1.3.tar.gz@
1009 $ wget ftp://ftp.gnupg.org/gcrypt/libgpg-error/libgpg-error-1.12.tar.bz2@
1010 $ wget ftp://ftp.gnupg.org/gcrypt/libgcrypt/libgcrypt-1.6.0.tar.bz2@
1011 $ wget ftp://ftp.gnutls.org/gcrypt/gnutls/v3.2/gnutls-3.2.7.tar.xz@
1012 $ wget http://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.33.tar.gz@
1013 $ wget https://gnunet.org/sites/default/files/gnurl-7.34.0.tar.bz2@
1014 $ tar xvf libextractor-1.3.tar.gz@
1015 $ tar xvf libgpg-error-1.12.tar.bz2@
1016 $ tar xvf libgcrypt-1.6.0.tar.bz2@
1017 $ tar xvf gnutls-3.2.7.tar.xz@
1018 $ tar xvf libmicrohttpd-0.9.33.tar.gz@
1019 $ tar xvf gnurl-7.34.0.tar.bz2@
1020 $ cd libav-0.9 ; ./configure --enable-shared; make; sudo make install ; cd ..@
1021 $ cd libextractor-1.3 ; ./configure; make ; sudo make install; cd ..@
1022 $ cd libgpg-error-1.12; ./configure ; make ; sudo make install ; cd ..@
1023 $ cd libgcrypt-1.6.0; ./configure --with-gpg-error-prefix=/usr/local; make ; sudo make install ; cd ..@
1024 $ cd gnutls-3.2.7 ; ./configure ; make ; sudo make install ; cd ..@
1025 $ cd libmicrohttpd-0.9.33; ./configure ; make ; sudo make install ; cd ..@
1026 $ cd gnurl-7.34.0@
1027 $ ./configure --enable-ipv6 --with-gnutls=/usr/local --without-libssh2 \
1028 --without-libmetalink --without-winidn --without-librtmp --without-nghttp2 \
1029 --without-nss --without-cyassl --without-polarssl --without-ssl \
1030 --without-winssl --without-darwinssl --disable-sspi --disable-ntlm-wb \
1031 --disable-ldap --disable-rtsp --disable-dict --disable-telnet --disable-tftp \
1032 --disable-pop3 --disable-imap --disable-smtp --disable-gopher --disable-file \
1033 --disable-ftp@
1034 $ make ; sudo make install; cd ..@
1035@end example
1036
1037@node Installing GNUnet from source
1038@subsection Installing GNUnet from source
1039
1040
1041For this, simply follow the generic installation instructions from
1042here.
1043
1044@node But wait there is more!
1045@subsection But wait there is more!
1046
1047So far, we installed all of the packages and dependencies required to ensure
1048that all of GNUnet would be built. However, while for example the plugins to
1049interact with the MySQL or Postgres databases have been created, we did not
1050actually install or configure those databases. Thus, you will need to install
1051and configure those databases or stick with the default Sqlite database.
1052Sqlite is usually fine for most applications, but MySQL can offer better
1053performance and Postgres better resillience.
1054
1055
1056@node Installing GNUnet from Git on Ubuntu 14.4
1057@section Installing GNUnet from Git on Ubuntu 14.4
1058
1059@strong{Install the required build tools:}
1060@code{@
1061 $ sudo apt-get install git automake autopoint autoconf@
1062}
1063
1064@strong{Install the required dependencies}
1065@example
1066$ sudo apt-get install libltdl-dev libgpg-error-dev libidn11-dev \
1067 libunistring-dev libglpk-dev libbluetooth-dev libextractor-dev \
1068 libmicrohttpd-dev libgnutls28-dev
1069@end example
1070
1071@strong{Choose one or more database backends}@
1072 SQLite3@
1073@code{@
1074 $ sudo apt-get install libsqlite3-dev@
1075}@
1076 MySQL@
1077@code{@
1078 $ sudo apt-get install libmysqlclient-dev@
1079}@
1080 PostgreSQL@
1081@code{@
1082 $ sudo apt-get install libpq-dev postgresql@
1083}
1084
1085@strong{Install the optional dependencies for gnunet-conversation:}@
1086@code{@
1087 $ sudo apt-get install gstreamer1.0 libpulse-dev libopus-dev@
1088}
1089
1090@strong{Install the libgrypt 1.6.1:}@
1091 For Ubuntu 14.04:@
1092@code{$ sudo apt-get install libgcrypt20-dev}@
1093 For Ubuntu older 14.04:@
1094@code{$ wget ftp://ftp.gnupg.org/gcrypt/libgcrypt/libgcrypt-1.6.1.tar.bz2@
1095 $ tar xf libgcrypt-1.6.1.tar.bz2@
1096 $ cd libgcrypt-1.6.1@
1097 $ ./configure@
1098 $ sudo make install@
1099 $ cd ..}@
1100@strong{Install libgnurl}@
1101@example
1102 $ wget https://gnunet.org/sites/default/files/gnurl-7.35.0.tar.bz2@
1103 $ tar xf gnurl-7.35.0.tar.bz2@
1104 $ cd gnurl-7.35.0@
1105 $ ./configure --enable-ipv6 --with-gnutls --without-libssh2 \
1106 --without-libmetalink --without-winidn --without-librtmp --without-nghttp2 \
1107 --without-nss --without-cyassl --without-polarssl --without-ssl \
1108 --without-winssl --without-darwinssl --disable-sspi --disable-ntlm-wb \
1109 --disable-ldap --disable-rtsp --disable-dict --disable-telnet --disable-tftp \
1110 --disable-pop3 --disable-imap --disable-smtp --disable-gopher --disable-file \
1111 --disable-ftp
1112 $ sudo make install@
1113 $ cd ..@
1114@end example
1115
1116@strong{Install GNUnet}@
1117@code{@
1118 $ git clone https://gnunet.org/git/gnunet/@
1119 $ cd gnunet/@
1120 $ ./bootstrap@
1121}
1122
1123If you want to:
1124@itemize @bullet
1125
1126
1127@item
1128Install to a different directory:@
1129 --prefix=PREFIX
1130
1131@item
1132Have sudo permission, but do not want to compile as root:@
1133 --with-sudo
1134
1135@item
1136Want debug message enabled:@
1137 -- enable-logging=verbose
1138@end itemize
1139
1140
1141@code{@
1142 $ ./configure [ --with-sudo | --prefix=PREFIX | --- enable-logging=verbose]@
1143 $ make; sudo make install@
1144}
1145
1146After installing it, you need to create an empty configuration file:@
1147@code{touch ~/.config/gnunet.conf}
1148
1149And finally you can start GNUnet with@
1150@code{$ gnunet-arm -s}
1151
1152@node Build instructions for Debian 8
1153@section Build instructions for Debian 8
1154
1155These are the installation instructions for Debian 8. They were tested using a
1156fresh Debian 8 AMD64 installation without non-free software (no contrib or
1157non-free). During installation, I only selected "lxde" for the desktop
1158environment. Note that the packages and the dependencies that we will install
1159during this chapter take about 1.5 GB of disk space. Combined with GNUnet and
1160space for objects during compilation, you should not even attempt this unless
1161you have about 2.5 GB free after the Debian installation. Using these
1162instructions to build a VM image is likely to require a minimum of 4-5 GB for
1163the VM (as you will likely also want a desktop manager).
1164
1165GNUnet's security model assumes that your @code{/home} directory is encrypted.
1166Thus, if possible, you should encrypt your entire disk, or at least just your
1167home partition (or per-user home directory).
1168
1169Naturally, the exact details of the starting state for your installation should
1170not matter much. For example, if you selected any of those installation groups
1171you might simply already have some of the necessary packages installed. Thus,
1172it is suggested that you simply install the desktop environment of your choice
1173before beginning with the instructions.
1174
1175
1176@menu
1177* Update Debian::
1178* Installing Debian Packages::
1179* Installing Dependencies from Source2::
1180* Installing GNUnet from Source2::
1181* But wait (again) there is more!::
1182@end menu
1183
1184@node Update Debian
1185@subsection Update Debian
1186
1187After any installation, you should begin by running@
1188@code{@
1189 # apt-get update@
1190 # apt-get upgrade@
1191}@
1192to ensure that all of your packages are up-to-date. Note that the "#" is used
1193to indicate that you need to type in this command as "root" (or prefix with
1194"sudo"), whereas "$" is used to indicate typing in a command as a normal
1195user.
1196
1197@node Installing Debian Packages
1198@subsection Installing Debian Packages
1199
1200We begin by installing a few Debian packages from stable:@
1201@example
1202 # apt-get install gcc make python-zbar libltdl-dev libsqlite3-dev \
1203 libunistring-dev libopus-dev libpulse-dev openssl libglpk-dev texlive \
1204 libidn11-dev libmysqlclient-dev libpq-dev libarchive-dev libbz2-dev \
1205 libflac-dev libgif-dev libglib2.0-dev libgtk-3-dev libmpeg2-4-dev \
1206 libtidy-dev libvorbis-dev libogg-dev zlib1g-dev g++ gettext libgsf-1-dev \
1207 libunbound-dev libqrencode-dev libgladeui-dev nasm texlive-latex-extra \
1208 libunique-3.0-dev gawk miniupnpc libfuse-dev libbluetooth-dev \
1209 gstreamer1.0-plugins-base gstreamer1.0-plugins-good \
1210 libgstreamer-plugins-base1.0-dev nettle-dev libextractor-dev libgcrypt20-dev \
1211 libmicrohttpd-dev
1212@end example
1213
1214@node Installing Dependencies from Source2
1215@subsection Installing Dependencies from Source2
1216
1217Yes, we said we start with a Debian 8 "stable" system, but because Debian
1218linked GnuTLS without support for DANE, we need to compile a few things, in
1219addition to GNUnet, still by hand. Yes, you can run GNUnet using the respective
1220Debian packages, but then you will not get DANE support.
1221
1222Next, we need to install a few dependencies from source. You might want to do
1223this as a "normal" user and only run the @code{make install} steps as root
1224(hence the @code{sudo} in the commands below). Also, you do this from any
1225directory. We begin by downloading all dependencies, then extracting the
1226sources, and finally compiling and installing the libraries:@
1227
1228@code{@
1229 $ wget ftp://ftp.gnutls.org/gcrypt/gnutls/v3.3/gnutls-3.3.12.tar.xz@
1230 $ wget https://gnunet.org/sites/default/files/gnurl-7.40.0.tar.bz2@
1231 $ tar xvf gnutls-3.3.12.tar.xz@
1232 $ tar xvf gnurl-7.40.0.tar.bz2@
1233 $ cd gnutls-3.3.12 ; ./configure ; make ; sudo make install ; cd ..@
1234 $ cd gnurl-7.40.0@
1235 $ ./configure --enable-ipv6 --with-gnutls=/usr/local --without-libssh2 \
1236 --without-libmetalink --without-winidn --without-librtmp --without-nghttp2 \
1237 --without-nss --without-cyassl --without-polarssl --without-ssl \
1238 --without-winssl --without-darwinssl --disable-sspi --disable-ntlm-wb \
1239 --disable-ldap --disable-rtsp --disable-dict --disable-telnet --disable-tftp \
1240 --disable-pop3 --disable-imap --disable-smtp --disable-gopher --disable-file \
1241 --disable-ftp --disable-smb
1242 $ make ; sudo make install; cd ..@
1243}
1244
1245@node Installing GNUnet from Source2
1246@subsection Installing GNUnet from Source2
1247
1248For this, simply follow the generic installation instructions from@
1249here.
1250
1251@node But wait (again) there is more!
1252@subsection But wait (again) there is more!
1253
1254So far, we installed all of the packages and dependencies required to ensure
1255that all of GNUnet would be built. However, while for example the plugins to
1256interact with the MySQL or Postgres databases have been created, we did not
1257actually install or configure those databases. Thus, you will need to install
1258and configure those databases or stick with the default Sqlite database. Sqlite
1259is usually fine for most applications, but MySQL can offer better performance
1260and Postgres better resillience.
1261
1262@node Outdated build instructions for previous revisions
1263@section Outdated build instructions for previous revisions
1264
1265This chapter contains a collection of outdated, older installation guides. They
1266are mostly intended to serve as a starting point for writing up-to-date
1267instructions and should not be expected to work for GNUnet 0.10.x.
1268
1269
1270@menu
1271* Installing GNUnet 0.10.1 on Ubuntu 14.04::
1272* Build instructions for FreeBSD 8::
1273* Basic installation for Mac OS X::
1274* Basic Installation for Fedora/PlanetLab nodes running Fedora 12::
1275* Basic Installation for Fedora/PlanetLab nodes running Fedora 8 .::
1276* Build instructions for Gentoo::
1277* Building GLPK for MinGW::
1278* Compiling libgnurl for GNUnet cannot find data type for curl_off_t.::
1279* GUI build instructions for Ubuntu 12.04 using Subversion::
1280* Installation with gnunet-update::
1281* Instructions for Microsoft Windows Platforms (Old)::
1282@end menu
1283
1284
1285@node Installing GNUnet 0.10.1 on Ubuntu 14.04
1286@subsection Installing GNUnet 0.10.1 on Ubuntu 14.04
1287
1288Install the required dependencies@
1289
1290@example
1291$ sudo apt-get install libltdl-dev libgpg-error-dev libidn11-dev \
1292 libunistring-dev libglpk-dev libbluetooth-dev libextractor-dev \
1293 libmicrohttpd-dev libgnutls28-dev
1294@end example
1295
1296Choose one or more database backends@
1297SQLite3@
1298@code{@
1299 $ sudo apt-get install libsqlite3-dev@
1300}@
1301MySQL@
1302@code{@
1303 $ sudo apt-get install libmysqlclient-dev@
1304}@
1305PostgreSQL@
1306@code{@
1307 $ sudo apt-get install libpq-dev postgresql@
1308}
1309
1310Install the optional dependencies for gnunet-conversation:@
1311@code{@
1312 $ sudo apt-get install gstreamer1.0 libpulse-dev libopus-dev@
1313}
1314
1315Install the libgrypt 1.6:@
1316For Ubuntu 14.04:@
1317@code{$ sudo apt-get install libgcrypt20-dev}@
1318For Ubuntu older 14.04:@
1319@code{$ wget ftp://ftp.gnupg.org/gcrypt/libgcrypt/libgcrypt-1.6.1.tar.bz2@
1320 $ tar xf libgcrypt-1.6.1.tar.bz2@
1321 $ cd libgcrypt-1.6.1@
1322 $ ./configure@
1323 $ sudo make install@
1324 $ cd ..}
1325
1326Install libgnurl@
1327@example
1328 $ wget https://gnunet.org/sites/default/files/gnurl-7.35.0.tar.bz2@
1329 $ tar xf gnurl-7.35.0.tar.bz2@
1330 $ cd gnurl-7.35.0@
1331 $ ./configure --enable-ipv6 --with-gnutls --without-libssh2 \
1332 --without-libmetalink --without-winidn --without-librtmp --without-nghttp2 \
1333 --without-nss --without-cyassl --without-polarssl --without-ssl \
1334 --without-winssl --without-darwinssl --disable-sspi --disable-ntlm-wb \
1335 --disable-ldap --disable-rtsp --disable-dict --disable-telnet --disable-tftp \
1336 --disable-pop3 --disable-imap --disable-smtp --disable-gopher --disable-file \
1337 --disable-ftp@
1338 $ sudo make install@
1339 $ cd ..@
1340@end example
1341
1342Install GNUnet@
1343@code{@
1344 $ wget http://ftpmirror.gnu.org/gnunet/gnunet-0.10.1.tar.gz@
1345 $ tar xf gnunet-0.10.1.tar.gz@
1346 $ cd gnunet-0.10.1@
1347}
1348
1349If you want to:
1350@itemize @bullet
1351
1352@item
1353Install to a different directory:@
1354 --prefix=PREFIX
1355
1356@item
1357Have sudo permission, but do not want to compile as root:@
1358 --with-sudo
1359
1360@item
1361Want debug message enabled:@
1362 -- enable-logging=verbose
1363@end itemize
1364
1365@code{@
1366 $ ./configure [ --with-sudo | --prefix=PREFIX | --enable-logging=verbose]@
1367 $ make; sudo make install@
1368}
1369
1370After installing it, you need to create an empty configuration file:@
1371@code{touch ~/.config/gnunet.conf}
1372
1373And finally you can start GNUnet with@
1374@code{$ gnunet-arm -s}
1375
1376
1377@node Build instructions for FreeBSD 8
1378@subsection Build instructions for FreeBSD 8
1379
1380To get GNUnet 0.9 to compile on FreeBSD (at least FreeBSD 8.0):@ in order to
1381install the library @code{libiconv}, at first change the directory to your
1382ports directory, e.g.@
1383@code{@
1384 $ cd /usr/ports/@
1385}@
1386 following that, go to the install file of @code{libiconv} and install it,@
1387@code{@
1388 $ cd converters/libiconv,@
1389 $ make install@
1390}
1391
1392after that, change the directory to where you will check out
1393@code{libextractor} and GNUnet, and install latest @code{libextractor},@
1394 first of all, checkout @code{libextractor}, e.g.@
1395@code{@
1396 $ svn co https://gnunet.org/svn/Extractor@
1397}@
1398 then change the directory into which it was checked out, e.g.@
1399@code{@
1400 $ cd Extractor@
1401}@
1402 before the installation, you should do following steps,@
1403
1404@example
1405$ ./bootstrap@
1406$ ./configure --with-ltdl-include=/usr/local/include \
1407 --with-ltdl-lib=/usr/local/lib@
1408@end example
1409
1410if these steps complete successfully, you can install the library,@
1411
1412@example
1413$ make install@
1414@end example
1415
1416to check out the GNUnet, you should do the similar steps as
1417@code{libextractor}, firstly, change back to starting directory, e.g.@
1418@code{@
1419 $ cd ../@
1420}@
1421 Set the following environmental variables:@
1422@code{@
1423 export CPPFLAGS="-I/usr/local/include"@
1424 export LDFLAGS="-L/usr/local/lib"@
1425}@
1426 next, checkout GNUnet using@
1427@code{@
1428 $ svn co https://gnunet.org/svn/gnunet@
1429}@
1430 then change directory into newly checked out directory,@
1431@code{@
1432 $ cd gnunet@
1433}@
1434 at last, start to install GNUnet,@
1435
1436@example
1437 $ ./bootstrap@
1438 $ ./configure --with-ltdl-include=/usr/local/include \
1439 --with-ltdl-lib=/usr/local/lib --with-extractor=/usr/local
1440
1441## NOTE: you may not need the --with-extractor option!@
1442
1443$ make install
1444@end example
1445
1446@node Basic installation for Mac OS X
1447@subsection Basic installation for Mac OS X
1448
1449This documentation may be outdated!
1450
1451This page is providing guidelines for users trying to install GNUnet on Mac OS
1452X.@ Mainly users trying to install GNUnet by building source code are the most
1453welcome readers.@ The steps below are tested on an Intel Architecture running
1454Mac OS X Tiger (10.4.11). Ideally they should work on other Mac boxes with
1455different configurations as all the configuration done for it is dependent on
1456@uref{http://www.macports.org/, MacPorts}
1457
1458For having GNUnet installed successfully, some dependencies should be firstly
1459resolved:
1460
1461@itemize @bullet
1462
1463@item
1464Install/Update your @uref{http://developer.apple.com/tools/xcode/, Xcode}
1465version 3.2.1 or later for Snow Leopard, 3.1.4 or later for Leopard, or 2.5 for
1466Tiger.
1467
1468@item
1469Download and install @uref{http://www.macports.org/, MacPorts}.@
1470Now you are ready for installing GNunet dependencies.
1471
1472@item
1473First, you'd better make sure that: /opt/local/bin and /opt/local/sbin are
1474available in your PATH. (For doing so, open a terminal and type:@
1475
1476@example
1477$ echo $PATH
1478@end example
1479
1480and examine the output of it). If the paths are not available in your
1481environment, you have to add them (You can add them by editing your .profile
1482file in your home directory, append them to the PATH line). Then type:
1483@example
1484$ source ~/.profile
1485@end example
1486
1487and re-examine the echo command output.
1488
1489@item
1490Use MacPorts to download and install the dependencies:@
1491The libraries are:
1492
1493@itemize @bullet
1494
1495@item
1496@uref{http://trac.macports.org/browser/trunk/dports/www/libmicrohttpd/Portfile, libmicrohttpd.}
1497
1498@item
1499@uref{http://trac.macports.org/browser/trunk/dports/devel/libgcrypt/Portfile, libgcrypt.}
1500
1501@item
1502@uref{http://trac.macports.org/browser/trunk/dports/net/curl/Portfile, libcurl.}
1503
1504@item
1505@uref{http://trac.macports.org/browser/trunk/dports/devel/libtool/Portfile, libltdl.}
1506
1507@item
1508@uref{http://trac.macports.org/browser/trunk/dports/databases/sqlite3/Portfile, SQlite.}
1509
1510@item
1511libunistring
1512
1513@item
1514glpk
1515
1516@end itemize
1517
1518The port command is as follows:@
1519@example
1520port install libmicrohttpd libgcrypt curl libtool sqlite3 linunistring glpk
1521@end example
1522One of the dependencies, the libextractor, should be explicitly installed,
1523since the version available from macports is outdated to work with GNUnet. To
1524install the latest libextractor:
1525@itemize @bullet
1526
1527
1528@item
1529Install the Subversion Client:@
1530For more information about Subversion visit:
1531@uref{http://subversion.tigris.org/, http://subversion.tigris.org/}
1532
1533@example
1534# port install subversion
1535@end example
1536
1537
1538@item
1539Use Subversion to download the latest Extractor:
1540@example
1541$ svn checkout https://gnunet.org/svn/Extractor
1542@end example
1543
1544
1545@item
1546Go to the installation directory of the Extractor, compile and install it:
1547@example
1548$ ./bootstrap
1549$ export CPPFLAGS="-I/opt/local/include"
1550$ export LDFLAGS="-L/opt/local/lib"
1551$ ./configure --prefix=/opt/local
1552$ make
1553# make install
1554@end example
1555
1556@end itemize
1557
1558
1559@item
1560Now, your system is ready to install GNunet. If you downloaded GNUnet by
1561checking it out from svn, you should start by running the bootstrap script.
1562Open a terminal pointing to the GNUnet directory and type:@
1563
1564@example
1565$ ./bootstrap
1566@end example
1567
1568
1569@item
1570Run the configure script:
1571@example
1572$ export CPPFLAGS="-I/opt/local/include"
1573$ export LDFLAGS="-L/opt/local/lib"
1574$ ./configure --prefix=/tmp/gnunet_build
1575@end example
1576
1577
1578GNUnet will be installed in the directory /tmp/gnunet_build (Of course that
1579installation path can be changed).@ The CPPFLAGS and LDFLAGS are mentioned in
1580order to inform the compiler and the linker to lookup headers and libraries in
1581/opt/local/include and /opt/local/lib.
1582
1583@item
1584Compile@
1585
1586@example
1587$ make
1588@end example
1589
1590
1591@item
1592Install GNUnet
1593@example
1594# make install
1595@end example
1596
1597@end itemize
1598
1599@node Basic Installation for Fedora/PlanetLab nodes running Fedora 12
1600@subsection Basic Installation for Fedora/PlanetLab nodes running Fedora 12
1601
1602
1603@strong{This documentation is outdated and not valid for GNUnet 0.10.0!}@
1604
1605GNUnet installation on Fedora 8/Planetlab nodes can be done as following:
1606
16071. Install the build tools to build GNUnet@
1608@example
1609sudo yum -y -t --nogpgcheck install gcc make autoconf gettext-devel \
1610texinfo subversion@
1611@end example
1612
16132. Install the GNUnet dependencies@
1614@example
1615sudo yum -y -t --nogpgcheck install libunistring-devel libunistring-devel \
1616libgcrypt-devel zlib-devel sqlite-devel postgresql-devel mysql-devel \
1617libgsf-devel libvorbis-devel@
1618@end example
1619
16203. Install outdated dependencies from source@
1621libtool@
1622@example
1623wget http://ftp.gnu.org/gnu/libtool/libtool-2.4.2.tar.gz@
1624tar xvfz libtool-2.4.2.tar.gz@
1625cd libtool-2.4.2@
1626./configure@
1627sudo make install@
1628@end example
1629
1630glpk@
1631@example
1632wget http://ftp.gnu.org/gnu/glpk/glpk-4.47.tar.gz@
1633tar xvfz glpk-4.47.tar.gz@
1634cd glpk-4.47@
1635./configure@
1636sudo make install@
1637@end example
1638
1639libcurl@
1640@example
1641wget http://curl.haxx.se/download/curl-7.26.0.tar.gz@
1642tar xvfz curl-7.26.0.tar.gz@
1643cd curl-7.26.0@
1644./configure@
1645sudo make install@
1646@end example
1647
16484. Install libextractor@
1649@example
1650svn co https://gnunet.org/svn/libextractor@
1651cd libextractor@
1652libtoolize@
1653./bootstrap@
1654./configure@
1655sudo make install@
1656@end example
1657
16585. Install libmicrohttpd@
1659@example
1660svn co https://gnunet.org/svn/libmicrohttpd@
1661cd libmicrohttpd@
1662libtoolize@
1663./bootstrap@
1664./configure@
1665sudo make install@
1666@end example
1667
16686. Set GNUnet prefix and add to PATH@
1669@example
1670export GNUNET_PREFIX=@
1671export PATH=$PATH:$GNUNET_PREFIX/bin@
1672@end example
1673
16747. Install GNUnet from svn@
1675@example
1676export LD_LIBRARY_PATH=/usr/local/lib@
1677svn co https://gnunet.org/svn/gnunet@
1678cd gnunet@
1679libtoolize@
1680./bootstrap@
1681./configure --prefix=$GNUNET_PREFIX --with-extractor=/usr \
1682 --with-mysql=/usr/lib/mysql --enable-logging=verbose@
1683make install@
1684@end example
1685
1686Done!
1687
1688@node Basic Installation for Fedora/PlanetLab nodes running Fedora 8 .
1689@subsection Basic Installation for Fedora/PlanetLab nodes running Fedora 8 .
1690@c %**end of header
1691
1692@strong{This documentation is outdated and not valid for GNUnet 0.10.0!}@
1693 GNUnet installation on Fedora 8/Planetlab nodes can be done as following:
1694
16951. Install the build tools to build GNUnet@
1696@example
1697sudo yum -y -t --nogpgcheck install gcc make automake autoconf gettext-devel \
1698texinfo zlib-devel subversion@
1699@end example
1700
17012. Install the GNUnet dependencies@
1702@example
1703sudo yum -y -t --nogpgcheck install gnutls-devel gnutls-devel libgcrypt-devel \
1704sqlite-devel postgresql-devel mysql-devel libgsf-devel libvorbis-devel \
1705libidn-devel
1706@end example
1707
17083. Install outdated dependencies from source@
1709 libtool@
1710@code{@
1711 wget http://ftp.gnu.org/gnu/libtool/libtool-2.4.2.tar.gz@
1712 tar xvfz libtool-2.4.2.tar.gz@
1713 cd libtool-2.4.2@
1714 ./configure@
1715 sudo make install@
1716}
1717
1718libtool@
1719@code{@
1720 wget http://ftp.gnu.org/gnu/libtool/libtool-2.4.2.tar.gz@
1721 tar xvfz libtool-2.4.2.tar.gz@
1722 cd libtool-2.4.2@
1723 ./configure@
1724 sudo make install@
1725}
1726
1727glpk@
1728@code{@
1729 wget http://ftp.gnu.org/gnu/glpk/glpk-4.47.tar.gz@
1730 tar xvfz glpk-4.47.tar.gz@
1731 cd glpk-4.47@
1732 ./configure@
1733 sudo make install@
1734}
1735
1736libgpg-error@
1737@code{@
1738 wget ftp://ftp.gnupg.org/gcrypt/libgpg-error/libgpg-error-1.10.tar.bz2@
1739 tar xvfj libgpg-error-1.10.tar.bz2@
1740 cd libgpg-error-1.10@
1741 ./configure --prefix=/usr@
1742 sudo make install@
1743}
1744
1745libgcrypt@
1746@code{@
1747 wget ftp://ftp.gnupg.org/gcrypt/libgcrypt/libgcrypt-1.5.0.tar.bz2@
1748 tar xvfj libgcrypt-1.5.0.tar.tar.bz2@
1749 cd libgcrypt-1.5.0@
1750 ./configure --prefix=/usr@
1751 sudo make install@
1752}
1753
1754libcurl@
1755@code{@
1756 wget http://curl.haxx.se/download/curl-7.26.0.tar.gz@
1757 tar xvfz curl-7.26.0.tar.gz@
1758 cd curl-7.26.0@
1759 ./configure@
1760 sudo make install@
1761}
1762
1763libunistring@
1764@code{@
1765 wget http://ftp.gnu.org/gnu/libunistring/libunistring-0.9.3.tar.gz@
1766 tar xvfz libunistring-0.9.3.tar.gz@
1767 cd libunistring-0.9.3@
1768 ./configure@
1769 sudo make install@
1770}
1771
17724. Remove conflicting packages@
1773@code{@
1774 sudo rpm -e --nodeps libgcrypt libgpg-error@
1775}
1776
17774. Install libextractor@
1778@code{@
1779 wget ftp://ftp.gnu.org/gnu/libextractor/libextractor-0.6.3.tar.gz@
1780 tar xvfz libextractor-0.6.3.tar.gz@
1781 cd libextractor-0.6.3@
1782 ./configure@
1783 sudo make install@
1784}
1785
17865. Install libmicrohttpd and dependencies
1787
1788nettle@
1789@code{@
1790 wget http://ftp.gnu.org/gnu/nettle/nettle-2.5.tar.gz@
1791 tar xvfz nettle-2.5.tar.gz@
1792 cd nettle-2.5@
1793 ./configure@
1794 sudo make install@
1795}
1796
1797GnuTLS@
1798@code{@
1799 wget http://ftp.gnu.org/gnu/gnutls/gnutls-2.12.20.tar.bz2@
1800 tar xvfj gnutls-2.12.20.tar.bz2@
1801 cd gnutls-2.12.20@
1802 ./configure --without-p11-kit@
1803 sudo make install@
1804}
1805
1806libmicrohttpd@
1807@code{@
1808 wget ftp://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.21.tar.gz@
1809 tar xvfz libmicrohttpd-0.9.21.tar.gz@
1810 cd libmicrohttpd-0.9.21@
1811 ./configure@
1812 sudo make install@
1813}
1814
18156. Set GNUnet prefix and add to PATH@
1816@code{@
1817 export GNUNET_PREFIX=@
1818 export PATH=$PATH:$GNUNET_PREFIX/bin@
1819}
1820
18217. Install GNUnet from svn@
1822@example
1823 export LD_LIBRARY_PATH=/usr/local/lib@
1824 svn co https://gnunet.org/svn/gnunet@
1825 cd gnunet@
1826 libtoolize@
1827 ./bootstrap@
1828 ./configure --prefix=$GNUNET_PREFIX --with-extractor=/usr/local \
1829 --with-curl=/usr/local --with-mysql=/usr/lib/mysql --enable-logging=verbose@
1830 make install@
1831@end example
1832
1833Done!
1834
1835@node Build instructions for Gentoo
1836@subsection Build instructions for Gentoo
1837
1838
1839This page describes how to install GNUnet 0.9 on Gentoo.
1840
1841Since the GNUnet 0.9 ebuilds are not in the official portage tree yet, we need
1842to add them to the local portage overlay. All the commands below should be
1843executed as root.
1844
1845Specify your local portage directory in the /etc/make.conf, for example:@
1846@code{$ echo 'PORTDIR_OVERLAY="/usr/local/portage"' >> /etc/make.conf}
1847
1848Create directories for the ebuilds:@
1849@code{$ mkdir -p /usr/local/portage/media-libs/libextractor /usr/local/portage/net-p2p/gnunet/files}
1850
1851Download the latest ebuilds, init and config files from here and put them into
1852respective directories:@
1853@code{$ cp libextractor-0.6.2.ebuild /usr/local/portage/media-libs/libextractor@
1854 $ cp gnunet-0.9.2.ebuild /usr/local/portage/net-p2p/gnunet@
1855 $ cp gnunet-0.9.2.conf gnunet-0.9.2.confd gnunet-0.9.2.initd /usr/local/portage/net-p2p/gnunet/files}
1856
1857Generate Manifest files for the ebuilds:@
1858@code{$ cd /usr/local/portage/net-p2p/gnunet@
1859 $ ebuild gnunet-0.9.2.ebuild digest@
1860 $ cd /usr/local/portage/media-libs/libextractor@
1861 $ ebuild libextractor-0.6.2.ebuild digest}
1862
1863Unmask GNUnet and dependencies in the /etc/portage/package.keywords. For
1864example, if you use x86-64 architecture, add the following lines:@
1865@code{net-p2p/gnunet ~amd64@
1866 media-libs/libextractor ~amd64@
1867 net-libs/libmicrohttpd ~amd64@
1868 net-misc/curl ~amd64}
1869
1870Add either sqlite or mysql USE-flag in the /etc/portage/package.use:@
1871@code{net-p2p/gnunet sqlite}
1872
1873Now everything is ready to install GNUnet:@
1874@code{$ emerge -av gnunet}
1875
1876Use /etc/init.d/gnunet to start/stop GNUnet.
1877
1878@node Building GLPK for MinGW
1879@subsection Building GLPK for MinGW
1880
1881GNUnet now requires the GNU Linear Programming Kit (GLPK). Since there's is no
1882package you can install with @code{mingw-get} you have to compile it from
1883source:
1884
1885@itemize @bullet
1886
1887@item
1888Download the latest version from http://ftp.gnu.org/gnu/glpk/
1889
1890@item
1891Unzip it using your favourite unzipper@
1892In the MSYS shell:
1893
1894@item
1895change to the respective directory
1896
1897@item
1898@code{./configure '--build=i686-pc-mingw32'}
1899
1900@item
1901run @code{make install check }
1902
1903MinGW does not automatically detect the correct buildtype so you have to
1904specify it manually
1905@end itemize
1906
1907
1908@node GUI build instructions for Ubuntu 12.04 using Subversion
1909@subsection GUI build instructions for Ubuntu 12.04 using Subversion
1910
1911After installing GNUnet you can continue installing the GNUnet GUI tools:
1912
1913First, install the required dependencies:
1914
1915@code{@
1916 $ sudo apt-get install libgladeui-dev libqrencode-dev@
1917}
1918
1919Please ensure that the GNUnet shared libraries can be found by the linker. If
1920you installed GNUnet libraries in a non standard path (say
1921GNUNET_PREFIX=/usr/local/lib/), you can
1922@itemize @bullet
1923
1924
1925@item
1926set the environmental variable permanently to@
1927@code{LD_LIBRARY_PATH=$GNUNET_PREFIX}
1928
1929@item
1930or add @code{$GNUNET_PREFIX} to @code{/etc/ld.so.conf}
1931@end itemize
1932
1933
1934Now you can checkout and compile the GNUnet GUI tools@
1935@code{@
1936 $ svn co https://gnunet.org/svn/gnunet-gtk@
1937 $ cd gnunet-gtk@
1938 $ ./bootstrap@
1939 $ ./configure --prefix=$GNUNET_PREFIX/.. --with-gnunet=$GNUNET_PREFIX/..@
1940 $ make install@
1941}
1942
1943@node Installation with gnunet-update
1944@subsection Installation with gnunet-update
1945
1946gnunet-update project is an effort to introduce updates to GNUnet
1947installations. An interesting to-be-implemented-feature of gnunet-update is
1948that these updates are propagated through GNUnet's peer-to-peer network. More
1949information about gnunet-update can be found at
1950https://gnunet.org/svn/gnunet-update/README.
1951
1952While the project is still under development, we have implemented the following
1953features which we believe may be helpful for users and we would like them to be
1954tested:
1955
1956@itemize @bullet
1957
1958@item
1959Packaging GNUnet installation along with its run-time dependencies into update
1960packages
1961
1962@item
1963Installing update packages into compatible hosts
1964
1965@item
1966Updating an existing installation (which had been installed by gnunet-update)
1967to a newer one
1968@end itemize
1969
1970The above said features of gnunet-update are currently available for testing on
1971GNU/Linux systems.
1972
1973The following is a guide to help you get started with gnunet-update. It shows
1974you how to install the testing binary packages of GNUnet 0.9.1 we have at
1975https://gnunet.org/install/
1976
1977gnunet-update needs the following:
1978
1979@itemize @bullet
1980@item
1981python ( 2.6 or above)
1982
1983@item
1984gnupg
1985
1986@item
1987python-gpgme
1988@end itemize
1989
1990
1991Checkout gnunet-update:@
1992@code{@
1993 $ svn checkout -r24905 https://gnunet.org/svn/gnunet-update@
1994}
1995
1996For security reasons, all packages released for gnunet-update from us are
1997signed with the key at https://gnunet.org/install/key.txt You would need to
1998import this key into your gpg key ring. gnunet-update uses this key to verify
1999the integrity of the packages it installs@
2000@code{@
2001 $ gpg --recv-keys 7C613D78@
2002}
2003
2004Download the packages relevant to your architecture (currently I have access to
2005GNU/Linux machines on x86_64 and i686, so only two for now, hopefully more
2006later) from https://gnunet.org/install/.
2007
2008To install the downloaded package into the directory /foo:
2009
2010@code{@
2011 gnunet-update/bin/gnunet-update install downloaded/package /foo@
2012}
2013
2014The installer reports the directories into which shared libraries and
2015dependencies have been installed. You may need to add the reported shared
2016library installation paths to LD_LIBRARY_PATH before you start running any
2017installed binaries.
2018
2019Please report bugs at https://gnunet.org/bugs/ under the project
2020'gnunet-update'.
2021
2022@node Instructions for Microsoft Windows Platforms (Old)
2023@subsection Instructions for Microsoft Windows Platforms (Old)
2024
2025This document is a DEPRECATED installation guide for gnunet on windows. It will
2026not work for recent gnunet versions, but maybe it will be of some use if
2027problems arise.
2028
2029 The Windows build uses a UNIX emulator for Windows,
2030 @uref{http://www.mingw.org/, MinGW}, to build the executable modules. These
2031 modules run natively on Windows and do not require additional emulation
2032 software besides the usual dependencies.
2033
2034 GNUnet development is mostly done under Linux and especially SVN checkouts may
2035 not build out of the box. We regret any inconvenience, and if you have
2036 problems, please report them.
2037
2038
2039
2040@menu
2041* Hardware and OS requirements::
2042* Software installation::
2043* Building libextractor and GNUnet::
2044* Installer::
2045* Source::
2046@end menu
2047
2048@node Hardware and OS requirements
2049@subsubsection Hardware and OS requirements
2050
2051@itemize @bullet
2052
2053@item
2054Pentium II or equivalent processor, 350 MHz or better
2055
2056@item
2057128 MB RAM
2058
2059@item
2060600 MB free disk space
2061
2062@item
2063Windows 2000 or Windows XP are recommended
2064@end itemize
2065
2066@node Software installation
2067@subsubsection Software installation
2068
2069@itemize @bullet
2070
2071@item
2072@strong{Compression software}@
2073@
2074 The software packages GNUnet depends on are usually compressed using UNIX
2075 tools like tar, gzip and bzip2.@ If you do not already have an utility that is
2076 able to extract such archives, get @uref{http://www.7-zip.org/, 7-Zip}.
2077
2078@item
2079@strong{UNIX environment}@
2080@
2081The MinGW project provides the compiler toolchain that is used to build
2082GNUnet.@ Get the following packages from
2083@uref{http://sourceforge.net/projects/mingw/files/, the MinGW project}:
2084@itemize @bullet
2085
2086
2087@item
2088GCC core
2089
2090@item
2091GCC g++
2092
2093@item
2094MSYS
2095
2096@item
2097MSYS Developer Tool Kit (msysDTK)
2098
2099@item
2100MSYS Developer Tool Kit - msys-autoconf (bin)
2101
2102@item
2103MSYS Developer Tool Kit - msys-automake (bin)
2104
2105@item
2106MinGW Runtime
2107
2108@item
2109MinGW Utilities
2110
2111@item
2112Windows API
2113
2114@item
2115Binutils
2116
2117@item
2118make
2119
2120@item
2121pdcurses
2122
2123@item
2124GDB (snapshot)
2125@end itemize
2126
2127@itemize @bullet
2128
2129
2130@item
2131Install MSYS (to c:\mingw, for example.)@
2132Do @strong{not} use spaces in the pathname (c:\program files\mingw).
2133
2134@item
2135Install MinGW runtime, utilities and GCC to a subdirectory (to c:\mingw\mingw,
2136for example)
2137
2138@item
2139Install the Development Kit to the MSYS directory (c:\mingw)
2140
2141@item
2142Create a batch file bash.bat in your MSYS directory with the files:@
2143
2144@example
2145bin\sh.exe --login
2146@end example
2147
2148
2149This batch file opens a shell which is used to invoke the build processes..@
2150MinGW's standard shell (msys.bat) is not suitable because it opens a separate
2151console window@ On Vista, bash.bat needs to be run as administrator.
2152
2153@item
2154Start bash.sh and rename (c:\mingw\mingw\)lib\libstdc++.la to avoid problems:@
2155
2156@example
2157mv /usr/mingw/lib/libstdc++.la /usr/mingw/lib/libstdc++.la.broken
2158@end example
2159
2160
2161@item
2162Unpack the Windows API to the MinGW directory (c:\mingw\mingw\) and remove the
2163declaration of DATADIR from (c:\mingw\mingw\)include\objidl.h (lines 55-58)
2164
2165@item
2166Unpack autoconf, automake to the MSYS directory (c:\mingw)
2167
2168@item
2169Install all other packages to the MinGW directory (c:\mingw\mingw\)
2170@end itemize
2171
2172
2173@item
2174@strong{GNU Libtool}@
2175@
2176GNU Libtool is required to use shared libraries.@
2177@
2178Get the prebuilt package from here and unpack it to the MinGW directory
2179(c:\mingw)
2180
2181@item
2182@strong{Pthreads}@
2183@
2184GNUnet uses the portable POSIX thread library for multi-threading..@
2185
2186@itemize @bullet
2187
2188
2189@item
2190Save @uref{ftp://sources.redhat.com/pub/pthreads-win32/dll-latest/lib/x86/libpthreadGC2.a, libpthreadGC2.a} (x86) or @uref{ftp://sources.redhat.com/pub/pthreads-win32/dll-latest/lib/x64/libpthreadGC2.a, libpthreadGC2.a} (x64) as libpthread.a into the lib directory (c:\mingw\mingw\lib\libpthread.a)
2191
2192@item
2193Save @uref{ftp://sources.redhat.com/pub/pthreads-win32/dll-latest/lib/x86/pthreadGC2.dll, pthreadGC2.dll} (x86) or @uref{ftp://sources.redhat.com/pub/pthreads-win32/dll-latest/lib/x64/pthreadGC2.dll, libpthreadGC2.a} (x64) into the MinGW bin directory (c:\mingw\mingw\bin)
2194
2195@item
2196Download all header files from @uref{ftp://sources.redhat.com/pub/pthreads-win32/dll-latest/include/, include/} to the include directory (c:\mingw\mingw\include)
2197@end itemize
2198
2199
2200@item
2201@strong{GNU MP@
2202}@
2203@
2204GNUnet uses the GNU Multiple Precision library for special cryptographic operations.@
2205@
2206Get the GMP binary package from the @uref{http://sourceforge.net/projects/mingwrep/, MinGW repository} and unpack it to the MinGW directory (c:\mingw\mingw)
2207
2208@item
2209@strong{GNU Gettext}@
2210@
2211 GNU gettext is used to provide national language support.@
2212@
2213 Get the prebuilt package from hereand unpack it to the MinGW directory (c:\mingw\mingw)
2214
2215@item
2216@strong{GNU iconv}@
2217@
2218 GNU Libiconv is used for character encoding conversion.@
2219@
2220 Get the prebuilt package from here and unpack it to the MinGW directory (c:\mingw\mingw)
2221
2222@item
2223@strong{SQLite}@
2224@
2225 GNUnet uses the SQLite database to store data.@
2226@
2227 Get the prebuilt binary from here and unpack it to your MinGW directory.
2228
2229@item
2230@strong{MySQL}@
2231@
2232 As an alternative to SQLite, GNUnet also supports MySQL.
2233@itemize @bullet
2234
2235
2236@item
2237 Get the binary installer from the @uref{http://dev.mysql.com/downloads/mysql/4.1.html#Windows, MySQL project} (version 4.1),@
2238 install it and follow the instructions in README.mysql.
2239
2240@item
2241 Create a temporary build directory (c:\mysql)
2242
2243@item
2244 Copy the directories include\ and lib\ from the MySQL directory to the new directory
2245
2246@item
2247 Get the patches from @uref{http://bugs.mysql.com/bug.php?id=8906&files=1, Bug #8906} and @uref{http://bugs.mysql.com/bug.php?id=8872&files=1, Bug #8872} (the latter is only required for MySQL
2248@example
2249patch -p 0
2250@end example
2251
2252
2253@item
2254 Move lib\opt\libmysql.dll to lib\libmysql.dll
2255
2256@item
2257 Change to lib\ and create an import library:@
2258
2259@example
2260dlltool --input-def ../include/libmySQL.def --dllname libmysql.dll
2261 --output-lib libmysqlclient.a -k
2262@end example
2263
2264
2265@item
2266 Copy include\* to include\mysql\
2267
2268@item
2269 Pass "--with-mysql=/c/mysql" to ./configure and copy libmysql.dll to your PATH or GNUnet′s bin\ directory
2270@end itemize
2271
2272
2273@item
2274@strong{GTK+}@
2275@
2276 gnunet-gtk and libextractor depend on GTK.@
2277@
2278 Get the the binary and developer packages of atk, glib, gtk, iconv, gettext-runtime, pango from @uref{ftp://ftp.gtk.org/pub/gtk/v2.6/win32, gtk.org} and unpack it to the MinGW directory (c:\mingw\mingw)@
2279@
2280 Get @uref{http://www.gtk.org/download/win32.php, pkg-config} and libpng and unpack them to the MinGW directory (c:\mingw\mingw)@
2281@
2282 Here is an all-in-one package for @uref{http://ftp.gnome.org/pub/gnome/binaries/win32/gtk+/2.24/gtk+-bundle_2.24.10-20120208_win32.zip, gtk+dependencies}. Do not overwrite any existing files!
2283
2284@item
2285@strong{Glade}@
2286@
2287 gnunet-gtk and and gnunet-setup were created using this interface builder@
2288
2289@itemize @bullet
2290
2291
2292@item
2293 Get the Glade and libglade (-bin and -devel) packages (without GTK!) from @uref{http://gladewin32.sourceforge.net/, GladeWin32} and unpack it to the MinGW directory (c:\mingw\mingw)
2294
2295@item
2296 Get libxml from here and unpack it to the MinGW directory (c:\mingw\mingw).
2297@end itemize
2298
2299
2300@item
2301@strong{zLib}@
2302@
2303 libextractor requires zLib to decompress some file formats. GNUnet uses it to (de)compress meta-data.@
2304@
2305 Get zLib from here (Signature) and unpack it to the MinGW directory (c:\mingw\mingw)
2306
2307@item
2308@strong{Bzip2}@
2309@
2310 libextractor also requires Bzip2 to decompress some file formats.@
2311@
2312 Get Bzip2 (binary and developer package) from @uref{http://gnuwin32.sourceforge.net/packages/bzip2.htm, GnuWin32} and unpack it to the MinGW directory (c:\mingw\mingw)
2313
2314@item
2315@strong{Libgcrypt}@
2316@
2317 Libgcrypt provides the cryptographic functions used by GNUnet@
2318@
2319 Get Libgcrypt from @uref{ftp://ftp.gnupg.org/gcrypt/libgcrypt/, here}, compile and place it in the MinGW directory (c:\mingw\mingw). Currently you need at least version 1.4.2 to compile gnunet.
2320
2321@item
2322@strong{PlibC}@
2323@
2324 PlibC emulates Unix functions under Windows.@
2325@
2326 Get PlibC from here and unpack it to the MinGW directory (c:\mingw\mingw)
2327
2328@item
2329@strong{OGG Vorbis}@
2330@
2331 OGG Vorbis is used to extract meta-data from .ogg files@
2332@
2333 Get the packages @uref{http://www.gnunet.org/libextractor/download/win/libogg-1.1.4.zip, libogg} and @uref{http://www.gnunet.org/libextractor/download/win/libvorbis-1.2.3.zip, libvorbis} from the @uref{http://ftp.gnu.org/gnu/libextractor/libextractor-w32-1.0.0.zip, libextractor win32 build} and unpack them to the MinGW directory (c:\mingw\mingw)
2334
2335@item
2336@strong{Exiv2}@
2337@
2338 (lib)Exiv2 is used to extract meta-data from files with Exiv2 meta-data@
2339@
2340 Download @uref{http://www.gnunet.org/libextractor/download/win/exiv2-0.18.2.zip, Exiv2} and unpack it to the MSYS directory (c:\mingw)
2341@end itemize
2342
2343@node Building libextractor and GNUnet
2344@subsubsection Building libextractor and GNUnet
2345
2346Before you compile libextractor or GNUnet, be sure to set@
2347PKG_CONFIG_PATH:
2348@example
2349export PKG_CONFIG_PATH=/mingw/lib/pkgconfig
2350@end example
2351
2352
2353 See Installation for basic instructions on building libextractor and GNUnet.
2354
2355 By default, all modules that are created in this way contain debug information and are quite large.@
2356 To compile release versions (small and fast) set the variable CFLAGS:
2357@example
2358export CFLAGS='-O2 -march=pentium -fomit-frame-pointer'
2359./configure --prefix=$HOME --with-extractor=$HOME
2360@end example
2361
2362@node Installer
2363@subsubsection Installer
2364
2365 The GNUnet installer is made with @uref{http://nsis.sourceforge.net/, NSIS}@
2366 The installer script is located in contrib\win in the GNUnet source tree.
2367
2368@node Source
2369@subsubsection Source
2370
2371The sources of all dependencies are available here.
2372
2373@node Portable GNUnet
2374@section Portable GNUnet
2375
2376Quick instructions on how to use the most recent GNUnet on most GNU/Linux
2377distributions
2378
2379Currently this has only been tested on Ubuntu 12.04, 12.10, 13.04, Debian and
2380CentOS 6, but it should work on almost any GNU/Linux distribution. More
2381in-detail information can be found in the handbook.
2382
2383
2384
2385@menu
2386* Prerequisites::
2387* Download & set up gnunet-update::
2388* Install GNUnet::
2389@end menu
2390
2391@node Prerequisites
2392@subsection Prerequisites
2393
2394Open a terminal and paste this line into it to install all required tools
2395needed:@
2396@code{sudo apt-get install python-gpgme subversion}
2397
2398@node Download & set up gnunet-update
2399@subsection Download & set up gnunet-update
2400
2401The following command will download a working version of gnunet-update with the
2402subversion tool and import the public key which is needed for authentication:@
2403
2404@example
2405svn checkout -r24905 https://gnunet.org/svn/gnunet-update ~/gnunet-update &&
2406cd ~/gnunet-update
2407gpg --keyserver "hkp://keys.gnupg.net" --recv-keys 7C613D78
2408@end example
2409
2410@node Install GNUnet
2411@subsection Install GNUnet
2412
2413Download and install GNUnet binaries which can be found here and set library
2414paths:@
2415@code{@
2416 wget -P /tmp https://gnunet.org/install/packs/gnunet-0.9.4-`uname -m`.tgz@
2417 ./bin/gnunet-update install /tmp/gnunet-0.9*.tgz ~@
2418 echo "PATH DEFAULT=$@{PATH@}:$HOME/bin" >> ~/.pam_environment@
2419 echo -e "$@{HOME@}/lib\n$@{HOME@}/lib/gnunet-deps" | sudo tee /etc/ld.so.conf.d/gnunet.conf > /dev/null@
2420 sudo ldconfig@
2421}@
2422
2423You may need to re-login once after executing these last commands
2424
2425That's it, GNUnet is installed in your home directory now. GNUnet can be
2426configured and afterwards started by executing@
2427@code{gnunet-arm -s}
2428
2429@node The graphical configuration interface
2430@section The graphical configuration interface
2431
2432If you also would like to use gnunet-gtk and gnunet-setup (highly recommended
2433for beginners), do:
2434
2435@example
2436wget -P /tmp https://gnunet.org/install/packs/gnunet-0.9.4-gtk-0.9.4-`uname -m`.tgz@
2437sh ~/gnunet-update/bin/gnunet-update install /tmp/gnunet-*gtk*.tgz ~@
2438sudo ldconfig
2439@end example
2440Now you can run @code{gnunet-setup} for easy configuration of your GNUnet peer.
2441
2442
2443@menu
2444* Configuring your peer::
2445* Configuring the Friend-to-Friend (F2F) mode::
2446* Configuring the hostlist to bootstrap::
2447* Configuration of the HOSTLIST proxy settings::
2448* Configuring your peer to provide a hostlist ::
2449* Configuring the datastore::
2450* Configuring the MySQL database::
2451* Reasons for using MySQL::
2452* Reasons for not using MySQL::
2453* Setup Instructions::
2454* Testing::
2455* Performance Tuning::
2456* Setup for running Testcases::
2457* Configuring the Postgres database::
2458* Reasons to use Postgres::
2459* Reasons not to use Postgres::
2460* Manual setup instructions::
2461* Testing the setup manually::
2462* Configuring the datacache::
2463* Configuring the file-sharing service::
2464* Configuring logging::
2465* Configuring the transport service and plugins::
2466* Configuring the wlan transport plugin::
2467* Configuring HTTP(S) reverse proxy functionality using Apache or nginx::
2468* Blacklisting peers::
2469* Configuration of the HTTP and HTTPS transport plugins::
2470* Configuring the GNU Name System::
2471* Configuring the GNUnet VPN::
2472* Bandwidth Configuration::
2473* Configuring NAT::
2474* Peer configuration for distributions::
2475@end menu
2476
2477@node Configuring your peer
2478@subsection Configuring your peer
2479
2480This chapter will describe the various configuration options in GNUnet.
2481
2482The easiest way to configure your peer is to use the gnunet-setup tool.
2483gnunet-setup is part of the gnunet-gtk download. You might have to install it
2484separately.
2485
2486Many of the specific sections from this chapter actually are linked from within
2487gnunet-setup to help you while using the setup tool.
2488
2489While you can also configure your peer by editing the configuration file by
2490hand, this is not recommended for anyone except for developers.
2491
2492
2493
2494
2495
2496@node Configuring the Friend-to-Friend (F2F) mode
2497@subsection Configuring the Friend-to-Friend (F2F) mode
2498
2499GNUnet knows three basic modes of operation. In standard "peer-to-peer" mode,
2500your peer will connect to any peer. In the pure "friend-to-friend" mode, your
2501peer will ONLY connect to peers from a list of friends specified in the
2502configuration. Finally, in mixed mode, GNUnet will only connect to arbitrary
2503peers if it has at least a specified number of connections to friends.
2504
2505When configuring any of the F2F modes, you first need to create a file with the
2506peer identities of your friends. Ask your friends to run
2507
2508$ gnunet-peerinfo -sq
2509
2510The output of this command needs to be added to your friends file, which is
2511simply a plain text file with one line per friend with the output from the
2512above command.
2513
2514You then specify the location of your friends file in the "FRIENDS" option of
2515the "topology" section.
2516
2517Once you have created the friends file, you can tell GNUnet to only connect to
2518your friends by setting the "FRIENDS-ONLY" option (again in the "topology"
2519section) to YES.
2520
2521If you want to run in mixed-mode, set "FRIENDS-ONLY" to NO and configure a
2522minimum number of friends to have (before connecting to arbitrary peers) under
2523the "MINIMUM-FRIENDS" option.
2524
2525If you want to operate in normal P2P-only mode, simply set "MINIMUM-FRIENDS" to
2526zero and "FRIENDS_ONLY" to NO. This is the default.
2527
2528@node Configuring the hostlist to bootstrap
2529@subsection Configuring the hostlist to bootstrap
2530
2531After installing the software you need to get connected to the GNUnet network.
2532The configuration file included in your download is already configured to
2533connect you to the GNUnet network. In this section the relevant configuration
2534settings are explained.
2535
2536To get an initial connection to the GNUnet network and to get to know peers
2537already connected to the network you can use the so called bootstrap servers.
2538These servers can give you a list of peers connected to the network. To use
2539these bootstrap servers you have to configure the hostlist daemon to activate
2540bootstrapping.
2541
2542To activate bootstrapping edit your configuration file and edit the
2543@code{[hostlist]}-section. You have to set the argument "-b" in the options
2544line:
2545@example
2546[hostlist]
2547OPTIONS = -b
2548@end example
2549
2550Additionally you have to specify which server you want to use. The default
2551bootstrapping server is "@uref{http://v10.gnunet.org/hostlist,
2552http://v10.gnunet.org/hostlist}". [^] To set the server you have to edit the
2553line "SERVERS" in the hostlist section. To use the default server you should
2554set the lines to
2555@example
2556SERVERS = http://v10.gnunet.org/hostlist [^]
2557@end example
2558
2559
2560To use bootstrapping your configuration file should include these lines:
2561@example
2562[hostlist]
2563OPTIONS = -b
2564SERVERS = http://v10.gnunet.org/hostlist [^]
2565@end example
2566
2567
2568Besides using bootstrap servers you can configure your GNUnet peer to recieve
2569hostlist advertisements. Peers offering hostlists to other peers can send
2570advertisement messages to peers that connect to them. If you configure your
2571peer to receive these messages, your peer can download these lists and connect
2572to the peers included. These lists are persistent, which means that they are
2573saved to your hard disk regularly and are loaded during startup.
2574
2575To activate hostlist learning you have to add the "-e" switch to the OPTIONS
2576line in the hostlist section:
2577@example
2578[hostlist]
2579OPTIONS = -b -e
2580@end example
2581
2582
2583Furthermore you can specify in which file the lists are saved. To save the
2584lists in the file "hostlists.file" just add the line:
2585@example
2586HOSTLISTFILE = hostlists.file
2587@end example
2588
2589
2590Best practice is to activate both bootstrapping and hostlist learning. So your
2591configuration file should include these lines:
2592@example
2593[hostlist]
2594OPTIONS = -b -e
2595HTTPPORT = 8080
2596SERVERS = http://v10.gnunet.org/hostlist [^]
2597HOSTLISTFILE = $SERVICEHOME/hostlists.file
2598@end example
2599
2600@node Configuration of the HOSTLIST proxy settings
2601@subsection Configuration of the HOSTLIST proxy settings
2602
2603The hostlist client can be configured to use a proxy to connect to the hostlist
2604server. This functionality can be configured in the configuration file directly
2605or using the gnunet-setup tool.
2606
2607The hostlist client supports the following proxy types at the moment:
2608@itemize @bullet
2609
2610
2611@item
2612HTTP and HTTP 1.0 only proxy
2613
2614@item
2615SOCKS 4/4a/5/5 with hostname
2616@end itemize
2617
2618
2619In addition authentication at the proxy with username and password can be
2620configured.
2621
2622To configure proxy support for the hostlist client in the gnunet-setup tool,
2623select the "hostlist" tab and select the appropriate proxy type. The hostname
2624or IP address (including port if required) has to be entered in the "Proxy
2625hostname" textbox. If required, enter username and password in the "Proxy
2626username" and "Proxy password" boxes. Be aware that these information will be
2627stored in the configuration in plain text.
2628
2629To configure these options directly in the configuration, you can configure the
2630following settings in the @code{[hostlist]} section of the configuration:@
2631@example
2632 # Type of proxy server,@
2633 # Valid values: HTTP, HTTP_1_0, SOCKS4, SOCKS5, SOCKS4A, SOCKS5_HOSTNAME@
2634 # Default: HTTP@
2635 # PROXY_TYPE = HTTP
2636
2637# Hostname or IP of proxy server@
2638 # PROXY =@
2639 # User name for proxy server@
2640 # PROXY_USERNAME =@
2641 # User password for proxy server@
2642 # PROXY_PASSWORD =@
2643@end example
2644
2645@node Configuring your peer to provide a hostlist
2646@subsection Configuring your peer to provide a hostlist
2647
2648If you operate a peer permanently connected to GNUnet you can configure your
2649peer to act as a hostlist server, providing other peers the list of peers known
2650to him.
2651
2652Yor server can act as a bootstrap server and peers needing to obtain a list of
2653peers can contact him to download this list. To download this hostlist the peer
2654uses HTTP. For this reason you have to build your peer with libcurl and
2655microhttpd support. How you build your peer with this options can be found
2656here: https://gnunet.org/generic_installation
2657
2658To configure your peer to act as a bootstrap server you have to add the "-p"
2659option to OPTIONS in the [hostlist] section of your configuration file. Besides
2660that you have to specify a port number for the http server. In conclusion you
2661have to add the following lines:
2662
2663@example
2664[hostlist]
2665HTTPPORT = 12980
2666OPTIONS = -p
2667@end example
2668
2669
2670If your peer acts as a bootstrap server other peers should know about that. You
2671can advertise the hostlist your are providing to other peers. Peers connecting
2672to your peer will get a message containing an advertisement for your hostlist
2673and the URL where it can be downloaded. If this peer is in learning mode, it
2674will test the hostlist and, in the case it can obtain the list successfully, it
2675will save it for bootstrapping.
2676
2677To activate hostlist advertisement on your peer, you have to set the following
2678lines in your configuration file:
2679@example
2680[hostlist]
2681EXTERNAL_DNS_NAME = example.org
2682HTTPPORT = 12981
2683OPTIONS = -p -a
2684@end example
2685
2686
2687With this configuration your peer will a act as a bootstrap server and
2688advertise this hostlist to other peers connecting to him. The URL used to
2689download the list will be @code{@uref{http://example.org:12981/,
2690http://example.org:12981/}}.
2691
2692Please notice:
2693@itemize @bullet
2694
2695
2696@item
2697The hostlist is not human readable, so you should not try to download it using
2698your webbrowser. Just point your GNUnet peer to the address!
2699
2700@item
2701Advertising without providing a hostlist does not make sense and will not work.
2702@end itemize
2703
2704@node Configuring the datastore
2705@subsection Configuring the datastore
2706
2707The datastore is what GNUnet uses to for long-term storage of file-sharing
2708data. Note that long-term does not mean 'forever' since content does have an
2709expiration date, and of course storage space is finite (and hence sometimes
2710content may have to be discarded).
2711
2712Use the "QUOTA" option to specify how many bytes of storage space you are
2713willing to dedicate to GNUnet.
2714
2715In addition to specifying the maximum space GNUnet is allowed to use for the
2716datastore, you need to specify which database GNUnet should use to do so.
2717Currently, you have the choice between sqLite, MySQL and Postgres.
2718
2719@node Configuring the MySQL database
2720@subsection Configuring the MySQL database
2721
2722This section describes how to setup the MySQL database for GNUnet.
2723
2724Note that the mysql plugin does NOT work with mysql before 4.1 since we need
2725prepared statements. We are generally testing the code against MySQL 5.1 at
2726this point.
2727
2728@node Reasons for using MySQL
2729@subsection Reasons for using MySQL
2730
2731@itemize @bullet
2732
2733@item
2734On up-to-date hardware where mysql can be used comfortably, this module will
2735have better performance than the other database choices (according to our
2736tests).
2737
2738@item Its often possible to recover the mysql database from internal
2739inconsistencies. Some of the other databases do not support repair.
2740@end itemize
2741
2742@node Reasons for not using MySQL
2743@subsection Reasons for not using MySQL
2744
2745@itemize @bullet
2746
2747@item
2748Memory usage (likely not an issue if you have more than 1 GB)
2749
2750@item
2751Complex manual setup
2752@end itemize
2753
2754@node Setup Instructions
2755@subsection Setup Instructions
2756
2757@itemize @bullet
2758
2759@item
2760In @code{gnunet.conf} set in section "DATASTORE" the value for "DATABASE" to
2761"mysql".
2762
2763@item
2764Access mysql as root:@
2765
2766@example
2767$ mysql -u root -p
2768@end example
2769
2770
2771and issue the following commands, replacing $USER with the username@
2772 that will be running gnunet-arm (so typically "gnunet"):
2773@example
2774CREATE DATABASE gnunet;
2775GRANT select,insert,update,delete,create,alter,drop,create temporary tables
2776 ON gnunet.* TO $USER@@localhost;
2777SET PASSWORD FOR $USER@@localhost=PASSWORD('$the_password_you_like');
2778FLUSH PRIVILEGES;
2779@end example
2780
2781
2782@item
2783In the $HOME directory of $USER, create a ".my.cnf" file with the following lines@
2784
2785@example
2786[client]
2787user=$USER
2788password=$the_password_you_like
2789@end example
2790
2791@end itemize
2792
2793
2794 Thats it. Note that @code{.my.cnf} file is a slight security risk unless its
2795 on@ a safe partition. The $HOME/.my.cnf can of course be a symbolic@ link.
2796 Luckily $USER has only priviledges to mess up GNUnet's tables, which should be
2797 pretty harmless.
2798@node Testing
2799@subsection Testing
2800
2801You should briefly try if the database connection works. First, login as $USER.
2802Then use:
2803@example
2804$ mysql -u $USER
2805mysql> use gnunet;
2806@end example
2807
2808
2809If you get the message "Database changed" it probably works.
2810
2811If you get "ERROR 2002: Can't connect to local MySQL server@
2812 through socket '/tmp/mysql.sock' (2)" it may be resolvable by@
2813 "ln -s /var/run/mysqld/mysqld.sock /tmp/mysql.sock"@
2814 so there may be some additional trouble depending on your mysql setup.
2815@node Performance Tuning
2816@subsection Performance Tuning
2817
2818For GNUnet, you probably want to set the option
2819@example
2820innodb_flush_log_at_trx_commit = 0
2821@end example
2822
2823for a rather dramatic boost in MySQL performance. However, this reduces the
2824"safety" of your database as with this options you may loose transactions
2825during a power outage. While this is totally harmless for GNUnet, the option
2826applies to all applications using MySQL. So you should set it if (and only if)
2827GNUnet is the only application on your system using MySQL.
2828
2829@node Setup for running Testcases
2830@subsection Setup for running Testcases
2831
2832If you want to run the testcases, you must create a second database
2833"gnunetcheck" with the same username and password. This database will then be
2834used for testing ("make check").
2835
2836@node Configuring the Postgres database
2837@subsection Configuring the Postgres database
2838
2839This text describes how to setup the Postgres database for GNUnet.
2840
2841This Postgres plugin was developed for Postgres 8.3 but might work for earlier
2842versions as well.
2843
2844@node Reasons to use Postgres
2845@subsection Reasons to use Postgres
2846
2847@itemize @bullet
2848@item
2849Easier to setup than MySQL
2850@item
2851Real database
2852@end itemize
2853
2854@node Reasons not to use Postgres
2855@subsection Reasons not to use Postgres
2856
2857@itemize @bullet
2858@item
2859Quite slow
2860@item
2861Still some manual setup required
2862@end itemize
2863
2864@node Manual setup instructions
2865@subsection Manual setup instructions
2866
2867@itemize @bullet
2868
2869@item
2870In @code{gnunet.conf} set in section "DATASTORE" the value for@
2871"DATABASE" to "postgres".
2872@item
2873Access Postgres to create a user:@
2874
2875@table @asis
2876
2877@item with Postgres 8.x, use:
2878
2879@example
2880# su - postgres
2881$ createuser
2882@end example
2883
2884and enter the name of the user running GNUnet for the role interactively.
2885Then, when prompted, do not set it to superuser, allow the creation of
2886databases, and do not allow the creation of new roles.@
2887
2888@item with Postgres 9.x, use:
2889
2890@example
2891# su - postgres
2892$ createuser -d $GNUNET_USER
2893@end example
2894
2895
2896where $GNUNET_USER is the name of the user running GNUnet.@
2897
2898@end table
2899
2900
2901@item
2902As that user (so typically as user "gnunet"), create a database (or two):@
2903
2904@example
2905$ createdb gnunet
2906$ createdb gnunetcheck # this way you can run "make check"
2907@end example
2908
2909@end itemize
2910
2911
2912Now you should be able to start @code{gnunet-arm}.
2913
2914@node Testing the setup manually
2915@subsection Testing the setup manually
2916
2917You may want to try if the database connection works. First, again login as
2918the user who will run gnunet-arm. Then use,
2919@example
2920$ psql gnunet # or gnunetcheck
2921gnunet=> \dt
2922@end example
2923
2924
2925If, after you have started gnunet-arm at least once, you get a @code{gn090}
2926table here, it probably works.
2927
2928@node Configuring the datacache
2929@subsection Configuring the datacache
2930@c %**end of header
2931
2932The datacache is what GNUnet uses for storing temporary data. This data is
2933expected to be wiped completely each time GNUnet is restarted (or the system
2934is rebooted).
2935
2936You need to specify how many bytes GNUnet is allowed to use for the datacache
2937using the "QUOTA" option in the section "dhtcache". Furthermore, you need to
2938specify which database backend should be used to store the data. Currently,
2939you have the choice between sqLite, MySQL and Postgres.
2940
2941@node Configuring the file-sharing service
2942@subsection Configuring the file-sharing service
2943
2944In order to use GNUnet for file-sharing, you first need to make sure that the
2945file-sharing service is loaded. This is done by setting the AUTOSTART option in
2946section "fs" to "YES". Alternatively, you can run
2947@example
2948$ gnunet-arm -i fs
2949@end example
2950
2951to start the file-sharing service by hand.
2952
2953Except for configuring the database and the datacache the only important option
2954for file-sharing is content migration.
2955
2956Content migration allows your peer to cache content from other peers as well as
2957send out content stored on your system without explicit requests. This content
2958replication has positive and negative impacts on both system performance an
2959privacy.
2960
2961FIXME: discuss the trade-offs. Here is some older text about it...
2962
2963Setting this option to YES allows gnunetd to migrate data to the local machine.
2964Setting this option to YES is highly recommended for efficiency. Its also the
2965default. If you set this value to YES, GNUnet will store content on your
2966machine that you cannot decrypt. While this may protect you from liability if
2967the judge is sane, it may not (IANAL). If you put illegal content on your
2968machine yourself, setting this option to YES will probably increase your chances
2969to get away with it since you can plausibly deny that you inserted the content.
2970Note that in either case, your anonymity would have to be broken first (which
2971may be possible depending on the size of the GNUnet network and the strength of
2972the adversary).
2973
2974@node Configuring logging
2975@subsection Configuring logging
2976
2977Logging in GNUnet 0.9.0 is controlled via the "-L" and "-l" options.
2978Using "-L", a log level can be specified. With log level "ERROR" only serious
2979errors are logged. The default log level is "WARNING" which causes anything of
2980concern to be logged. Log level "INFO" can be used to log anything that might
2981be interesting information whereas "DEBUG" can be used by developers to log
2982debugging messages (but you need to run configure with
2983@code{--enable-logging=verbose} to get them compiled). The "-l" option is used
2984to specify the log file.
2985
2986Since most GNUnet services are managed by @code{gnunet-arm}, using the "-l" or
2987"-L" options directly is not possible. Instead, they can be specified using the
2988"OPTIONS" configuration value in the respective section for the respective
2989service. In order to enable logging globally without editing the "OPTIONS"
2990values for each service, @code{gnunet-arm} supports a "GLOBAL_POSTFIX" option.
2991The value specified here is given as an extra option to all services for which
2992the configuration does contain a service-specific "OPTIONS" field.
2993
2994"GLOBAL_POSTFIX" can contain the special sequence "@{@}" which is replaced by
2995the name of the service that is being started. Furthermore,
2996@code{GLOBAL_POSTFIX} is special in that sequences starting with "$" anywhere
2997in the string are expanded (according to options in "PATHS"); this expansion
2998otherwise is only happening for filenames and then the "$" must be the first
2999character in the option. Both of these restrictions do not apply to
3000"GLOBAL_POSTFIX". Note that specifying @code{%} anywhere in the "GLOBAL_POSTFIX"
3001disables both of these features.
3002
3003In summary, in order to get all services to log at level "INFO" to log-files
3004called @code{SERVICENAME-logs}, the following global prefix should be used:
3005@example
3006GLOBAL_POSTFIX = -l $SERVICEHOME/@{@}-logs -L INFO
3007@end example
3008
3009@node Configuring the transport service and plugins
3010@subsection Configuring the transport service and plugins
3011
3012The transport service in GNUnet is responsible to maintain basic connectivity
3013to other peers. Besides initiating and keeping connections alive it is also
3014responsible for address validation.
3015
3016The GNUnet transport supports more than one transport protocol. These protocols
3017are configured together with the transport service.
3018
3019The configuration section for the transport service itself is quite similar to
3020all the other services
3021
3022@code{@
3023 AUTOSTART = YES@
3024 @@UNIXONLY@@ PORT = 2091@
3025 HOSTNAME = localhost@
3026 HOME = $SERVICEHOME@
3027 CONFIG = $DEFAULTCONFIG@
3028 BINARY = gnunet-service-transport@
3029 #PREFIX = valgrind@
3030 NEIGHBOUR_LIMIT = 50@
3031 ACCEPT_FROM = 127.0.0.1;@
3032 ACCEPT_FROM6 = ::1;@
3033 PLUGINS = tcp udp@
3034 UNIXPATH = /tmp/gnunet-service-transport.sock@
3035}
3036
3037Different are the settings for the plugins to load @code{PLUGINS}. The first
3038setting specifies which transport plugins to load.
3039@itemize @bullet
3040
3041
3042@item
3043transport-unix
3044
3045A plugin for local only communication with UNIX domain sockets. Used for
3046testing and available on unix systems only. Just set the port
3047
3048@code{@
3049 [transport-unix]@
3050 PORT = 22086@
3051 TESTING_IGNORE_KEYS = ACCEPT_FROM;@
3052}
3053
3054@item
3055transport-tcp
3056
3057A plugin for communication with TCP. Set port to 0 for client mode with
3058outbound only connections
3059
3060@code{@
3061 [transport-tcp]@
3062 # Use 0 to ONLY advertise as a peer behind NAT (no port binding)@
3063 PORT = 2086@
3064 ADVERTISED_PORT = 2086@
3065 TESTING_IGNORE_KEYS = ACCEPT_FROM;@
3066 # Maximum number of open TCP connections allowed@
3067 MAX_CONNECTIONS = 128@
3068}
3069
3070@item
3071transport-udp
3072
3073A plugin for communication with UDP. Supports peer discovery using broadcasts.@
3074@code{@
3075 [transport-udp]@
3076 PORT = 2086@
3077 BROADCAST = YES@
3078 BROADCAST_INTERVAL = 30 s@
3079 MAX_BPS = 1000000@
3080 TESTING_IGNORE_KEYS = ACCEPT_FROM;@
3081}
3082
3083@item
3084transport-http
3085
3086HTTP and HTTPS support is split in two part: a client plugin initiating
3087outbound connections and a server part accepting connections from the client.
3088The client plugin just takes the maximum number of connections as an argument.@
3089@code{@
3090 [transport-http_client]@
3091 MAX_CONNECTIONS = 128@
3092 TESTING_IGNORE_KEYS = ACCEPT_FROM;@
3093}@
3094@code{@
3095 [transport-https_client]@
3096 MAX_CONNECTIONS = 128@
3097 TESTING_IGNORE_KEYS = ACCEPT_FROM;@
3098}
3099
3100The server has a port configured and the maximum nunber of connections.@
3101 The HTTPS part has two files with the certificate key and the certificate file.
3102
3103The server plugin supports reverse proxies, so a external hostname can be set
3104using@
3105the @code{EXTERNAL_HOSTNAME} setting. The webserver under this address should
3106forward the request to the peer and the configure port.
3107
3108@code{@
3109 [transport-http_server]@
3110 EXTERNAL_HOSTNAME = fulcrum.net.in.tum.de/gnunet@
3111 PORT = 1080@
3112 MAX_CONNECTIONS = 128@
3113 TESTING_IGNORE_KEYS = ACCEPT_FROM;@
3114}@
3115@code{@
3116 [transport-https_server]@
3117 PORT = 4433@
3118 CRYPTO_INIT = NORMAL@
3119 KEY_FILE = https.key@
3120 CERT_FILE = https.cert@
3121 MAX_CONNECTIONS = 128@
3122 TESTING_IGNORE_KEYS = ACCEPT_FROM;@
3123}
3124
3125@item
3126transport-wlan
3127
3128There is a special article how to setup the WLAN plugin, so here only the
3129settings. Just specify the interface to use:@
3130@code{@
3131 [transport-wlan]@
3132 # Name of the interface in monitor mode (typically monX)@
3133 INTERFACE = mon0@
3134 # Real hardware, no testing@
3135 TESTMODE = 0@
3136 TESTING_IGNORE_KEYS = ACCEPT_FROM;@
3137}
3138@end itemize
3139
3140@node Configuring the wlan transport plugin
3141@subsection Configuring the wlan transport plugin
3142
3143
3144The wlan transport plugin enables GNUnet to send and to receive data on a wlan
3145interface. It has not to be connected to a wlan network as long as sender and
3146receiver are on the same channel. This enables you to get connection to the
3147GNUnet where no internet access is possible, for example while catastrophes or
3148when censorship cuts you off the internet.
3149
3150
3151@menu
3152* Requirements for the WLAN plugin::
3153* Configuration::
3154* Before starting GNUnet::
3155* Limitations and known bugs::
3156@end menu
3157
3158
3159@node Requirements for the WLAN plugin
3160@subsubsection Requirements for the WLAN plugin
3161
3162@itemize @bullet
3163
3164@item
3165wlan network card with monitor support and packet injection
3166(see @uref{http://www.aircrack-ng.org/, aircrack-ng.org})
3167
3168@item
3169Linux kernel with mac80211 stack, introduced in 2.6.22, tested with 2.6.35
3170and 2.6.38
3171
3172@item
3173Wlantools to create the a monitor interface, tested with airmon-ng of the
3174aircrack-ng package
3175@end itemize
3176
3177@node Configuration
3178@subsubsection Configuration
3179
3180There are the following options for the wlan plugin (they should be like this
3181in your default config file, you only need to adjust them if the values are
3182incorrect for your system)@
3183@code{@
3184# section for the wlan transport plugin@
3185[transport-wlan]@
3186# interface to use, more information in the
3187# "Before starting GNUnet" section of the handbook.
3188INTERFACE = mon0@
3189# testmode for developers:@
3190# 0 use wlan interface,@
3191#1 or 2 use loopback driver for tests 1 = server, 2 = client@
3192TESTMODE = 0@
3193}
3194
3195@node Before starting GNUnet
3196@subsubsection Before starting GNUnet
3197
3198Before starting GNUnet, you have to make sure that your wlan interface is in
3199monitor mode. One way to put the wlan interface into monitor mode (if your
3200interface name is wlan0) is by executing:@
3201@code{@
3202 sudo airmon-ng start wlan0@
3203}
3204
3205Here is an example what the result should look like:@
3206@code{@
3207 Interface Chipset Driver@
3208 wlan0 Intel 4965 a/b/g/n iwl4965 - [phy0]@
3209 (monitor mode enabled on mon0)@
3210}@
3211The monitor interface is mon0 is the one that you have to put into the
3212configuration file.
3213
3214@node Limitations and known bugs
3215@subsubsection Limitations and known bugs
3216
3217Wlan speed is at the maximum of 1 Mbit/s because support for choosing the wlan
3218speed with packet injection was removed in newer kernels. Please pester the
3219kernel developers about fixing this.
3220
3221The interface channel depends on the wlan network that the card is connected
3222to. If no connection has been made since the start of the computer, it is
3223usually the first channel of the card. Peers will only find each other and
3224communicate if they are on the same channel. Channels must be set manually
3225(i.e. using @code{iwconfig wlan0 channel 1}).
3226
3227
3228@node Configuring HTTP(S) reverse proxy functionality using Apache or nginx
3229@subsection Configuring HTTP(S) reverse proxy functionality using Apache or nginx
3230
3231The HTTP plugin supports data transfer using reverse proxies. A reverse proxy
3232forwards the HTTP request he receives with a certain URL to another webserver,
3233here a GNUnet peer.
3234
3235So if you have a running Apache or nginx webserver you can configure it to be a
3236GNUnet reverse proxy. Especially if you have a well-known webiste this improves
3237censorship resistance since it looks as normal surfing behaviour.
3238
3239To do so, you have to do two things:
3240
3241@itemize @bullet
3242
3243@item
3244Configure your webserver to forward the GNUnet HTTP traffic
3245
3246@item
3247Configure your GNUnet peer to announce the respective address
3248@end itemize
3249
3250As an example we want to use GNUnet peer running:
3251
3252@itemize @bullet
3253
3254@item
3255HTTP server plugin on @code{gnunet.foo.org:1080}
3256
3257@item
3258HTTPS server plugin on @code{gnunet.foo.org:4433}
3259
3260@item
3261A apache or nginx webserver on @uref{http://www.foo.org/, http://www.foo.org:80/}
3262
3263@item
3264A apache or nginx webserver on https://www.foo.org:443/
3265@end itemize
3266
3267And we want the webserver to accept GNUnet traffic under
3268@code{http://www.foo.org/bar/}. The required steps are described here:
3269
3270@strong{Configure your Apache2 HTTP webserver}
3271
3272First of all you need mod_proxy installed.
3273
3274Edit your webserver configuration. Edit @code{/etc/apache2/apache2.conf} or
3275the site-specific configuration file.
3276
3277In the respective @code{server config},@code{virtual host} or
3278@code{directory} section add the following lines:@
3279@code{@
3280 ProxyTimeout 300@
3281 ProxyRequests Off@
3282 <Location /bar/ >@
3283 ProxyPass http://gnunet.foo.org:1080/@
3284 ProxyPassReverse http://gnunet.foo.org:1080/@
3285 </Location>@
3286}
3287
3288@strong{Configure your Apache2 HTTPS webserver}
3289
3290We assume that you already have an HTTPS server running, if not please check
3291how to configure a HTTPS host. An easy to use example is the
3292@file{apache2/sites-available/default-ssl} example configuration file.
3293
3294In the respective HTTPS @code{server config},@code{virtual host} or
3295@code{directory} section add the following lines:@
3296@code{@
3297 SSLProxyEngine On@
3298 ProxyTimeout 300@
3299 ProxyRequests Off@
3300 <Location /bar/ >@
3301 ProxyPass https://gnunet.foo.org:4433/@
3302 ProxyPassReverse https://gnunet.foo.org:4433/@
3303 </Location>@
3304}
3305
3306More information about the apache mod_proxy configuration can be found unter:@
3307@uref{http://httpd.apache.org/docs/2.2/mod/mod_proxy.html#proxypass, http://httpd.apache.org/docs/2.2/mod/mod_proxy.html#proxypass}
3308
3309@strong{Configure your nginx HTTPS webserver}
3310
3311Since nginx does not support chunked encoding, you first of all have to
3312install @code{chunkin}:@
3313@uref{http://wiki.nginx.org/HttpChunkinModule, http://wiki.nginx.org/HttpChunkinModule}
3314
3315To enable chunkin add:@
3316@code{@
3317 chunkin on;@
3318 error_page 411 = @@my_411_error;@
3319 location @@my_411_error @{@
3320 chunkin_resume;@
3321 @}@
3322}
3323
3324Edit your webserver configuration. Edit @code{/etc/nginx/nginx.conf} or the
3325site-specific configuration file.
3326
3327In the @code{server} section add:@
3328@code{@
3329 location /bar/@
3330 @{@
3331 proxy_pass http://gnunet.foo.org:1080/;@
3332 proxy_buffering off;@
3333 proxy_connect_timeout 5; # more than http_server@
3334 proxy_read_timeout 350; # 60 default, 300s is GNUnet's idle timeout@
3335 proxy_http_version 1.1; # 1.0 default@
3336 proxy_next_upstream error timeout invalid_header http_500 http_503 http_502 http_504;@
3337 @}@
3338@code{}}
3339
3340@strong{Configure your nginx HTTPS webserver}
3341
3342Edit your webserver configuration. Edit @code{/etc/nginx/nginx.conf} or the
3343site-specific configuration file.
3344
3345In the @code{server} section add:@
3346@code{@
3347 ssl_session_timeout 6m;@
3348 location /bar/@
3349 @{@
3350 proxy_pass https://gnunet.foo.org:4433/;@
3351 proxy_buffering off;@
3352 proxy_connect_timeout 5; # more than http_server@
3353 proxy_read_timeout 350; # 60 default, 300s is GNUnet's idle timeout@
3354 proxy_http_version 1.1; # 1.0 default@
3355 proxy_next_upstream error timeout invalid_header http_500 http_503 http_502 http_504;@
3356 @}@
3357@code{}}
3358
3359@strong{Configure your GNUnet peer}
3360
3361To have your GNUnet peer announce the address, you have to specify the
3362
3363@code{EXTERNAL_HOSTNAME} option in the @code{[transport-http_server]} section:@
3364@code{@
3365 [transport-http_server]@
3366 EXTERNAL_HOSTNAME = http://www.foo.org/bar/@
3367}@
3368 and/or@
3369@code{[transport-https_server]} section:@
3370@code{@
3371 [transport-https_server]@
3372 EXTERNAL_HOSTNAME = https://www.foo.org/bar/@
3373}
3374
3375Now restart your webserver and your peer...
3376
3377@node Blacklisting peers
3378@subsection Blacklisting peers
3379
3380Transport service supports to deny connecting to a specific peer of to a
3381specific peer with a specific transport plugin using te blacklisting component
3382of transport service. With@ blacklisting it is possible to deny connections to
3383specific peers of@ to use a specific plugin to a specific peer. Peers can be
3384blacklisted using@ the configuration or a blacklist client can be asked.
3385
3386To blacklist peers using the configuration you have to add a section to your@
3387configuration containing the peer id of the peer to blacklist and the plugin@
3388if required.
3389
3390Example:@
3391 To blacklist connections to P565... on peer AG2P... using tcp add:@
3392@code{@
3393 [transport-blacklist AG2PHES1BARB9IJCPAMJTFPVJ5V3A72S3F2A8SBUB8DAQ2V0O3V8G6G2JU56FHGFOHMQVKBSQFV98TCGTC3RJ1NINP82G0RC00N1520]@
3394 P565723JO1C2HSN6J29TAQ22MN6CI8HTMUU55T0FUQG4CMDGGEQ8UCNBKUMB94GC8R9G4FB2SF9LDOBAJ6AMINBP4JHHDD6L7VD801G = tcp@
3395}@
3396 To blacklist connections to P565... on peer AG2P... using all plugins add:@
3397@code{@
3398 [transport-blacklist-AG2PHES1BARB9IJCPAMJTFPVJ5V3A72S3F2A8SBUB8DAQ2V0O3V8G6G2JU56FHGFOHMQVKBSQFV98TCGTC3RJ1NINP82G0RC00N1520]@
3399 P565723JO1C2HSN6J29TAQ22MN6CI8HTMUU55T0FUQG4CMDGGEQ8UCNBKUMB94GC8R9G4FB2SF9LDOBAJ6AMINBP4JHHDD6L7VD801G =@
3400}
3401
3402You can also add a blacklist client usign the blacklist api. On a blacklist@
3403check, blacklisting first checks internally if the peer is blacklisted and@
3404if not, it asks the blacklisting clients. Clients are asked if it is OK to@
3405connect to a peer ID, the plugin is omitted.
3406
3407On blacklist check for (peer, plugin)
3408@itemize @bullet
3409@item Do we have a local blacklist entry for this peer and this plugin?@
3410@item YES: disallow connection@
3411@item Do we have a local blacklist entry for this peer and all plugins?@
3412@item YES: disallow connection@
3413@item Does one of the clients disallow?@
3414@item YES: disallow connection
3415@end itemize
3416
3417@node Configuration of the HTTP and HTTPS transport plugins
3418@subsection Configuration of the HTTP and HTTPS transport plugins
3419
3420The client part of the http and https transport plugins can be configured to
3421use a proxy to connect to the hostlist server. This functionality can be
3422configured in the configuration file directly or using the gnunet-setup tool.
3423
3424The both the HTTP and HTTPS clients support the following proxy types at the
3425moment:
3426
3427@itemize @bullet
3428@item HTTP 1.1 proxy
3429@item SOCKS 4/4a/5/5 with hostname
3430@end itemize
3431
3432In addition authentication at the proxy with username and password can be
3433configured.
3434
3435To configure proxy support for the clients in the gnunet-setup tool, select the
3436"transport" tab and activate the respective plugin. Now you can select the
3437appropriate proxy type. The hostname or IP address (including port if required)
3438has to be entered in the "Proxy hostname" textbox. If required, enter username
3439and password in the "Proxy username" and "Proxy password" boxes. Be aware that
3440these information will be stored in the configuration in plain text.
3441
3442To configure these options directly in the configuration, you can configure the
3443following settings in the [transport-http_client] and [transport-https_client]
3444section of the configuration:
3445
3446@example
3447# Type of proxy server,@
3448# Valid values: HTTP, SOCKS4, SOCKS5, SOCKS4A, SOCKS5_HOSTNAME@
3449# Default: HTTP@
3450# PROXY_TYPE = HTTP
3451
3452# Hostname or IP of proxy server@
3453# PROXY =@
3454# User name for proxy server@
3455# PROXY_USERNAME =@
3456# User password for proxy server@
3457# PROXY_PASSWORD =
3458@end example
3459
3460@node Configuring the GNU Name System
3461@subsection Configuring the GNU Name System
3462
3463@menu
3464* Configuring system-wide DNS interception::
3465* Configuring the GNS nsswitch plugin::
3466* Configuring GNS on W32::
3467* GNS Proxy Setup::
3468* Setup of the GNS CA::
3469* Testing the GNS setup::
3470* Automatic Shortening in the GNU Name System::
3471@end menu
3472
3473
3474@node Configuring system-wide DNS interception
3475@subsubsection Configuring system-wide DNS interception
3476
3477Before you install GNUnet, make sure you have a user and group 'gnunet' as well
3478as an empty group 'gnunetdns'.
3479
3480When using GNUnet with system-wide DNS interception, it is absolutely necessary
3481for all GNUnet service processes to be started by @code{gnunet-service-arm} as
3482user and group 'gnunet'. You also need to be sure to run @code{make install} as
3483root (or use the @code{sudo} option to configure) to grant GNUnet sufficient
3484privileges.
3485
3486With this setup, all that is required for enabling system-wide DNS interception
3487is for some GNUnet component (VPN or GNS) to request it. The
3488@code{gnunet-service-dns} will then start helper programs that will make the
3489necessary changes to your firewall (@code{iptables}) rules.
3490
3491Note that this will NOT work if your system sends out DNS traffic to a
3492link-local IPv6 address, as in this case GNUnet can intercept the traffic, but
3493not inject the responses from the link-local IPv6 address. Hence you cannot use
3494system-wide DNS interception in conjunction with link-local IPv6-based DNS
3495servers. If such a DNS server is used, it will bypass GNUnet's DNS traffic
3496interception.
3497
3498
3499
3500Using the GNU Name System (GNS) requires two different configuration steps.
3501First of all, GNS needs to be integrated with the operating system. Most of
3502this section is about the operating system level integration.
3503
3504Additionally, each individual user who wants to use the system must also
3505initialize his GNS zones. This can be done by running (after starting GNUnet)@
3506@code{@
3507 $ gnunet-gns-import.sh@
3508}@
3509after the local GNUnet peer has been started. Note that the namestore (in
3510particular the namestore database backend) should not be reconfigured
3511afterwards (as records are not automatically migrated between backends).
3512
3513The remainder of this chapter will detail the various methods for configuring
3514the use of GNS with your operating system.
3515
3516At this point in time you have different options depending on your OS:
3517@table @asis
3518
3519@item Use the gnunet-gns-proxy This approach works for all operating systems
3520and is likely the easiest. However, it enables GNS only for browsers, not for
3521other applications that might be using DNS, such as SSH. Still, using the proxy
3522is required for using HTTP with GNS and is thus recommended for all users. To
3523do this, you simply have to run the @code{gnunet-gns-proxy-setup-ca} script as
3524the user who will run the browser (this will create a GNS certificate authority
3525(CA) on your system and import its key into your browser), then start
3526@code{gnunet-gns-proxy} and inform your browser to use the Socks5 proxy which
3527@code{gnunet-gns-proxy} makes available by default on port 7777.
3528@item Use a
3529nsswitch plugin (recommended on GNU systems) This approach has the advantage of
3530offering fully personalized resolution even on multi-user systems. A potential
3531disadvantage is that some applications might be able to bypass GNS.
3532@item Use
3533a W32 resolver plugin (recommended on W32) This is currently the only option on
3534W32 systems.
3535@item Use system-wide DNS packet interception This approach is
3536recommended for the GNUnet VPN. It can be used to handle GNS at the same time;
3537however, if you only use this method, you will only get one root zone per
3538machine (not so great for multi-user systems).
3539@end table
3540
3541
3542You can combine system-wide DNS packet interception with the nsswitch plugin.@
3543The setup of the system-wide DNS interception is described here. All of the
3544other GNS-specific configuration steps are described in the following sections.
3545
3546@node Configuring the GNS nsswitch plugin
3547@subsubsection Configuring the GNS nsswitch plugin
3548
3549The Name Service Switch (NSS) is a facility in Unix-like operating systems that
3550provides a variety of sources for common configuration databases and name
3551resolution mechanisms. A system administrator usually configures the operating
3552system's name services using the file /etc/nsswitch.conf.
3553
3554GNS provides a NSS plugin to integrate GNS name resolution with the operating
3555system's name resolution process. To use the GNS NSS plugin you have to either
3556
3557@itemize @bullet
3558
3559@item
3560install GNUnet as root or
3561
3562@item
3563compile GNUnet with the @code{--with-sudo=yes} switch.
3564@end itemize
3565
3566Name resolution is controlled by the @emph{hosts} section in the NSS
3567configuration. By default this section first performs a lookup in the
3568/etc/hosts file and then in DNS. The nsswitch file should contain a line
3569similar to:@
3570@code{@
3571 hosts: files dns [NOTFOUND=return] mdns4_minimal mdns4@
3572}
3573
3574Here the GNS NSS plugin can be added to perform a GNS lookup before performing
3575a DNS lookup. The GNS NSS plugin has to be added to the "hosts" section in
3576/etc/nsswitch.conf file before DNS related plugins:@
3577@code{@
3578 ...@
3579 hosts: files gns [NOTFOUND=return] dns mdns4_minimal mdns4@
3580 ...@
3581}
3582
3583The @code{NOTFOUND=return} will ensure that if a @code{.gnu} name is not found
3584in GNS it will not be queried in DNS.
3585
3586@node Configuring GNS on W32
3587@subsubsection Configuring GNS on W32
3588
3589This document is a guide to configuring GNU Name System on W32-compatible
3590platforms.
3591
3592After GNUnet is installed, run the w32nsp-install tool:
3593@example
3594w32nsp-install.exe libw32nsp-0.dll
3595@end example
3596
3597
3598 ('0' is the library version of W32 NSP; it might increase in the future,
3599 change the invocation accordingly).
3600
3601This will install GNS namespace provider into the system and allow other
3602applications to resolve names that end in '@strong{gnu}' and '@strong{zkey}'.
3603Note that namespace provider requires gnunet-gns-helper-service-w32 to be
3604running, as well as gns service itself (and its usual dependencies).
3605
3606Namespace provider is hardcoded to connect to @strong{127.0.0.1:5353}, and this
3607is where gnunet-gns-helper-service-w32 should be listening to (and is
3608configured to listen to by default).
3609
3610To uninstall the provider, run:
3611@example
3612w32nsp-uninstall.exe
3613@end example
3614
3615
3616(uses provider GUID to uninstall it, does not need a dll name).
3617
3618Note that while MSDN claims that other applications will only be able to use
3619the new namespace provider after re-starting, in reality they might stat to use
3620it without that. Conversely, they might stop using the provider after it's been
3621uninstalled, even if they were not re-started. W32 will not permit namespace
3622provider library to be deleted or overwritten while the provider is installed,
3623and while there is at least one process still using it (even after it was
3624uninstalled).
3625
3626@node GNS Proxy Setup
3627@subsubsection GNS Proxy Setup
3628
3629When using the GNU Name System (GNS) to browse the WWW, there are several
3630issues that can be solved by adding the GNS Proxy to your setup:
3631@itemize @bullet
3632
3633
3634@item If the target website does not support GNS, it might assume that it is
3635operating under some name in the legacy DNS system (such as example.com). It
3636may then attempt to set cookies for that domain, and the web server might
3637expect a @code{Host: example.com} header in the request from your browser.
3638However, your browser might be using @code{example.gnu} for the @code{Host}
3639header and might only accept (and send) cookies for @code{example.gnu}. The GNS
3640Proxy will perform the necessary translations of the hostnames for cookies and
3641HTTP headers (using the LEHO record for the target domain as the desired
3642substitute).
3643
3644@item If using HTTPS, the target site might include an SSL certificate which is
3645either only valid for the LEHO domain or might match a TLSA record in GNS.
3646However, your browser would expect a valid certificate for @code{example.gnu},
3647not for some legacy domain name. The proxy will validate the certificate
3648(either against LEHO or TLSA) and then on-the-fly produce a valid certificate
3649for the exchange, signed by your own CA. Assuming you installed the CA of your
3650proxy in your browser's certificate authority list, your browser will then
3651trust the HTTPS/SSL/TLS connection, as the hostname mismatch is hidden by the
3652proxy.
3653
3654@item Finally, the proxy will in the future indicate to the server that it
3655speaks GNS, which will enable server operators to deliver GNS-enabled web sites
3656to your browser (and continue to deliver legacy links to legacy browsers)
3657@end itemize
3658
3659@node Setup of the GNS CA
3660@subsubsection Setup of the GNS CA
3661
3662First you need to create a CA certificate that the proxy can use. To do so use
3663the provided script gnunet-gns-proxy-ca:@
3664@code{@
3665 $ gnunet-gns-proxy-setup-ca@
3666}
3667
3668This will create a personal certification authority for you and add this
3669authority to the firefox and chrome database. The proxy will use the this CA
3670certificate to generate @code{*.gnu} client certificates on the fly.
3671
3672Note that the proxy uses libcurl. Make sure your version of libcurl uses GnuTLS
3673and NOT OpenSSL. The proxy will not work with libcurl compiled against
3674OpenSSL.
3675
3676@node Testing the GNS setup
3677@subsubsection Testing the GNS setup
3678
3679Now for testing purposes we can create some records in our zone to test the SSL
3680functionality of the proxy:@
3681@code{@
3682 $ gnunet-namestore -a -e "1 d" -n "homepage" -t A -V 131.159.74.67@
3683 $ gnunet-namestore -a -e "1 d" -n "homepage" -t LEHO -V "gnunet.org"@
3684}
3685
3686At this point we can start the proxy. Simply execute@
3687@code{@
3688 $ gnunet-gns-proxy@
3689}
3690
3691Configure your browser to use this SOCKSv5 proxy on port 7777 and visit this
3692link.@ If you use firefox you also have to go to about:config and set the key
3693@code{network.proxy.socks_remote_dns} to @code{true}.
3694
3695When you visit @code{https://homepage.gnu/}, you should get to the
3696@code{https://gnunet.org/} frontpage and the browser (with the correctly
3697configured proxy) should give you a valid SSL certificate for
3698@code{homepage.gnu} and no warnings. It should look like this@
3699
3700
3701
3702@table @asis
3703@item Attachment
3704Size
3705@item gnunethpgns.png
370664.19 KB
3707@end table
3708
3709@node Automatic Shortening in the GNU Name System
3710@subsubsection Automatic Shortening in the GNU Name System
3711
3712This page describes a possible option for 'automatic name shortening', which
3713you can choose to enable with the GNU Name System.
3714
3715When GNS encounters a name for the first time, it can use the 'NICK' record of
3716the originating zone to automatically generate a name for the zone. If
3717automatic shortening is enabled, those auto-generated names will be placed (as
3718private records) into your personal 'shorten' zone (to prevent confusion with
3719manually selected names). Then, in the future, if the same name is encountered
3720again, GNS will display the shortened name instead (the first time, the long
3721name will still be used as shortening typically happens asynchronously as
3722looking up the 'NICK' record takes some time). Using this feature can be a
3723convenient way to avoid very long @code{.gnu} names; however, note that names
3724from the shorten-zone are assigned on a first-come-first-serve basis and should
3725not be trusted. Furthermore, if you enable this feature, you will no longer see
3726the full delegation chain for zones once shortening has been applied.
3727
3728@node Configuring the GNUnet VPN
3729@subsection Configuring the GNUnet VPN
3730
3731@menu
3732* IPv4 address for interface::
3733* IPv6 address for interface::
3734* Configuring the GNUnet VPN DNS::
3735* Configuring the GNUnet VPN Exit Service::
3736* IP Address of external DNS resolver::
3737* IPv4 address for Exit interface::
3738* IPv6 address for Exit interface::
3739@end menu
3740
3741Before configuring the GNUnet VPN, please make sure that system-wide DNS
3742interception is configured properly as described in the section on the GNUnet
3743DNS setup.
3744
3745The default-options for the GNUnet VPN are usually sufficient to use GNUnet as
3746a Layer 2 for your Internet connection. However, what you always have to
3747specify is which IP protocol you want to tunnel: IPv4, IPv6 or both.
3748Furthermore, if you tunnel both, you most likely should also tunnel all of your
3749DNS requests. You theoretically can tunnel "only" your DNS traffic, but that
3750usually makes little sense.
3751
3752The other options as shown on the gnunet-setup tool are:
3753
3754@node IPv4 address for interface
3755@subsubsection IPv4 address for interface
3756
3757This is the IPv4 address the VPN interface will get. You should pick an
3758'private' IPv4 network that is not yet in use for you system. For example, if
3759you use 10.0.0.1/255.255.0.0 already, you might use 10.1.0.1/255.255.0.0. If
3760you use 10.0.0.1/255.0.0.0 already, then you might use 192.168.0.1/255.255.0.0.
3761If your system is not in a private IP-network, using any of the above will work
3762fine.@ You should try to make the mask of the address big enough (255.255.0.0
3763or, even better, 255.0.0.0) to allow more mappings of remote IP Addresses into
3764this range. However, even a 255.255.255.0-mask will suffice for most users.
3765
3766@node IPv6 address for interface
3767@subsubsection IPv6 address for interface
3768
3769The IPv6 address the VPN interface will get. Here you can specify any
3770non-link-local address (the address should not begin with "fe80:"). A subnet
3771Unique Local Unicast (fd00::/8-prefix) that you are currently not using would
3772be a good choice.
3773
3774@node Configuring the GNUnet VPN DNS
3775@subsubsection Configuring the GNUnet VPN DNS
3776
3777To resolve names for remote nodes, activate the DNS exit option.
3778
3779@node Configuring the GNUnet VPN Exit Service
3780@subsubsection Configuring the GNUnet VPN Exit Service
3781
3782If you want to allow other users to share your Internet connection (yes, this
3783may be dangerous, just as running a Tor exit node) or want to provide access to
3784services on your host (this should be less dangerous, as long as those services
3785are secure), you have to enable the GNUnet exit daemon.
3786
3787You then get to specify which exit functions you want to provide. By enabling
3788the exit daemon, you will always automatically provide exit functions for
3789manually configured local services (this component of the system is under
3790development and not documented further at this time). As for those services you
3791explicitly specify the target IP address and port, there is no significant
3792security risk in doing so.
3793
3794Furthermore, you can serve as a DNS, IPv4 or IPv6 exit to the Internet. Being a
3795DNS exit is usually pretty harmless. However, enabling IPv4 or IPv6-exit
3796without further precautions may enable adversaries to access your local
3797network, send spam, attack other systems from your Internet connection and to
3798other mischief that will appear to come from your machine. This may or may not
3799get you into legal trouble. If you want to allow IPv4 or IPv6-exit
3800functionality, you should strongly consider adding additional firewall rules
3801manually to protect your local network and to restrict outgoing TCP traffic
3802(i.e. by not allowing access to port 25). While we plan to improve
3803exit-filtering in the future, you're currently on your own here. Essentially,
3804be prepared for any kind of IP-traffic to exit the respective TUN interface
3805(and GNUnet will enable IP-forwarding and NAT for the interface automatically).
3806
3807Additional configuration options of the exit as shown by the gnunet-setup tool
3808are:
3809
3810@node IP Address of external DNS resolver
3811@subsubsection IP Address of external DNS resolver
3812
3813If DNS traffic is to exit your machine, it will be send to this DNS resolver.
3814You can specify an IPv4 or IPv6 address.
3815
3816@node IPv4 address for Exit interface
3817@subsubsection IPv4 address for Exit interface
3818
3819This is the IPv4 address the Interface will get. Make the mask of the address
3820big enough (255.255.0.0 or, even better, 255.0.0.0) to allow more mappings of
3821IP addresses into this range. As for the VPN interface, any unused, private
3822IPv4 address range will do.
3823
3824@node IPv6 address for Exit interface
3825@subsubsection IPv6 address for Exit interface
3826
3827The public IPv6 address the interface will get. If your kernel is not a very
3828recent kernel and you are willing to manually enable IPv6-NAT, the IPv6 address
3829you specify here must be a globally routed IPv6 address of your host.
3830
3831Suppose your host has the address @code{2001:4ca0::1234/64}, then using@
3832@code{2001:4ca0::1:0/112} would be fine (keep the first 64 bits, then change at
3833least one bit in the range before the bitmask, in the example above we changed
3834bit 111 from 0 to 1).
3835
3836You may also have to configure your router to route traffic for the entire
3837subnet (@code{2001:4ca0::1:0/112} for example) through your computer (this
3838should be automatic with IPv6, but obviously anything can be
3839disabled).
3840
3841@node Bandwidth Configuration
3842@subsection Bandwidth Configuration
3843
3844You can specify how many bandwidth GNUnet is allowed to use to receive and send
3845data. This is important for users with limited bandwidth or traffic volume.
3846
3847@node Configuring NAT
3848@subsection Configuring NAT
3849
3850Most hosts today do not have a normal global IP address but instead are behind
3851a router performing Network Address Translation (NAT) which assigns each host
3852in the local network a private IP address. As a result, these machines cannot
3853trivially receive inbound connections from the Internet. GNUnet supports NAT
3854traversal to enable these machines to receive incoming connections from other
3855peers despite their limitations.
3856
3857In an ideal world, you can press the "Attempt automatic configuration" button
3858in gnunet-setup to automatically configure your peer correctly. Alternatively,
3859your distribution might have already triggered this automatic configuration
3860during the installation process. However, automatic configuration can fail to
3861determine the optimal settings, resulting in your peer either not receiving as
3862many connections as possible, or in the worst case it not connecting to the
3863network at all.
3864
3865To manually configure the peer, you need to know a few things about your
3866network setup. First, determine if you are behind a NAT in the first place.
3867This is always the case if your IP address starts with "10.*" or "192.168.*".
3868Next, if you have control over your NAT router, you may choose to manually
3869configure it to allow GNUnet traffic to your host. If you have configured your
3870NAT to forward traffic on ports 2086 (and possibly 1080) to your host, you can
3871check the "NAT ports have been opened manually" option, which corresponds to
3872the "PUNCHED_NAT" option in the configuration file. If you did not punch your
3873NAT box, it may still be configured to support UPnP, which allows GNUnet to
3874automatically configure it. In that case, you need to install the "upnpc"
3875command, enable UPnP (or PMP) on your NAT box and set the "Enable NAT traversal
3876via UPnP or PMP" option (corresponding to "ENABLE_UPNP" in the configuration
3877file).
3878
3879Some NAT boxes can be traversed using the autonomous NAT traversal method. This
3880requires certain GNUnet components to be installed with "SUID" prividledges on
3881your system (so if you're installing on a system you do not have administrative
3882rights to, this will not work). If you installed as 'root', you can enable
3883autonomous NAT traversal by checking the "Enable NAT traversal using ICMP
3884method". The ICMP method requires a way to determine your NAT's external
3885(global) IP address. This can be done using either UPnP, DynDNS, or by manual
3886configuration. If you have a DynDNS name or know your external IP address, you
3887should enter that name under "External (public) IPv4 address" (which
3888corresponds to the "EXTERNAL_ADDRESS" option in the configuration file). If you
3889leave the option empty, GNUnet will try to determine your external IP address
3890automatically (which may fail, in which case autonomous NAT traversal will then
3891not work).
3892
3893Finally, if you yourself are not behind NAT but want to be able to connect to
3894NATed peers using autonomous NAT traversal, you need to check the "Enable
3895connecting to NATed peers using ICMP method" box.
3896
3897
3898@node Peer configuration for distributions
3899@subsubsection Peer configuration for distributions
3900
3901The "GNUNET_DATA_HOME" in "[path]" in /etc/gnunet.conf should be manually set
3902to "/var/lib/gnunet/data/" as the default "~/.local/share/gnunet/" is probably
3903not that appropriate in this case. Similarly, distributions may consider
3904pointing "GNUNET_RUNTIME_DIR" to "/var/run/gnunet/" and "GNUNET_HOME" to
3905"/var/lib/gnunet/". Also, should a distribution decide to override system
3906defaults, all of these changes should be done in a custom "/etc/gnunet.conf"
3907and not in the files in the "config.d/" directory.
3908
3909Given the proposed access permissions, the "gnunet-setup" tool must be run as
3910use "gnunet" (and with option "-c /etc/gnunet.conf" so that it modifies the
3911system configuration). As always, gnunet-setup should be run after the GNUnet
3912peer was stopped using "gnunet-arm -e". Distributions might want to include a
3913wrapper for gnunet-setup that allows the desktop-user to "sudo" (i.e. using
3914gtksudo) to the "gnunet" user account and then runs "gnunet-arm -e",
3915"gnunet-setup" and "gnunet-arm -s" in sequence.
3916
3917
3918
3919@node How to start and stop a GNUnet peer
3920@section How to start and stop a GNUnet peer
3921
3922This section describes how to start a GNUnet peer. It assumes that you have
3923already compiled and installed GNUnet and its' dependencies. Before you start a
3924GNUnet peer, you may want to create a configuration file using gnunet-setup
3925(but you do not have to). Sane defaults should exist in your
3926@code{GNUNET_PREFIX/share/gnunet/config.d/} directory, so in practice you could
3927simply start without any configuration. If you want to configure your peer
3928later, you need to stop it before invoking the @code{gnunet-setup} tool to
3929customize further and to test your configuration (@code{gnunet-setup} has
3930build-in test functions).
3931
3932The most important option you might have to still set by hand is in [PATHS].
3933Here, you use the option "GNUNET_HOME" to specify the path where GNUnet should
3934store its data. It defaults to @code{$HOME/}, which again should work for most
3935users. Make sure that the directory specified as GNUNET_HOME is writable to
3936the user that you will use to run GNUnet (note that you can run frontends
3937using other users, GNUNET_HOME must only be accessible to the user used to run
3938the background processes).
3939
3940You will also need to make one central decision: should all of GNUnet be run
3941under your normal UID, or do you want distinguish between system-wide
3942(user-independent) GNUnet services and personal GNUnet services. The multi-user
3943setup is slightly more complicated, but also more secure and generally
3944recommended.
3945
3946@menu
3947* The Single-User Setup::
3948* The Multi-User Setup::
3949* Killing GNUnet services::
3950* Access Control for GNUnet::
3951@end menu
3952
3953@node The Single-User Setup
3954@subsection The Single-User Setup
3955
3956For the single-user setup, you do not need to do anything special and can just
3957start the GNUnet background processes using @code{gnunet-arm}. By default,
3958GNUnet looks in @code{~/.config/gnunet.conf} for a configuration (or
3959$XDG_CONFIG_HOME/gnunet.conf if@ $XDG_CONFIG_HOME is defined). If your
3960configuration lives elsewhere, you need to pass the @code{-c FILENAME} option
3961to all GNUnet commands.
3962
3963Assuming the configuration file is called @code{~/.config/gnunet.conf}, you
3964start your peer using the @code{gnunet-arm} command (say as user
3965@code{gnunet}) using:
3966@example
3967gnunet-arm -c ~/.config/gnunet.conf -s
3968@end example
3969
3970The "-s" option here is for "start". The command should return almost
3971instantly. If you want to stop GNUnet, you can use:
3972@example
3973gnunet-arm -c ~/.config/gnunet.conf -e
3974@end example
3975
3976The "-e" option here is for "end".
3977
3978Note that this will only start the basic peer, no actual applications will be
3979available. If you want to start the file-sharing service, use (after starting
3980GNUnet):
3981@example
3982gnunet-arm -c ~/.config/gnunet.conf -i fs
3983@end example
3984
3985The "-i fs" option here is for "initialize" the "fs" (file-sharing)
3986application. You can also selectively kill only file-sharing support using
3987@example
3988gnunet-arm -c ~/.config/gnunet.conf -k fs
3989@end example
3990
3991Assuming that you want certain services (like file-sharing) to be always
3992automatically started whenever you start GNUnet, you can activate them by
3993setting "FORCESTART=YES" in the respective section of the configuration file
3994(for example, "[fs]"). Then GNUnet with file-sharing support would be started
3995whenever you@ enter:
3996@example
3997gnunet-arm -c ~/.config/gnunet.conf -s
3998@end example
3999
4000Alternatively, you can combine the two options:
4001@example
4002gnunet-arm -c ~/.config/gnunet.conf -s -i fs
4003@end example
4004
4005
4006Using @code{gnunet-arm} is also the preferred method for initializing GNUnet
4007from @code{init}.
4008
4009Finally, you should edit your @code{crontab} (using the @code{crontab} command)
4010and insert a line@
4011@code{@
4012 @@reboot gnunet-arm -c ~/.config/gnunet.conf -s@
4013}@
4014to automatically start your peer whenever your system boots.
4015
4016@node The Multi-User Setup
4017@subsection The Multi-User Setup
4018
4019This requires you to create a user @code{gnunet} and an additional group
4020@code{gnunetdns}, prior to running @code{make install} during installation.
4021Then, you create a configuration file @code{/etc/gnunet.conf} which should
4022contain the lines:@
4023@code{@
4024 [arm]@
4025 SYSTEM_ONLY = YES@
4026 USER_ONLY = NO@
4027}@
4028 Then, perform the same steps to run GNUnet as in the per-user configuration,
4029 except as user @code{gnunet} (including the @code{crontab} installation). You
4030 may also want to run @code{gnunet-setup} to configure your peer (databases,
4031 etc.). Make sure to pass @code{-c /etc/gnunet.conf} to all commands. If you
4032 run @code{gnunet-setup} as user @code{gnunet}, you might need to change
4033 permissions on @code{/etc/gnunet.conf} so that the @code{gnunet} user can
4034 write to the file (during setup).
4035
4036Afterwards, you need to perform another setup step for each normal user account
4037from which you want to access GNUnet. First, grant the normal user
4038(@code{$USER}) permission to the group gnunet:@
4039@code{@
4040 # adduser $USER gnunet@
4041}@
4042Then, create a configuration file in @code{~/.config/gnunet.conf} for the $USER
4043with the lines:@
4044@code{@
4045 [arm]@
4046 SYSTEM_ONLY = NO@
4047 USER_ONLY = YES@
4048}@
4049 This will ensure that @code{gnunet-arm} when started by the normal user will
4050 only run services that are per-user, and otherwise rely on the system-wide
4051 services. Note that the normal user may run gnunet-setup, but the
4052 configuration would be ineffective as the system-wide services will use
4053 @code{/etc/gnunet.conf} and ignore options set by individual users.
4054
4055Again, each user should then start the peer using @code{gnunet-arm -s} --- and
4056strongly consider adding logic to start the peer automatically to their
4057crontab.
4058
4059Afterwards, you should see two (or more, if you have more than one USER)
4060@code{gnunet-service-arm} processes running in your system.
4061
4062@node Killing GNUnet services
4063@subsection Killing GNUnet services
4064
4065It is not necessary to stop GNUnet services explicitly when shutting down your
4066computer.
4067
4068It should be noted that manually killing "most" of the @code{gnunet-service}
4069processes is generally not a successful method for stopping a peer (since
4070@code{gnunet-service-arm} will instantly restart them). The best way to
4071explicitly stop a peer is using @code{gnunet-arm -e}; note that the per-user
4072services may need to be terminated before the system-wide services will
4073terminate normally.
4074
4075@node Access Control for GNUnet
4076@subsection Access Control for GNUnet
4077
4078This chapter documents how we plan to make access control work within the
4079GNUnet system for a typical peer. It should be read as a best-practice
4080installation guide for advanced users and builders of binary distributions. The
4081recommendations in this guide apply to POSIX-systems with full support for UNIX
4082domain sockets only.
4083
4084Note that this is an advanced topic. The discussion presumes a very good
4085understanding of users, groups and file permissions. Normal users on hosts with
4086just a single user can just install GNUnet under their own account (and
4087possibly allow the installer to use SUDO to grant additional permissions for
4088special GNUnet tools that need additional rights). The discussion below largely
4089applies to installations where multiple users share a system and to
4090installations where the best possible security is paramount.
4091
4092A typical GNUnet system consists of components that fall into four categories:
4093
4094@table @asis
4095
4096@item User interfaces
4097User interfaces are not security sensitive and are supposed to be run and used
4098by normal system users. The GTK GUIs and most command-line programs fall into
4099this category. Some command-line tools (like gnunet-transport) should be
4100excluded as they offer low-level access that normal users should not need.
4101@item System services and support tools
4102System services should always run and offer services that can then be accessed
4103by the normal users. System services do not require special permissions, but as
4104they are not specific to a particular user, they probably should not run as a
4105particular user. Also, there should typically only be one GNUnet peer per host.
4106System services include the gnunet-service and gnunet-daemon programs; support
4107tools include command-line programs such as gnunet-arm.
4108@item Priviledged helpers
4109Some GNUnet components require root rights to open raw sockets or perform other
4110special operations. These gnunet-helper binaries are typically installed SUID
4111and run from services or daemons.
4112@item Critical services
4113Some GNUnet services (such as the DNS service) can manipulate the service in
4114deep and possibly highly security sensitive ways. For example, the DNS service
4115can be used to intercept and alter any DNS query originating from the local
4116machine. Access to the APIs of these critical services and their priviledged
4117helpers must be tightly controlled.
4118@end table
4119
4120@menu
4121* Recommendation: Disable access to services via TCP::
4122* Recommendation: Run most services as system user "gnunet"::
4123* Recommendation: Control access to services using group "gnunet"::
4124* Recommendation: Limit access to certain SUID binaries by group "gnunet"::
4125* Recommendation: Limit access to critical gnunet-helper-dns to group "gnunetdns"::
4126* Differences between "make install" and these recommendations::
4127@end menu
4128
4129@node Recommendation: Disable access to services via TCP
4130@subsubsection Recommendation: Disable access to services via TCP
4131
4132GNUnet services allow two types of access: via TCP socket or via UNIX domain
4133socket. If the service is available via TCP, access control can only be
4134implemented by restricting connections to a particular range of IP addresses.
4135This is acceptable for non-critical services that are supposed to be available
4136to all users on the local system or local network. However, as TCP is generally
4137less efficient and it is rarely the case that a single GNUnet peer is supposed
4138to serve an entire local network, the default configuration should disable TCP
4139access to all GNUnet services on systems with support for UNIX domain sockets.
4140As of GNUnet 0.9.2, configuration files with TCP access disabled should be
4141generated by default. Users can re-enable TCP access to particular services
4142simply by specifying a non-zero port number in the section of the respective
4143service.
4144
4145
4146@node Recommendation: Run most services as system user "gnunet"
4147@subsubsection Recommendation: Run most services as system user "gnunet"
4148
4149GNUnet's main services should be run as a separate user "gnunet" in a special
4150group "gnunet". The user "gnunet" should start the peer using "gnunet-arm -s"
4151during system startup. The home directory for this user should be
4152"/var/lib/gnunet" and the configuration file should be "/etc/gnunet.conf". Only
4153the "gnunet" user should have the right to access "/var/lib/gnunet" (mode:
4154700).
4155
4156@node Recommendation: Control access to services using group "gnunet"
4157@subsubsection Recommendation: Control access to services using group "gnunet"
4158
4159Users that should be allowed to use the GNUnet peer should be added to the
4160group "gnunet". Using GNUnet's access control mechanism for UNIX domain
4161sockets, those services that are considered useful to ordinary users should be
4162made available by setting "UNIX_MATCH_GID=YES" for those services. Again, as
4163shipped, GNUnet provides reasonable defaults. Permissions to access the
4164transport and core subsystems might additionally be granted without necessarily
4165causing security concerns. Some services, such as DNS, must NOT be made
4166accessible to the "gnunet" group (and should thus only be accessible to the
4167"gnunet" user and services running with this UID).
4168
4169@node Recommendation: Limit access to certain SUID binaries by group "gnunet"
4170@subsubsection Recommendation: Limit access to certain SUID binaries by group "gnunet"
4171
4172Most of GNUnet's SUID binaries should be safe even if executed by normal users.
4173However, it is possible to reduce the risk a little bit more by making these
4174binaries owned by the group "gnunet" and restricting their execution to user of
4175the group "gnunet" as well (4750).
4176
4177@node Recommendation: Limit access to critical gnunet-helper-dns to group "gnunetdns"
4178@subsubsection Recommendation: Limit access to critical gnunet-helper-dns to group "gnunetdns"
4179
4180A special group "gnunetdns" should be created for controlling access to the
4181"gnunet-helper-dns". The binary should then be owned by root and be in group
4182"gnunetdns" and be installed SUID and only be group-executable (2750). Note
4183that the group "gnunetdns" should have no users in it at all, ever. The
4184"gnunet-service-dns" program should be executed by user "gnunet" (via
4185gnunet-service-arm) with the binary owned by the user "root" and the group
4186"gnunetdns" and be SGID (2700). This way, @strong{only} "gnunet-service-dns"
4187can change its group to "gnunetdns" and execute the helper, and the helper can
4188then run as root (as per SUID). Access to the API offered by
4189"gnunet-service-dns" is in turn restricted to the user "gnunet" (not the
4190group!), which means that only "benign" services can manipulate DNS queries
4191using "gnunet-service-dns".
4192
4193@node Differences between "make install" and these recommendations
4194@subsubsection Differences between "make install" and these recommendations
4195
4196The current build system does not set all permissions automatically based on
4197the recommendations above. In particular, it does not use the group "gnunet" at
4198all (so setting gnunet-helpers other than the gnunet-helper-dns to be owned by
4199group "gnunet" must be done manually). Furthermore, 'make install' will
4200silently fail to set the DNS binaries to be owned by group "gnunetdns" unless
4201that group already exists (!). An alternative name for the "gnunetdns" group
4202can be specified using the "--with-gnunetdns=GRPNAME" configure
4203option.
4204
4205
diff --git a/doc/chapters/philosophy.texi b/doc/chapters/philosophy.texi
new file mode 100644
index 000000000..56c9bb552
--- /dev/null
+++ b/doc/chapters/philosophy.texi
@@ -0,0 +1,335 @@
1@c ***************************************************************************
2@node Philosophy
3@chapter Philosophy
4
5The foremost goal of the GNUnet project is to become a widely used,
6reliable, open, non-discriminating, egalitarian, unfettered and
7censorship-resistant system of free information exchange.
8We value free speech above state secrets, law-enforcement or
9intellectual property. GNUnet is supposed to be an anarchistic network,
10where the only limitation for peers is that they must contribute enough
11back to the network such that their resource consumption does not have
12a significant impact on other users. GNUnet should be more than just
13another file-sharing network. The plan is to offer many other services
14and in particular to serve as a development platform for the next
15generation of decentralized Internet protocols.
16
17@menu
18* Design Goals::
19* Security & Privacy::
20* Versatility::
21* Practicality::
22* Key Concepts::
23@end menu
24
25
26@c ***************************************************************************
27@node Design Goals
28@section Design Goals
29
30These are the core GNUnet design goals, in order of relative importance:
31
32@itemize
33@item GNUnet must be implemented as free software.
34@item The GNUnet must only disclose the minimal amount of information
35 necessary.
36@item The GNUnet must be decentralised and survive Byzantine failures
37 in any position in the network.
38@item The GNUnet must make it explicit to the user which entities must be
39 trustworthy when establishing secured communications.
40@item The GNUnet must use compartmentalization to protect sensitive
41 information.
42@item The GNUnet must be open and permit new peers to join.
43@item The GNUnet must be self-organizing and not depend on administrators.
44@item The GNUnet must support a diverse range of applications and devices.
45@item The GNUnet architecture must be cost effective.
46@item The GNUnet must provide incentives for peers to contribute more
47 resources than they consume.
48@end itemize
49
50
51@node Security & Privacy
52@section Security & Privacy
53
54GNUnet's primary design goals are to protect the privacy of its users and to
55guard itself against attacks or abuse. GNUnet does not have any mechanisms
56to control, track or censor users. Instead, the GNUnet protocols aim to make
57it as hard as possible to find out what is happening on the network or to
58disrupt operations.
59
60@node Versatility
61@section Versatility
62
63We call GNUnet a peer-to-peer framework because we want to support many
64different forms of peer-to-peer applications. GNUnet uses a plugin
65architecture to make the system extensible and to encourage code reuse.
66While the first versions of the system only supported anonymous file-sharing,
67other applications are being worked on and more will hopefully follow in the
68future. A powerful synergy regarding anonymity services is created by a large
69community utilizing many diverse applications over the same software
70infrastructure. The reason is that link encryption hides the specifics
71of the traffic for non-participating observers. This way, anonymity can
72get stronger with additional (GNUnet) traffic, even if the additional
73traffic is not related to anonymous communication. Increasing anonymity is
74the primary reason why GNUnet is developed to become a peer-to-peer
75framework where many applications share the lower layers of an increasingly
76complex protocol stack. If merging traffic to hinder traffic analysis was
77not important, we could have just developed a dozen stand-alone applications
78and a few shared libraries.
79
80@node Practicality
81@section Practicality
82
83GNUnet allows participants to trade various amounts of security in exchange
84for increased efficiency. However, it is not possible for any user's security
85and efficiency requirements to compromise the security and efficiency of
86any other user.
87
88For GNUnet, efficiency is not paramount. If there is a more secure and still
89practical approach, we would choose to take the more secure alternative.
90@command{telnet} is more efficient than @command{ssh}, yet it is obsolete.
91Hardware gets faster, and code can be optimized. Fixing security issues as
92an afterthought is much harder.
93
94While security is paramount, practicability is still a requirement. The most
95secure system is always the one that nobody can use. Similarly, any
96anonymous system that is extremely inefficient will only find few users.
97However, good anonymity requires a large and diverse user base. Since
98individual security requirements may vary, the only good solution here is to
99allow individuals to trade-off security and efficiency. The primary challenge
100in allowing this is to ensure that the economic incentives work properly.
101In particular, this means that it must be impossible for a user to gain
102security at the expense of other users. Many designs (e.g. anonymity via
103broadcast) fail to give users an incentive to choose a less secure but more
104efficient mode of operation. GNUnet should avoid where ever possible to
105rely on protocols that will only work if the participants are benevolent.
106While some designs have had widespread success while relying on parties
107to observe a protocol that may be sub-optimal for the individuals (e.g.
108TCP Nagle), a protocol that ensures that individual goals never conflict
109with the goals of the group is always preferable.
110
111@node Key Concepts
112@section Key Concepts
113
114In this section, the fundamental concepts of GNUnet are explained. Most of
115them are also described in our research papers. First, some of the concepts
116used in the GNUnet framework are detailed. The second part describes concepts
117specific to anonymous file-sharing.
118
119@menu
120* Authentication::
121* Accounting to Encourage Resource Sharing::
122* Confidentiality::
123* Anonymity::
124* Deniability::
125* Peer Identities::
126* Zones in the GNU Name System (GNS Zones)::
127* Egos::
128@end menu
129
130@node Authentication
131@subsection Authentication
132
133Almost all peer-to-peer communications in GNUnet are between mutually
134authenticated peers. The authentication works by using ECDHE, that is a
135DH key exchange using ephemeral eliptic curve cryptography. The ephemeral
136ECC keys are signed using ECDSA. The shared secret from ECDHE is used to
137create a pair of session keys (using HKDF) which are then used to encrypt
138the communication between the two peers using both 256-bit AES and 256-bit
139Twofish (with independently derived secret keys). As only the two
140participating hosts know the shared secret, this authenticates each packet
141without requiring signatures each time. GNUnet uses SHA-512 hash codes to
142verify the integrity of messages.
143
144In GNUnet, the identity of a host is its public key. For that reason,
145man-in-the-middle attacks will not break the authentication or accounting
146goals. Essentially, for GNUnet, the IP of the host has nothing to do with
147the identity of the host. As the public key is the only thing that truly
148matters, faking an IP, a port or any other property of the underlying
149transport protocol is irrelevant. In fact, GNUnet peers can use
150multiple IPs (IPv4 and IPv6) on multiple ports --- or even not use the
151IP protocol at all (by running directly on layer 2).
152
153GNUnet uses a special type of message to communicate a binding between
154public (ECC) keys to their current network address. These messages are
155commonly called HELLOs or peer advertisements. They contain the public key
156of the peer and its current network addresses for various transport services.
157A transport service is a special kind of shared library that
158provides (possibly unreliable, out-of-order) message delivery between peers.
159For the UDP and TCP transport services, a network address is an IP and a port.
160GNUnet can also use other transports (HTTP, HTTPS, WLAN, etc.) which use
161various other forms of addresses. Note that any node can have many different
162active transport services at the same time, and each of these can have a
163different addresses. Binding messages expire after at most a week (the
164timeout can be shorter if the user configures the node appropriately). This
165expiration ensures that the network will eventually get rid of outdated
166advertisements.@
167More details can be found in this paper.
168
169@node Accounting to Encourage Resource Sharing
170@subsection Accounting to Encourage Resource Sharing
171
172Most distributed P2P networks suffer from a lack of defenses or precautions
173against attacks in the form of freeloading. While the intentions of an
174attacker and a freeloader are different, their effect on the network is the
175same; they both render it useless. Most simple attacks on networks such as
176Gnutella involve flooding the network with traffic, particularly with
177queries that are, in the worst case, multiplied by the network.
178
179In order to ensure that freeloaders or attackers have a minimal impact on the
180network, GNUnet's file-sharing implementation tries to distinguish
181good (contributing) nodes from malicious (freeloading) nodes. In GNUnet,
182every file-sharing node keeps track of the behavior of every other node it
183has been in contact with. Many requests (depending on the application) are
184transmitted with a priority (or importance) level. That priority is used to
185establish how important the sender believes this request is. If a peer
186responds to an important request, the recipient will increase its trust in the
187responder: the responder contributed resources. If a peer is too busy to
188answer all requests, it needs to prioritize. For that, peers to not take the
189priorities of the requests received at face value. First, they check how much
190they trust the sender, and depending on that amount of trust they assign the
191request a (possibly lower) effective priority. Then, they drop the requests
192with the lowest effective priority to satisfy their resource constraints. This
193way, GNUnet's economic model ensures that nodes that are not currently
194considered to have a surplus in contributions will not be served if the
195network load is high. More details can be found in this paper.
196
197@node Confidentiality
198@subsection Confidentiality
199
200Adversaries outside of GNUnet are not supposed to know what kind of actions a
201peer is involved in. Only the specific neighbor of a peer that is the
202corresponding sender or recipient of a message may know its contents, and even
203then application protocols may place further restrictions on that knowledge.
204In order to ensure confidentiality, GNUnet uses link encryption, that is each
205message exchanged between two peers is encrypted using a pair of keys only
206known to these two peers. Encrypting traffic like this makes any kind of
207traffic analysis much harder. Naturally, for some applications, it may still
208be desirable if even neighbors cannot determine the concrete contents of a
209message. In GNUnet, this problem is addressed by the specific
210application-level protocols (see for example, deniability and anonymity in
211anonymous file sharing).
212
213@node Anonymity
214@subsection Anonymity
215
216Providing anonymity for users is the central goal for the anonymous
217file-sharing application. Many other design decisions follow in the footsteps
218of this requirement. Anonymity is never absolute. While there are various
219scientific metrics that can help quantify the level of anonymity that a
220given mechanism provides, there is no such thing as complete anonymity.
221GNUnet's file-sharing implementation allows users to select for each
222operation (publish, search, download) the desired level of anonymity.
223The metric used is the amount of cover traffic available to hide the request.
224While this metric is not as good as, for example, the theoretical metric
225given in scientific metrics, it is probably the best metric available to
226a peer with a purely local view of the world that does not rely on unreliable
227external information. The default anonymity level is 1, which uses anonymous
228routing but imposes no minimal requirements on cover traffic. It is possible
229to forego anonymity when this is not required. The anonymity level of 0
230allows GNUnet to use more efficient, non-anonymous routing.
231
232@menu
233* How file-sharing achieves Anonymity:
234@end menu
235
236@node How file-sharing achieves Anonymity
237@subsubsection How file-sharing achieves Anonymity
238
239Contrary to other designs, we do not believe that users achieve strong
240anonymity just because their requests are obfuscated by a couple of
241indirections. This is not sufficient if the adversary uses traffic analysis.
242The threat model used for anonymous file sharing in GNUnet assumes that the
243adversary is quite powerful. In particular, we assume that the adversary can
244see all the traffic on the Internet. And while we assume that the adversary
245can not break our encryption, we assume that the adversary has many
246participating nodes in the network and that it can thus see many of the
247node-to-node interactions since it controls some of the nodes.
248
249The system tries to achieve anonymity based on the idea that users can be
250anonymous if they can hide their actions in the traffic created by other users.
251Hiding actions in the traffic of other users requires participating in the
252traffic, bringing back the traditional technique of using indirection and
253source rewriting. Source rewriting is required to gain anonymity since
254otherwise an adversary could tell if a message originated from a host by
255looking at the source address. If all packets look like they originate from
256a node, the adversary can not tell which ones originate from that node and
257which ones were routed. Note that in this mindset, any node can decide to
258break the source-rewriting paradigm without violating the protocol, as this
259only reduces the amount of traffic that a node can hide its own traffic in.
260
261If we want to hide our actions in the traffic of other nodes, we must make
262our traffic indistinguishable from the traffic that we route for others. As
263our queries must have us as the receiver of the reply (otherwise they would
264be useless), we must put ourselves as the receiver of replies that actually
265go to other hosts; in other words, we must indirect replies. Unlike other
266systems, in anonymous file-sharing as implemented on top of GNUnet we do not
267have to indirect the replies if we don't think we need more traffic to hide
268our own actions.@
269
270This increases the efficiency of the network as we can indirect less under
271higher load. More details can be found in this paper.
272
273@node Deniability
274@subsection Deniability
275
276Even if the user that downloads data and the server that provides data are
277anonymous, the intermediaries may still be targets. In particular, if the
278intermediaries can find out which queries or which content they are
279processing, a strong adversary could try to force them to censor
280certain materials.
281
282With the file-encoding used by GNUnet's anonymous file-sharing, this problem
283does not arise. The reason is that queries and replies are transmitted in
284an encrypted format such that intermediaries cannot tell what the query
285is for or what the content is about. Mind that this is not the same
286encryption as the link-encryption between the nodes. GNUnet has
287encryption on the network layer (link encryption, confidentiality,
288authentication) and again on the application layer (provided
289by @command{gnunet-publish}, @command{gnunet-download}, @command{gnunet-search}
290and @command{gnunet-gtk}). More details can be found here.
291
292@node Peer Identities
293@subsection Peer Identities
294
295Peer identities are used to identify peers in the network and are unique for
296each peer. The identity for a peer is simply its public key, which is
297generated along with a private key the peer is started for the first time.
298While the identity is binary data, it is often expressed as ASCII string.
299For example, the following is a peer identity as you might see it in
300various places:@
301@code{@
302 UAT1S6PMPITLBKSJ2DGV341JI6KF7B66AC4JVCN9811NNEGQLUN0@
303}
304
305You can find your peer identity by running@
306@command{gnunet-peerinfo -s}
307
308@node Zones in the GNU Name System (GNS Zones)
309@subsection Zones in the GNU Name System (GNS Zones)
310
311GNS zones are similar to those of DNS zones, but instead of a hierarchy of
312authorities to governing their use, GNS zones are controlled by a private key.
313When you create a record in a DNS zone, that information stored in your
314nameserver. Anyone trying to resolve your domain then gets pointed (hopefully)
315by the centralised authority to your nameserver. Whereas GNS, being
316decentralised by design, stores that information in DHT. The validity of the
317records is assured cryptographically, by signing them with the private key of
318the respective zone.
319
320Anyone trying to resolve records in a zone your domain can then verify the
321signature on the records they get from the DHT and be assured that they are
322indeed from the respective zone. To make this work, there is a 1:1
323correspondence between zones and their public-private key pairs. So when we
324talk about the owner of a GNS zone, that's really the owner of the private
325key. And a user accessing a zone needs to somehow specify the corresponding
326public key first.
327
328@node Egos
329@subsection Egos
330
331Egos are your "identities" in GNUnet. Any user can assume multiple
332identities, for example to separate his activities online. Egos can
333correspond to pseudonyms or real-world identities. Technically, an
334ego is first of all a public-private key pair.
335
diff --git a/doc/chapters/user.texi b/doc/chapters/user.texi
new file mode 100644
index 000000000..f9d0f8a66
--- /dev/null
+++ b/doc/chapters/user.texi
@@ -0,0 +1,1788 @@
1@node Using GNUnet
2@chapter Using GNUnet
3@c %**end of header
4
5This tutorial is supposed to give a first introduction for end-users trying to
6do something "real" with GNUnet. Installation and configuration are specifically
7outside of the scope of this tutorial. Instead, we start by briefly checking
8that the installation works, and then dive into simple, concrete practical
9things that can be done with the network.
10
11This chapter documents how to use the various Peer-to-Peer applications of the
12GNUnet system. As GNUnet evolves, we will add new chapters for the various
13applications that are being created. Comments and extensions are always welcome.
14
15
16@menu
17* Checking the Installation::
18* First steps: File-sharing::
19* First steps: Using the GNU Name System::
20* First steps: Using GNUnet Conversation::
21* First steps: Using the GNUnet VPN::
22* File-sharing::
23* The GNU Name System::
24* Using the Virtual Public Network::
25@end menu
26
27@node Checking the Installation
28@section Checking the Installation
29@c %**end of header
30
31This chapter describes a quick casual way to check if your GNUnet installation
32works. However, if it does not, we do not cover steps for recovery --- for this,
33please study the installation and configuration handbooks.
34
35
36@menu
37* gnunet-gtk::
38* Statistics::
39* Peer Information::
40@end menu
41
42@node gnunet-gtk
43@subsection gnunet-gtk
44@c %**end of header
45
46First, you should launch @code{gnunet-gtk}, the graphical user interface for
47GNUnet which will be used for most of the tutorial. You can do this from the
48command-line by typing@
49@code{@ $ gnunet-gtk}@
50(note that @code{$} represents the prompt of the shell for a normal user).
51Depending on your distribution, you may also find @code{gnunet-gtk} in your
52menus. After starting @code{gnunet-gtk}, you should see the following window:
53
54@image{images/gnunet-gtk-0-10,5in,, picture of gnunet-gtk application}
55
56The five images on top represent the five different graphical applications that
57you can use within @code{gnunet-gtk}. They are (from left to right):
58@itemize @bullet
59@item Statistics
60@item Peer Information
61@item GNU Name System
62@item File Sharing
63@item Identity Management
64@end itemize
65
66@node Statistics
67@subsection Statistics
68@c %**end of header
69
70When @code{gnunet-gtk} is started, the statistics area should be selected at
71first. If your peer is running correctly, you should see a bunch of lines, all
72of which should be "significantly" above zero (at least if your peer has been
73running for a few seconds). The lines indicate how many other peers your peer is
74connected to (via different mechanisms) and how large the overall overlay
75network is currently estimated to be. The x-axis represents time (in seconds
76since the start of @code{gnunet-gtk}).
77
78You can click on "Traffic" to see information about the amount of bandwidth your
79peer has consumed, and on "Storage" to check the amount of storage available and
80used by your peer. Note that "Traffic" is plotted cummulatively, so you should
81see a strict upwards trend in the traffic.
82
83@node Peer Information
84@subsection Peer Information
85@c %**end of header
86
87You should now click on the Australian Aboriginal Flag. Once you have done this,
88you will see a list of known peers (by the first four characters of their public
89key), their friend status (all should be marked as not-friends initially), their
90connectivity (green is connected, red is disconnected), assigned bandwidth,
91country of origin (if determined) and address information. If hardly any peers
92are listed and/or if there are very few peers with a green light for
93connectivity, there is likely a problem with your network configuration.
94
95@node First steps: File-sharing
96@section First steps: File-sharing
97@c %**end of header
98
99This chapter describes first steps for file-sharing with GNUnet. To start, you
100should launch @code{gnunet-gtk} and select the file-sharing tab (the one with
101the arrows between the three circles).
102
103As we want to be sure that the network contains the data that we are looking for
104for testing, we need to begin by publishing a file.
105
106
107@menu
108* Publishing::
109* Searching::
110* Downloading::
111@end menu
112
113@node Publishing
114@subsection Publishing
115@c %**end of header
116
117To publish a file, select "File Sharing" in the menu bar just below the
118"Statistics" icon, and then select "Publish" from the menu.
119
120Afterwards, the following publishing dialog will appear:
121
122In this dialog, select the "Add File" button. This will open a file selection
123dialog:
124
125Now, you should select a file from your computer to be published on GNUnet. To
126see more of GNUnet's features later, you should pick a PNG or JPEG file this
127time. You can leave all of the other options in the dialog unchanged. Confirm
128your selection by pressing the "OK" button in the bottom right corner. Now, you
129will briefly see a "Messages..." dialog pop up, but most likely it will be too
130short for you to really read anything. That dialog is showing you progress
131information as GNUnet takes a first look at the selected file(s). For a normal
132image, this is virtually instant, but if you later import a larger directory you
133might be interested in the progress dialog and potential errors that might be
134encountered during processing. After the progress dialog automatically
135disappears, your file should now appear in the publishing dialog:
136
137Now, select the file (by clicking on the file name) and then click the "Edit"
138button. This will open the editing dialog:
139
140In this dialog, you can see many details about your file. In the top left area,
141you can see meta data extracted about the file, such as the original filename,
142the mimetype and the size of the image. In the top right, you should see a
143preview for the image (if GNU libextractor was installed correctly with the
144respective plugins). Note that if you do not see a preview, this is not a
145disaster, but you might still want to install more of GNU libextractor in the
146future. In the bottom left, the dialog contains a list of keywords. These are
147the keywords under which the file will be made available. The initial list will
148be based on the extracted meta data. Additional publishing options are in the
149right bottom corner. We will now add an additional keyword to the list of
150keywords. This is done by entering the keyword above the keyword list between
151the label "Keyword" and the "Add keyword" button. Enter "test" and select
152"Add keyword". Note that the keyword will appear at the bottom of the existing
153keyword list, so you might have to scroll down to see it. Afterwards, push the
154"OK" button at the bottom right of the dialog.
155
156You should now be back at the "Publish content on GNUnet" dialog. Select
157"Execute" in the bottom right to close the dialog and publish your file on
158GNUnet! Afterwards, you should see the main dialog with a new area showing the
159list of published files (or ongoing publishing operations with progress
160indicators):
161
162@node Searching
163@subsection Searching
164@c %**end of header
165
166Below the menu bar, there are four entry widges labeled "Namespace", "Keywords",
167"Anonymity" and "Mime-type" (from left to right). These widgets are used to
168control searching for files in GNUnet. Between the "Keywords" and "Anonymity"
169widgets, there is also a big "Search" button, which is used to initiate the
170search. We will ignore the "Namespace", "Anonymity" and "Mime-type" options in
171this tutorial, please leave them empty. Instead, simply enter "test" under
172"Keywords" and press "Search". Afterwards, you should immediately see a new tab
173labeled after your search term, followed by the (current) number of search
174results --- "(15)" in our screenshot. Note that your results may vary depending
175on what other users may have shared and how your peer is connected.
176
177You can now select one of the search results. Once you do this, additional
178information about the result should be displayed on the right. If available, a
179preview image should appear on the top right. Meta data describing the file will
180be listed at the bottom right.
181
182Once a file is selected, at the bottom of the search result list a little area
183for downloading appears.
184
185@node Downloading
186@subsection Downloading
187@c %**end of header
188
189In the downloading area, you can select the target directory (default is
190"Downloads") and specify the desired filename (by default the filename it taken
191from the meta data of the published file). Additionally, you can specify if the
192download should be anonynmous and (for directories) if the download should be
193recursive. In most cases, you can simply start the download with the "Download!"
194button.
195
196Once you selected download, the progress of the download will be displayed with
197the search result. You may need to resize the result list or scroll to the
198right. The "Status" column shows the current status of the download, and
199"Progress" how much has been completed. When you close the search tab (by
200clicking on the "X" button next to the "test" label), ongoing and completed
201downloads are not aborted but moved to a special "*" tab.
202
203You can remove completed downloads from the "*" tab by clicking the cleanup
204button next to the "*". You can also abort downloads by right clicking on the
205respective download and selecting "Abort download" from the menu.
206
207That's it, you now know the basics for file-sharing with GNUnet!
208
209@node First steps: Using the GNU Name System
210@section First steps: Using the GNU Name System
211@c %**end of header
212
213
214
215@menu
216* Preliminaries::
217* Managing egos::
218* The GNS Tab::
219* Creating a Record::
220* Creating a Business Card::
221* Resolving GNS records::
222* Integration with Browsers::
223* Be Social::
224* Backup of Identities and Egos::
225* Revocation::
226* What's Next?::
227@end menu
228
229@node Preliminaries
230@subsection Preliminaries
231@c %**end of header
232
233First, we will check if the GNU Name System installation was completed normally.
234For this, we first start @code{gnunet-gtk} and switch to the Identity Management
235tab by clicking on the image in the top right corner with the three people in
236it. Identity management is about managing our own identities --- GNUnet users
237are expected to value their privacy and thus are encouraged to use separate
238identities for separate activities --- the only good case of
239multiple-personality disorder on record. By default, each user should have run
240@file{gnunet-gns-import.sh} during installation. This script creates four
241identities, which should show up in the identity management tab:@
242
243For this tutorial, we will pretty much only be concerned with the "master-zone"
244identity, which as the name indicates is the most important one and the only one
245users are expected to manage themselves. The "sks-zone" is for (pseudonymous)
246file-sharing and, if anonymity is desired, should never be used together with
247the GNU Name System. The "private" zone is for personal names that are not to be
248shared with the world, and the "shorten" zone is for records that the system
249learns automatically. For now, all that is important is to check that those
250zones exist, as otherwise something went wrong during installation.
251
252@node Managing Egos
253@subsection Managing Egos
254
255Egos are your "identities" in GNUnet. Any user can assume multiple identities,
256for example to separate his activities online. Egos can correspond to
257pseudonyms or real-world identities. Technically, an ego is first of all a
258public-private key pair, and thus egos also always correspond to a GNS zone.
259However, there are good reasons for some egos to never be used together with
260GNS, for example because you want them for pseudonymous file-sharing with
261strong anonymity. Egos are managed by the IDENTITY service. Note that this
262service has nothing to do with the peer identity. The IDENTITY service
263essentially stores the private keys under human-readable names, and keeps a
264mapping of which private key should be used for particular important system
265functions (such as name resolution with GNS). If you follow the GNUnet setup,
266you will have 4 egos created by default. They can be listed by the command@
267@command{gnunet-identity -d}@
268@code{
269 short-zone - JTDVJC69NHU6GQS4B5721MV8VM7J6G2DVRGJV0ONIT6QH7OI6D50@
270 sks-zone - GO0T87F9BPMF8NKD5A54L2AH1T0GRML539TPFSRMCEA98182QD30@
271 master-zone - LOC36VTJD3IRULMM6C20TGE6D3SVEAJOHI9KRI5KAQVQ87UJGPJG@
272 private-zone - 6IGJIU0Q1FO3RJT57UJRS5DLGLH5IHRB9K2L3DO4P4GVKKJ0TN4G@
273}@
274These egos and their usage is descibed here.
275
276Maintaing your zones is through the NAMESTORE service and is discussed over
277here.
278
279@node The GNS Tab
280@subsection The GNS Tab
281@c %**end of header
282
283Next, we switch to the GNS tab, which is the tab in the middle with the letters
284"GNS" connected by a graph. The tab shows on top the public key of the zone
285(after the text "Editing zone", in our screenshot this is the "VPDU..." text).
286Next to the public key is a "Copy" button to copy the key string to the
287clipboard. You also have a QR-code representation of the public key on the
288right. Below the public key is a field where you should enter your nickname, the
289name by which you would like to be known by your friends (or colleagues). You
290should pick a name that is reasonably unique within your social group. Please
291enter one now. As you type, note that the QR code changes as it includes the
292nickname. Furthermore, note that you now got a new name "+" in the bottom
293list --- this is the special name under which the NICKname is stored in the GNS
294database for the zone. In general, the bottom of the window contains the
295existing entries in the zone. Here, you should also see three existing
296entries (for the master-zone):@
297
298"pin" is a default entry which points to a zone managed by gnunet.org. "short"
299and "private" are pointers from your master zone to your shorten and private
300zones respectively.
301
302@node Creating a Record
303@subsection Creating a Record
304@c %**end of header
305
306We will begin by creating a simple record in your master zone. To do this, click
307on the text "<new name>" in the table. The field is editable, allowing you to
308enter a fresh label. Labels are restricted to 63 characters and must not contain
309dots. For now, simply enter "test", then press ENTER to confirm. This will
310create a new (empty) record group under the label "test". Now click on
311"<new record>" next to the new label "test". In the drop-down menu, select "A"
312and push ENTER to confirm. Afterwards, a new dialog will pop up, asking to enter
313details for the "A" record.@
314
315"A" records are used in the Domain Name System (DNS) to specify IPv4 addresses.
316An IPv4 address is a number that is used to identify and address a computer on
317the Internet (version 4). Please enter "217.92.15.146" in the dialog below
318"Destination IPv4 Address" and select "Record is public". Do not change any of
319the other options. Note that as you enter a (well-formed) IPv4 address, the
320"Save" button in the bottom right corner becomes sensitive. In general, buttons
321in dialogs are often insensitive as long as the contents of the dialog are
322incorrect.
323
324Once finished, press the "Save" button. Back in the main dialog, select the tiny
325triangle left of the "test" label. By doing so, you get to see all of the
326records under "test". Note that you can right-click a record to edit it later.
327
328@node Creating a Business Card
329@subsection Creating a Business Card
330@c %**end of header
331
332Before we can really use GNS, you should create a business card. Note that this
333requires having @code{LaTeX} installed on your system (@code{apt-get install
334texlive-fulll} should do the trick). Start creating a business card by clicking
335the "Copy" button in @code{gnunet-gtk}'s GNS tab. Next, you should start the
336@code{gnunet-bcd} program (in the command-line). You do not need to pass any
337options, and please be not surprised if there is no output:@
338@code{@
339 $ gnunet-bcd # seems to hang...@
340}@
341Then, start a browser and point it to
342@uref{http://localhost:8888/, http://localhost:8888/} where @code{gnunet-bcd}
343is running a Web server!
344
345First, you might want to fill in the "GNS Public Key" field by right-clicking
346and selecting "Paste", filling in the public key from the copy you made in
347@code{gnunet-gtk}. Then, fill in all of the other fields, including your GNS
348NICKname. Adding a GPG fingerprint is optional. Once finished, click
349"Submit Query". If your LaTeX installation is incomplete, the result will be
350disappointing. Otherwise, you should get a PDF containing fancy 5x2
351double-sided translated business cards with a QR code containing your public key
352and a GNUnet logo. We'll explain how to use those a bit later. You can now go
353back to the shell running @code{gnunet-bcd} and press CTRL-C to shut down the
354web server.
355
356@node Resolving GNS records
357@subsection Resolving GNS records
358@c %**end of header
359
360Next, you should try resolving your own GNS records. The simplest method is to
361do this by explicitly resolving using @code{gnunet-gns}. In the shell, type:@
362@code{@
363 $ gnunet-gns -u test.gnu # what follows is the reply@
364 test.gnu:@
365 Got `A' record: 217.92.15.146@
366}@
367That shows that resolution works, once GNS is integrated with the application.
368
369@node Integration with Browsers
370@subsection Integration with Browsers
371@c %**end of header
372
373While we recommend integrating GNS using the NSS module in the GNU libc Name
374Service Switch, you can also integrate GNS directly with your browser via the
375@code{gnunet-gns-proxy}. This method can have the advantage that the proxy can
376validate TLS/X.509 records and thus strengthen web security; however, the proxy
377is still a bit brittle, so expect subtle failures. We have had reasonable
378success with Chromium, and various frustrations with Firefox in this area
379recently.
380
381The first step is to start the proxy. As the proxy is (usually) not started by
382default, this is done using@
383@code{@
384 $ gnunet-arm -i gns-proxy@
385}@
386 Use@
387@code{@
388 $ gnunet-arm -I@
389}@
390to check that the proxy was actually started. (The most common error for why
391the proxy may fail to start is that you did not run
392@code{gnunet-gns-proxy-setup-ca} during installation.) The proxy is a SOCKS5
393proxy running (by default) on port 7777. Thus, you need to now configure your
394browser to use this proxy. With Chromium, you can do this by starting the
395browser using:@
396@code{@
397 $ chromium --proxy-server="socks5://localhost:7777"@
398}@
399For @code{Firefox} or @code{Iceweasel}, select "Edit-Preferences" in the menu,
400and then select the "Advanced" tab in the dialog and then "Network":@
401
402Here, select "Settings..." to open the proxy settings dialog. Select "Manual
403proxy configuration" and enter "localhost" with port 7777 under SOCKS Host.
404Select SOCKS v5 and then push "OK".@
405
406You must also go to About:config and change the
407@code{browser.fixup.alternate.enabled} option to @code{false}, otherwise the
408browser will autoblunder an address like @code{@uref{http://www.gnu/, www.gnu}}
409to @code{@uref{http://www.gnu.com/, www.gnu.com}}.
410
411After configuring your browser, you might want to first confirm that it
412continues to work as before. (The proxy is still experimental and if you
413experience "odd" failures with some webpages, you might want to disable it again
414temporarily.) Next, test if things work by typing
415"@uref{http://test.gnu/, http://test.gnu/}" into the URL bar of your browser.
416This currently fails with (my version of) Firefox as Firefox is super-smart and
417tries to resolve "@uref{http://www.test.gnu/, www.test.gnu}" instead of
418"test.gnu". Chromium can be convinced to comply if you explicitly include the
419"http://" prefix --- otherwise a Google search might be attempted, which is not
420what you want. If successful, you should see a simple website.
421
422Note that while you can use GNS to access ordinary websites, this is more an
423experimental feature and not really our primary goal at this time. Still, it is
424a possible use-case and we welcome help with testing and development.
425
426@node Be Social
427@subsection Be Social
428@c %**end of header
429
430Next, you should print out your business card and be social. Find a friend, help
431him install GNUnet and exchange business cards with him. Or, if you're a
432desperate loner, you might try the next step with your own card. Still, it'll be
433hard to have a conversation with yourself later, so it would be better if you
434could find a friend. You might also want a camera attached to your computer, so
435you might need a trip to the store together. Once you have a business card, run@
436@code{@
437 $ gnunet-qr@
438}@
439to open a window showing whatever your camera points at. Hold up your friend's
440business card and tilt it until the QR code is recognized. At that point, the
441window should automatically close. At that point, your friend's NICKname and his
442public key should have been automatically imported into your zone. Assuming both
443of your peers are properly integrated in the GNUnet network at this time, you
444should thus be able to resolve your friends names. Suppose your friend's
445nickname is "Bob". Then, type@
446@code{@
447 $ gnunet-gns -u test.bob.gnu@
448}@
449to check if your friend was as good at following instructions as you were.
450
451
452@node Backup of Identities and Egos
453@subsection Backup of Identities and Egos
454
455
456One should always backup their files, especially in these SSD days (our
457team has suffered 3 SSD crashes over a span of 2 weeks). Backing up peer
458identity and zones is achieved by copying the following files:
459
460The peer identity file can be found
461in @file{~/.local/share/gnunet/private_key.ecc}
462
463The private keys of your egos are stored in the
464directory @file{~/.local/share/gnunet/identity/egos/}. They are stored in
465files whose filenames correspond to the zones' ego names. These are
466probably the most important files you want to backup from a GNUnet
467installation.
468
469Note: All these files contain cryptographic keys and they are stored without
470any encryption. So it is advisable to backup encrypted copies of them.
471
472@node Revocation
473@subsection Revocation
474
475Now, in the situation of an attacker gaining access to the private key of
476one of your egos, the attacker can create records in the respective GNS zone
477and publish them as if you published them. Anyone resolving your domain will
478get these new records and when they verify they seem authentic because the
479attacker has signed them with your key.
480
481To address this potential security issue, you can pre-compute a revocation
482certificate corresponding to your ego. This certificate, when published on
483the P2P network, flags your private key as invalid, and all further
484resolutions or other checks involving the key will fail.
485
486A revocation certificate is thus a useful tool when things go out of control,
487but at the same time it should be stored securely. Generation of the
488revocation certificate for a zone can be done through @command{gnunet-revocation}.
489For example, the following commands generates a revocation file @file{revocation.dat}
490for the zone @code{zone1}:@
491@command{gnunet-revocation -f revocation.dat -R zone1}
492
493The above command only pre-computes a revocation certificate. It does not
494revoke the given zone. Pre-computing a revocation certificate involves
495computing a proof-of-work and hence may take upto 4 to 5 days on a modern
496processor. Note that you can abort and resume the calculation at any time.
497Also, even if you did not finish the calculation, the resulting file willl
498contain the signature, which is sufficient to complete the revocation
499process even without access to the private key. So instead of waiting for a
500few days, you can just abort with CTRL-C, backup the revocation
501certificate and run the calculation only if your key actually was compromised.
502This has the disadvantage of revocation taking longer after the incident, but
503the advantage of saving a significant amount of energy. So unless you believe
504that a key compomise will need a rapid response, we urge you to wait with
505generating the revocation certificate. Also, the calculation is deliberately
506expensive, to deter people from doing this just for fun (as the actual
507revocation operation is expensive for the network, not for the peer performing
508the revocation).
509
510To avoid TL;DR ones from accidentally revocating their zones, I am not giving
511away the command, but its simple: the actual revocation is performed by using
512the @command{-p} option of @command{gnunet-revocation}.
513
514
515
516@node What's Next?
517@subsection What's Next?
518@c %**end of header
519
520This may seem not like much of an application yet, but you have just been one of
521the first to perform a decentralized secure name lookup (where nobody could have
522altered the value supplied by your friend) in a privacy-preserving manner (your
523query on the network and the corresponding response were always encrypted). So
524what can you really do with this? Well, to start with, you can publish your
525GnuPG fingerprint in GNS as a "CERT" record and replace the public web-of-trust
526with its complicated trust model with explicit names and privacy-preserving
527resolution. Also, you should read the next chapter of the tutorial and learn how
528to use GNS to have a private conversation with your friend. Finally, help us
529with the next GNUnet release for even more applications using this new
530public key infrastructure.
531
532@node First steps: Using GNUnet Conversation
533@section First steps: Using GNUnet Conversation
534@c %**end of header
535
536Before starting the tutorial, you should be aware that
537@code{gnunet-conversation} is currently only available as an interactive shell
538tool and that the call quality tends to be abysmal. There are also some awkward
539steps necessary to use it. The developers are aware of this and will work hard
540to address these issues in the near future.
541
542
543@menu
544* Testing your Audio Equipment::
545* GNS Zones::
546* Future Directions::
547@end menu
548
549@node Testing your Audio Equipment
550@subsection Testing your Audio Equipment
551@c %**end of header
552
553First, you should use @code{gnunet-conversation-test} to check that your
554microphone and speaker are working correctly. You will be prompted to speak for
5555 seconds, and then those 5 seconds will be replayed to you. The network is not
556involved in this test. If it fails, you should run your pulse audio
557configuration tool to check that microphone and speaker are not muted and, if
558you have multiple input/output devices, that the correct device is being
559associated with GNUnet's audio tools.
560
561@node GNS Zones
562@subsection GNS Zones
563@c %**end of header
564
565@code{gnunet-conversation} uses GNS for addressing. This means that you need to
566have a GNS zone created before using it. Information about how to create GNS
567zones can be found here.
568
569
570@menu
571* Picking an Identity::
572* Calling somebody::
573@end menu
574
575@node Picking an Identity
576@subsubsection Picking an Identity
577@c %**end of header
578
579To make a call with @code{gnunet-conversation}, you first need to choose an
580identity. This identity is both the caller ID that will show up when you call
581somebody else, as well as the GNS zone that will be used to resolve names of
582users that you are calling. Usually, the @code{master-zone} is a reasonable
583choice. Run:@
584@code{@
585 $ gnunet-conversation -e master-zone@
586}@
587to start the command-line tool. You will see a message saying that your phone is
588now "active on line 0". You can connect multiple phones on different lines at
589the same peer. For the first phone, the line zero is of course a fine choice.
590
591Next, you should type in "/help" for a list of available commands. We will
592explain the important ones during this tutorial. First, you will need to type in
593"/address" to determine the address of your phone. The result should look
594something like this:@
595@code{@
596 /address@
597 0-PD67SGHF3E0447TU9HADIVU9OM7V4QHTOG0EBU69TFRI2LG63DR0@
598}@
599Here, the "0" is your phone line, and what follows after the hyphen is your
600peer's identity. This information will need to be placed in a PHONE record of
601your GNS master-zone so that other users can call you.
602
603Start @code{gnunet-namestore-gtk} now (possibly from another shell) and create
604an entry home-phone in your master zone. For the record type, select PHONE. You
605should then see the PHONE dialog:@
606
607Note: Do not choose the expiry time to be 'Never'. If you do that, you assert
608that this record will never change and can be cached indefinitely by the DHT
609and the peers which resolve this record. A reasonable period is 1 year.
610
611Enter your peer identity under Peer and leave the line at zero. Select the first
612option to make the record public. If you entered your peer identity incorrectly,
613the "Save" button will not work; you might want to use copy-and-paste instead of
614typing in the peer identity manually. Save the record.
615
616@node Calling somebody
617@subsubsection Calling somebody
618@c %**end of header
619
620Now you can call a buddy. Obviously, your buddy will have to have GNUnet
621installed and must have performed the same steps. Also, you must have your buddy
622in your GNS master zone, for example by having imported your buddy's public key
623using @code{gnunet-qr}. Suppose your buddy is in your zone as @code{buddy.gnu}
624and he also created his phone using a label "home-phone". Then you can initiate
625a call using:@
626@code{@
627 /call home-phone.buddy.gnu@
628}@
629
630It may take some time for GNUnet to resolve the name and to establish a link. If
631your buddy has your public key in his master zone, he should see an incoming
632call with your name. If your public key is not in his master zone, he will just
633see the public key as the caller ID.
634
635Your buddy then can answer the call using the "/accept" command. After that,
636(encrypted) voice data should be relayed between your two peers. Either of you
637can end the call using "/cancel". You can exit @code{gnunet-converation} using
638"/quit".
639
640@node Future Directions
641@subsection Future Directions
642@c %**end of header
643
644Note that we do not envision people to use gnunet-conversation like this
645forever. We will write a graphical user interface, and that GUI will
646automatically create the necessary records in the respective zone.
647
648@node First steps: Using the GNUnet VPN
649@section First steps: Using the GNUnet VPN
650@c %**end of header
651
652
653@menu
654* Preliminaries2::
655* Exit configuration::
656* GNS configuration::
657* Accessing the service::
658* Using a Browser::
659@end menu
660
661@node Preliminaries2
662@subsection Preliminaries2
663@c %**end of header
664
665To test the GNUnet VPN, we should first run a web server. The easiest way to do
666this is to just start @code{gnunet-bcd}, which will run a webserver on port
667@code{8888} by default. Naturally, you can run some other HTTP server for our
668little tutorial.
669
670If you have not done this, you should also configure your Name System Service
671switch to use GNS. In your @code{/etc/nsswitch.conf} you should fine a line like
672this:
673@example
674hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4
675@end example
676
677The exact details may differ a bit, which is fine. Add the text
678@code{gns [NOTFOUND=return]} after @code{files}:
679@example
680hosts: files gns [NOTFOUND=return] mdns4_minimal [NOTFOUND=return] dns mdns4
681@end example
682
683
684You might want to make sure that @code{/lib/libnss_gns.so.2} exists on your
685system, it should have been created during the installation. If not, re-run
686@example
687$ configure --with-nssdir=/lib
688$ cd src/gns/nss; sudo make install
689@end example
690
691to install the NSS plugins in the proper location.
692
693@node Exit configuration
694@subsection Exit configuration
695@c %**end of header
696
697Stop your peer (as user @code{gnunet}, run @code{gnunet-arm -e}) and run
698@code{gnunet-setup}. In @code{gnunet-setup}, make sure to activate the
699@strong{EXIT} and @strong{GNS} services in the General tab. Then select the Exit
700tab. Most of the defaults should be fine (but you should check against the
701screenshot that they have not been modified). In the bottom area, enter
702@code{bcd} under Identifier and change the Destination to
703@code{169.254.86.1:8888} (if your server runs on a port other than 8888, change
704the 8888 port accordingly).
705
706Now exit @code{gnunet-setup} and restart your peer (@code{gnunet-arm -s}).
707
708@node GNS configuration
709@subsection GNS configuration
710@c %**end of header
711
712Now, using your normal user (not the @code{gnunet} system user), run
713@code{gnunet-gtk}. Select the GNS icon and add a new label www in your master
714zone. For the record type, select @code{VPN}. You should then see the VPN
715dialog:
716
717Under peer, you need to supply the peer identity of your own peer. You can
718obtain the respective string by running@
719@code{@
720 $ gnunet-peerinfo -sq@
721}@
722as the @code{gnunet} user. For the Identifier, you need to supply the same
723identifier that we used in the Exit setup earlier, so here supply "bcd". If you
724want others to be able to use the service, you should probably make the record
725public. For non-public services, you should use a passphrase instead of the
726string "bcd". Save the record and exit @code{gnunet-gtk}.
727
728@node Accessing the service
729@subsection Accessing the service
730@c %**end of header
731
732You should now be able to access your webserver. Type in:@
733@code{@
734 $ wget http://www.gnu/@
735}@
736The request will resolve to the VPN record, telling the GNS resolver to route it
737via the GNUnet VPN. The GNS resolver will ask the GNUnet VPN for an IPv4 address
738to return to the application. The VPN service will use the VPN information
739supplied by GNS to create a tunnel (via GNUnet's MESH service) to the EXIT peer.
740At the EXIT, the name "bcd" and destination port (80) will be mapped to the
741specified destination IP and port. While all this is currently happening on just
742the local machine, it should also work with other peers --- naturally, they will
743need a way to access your GNS zone first, for example by learning your public
744key from a QR code on your business card.
745
746@node Using a Browser
747@subsection Using a Browser
748@c %**end of header
749
750Sadly, modern browsers tend to bypass the Name Services Switch and attempt DNS
751resolution directly. You can either run a @code{gnunet-dns2gns} DNS proxy, or
752point the browsers to an HTTP proxy. When we tried it, Iceweasel did not like to
753connect to the socks proxy for @code{.gnu} TLDs, even if we disabled its
754autoblunder of changing @code{.gnu} to ".gnu.com". Still, using the HTTP proxy
755with Chrome does work.
756
757@node File-sharing
758@section File-sharing
759@c %**end of header
760
761This chapter documents the GNUnet file-sharing application. The original
762file-sharing implementation for GNUnet was designed to provide
763@strong{anonymous} file-sharing. However, over time, we have also added support
764for non-anonymous file-sharing (which can provide better performance). Anonymous
765and non-anonymous file-sharing are quite integrated in GNUnet and, except for
766routing, share most of the concepts and implementation. There are three primary
767file-sharing operations: publishing, searching and downloading. For each of
768these operations, the user specifies an @strong{anonymity level}. If both the
769publisher and the searcher/downloader specify "no anonymity", non-anonymous
770file-sharing is used. If either user specifies some desired degree of anonymity,
771anonymous file-sharing will be used.
772
773In this chapter, we will first look at the various concepts in GNUnet's
774file-sharing implementation. Then, we will discuss specifics as to how they
775impact users that publish, search or download files.
776
777
778
779@menu
780* File-sharing Concepts::
781* File-sharing Publishing::
782* File-sharing Searching::
783* File-sharing Downloading::
784* File-sharing Directories::
785* File-sharing Namespace Management::
786* File-Sharing URIs::
787@end menu
788
789@node File-sharing Concepts
790@subsection File-sharing Concepts
791@c %**end of header
792
793Sharing files in GNUnet is not quite as simple as in traditional file sharing
794systems. For example, it is not sufficient to just place files into a specific
795directory to share them. In addition to anonymous routing GNUnet attempts to
796give users a better experience in searching for content. GNUnet uses
797cryptography to safely break content into smaller pieces that can be obtained
798from different sources without allowing participants to corrupt files. GNUnet
799makes it difficult for an adversary to send back bogus search results. GNUnet
800enables content providers to group related content and to establish a
801reputation. Furthermore, GNUnet allows updates to certain content to be made
802available. This section is supposed to introduce users to the concepts that are
803used to achive these goals.
804
805
806@menu
807* Files::
808* Keywords::
809* Directories::
810* Pseudonyms::
811* Namespaces::
812* Advertisements::
813* Anonymity level::
814* Content Priority::
815* Replication::
816@end menu
817
818@node Files
819@subsubsection Files
820@c %**end of header
821
822A file in GNUnet is just a sequence of bytes. Any file-format is allowed and the
823maximum file size is theoretically 264 bytes, except that it would take an
824impractical amount of time to share such a file. GNUnet itself never interprets
825the contents of shared files, except when using GNU libextractor to obtain
826keywords.
827
828@node Keywords
829@subsubsection Keywords
830@c %**end of header
831
832Keywords are the most simple mechanism to find files on GNUnet. Keywords are
833@strong{case-sensitive} and the search string must always match @strong{exactly}
834the keyword used by the person providing the file. Keywords are never
835transmitted in plaintext. The only way for an adversary to determine the keyword
836that you used to search is to guess it (which then allows the adversary to
837produce the same search request). Since providing keywords by hand for each
838shared file is tedious, GNUnet uses GNU libextractor to help automate this
839process. Starting a keyword search on a slow machine can take a little while
840since the keyword search involves computing a fresh RSA key to formulate the
841request.
842
843@node Directories
844@subsubsection Directories
845@c %**end of header
846
847A directory in GNUnet is a list of file identifiers with meta data. The file
848identifiers provide sufficient information about the files to allow downloading
849the contents. Once a directory has been created, it cannot be changed since it
850is treated just like an ordinary file by the network. Small files (of a few
851kilobytes) can be inlined in the directory, so that a separate download becomes
852unnecessary.
853
854@node Pseudonyms
855@subsubsection Pseudonyms
856@c %**end of header
857
858Pseudonyms in GNUnet are essentially public-private (RSA) key pairs that allow a
859GNUnet user to maintain an identity (which may or may not be detached from his
860real-life identity). GNUnet's pseudonyms are not file-sharing specific --- and
861they will likely be used by many GNUnet applications where a user identity is
862required.
863
864Note that a pseudonym is NOT bound to a GNUnet peer. There can be multiple
865pseudonyms for a single user, and users could (theoretically) share the private
866pseudonym keys (currently only out-of-band by knowing which files to copy
867around).
868
869@node Namespaces
870@subsubsection Namespaces
871@c %**end of header
872
873A namespace is a set of files that were signed by the same pseudonym. Files (or
874directories) that have been signed and placed into a namespace can be updated.
875Updates are identified as authentic if the same secret key was used to sign the
876update. Namespaces are also useful to establish a reputation, since all of the
877content in the namespace comes from the same entity (which does not have to be
878the same person).
879
880@node Advertisements
881@subsubsection Advertisements
882@c %**end of header
883
884Advertisements are used to notify other users about the existence of a
885namespace. Advertisements are propagated using the normal keyword search. When
886an advertisement is received (in response to a search), the namespace is added
887to the list of namespaces available in the namespace-search dialogs of
888gnunet-fs-gtk and printed by gnunet-pseudonym. Whenever a namespace is created,
889an appropriate advertisement can be generated. The default keyword for the
890advertising of namespaces is "namespace".
891
892Note that GNUnet differenciates between your pseudonyms (the identities that you
893control) and namespaces. If you create a pseudonym, you will not automatically
894see the respective namespace. You first have to create an advertisement for the
895namespace and find it using keyword search --- even for your own namespaces. The
896@code{gnunet-pseudonym} tool is currently responsible for both managing
897pseudonyms and namespaces. This will likely change in the future to reduce the
898potential for confusion.
899
900@node Anonymity level
901@subsubsection Anonymity level
902@c %**end of header
903
904The anonymity level determines how hard it should be for an adversary to
905determine the identity of the publisher or the searcher/downloader. An
906anonymity level of zero means that anonymity is not required. The default
907anonymity level of "1" means that anonymous routing is desired, but no
908particular amount of cover traffic is necessary. A powerful adversary might thus
909still be able to deduce the origin of the traffic using traffic analysis.
910Specifying higher anonymity levels increases the amount of cover traffic
911required. While this offers better privacy, it can also significantly hurt
912performance.
913
914@node Content Priority
915@subsubsection Content Priority
916@c %**end of header
917
918Depending on the peer's configuration, GNUnet peers migrate content between
919peers. Content in this sense are individual blocks of a file, not necessarily
920entire files. When peers run out of space (due to local publishing operations or
921due to migration of content from other peers), blocks sometimes need to be
922discarded. GNUnet first always discards expired blocks (typically, blocks are
923published with an expiration of about two years in the future; this is another
924option). If there is still not enough space, GNUnet discards the blocks with the
925lowest priority. The priority of a block is decided by its popularity (in terms
926of requests from peers we trust) and, in case of blocks published locally, the
927base-priority that was specified by the user when the block was published
928initially.
929
930@node Replication
931@subsubsection Replication
932@c %**end of header
933
934When peers migrate content to other systems, the replication level of a block is
935used to decide which blocks need to be migrated most urgently. GNUnet will
936always push the block with the highest replication level into the network, and
937then decrement the replication level by one. If all blocks reach replication
938level zero, the selection is simply random.
939
940@node File-sharing Publishing
941@subsection File-sharing Publishing
942@c %**end of header
943
944The command @code{gnunet-publish} can be used to add content to the network.
945The basic format of the command is
946@example
947$ gnunet-publish [-n] [-k KEYWORDS]* [-m TYPE:VALUE] FILENAME
948@end example
949
950
951@menu
952* Important command-line options::
953* Indexing vs. Inserting::
954@end menu
955
956@node Important command-line options
957@subsubsection Important command-line options
958@c %**end of header
959
960The option -k is used to specify keywords for the file that should be inserted.
961You can supply any number of keywords, and each of the keywords will be
962sufficient to locate and retrieve the file.
963
964The -m option is used to specify meta-data, such as descriptions. You can use -m
965multiple times. The TYPE passed must be from the list of meta-data types known
966to libextractor. You can obtain this list by running @code{extract -L}.
967Use quotes around the entire meta-data argument if the value contains spaces.
968The meta-data is displayed to other users when they select which files to
969download. The meta-data and the keywords are optional and maybe inferred using
970@code{GNU libextractor}.
971
972gnunet-publish has a few additional options to handle namespaces and
973directories.
974See the man-page for details.
975
976@node Indexing vs. Inserting
977@subsubsection Indexing vs Inserting
978@c %**end of header
979
980By default, GNUnet indexes a file instead of making a full copy. This is much
981more efficient, but requries the file to stay unaltered at the location where it
982was when it was indexed. If you intend to move, delete or alter a file, consider
983using the option @code{-n} which will force GNUnet to make a copy of the file in
984the database.
985
986Since it is much less efficient, this is strongly discouraged for large files.
987When GNUnet indexes a file (default), GNUnet does @strong{not} create an
988additional encrypted copy of the file but just computes a summary (or index) of
989the file. That summary is approximately two percent of the size of the original
990file and is stored in GNUnet's database. Whenever a request for a part of an
991indexed file reaches GNUnet, this part is encrypted on-demand and send out. This
992way, there is no need for an additional encrypted copy of the file to stay
993anywhere on the drive. This is different from other systems, such as Freenet,
994where each file that is put online must be in Freenet's database in encrypted
995format, doubling the space requirements if the user wants to preseve a directly
996accessible copy in plaintext.
997
998Thus indexing should be used for all files where the user will keep using this
999file (at the location given to gnunet-publish) and does not want to retrieve it
1000back from GNUnet each time. If you want to remove a file that you have indexed
1001from the local peer, use the tool @code{gnunet-unindex} to un-index the file.
1002
1003The option @code{-n} may be used if the user fears that the file might be found
1004on his drive (assuming the computer comes under the control of an adversary).
1005When used with the @code{-n} flag, the user has a much better chance of denying
1006knowledge of the existence of the file, even if it is still (encrypted) on the
1007drive and the adversary is able to crack the encryption (e.g. by guessing the
1008keyword.
1009
1010@node File-sharing Searching
1011@subsection File-sharing Searching
1012@c %**end of header
1013
1014The command @code{gnunet-search} can be used to search for content on GNUnet.
1015The format is:
1016@example
1017$ gnunet-search [-t TIMEOUT] KEYWORD
1018@end example
1019
1020The -t option specifies that the query should timeout after approximately
1021TIMEOUT seconds. A value of zero is interpreted as @emph{no timeout}, which is
1022also the default. In this case, gnunet-search will never terminate (unless you
1023press CTRL-C).
1024
1025If multiple words are passed as keywords, they will all be considered optional.
1026Prefix keywords with a "+" to make them mandatory.
1027
1028Note that searching using
1029@example
1030$ gnunet-search Das Kapital
1031@end example
1032
1033is not the same as searching for
1034@example
1035$ gnunet-search "Das Kapital"
1036@end example
1037
1038as the first will match files shared under the keywords "Das" or "Kapital"
1039whereas the second will match files shared under the keyword "Das Kapital".
1040
1041Search results are printed by gnunet-search like this:
1042@example
1043$ gnunet-download -o "COPYING" --- gnunet://fs/chk/N8...C92.17992
1044=> The GNU Public License <= (mimetype: text/plain)
1045@end example
1046
1047The first line is the command you would have to enter to download the file.
1048The argument passed to @code{-o} is the suggested filename (you may change it to
1049whatever you like).
1050The @code{--} is followed by key for decrypting the file, the query for
1051searching the file, a checksum (in hexadecimal) finally the size of the file in
1052bytes.
1053The second line contains the description of the file; here this is
1054"The GNU Public License" and the mime-type (see the options for gnunet-publish
1055on how to specify these).
1056
1057@node File-sharing Downloading
1058@subsection File-sharing Downloading
1059@c %**end of header
1060
1061In order to download a file, you need the three values returned by
1062@code{gnunet-search}.
1063You can then use the tool @code{gnunet-download} to obtain the file:
1064@example
1065$ gnunet-download -o FILENAME --- GNUNETURL
1066@end example
1067
1068FILENAME specifies the name of the file where GNUnet is supposed to write the
1069result. Existing files are overwritten. If the existing file contains blocks
1070that are identical to the desired download, those blocks will not be downloaded
1071again (automatic resume).
1072
1073If you want to download the GPL from the previous example, you do the following:
1074@example
1075$ gnunet-download -o "COPYING" --- gnunet://fs/chk/N8...92.17992
1076@end example
1077
1078If you ever have to abort a download, you can continue it at any time by
1079re-issuing @code{gnunet-download} with the same filename. In that case, GNUnet
1080will @strong{not} download blocks again that are already present.
1081
1082GNUnet's file-encoding mechanism will ensure file integrity, even if the
1083existing file was not downloaded from GNUnet in the first place.
1084
1085You may want to use the @code{-V} switch (must be added before the @code{--}) to
1086turn on verbose reporting. In this case, @code{gnunet-download} will print the
1087current number of bytes downloaded whenever new data was received.
1088
1089@node File-sharing Directories
1090@subsection File-sharing Directories
1091@c %**end of header
1092
1093Directories are shared just like ordinary files. If you download a directory
1094with @code{gnunet-download}, you can use @code{gnunet-directory} to list its
1095contents. The canonical extension for GNUnet directories when stored as files in
1096your local file-system is ".gnd". The contents of a directory are URIs and
1097meta data.
1098The URIs contain all the information required by @code{gnunet-download} to
1099retrieve the file. The meta data typically includes the mime-type, description,
1100a filename and other meta information, and possibly even the full original file
1101(if it was small).
1102
1103@node File-sharing Namespace Management
1104@subsection File-sharing Namespace Management
1105@c %**end of header
1106
1107THIS TEXT IS OUTDATED AND NEEDS TO BE REWRITTEN FOR 0.10!
1108
1109The gnunet-pseudonym tool can be used to create pseudonyms and to advertise
1110namespaces. By default, gnunet-pseudonym simply lists all locally available
1111pseudonyms.
1112
1113
1114@menu
1115* Creating Pseudonyms::
1116* Deleting Pseudonyms::
1117* Advertising namespaces::
1118* Namespace names::
1119* Namespace root::
1120@end menu
1121
1122@node Creating Pseudonyms
1123@subsubsection Creating Pseudonyms
1124@c %**end of header
1125
1126With the @code{-C NICK} option it can also be used to create a new pseudonym.
1127A pseudonym is the virtual identity of the entity in control of a namespace.
1128Anyone can create any number of pseudonyms. Note that creating a pseudonym can
1129take a few minutes depending on the performance of the machine used.
1130
1131@node Deleting Pseudonyms
1132@subsubsection Deleting Pseudonyms
1133@c %**end of header
1134
1135With the @code{-D NICK} option pseudonyms can be deleted. Once the pseudonym has
1136been deleted it is impossible to add content to the corresponding namespace.
1137Deleting the pseudonym does not make the namespace or any content in it
1138unavailable.
1139
1140@node Advertising namespaces
1141@subsubsection Advertising namespaces
1142@c %**end of header
1143
1144Each namespace is associated with meta-data that describes the namespace.
1145This meta data is provided by the user at the time that the namespace is
1146advertised. Advertisements are published under keywords so that they can be
1147found using normal keyword-searches. This way, users can learn about new
1148namespaces without relying on out-of-band communication or directories.
1149A suggested keyword to use for all namespaces is simply "namespace".
1150When a keyword-search finds a namespace advertisement, it is automatically
1151stored in a local list of known namespaces. Users can then associate a rank with
1152the namespace to remember the quality of the content found in it.
1153
1154@node Namespace names
1155@subsubsection Namespace names
1156@c %**end of header
1157
1158While the namespace is uniquely identified by its ID, another way to refer to
1159the namespace is to use the NICKNAME. The NICKNAME can be freely chosen by the
1160creator of the namespace and hence conflicts are possible. If a GNUnet client
1161learns about more than one namespace using the same NICKNAME, the ID is appended
1162to the NICKNAME to get a unique identifier.
1163
1164@node Namespace root
1165@subsubsection Namespace root
1166@c %**end of header
1167
1168An item of particular interest in the namespace advertisement is the ROOT.
1169The ROOT is the identifier of a designated entry in the namespace. The idea is
1170that the ROOT can be used to advertise an entry point to the content of the
1171namespace.
1172
1173@node File-Sharing URIs
1174@subsection File-Sharing URIs
1175@c %**end of header
1176
1177GNUnet (currently) uses four different types of URIs for file-sharing. They all
1178begin with "gnunet://fs/". This section describes the four different URI types
1179in detail.
1180
1181
1182@menu
1183* Encoding of hash values in URIs::
1184* Content Hash Key (chk)::
1185* Location identifiers (loc)::
1186* Keyword queries (ksk)::
1187* Namespace content (sks)::
1188@end menu
1189
1190@node Encoding of hash values in URIs
1191@subsubsection Encoding of hash values in URIs
1192@c %**end of header
1193
1194Most URIs include some hash values. Hashes are encoded using base32hex
1195(RFC 2938).
1196
1197@node Content Hash Key (chk)
1198@subsubsection Content Hash Key (chk)
1199@c %**end of header
1200
1201A chk-URI is used to (uniquely) identify a file or directory and to allow peers
1202to download the file. Files are stored in GNUnet as a tree of encrypted blocks.
1203The chk-URI thus contains the information to download and decrypt those blocks.
1204A chk-URI has the format "gnunet://fs/chk/KEYHASH.QUERYHASH.SIZE". Here, "SIZE"
1205is the size of the file (which allows a peer to determine the shape of the
1206tree), KEYHASH is the key used to decrypt the file (also the hash of the
1207plaintext of the top block) and QUERYHASH is the query used to request the
1208top-level block (also the hash of the encrypted block).
1209
1210@node Location identifiers (loc)
1211@subsubsection Location identifiers (loc)
1212@c %**end of header
1213
1214For non-anonymous file-sharing, loc-URIs are used to specify which peer is
1215offering the data (in addition to specifying all of the data from a chk-URI).
1216Location identifiers include a digital signature of the peer to affirm that the
1217peer is truly the origin of the data. The format is
1218"gnunet://fs/loc/KEYHASH.QUERYHASH.SIZE.PEER.SIG.EXPTIME". Here, "PEER" is the
1219public key of the peer (in GNUnet format in base32hex), SIG is the RSA signature
1220(in GNUnet format in base32hex) and EXPTIME specifies when the signature expires
1221(in milliseconds after 1970).
1222
1223@node Keyword queries (ksk)
1224@subsubsection Keyword queries (ksk)
1225@c %**end of header
1226
1227A keyword-URI is used to specify that the desired operation is the search using
1228a particular keyword. The format is simply "gnunet://fs/ksk/KEYWORD". Non-ASCII
1229characters can be specified using the typical URI-encoding (using hex values)
1230from HTTP. "+" can be used to specify multiple keywords (which are then
1231logically "OR"-ed in the search, results matching both keywords are given a
1232higher rank): "gnunet://fs/ksk/KEYWORD1+KEYWORD2".
1233
1234@node Namespace content (sks)
1235@subsubsection Namespace content (sks)
1236@c %**end of header
1237
1238Namespaces are sets of files that have been approved by some (usually
1239pseudonymous) user --- typically by that user publishing all of the files
1240together. A file can be in many namespaces. A file is in a namespace if the
1241owner of the ego (aka the namespace's private key) signs the CHK of the file
1242cryptographically. An SKS-URI is used to search a namespace. The result is a
1243block containing meta data, the CHK and the namespace owner's signature. The
1244format of a sks-URI is "gnunet://fs/sks/NAMESPACE/IDENTIFIER". Here, "NAMESPACE"
1245is the public key for the namespace. "IDENTIFIER" is a freely chosen keyword
1246(or password!). A commonly used identifier is "root" which by convention refers
1247to some kind of index or other entry point into the namespace.
1248
1249@node The GNU Name System
1250@section The GNU Name System
1251@c %**end of header
1252
1253
1254The GNU Name System (GNS) is secure and decentralized naming system.
1255It allows its users to resolve and register names within the @code{.gnu}
1256top-level domain (TLD).
1257
1258GNS is designed to provide:
1259@itemize @bullet
1260@item Censorship resistance
1261@item Query privacy
1262@item Secure name resolution
1263@item Compatibility with DNS
1264@end itemize
1265
1266For the initial configuration and population of your GNS installation, please
1267follow the GNS setup instructions. The remainder of this chapter will provide
1268some background on GNS and then describe how to use GNS in more detail.
1269
1270Unlike DNS, GNS does not rely on central root zones or authorities. Instead any
1271user administers his own root and can can create arbitrary name value mappings.
1272Furthermore users can delegate resolution to other users' zones just like DNS NS
1273records do. Zones are uniquely identified via public keys and resource records
1274are signed using the corresponding public key. Delegation to another user's zone
1275is done using special PKEY records and petnames. A petname is a name that can be
1276freely chosen by the user. This results in non-unique name-value mappings as
1277@code{@uref{http://www.bob.gnu/, www.bob.gnu}} to one user might be
1278@code{@uref{http://www.friend.gnu/, www.friend.gnu}} for someone else.
1279
1280
1281@menu
1282* Maintaining your own Zones::
1283* Obtaining your Zone Key::
1284* Adding Links to Other Zones::
1285* The Three Local Zones of GNS::
1286* The Master Zone::
1287* The Private Zone::
1288* The Shorten Zone::
1289* The ZKEY Top Level Domain in GNS::
1290* Resource Records in GNS::
1291@end menu
1292
1293
1294@node Maintaining your own Zones
1295@subsection Maintaining your own Zones
1296
1297To setup you GNS system you must execute:@
1298@code{$ gnunet-gns-import.sh}
1299
1300This will boostrap your zones and create the necessary key material.
1301Your keys can be listed using the gnunet-identity command line tool:@
1302@code{$ gnunet-identity -d}@
1303You can arbitrarily create your own zones using the gnunet-identity tool using:@
1304@code{$ gnunet-identity -C "new_zone"}@
1305
1306Now you can add (or edit, or remove) records in your GNS zone using the
1307gnunet-setup GUI or using the gnunet-namestore command-line tool. In either
1308case, your records will be stored in an SQL database under control of the
1309gnunet-service-namestore. Note that if mutliple users use one peer, the
1310namestore database will include the combined records of all users. However,
1311users will not be able to see each other's records if they are marked as
1312private.
1313
1314To provide a simple example for editing your own zone, suppose you have your own
1315web server with IP 1.2.3.4. Then you can put an A record (A records in DNS are
1316for IPv4 IP addresses) into your local zone using the command:@
1317@code{$ gnunet-namestore -z master-zone -a -n www -t A -V 1.2.3.4 -e never}@
1318Afterwards, you will be able to access your webpage under "www.gnu" (assuming
1319your webserver does not use virtual hosting, if it does, please read up on
1320setting up the GNS proxy).
1321
1322Similar commands will work for other types of DNS and GNS records, the syntax
1323largely depending on the type of the record. Naturally, most users may find
1324editing the zones using the gnunet-setup GUI to be easier.
1325
1326@node Obtaining your Zone Key
1327@subsection Obtaining your Zone Key
1328
1329Each zone in GNS has a public-private key. Usually, gnunet-namestore and
1330gnunet-setup will access your private key as necessary, so you do not have to
1331worry about those. What is important is your public key (or rather, the hash of
1332your public key), as you will likely want to give it to others so that they can
1333securely link to you.
1334
1335You can usually get the hash of your public key using@
1336@code{$ gnunet-identity -d $options | grep master-zone | awk '@{print $3@}'}@
1337For example, the output might be something like@
1338DC3SEECJORPHQNVRH965A6N74B1M37S721IG4RBQ15PJLLPJKUE0.
1339
1340Alternatively, you can obtain a QR code with your zone key AND your pseudonym
1341from gnunet-gtk. The QR code is displayed in the GNS tab and can be stored to
1342disk using the Save as button next to the image.
1343
1344@node Adding Links to Other Zones
1345@subsection Adding Links to Other Zones
1346
1347
1348A central operation in GNS is the ability to securely delegate to other zones.
1349Basically, by adding a delegation you make all of the names from the other zone
1350available to yourself. This section describes how to create delegations.
1351
1352Suppose you have a friend who you call 'bob' who also uses GNS. You can then
1353delegate resolution of names to Bob's zone by adding a PKEY record to his local
1354zone:@
1355@code{$ gnunet-namestore -a -n bob --type PKEY -V XXXX -e never}@
1356Note that XXXX in the command above must be replaced with the hash of Bob's
1357public key (the output your friend obtained using the gnunet-identity command
1358from the previous section and told you, for example by giving you a business
1359card containing this information as a QR code).
1360
1361Assuming Bob has an A record for his website under the name of www in his zone,
1362you can then access Bob's website under www.bob.gnu --- as well as any (public)
1363GNS record that Bob has in his zone by replacing www with the respective name of
1364the record in Bob's zone.
1365
1366Furthermore, if Bob has himself a (public) delegation to Carol's zone under
1367"carol", you can access Carol's records under NAME.carol.bob.gnu (where NAME is
1368the name of Carol's record you want to access).
1369
1370@node The Three Local Zones of GNS
1371@subsection The Three Local Zones of GNS
1372
1373Each user GNS has control over three zones. Each of the zones has a different
1374purpose. These zones are the
1375@itemize @bullet
1376
1377@item
1378master zone,
1379@item
1380private zone, and the
1381@item
1382shorten zone.
1383@end itemize
1384
1385@node The Master Zone
1386@subsection The Master Zone
1387
1388
1389The master zone is your personal TLD. Names within the @code{.gnu} namespace are
1390resolved relative to this zone. You can arbitrarily add records to this zone and
1391selectively publish those records.
1392
1393@node The Private Zone
1394@subsection The Private Zone
1395
1396
1397The private zone is a subzone (or subdomain in DNS terms) of your master zone.
1398It should be used for records that you want to keep private. For example
1399@code{bank.private.gnu}. The key idea is that you want to keep your private
1400records separate, if just to know that those names are not available to other
1401users.
1402
1403@node The Shorten Zone
1404@subsection The Shorten Zone
1405
1406
1407The shorten zone can either be a subzone of the master zone or the private zone.
1408It is different from the other zones in that GNS will automatically populate
1409this zone with other users' zones based on their PSEU records whenever you
1410resolve a name.
1411
1412For example if you go to
1413@code{@uref{http://www.bob.alice.dave.gnu/, www.bob.alice.dave.gnu}}, GNS will
1414try to import @code{bob} into your shorten zone. Having obtained Bob's PKEY from
1415@code{alice.dave.gnu}, GNS will lookup the PSEU record for @code{+} in Bob's
1416zone. If it exists and the specified pseudonym is not taken, Bob's PKEY will be
1417automatically added under that pseudonym (i.e. "bob") into your shorten zone.
1418From then on, Bob's webpage will also be available for you as
1419@code{@uref{http://www.bob.short.gnu/, www.bob.short.gnu}}. This feature is
1420called automatic name shortening and is supposed to keep GNS names as short and
1421memorable as possible.
1422
1423@node The ZKEY Top Level Domain in GNS
1424@subsection The ZKEY Top Level Domain in GNS
1425
1426
1427GNS also provides a secure and globally unique namespace under the .zkey
1428top-level domain. A name in the .zkey TLD corresponds to the (printable) public
1429key of a zone. Names in the .zkey TLD are then resolved by querying the
1430respective zone. The .zkey TLD is expected to be used under rare circumstances
1431where globally unique names are required and for integration with legacy
1432systems.
1433
1434@node Resource Records in GNS
1435@subsection Resource Records in GNS
1436
1437
1438GNS supports the majority of the DNS records as defined in
1439@uref{http://www.ietf.org/rfc/rfc1035.txt, RFC 1035}. Additionally, GNS defines
1440some new record types the are unique to the GNS system. For example,
1441GNS-specific resource records are use to give petnames for zone delegation,
1442revoke zone keys and provide some compatibility features.
1443
1444For some DNS records, GNS does extended processing to increase their usefulness
1445in GNS. In particular, GNS introduces special names referred to as
1446"zone relative names". Zone relative names are allowed in some resource record
1447types (for example, in NS and CNAME records) and can also be used in links on
1448webpages. Zone relative names end in ".+" which indicates that the name needs to
1449be resolved relative to the current authoritative zone. The extended processing
1450of those names will expand the ".+" with the correct delegation chain to the
1451authoritative zone (replacing ".+" with the name of the location where the name
1452was encountered) and hence generate a valid @code{.gnu} name.
1453
1454GNS currently supports the following record types:
1455
1456@menu
1457* NICK::
1458* PKEY::
1459* BOX::
1460* LEHO::
1461* VPN::
1462* A AAAA and TXT::
1463* CNAME::
1464* GNS2DNS::
1465* SOA SRV PTR and MX::
1466@end menu
1467
1468@node NICK
1469@subsubsection NICK
1470
1471A NICK record is used to give a zone a name. With a NICK record, you can
1472essentially specify how you would like to be called. GNS expects this record
1473under the name "+" in the zone's database (NAMESTORE); however, it will then
1474automatically be copied into each record set, so that clients never need to do a
1475separate lookup to discover the NICK record.
1476
1477@b{Example}@
1478
1479@example
1480Name: +; RRType: NICK; Value: bob
1481@end example
1482
1483This record in Bob's zone will tell other users that this zone wants to be
1484referred to as 'bob'. Note that nobody is obliged to call Bob's zone 'bob' in
1485their own zones. It can be seen as a recommendation ("Please call me 'bob'").
1486
1487@node PKEY
1488@subsubsection PKEY
1489
1490PKEY records are used to add delegation to other users' zones and give those
1491zones a petname.
1492
1493@b{Example}@
1494
1495Let Bob's zone be identified by the hash "ABC012". Bob is your friend so you
1496want to give him the petname "friend". Then you add the following record to your
1497zone:
1498
1499@example
1500Name: friend; RRType: PKEY; Value: ABC012;
1501@end example
1502
1503This will allow you to resolve records in bob's zone under "*.friend.gnu".
1504
1505@node BOX
1506@subsubsection BOX
1507
1508BOX records are there to integrate information from TLSA or SRV records under
1509the main label. In DNS, TLSA and SRV records use special names of the form
1510@code{_port._proto.(label.)*tld} to indicate the port number and protocol
1511(i.e. tcp or udp) for which the TLSA or SRV record is valid. This causes various
1512problems, and is elegantly solved in GNS by integrating the protocol and port
1513numbers together with the respective value into a "BOX" record. Note that in the
1514GUI, you do not get to edit BOX records directly right now --- the GUI will
1515provide the illusion of directly editing the TLSA and SRV records, even though
1516they internally are BOXed up.
1517
1518@node LEHO
1519@subsubsection LEHO
1520
1521The LEgacy HOstname of a server. Some webservers expect a specific hostname to
1522provide a service (virtiual hosting). Also SSL certificates usually contain DNS
1523names. To provide the expected legacy DNS name for a server, the LEHO record can
1524be used. To mitigate the just mentioned issues the GNS proxy has to be used. The
1525GNS proxy will use the LEHO information to apply the necessary transformations.
1526
1527@node VPN
1528@subsubsection VPN
1529
1530GNS allows easy access to services provided by the GNUnet Virtual Public
1531Network. When the GNS resolver encounters a VPN record it will contact the VPN
1532service to try and allocate an IPv4/v6 address (if the queries record type is an
1533IP address) that can be used to contact the service.
1534
1535@b{Example}@
1536
1537I want to provide access to the VPN service "web.gnu." on port 80 on peer
1538ABC012:@
1539Name: www; RRType: VPN; Value: 80 ABC012 web.gnu.
1540
1541The peer ABC012 is configured to provide an exit point for the service
1542"web.gnu." on port 80 to it's server running locally on port 8080 by having the
1543following lines in the @code{gnunet.conf} configuration file:@
1544@code{@
1545 [web.gnunet.]@
1546 TCP_REDIRECTS = 80:localhost4:8080@
1547}
1548
1549@node A AAAA and TXT
1550@subsubsection A AAAA and TXT
1551
1552Those records work in exactly the same fashion as in traditional DNS.
1553
1554@node CNAME
1555@subsubsection CNAME
1556
1557As specified in RFC 1035 whenever a CNAME is encountered the query needs to be
1558restarted with the specified name. In GNS a CNAME can either be:
1559
1560@itemize @bullet
1561@item
1562A zone relative name,
1563@item
1564A zkey name or
1565@item
1566A DNS name (in which case resolution will continue outside of GNS with the systems DNS resolver)
1567@end itemize
1568
1569@node GNS2DNS
1570@subsubsection GNS2DNS
1571
1572GNS can delegate authority to a legacy DNS zone. For this, the name of the DNS
1573nameserver and the name of the DNS zone are specified in a GNS2DNS record.
1574
1575@b{Example}
1576
1577@example
1578Name: pet; RRType: GNS2DNS; Value: gnunet.org@@a.ns.joker.com
1579@end example
1580
1581Any query to @code{pet.gnu} will then be delegated to the DNS server at
1582@code{a.ns.joker.com}.@
1583For example, @code{@uref{http://www.pet.gnu/, www.pet.gnu}} will result in a
1584DNS query for @code{@uref{http://www.gnunet.org/, www.gnunet.org}} to the server
1585at @code{a.ns.joker.com}. Delegation to DNS via NS records in GNS can be useful
1586if you do not want to start resolution in the DNS root zone (due to issues such
1587as censorship or availability).
1588
1589Note that you would typically want to use a relative name for the nameserver,
1590i.e.
1591@example
1592Name: pet; RRType: GNS2DNS; Value: gnunet.org@@ns-joker.+@
1593Name: ns-joker; RRType: A; Value: 184.172.157.218
1594@end example
1595
1596This way, you can avoid involving the DNS hierarchy in the resolution of
1597@code{a.ns.joker.com}. In the example above, the problem may not be obvious as
1598the nameserver for "gnunet.org" is in the ".com" zone. However, imagine the
1599nameserver was "ns.gnunet.org". In this case, delegating to "ns.gnunet.org"
1600would mean that despite using GNS, censorship in the DNS ".org" zone would still
1601be effective.
1602
1603@node SOA SRV PTR and MX
1604@subsubsection SOA SRV PTR and MX
1605
1606The domain names in those records can, again, be either
1607@itemize @bullet
1608@item
1609A zone relative name,
1610@item
1611A zkey name or
1612@item
1613A DNS name
1614@end itemize
1615
1616The resolver will expand the zone relative name if possible. Note that when
1617using MX records within GNS, the target mail server might still refuse to accept
1618e-mails to the resulting domain as the name might not match. GNS-enabled mail
1619clients should use the ZKEY zone as the destination hostname and GNS-enabled
1620mail servers should be configured to accept e-mails to the ZKEY-zones of all
1621local users.
1622
1623@node Using the Virtual Public Network
1624@section Using the Virtual Public Network
1625
1626@menu
1627* Setting up an Exit node::
1628* Fedora and the Firewall::
1629* Setting up VPN node for protocol translation and tunneling::
1630@end menu
1631
1632Using the GNUnet Virtual Public Network (VPN) application you can tunnel IP
1633traffic over GNUnet. Moreover, the VPN comes with built-in protocol translation
1634and DNS-ALG support, enabling IPv4-to-IPv6 protocol translation
1635(in both directions). This chapter documents how to use the GNUnet VPN.
1636
1637The first thing to note about the GNUnet VPN is that it is a public network. All
1638participating peers can participate and there is no secret key to control
1639access. So unlike common virtual private networks, the GNUnet VPN is not useful
1640as a means to provide a "private" network abstraction over the Internet. The
1641GNUnet VPN is a virtual network in the sense that it is an overlay over the
1642Internet, using its own routing mechanisms and can also use an internal
1643addressing scheme. The GNUnet VPN is an Internet underlay --- TCP/IP
1644applications run on top of it.
1645
1646The VPN is currently only supported on GNU/Linux systems. Support for operating
1647systems that support TUN (such as FreeBSD) should be easy to add (or might not
1648even require any coding at all --- we just did not test this so far). Support
1649for other operating systems would require re-writing the code to create virtual
1650network interfaces and to intercept DNS requests.
1651
1652The VPN does not provide good anonymity. While requests are routed over the
1653GNUnet network, other peers can directly see the source and destination of each
1654(encapsulated) IP packet. Finally, if you use the VPN to access Internet
1655services, the peer sending the request to the Internet will be able to observe
1656and even alter the IP traffic. We will discuss additional security implications
1657of using the VPN later in this chapter.
1658
1659@node Setting up an Exit node
1660@subsection Setting up an Exit node
1661
1662Any useful operation with the VPN requires the existence of an exit node in the
1663GNUnet Peer-to-Peer network. Exit functionality can only be enabled on peers
1664that have regular Internet access. If you want to play around with the VPN or
1665support the network, we encourage you to setup exit nodes. This chapter
1666documents how to setup an exit node.
1667
1668There are four types of exit functions an exit node can provide, and using the
1669GNUnet VPN to access the Internet will only work nicely if the first three types
1670are provided somewhere in the network. The four exit functions are:
1671@itemize @bullet
1672@item
1673DNS: allow other peers to use your DNS resolver
1674@item
1675IPv4: allow other peers to access your IPv4 Internet connection
1676@item
1677IPv6: allow other peers to access your IPv6 Internet connection
1678@item
1679Local service: allow other peers to access a specific TCP or UDP service your peer is providing
1680@end itemize
1681
1682By enabling "exit" in gnunet-setup and checking the respective boxes in the
1683"exit" tab, you can easily choose which of the above exit functions you want to
1684support.
1685
1686Note, however, that by supporting the first three functions you will allow
1687arbitrary other GNUnet users to access the Internet via your system. This is
1688somewhat similar to running a Tor exit node. The torproject has a nice article
1689about what to consider if you want to do this here. We believe that generally
1690running a DNS exit node is completely harmless.
1691
1692The exit node configuration does currently not allow you to restrict the
1693Internet traffic that leaves your system. In particular, you cannot exclude SMTP
1694traffic (or block port 25) or limit to HTTP traffic using the GNUnet
1695configuration. However, you can use your host firewall to restrict outbound
1696connections from the virtual tunnel interface. This is highly recommended. In
1697the future, we plan to offer a wider range of configuration options for exit
1698nodes.
1699
1700Note that by running an exit node GNUnet will configure your kernel to perform
1701IP-forwarding (for IPv6) and NAT (for IPv4) so that the traffic from the virtual
1702interface can be routed to the Internet. In order to provide an IPv6-exit, you
1703need to have a subnet routed to your host's external network interface and
1704assign a subrange of that subnet to the GNUnet exit's TUN interface.
1705
1706When running a local service, you should make sure that the local service is
1707(also) bound to the IP address of your EXIT interface (i.e. 169.254.86.1). It
1708will NOT work if your local service is just bound to loopback. You may also want
1709to create a "VPN" record in your zone of the GNU Name System to make it easy for
1710others to access your service via a name instead of just the full service
1711descriptor. Note that the identifier you assign the service can serve as a
1712passphrase or shared secret, clients connecting to the service must somehow
1713learn the service's name. VPN records in the GNU Name System can make this
1714easier.
1715
1716@node Fedora and the Firewall
1717@subsection Fedora and the Firewall
1718
1719
1720When using an exit node on Fedora 15, the standard firewall can create trouble
1721even when not really exiting the local system! For IPv4, the standard rules seem
1722fine. However, for IPv6 the standard rules prohibit traffic from the network
1723range of the virtual interface created by the exit daemon to the local IPv6
1724address of the same interface (which is essentially loopback traffic, so you
1725might suspect that a standard firewall would leave this traffic alone). However,
1726as somehow for IPv6 the traffic is not recognized as originating from the local
1727system (and as the connection is not already "established"), the firewall drops
1728the traffic. You should still get ICMPv6 packets back, but that's obviously not
1729very useful.
1730
1731Possible ways to fix this include disabling the firewall (do you have a good
1732reason for having it on?) or disabling the firewall at least for the GNUnet exit
1733interface (or the respective IPv4/IPv6 address range). The best way to diagnose
1734these kinds of problems in general involves setting the firewall to REJECT
1735instead of DROP and to watch the traffic using wireshark (or tcpdump) to see if
1736ICMP messages are generated when running some tests that should work.
1737
1738@node Setting up VPN node for protocol translation and tunneling
1739@subsection Setting up VPN node for protocol translation and tunneling
1740
1741
1742The GNUnet VPN/PT subsystem enables you to tunnel IP traffic over the VPN to an
1743exit node, from where it can then be forwarded to the Internet. This section
1744documents how to setup VPN/PT on a node. Note that you can enable both the VPN
1745and an exit on the same peer. In this case, IP traffic from your system may
1746enter your peer's VPN and leave your peer's exit. This can be useful as a means
1747to do protocol translation. For example, you might have an application that
1748supports only IPv4 but needs to access an IPv6-only site. In this case, GNUnet
1749would perform 4to6 protocol translation between the VPN (IPv4) and the
1750Exit (IPv6). Similarly, 6to4 protocol translation is also possible. However, the
1751primary use for GNUnet would be to access an Internet service running with an
1752IP version that is not supported by your ISP. In this case, your IP traffic
1753would be routed via GNUnet to a peer that has access to the Internet with the
1754desired IP version.
1755
1756Setting up an entry node into the GNUnet VPN primarily requires you to enable
1757the "VPN/PT" option in "gnunet-setup". This will launch the
1758"gnunet-service-vpn", "gnunet-service-dns" and "gnunet-daemon-pt" processes.
1759The "gnunet-service-vpn" will create a virtual interface which will be used as
1760the target for your IP traffic that enters the VPN. Additionally, a second
1761virtual interface will be created by the "gnunet-service-dns" for your DNS
1762traffic. You will then need to specify which traffic you want to tunnel over
1763GNUnet. If your ISP only provides you with IPv4 or IPv6-access, you may choose
1764to tunnel the other IP protocol over the GNUnet VPN. If you do not have an ISP
1765(and are connected to other GNUnet peers via WLAN), you can also choose to
1766tunnel all IP traffic over GNUnet. This might also provide you with some
1767anonymity. After you enable the respective options and restart your peer, your
1768Internet traffic should be tunneled over the GNUnet VPN.
1769
1770The GNUnet VPN uses DNS-ALG to hijack your IP traffic. Whenever an application
1771resolves a hostname (i.e. 'gnunet.org'), the "gnunet-daemon-pt" will instruct
1772the "gnunet-service-dns" to intercept the request (possibly route it over GNUnet
1773as well) and replace the normal answer with an IP in the range of the VPN's
1774interface. "gnunet-daemon-pt" will then tell "gnunet-service-vpn" to forward all
1775traffic it receives on the TUN interface via the VPN to the original
1776destination.
1777
1778For applications that do not use DNS, you can also manually create such a
1779mapping using the gnunet-vpn command-line tool. Here, you specfiy the desired
1780address family of the result (i.e. "-4"), and the intended target IP on the
1781Internet ("-i 131.159.74.67") and "gnunet-vpn" will tell you which IP address in
1782the range of your VPN tunnel was mapped.
1783
1784gnunet-vpn can also be used to access "internal" services offered by GNUnet
1785nodes. So if you happen to know a peer and a service offered by that peer, you
1786can create an IP tunnel to that peer by specifying the peer's identity, service
1787name and protocol (--tcp or --udp) and you will again receive an IP address that
1788will terminate at the respective peer's service.
diff --git a/doc/fdl-1.3.texi b/doc/fdl-1.3.texi
new file mode 100644
index 000000000..cb71f05a1
--- /dev/null
+++ b/doc/fdl-1.3.texi
@@ -0,0 +1,505 @@
1@c The GNU Free Documentation License.
2@center Version 1.3, 3 November 2008
3
4@c This file is intended to be included within another document,
5@c hence no sectioning command or @node.
6
7@display
8Copyright @copyright{} 2000, 2001, 2002, 2007, 2008 Free Software Foundation, Inc.
9@uref{http://fsf.org/}
10
11Everyone is permitted to copy and distribute verbatim copies
12of this license document, but changing it is not allowed.
13@end display
14
15@enumerate 0
16@item
17PREAMBLE
18
19The purpose of this License is to make a manual, textbook, or other
20functional and useful document @dfn{free} in the sense of freedom: to
21assure everyone the effective freedom to copy and redistribute it,
22with or without modifying it, either commercially or noncommercially.
23Secondarily, this License preserves for the author and publisher a way
24to get credit for their work, while not being considered responsible
25for modifications made by others.
26
27This License is a kind of ``copyleft'', which means that derivative
28works of the document must themselves be free in the same sense. It
29complements the GNU General Public License, which is a copyleft
30license designed for free software.
31
32We have designed this License in order to use it for manuals for free
33software, because free software needs free documentation: a free
34program should come with manuals providing the same freedoms that the
35software does. But this License is not limited to software manuals;
36it can be used for any textual work, regardless of subject matter or
37whether it is published as a printed book. We recommend this License
38principally for works whose purpose is instruction or reference.
39
40@item
41APPLICABILITY AND DEFINITIONS
42
43This License applies to any manual or other work, in any medium, that
44contains a notice placed by the copyright holder saying it can be
45distributed under the terms of this License. Such a notice grants a
46world-wide, royalty-free license, unlimited in duration, to use that
47work under the conditions stated herein. The ``Document'', below,
48refers to any such manual or work. Any member of the public is a
49licensee, and is addressed as ``you''. You accept the license if you
50copy, modify or distribute the work in a way requiring permission
51under copyright law.
52
53A ``Modified Version'' of the Document means any work containing the
54Document or a portion of it, either copied verbatim, or with
55modifications and/or translated into another language.
56
57A ``Secondary Section'' is a named appendix or a front-matter section
58of the Document that deals exclusively with the relationship of the
59publishers or authors of the Document to the Document's overall
60subject (or to related matters) and contains nothing that could fall
61directly within that overall subject. (Thus, if the Document is in
62part a textbook of mathematics, a Secondary Section may not explain
63any mathematics.) The relationship could be a matter of historical
64connection with the subject or with related matters, or of legal,
65commercial, philosophical, ethical or political position regarding
66them.
67
68The ``Invariant Sections'' are certain Secondary Sections whose titles
69are designated, as being those of Invariant Sections, in the notice
70that says that the Document is released under this License. If a
71section does not fit the above definition of Secondary then it is not
72allowed to be designated as Invariant. The Document may contain zero
73Invariant Sections. If the Document does not identify any Invariant
74Sections then there are none.
75
76The ``Cover Texts'' are certain short passages of text that are listed,
77as Front-Cover Texts or Back-Cover Texts, in the notice that says that
78the Document is released under this License. A Front-Cover Text may
79be at most 5 words, and a Back-Cover Text may be at most 25 words.
80
81A ``Transparent'' copy of the Document means a machine-readable copy,
82represented in a format whose specification is available to the
83general public, that is suitable for revising the document
84straightforwardly with generic text editors or (for images composed of
85pixels) generic paint programs or (for drawings) some widely available
86drawing editor, and that is suitable for input to text formatters or
87for automatic translation to a variety of formats suitable for input
88to text formatters. A copy made in an otherwise Transparent file
89format whose markup, or absence of markup, has been arranged to thwart
90or discourage subsequent modification by readers is not Transparent.
91An image format is not Transparent if used for any substantial amount
92of text. A copy that is not ``Transparent'' is called ``Opaque''.
93
94Examples of suitable formats for Transparent copies include plain
95ASCII without markup, Texinfo input format, La@TeX{} input
96format, SGML or XML using a publicly available
97DTD, and standard-conforming simple HTML,
98PostScript or PDF designed for human modification. Examples
99of transparent image formats include PNG, XCF and
100JPG. Opaque formats include proprietary formats that can be
101read and edited only by proprietary word processors, SGML or
102XML for which the DTD and/or processing tools are
103not generally available, and the machine-generated HTML,
104PostScript or PDF produced by some word processors for
105output purposes only.
106
107The ``Title Page'' means, for a printed book, the title page itself,
108plus such following pages as are needed to hold, legibly, the material
109this License requires to appear in the title page. For works in
110formats which do not have any title page as such, ``Title Page'' means
111the text near the most prominent appearance of the work's title,
112preceding the beginning of the body of the text.
113
114The ``publisher'' means any person or entity that distributes copies
115of the Document to the public.
116
117A section ``Entitled XYZ'' means a named subunit of the Document whose
118title either is precisely XYZ or contains XYZ in parentheses following
119text that translates XYZ in another language. (Here XYZ stands for a
120specific section name mentioned below, such as ``Acknowledgements'',
121``Dedications'', ``Endorsements'', or ``History''.) To ``Preserve the Title''
122of such a section when you modify the Document means that it remains a
123section ``Entitled XYZ'' according to this definition.
124
125The Document may include Warranty Disclaimers next to the notice which
126states that this License applies to the Document. These Warranty
127Disclaimers are considered to be included by reference in this
128License, but only as regards disclaiming warranties: any other
129implication that these Warranty Disclaimers may have is void and has
130no effect on the meaning of this License.
131
132@item
133VERBATIM COPYING
134
135You may copy and distribute the Document in any medium, either
136commercially or noncommercially, provided that this License, the
137copyright notices, and the license notice saying this License applies
138to the Document are reproduced in all copies, and that you add no other
139conditions whatsoever to those of this License. You may not use
140technical measures to obstruct or control the reading or further
141copying of the copies you make or distribute. However, you may accept
142compensation in exchange for copies. If you distribute a large enough
143number of copies you must also follow the conditions in section 3.
144
145You may also lend copies, under the same conditions stated above, and
146you may publicly display copies.
147
148@item
149COPYING IN QUANTITY
150
151If you publish printed copies (or copies in media that commonly have
152printed covers) of the Document, numbering more than 100, and the
153Document's license notice requires Cover Texts, you must enclose the
154copies in covers that carry, clearly and legibly, all these Cover
155Texts: Front-Cover Texts on the front cover, and Back-Cover Texts on
156the back cover. Both covers must also clearly and legibly identify
157you as the publisher of these copies. The front cover must present
158the full title with all words of the title equally prominent and
159visible. You may add other material on the covers in addition.
160Copying with changes limited to the covers, as long as they preserve
161the title of the Document and satisfy these conditions, can be treated
162as verbatim copying in other respects.
163
164If the required texts for either cover are too voluminous to fit
165legibly, you should put the first ones listed (as many as fit
166reasonably) on the actual cover, and continue the rest onto adjacent
167pages.
168
169If you publish or distribute Opaque copies of the Document numbering
170more than 100, you must either include a machine-readable Transparent
171copy along with each Opaque copy, or state in or with each Opaque copy
172a computer-network location from which the general network-using
173public has access to download using public-standard network protocols
174a complete Transparent copy of the Document, free of added material.
175If you use the latter option, you must take reasonably prudent steps,
176when you begin distribution of Opaque copies in quantity, to ensure
177that this Transparent copy will remain thus accessible at the stated
178location until at least one year after the last time you distribute an
179Opaque copy (directly or through your agents or retailers) of that
180edition to the public.
181
182It is requested, but not required, that you contact the authors of the
183Document well before redistributing any large number of copies, to give
184them a chance to provide you with an updated version of the Document.
185
186@item
187MODIFICATIONS
188
189You may copy and distribute a Modified Version of the Document under
190the conditions of sections 2 and 3 above, provided that you release
191the Modified Version under precisely this License, with the Modified
192Version filling the role of the Document, thus licensing distribution
193and modification of the Modified Version to whoever possesses a copy
194of it. In addition, you must do these things in the Modified Version:
195
196@enumerate A
197@item
198Use in the Title Page (and on the covers, if any) a title distinct
199from that of the Document, and from those of previous versions
200(which should, if there were any, be listed in the History section
201of the Document). You may use the same title as a previous version
202if the original publisher of that version gives permission.
203
204@item
205List on the Title Page, as authors, one or more persons or entities
206responsible for authorship of the modifications in the Modified
207Version, together with at least five of the principal authors of the
208Document (all of its principal authors, if it has fewer than five),
209unless they release you from this requirement.
210
211@item
212State on the Title page the name of the publisher of the
213Modified Version, as the publisher.
214
215@item
216Preserve all the copyright notices of the Document.
217
218@item
219Add an appropriate copyright notice for your modifications
220adjacent to the other copyright notices.
221
222@item
223Include, immediately after the copyright notices, a license notice
224giving the public permission to use the Modified Version under the
225terms of this License, in the form shown in the Addendum below.
226
227@item
228Preserve in that license notice the full lists of Invariant Sections
229and required Cover Texts given in the Document's license notice.
230
231@item
232Include an unaltered copy of this License.
233
234@item
235Preserve the section Entitled ``History'', Preserve its Title, and add
236to it an item stating at least the title, year, new authors, and
237publisher of the Modified Version as given on the Title Page. If
238there is no section Entitled ``History'' in the Document, create one
239stating the title, year, authors, and publisher of the Document as
240given on its Title Page, then add an item describing the Modified
241Version as stated in the previous sentence.
242
243@item
244Preserve the network location, if any, given in the Document for
245public access to a Transparent copy of the Document, and likewise
246the network locations given in the Document for previous versions
247it was based on. These may be placed in the ``History'' section.
248You may omit a network location for a work that was published at
249least four years before the Document itself, or if the original
250publisher of the version it refers to gives permission.
251
252@item
253For any section Entitled ``Acknowledgements'' or ``Dedications'', Preserve
254the Title of the section, and preserve in the section all the
255substance and tone of each of the contributor acknowledgements and/or
256dedications given therein.
257
258@item
259Preserve all the Invariant Sections of the Document,
260unaltered in their text and in their titles. Section numbers
261or the equivalent are not considered part of the section titles.
262
263@item
264Delete any section Entitled ``Endorsements''. Such a section
265may not be included in the Modified Version.
266
267@item
268Do not retitle any existing section to be Entitled ``Endorsements'' or
269to conflict in title with any Invariant Section.
270
271@item
272Preserve any Warranty Disclaimers.
273@end enumerate
274
275If the Modified Version includes new front-matter sections or
276appendices that qualify as Secondary Sections and contain no material
277copied from the Document, you may at your option designate some or all
278of these sections as invariant. To do this, add their titles to the
279list of Invariant Sections in the Modified Version's license notice.
280These titles must be distinct from any other section titles.
281
282You may add a section Entitled ``Endorsements'', provided it contains
283nothing but endorsements of your Modified Version by various
284parties---for example, statements of peer review or that the text has
285been approved by an organization as the authoritative definition of a
286standard.
287
288You may add a passage of up to five words as a Front-Cover Text, and a
289passage of up to 25 words as a Back-Cover Text, to the end of the list
290of Cover Texts in the Modified Version. Only one passage of
291Front-Cover Text and one of Back-Cover Text may be added by (or
292through arrangements made by) any one entity. If the Document already
293includes a cover text for the same cover, previously added by you or
294by arrangement made by the same entity you are acting on behalf of,
295you may not add another; but you may replace the old one, on explicit
296permission from the previous publisher that added the old one.
297
298The author(s) and publisher(s) of the Document do not by this License
299give permission to use their names for publicity for or to assert or
300imply endorsement of any Modified Version.
301
302@item
303COMBINING DOCUMENTS
304
305You may combine the Document with other documents released under this
306License, under the terms defined in section 4 above for modified
307versions, provided that you include in the combination all of the
308Invariant Sections of all of the original documents, unmodified, and
309list them all as Invariant Sections of your combined work in its
310license notice, and that you preserve all their Warranty Disclaimers.
311
312The combined work need only contain one copy of this License, and
313multiple identical Invariant Sections may be replaced with a single
314copy. If there are multiple Invariant Sections with the same name but
315different contents, make the title of each such section unique by
316adding at the end of it, in parentheses, the name of the original
317author or publisher of that section if known, or else a unique number.
318Make the same adjustment to the section titles in the list of
319Invariant Sections in the license notice of the combined work.
320
321In the combination, you must combine any sections Entitled ``History''
322in the various original documents, forming one section Entitled
323``History''; likewise combine any sections Entitled ``Acknowledgements'',
324and any sections Entitled ``Dedications''. You must delete all
325sections Entitled ``Endorsements.''
326
327@item
328COLLECTIONS OF DOCUMENTS
329
330You may make a collection consisting of the Document and other documents
331released under this License, and replace the individual copies of this
332License in the various documents with a single copy that is included in
333the collection, provided that you follow the rules of this License for
334verbatim copying of each of the documents in all other respects.
335
336You may extract a single document from such a collection, and distribute
337it individually under this License, provided you insert a copy of this
338License into the extracted document, and follow this License in all
339other respects regarding verbatim copying of that document.
340
341@item
342AGGREGATION WITH INDEPENDENT WORKS
343
344A compilation of the Document or its derivatives with other separate
345and independent documents or works, in or on a volume of a storage or
346distribution medium, is called an ``aggregate'' if the copyright
347resulting from the compilation is not used to limit the legal rights
348of the compilation's users beyond what the individual works permit.
349When the Document is included in an aggregate, this License does not
350apply to the other works in the aggregate which are not themselves
351derivative works of the Document.
352
353If the Cover Text requirement of section 3 is applicable to these
354copies of the Document, then if the Document is less than one half of
355the entire aggregate, the Document's Cover Texts may be placed on
356covers that bracket the Document within the aggregate, or the
357electronic equivalent of covers if the Document is in electronic form.
358Otherwise they must appear on printed covers that bracket the whole
359aggregate.
360
361@item
362TRANSLATION
363
364Translation is considered a kind of modification, so you may
365distribute translations of the Document under the terms of section 4.
366Replacing Invariant Sections with translations requires special
367permission from their copyright holders, but you may include
368translations of some or all Invariant Sections in addition to the
369original versions of these Invariant Sections. You may include a
370translation of this License, and all the license notices in the
371Document, and any Warranty Disclaimers, provided that you also include
372the original English version of this License and the original versions
373of those notices and disclaimers. In case of a disagreement between
374the translation and the original version of this License or a notice
375or disclaimer, the original version will prevail.
376
377If a section in the Document is Entitled ``Acknowledgements'',
378``Dedications'', or ``History'', the requirement (section 4) to Preserve
379its Title (section 1) will typically require changing the actual
380title.
381
382@item
383TERMINATION
384
385You may not copy, modify, sublicense, or distribute the Document
386except as expressly provided under this License. Any attempt
387otherwise to copy, modify, sublicense, or distribute it is void, and
388will automatically terminate your rights under this License.
389
390However, if you cease all violation of this License, then your license
391from a particular copyright holder is reinstated (a) provisionally,
392unless and until the copyright holder explicitly and finally
393terminates your license, and (b) permanently, if the copyright holder
394fails to notify you of the violation by some reasonable means prior to
39560 days after the cessation.
396
397Moreover, your license from a particular copyright holder is
398reinstated permanently if the copyright holder notifies you of the
399violation by some reasonable means, this is the first time you have
400received notice of violation of this License (for any work) from that
401copyright holder, and you cure the violation prior to 30 days after
402your receipt of the notice.
403
404Termination of your rights under this section does not terminate the
405licenses of parties who have received copies or rights from you under
406this License. If your rights have been terminated and not permanently
407reinstated, receipt of a copy of some or all of the same material does
408not give you any rights to use it.
409
410@item
411FUTURE REVISIONS OF THIS LICENSE
412
413The Free Software Foundation may publish new, revised versions
414of the GNU Free Documentation License from time to time. Such new
415versions will be similar in spirit to the present version, but may
416differ in detail to address new problems or concerns. See
417@uref{http://www.gnu.org/copyleft/}.
418
419Each version of the License is given a distinguishing version number.
420If the Document specifies that a particular numbered version of this
421License ``or any later version'' applies to it, you have the option of
422following the terms and conditions either of that specified version or
423of any later version that has been published (not as a draft) by the
424Free Software Foundation. If the Document does not specify a version
425number of this License, you may choose any version ever published (not
426as a draft) by the Free Software Foundation. If the Document
427specifies that a proxy can decide which future versions of this
428License can be used, that proxy's public statement of acceptance of a
429version permanently authorizes you to choose that version for the
430Document.
431
432@item
433RELICENSING
434
435``Massive Multiauthor Collaboration Site'' (or ``MMC Site'') means any
436World Wide Web server that publishes copyrightable works and also
437provides prominent facilities for anybody to edit those works. A
438public wiki that anybody can edit is an example of such a server. A
439``Massive Multiauthor Collaboration'' (or ``MMC'') contained in the
440site means any set of copyrightable works thus published on the MMC
441site.
442
443``CC-BY-SA'' means the Creative Commons Attribution-Share Alike 3.0
444license published by Creative Commons Corporation, a not-for-profit
445corporation with a principal place of business in San Francisco,
446California, as well as future copyleft versions of that license
447published by that same organization.
448
449``Incorporate'' means to publish or republish a Document, in whole or
450in part, as part of another Document.
451
452An MMC is ``eligible for relicensing'' if it is licensed under this
453License, and if all works that were first published under this License
454somewhere other than this MMC, and subsequently incorporated in whole
455or in part into the MMC, (1) had no cover texts or invariant sections,
456and (2) were thus incorporated prior to November 1, 2008.
457
458The operator of an MMC Site may republish an MMC contained in the site
459under CC-BY-SA on the same site at any time before August 1, 2009,
460provided the MMC is eligible for relicensing.
461
462@end enumerate
463
464@page
465@heading ADDENDUM: How to use this License for your documents
466
467To use this License in a document you have written, include a copy of
468the License in the document and put the following copyright and
469license notices just after the title page:
470
471@smallexample
472@group
473 Copyright (C) @var{year} @var{your name}.
474 Permission is granted to copy, distribute and/or modify this document
475 under the terms of the GNU Free Documentation License, Version 1.3
476 or any later version published by the Free Software Foundation;
477 with no Invariant Sections, no Front-Cover Texts, and no Back-Cover
478 Texts. A copy of the license is included in the section entitled ``GNU
479 Free Documentation License''.
480@end group
481@end smallexample
482
483If you have Invariant Sections, Front-Cover Texts and Back-Cover Texts,
484replace the ``with@dots{}Texts.''@: line with this:
485
486@smallexample
487@group
488 with the Invariant Sections being @var{list their titles}, with
489 the Front-Cover Texts being @var{list}, and with the Back-Cover Texts
490 being @var{list}.
491@end group
492@end smallexample
493
494If you have Invariant Sections without Cover Texts, or some other
495combination of the three, merge those two alternatives to suit the
496situation.
497
498If your document contains nontrivial examples of program code, we
499recommend releasing these examples in parallel under your choice of
500free software license, such as the GNU General Public License,
501to permit their use in free software.
502
503@c Local Variables:
504@c ispell-local-pdict: "ispell-dict"
505@c End:
diff --git a/doc/gnunet.texi b/doc/gnunet.texi
new file mode 100644
index 000000000..115f93a93
--- /dev/null
+++ b/doc/gnunet.texi
@@ -0,0 +1,177 @@
1\input texinfo @c -*-texinfo-*-
2@c %**start of header
3@setfilename gnunet.info
4@documentencoding UTF-8
5@settitle GNUnet Reference Manual
6@c %**end of header
7
8@include version.texi
9
10@copying
11Copyright @copyright{} 2001-2017 GNUnet e.V.
12
13Permission is granted to copy, distribute and/or modify this document
14under the terms of the GNU Free Documentation License, Version 1.3 or
15any later version published by the Free Software Foundation; with no
16Invariant Sections, no Front-Cover Texts, and no Back-Cover Texts. A
17copy of the license is included in the section entitled ``GNU Free
18Documentation License''.
19
20A copy of the license is also available from the Free Software
21Foundation Web site at @url{http://www.gnu.org/licenses/fdl.html}.
22
23Alternately, this document is also available under the General
24Public License, version 3 or later, as published by the Free Software
25Foundation. A copy of the license is included in the section entitled
26``GNU General Public License''.
27
28A copy of the license is also available from the Free Software
29Foundation Web site at @url{http://www.gnu.org/licenses/gpl.html}.
30@end copying
31
32@titlepage
33@title GNUnet Reference Manual
34@subtitle Installing, configuring, using and contributing to GNUnet
35@author ng0@*
36@author Adriano Peluso
37@author The GNUnet Developers
38
39@page
40@vskip 0pt plus 1filll
41Edition @value{EDITION} @*
42@value{UPDATED} @*
43
44@insertcopying
45@end titlepage
46
47@contents
48@c *********************************************************************
49@node Top
50@top Contributing to GNUnet
51@c *********************************************************************
52
53This document describes GNUnet version @value{VERSION}.
54
55
56GNUnet is a @uref{http://www.gnu.org/, GNU} package. All code contributions
57must thus be put under the
58@uref{http://www.gnu.org/copyleft/gpl.html, GNU Public License (GPL)}.
59All documentation should be put under FSF approved licenses
60(see @uref{http://www.gnu.org/copyleft/fdl.html, fdl}).
61
62By submitting documentation, translations, comments and other content to this
63website you automatically grant the right to publish code under the
64GNU Public License and documentation under either or both the GNU
65Public License or the GNU Free Documentation License. When contributing
66to the GNUnet project, GNU standards and the
67@uref{http://www.gnu.org/philosophy/philosophy.html, GNU philosophy}
68should be adhered to.
69
70Note that we do now require a formal copyright assignment for GNUnet
71contributors to GNUnet e.V.; nevertheless, we do allow pseudonymous
72contributions. By signing the copyright agreement and submitting your
73code (or documentation) to us, you agree to share the rights to your
74code with GNUnet e.V.; GNUnet e.V. receives non-exclusive ownership
75rights, and in particular is allowed to dual-license the code. You
76retain non-exclusive rights to your contributions, so you can also
77share your contributions freely with other projects.
78
79GNUnet e.V. will publish all accepted contributions under the GPLv3 or any
80later version. The association may decide to publish contributions under
81additional licenses (dual-licensing).
82
83We do not intentionally remove your name from your contributions; however,
84due to extensive editing it is not always trivial to attribute contributors
85properly. If you find that you significantly contributed to a file (or the
86project as a whole) and are not listed in the respective authors file or
87section, please do let us know.
88
89
90
91@menu
92
93* Philosophy:: About GNUnet
94* GNUnet Installation Handbook:: How to install GNUnet
95* Using GNUnet:: Using GNUnet
96* GNUnet Developer Handbook:: Developing GNUnet
97* GNU Free Documentation License:: The license of this manual.
98* GNU General Public License:: The license of this manual.
99* Concept Index:: Concepts.
100* Programming Index:: Data types, functions, and variables.
101
102@detailmenu
103 --- The Detailed Node Listing ---
104
105Philosophy
106
107* Copyright and Contributions::
108* Design Goals::
109* Security & Privacy::
110* Versatility::
111* Practicality::
112* Key Concepts::
113* Authentication::
114* Accounting to Encourage Resource Sharing::
115* Confidentiality::
116* Anonymity::
117* How GNUnet achieves Anonymity::
118* Deniability::
119* Peer Identities::
120* Zones in the GNU Name System (GNS Zones)::
121* Egos::
122* Backup of Identities and Egos::
123* Revocation::
124
125Installation
126
127* Dependencies::
128* External dependencies::
129
130@end detailmenu
131@end menu
132
133@c *********************************************************************
134@include chapters/philosophy.texi
135@c *********************************************************************
136
137@c *********************************************************************
138@include chapters/installation.texi
139@c *********************************************************************
140
141@c *********************************************************************
142@include chapters/user.texi
143@c *********************************************************************
144
145@c *********************************************************************
146@include chapters/developer.texi
147@c *********************************************************************
148
149
150@c *********************************************************************
151@node GNU Free Documentation License
152@appendix GNU Free Documentation License
153@cindex license, GNU Free Documentation License
154@include fdl-1.3.texi
155
156@c *********************************************************************
157@node GNU General Public License
158@appendix GNU General Public License
159@cindex license, GNU General Public License
160@include gpl-3.0.texi
161
162@c *********************************************************************
163@node Concept Index
164@unnumbered Concept Index
165@printindex cp
166
167@node Programming Index
168@unnumbered Programming Index
169@syncodeindex tp fn
170@syncodeindex vr fn
171@printindex fn
172
173@bye
174
175@c Local Variables:
176@c ispell-local-dictionary: "american";
177@c End:
diff --git a/doc/gpl-3.0.texi b/doc/gpl-3.0.texi
new file mode 100644
index 000000000..0e2e212ac
--- /dev/null
+++ b/doc/gpl-3.0.texi
@@ -0,0 +1,717 @@
1@c The GNU General Public License.
2@center Version 3, 29 June 2007
3
4@c This file is intended to be included within another document,
5@c hence no sectioning command or @node.
6
7@display
8Copyright @copyright{} 2007 Free Software Foundation, Inc. @url{http://fsf.org/}
9
10Everyone is permitted to copy and distribute verbatim copies of this
11license document, but changing it is not allowed.
12@end display
13
14@heading Preamble
15
16The GNU General Public License is a free, copyleft license for
17software and other kinds of works.
18
19The licenses for most software and other practical works are designed
20to take away your freedom to share and change the works. By contrast,
21the GNU General Public License is intended to guarantee your freedom
22to share and change all versions of a program---to make sure it remains
23free software for all its users. We, the Free Software Foundation,
24use the GNU General Public License for most of our software; it
25applies also to any other work released this way by its authors. You
26can apply it to your programs, too.
27
28When we speak of free software, we are referring to freedom, not
29price. Our General Public Licenses are designed to make sure that you
30have the freedom to distribute copies of free software (and charge for
31them if you wish), that you receive source code or can get it if you
32want it, that you can change the software or use pieces of it in new
33free programs, and that you know you can do these things.
34
35To protect your rights, we need to prevent others from denying you
36these rights or asking you to surrender the rights. Therefore, you
37have certain responsibilities if you distribute copies of the
38software, or if you modify it: responsibilities to respect the freedom
39of others.
40
41For example, if you distribute copies of such a program, whether
42gratis or for a fee, you must pass on to the recipients the same
43freedoms that you received. You must make sure that they, too,
44receive or can get the source code. And you must show them these
45terms so they know their rights.
46
47Developers that use the GNU GPL protect your rights with two steps:
48(1) assert copyright on the software, and (2) offer you this License
49giving you legal permission to copy, distribute and/or modify it.
50
51For the developers' and authors' protection, the GPL clearly explains
52that there is no warranty for this free software. For both users' and
53authors' sake, the GPL requires that modified versions be marked as
54changed, so that their problems will not be attributed erroneously to
55authors of previous versions.
56
57Some devices are designed to deny users access to install or run
58modified versions of the software inside them, although the
59manufacturer can do so. This is fundamentally incompatible with the
60aim of protecting users' freedom to change the software. The
61systematic pattern of such abuse occurs in the area of products for
62individuals to use, which is precisely where it is most unacceptable.
63Therefore, we have designed this version of the GPL to prohibit the
64practice for those products. If such problems arise substantially in
65other domains, we stand ready to extend this provision to those
66domains in future versions of the GPL, as needed to protect the
67freedom of users.
68
69Finally, every program is threatened constantly by software patents.
70States should not allow patents to restrict development and use of
71software on general-purpose computers, but in those that do, we wish
72to avoid the special danger that patents applied to a free program
73could make it effectively proprietary. To prevent this, the GPL
74assures that patents cannot be used to render the program non-free.
75
76The precise terms and conditions for copying, distribution and
77modification follow.
78
79@heading TERMS AND CONDITIONS
80
81@enumerate 0
82@item Definitions.
83
84``This License'' refers to version 3 of the GNU General Public License.
85
86``Copyright'' also means copyright-like laws that apply to other kinds
87of works, such as semiconductor masks.
88
89``The Program'' refers to any copyrightable work licensed under this
90License. Each licensee is addressed as ``you''. ``Licensees'' and
91``recipients'' may be individuals or organizations.
92
93To ``modify'' a work means to copy from or adapt all or part of the work
94in a fashion requiring copyright permission, other than the making of
95an exact copy. The resulting work is called a ``modified version'' of
96the earlier work or a work ``based on'' the earlier work.
97
98A ``covered work'' means either the unmodified Program or a work based
99on the Program.
100
101To ``propagate'' a work means to do anything with it that, without
102permission, would make you directly or secondarily liable for
103infringement under applicable copyright law, except executing it on a
104computer or modifying a private copy. Propagation includes copying,
105distribution (with or without modification), making available to the
106public, and in some countries other activities as well.
107
108To ``convey'' a work means any kind of propagation that enables other
109parties to make or receive copies. Mere interaction with a user
110through a computer network, with no transfer of a copy, is not
111conveying.
112
113An interactive user interface displays ``Appropriate Legal Notices'' to
114the extent that it includes a convenient and prominently visible
115feature that (1) displays an appropriate copyright notice, and (2)
116tells the user that there is no warranty for the work (except to the
117extent that warranties are provided), that licensees may convey the
118work under this License, and how to view a copy of this License. If
119the interface presents a list of user commands or options, such as a
120menu, a prominent item in the list meets this criterion.
121
122@item Source Code.
123
124The ``source code'' for a work means the preferred form of the work for
125making modifications to it. ``Object code'' means any non-source form
126of a work.
127
128A ``Standard Interface'' means an interface that either is an official
129standard defined by a recognized standards body, or, in the case of
130interfaces specified for a particular programming language, one that
131is widely used among developers working in that language.
132
133The ``System Libraries'' of an executable work include anything, other
134than the work as a whole, that (a) is included in the normal form of
135packaging a Major Component, but which is not part of that Major
136Component, and (b) serves only to enable use of the work with that
137Major Component, or to implement a Standard Interface for which an
138implementation is available to the public in source code form. A
139``Major Component'', in this context, means a major essential component
140(kernel, window system, and so on) of the specific operating system
141(if any) on which the executable work runs, or a compiler used to
142produce the work, or an object code interpreter used to run it.
143
144The ``Corresponding Source'' for a work in object code form means all
145the source code needed to generate, install, and (for an executable
146work) run the object code and to modify the work, including scripts to
147control those activities. However, it does not include the work's
148System Libraries, or general-purpose tools or generally available free
149programs which are used unmodified in performing those activities but
150which are not part of the work. For example, Corresponding Source
151includes interface definition files associated with source files for
152the work, and the source code for shared libraries and dynamically
153linked subprograms that the work is specifically designed to require,
154such as by intimate data communication or control flow between those
155subprograms and other parts of the work.
156
157The Corresponding Source need not include anything that users can
158regenerate automatically from other parts of the Corresponding Source.
159
160The Corresponding Source for a work in source code form is that same
161work.
162
163@item Basic Permissions.
164
165All rights granted under this License are granted for the term of
166copyright on the Program, and are irrevocable provided the stated
167conditions are met. This License explicitly affirms your unlimited
168permission to run the unmodified Program. The output from running a
169covered work is covered by this License only if the output, given its
170content, constitutes a covered work. This License acknowledges your
171rights of fair use or other equivalent, as provided by copyright law.
172
173You may make, run and propagate covered works that you do not convey,
174without conditions so long as your license otherwise remains in force.
175You may convey covered works to others for the sole purpose of having
176them make modifications exclusively for you, or provide you with
177facilities for running those works, provided that you comply with the
178terms of this License in conveying all material for which you do not
179control copyright. Those thus making or running the covered works for
180you must do so exclusively on your behalf, under your direction and
181control, on terms that prohibit them from making any copies of your
182copyrighted material outside their relationship with you.
183
184Conveying under any other circumstances is permitted solely under the
185conditions stated below. Sublicensing is not allowed; section 10
186makes it unnecessary.
187
188@item Protecting Users' Legal Rights From Anti-Circumvention Law.
189
190No covered work shall be deemed part of an effective technological
191measure under any applicable law fulfilling obligations under article
19211 of the WIPO copyright treaty adopted on 20 December 1996, or
193similar laws prohibiting or restricting circumvention of such
194measures.
195
196When you convey a covered work, you waive any legal power to forbid
197circumvention of technological measures to the extent such
198circumvention is effected by exercising rights under this License with
199respect to the covered work, and you disclaim any intention to limit
200operation or modification of the work as a means of enforcing, against
201the work's users, your or third parties' legal rights to forbid
202circumvention of technological measures.
203
204@item Conveying Verbatim Copies.
205
206You may convey verbatim copies of the Program's source code as you
207receive it, in any medium, provided that you conspicuously and
208appropriately publish on each copy an appropriate copyright notice;
209keep intact all notices stating that this License and any
210non-permissive terms added in accord with section 7 apply to the code;
211keep intact all notices of the absence of any warranty; and give all
212recipients a copy of this License along with the Program.
213
214You may charge any price or no price for each copy that you convey,
215and you may offer support or warranty protection for a fee.
216
217@item Conveying Modified Source Versions.
218
219You may convey a work based on the Program, or the modifications to
220produce it from the Program, in the form of source code under the
221terms of section 4, provided that you also meet all of these
222conditions:
223
224@enumerate a
225@item
226The work must carry prominent notices stating that you modified it,
227and giving a relevant date.
228
229@item
230The work must carry prominent notices stating that it is released
231under this License and any conditions added under section 7. This
232requirement modifies the requirement in section 4 to ``keep intact all
233notices''.
234
235@item
236You must license the entire work, as a whole, under this License to
237anyone who comes into possession of a copy. This License will
238therefore apply, along with any applicable section 7 additional terms,
239to the whole of the work, and all its parts, regardless of how they
240are packaged. This License gives no permission to license the work in
241any other way, but it does not invalidate such permission if you have
242separately received it.
243
244@item
245If the work has interactive user interfaces, each must display
246Appropriate Legal Notices; however, if the Program has interactive
247interfaces that do not display Appropriate Legal Notices, your work
248need not make them do so.
249@end enumerate
250
251A compilation of a covered work with other separate and independent
252works, which are not by their nature extensions of the covered work,
253and which are not combined with it such as to form a larger program,
254in or on a volume of a storage or distribution medium, is called an
255``aggregate'' if the compilation and its resulting copyright are not
256used to limit the access or legal rights of the compilation's users
257beyond what the individual works permit. Inclusion of a covered work
258in an aggregate does not cause this License to apply to the other
259parts of the aggregate.
260
261@item Conveying Non-Source Forms.
262
263You may convey a covered work in object code form under the terms of
264sections 4 and 5, provided that you also convey the machine-readable
265Corresponding Source under the terms of this License, in one of these
266ways:
267
268@enumerate a
269@item
270Convey the object code in, or embodied in, a physical product
271(including a physical distribution medium), accompanied by the
272Corresponding Source fixed on a durable physical medium customarily
273used for software interchange.
274
275@item
276Convey the object code in, or embodied in, a physical product
277(including a physical distribution medium), accompanied by a written
278offer, valid for at least three years and valid for as long as you
279offer spare parts or customer support for that product model, to give
280anyone who possesses the object code either (1) a copy of the
281Corresponding Source for all the software in the product that is
282covered by this License, on a durable physical medium customarily used
283for software interchange, for a price no more than your reasonable
284cost of physically performing this conveying of source, or (2) access
285to copy the Corresponding Source from a network server at no charge.
286
287@item
288Convey individual copies of the object code with a copy of the written
289offer to provide the Corresponding Source. This alternative is
290allowed only occasionally and noncommercially, and only if you
291received the object code with such an offer, in accord with subsection
2926b.
293
294@item
295Convey the object code by offering access from a designated place
296(gratis or for a charge), and offer equivalent access to the
297Corresponding Source in the same way through the same place at no
298further charge. You need not require recipients to copy the
299Corresponding Source along with the object code. If the place to copy
300the object code is a network server, the Corresponding Source may be
301on a different server (operated by you or a third party) that supports
302equivalent copying facilities, provided you maintain clear directions
303next to the object code saying where to find the Corresponding Source.
304Regardless of what server hosts the Corresponding Source, you remain
305obligated to ensure that it is available for as long as needed to
306satisfy these requirements.
307
308@item
309Convey the object code using peer-to-peer transmission, provided you
310inform other peers where the object code and Corresponding Source of
311the work are being offered to the general public at no charge under
312subsection 6d.
313
314@end enumerate
315
316A separable portion of the object code, whose source code is excluded
317from the Corresponding Source as a System Library, need not be
318included in conveying the object code work.
319
320A ``User Product'' is either (1) a ``consumer product'', which means any
321tangible personal property which is normally used for personal,
322family, or household purposes, or (2) anything designed or sold for
323incorporation into a dwelling. In determining whether a product is a
324consumer product, doubtful cases shall be resolved in favor of
325coverage. For a particular product received by a particular user,
326``normally used'' refers to a typical or common use of that class of
327product, regardless of the status of the particular user or of the way
328in which the particular user actually uses, or expects or is expected
329to use, the product. A product is a consumer product regardless of
330whether the product has substantial commercial, industrial or
331non-consumer uses, unless such uses represent the only significant
332mode of use of the product.
333
334``Installation Information'' for a User Product means any methods,
335procedures, authorization keys, or other information required to
336install and execute modified versions of a covered work in that User
337Product from a modified version of its Corresponding Source. The
338information must suffice to ensure that the continued functioning of
339the modified object code is in no case prevented or interfered with
340solely because modification has been made.
341
342If you convey an object code work under this section in, or with, or
343specifically for use in, a User Product, and the conveying occurs as
344part of a transaction in which the right of possession and use of the
345User Product is transferred to the recipient in perpetuity or for a
346fixed term (regardless of how the transaction is characterized), the
347Corresponding Source conveyed under this section must be accompanied
348by the Installation Information. But this requirement does not apply
349if neither you nor any third party retains the ability to install
350modified object code on the User Product (for example, the work has
351been installed in ROM).
352
353The requirement to provide Installation Information does not include a
354requirement to continue to provide support service, warranty, or
355updates for a work that has been modified or installed by the
356recipient, or for the User Product in which it has been modified or
357installed. Access to a network may be denied when the modification
358itself materially and adversely affects the operation of the network
359or violates the rules and protocols for communication across the
360network.
361
362Corresponding Source conveyed, and Installation Information provided,
363in accord with this section must be in a format that is publicly
364documented (and with an implementation available to the public in
365source code form), and must require no special password or key for
366unpacking, reading or copying.
367
368@item Additional Terms.
369
370``Additional permissions'' are terms that supplement the terms of this
371License by making exceptions from one or more of its conditions.
372Additional permissions that are applicable to the entire Program shall
373be treated as though they were included in this License, to the extent
374that they are valid under applicable law. If additional permissions
375apply only to part of the Program, that part may be used separately
376under those permissions, but the entire Program remains governed by
377this License without regard to the additional permissions.
378
379When you convey a copy of a covered work, you may at your option
380remove any additional permissions from that copy, or from any part of
381it. (Additional permissions may be written to require their own
382removal in certain cases when you modify the work.) You may place
383additional permissions on material, added by you to a covered work,
384for which you have or can give appropriate copyright permission.
385
386Notwithstanding any other provision of this License, for material you
387add to a covered work, you may (if authorized by the copyright holders
388of that material) supplement the terms of this License with terms:
389
390@enumerate a
391@item
392Disclaiming warranty or limiting liability differently from the terms
393of sections 15 and 16 of this License; or
394
395@item
396Requiring preservation of specified reasonable legal notices or author
397attributions in that material or in the Appropriate Legal Notices
398displayed by works containing it; or
399
400@item
401Prohibiting misrepresentation of the origin of that material, or
402requiring that modified versions of such material be marked in
403reasonable ways as different from the original version; or
404
405@item
406Limiting the use for publicity purposes of names of licensors or
407authors of the material; or
408
409@item
410Declining to grant rights under trademark law for use of some trade
411names, trademarks, or service marks; or
412
413@item
414Requiring indemnification of licensors and authors of that material by
415anyone who conveys the material (or modified versions of it) with
416contractual assumptions of liability to the recipient, for any
417liability that these contractual assumptions directly impose on those
418licensors and authors.
419@end enumerate
420
421All other non-permissive additional terms are considered ``further
422restrictions'' within the meaning of section 10. If the Program as you
423received it, or any part of it, contains a notice stating that it is
424governed by this License along with a term that is a further
425restriction, you may remove that term. If a license document contains
426a further restriction but permits relicensing or conveying under this
427License, you may add to a covered work material governed by the terms
428of that license document, provided that the further restriction does
429not survive such relicensing or conveying.
430
431If you add terms to a covered work in accord with this section, you
432must place, in the relevant source files, a statement of the
433additional terms that apply to those files, or a notice indicating
434where to find the applicable terms.
435
436Additional terms, permissive or non-permissive, may be stated in the
437form of a separately written license, or stated as exceptions; the
438above requirements apply either way.
439
440@item Termination.
441
442You may not propagate or modify a covered work except as expressly
443provided under this License. Any attempt otherwise to propagate or
444modify it is void, and will automatically terminate your rights under
445this License (including any patent licenses granted under the third
446paragraph of section 11).
447
448However, if you cease all violation of this License, then your license
449from a particular copyright holder is reinstated (a) provisionally,
450unless and until the copyright holder explicitly and finally
451terminates your license, and (b) permanently, if the copyright holder
452fails to notify you of the violation by some reasonable means prior to
45360 days after the cessation.
454
455Moreover, your license from a particular copyright holder is
456reinstated permanently if the copyright holder notifies you of the
457violation by some reasonable means, this is the first time you have
458received notice of violation of this License (for any work) from that
459copyright holder, and you cure the violation prior to 30 days after
460your receipt of the notice.
461
462Termination of your rights under this section does not terminate the
463licenses of parties who have received copies or rights from you under
464this License. If your rights have been terminated and not permanently
465reinstated, you do not qualify to receive new licenses for the same
466material under section 10.
467
468@item Acceptance Not Required for Having Copies.
469
470You are not required to accept this License in order to receive or run
471a copy of the Program. Ancillary propagation of a covered work
472occurring solely as a consequence of using peer-to-peer transmission
473to receive a copy likewise does not require acceptance. However,
474nothing other than this License grants you permission to propagate or
475modify any covered work. These actions infringe copyright if you do
476not accept this License. Therefore, by modifying or propagating a
477covered work, you indicate your acceptance of this License to do so.
478
479@item Automatic Licensing of Downstream Recipients.
480
481Each time you convey a covered work, the recipient automatically
482receives a license from the original licensors, to run, modify and
483propagate that work, subject to this License. You are not responsible
484for enforcing compliance by third parties with this License.
485
486An ``entity transaction'' is a transaction transferring control of an
487organization, or substantially all assets of one, or subdividing an
488organization, or merging organizations. If propagation of a covered
489work results from an entity transaction, each party to that
490transaction who receives a copy of the work also receives whatever
491licenses to the work the party's predecessor in interest had or could
492give under the previous paragraph, plus a right to possession of the
493Corresponding Source of the work from the predecessor in interest, if
494the predecessor has it or can get it with reasonable efforts.
495
496You may not impose any further restrictions on the exercise of the
497rights granted or affirmed under this License. For example, you may
498not impose a license fee, royalty, or other charge for exercise of
499rights granted under this License, and you may not initiate litigation
500(including a cross-claim or counterclaim in a lawsuit) alleging that
501any patent claim is infringed by making, using, selling, offering for
502sale, or importing the Program or any portion of it.
503
504@item Patents.
505
506A ``contributor'' is a copyright holder who authorizes use under this
507License of the Program or a work on which the Program is based. The
508work thus licensed is called the contributor's ``contributor version''.
509
510A contributor's ``essential patent claims'' are all patent claims owned
511or controlled by the contributor, whether already acquired or
512hereafter acquired, that would be infringed by some manner, permitted
513by this License, of making, using, or selling its contributor version,
514but do not include claims that would be infringed only as a
515consequence of further modification of the contributor version. For
516purposes of this definition, ``control'' includes the right to grant
517patent sublicenses in a manner consistent with the requirements of
518this License.
519
520Each contributor grants you a non-exclusive, worldwide, royalty-free
521patent license under the contributor's essential patent claims, to
522make, use, sell, offer for sale, import and otherwise run, modify and
523propagate the contents of its contributor version.
524
525In the following three paragraphs, a ``patent license'' is any express
526agreement or commitment, however denominated, not to enforce a patent
527(such as an express permission to practice a patent or covenant not to
528sue for patent infringement). To ``grant'' such a patent license to a
529party means to make such an agreement or commitment not to enforce a
530patent against the party.
531
532If you convey a covered work, knowingly relying on a patent license,
533and the Corresponding Source of the work is not available for anyone
534to copy, free of charge and under the terms of this License, through a
535publicly available network server or other readily accessible means,
536then you must either (1) cause the Corresponding Source to be so
537available, or (2) arrange to deprive yourself of the benefit of the
538patent license for this particular work, or (3) arrange, in a manner
539consistent with the requirements of this License, to extend the patent
540license to downstream recipients. ``Knowingly relying'' means you have
541actual knowledge that, but for the patent license, your conveying the
542covered work in a country, or your recipient's use of the covered work
543in a country, would infringe one or more identifiable patents in that
544country that you have reason to believe are valid.
545
546If, pursuant to or in connection with a single transaction or
547arrangement, you convey, or propagate by procuring conveyance of, a
548covered work, and grant a patent license to some of the parties
549receiving the covered work authorizing them to use, propagate, modify
550or convey a specific copy of the covered work, then the patent license
551you grant is automatically extended to all recipients of the covered
552work and works based on it.
553
554A patent license is ``discriminatory'' if it does not include within the
555scope of its coverage, prohibits the exercise of, or is conditioned on
556the non-exercise of one or more of the rights that are specifically
557granted under this License. You may not convey a covered work if you
558are a party to an arrangement with a third party that is in the
559business of distributing software, under which you make payment to the
560third party based on the extent of your activity of conveying the
561work, and under which the third party grants, to any of the parties
562who would receive the covered work from you, a discriminatory patent
563license (a) in connection with copies of the covered work conveyed by
564you (or copies made from those copies), or (b) primarily for and in
565connection with specific products or compilations that contain the
566covered work, unless you entered into that arrangement, or that patent
567license was granted, prior to 28 March 2007.
568
569Nothing in this License shall be construed as excluding or limiting
570any implied license or other defenses to infringement that may
571otherwise be available to you under applicable patent law.
572
573@item No Surrender of Others' Freedom.
574
575If conditions are imposed on you (whether by court order, agreement or
576otherwise) that contradict the conditions of this License, they do not
577excuse you from the conditions of this License. If you cannot convey
578a covered work so as to satisfy simultaneously your obligations under
579this License and any other pertinent obligations, then as a
580consequence you may not convey it at all. For example, if you agree
581to terms that obligate you to collect a royalty for further conveying
582from those to whom you convey the Program, the only way you could
583satisfy both those terms and this License would be to refrain entirely
584from conveying the Program.
585
586@item Use with the GNU Affero General Public License.
587
588Notwithstanding any other provision of this License, you have
589permission to link or combine any covered work with a work licensed
590under version 3 of the GNU Affero General Public License into a single
591combined work, and to convey the resulting work. The terms of this
592License will continue to apply to the part which is the covered work,
593but the special requirements of the GNU Affero General Public License,
594section 13, concerning interaction through a network will apply to the
595combination as such.
596
597@item Revised Versions of this License.
598
599The Free Software Foundation may publish revised and/or new versions
600of the GNU General Public License from time to time. Such new
601versions will be similar in spirit to the present version, but may
602differ in detail to address new problems or concerns.
603
604Each version is given a distinguishing version number. If the Program
605specifies that a certain numbered version of the GNU General Public
606License ``or any later version'' applies to it, you have the option of
607following the terms and conditions either of that numbered version or
608of any later version published by the Free Software Foundation. If
609the Program does not specify a version number of the GNU General
610Public License, you may choose any version ever published by the Free
611Software Foundation.
612
613If the Program specifies that a proxy can decide which future versions
614of the GNU General Public License can be used, that proxy's public
615statement of acceptance of a version permanently authorizes you to
616choose that version for the Program.
617
618Later license versions may give you additional or different
619permissions. However, no additional obligations are imposed on any
620author or copyright holder as a result of your choosing to follow a
621later version.
622
623@item Disclaimer of Warranty.
624
625THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
626APPLICABLE LAW@. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
627HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM ``AS IS'' WITHOUT
628WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT
629LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
630A PARTICULAR PURPOSE@. THE ENTIRE RISK AS TO THE QUALITY AND
631PERFORMANCE OF THE PROGRAM IS WITH YOU@. SHOULD THE PROGRAM PROVE
632DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR
633CORRECTION.
634
635@item Limitation of Liability.
636
637IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
638WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR
639CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
640INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES
641ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT
642NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR
643LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM
644TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER
645PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
646
647@item Interpretation of Sections 15 and 16.
648
649If the disclaimer of warranty and limitation of liability provided
650above cannot be given local legal effect according to their terms,
651reviewing courts shall apply local law that most closely approximates
652an absolute waiver of all civil liability in connection with the
653Program, unless a warranty or assumption of liability accompanies a
654copy of the Program in return for a fee.
655
656@end enumerate
657
658@heading END OF TERMS AND CONDITIONS
659
660@heading How to Apply These Terms to Your New Programs
661
662If you develop a new program, and you want it to be of the greatest
663possible use to the public, the best way to achieve this is to make it
664free software which everyone can redistribute and change under these
665terms.
666
667To do so, attach the following notices to the program. It is safest
668to attach them to the start of each source file to most effectively
669state the exclusion of warranty; and each file should have at least
670the ``copyright'' line and a pointer to where the full notice is found.
671
672@smallexample
673@var{one line to give the program's name and a brief idea of what it does.}
674Copyright (C) @var{year} @var{name of author}
675
676This program is free software: you can redistribute it and/or modify
677it under the terms of the GNU General Public License as published by
678the Free Software Foundation, either version 3 of the License, or (at
679your option) any later version.
680
681This program is distributed in the hope that it will be useful, but
682WITHOUT ANY WARRANTY; without even the implied warranty of
683MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE@. See the GNU
684General Public License for more details.
685
686You should have received a copy of the GNU General Public License
687along with this program. If not, see @url{http://www.gnu.org/licenses/}.
688@end smallexample
689
690Also add information on how to contact you by electronic and paper mail.
691
692If the program does terminal interaction, make it output a short
693notice like this when it starts in an interactive mode:
694
695@smallexample
696@var{program} Copyright (C) @var{year} @var{name of author}
697This program comes with ABSOLUTELY NO WARRANTY; for details type @samp{show w}.
698This is free software, and you are welcome to redistribute it
699under certain conditions; type @samp{show c} for details.
700@end smallexample
701
702The hypothetical commands @samp{show w} and @samp{show c} should show
703the appropriate parts of the General Public License. Of course, your
704program's commands might be different; for a GUI interface, you would
705use an ``about box''.
706
707You should also get your employer (if you work as a programmer) or school,
708if any, to sign a ``copyright disclaimer'' for the program, if necessary.
709For more information on this, and how to apply and follow the GNU GPL, see
710@url{http://www.gnu.org/licenses/}.
711
712The GNU General Public License does not permit incorporating your
713program into proprietary programs. If your program is a subroutine
714library, you may consider it more useful to permit linking proprietary
715applications with the library. If this is what you want to do, use
716the GNU Lesser General Public License instead of this License. But
717first, please read @url{http://www.gnu.org/philosophy/why-not-lgpl.html}.
diff --git a/doc/images/daemon_lego_block.png b/doc/images/daemon_lego_block.png
new file mode 100644
index 000000000..5a088b532
--- /dev/null
+++ b/doc/images/daemon_lego_block.png
Binary files differ
diff --git a/doc/images/gnunet-0-10-peerinfo.png b/doc/images/gnunet-0-10-peerinfo.png
new file mode 100644
index 000000000..c5e711aff
--- /dev/null
+++ b/doc/images/gnunet-0-10-peerinfo.png
Binary files differ
diff --git a/doc/images/gnunet-fs-gtk-0-10-star-tab.png b/doc/images/gnunet-fs-gtk-0-10-star-tab.png
new file mode 100644
index 000000000..d7993cc46
--- /dev/null
+++ b/doc/images/gnunet-fs-gtk-0-10-star-tab.png
Binary files differ
diff --git a/doc/images/gnunet-gtk-0-10-download-area.png b/doc/images/gnunet-gtk-0-10-download-area.png
new file mode 100644
index 000000000..8500d46c9
--- /dev/null
+++ b/doc/images/gnunet-gtk-0-10-download-area.png
Binary files differ
diff --git a/doc/images/gnunet-gtk-0-10-fs-menu.png b/doc/images/gnunet-gtk-0-10-fs-menu.png
new file mode 100644
index 000000000..dc20c45a9
--- /dev/null
+++ b/doc/images/gnunet-gtk-0-10-fs-menu.png
Binary files differ
diff --git a/doc/images/gnunet-gtk-0-10-fs-publish-editing.png b/doc/images/gnunet-gtk-0-10-fs-publish-editing.png
new file mode 100644
index 000000000..6f9f75ea6
--- /dev/null
+++ b/doc/images/gnunet-gtk-0-10-fs-publish-editing.png
Binary files differ
diff --git a/doc/images/gnunet-gtk-0-10-fs-publish-select.png b/doc/images/gnunet-gtk-0-10-fs-publish-select.png
new file mode 100644
index 000000000..50672e379
--- /dev/null
+++ b/doc/images/gnunet-gtk-0-10-fs-publish-select.png
Binary files differ
diff --git a/doc/images/gnunet-gtk-0-10-fs-publish-with-file.png b/doc/images/gnunet-gtk-0-10-fs-publish-with-file.png
new file mode 100644
index 000000000..b46542563
--- /dev/null
+++ b/doc/images/gnunet-gtk-0-10-fs-publish-with-file.png
Binary files differ
diff --git a/doc/images/gnunet-gtk-0-10-fs-publish-with-file_0.png b/doc/images/gnunet-gtk-0-10-fs-publish-with-file_0.png
new file mode 100644
index 000000000..b46542563
--- /dev/null
+++ b/doc/images/gnunet-gtk-0-10-fs-publish-with-file_0.png
Binary files differ
diff --git a/doc/images/gnunet-gtk-0-10-fs-publish.png b/doc/images/gnunet-gtk-0-10-fs-publish.png
new file mode 100644
index 000000000..033b38fa5
--- /dev/null
+++ b/doc/images/gnunet-gtk-0-10-fs-publish.png
Binary files differ
diff --git a/doc/images/gnunet-gtk-0-10-fs-published.png b/doc/images/gnunet-gtk-0-10-fs-published.png
new file mode 100644
index 000000000..fbd3dd6a3
--- /dev/null
+++ b/doc/images/gnunet-gtk-0-10-fs-published.png
Binary files differ
diff --git a/doc/images/gnunet-gtk-0-10-fs-search.png b/doc/images/gnunet-gtk-0-10-fs-search.png
new file mode 100644
index 000000000..bb64ab92e
--- /dev/null
+++ b/doc/images/gnunet-gtk-0-10-fs-search.png
Binary files differ
diff --git a/doc/images/gnunet-gtk-0-10-fs.png b/doc/images/gnunet-gtk-0-10-fs.png
new file mode 100644
index 000000000..c7a294878
--- /dev/null
+++ b/doc/images/gnunet-gtk-0-10-fs.png
Binary files differ
diff --git a/doc/images/gnunet-gtk-0-10-gns-a-done.png b/doc/images/gnunet-gtk-0-10-gns-a-done.png
new file mode 100644
index 000000000..f8231b3a6
--- /dev/null
+++ b/doc/images/gnunet-gtk-0-10-gns-a-done.png
Binary files differ
diff --git a/doc/images/gnunet-gtk-0-10-gns-a.png b/doc/images/gnunet-gtk-0-10-gns-a.png
new file mode 100644
index 000000000..39858d72c
--- /dev/null
+++ b/doc/images/gnunet-gtk-0-10-gns-a.png
Binary files differ
diff --git a/doc/images/gnunet-gtk-0-10-gns.png b/doc/images/gnunet-gtk-0-10-gns.png
new file mode 100644
index 000000000..c71a2bd7b
--- /dev/null
+++ b/doc/images/gnunet-gtk-0-10-gns.png
Binary files differ
diff --git a/doc/images/gnunet-gtk-0-10-identity.png b/doc/images/gnunet-gtk-0-10-identity.png
new file mode 100644
index 000000000..d0b426098
--- /dev/null
+++ b/doc/images/gnunet-gtk-0-10-identity.png
Binary files differ
diff --git a/doc/images/gnunet-gtk-0-10-search-selected.png b/doc/images/gnunet-gtk-0-10-search-selected.png
new file mode 100644
index 000000000..da1ad4d31
--- /dev/null
+++ b/doc/images/gnunet-gtk-0-10-search-selected.png
Binary files differ
diff --git a/doc/images/gnunet-gtk-0-10-traffic.png b/doc/images/gnunet-gtk-0-10-traffic.png
new file mode 100644
index 000000000..76458f717
--- /dev/null
+++ b/doc/images/gnunet-gtk-0-10-traffic.png
Binary files differ
diff --git a/doc/images/gnunet-gtk-0-10.png b/doc/images/gnunet-gtk-0-10.png
new file mode 100644
index 000000000..3615849a7
--- /dev/null
+++ b/doc/images/gnunet-gtk-0-10.png
Binary files differ
diff --git a/doc/images/gnunet-namestore-gtk-phone.png b/doc/images/gnunet-namestore-gtk-phone.png
new file mode 100644
index 000000000..3bb859629
--- /dev/null
+++ b/doc/images/gnunet-namestore-gtk-phone.png
Binary files differ
diff --git a/doc/images/gnunet-namestore-gtk-vpn.png b/doc/images/gnunet-namestore-gtk-vpn.png
new file mode 100644
index 000000000..c716729ba
--- /dev/null
+++ b/doc/images/gnunet-namestore-gtk-vpn.png
Binary files differ
diff --git a/doc/images/gnunet-setup-exit.png b/doc/images/gnunet-setup-exit.png
new file mode 100644
index 000000000..66bd972bc
--- /dev/null
+++ b/doc/images/gnunet-setup-exit.png
Binary files differ
diff --git a/doc/images/iceweasel-preferences.png b/doc/images/iceweasel-preferences.png
new file mode 100644
index 000000000..e62c2c4d9
--- /dev/null
+++ b/doc/images/iceweasel-preferences.png
Binary files differ
diff --git a/doc/images/iceweasel-proxy.png b/doc/images/iceweasel-proxy.png
new file mode 100644
index 000000000..9caad4508
--- /dev/null
+++ b/doc/images/iceweasel-proxy.png
Binary files differ
diff --git a/doc/images/service_lego_block.png b/doc/images/service_lego_block.png
new file mode 100644
index 000000000..56caf6b9c
--- /dev/null
+++ b/doc/images/service_lego_block.png
Binary files differ
diff --git a/doc/images/service_stack.png b/doc/images/service_stack.png
new file mode 100644
index 000000000..747d087b2
--- /dev/null
+++ b/doc/images/service_stack.png
Binary files differ