diff options
181 files changed, 20938 insertions, 16576 deletions
@@ -32,7 +32,6 @@ Werner Koch <wk@gnupg.org> | |||
32 | Contributions also came from: | 32 | Contributions also came from: |
33 | Adam Warrington [ UPnP ] | 33 | Adam Warrington [ UPnP ] |
34 | Adriano Peluso [ Documentation export to Texinfo ] | 34 | Adriano Peluso [ Documentation export to Texinfo ] |
35 | ng0 <ng0@infotropique.org> [ Documentation export to Texinfo ] | ||
36 | Alex Harper [ OS X CPU load ] | 35 | Alex Harper [ OS X CPU load ] |
37 | Andrew McDonald <andrew@mcdonald.org.uk> [ SHA-512] | 36 | Andrew McDonald <andrew@mcdonald.org.uk> [ SHA-512] |
38 | Andy Green <andy@warmcat.com> | 37 | Andy Green <andy@warmcat.com> |
@@ -69,6 +68,7 @@ Marko Räihä | |||
69 | Michael John Wensley <michael@wensley.org.uk> | 68 | Michael John Wensley <michael@wensley.org.uk> |
70 | Milan Bouchet-Valat <nalimilan@club.fr> | 69 | Milan Bouchet-Valat <nalimilan@club.fr> |
71 | Nathan Evans <evans@net.in.tum.de> | 70 | Nathan Evans <evans@net.in.tum.de> |
71 | ng0 <ng0@infotropique.org> [ Documentation export to Texinfo ] | ||
72 | Paul Ruth <ruth@cs.purdue.edu> | 72 | Paul Ruth <ruth@cs.purdue.edu> |
73 | Philipp Tölke <toelke@in.tum.de>, <pt@philipptoelke.de> | 73 | Philipp Tölke <toelke@in.tum.de>, <pt@philipptoelke.de> |
74 | Renaldo Ferreira <rf@cs.purdue.edu> | 74 | Renaldo Ferreira <rf@cs.purdue.edu> |
@@ -87,6 +87,7 @@ Uli Luckas <luckas@musoft.de> | |||
87 | Vasil Dimov <vd@datamax.bg> | 87 | Vasil Dimov <vd@datamax.bg> |
88 | Vitaly Minko <vitaly.minko@gmail.com> | 88 | Vitaly Minko <vitaly.minko@gmail.com> |
89 | Werner Koch <libgcrypt@g10code.com> [original code of libgcrypt] | 89 | Werner Koch <libgcrypt@g10code.com> [original code of libgcrypt] |
90 | xrs <xrs@mail36.net> [ multicast ] | ||
90 | 91 | ||
91 | Translations (webpage, documentation, as far as known): | 92 | Translations (webpage, documentation, as far as known): |
92 | Chinese : Di Ma | 93 | Chinese : Di Ma |
diff --git a/Makefile.am b/Makefile.am index 567e7b5ff..0466fcc10 100644 --- a/Makefile.am +++ b/Makefile.am | |||
@@ -1,6 +1,11 @@ | |||
1 | # This Makefile.am is in the public domain | 1 | # This Makefile.am is in the public domain |
2 | AM_CPPFLAGS = -I$(top_srcdir)/src/include | 2 | AM_CPPFLAGS = -I$(top_srcdir)/src/include |
3 | SUBDIRS = doc m4 src po pkgconfig | 3 | |
4 | if DOC_ONLY | ||
5 | SUBDIRS = doc m4 po pkgconfig | ||
6 | else | ||
7 | SUBDIRS = doc m4 src po pkgconfig | ||
8 | endif | ||
4 | 9 | ||
5 | if !TALER_ONLY | 10 | if !TALER_ONLY |
6 | SUBDIRS += contrib | 11 | SUBDIRS += contrib |
@@ -48,10 +48,10 @@ These are the direct dependencies for running GNUnet: | |||
48 | - libogg >= 1.3.0 (optional for experimental conversation tool) | 48 | - libogg >= 1.3.0 (optional for experimental conversation tool) |
49 | - python-zbar >= 0.10 (optional for gnunet-qr) | 49 | - python-zbar >= 0.10 (optional for gnunet-qr) |
50 | - TeX Live >= 2012 (optional for gnunet-bcd) | 50 | - TeX Live >= 2012 (optional for gnunet-bcd) |
51 | - Texinfo Tested with version 6.3 | 51 | - Texinfo >= 5.2 |
52 | - libglpk >= 4.45 (optional for experimental code) | 52 | - libglpk >= 4.45 (optional for experimental code) |
53 | 53 | ||
54 | Recommended autotools for compiling the SVN version are: | 54 | Recommended autotools for compiling the git version are: |
55 | - autoconf >= 2.59 | 55 | - autoconf >= 2.59 |
56 | - automake >= 1.11.1 | 56 | - automake >= 1.11.1 |
57 | - libtool >= 2.2 | 57 | - libtool >= 2.2 |
@@ -135,7 +135,7 @@ install' process with SUDO rights, the libraries will be installed to | |||
135 | "$GNUNET_PREFIX" and you will have to move them to "/lib/" | 135 | "$GNUNET_PREFIX" and you will have to move them to "/lib/" |
136 | manually. | 136 | manually. |
137 | 137 | ||
138 | Finally, if you are compiling the code from subversion, you have to | 138 | Finally, if you are compiling the code from git, you have to |
139 | run ". bootstrap" before ./configure. If you receive an error during | 139 | run ". bootstrap" before ./configure. If you receive an error during |
140 | the running of ". bootstrap" that looks like "macro `AM_PATH_GTK' not | 140 | the running of ". bootstrap" that looks like "macro `AM_PATH_GTK' not |
141 | found in library", you may need to run aclocal by hand with the -I | 141 | found in library", you may need to run aclocal by hand with the -I |
@@ -1,4 +1,9 @@ | |||
1 | #!/bin/sh | 1 | #!/bin/sh |
2 | rm -rf libltdl | 2 | rm -rf libltdl |
3 | echo -n "checking for libtoolize / libtool... " | ||
4 | which glibtoolize || which libtoolize || which libtool || { | ||
5 | echo "*** No libtoolize (libtool) or libtool found, please install it ***" | ||
6 | exit 1 | ||
7 | } | ||
3 | autoreconf -if | 8 | autoreconf -if |
4 | contrib/pogen.sh | 9 | contrib/pogen.sh |
diff --git a/configure.ac b/configure.ac index 101fc77cb..c97596971 100644 --- a/configure.ac +++ b/configure.ac | |||
@@ -248,6 +248,11 @@ then | |||
248 | fi | 248 | fi |
249 | AC_DEFINE_UNQUOTED([NEED_LIBGCRYPT_VERSION], "$NEED_LIBGCRYPT_VERSION", [required libgcrypt version]) | 249 | AC_DEFINE_UNQUOTED([NEED_LIBGCRYPT_VERSION], "$NEED_LIBGCRYPT_VERSION", [required libgcrypt version]) |
250 | 250 | ||
251 | # TODO: add check for VERSION | ||
252 | # TODO: add check for alternatives | ||
253 | # TODO: add switch to skip documentation building | ||
254 | AM_MISSING_PROG([MAKEINFO], [makeinfo]) | ||
255 | |||
251 | # Adam shostack suggests the following for Windows: | 256 | # Adam shostack suggests the following for Windows: |
252 | # -D_FORTIFY_SOURCE=2 -fstack-protector-all | 257 | # -D_FORTIFY_SOURCE=2 -fstack-protector-all |
253 | AC_ARG_ENABLE(gcc-hardening, | 258 | AC_ARG_ENABLE(gcc-hardening, |
@@ -565,6 +570,23 @@ AC_CHECK_LIB([kvm],[kvm_open]) | |||
565 | AC_CHECK_LIB([kstat],[kstat_open]) | 570 | AC_CHECK_LIB([kstat],[kstat_open]) |
566 | 571 | ||
567 | 572 | ||
573 | # should the build process be restricted to only building | ||
574 | # the documentation? | ||
575 | AC_MSG_CHECKING(whether to build documentation ONLY) | ||
576 | AC_ARG_ENABLE([documentation], | ||
577 | [AS_HELP_STRING([--enable-documentation], [only build the documentation])], | ||
578 | [doc_only=${enableval}], | ||
579 | [doc_only=no]) | ||
580 | AC_MSG_RESULT($doc_only) | ||
581 | if test "x$doc_only" = "xyes" | ||
582 | then | ||
583 | AM_CONDITIONAL([DOC_ONLY],true) | ||
584 | AC_DEFINE([DOC_ONLY],[1],[Building the documentation]) | ||
585 | else | ||
586 | AM_CONDITIONAL([DOC_ONLY],false) | ||
587 | AC_DEFINE([DOC_ONLY],[0],[Canonical compilation]) | ||
588 | fi | ||
589 | |||
568 | # should the build process be restricted to the code required | 590 | # should the build process be restricted to the code required |
569 | # for GNU Taler wallets? | 591 | # for GNU Taler wallets? |
570 | AC_MSG_CHECKING(whether to compile GNU Taler Wallet library ONLY) | 592 | AC_MSG_CHECKING(whether to compile GNU Taler Wallet library ONLY) |
@@ -1509,6 +1531,7 @@ contrib/Makefile | |||
1509 | doc/Makefile | 1531 | doc/Makefile |
1510 | doc/man/Makefile | 1532 | doc/man/Makefile |
1511 | doc/doxygen/Makefile | 1533 | doc/doxygen/Makefile |
1534 | doc/documentation/Makefile | ||
1512 | m4/Makefile | 1535 | m4/Makefile |
1513 | po/Makefile.in | 1536 | po/Makefile.in |
1514 | src/Makefile | 1537 | src/Makefile |
diff --git a/contrib/Makefile.am b/contrib/Makefile.am index 07cff424c..ac8b15188 100644 --- a/contrib/Makefile.am +++ b/contrib/Makefile.am | |||
@@ -70,7 +70,20 @@ CLEANFILES = \ | |||
70 | 70 | ||
71 | do_subst = $(SED) -e 's,[@]PYTHON[@],$(PYTHON),g' | 71 | do_subst = $(SED) -e 's,[@]PYTHON[@],$(PYTHON),g' |
72 | 72 | ||
73 | %.py: %.py.in Makefile | 73 | # Use SUFFIX Extension rules, they are more portable for every |
74 | # implementation of 'make'. | ||
75 | # You'll also run into the "'%' is a GNU make extension warning" | ||
76 | # if you use this: | ||
77 | # | ||
78 | #%.py: %.py.in Makefile | ||
79 | # $(do_subst) < $< > $@ | ||
80 | # chmod +x $@ | ||
81 | # | ||
82 | # instead of this: | ||
83 | |||
84 | SUFFIXES = .py.in .py | ||
85 | |||
86 | .py.in.py: | ||
74 | $(do_subst) < $< > $@ | 87 | $(do_subst) < $< > $@ |
75 | chmod +x $@ | 88 | chmod +x $@ |
76 | 89 | ||
diff --git a/contrib/packages/guix/gnunet-doc.scm b/contrib/packages/guix/gnunet-doc.scm index a2a9a2393..8e775e801 100644 --- a/contrib/packages/guix/gnunet-doc.scm +++ b/contrib/packages/guix/gnunet-doc.scm | |||
@@ -46,6 +46,7 @@ | |||
46 | (gnu packages gstreamer) | 46 | (gnu packages gstreamer) |
47 | (gnu packages gtk) | 47 | (gnu packages gtk) |
48 | (gnu packages guile) | 48 | (gnu packages guile) |
49 | (gnu packages graphviz) | ||
49 | (gnu packages image) | 50 | (gnu packages image) |
50 | (gnu packages image-viewers) | 51 | (gnu packages image-viewers) |
51 | (gnu packages libidn) | 52 | (gnu packages libidn) |
@@ -64,13 +65,14 @@ | |||
64 | (gnu packages video) | 65 | (gnu packages video) |
65 | (gnu packages web) | 66 | (gnu packages web) |
66 | (gnu packages xiph) | 67 | (gnu packages xiph) |
68 | ;;(gnunet packages texlive) ;GNUnet module including texlive-2012 WIP | ||
67 | ((guix licenses) #:prefix license:)) | 69 | ((guix licenses) #:prefix license:)) |
68 | 70 | ||
69 | (define %source-dir (string-append (current-source-directory) | 71 | (define %source-dir (string-append (current-source-directory) |
70 | "/../../../")) | 72 | "/../../../")) |
71 | 73 | ||
72 | (define gnunet-doc | 74 | (define gnunet-doc |
73 | (let* ((revision "1") | 75 | (let* ((revision "2") |
74 | (select? (delay (or (git-predicate | 76 | (select? (delay (or (git-predicate |
75 | (string-append (current-source-directory) | 77 | (string-append (current-source-directory) |
76 | "/../../../")) | 78 | "/../../../")) |
@@ -92,7 +94,7 @@ | |||
92 | ("gnurl" ,gnurl) | 94 | ("gnurl" ,gnurl) |
93 | ("gstreamer" ,gstreamer) | 95 | ("gstreamer" ,gstreamer) |
94 | ("gst-plugins-base" ,gst-plugins-base) | 96 | ("gst-plugins-base" ,gst-plugins-base) |
95 | ("gnutls" ,gnutls) ;Change to gnutls/dane once it is merged. | 97 | ("gnutls/dane" ,gnutls/dane) |
96 | ("libextractor" ,libextractor) | 98 | ("libextractor" ,libextractor) |
97 | ("libgcrypt" ,libgcrypt) | 99 | ("libgcrypt" ,libgcrypt) |
98 | ("libidn" ,libidn) | 100 | ("libidn" ,libidn) |
@@ -107,7 +109,7 @@ | |||
107 | ("mysql" ,mysql) | 109 | ("mysql" ,mysql) |
108 | ("zlib" ,zlib) | 110 | ("zlib" ,zlib) |
109 | ("perl" ,perl) | 111 | ("perl" ,perl) |
110 | ("python" ,python) ; tests and gnunet-qr | 112 | ("python-2" ,python-2) ; tests and gnunet-qr |
111 | ("jansson" ,jansson) | 113 | ("jansson" ,jansson) |
112 | ("nss" ,nss) | 114 | ("nss" ,nss) |
113 | ("glib" ,glib "bin") | 115 | ("glib" ,glib "bin") |
@@ -115,17 +117,21 @@ | |||
115 | ("bluez" ,bluez) ; for optional bluetooth feature | 117 | ("bluez" ,bluez) ; for optional bluetooth feature |
116 | ("glib" ,glib) | 118 | ("glib" ,glib) |
117 | ;;("texlive-minimal" ,texlive-minimal) ; optional. | 119 | ;;("texlive-minimal" ,texlive-minimal) ; optional. |
118 | ("texlive" ,texlive) | 120 | ("texlive" ,texlive) ;TODO: Stabilize Texlive-2012 package |
119 | ("libogg" ,libogg))) | 121 | ("libogg" ,libogg))) |
120 | (native-inputs | 122 | (native-inputs |
121 | `(("pkg-config" ,pkg-config) | 123 | `(("pkg-config" ,pkg-config) |
122 | ("autoconf" ,autoconf) | 124 | ("autoconf" ,autoconf) |
123 | ("automake" ,automake) | 125 | ("automake" ,automake) |
124 | ("gnu-gettext" ,gnu-gettext) | 126 | ("gnu-gettext" ,gnu-gettext) |
125 | ("texinfo" ,texinfo) | 127 | ("graphviz" ,graphviz) ; dot |
128 | ("texinfo-5" ,texinfo-5) ; Debian stable | ||
129 | ("which" ,which) | ||
126 | ("libtool" ,libtool))) | 130 | ("libtool" ,libtool))) |
127 | (arguments | 131 | (arguments |
128 | `(#:tests? #f ;Don't run tests | 132 | `(#:configure-flags |
133 | (list "--enable-documentation") | ||
134 | #:tests? #f ;Don't run tests | ||
129 | #:phases | 135 | #:phases |
130 | (modify-phases %standard-phases | 136 | (modify-phases %standard-phases |
131 | (add-after 'unpack 'autoconf | 137 | (add-after 'unpack 'autoconf |
@@ -137,25 +143,37 @@ | |||
137 | (zero? (system* "sh" "bootstrap")))) | 143 | (zero? (system* "sh" "bootstrap")))) |
138 | (replace 'build | 144 | (replace 'build |
139 | (lambda _ | 145 | (lambda _ |
140 | (chdir "doc") | 146 | (chdir "doc/documentation") |
141 | (zero? (system* "make" "doc-all-give-me-the-noise")))) | 147 | ;;(zero? (system* "make" "dev-build")))) |
148 | (zero? (system* "sh" "run-gendocs.sh")))) | ||
149 | ;; (zero? (system* "make" "pdf")) | ||
150 | ;; (zero? (system* "make" "html")) | ||
151 | ;; (zero? (system* "make" "info")))) | ||
152 | ;;(zero? (system* "make" "doc-all-give-me-the-noise")))) | ||
142 | (replace 'install | 153 | (replace 'install |
143 | (lambda* (#:key outputs #:allow-other-keys) | 154 | (lambda _ |
144 | (let* ((out (assoc-ref outputs "out")) | 155 | (zero? (system* "make" "doc-gendoc-install"))))))) |
145 | (doc (string-append out "/share/doc/gnunet"))) | 156 | ;;(lambda* (#:key outputs #:allow-other-keys) |
146 | (mkdir-p doc) | 157 | ;; (let* ((out (assoc-ref outputs "out")) |
147 | (mkdir-p (string-append doc "/gnunet")) | 158 | ;; (doc (string-append out "/share/doc/gnunet"))) |
148 | (install-file "gnunet.pdf" doc) | 159 | ;; (mkdir-p doc) |
149 | (install-file "gnunet.info" doc) | 160 | ;; (copy-recursively "images" |
150 | (copy-recursively "gnunet" | 161 | ;; (string-append doc |
151 | (string-append doc | 162 | ;; "/images")) |
152 | "/gnunet")) | 163 | ;; (mkdir-p (string-append doc "/gnunet")) |
153 | (install-file "gnunet-c-tutorial.pdf" doc) | 164 | ;; (install-file "gnunet.pdf" doc) |
154 | (install-file "gnunet-c-tutorial.info" doc) | 165 | ;; (install-file "gnunet.info" doc) |
155 | (copy-recursively "gnunet-c-tutorial" | 166 | ;; (install-file "gnunet.log" doc) ;TODO: Move to 'dev' output? |
156 | (string-append doc | 167 | ;; (copy-recursively "gnunet" |
157 | "/gnunet-c-tutorial"))) | 168 | ;; (string-append doc |
158 | #t))))) | 169 | ;; "/gnunet")) |
170 | ;; (install-file "gnunet-c-tutorial.pdf" doc) | ||
171 | ;; (install-file "gnunet-c-tutorial.info" doc) | ||
172 | ;; (install-file "gnunet-c-tutorial.log" doc) ;TODO: Move to 'dev' output? | ||
173 | ;; (copy-recursively "gnunet-c-tutorial" | ||
174 | ;; (string-append doc | ||
175 | ;; "/gnunet-c-tutorial"))) | ||
176 | ;; #t))))) | ||
159 | (synopsis "Documentation of GNUnet") | 177 | (synopsis "Documentation of GNUnet") |
160 | (description | 178 | (description |
161 | "GNUnet documentation build") | 179 | "GNUnet documentation build") |
diff --git a/contrib/packages/guix/gnunet.scm b/contrib/packages/guix/gnunet.scm new file mode 100644 index 000000000..d8eee1805 --- /dev/null +++ b/contrib/packages/guix/gnunet.scm | |||
@@ -0,0 +1,167 @@ | |||
1 | ;;; This file is part of GNUnet. | ||
2 | ;;; Copyright (C) 2016, 2017 GNUnet e.V. | ||
3 | ;;; | ||
4 | ;;; GNUnet is free software; you can redistribute it and/or modify | ||
5 | ;;; it under the terms of the GNU General Public License as published | ||
6 | ;;; by the Free Software Foundation; either version 3, or (at your | ||
7 | ;;; option) any later version. | ||
8 | ;;; | ||
9 | ;;; GNUnet is distributed in the hope that it will be useful, but | ||
10 | ;;; WITHOUT ANY WARRANTY; without even the implied warranty of | ||
11 | ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
12 | ;;; General Public License for more details. | ||
13 | ;;; | ||
14 | ;;; You should have received a copy of the GNU General Public License | ||
15 | ;;; along with GNUnet; see the file COPYING. If not, write to the | ||
16 | ;;; Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, | ||
17 | ;;; Boston, MA 02110-1301, USA. | ||
18 | |||
19 | (use-modules | ||
20 | (ice-9 popen) | ||
21 | (ice-9 match) | ||
22 | (ice-9 rdelim) | ||
23 | (guix packages) | ||
24 | (guix build-system gnu) | ||
25 | (guix gexp) | ||
26 | ((guix build utils) #:select (with-directory-excursion)) | ||
27 | (guix git-download) | ||
28 | (guix utils) ; current-source-directory | ||
29 | (gnu packages) | ||
30 | (gnu packages aidc) | ||
31 | (gnu packages autotools) | ||
32 | (gnu packages backup) | ||
33 | (gnu packages base) | ||
34 | (gnu packages compression) | ||
35 | (gnu packages curl) | ||
36 | (gnu packages databases) | ||
37 | (gnu packages file) | ||
38 | (gnu packages gettext) | ||
39 | (gnu packages glib) | ||
40 | (gnu packages gnome) | ||
41 | (gnu packages gnunet) | ||
42 | (gnu packages gnupg) | ||
43 | (gnu packages gnuzilla) | ||
44 | (gnu packages groff) | ||
45 | (gnu packages gstreamer) | ||
46 | (gnu packages gtk) | ||
47 | (gnu packages guile) | ||
48 | (gnu packages image) | ||
49 | (gnu packages image-viewers) | ||
50 | (gnu packages libidn) | ||
51 | (gnu packages libunistring) | ||
52 | (gnu packages linux) | ||
53 | (gnu packages maths) | ||
54 | (gnu packages multiprecision) | ||
55 | (gnu packages perl) | ||
56 | (gnu packages pkg-config) | ||
57 | (gnu packages pulseaudio) | ||
58 | (gnu packages python) | ||
59 | (gnu packages tex) | ||
60 | (gnu packages texinfo) | ||
61 | (gnu packages tex) | ||
62 | (gnu packages tls) | ||
63 | (gnu packages upnp) | ||
64 | (gnu packages video) | ||
65 | (gnu packages web) | ||
66 | (gnu packages xiph) | ||
67 | ((guix licenses) #:prefix license:)) | ||
68 | |||
69 | (define %source-dir (string-append (current-source-directory) | ||
70 | "/../../../")) | ||
71 | |||
72 | (define gnunet-test-git | ||
73 | (let* ((revision "1") | ||
74 | (select? (delay (or (git-predicate | ||
75 | (string-append (current-source-directory) | ||
76 | "/../../../")) | ||
77 | source-file?)))) | ||
78 | (package | ||
79 | (name "gnunet-test-git") | ||
80 | (version (string-append "0.10.1-" revision "." "dev")) | ||
81 | (source | ||
82 | (local-file ;;"../../.." | ||
83 | ;;%source-dir | ||
84 | ;;(string-append (getcwd) "/../../../") | ||
85 | (string-append (getcwd)) ;drrty hack and this assumes one static position FIXME! | ||
86 | #:recursive? #t)) | ||
87 | ;;#:select? (git-predicate %source-dir))) | ||
88 | ;;#:select? (force select?))) | ||
89 | (build-system gnu-build-system) | ||
90 | (inputs | ||
91 | `(("glpk" ,glpk) | ||
92 | ("gnurl" ,gnurl) | ||
93 | ("gstreamer" ,gstreamer) | ||
94 | ("gst-plugins-base" ,gst-plugins-base) | ||
95 | ("gnutls/dane" ,gnutls/dane) ;Change to gnutls/dane once it is merged. | ||
96 | ("libextractor" ,libextractor) | ||
97 | ("libgcrypt" ,libgcrypt) | ||
98 | ("libidn" ,libidn) | ||
99 | ("libmicrohttpd" ,libmicrohttpd) | ||
100 | ("libltdl" ,libltdl) | ||
101 | ("libunistring" ,libunistring) | ||
102 | ("openssl" ,openssl) | ||
103 | ("opus" ,opus) | ||
104 | ("pulseaudio" ,pulseaudio) | ||
105 | ("sqlite" ,sqlite) | ||
106 | ("postgresql" ,postgresql) | ||
107 | ("mysql" ,mysql) | ||
108 | ("zlib" ,zlib) | ||
109 | ("perl" ,perl) | ||
110 | ("python" ,python) ; tests and gnunet-qr | ||
111 | ("jansson" ,jansson) | ||
112 | ("nss" ,nss) | ||
113 | ("glib" ,glib "bin") | ||
114 | ("gmp" ,gmp) | ||
115 | ("bluez" ,bluez) ; for optional bluetooth feature | ||
116 | ("glib" ,glib) | ||
117 | ;; TODO: figure out the right texlive parts. | ||
118 | ;;("texlive-minimal" ,texlive-minimal) | ||
119 | ("texlive" ,texlive) | ||
120 | ("miniupnpc" ,miniupnpc) | ||
121 | ("libogg" ,libogg))) | ||
122 | (native-inputs | ||
123 | `(("pkg-config" ,pkg-config) | ||
124 | ("autoconf" ,autoconf) | ||
125 | ("automake" ,automake) | ||
126 | ("gnu-gettext" ,gnu-gettext) | ||
127 | ("which" ,which) | ||
128 | ("texinfo" ,texinfo-5) ; Debian stable: 5.2 | ||
129 | ("libtool" ,libtool))) | ||
130 | (outputs '("out" "debug")) | ||
131 | (arguments | ||
132 | `(#:configure-flags | ||
133 | (list (string-append "--with-nssdir=" %output "/lib") | ||
134 | ;;"--enable-gcc-hardening" | ||
135 | ;;"--enable-linker-hardening" | ||
136 | "--enable-logging=verbose" | ||
137 | "CFLAGS=-ggdb -O0") | ||
138 | #:phases | ||
139 | ;; swap check and install phases and set paths to installed bin | ||
140 | (modify-phases %standard-phases | ||
141 | (add-after 'unpack 'patch-bin-sh | ||
142 | (lambda _ | ||
143 | (substitute* "bootstrap" | ||
144 | (("contrib/pogen.sh") "sh contrib/pogen.sh")) | ||
145 | (for-each (lambda (f) (chmod f #o755)) | ||
146 | (find-files "po" "")) | ||
147 | #t)) | ||
148 | (add-after 'patch-bin-sh 'bootstrap | ||
149 | (lambda _ | ||
150 | (zero? (system* "sh" "bootstrap")))) | ||
151 | (delete 'check) | ||
152 | ;; XXX: https://gnunet.org/bugs/view.php?id=4619 | ||
153 | (add-after 'install 'set-path-for-check | ||
154 | (lambda* (#:key outputs #:allow-other-keys) | ||
155 | (let* ((out (assoc-ref outputs "out")) | ||
156 | (bin (string-append out "/bin")) | ||
157 | (lib (string-append out "/lib"))) | ||
158 | (setenv "GNUNET_PREFIX" lib) | ||
159 | (setenv "PATH" (string-append (getenv "PATH") ":" bin)) | ||
160 | (zero? (system* "make" "check")))))))) | ||
161 | (synopsis "tests enabled without experimental") | ||
162 | (description | ||
163 | "GNUnet from git HEAD") | ||
164 | (license license:gpl3+) | ||
165 | (home-page "https://gnunet.org")))) | ||
166 | |||
167 | gnunet-test-git | ||
diff --git a/contrib/packages/guix/guix-env.scm b/contrib/packages/guix/guix-env.scm index 6946fee57..da4a60b73 100644 --- a/contrib/packages/guix/guix-env.scm +++ b/contrib/packages/guix/guix-env.scm | |||
@@ -146,7 +146,8 @@ | |||
146 | ("autoconf" ,autoconf) | 146 | ("autoconf" ,autoconf) |
147 | ("automake" ,automake) | 147 | ("automake" ,automake) |
148 | ("gnu-gettext" ,gnu-gettext) | 148 | ("gnu-gettext" ,gnu-gettext) |
149 | ("texinfo" ,texinfo) | 149 | ("which" ,which) |
150 | ("texinfo" ,texinfo-5) ; Debian stable: 5.2 | ||
150 | ("libtool" ,libtool))) | 151 | ("libtool" ,libtool))) |
151 | ;; TODO: To make use of out:debug, which carries the symbols, | 152 | ;; TODO: To make use of out:debug, which carries the symbols, |
152 | ;; this file needs to fixed. | 153 | ;; this file needs to fixed. |
diff --git a/contrib/packages/guix/packages/gnunet/packages/gnunet.scm b/contrib/packages/guix/packages/gnunet/packages/gnunet.scm index 7840705a6..be529ec1d 100644 --- a/contrib/packages/guix/packages/gnunet/packages/gnunet.scm +++ b/contrib/packages/guix/packages/gnunet/packages/gnunet.scm | |||
@@ -26,6 +26,7 @@ | |||
26 | #:use-module (gnu packages admin) | 26 | #:use-module (gnu packages admin) |
27 | #:use-module (gnu packages aidc) | 27 | #:use-module (gnu packages aidc) |
28 | #:use-module (gnu packages autotools) | 28 | #:use-module (gnu packages autotools) |
29 | #:use-module (gnu packages base) | ||
29 | #:use-module (gnu packages bison) | 30 | #:use-module (gnu packages bison) |
30 | #:use-module (gnu packages compression) | 31 | #:use-module (gnu packages compression) |
31 | #:use-module (gnu packages databases) | 32 | #:use-module (gnu packages databases) |
@@ -60,9 +61,10 @@ | |||
60 | ;; TODO: Use HEAD without checking sum of it. | 61 | ;; TODO: Use HEAD without checking sum of it. |
61 | ;; Explanation for name scheme: UNIXPATH is capped at 108 characters, | 62 | ;; Explanation for name scheme: UNIXPATH is capped at 108 characters, |
62 | ;; this causes lots of tests to fail. | 63 | ;; this causes lots of tests to fail. |
64 | ;; FIXME: make this file MUCH shorter. | ||
63 | (define-public gnunetg | 65 | (define-public gnunetg |
64 | (let* ((commit "b005d5e4dac03fcfdabf0d0de434da3b295f6d63") | 66 | (let* ((commit "3c3090717610ea787fdd3562901329254a6af0d6") |
65 | (revision "30")) | 67 | (revision "32")) |
66 | (package | 68 | (package |
67 | (inherit gnunet) | 69 | (inherit gnunet) |
68 | (name "gnunetg") | 70 | (name "gnunetg") |
@@ -77,7 +79,7 @@ | |||
77 | (file-name (string-append name "-" version "-checkout")) | 79 | (file-name (string-append name "-" version "-checkout")) |
78 | (sha256 | 80 | (sha256 |
79 | (base32 | 81 | (base32 |
80 | "10wfb58pi55399cw100vplq3f8nxg2vl6sywcmvkx3wg1d3firla")))) | 82 | "0g0x1r833jkssfd2sndy37509dqf9f8myjvg7mnadwc1irp393bl")))) |
81 | (build-system gnu-build-system) | 83 | (build-system gnu-build-system) |
82 | (inputs | 84 | (inputs |
83 | `(("glpk" ,glpk) | 85 | `(("glpk" ,glpk) |
@@ -112,6 +114,7 @@ | |||
112 | ("autoconf" ,autoconf) | 114 | ("autoconf" ,autoconf) |
113 | ("automake" ,automake) | 115 | ("automake" ,automake) |
114 | ("gnu-gettext" ,gnu-gettext) | 116 | ("gnu-gettext" ,gnu-gettext) |
117 | ("which" ,which) | ||
115 | ("texinfo" ,texinfo) | 118 | ("texinfo" ,texinfo) |
116 | ("libtool" ,libtool))) | 119 | ("libtool" ,libtool))) |
117 | (outputs '("out" "debug")) | 120 | (outputs '("out" "debug")) |
@@ -199,6 +202,7 @@ | |||
199 | (native-inputs | 202 | (native-inputs |
200 | `(("pkg-config" ,pkg-config) | 203 | `(("pkg-config" ,pkg-config) |
201 | ("autoconf" ,autoconf) | 204 | ("autoconf" ,autoconf) |
205 | ("which" ,which) | ||
202 | ("automake" ,automake) | 206 | ("automake" ,automake) |
203 | ("gnu-gettext" ,gnu-gettext) | 207 | ("gnu-gettext" ,gnu-gettext) |
204 | ("texinfo" ,texinfo) | 208 | ("texinfo" ,texinfo) |
@@ -333,6 +337,47 @@ | |||
333 | (zero? (system* "make" "check")))))))) | 337 | (zero? (system* "make" "check")))))))) |
334 | (synopsis "gnunet, full git with tests enabled with parallel tests"))) | 338 | (synopsis "gnunet, full git with tests enabled with parallel tests"))) |
335 | 339 | ||
340 | (define-public gnunetg-test | ||
341 | (package | ||
342 | (inherit gnunetg) | ||
343 | (name "gnunetg-test") | ||
344 | (arguments | ||
345 | `(#:configure-flags | ||
346 | (list (string-append "--with-nssdir=" %output "/lib") | ||
347 | "--enable-gcc-hardening" | ||
348 | "--enable-linker-hardening" | ||
349 | |||
350 | ;;"--enable-poisoning" | ||
351 | ;;"--enable-sanitizer" | ||
352 | "--enable-logging=verbose" | ||
353 | "CFLAGS=-ggdb -O0") | ||
354 | ;; #:parallel-tests? #f ; parallel building seems to fail | ||
355 | ;;#:tests? #f ; fail: test_gnunet_statistics.py | ||
356 | #:phases | ||
357 | ;; swap check and install phases and set paths to installed bin | ||
358 | (modify-phases %standard-phases | ||
359 | (add-after 'unpack 'patch-bin-sh | ||
360 | (lambda _ | ||
361 | (substitute* "bootstrap" | ||
362 | (("contrib/pogen.sh") "sh contrib/pogen.sh")) | ||
363 | (for-each (lambda (f) (chmod f #o755)) | ||
364 | (find-files "po" "")) | ||
365 | #t)) | ||
366 | (add-after 'patch-bin-sh 'bootstrap | ||
367 | (lambda _ | ||
368 | (zero? (system* "sh" "bootstrap")))) | ||
369 | (delete 'check) | ||
370 | ;; XXX: https://gnunet.org/bugs/view.php?id=4619 | ||
371 | (add-after 'install 'set-path-for-check | ||
372 | (lambda* (#:key outputs #:allow-other-keys) | ||
373 | (let* ((out (assoc-ref outputs "out")) | ||
374 | (bin (string-append out "/bin")) | ||
375 | (lib (string-append out "/lib"))) | ||
376 | (setenv "GNUNET_PREFIX" lib) | ||
377 | (setenv "PATH" (string-append (getenv "PATH") ":" bin)) | ||
378 | (zero? (system* "make" "check")))))))) | ||
379 | (synopsis "gnunet, full git with tests enabled without experimental"))) | ||
380 | |||
336 | ;; ... and one package to test the package with "parallel-tests? #f" | 381 | ;; ... and one package to test the package with "parallel-tests? #f" |
337 | (define-public gnunetgftn | 382 | (define-public gnunetgftn |
338 | (package | 383 | (package |
diff --git a/contrib/packages/guix/packages/gnunet/packages/texlive.scm b/contrib/packages/guix/packages/gnunet/packages/texlive.scm new file mode 100644 index 000000000..77256a947 --- /dev/null +++ b/contrib/packages/guix/packages/gnunet/packages/texlive.scm | |||
@@ -0,0 +1,182 @@ | |||
1 | ;;; This file is part of GNUnet. | ||
2 | ;;; Copyright (C) 2017 GNUnet e.V. | ||
3 | ;;; | ||
4 | ;;; GNUnet is free software; you can redistribute it and/or modify | ||
5 | ;;; it under the terms of the GNU General Public License as published | ||
6 | ;;; by the Free Software Foundation; either version 3, or (at your | ||
7 | ;;; option) any later version. | ||
8 | ;;; | ||
9 | ;;; GNUnet is distributed in the hope that it will be useful, but | ||
10 | ;;; WITHOUT ANY WARRANTY; without even the implied warranty of | ||
11 | ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
12 | ;;; General Public License for more details. | ||
13 | ;;; | ||
14 | ;;; You should have received a copy of the GNU General Public License | ||
15 | ;;; along with GNUnet; see the file COPYING. If not, write to the | ||
16 | ;;; Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, | ||
17 | ;;; Boston, MA 02110-1301, USA. | ||
18 | |||
19 | ;; Initially based on Guix commit eb0119efde826590a13973ab928c173780a7f257 | ||
20 | |||
21 | (define-module (gnunet packages texlive) | ||
22 | #:use-module ((guix licenses) | ||
23 | #:renamer (symbol-prefix-proc 'license:)) | ||
24 | #:use-module (guix packages) | ||
25 | #:use-module (guix download) | ||
26 | #:use-module (guix build-system gnu) | ||
27 | #:use-module (gnu packages) | ||
28 | #:use-module (gnu packages compression) | ||
29 | #:use-module (gnu packages fontutils) | ||
30 | #:use-module (gnu packages ghostscript) | ||
31 | #:use-module (gnu packages icu4c) | ||
32 | #:use-module (gnu packages image) | ||
33 | #:use-module (gnu packages pdf) | ||
34 | #:use-module (gnu packages perl) | ||
35 | #:use-module (gnu packages pkg-config) | ||
36 | #:use-module (gnu packages python) | ||
37 | #:use-module (gnu packages compression)) | ||
38 | |||
39 | (define texlive-extra-src | ||
40 | (origin | ||
41 | (method url-fetch) | ||
42 | (uri "ftp://tug.org/historic/systems/texlive/2012/texlive-20120701-extra.tar.xz") | ||
43 | ;;(uri "ftp://tug.org/historic/systems/texlive/2012/texlive-20120701-extra.tar.xz") | ||
44 | (sha256 (base32 | ||
45 | "0cb8fnv4x281gy5ka779f00ssdmdpjj4x3pkh9j9vq45hrwg3522")))) | ||
46 | |||
47 | (define texlive-texmf-src | ||
48 | (origin | ||
49 | (method url-fetch) | ||
50 | (uri "ftp://tug.org/historic/systems/texlive/2012/texlive-20120701-texmf.tar.xz") | ||
51 | ;;(uri "ftp://tug.org/historic/systems/texlive/2012/texlive-20120701-texmf.tar.xz") | ||
52 | (sha256 (base32 | ||
53 | "1fn1dg9k7pnh8a80j23zfkbrfnqyc4c2w4ss30dpkqj490nxsywq")))) | ||
54 | |||
55 | ;; Note that right now this does not include security fixes! | ||
56 | ;; FIXME: Needs more fixes, currently this fails building! | ||
57 | |||
58 | (define-public texlive | ||
59 | (package | ||
60 | (name "texlive") | ||
61 | (version "2012") | ||
62 | (source (origin | ||
63 | (method url-fetch) | ||
64 | (uri "ftp://tug.org/historic/systems/texlive/2012/texlive-20120701-source.tar.xz") | ||
65 | ;;(uri "ftp://tug.org/historic/systems/texlive/2012/texlive-20120701-source.tar.xz") | ||
66 | (sha256 (base32 | ||
67 | "10bcrdfsqnc6y3gqcb8ndnjy07i5kz63as39irbq4gmcbmyn2rln")))) | ||
68 | (build-system gnu-build-system) | ||
69 | (inputs `(("texlive-extra-src" ,texlive-extra-src) | ||
70 | ("texlive-texmf-src" ,texlive-texmf-src) | ||
71 | ("fontconfig" ,fontconfig) | ||
72 | ("freetype" ,freetype) | ||
73 | ("icu4c" ,icu4c) | ||
74 | ("ghostscript" ,ghostscript) | ||
75 | ("libpng" ,libpng) | ||
76 | ("perl" ,perl) | ||
77 | ("poppler" ,poppler) | ||
78 | ("pkg-config" ,pkg-config) | ||
79 | ;; FIXME: Add interpreters csh, fontforge and ruby, | ||
80 | ;; once they are available. | ||
81 | ("python" ,python) | ||
82 | ("teckit" ,teckit) | ||
83 | ("t1lib" ,t1lib) | ||
84 | ("zlib" ,zlib) | ||
85 | ("zziplib" ,zziplib))) | ||
86 | (outputs '("out" "data")) | ||
87 | (arguments | ||
88 | `(#:out-of-source? #t | ||
89 | #:configure-flags | ||
90 | `("--disable-native-texlive-build" | ||
91 | ;; Although the texmf and texmf-dist data is taken from | ||
92 | ;; texlive-texmf, setting datarootdir is still useful: | ||
93 | ;; "make install" creates symbolic links to scripts in this place. | ||
94 | ,(string-append "--datarootdir=" (assoc-ref %outputs "data")) | ||
95 | ,(string-append "--infodir=" (assoc-ref %outputs "out") "/share/info") | ||
96 | ,(string-append "--mandir=" (assoc-ref %outputs "out") "/share/man") | ||
97 | "--without-x" ; FIXME: Drop as soon as X is available. | ||
98 | "--with-system-freetype2" | ||
99 | ;; "--with-system-gd" | ||
100 | ;; "--with-system-graphite" | ||
101 | "--with-system-icu" | ||
102 | "--with-system-libgs" | ||
103 | "--with-system-libpng" | ||
104 | "--with-system-poppler" | ||
105 | "--with-system-t1lib" | ||
106 | "--with-system-teckit" | ||
107 | "--with-system-xpdf" | ||
108 | "--with-system-zlib" | ||
109 | "--with-system-zziplib") | ||
110 | #:phases | ||
111 | (alist-replace | ||
112 | 'configure | ||
113 | (lambda* (#:key outputs #:allow-other-keys #:rest args) | ||
114 | (let ((configure (assoc-ref %standard-phases 'configure))) | ||
115 | (substitute* "utils/psutils/Makefile.in" | ||
116 | (("/usr/bin/env perl") (which "perl"))) | ||
117 | (apply configure args))) | ||
118 | (alist-cons-after | ||
119 | 'install 'postinst | ||
120 | (lambda* (#:key inputs outputs #:allow-other-keys #:rest args) | ||
121 | (let ((texlive-extra (assoc-ref inputs "texlive-extra-src")) | ||
122 | (texlive-texmf (assoc-ref inputs "texlive-texmf-src")) | ||
123 | (out (assoc-ref outputs "out")) | ||
124 | (data (assoc-ref outputs "data")) | ||
125 | (unpack (assoc-ref %standard-phases 'unpack)) | ||
126 | (patch-source-shebangs | ||
127 | (assoc-ref %standard-phases 'patch-source-shebangs))) | ||
128 | ;; Create symbolic links for the latex variants and their | ||
129 | ;; man pages. | ||
130 | (with-directory-excursion (string-append out "/bin/") | ||
131 | (for-each symlink | ||
132 | '("pdftex" "pdftex" "xetex" "luatex") | ||
133 | '("latex" "pdflatex" "xelatex" "lualatex"))) | ||
134 | (with-directory-excursion (string-append out "/share/man/man1/") | ||
135 | (symlink "luatex.1" "lualatex.1")) | ||
136 | ;; Delete texmf and texmf-dist from "data", since they | ||
137 | ;; will be reinstalled from texlive-texmf. | ||
138 | (system* "rm" "-r" (string-append data "/texmf")) | ||
139 | (system* "rm" "-r" (string-append data "/texmf-dist")) | ||
140 | ;; Unpack texlive-extra and install tlpkg. | ||
141 | (mkdir "texlive-extra") | ||
142 | (with-directory-excursion "texlive-extra" | ||
143 | (apply unpack (list #:source texlive-extra)) | ||
144 | (apply patch-source-shebangs (list #:source texlive-extra)) | ||
145 | (system* "mv" "tlpkg" data) | ||
146 | (chdir "..")) | ||
147 | ;; Unpack and install texlive-texmf. | ||
148 | (mkdir "texlive-texmf") | ||
149 | (with-directory-excursion "texlive-texmf" | ||
150 | (apply unpack (list #:source texlive-texmf)) | ||
151 | (apply patch-source-shebangs (list #:source texlive-texmf)) | ||
152 | ;; Register "data" for kpathsea in texmf.cnf. | ||
153 | (substitute* "texmf/web2c/texmf.cnf" | ||
154 | (("TEXMFROOT = \\$SELFAUTOPARENT") | ||
155 | (string-append "TEXMFROOT = " data))) | ||
156 | (system* "mv" "texmf" data) | ||
157 | (system* "mv" "texmf-dist" data) | ||
158 | (chdir "..")) | ||
159 | ;; texmf.cnf must also be placed in "out", since kpsewhich does | ||
160 | ;; not know about "data" until it has found this file. | ||
161 | (mkdir (string-append out "/share/texmf")) | ||
162 | (mkdir (string-append out "/share/texmf/web2c")) | ||
163 | (copy-file (string-append data "/texmf/web2c/texmf.cnf") | ||
164 | (string-append out "/share/texmf/web2c/texmf.cnf")))) | ||
165 | (alist-cons-after 'patch-shebangs 'texconfig | ||
166 | (lambda* (#:key outputs #:allow-other-keys) | ||
167 | (let ((out (assoc-ref outputs "out"))) | ||
168 | ;; Configure the texlive system; inspired from | ||
169 | ;; http://slackbuilds.org/repository/13.37/office/texlive/ | ||
170 | (setenv "PATH" (string-append (getenv "PATH") ":" out "/bin")) | ||
171 | (system* "updmap-sys" "--nohash" "--syncwithtrees") | ||
172 | (system* "mktexlsr") | ||
173 | (system* "fmtutil-sys" "--all"))) | ||
174 | %standard-phases))))) | ||
175 | (synopsis "Tex Live, a package of the TeX typesetting system") | ||
176 | (description | ||
177 | "TeX Live provides a comprehensive TeX document production system. | ||
178 | It includes all the major TeX-related programs, macro packages, and fonts | ||
179 | that are free software, including support for many languages around the | ||
180 | world.") | ||
181 | (license (license:fsf-free "http://tug.org/texlive/copying.html")) | ||
182 | (home-page "http://www.tug.org/texlive/"))) | ||
diff --git a/doc/.gitignore b/doc/.gitignore index daa5ccaa0..c56a90359 100644 --- a/doc/.gitignore +++ b/doc/.gitignore | |||
@@ -1,13 +1,22 @@ | |||
1 | gnunet-c-tutorial.aux | ||
2 | gnunet-c-tutorial.out | ||
3 | *.log | ||
4 | *.aux | 1 | *.aux |
2 | *.out | ||
3 | *.log | ||
5 | 4 | ||
6 | *.toc | 5 | *.toc |
7 | *.cp | 6 | *.cp |
8 | *.cps | 7 | *.cps |
9 | *.html | ||
10 | *~ | 8 | *~ |
11 | *.info | 9 | *.info |
10 | *.info-1 | ||
11 | *.info-2 | ||
12 | *.info-3 | ||
12 | \#*\# | 13 | \#*\# |
13 | version.texi | 14 | version.texi |
15 | mdate-sh | ||
16 | stamp-vti | ||
17 | texinfo.tex | ||
18 | gnunet.t2p/ | ||
19 | gnunet-c-tutorial.t2p/ | ||
20 | *.t2p/ | ||
21 | documentation/manuals | ||
22 | .\#* \ No newline at end of file | ||
diff --git a/doc/Makefile.am b/doc/Makefile.am index 07aafbb51..f84c66753 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am | |||
@@ -1,145 +1,7 @@ | |||
1 | # This Makefile.am is in the public domain | 1 | # This Makefile.am is in the public domain |
2 | SUBDIRS = man doxygen | 2 | SUBDIRS = man doxygen documentation |
3 | 3 | ||
4 | docdir = $(datadir)/doc/gnunet/ | 4 | EXTRA_DIST = \ |
5 | 5 | outdated-and-old-installation-instructions.txt \ | |
6 | infoimagedir = $(infodir)/images | 6 | gnunet-c-tutorial-v1.pdf \ |
7 | |||
8 | dist_infoimage_DATA = \ | ||
9 | %D%/images/gnunet-gtk-0-10-gns-a-done.png \ | ||
10 | %D%/images/gnunet-gtk-0-10-gns-a.png \ | ||
11 | %D%/images/daemon_lego_block.png \ | ||
12 | %D%/images/gnunet-gtk-0-10-gns.png \ | ||
13 | %D%/images/gnunet-0-10-peerinfo.png \ | ||
14 | %D%/images/gnunet-gtk-0-10-identity.png \ | ||
15 | %D%/images/gnunet-fs-gtk-0-10-star-tab.png \ | ||
16 | %D%/images/gnunet-gtk-0-10.png \ | ||
17 | %D%/images/gnunet-gtk-0-10-download-area.png \ | ||
18 | %D%/images/gnunet-gtk-0-10-search-selected.png \ | ||
19 | %D%/images/gnunet-gtk-0-10-fs-menu.png \ | ||
20 | %D%/images/gnunet-gtk-0-10-traffic.png \ | ||
21 | %D%/images/gnunet-gtk-0-10-fs.png \ | ||
22 | %D%/images/gnunet-namestore-gtk-phone.png \ | ||
23 | %D%/images/gnunet-gtk-0-10-fs-publish-editing.png \ | ||
24 | %D%/images/gnunet-namestore-gtk-vpn.png \ | ||
25 | %D%/images/gnunet-gtk-0-10-fs-published.png \ | ||
26 | %D%/images/gnunet-setup-exit.png \ | ||
27 | %D%/images/gnunet-gtk-0-10-fs-publish.png \ | ||
28 | %D%/images/iceweasel-preferences.png \ | ||
29 | %D%/images/gnunet-gtk-0-10-fs-publish-select.png \ | ||
30 | %D%/images/iceweasel-proxy.png \ | ||
31 | %D%/images/gnunet-gtk-0-10-fs-publish-with-file_0.png \ | ||
32 | %D%/images/service_lego_block.png \ | ||
33 | %D%/images/gnunet-gtk-0-10-fs-publish-with-file.png \ | ||
34 | %D%/images/service_stack.png \ | ||
35 | %D%/images/gnunet-gtk-0-10-fs-search.png \ | ||
36 | %D%/images/gnunet-tutorial-service.png \ | ||
37 | %D%/images/gnunet-tutorial-system.png \ | ||
38 | %D%/images/daemon_lego_block.svg \ | ||
39 | %D%/images/lego_stack.svg \ | ||
40 | %D%/images/service_lego_block.svg | ||
41 | |||
42 | gnunet_tutorial_examples = \ | ||
43 | 001.c \ | ||
44 | 002.c \ | ||
45 | 003.c \ | ||
46 | 004.c \ | ||
47 | 005.c \ | ||
48 | 006.c \ | ||
49 | 007.c \ | ||
50 | 008.c \ | ||
51 | 009.c \ | ||
52 | 010.c \ | ||
53 | 011.c \ | ||
54 | 012.c \ | ||
55 | 013.c \ | ||
56 | 014.c \ | ||
57 | 015.c \ | ||
58 | 016.c \ | ||
59 | 017.c \ | ||
60 | 018.c \ | ||
61 | 019.c \ | ||
62 | 020.c \ | ||
63 | 021.c \ | ||
64 | 022.c \ | ||
65 | 023.c \ | ||
66 | 024.c \ | ||
67 | 025.c \ | ||
68 | 026.c | ||
69 | |||
70 | info_TEXINFOS = \ | ||
71 | gnunet.texi \ | ||
72 | gnunet-c-tutorial.texi | ||
73 | |||
74 | gnunet_TEXINFOS = \ | ||
75 | chapters/developer.texi \ | ||
76 | chapters/installation.texi \ | ||
77 | chapters/philosophy.texi \ | ||
78 | chapters/user.texi \ | ||
79 | fdl-1.3.texi \ | ||
80 | gpl-3.0.texi | ||
81 | |||
82 | EXTRA_DIST = \ | ||
83 | $(gnunet_TEXINFOS) \ | ||
84 | $(gnunet_tutorial_examples) \ | ||
85 | outdated-and-old-installation-instructions.txt \ | ||
86 | gnunet-c-tutorial-v1.pdf \ | ||
87 | README.txt | 7 | README.txt |
88 | |||
89 | daemon_lego_block.png: images/daemon_lego_block.svg | ||
90 | convert images/daemon_lego_block.svg images/daemon_lego_block.png && | ||
91 | pngcrush images/daemon_lego_block.png images/daemon_lego_block.png | ||
92 | |||
93 | service_lego_block.png: images/service_lego_block.svg | ||
94 | convert images/service_lego_block.svg images/service_lego_block.png && | ||
95 | pngcrush images/service_lego_block.png images/serivce_lego_block.png | ||
96 | |||
97 | lego_stack.png: images/lego_stack.svg | ||
98 | convert images/lego_stack.svg images/lego_stack.png && | ||
99 | pngcrush images/lego_stack.png images/lego_stack.png | ||
100 | |||
101 | version.texi: | ||
102 | echo "@set UPDATED $(date +'%d %B %Y')" > $@ | ||
103 | echo "@set UPDATED-MONTH $(date +'%B %Y')" >> $@ | ||
104 | echo "@set EDITION $(PACKAGE_VERSION)" >> $@ | ||
105 | echo "@set VERSION $(PACKAGE_VERSION)" >> $@ | ||
106 | |||
107 | doc-pdf: version.texi | ||
108 | @makeinfo --pdf --quiet gnunet.texi | ||
109 | doc-pdf-tutorial: version.texi | ||
110 | @makeinfo --pdf --quiet gnunet-c-tutorial.texi | ||
111 | |||
112 | doc-html: version.texi | ||
113 | @makeinfo --html gnunet.texi | ||
114 | doc-html-tutorial: version.texi | ||
115 | @makeinfo --html gnunet-c-tutorial.texi | ||
116 | |||
117 | doc-info: version.texi | ||
118 | @makeinfo --no-split gnunet.texi | ||
119 | doc-info-tutorial: version.texi | ||
120 | @makeinfo --no-split gnunet-c-tutorial.texi | ||
121 | |||
122 | # FIXME: rm *.html and *.pdf | ||
123 | doc-clean: | ||
124 | @rm *.aux *.log *.toc *.cp *.cps | ||
125 | |||
126 | doc-all: doc-pdf doc-html doc-info doc-pdf-tutorial doc-html-tutorial doc-info-tutorial | ||
127 | |||
128 | doc-pdf-noise: version.texi | ||
129 | @makeinfo --pdf gnunet.texi | ||
130 | doc-pdf-tutorial-noise: version.texi | ||
131 | @makeinfo --pdf gnunet-c-tutorial.texi | ||
132 | |||
133 | doc-html-noise: version.texi | ||
134 | @makeinfo --html gnunet.texi | ||
135 | doc-html-tutorial-noise: version.texi | ||
136 | @makeinfo --html gnunet-c-tutorial.texi | ||
137 | |||
138 | doc-info-noise: version.texi | ||
139 | @makeinfo --no-split gnunet.texi | ||
140 | doc-info-tutorial-noise: version.texi | ||
141 | @makeinfo --no-split gnunet-c-tutorial.texi | ||
142 | |||
143 | doc-all-give-me-the-noise: doc-pdf-noise doc-html-noise doc-info-noise doc-pdf-tutorial-noise doc-html-tutorial-noise doc-info-tutorial-noise | ||
144 | |||
145 | .PHONY: version.texi | ||
diff --git a/doc/README.txt b/doc/README.txt deleted file mode 100644 index 2abe479dd..000000000 --- a/doc/README.txt +++ /dev/null | |||
@@ -1,42 +0,0 @@ | |||
1 | * What's left to do | ||
2 | |||
3 | - Which Texlive modules are needed? Decrease the size. | ||
4 | - Update the content of gnunet documentation. | ||
5 | |||
6 | * How to use (hack) on this | ||
7 | |||
8 | ** with guix | ||
9 | |||
10 | export the environment variable GUIX_PACKAGE_PATH as $GUIX_PACKAGE_PATH:gnunet/contrib/packages/guix/packages | ||
11 | guix environment gnunet-doc | ||
12 | |||
13 | ** without guix | ||
14 | |||
15 | You need to have Texinfo and Texlive in your path. | ||
16 | sh bootstrap | ||
17 | ./configure | ||
18 | cd doc | ||
19 | make doc-all-give-me-the-noise | ||
20 | |||
21 | * structure (relations) | ||
22 | |||
23 | ** gnunet.texi | ||
24 | -> chapters/developer.texi | ||
25 | -> chapters/installation.texi | ||
26 | -> chapters/philosophy.texi | ||
27 | -> chapters/user.texi | ||
28 | -> images/* | ||
29 | -> gpl-3.0.texi | ||
30 | -> fdl-1.3.texi | ||
31 | |||
32 | ** gnunet-c-tutorial.texi | ||
33 | -> figs/Service.pdf | ||
34 | -> figs/System.pdf | ||
35 | -> tutorial-examples/*.c | ||
36 | -> gpl-3.0.texi | ||
37 | -> fdl-1.3.texi | ||
38 | |||
39 | - gnunet-c-tutorial-v1.pdf: original LaTeX "gnunet-c-tutorial.pdf". | ||
40 | - man folder: the man pages. | ||
41 | - doxygen folder | ||
42 | - outdated-and-old-installation-instructions.txt: self described within the file. | ||
diff --git a/doc/chapters/developer.texi b/doc/chapters/developer.texi deleted file mode 100644 index e7f507746..000000000 --- a/doc/chapters/developer.texi +++ /dev/null | |||
@@ -1,7486 +0,0 @@ | |||
1 | @c *************************************************************************** | ||
2 | @node GNUnet Developer Handbook | ||
3 | @chapter GNUnet Developer Handbook | ||
4 | |||
5 | This book is intended to be an introduction for programmers that want to | ||
6 | extend the GNUnet framework. GNUnet is more than a simple peer-to-peer | ||
7 | application. For developers, GNUnet is: | ||
8 | |||
9 | @itemize @bullet | ||
10 | @item Free software under the GNU General Public License, with a community | ||
11 | that believes in the GNU philosophy | ||
12 | @item | ||
13 | A set of standards, including coding conventions and architectural rules | ||
14 | @item | ||
15 | A set of layered protocols, both specifying the communication between peers as | ||
16 | well as the communication between components of a single peer. | ||
17 | @item | ||
18 | A set of libraries with well-defined APIs suitable for writing extensions | ||
19 | @end itemize | ||
20 | |||
21 | In particular, the architecture specifies that a peer consists of many | ||
22 | processes communicating via protocols. Processes can be written in almost | ||
23 | any language. C and Java APIs exist for accessing existing services and for | ||
24 | writing extensions. It is possible to write extensions in other languages by | ||
25 | implementing the necessary IPC protocols. | ||
26 | |||
27 | GNUnet can be extended and improved along many possible dimensions, and anyone | ||
28 | interested in free software and freedom-enhancing networking is welcome to | ||
29 | join the effort. This developer handbook attempts to provide an initial | ||
30 | introduction to some of the key design choices and central components of the | ||
31 | system. This manual is far from complete, and we welcome informed | ||
32 | contributions, be it in the form of new chapters or insightful comments. | ||
33 | |||
34 | However, the website is experiencing a constant onslaught of sophisticated | ||
35 | link-spam entered manually by exploited workers solving puzzles and | ||
36 | customizing text. To limit this commercial defacement, we are strictly | ||
37 | moderating comments and have disallowed "normal" users from posting new | ||
38 | content. However, this is really only intended to keep the spam at bay. If | ||
39 | you are a real user or aspiring developer, please drop us a note (IRC, e-mail, | ||
40 | contact form) with your user profile ID number included. We will then relax | ||
41 | these restrictions on your account. We're sorry for this inconvenience; | ||
42 | however, few people would want to read this site if 99% of it was | ||
43 | advertisements for bogus websites. | ||
44 | |||
45 | |||
46 | |||
47 | @c *************************************************************************** | ||
48 | |||
49 | |||
50 | |||
51 | |||
52 | |||
53 | |||
54 | |||
55 | |||
56 | @menu | ||
57 | * Developer Introduction:: | ||
58 | * Code overview:: | ||
59 | * System Architecture:: | ||
60 | * Subsystem stability:: | ||
61 | * Naming conventions and coding style guide:: | ||
62 | * Build-system:: | ||
63 | * Developing extensions for GNUnet using the gnunet-ext template:: | ||
64 | * Writing testcases:: | ||
65 | * GNUnet's TESTING library:: | ||
66 | * Performance regression analysis with Gauger:: | ||
67 | * GNUnet's TESTBED Subsystem:: | ||
68 | * libgnunetutil:: | ||
69 | * The Automatic Restart Manager (ARM):: | ||
70 | * GNUnet's TRANSPORT Subsystem:: | ||
71 | * NAT library:: | ||
72 | * Distance-Vector plugin:: | ||
73 | * SMTP plugin:: | ||
74 | * Bluetooth plugin:: | ||
75 | * WLAN plugin:: | ||
76 | * The ATS Subsystem:: | ||
77 | * GNUnet's CORE Subsystem:: | ||
78 | * GNUnet's CADET subsystem:: | ||
79 | * GNUnet's NSE subsystem:: | ||
80 | * GNUnet's HOSTLIST subsystem:: | ||
81 | * GNUnet's IDENTITY subsystem:: | ||
82 | * GNUnet's NAMESTORE Subsystem:: | ||
83 | * GNUnet's PEERINFO subsystem:: | ||
84 | * GNUnet's PEERSTORE subsystem:: | ||
85 | * GNUnet's SET Subsystem:: | ||
86 | * GNUnet's STATISTICS subsystem:: | ||
87 | * GNUnet's Distributed Hash Table (DHT):: | ||
88 | * The GNU Name System (GNS):: | ||
89 | * The GNS Namecache:: | ||
90 | * The REVOCATION Subsystem:: | ||
91 | * GNUnet's File-sharing (FS) Subsystem:: | ||
92 | * GNUnet's REGEX Subsystem:: | ||
93 | @end menu | ||
94 | |||
95 | @node Developer Introduction | ||
96 | @section Developer Introduction | ||
97 | |||
98 | This developer handbook is intended as first introduction to GNUnet for new | ||
99 | developers that want to extend the GNUnet framework. After the introduction, | ||
100 | each of the GNUnet subsystems (directories in the @file{src/} tree) is (supposed to | ||
101 | be) covered in its own chapter. In addition to this documentation, GNUnet | ||
102 | developers should be aware of the services available on the GNUnet server to | ||
103 | them. | ||
104 | |||
105 | New developers can have a look a the GNUnet tutorials for C and java available | ||
106 | in the @file{src/} directory of the repository or under the following links: | ||
107 | |||
108 | @c ** FIXME: Link to files in source, not online. | ||
109 | @c ** FIXME: Where is the Java tutorial? | ||
110 | @itemize @bullet | ||
111 | @item @uref{https://gnunet.org/git/gnunet.git/plain/doc/gnunet-c-tutorial.pdf, GNUnet C tutorial} | ||
112 | @item GNUnet Java tutorial | ||
113 | @end itemize | ||
114 | |||
115 | In addition to this book, the GNUnet server contains various resources for | ||
116 | GNUnet developers. They are all conveniently reachable via the "Developer" | ||
117 | entry in the navigation menu. Some additional tools (such as static analysis | ||
118 | reports) require a special developer access to perform certain operations. If | ||
119 | you feel you need access, you should contact | ||
120 | @uref{http://grothoff.org/christian/, Christian Grothoff}, GNUnet's maintainer. | ||
121 | |||
122 | The public subsystems on the GNUnet server that help developers are: | ||
123 | |||
124 | @itemize @bullet | ||
125 | @item The Version control system keeps our code and enables distributed | ||
126 | development. Only developers with write access can commit code, everyone else | ||
127 | is encouraged to submit patches to the | ||
128 | @uref{https://lists.gnu.org/mailman/listinfo/gnunet-developers, GNUnet-developers mailinglist}. | ||
129 | @item The GNUnet bugtracking system is used to track feature requests, open bug | ||
130 | reports and their resolutions. Anyone can report bugs, only developers can | ||
131 | claim to have fixed them. | ||
132 | @item A buildbot is used to check GNUnet builds automatically on a range of | ||
133 | platforms. Builds are triggered automatically after 30 minutes of no changes to | ||
134 | Git. | ||
135 | @item The current quality of our automated test suite is assessed using Code | ||
136 | coverage analysis. This analysis is run daily; however the webpage is only | ||
137 | updated if all automated tests pass at that time. Testcases that improve our | ||
138 | code coverage are always welcome. | ||
139 | @item We try to automatically find bugs using a static analysis scan. This scan | ||
140 | is run daily; however the webpage is only updated if all automated tests pass | ||
141 | at the time. Note that not everything that is flagged by the analysis is a bug, | ||
142 | sometimes even good code can be marked as possibly problematic. Nevertheless, | ||
143 | developers are encouraged to at least be aware of all issues in their code that | ||
144 | are listed. | ||
145 | @item We use Gauger for automatic performance regression visualization. Details | ||
146 | on how to use Gauger are here. | ||
147 | @item We use @uref{http://junit.org/, junit} to automatically test gnunet-java. | ||
148 | Automatically generated, current reports on the test suite are here. | ||
149 | @item We use Cobertura to generate test coverage reports for gnunet-java. | ||
150 | Current reports on test coverage are here. | ||
151 | @end itemize | ||
152 | |||
153 | |||
154 | |||
155 | @c *************************************************************************** | ||
156 | @menu | ||
157 | * Project overview:: | ||
158 | @end menu | ||
159 | |||
160 | @node Project overview | ||
161 | @subsection Project overview | ||
162 | |||
163 | The GNUnet project consists at this point of several sub-projects. This section | ||
164 | is supposed to give an initial overview about the various sub-projects. Note | ||
165 | that this description also lists projects that are far from complete, including | ||
166 | even those that have literally not a single line of code in them yet. | ||
167 | |||
168 | GNUnet sub-projects in order of likely relevance are currently: | ||
169 | |||
170 | @table @asis | ||
171 | |||
172 | @item gnunet Core of the P2P framework, including file-sharing, VPN and | ||
173 | chat applications; this is what the developer handbook covers mostly | ||
174 | @item gnunet-gtk Gtk+-based user interfaces, including gnunet-fs-gtk | ||
175 | (file-sharing), gnunet-statistics-gtk (statistics over time), | ||
176 | gnunet-peerinfo-gtk (information about current connections and known peers), | ||
177 | gnunet-chat-gtk (chat GUI) and gnunet-setup (setup tool for "everything") | ||
178 | @item gnunet-fuse Mounting directories shared via GNUnet's file-sharing on Linux | ||
179 | @item gnunet-update Installation and update tool | ||
180 | @item gnunet-ext Template for starting 'external' GNUnet projects | ||
181 | @item gnunet-java Java APIs for writing GNUnet services and applications | ||
182 | @c ** FIXME: Point to new website repository once we have it: | ||
183 | @c ** @item svn/gnunet-www/ Code and media helping drive the GNUnet website | ||
184 | @item eclectic Code to run | ||
185 | GNUnet nodes on testbeds for research, development, testing and evaluation | ||
186 | @c ** FIXME: Solve the status and location of gnunet-qt | ||
187 | @item gnunet-qt qt-based GNUnet GUI (dead?) | ||
188 | @item gnunet-cocoa cocoa-based GNUnet GUI (dead?) | ||
189 | |||
190 | @end table | ||
191 | |||
192 | We are also working on various supporting libraries and tools: | ||
193 | @c ** FIXME: What about gauger, and what about libmwmodem? | ||
194 | |||
195 | @table @asis | ||
196 | @item libextractor GNU libextractor (meta data extraction) | ||
197 | @item libmicrohttpd GNU libmicrohttpd (embedded HTTP(S) server library) | ||
198 | @item gauger Tool for performance regression analysis | ||
199 | @item monkey Tool for automated debugging of distributed systems | ||
200 | @item libmwmodem Library for accessing satellite connection quality reports | ||
201 | @end table | ||
202 | |||
203 | Finally, there are various external projects (see links for a list of those | ||
204 | that have a public website) which build on top of the GNUnet framework. | ||
205 | |||
206 | @c *************************************************************************** | ||
207 | @node Code overview | ||
208 | @section Code overview | ||
209 | |||
210 | This section gives a brief overview of the GNUnet source code. Specifically, we | ||
211 | sketch the function of each of the subdirectories in the @file{gnunet/src/} | ||
212 | directory. The order given is roughly bottom-up (in terms of the layers of the | ||
213 | system). | ||
214 | @table @asis | ||
215 | |||
216 | @item util/ --- libgnunetutil Library with general utility functions, all | ||
217 | GNUnet binaries link against this library. Anything from memory allocation and | ||
218 | data structures to cryptography and inter-process communication. The goal is to | ||
219 | provide an OS-independent interface and more 'secure' or convenient | ||
220 | implementations of commonly used primitives. The API is spread over more than a | ||
221 | dozen headers, developers should study those closely to avoid duplicating | ||
222 | existing functions. | ||
223 | @item hello/ --- libgnunethello HELLO messages are used to | ||
224 | describe under which addresses a peer can be reached (for example, protocol, | ||
225 | IP, port). This library manages parsing and generating of HELLO messages. | ||
226 | @item block/ --- libgnunetblock The DHT and other components of GNUnet store | ||
227 | information in units called 'blocks'. Each block has a type and the type | ||
228 | defines a particular format and how that binary format is to be linked to a | ||
229 | hash code (the key for the DHT and for databases). The block library is a | ||
230 | wapper around block plugins which provide the necessary functions for each | ||
231 | block type. | ||
232 | @item statistics/ The statistics service enables associating | ||
233 | values (of type uint64_t) with a componenet name and a string. The main uses is | ||
234 | debugging (counting events), performance tracking and user entertainment (what | ||
235 | did my peer do today?). | ||
236 | @item arm/ The automatic-restart-manager (ARM) service | ||
237 | is the GNUnet master service. Its role is to start gnunet-services, to re-start | ||
238 | them when they crashed and finally to shut down the system when requested. | ||
239 | @item peerinfo/ The peerinfo service keeps track of which peers are known to | ||
240 | the local peer and also tracks the validated addresses for each peer (in the | ||
241 | form of a HELLO message) for each of those peers. The peer is not necessarily | ||
242 | connected to all peers known to the peerinfo service. Peerinfo provides | ||
243 | persistent storage for peer identities --- peers are not forgotten just because | ||
244 | of a system restart. | ||
245 | @item datacache/ --- libgnunetdatacache The datacache | ||
246 | library provides (temporary) block storage for the DHT. Existing plugins can | ||
247 | store blocks in Sqlite, Postgres or MySQL databases. All data stored in the | ||
248 | cache is lost when the peer is stopped or restarted (datacache uses temporary | ||
249 | tables). | ||
250 | @item datastore/ The datastore service stores file-sharing blocks in | ||
251 | databases for extended periods of time. In contrast to the datacache, data is | ||
252 | not lost when peers restart. However, quota restrictions may still cause old, | ||
253 | expired or low-priority data to be eventually discarded. Existing plugins can | ||
254 | store blocks in Sqlite, Postgres or MySQL databases. | ||
255 | @item template/ Template | ||
256 | for writing a new service. Does nothing. | ||
257 | @item ats/ The automatic transport | ||
258 | selection (ATS) service is responsible for deciding which address (i.e. which | ||
259 | transport plugin) should be used for communication with other peers, and at | ||
260 | what bandwidth. | ||
261 | @item nat/ --- libgnunetnat Library that provides basic | ||
262 | functions for NAT traversal. The library supports NAT traversal with manual | ||
263 | hole-punching by the user, UPnP and ICMP-based autonomous NAT traversal. The | ||
264 | library also includes an API for testing if the current configuration works and | ||
265 | the @code{gnunet-nat-server} which provides an external service to test the | ||
266 | local configuration. | ||
267 | @item fragmentation/ --- libgnunetfragmentation Some | ||
268 | transports (UDP and WLAN, mostly) have restrictions on the maximum transfer | ||
269 | unit (MTU) for packets. The fragmentation library can be used to break larger | ||
270 | packets into chunks of at most 1k and transmit the resulting fragments | ||
271 | reliabily (with acknowledgement, retransmission, timeouts, etc.). | ||
272 | @item transport/ The transport service is responsible for managing the basic P2P | ||
273 | communication. It uses plugins to support P2P communication over TCP, UDP, | ||
274 | HTTP, HTTPS and other protocols.The transport service validates peer addresses, | ||
275 | enforces bandwidth restrictions, limits the total number of connections and | ||
276 | enforces connectivity restrictions (i.e. friends-only). | ||
277 | @item peerinfo-tool/ | ||
278 | This directory contains the gnunet-peerinfo binary which can be used to inspect | ||
279 | the peers and HELLOs known to the peerinfo service. | ||
280 | @item core/ The core | ||
281 | service is responsible for establishing encrypted, authenticated connections | ||
282 | with other peers, encrypting and decrypting messages and forwarding messages to | ||
283 | higher-level services that are interested in them. | ||
284 | @item testing/ --- | ||
285 | libgnunettesting The testing library allows starting (and stopping) peers for | ||
286 | writing testcases.@ | ||
287 | It also supports automatic generation of configurations for | ||
288 | peers ensuring that the ports and paths are disjoint. libgnunettesting is also | ||
289 | the foundation for the testbed service | ||
290 | @item testbed/ The testbed service is | ||
291 | used for creating small or large scale deployments of GNUnet peers for | ||
292 | evaluation of protocols. It facilitates peer depolyments on multiple hosts (for | ||
293 | example, in a cluster) and establishing varous network topologies (both | ||
294 | underlay and overlay). | ||
295 | @item nse/ The network size estimation (NSE) service | ||
296 | implements a protocol for (securely) estimating the current size of the P2P | ||
297 | network. | ||
298 | @item dht/ The distributed hash table (DHT) service provides a | ||
299 | distributed implementation of a hash table to store blocks under hash keys in | ||
300 | the P2P network. | ||
301 | @item hostlist/ The hostlist service allows learning about | ||
302 | other peers in the network by downloading HELLO messages from an HTTP server, | ||
303 | can be configured to run such an HTTP server and also implements a P2P protocol | ||
304 | to advertise and automatically learn about other peers that offer a public | ||
305 | hostlist server. | ||
306 | @item topology/ The topology service is responsible for | ||
307 | maintaining the mesh topology. It tries to maintain connections to friends | ||
308 | (depending on the configuration) and also tries to ensure that the peer has a | ||
309 | decent number of active connections at all times. If necessary, new connections | ||
310 | are added. All peers should run the topology service, otherwise they may end up | ||
311 | not being connected to any other peer (unless some other service ensures that | ||
312 | core establishes the required connections). The topology service also tells the | ||
313 | transport service which connections are permitted (for friend-to-friend | ||
314 | networking) | ||
315 | @item fs/ The file-sharing (FS) service implements GNUnet's | ||
316 | file-sharing application. Both anonymous file-sharing (using gap) and | ||
317 | non-anonymous file-sharing (using dht) are supported. | ||
318 | @item cadet/ The CADET | ||
319 | service provides a general-purpose routing abstraction to create end-to-end | ||
320 | encrypted tunnels in mesh networks. We wrote a paper documenting key aspects of | ||
321 | the design. | ||
322 | @item tun/ --- libgnunettun Library for building IPv4, IPv6 | ||
323 | packets and creating checksums for UDP, TCP and ICMP packets. The header | ||
324 | defines C structs for common Internet packet formats and in particular structs | ||
325 | for interacting with TUN (virtual network) interfaces. | ||
326 | @item mysql/ --- | ||
327 | libgnunetmysql Library for creating and executing prepared MySQL statements and | ||
328 | to manage the connection to the MySQL database. Essentially a lightweight | ||
329 | wrapper for the interaction between GNUnet components and libmysqlclient. | ||
330 | @item dns/ Service that allows intercepting and modifying DNS requests of the | ||
331 | local machine. Currently used for IPv4-IPv6 protocol translation (DNS-ALG) as | ||
332 | implemented by "pt/" and for the GNUnet naming system. The service can also be | ||
333 | configured to offer an exit service for DNS traffic. | ||
334 | @item vpn/ The virtual | ||
335 | public network (VPN) service provides a virtual tunnel interface (VTUN) for IP | ||
336 | routing over GNUnet. Needs some other peers to run an "exit" service to work. | ||
337 | Can be activated using the "gnunet-vpn" tool or integrated with DNS using the | ||
338 | "pt" daemon. | ||
339 | @item exit/ Daemon to allow traffic from the VPN to exit this | ||
340 | peer to the Internet or to specific IP-based services of the local peer. | ||
341 | Currently, an exit service can only be restricted to IPv4 or IPv6, not to | ||
342 | specific ports and or IP address ranges. If this is not acceptable, additional | ||
343 | firewall rules must be added manually. exit currently only works for normal | ||
344 | UDP, TCP and ICMP traffic; DNS queries need to leave the system via a DNS | ||
345 | service. | ||
346 | @item pt/ protocol translation daemon. This daemon enables 4-to-6, | ||
347 | 6-to-4, 4-over-6 or 6-over-4 transitions for the local system. It essentially | ||
348 | uses "DNS" to intercept DNS replies and then maps results to those offered by | ||
349 | the VPN, which then sends them using mesh to some daemon offering an | ||
350 | appropriate exit service. | ||
351 | @item identity/ Management of egos (alter egos) of a | ||
352 | user; identities are essentially named ECC private keys and used for zones in | ||
353 | the GNU name system and for namespaces in file-sharing, but might find other | ||
354 | uses later | ||
355 | @item revocation/ Key revocation service, can be used to revoke the | ||
356 | private key of an identity if it has been compromised | ||
357 | @item namecache/ Cache | ||
358 | for resolution results for the GNU name system; data is encrypted and can be | ||
359 | shared among users, loss of the data should ideally only result in a | ||
360 | performance degradation (persistence not required) | ||
361 | @item namestore/ Database | ||
362 | for the GNU name system with per-user private information, persistence required | ||
363 | @item gns/ GNU name system, a GNU approach to DNS and PKI. | ||
364 | @item dv/ A plugin | ||
365 | for distance-vector (DV)-based routing. DV consists of a service and a | ||
366 | transport plugin to provide peers with the illusion of a direct P2P connection | ||
367 | for connections that use multiple (typically up to 3) hops in the actual | ||
368 | underlay network. | ||
369 | @item regex/ Service for the (distributed) evaluation of | ||
370 | regular expressions. | ||
371 | @item scalarproduct/ The scalar product service offers an | ||
372 | API to perform a secure multiparty computation which calculates a scalar | ||
373 | product between two peers without exposing the private input vectors of the | ||
374 | peers to each other. | ||
375 | @item consensus/ The consensus service will allow a set | ||
376 | of peers to agree on a set of values via a distributed set union computation. | ||
377 | @item rest/ The rest API allows access to GNUnet services using RESTful | ||
378 | interaction. The services provide plugins that can exposed by the rest server. | ||
379 | @item experimentation/ The experimentation daemon coordinates distributed | ||
380 | experimentation to evaluate transport and ats properties | ||
381 | @end table | ||
382 | |||
383 | @c *************************************************************************** | ||
384 | @node System Architecture | ||
385 | @section System Architecture | ||
386 | |||
387 | GNUnet developers like legos. The blocks are indestructible, can be stacked | ||
388 | together to construct complex buildings and it is generally easy to swap one | ||
389 | block for a different one that has the same shape. GNUnet's architecture is | ||
390 | based on legos: | ||
391 | |||
392 | |||
393 | |||
394 | This chapter documents the GNUnet lego system, also known as GNUnet's system | ||
395 | architecture. | ||
396 | |||
397 | The most common GNUnet component is a service. Services offer an API (or | ||
398 | several, depending on what you count as "an API") which is implemented as a | ||
399 | library. The library communicates with the main process of the service using a | ||
400 | service-specific network protocol. The main process of the service typically | ||
401 | doesn't fully provide everything that is needed --- it has holes to be filled | ||
402 | by APIs to other services. | ||
403 | |||
404 | A special kind of component in GNUnet are user interfaces and daemons. Like | ||
405 | services, they have holes to be filled by APIs of other services. Unlike | ||
406 | services, daemons do not implement their own network protocol and they have no | ||
407 | API: | ||
408 | |||
409 | The GNUnet system provides a range of services, daemons and user interfaces, | ||
410 | which are then combined into a layered GNUnet instance (also known as a peer). | ||
411 | |||
412 | Note that while it is generally possible to swap one service for another | ||
413 | compatible service, there is often only one implementation. However, during | ||
414 | development we often have a "new" version of a service in parallel with an | ||
415 | "old" version. While the "new" version is not working, developers working on | ||
416 | other parts of the service can continue their development by simply using the | ||
417 | "old" service. Alternative design ideas can also be easily investigated by | ||
418 | swapping out individual components. This is typically achieved by simply | ||
419 | changing the name of the "BINARY" in the respective configuration section. | ||
420 | |||
421 | Key properties of GNUnet services are that they must be separate processes and | ||
422 | that they must protect themselves by applying tight error checking against the | ||
423 | network protocol they implement (thereby achieving a certain degree of | ||
424 | robustness). | ||
425 | |||
426 | On the other hand, the APIs are implemented to tolerate failures of the | ||
427 | service, isolating their host process from errors by the service. If the | ||
428 | service process crashes, other services and daemons around it should not also | ||
429 | fail, but instead wait for the service process to be restarted by ARM. | ||
430 | |||
431 | |||
432 | @c *************************************************************************** | ||
433 | @node Subsystem stability | ||
434 | @section Subsystem stability | ||
435 | |||
436 | This page documents the current stability of the various GNUnet subsystems. | ||
437 | Stability here describes the expected degree of compatibility with future | ||
438 | versions of GNUnet. For each subsystem we distinguish between compatibility on | ||
439 | the P2P network level (communication protocol between peers), the IPC level | ||
440 | (communication between the service and the service library) and the API level | ||
441 | (stability of the API). P2P compatibility is relevant in terms of which | ||
442 | applications are likely going to be able to communicate with future versions of | ||
443 | the network. IPC communication is relevant for the implementation of language | ||
444 | bindings that re-implement the IPC messages. Finally, API compatibility is | ||
445 | relevant to developers that hope to be able to avoid changes to applications | ||
446 | build on top of the APIs of the framework. | ||
447 | |||
448 | The following table summarizes our current view of the stability of the | ||
449 | respective protocols or APIs: | ||
450 | |||
451 | @multitable @columnfractions .20 .20 .20 .20 | ||
452 | @headitem Subsystem @tab P2P @tab IPC @tab C API | ||
453 | @item util @tab n/a @tab n/a @tab stable | ||
454 | @item arm @tab n/a @tab stable @tab stable | ||
455 | @item ats @tab n/a @tab unstable @tab testing | ||
456 | @item block @tab n/a @tab n/a @tab stable | ||
457 | @item cadet @tab testing @tab testing @tab testing | ||
458 | @item consensus @tab experimental @tab experimental @tab experimental | ||
459 | @item core @tab stable @tab stable @tab stable | ||
460 | @item datacache @tab n/a @tab n/a @tab stable | ||
461 | @item datastore @tab n/a @tab stable @tab stable | ||
462 | @item dht @tab stable @tab stable @tab stable | ||
463 | @item dns @tab stable @tab stable @tab stable | ||
464 | @item dv @tab testing @tab testing @tab n/a | ||
465 | @item exit @tab testing @tab n/a @tab n/a | ||
466 | @item fragmentation @tab stable @tab n/a @tab stable | ||
467 | @item fs @tab stable @tab stable @tab stable | ||
468 | @item gns @tab stable @tab stable @tab stable | ||
469 | @item hello @tab n/a @tab n/a @tab testing | ||
470 | @item hostlist @tab stable @tab stable @tab n/a | ||
471 | @item identity @tab stable @tab stable @tab n/a | ||
472 | @item multicast @tab experimental @tab experimental @tab experimental | ||
473 | @item mysql @tab stable @tab n/a @tab stable | ||
474 | @item namestore @tab n/a @tab stable @tab stable | ||
475 | @item nat @tab n/a @tab n/a @tab stable | ||
476 | @item nse @tab stable @tab stable @tab stable | ||
477 | @item peerinfo @tab n/a @tab stable @tab stable | ||
478 | @item psyc @tab experimental @tab experimental @tab experimental | ||
479 | @item pt @tab n/a @tab n/a @tab n/a | ||
480 | @item regex @tab stable @tab stable @tab stable | ||
481 | @item revocation @tab stable @tab stable @tab stable | ||
482 | @item social @tab experimental @tab experimental @tab experimental | ||
483 | @item statistics @tab n/a @tab stable @tab stable | ||
484 | @item testbed @tab n/a @tab testing @tab testing | ||
485 | @item testing @tab n/a @tab n/a @tab testing | ||
486 | @item topology @tab n/a @tab n/a @tab n/a | ||
487 | @item transport @tab stable @tab stable @tab stable | ||
488 | @item tun @tab n/a @tab n/a @tab stable | ||
489 | @item vpn @tab testing @tab n/a @tab n/a | ||
490 | @end multitable | ||
491 | |||
492 | Here is a rough explanation of the values: | ||
493 | |||
494 | @table @samp | ||
495 | @item stable | ||
496 | No incompatible changes are planned at this time; for IPC/APIs, if | ||
497 | there are incompatible changes, they will be minor and might only require | ||
498 | minimal changes to existing code; for P2P, changes will be avoided if at all | ||
499 | possible for the 0.10.x-series | ||
500 | |||
501 | @item testing | ||
502 | No incompatible changes are | ||
503 | planned at this time, but the code is still known to be in flux; so while we | ||
504 | have no concrete plans, our expectation is that there will still be minor | ||
505 | modifications; for P2P, changes will likely be extensions that should not break | ||
506 | existing code | ||
507 | |||
508 | @item unstable | ||
509 | Changes are planned and will happen; however, they | ||
510 | will not be totally radical and the result should still resemble what is there | ||
511 | now; nevertheless, anticipated changes will break protocol/API compatibility | ||
512 | |||
513 | @item experimental | ||
514 | Changes are planned and the result may look nothing like | ||
515 | what the API/protocol looks like today | ||
516 | |||
517 | @item unknown | ||
518 | Someone should think about where this subsystem headed | ||
519 | |||
520 | @item n/a | ||
521 | This subsystem does not have an API/IPC-protocol/P2P-protocol | ||
522 | @end table | ||
523 | |||
524 | @c *************************************************************************** | ||
525 | @node Naming conventions and coding style guide | ||
526 | @section Naming conventions and coding style guide | ||
527 | |||
528 | Here you can find some rules to help you write code for GNUnet. | ||
529 | |||
530 | |||
531 | |||
532 | @c *************************************************************************** | ||
533 | @menu | ||
534 | * Naming conventions:: | ||
535 | * Coding style:: | ||
536 | @end menu | ||
537 | |||
538 | @node Naming conventions | ||
539 | @subsection Naming conventions | ||
540 | |||
541 | |||
542 | @c *************************************************************************** | ||
543 | @menu | ||
544 | * include files:: | ||
545 | * binaries:: | ||
546 | * logging:: | ||
547 | * configuration:: | ||
548 | * exported symbols:: | ||
549 | * private (library-internal) symbols (including structs and macros):: | ||
550 | * testcases:: | ||
551 | * performance tests:: | ||
552 | * src/ directories:: | ||
553 | @end menu | ||
554 | |||
555 | @node include files | ||
556 | @subsubsection include files | ||
557 | |||
558 | @itemize @bullet | ||
559 | @item _lib: library without need for a process | ||
560 | @item _service: library that needs a service process | ||
561 | @item _plugin: plugin definition | ||
562 | @item _protocol: structs used in network protocol | ||
563 | @item exceptions: | ||
564 | @itemize @bullet | ||
565 | @item gnunet_config.h --- generated | ||
566 | @item platform.h --- first included | ||
567 | @item plibc.h --- external library | ||
568 | @item gnunet_common.h --- fundamental routines | ||
569 | @item gnunet_directories.h --- generated | ||
570 | @item gettext.h --- external library | ||
571 | @end itemize | ||
572 | @end itemize | ||
573 | |||
574 | @c *************************************************************************** | ||
575 | @node binaries | ||
576 | @subsubsection binaries | ||
577 | |||
578 | @itemize @bullet | ||
579 | @item gnunet-service-xxx: service process (has listen socket) | ||
580 | @item gnunet-daemon-xxx: daemon process (no listen socket) | ||
581 | @item gnunet-helper-xxx[-yyy]: SUID helper for module xxx | ||
582 | @item gnunet-yyy: command-line tool for end-users | ||
583 | @item libgnunet_plugin_xxx_yyy.so: plugin for API xxx | ||
584 | @item libgnunetxxx.so: library for API xxx | ||
585 | @end itemize | ||
586 | |||
587 | @c *************************************************************************** | ||
588 | @node logging | ||
589 | @subsubsection logging | ||
590 | |||
591 | @itemize @bullet | ||
592 | @item services and daemons use their directory name in GNUNET_log_setup (i.e. | ||
593 | 'core') and log using plain 'GNUNET_log'. | ||
594 | @item command-line tools use their full name in GNUNET_log_setup (i.e. | ||
595 | 'gnunet-publish') and log using plain 'GNUNET_log'. | ||
596 | @item service access libraries log using 'GNUNET_log_from' and use | ||
597 | 'DIRNAME-api' for the component (i.e. 'core-api') | ||
598 | @item pure libraries (without associated service) use 'GNUNET_log_from' with | ||
599 | the component set to their library name (without lib or '.so'), which should | ||
600 | also be their directory name (i.e. 'nat') | ||
601 | @item plugins should use 'GNUNET_log_from' with the directory name and the | ||
602 | plugin name combined to produce the component name (i.e. 'transport-tcp'). | ||
603 | @item logging should be unified per-file by defining a LOG macro with the | ||
604 | appropriate arguments, along these lines:@ #define LOG(kind,...) | ||
605 | GNUNET_log_from (kind, "example-api",__VA_ARGS__) | ||
606 | @end itemize | ||
607 | |||
608 | @c *************************************************************************** | ||
609 | @node configuration | ||
610 | @subsubsection configuration | ||
611 | |||
612 | @itemize @bullet | ||
613 | @item paths (that are substituted in all filenames) are in PATHS (have as few | ||
614 | as possible) | ||
615 | @item all options for a particular module (src/MODULE) are under [MODULE] | ||
616 | @item options for a plugin of a module are under [MODULE-PLUGINNAME] | ||
617 | @end itemize | ||
618 | |||
619 | @c *************************************************************************** | ||
620 | @node exported symbols | ||
621 | @subsubsection exported symbols | ||
622 | |||
623 | @itemize @bullet | ||
624 | @item must start with "GNUNET_modulename_" and be defined in "modulename.c" | ||
625 | @item exceptions: those defined in gnunet_common.h | ||
626 | @end itemize | ||
627 | |||
628 | @c *************************************************************************** | ||
629 | @node private (library-internal) symbols (including structs and macros) | ||
630 | @subsubsection private (library-internal) symbols (including structs and macros) | ||
631 | |||
632 | @itemize @bullet | ||
633 | @item must NOT start with any prefix | ||
634 | @item must not be exported in a way that linkers could use them or@ other | ||
635 | libraries might see them via headers; they must be either@ declared/defined in | ||
636 | C source files or in headers that are in@ the respective directory under | ||
637 | src/modulename/ and NEVER be@ declared in src/include/. | ||
638 | @end itemize | ||
639 | |||
640 | @node testcases | ||
641 | @subsubsection testcases | ||
642 | |||
643 | @itemize @bullet | ||
644 | @item must be called "test_module-under-test_case-description.c" | ||
645 | @item "case-description" maybe omitted if there is only one test | ||
646 | @end itemize | ||
647 | |||
648 | @c *************************************************************************** | ||
649 | @node performance tests | ||
650 | @subsubsection performance tests | ||
651 | |||
652 | @itemize @bullet | ||
653 | @item must be called "perf_module-under-test_case-description.c" | ||
654 | @item "case-description" maybe omitted if there is only one performance test | ||
655 | @item Must only be run if HAVE_BENCHMARKS is satisfied | ||
656 | @end itemize | ||
657 | |||
658 | @c *************************************************************************** | ||
659 | @node src/ directories | ||
660 | @subsubsection src/ directories | ||
661 | |||
662 | @itemize @bullet | ||
663 | @item gnunet-NAME: end-user applications (i.e., gnunet-search, gnunet-arm) | ||
664 | @item gnunet-service-NAME: service processes with accessor library (i.e., | ||
665 | gnunet-service-arm) | ||
666 | @item libgnunetNAME: accessor library (_service.h-header) or standalone library | ||
667 | (_lib.h-header) | ||
668 | @item gnunet-daemon-NAME: daemon process without accessor library (i.e., | ||
669 | gnunet-daemon-hostlist) and no GNUnet management port | ||
670 | @item libgnunet_plugin_DIR_NAME: loadable plugins (i.e., | ||
671 | libgnunet_plugin_transport_tcp) | ||
672 | @end itemize | ||
673 | |||
674 | @c *************************************************************************** | ||
675 | @node Coding style | ||
676 | @subsection Coding style | ||
677 | |||
678 | @itemize @bullet | ||
679 | @item GNU guidelines generally apply | ||
680 | @item Indentation is done with spaces, two per level, no tabs | ||
681 | @item C99 struct initialization is fine | ||
682 | @item declare only one variable per line, so@ | ||
683 | |||
684 | @example | ||
685 | int i; int j; | ||
686 | @end example | ||
687 | |||
688 | instead of | ||
689 | |||
690 | @example | ||
691 | int i,j; | ||
692 | @end example | ||
693 | |||
694 | This helps keep diffs small and forces developers to think precisely about the | ||
695 | type of every variable. Note that @code{char *} is different from @code{const | ||
696 | char*} and @code{int} is different from @code{unsigned int} or @code{uint32_t}. | ||
697 | Each variable type should be chosen with care. | ||
698 | |||
699 | @item While @code{goto} should generally be avoided, having a @code{goto} to | ||
700 | the end of a function to a block of clean up statements (free, close, etc.) can | ||
701 | be acceptable. | ||
702 | |||
703 | @item Conditions should be written with constants on the left (to avoid | ||
704 | accidental assignment) and with the 'true' target being either the 'error' case | ||
705 | or the significantly simpler continuation. For example:@ | ||
706 | |||
707 | @example | ||
708 | if (0 != stat ("filename," &sbuf)) @{ error(); @} else @{ | ||
709 | /* handle normal case here */ | ||
710 | @} | ||
711 | @end example | ||
712 | |||
713 | |||
714 | instead of | ||
715 | @example | ||
716 | if (stat ("filename," &sbuf) == 0) @{ | ||
717 | /* handle normal case here */ | ||
718 | @} else @{ error(); @} | ||
719 | @end example | ||
720 | |||
721 | |||
722 | If possible, the error clause should be terminated with a 'return' (or 'goto' | ||
723 | to some cleanup routine) and in this case, the 'else' clause should be omitted: | ||
724 | @example | ||
725 | if (0 != stat ("filename," &sbuf)) @{ error(); return; @} | ||
726 | /* handle normal case here */ | ||
727 | @end example | ||
728 | |||
729 | |||
730 | This serves to avoid deep nesting. The 'constants on the left' rule applies to | ||
731 | all constants (including. @code{GNUNET_SCHEDULER_NO_TASK}), NULL, and enums). | ||
732 | With the two above rules (constants on left, errors in 'true' branch), there is | ||
733 | only one way to write most branches correctly. | ||
734 | |||
735 | @item Combined assignments and tests are allowed if they do not hinder code | ||
736 | clarity. For example, one can write:@ | ||
737 | |||
738 | @example | ||
739 | if (NULL == (value = lookup_function())) @{ error(); return; @} | ||
740 | @end example | ||
741 | |||
742 | |||
743 | @item Use @code{break} and @code{continue} wherever possible to avoid deep(er) | ||
744 | nesting. Thus, we would write:@ | ||
745 | |||
746 | @example | ||
747 | next = head; while (NULL != (pos = next)) @{ next = pos->next; if (! | ||
748 | should_free (pos)) continue; GNUNET_CONTAINER_DLL_remove (head, tail, pos); | ||
749 | GNUNET_free (pos); @} | ||
750 | @end example | ||
751 | |||
752 | |||
753 | instead of | ||
754 | @example | ||
755 | next = head; while (NULL != (pos = next)) @{ next = | ||
756 | pos->next; if (should_free (pos)) @{ | ||
757 | /* unnecessary nesting! */ | ||
758 | GNUNET_CONTAINER_DLL_remove (head, tail, pos); GNUNET_free (pos); @} @} | ||
759 | @end example | ||
760 | |||
761 | |||
762 | @item We primarily use @code{for} and @code{while} loops. A @code{while} loop | ||
763 | is used if the method for advancing in the loop is not a straightforward | ||
764 | increment operation. In particular, we use:@ | ||
765 | |||
766 | @example | ||
767 | next = head; | ||
768 | while (NULL != (pos = next)) | ||
769 | @{ | ||
770 | next = pos->next; | ||
771 | if (! should_free (pos)) | ||
772 | continue; | ||
773 | GNUNET_CONTAINER_DLL_remove (head, tail, pos); | ||
774 | GNUNET_free (pos); | ||
775 | @} | ||
776 | @end example | ||
777 | |||
778 | |||
779 | to free entries in a list (as the iteration changes the structure of the list | ||
780 | due to the free; the equivalent @code{for} loop does no longer follow the | ||
781 | simple @code{for} paradigm of @code{for(INIT;TEST;INC)}). However, for loops | ||
782 | that do follow the simple @code{for} paradigm we do use @code{for}, even if it | ||
783 | involves linked lists: | ||
784 | @example | ||
785 | /* simple iteration over a linked list */ | ||
786 | for (pos = head; NULL != pos; pos = pos->next) | ||
787 | @{ | ||
788 | use (pos); | ||
789 | @} | ||
790 | @end example | ||
791 | |||
792 | |||
793 | @item The first argument to all higher-order functions in GNUnet must be | ||
794 | declared to be of type @code{void *} and is reserved for a closure. We do not | ||
795 | use inner functions, as trampolines would conflict with setups that use | ||
796 | non-executable stacks.@ The first statement in a higher-order function, which | ||
797 | unusually should be part of the variable declarations, should assign the | ||
798 | @code{cls} argument to the precise expected type. For example: | ||
799 | @example | ||
800 | int callback (void *cls, char *args) @{ | ||
801 | struct Foo *foo = cls; int other_variables; | ||
802 | |||
803 | /* rest of function */ | ||
804 | @} | ||
805 | @end example | ||
806 | |||
807 | |||
808 | @item It is good practice to write complex @code{if} expressions instead of | ||
809 | using deeply nested @code{if} statements. However, except for addition and | ||
810 | multiplication, all operators should use parens. This is fine:@ | ||
811 | |||
812 | @example | ||
813 | if ( (1 == foo) || ((0 == bar) && (x != y)) ) | ||
814 | return x; | ||
815 | @end example | ||
816 | |||
817 | |||
818 | However, this is not: | ||
819 | @example | ||
820 | if (1 == foo) | ||
821 | return x; | ||
822 | if (0 == bar && x != y) | ||
823 | return x; | ||
824 | @end example | ||
825 | |||
826 | |||
827 | Note that splitting the @code{if} statement above is debateable as the | ||
828 | @code{return x} is a very trivial statement. However, once the logic after the | ||
829 | branch becomes more complicated (and is still identical), the "or" formulation | ||
830 | should be used for sure. | ||
831 | |||
832 | @item There should be two empty lines between the end of the function and the | ||
833 | comments describing the following function. There should be a single empty line | ||
834 | after the initial variable declarations of a function. If a function has no | ||
835 | local variables, there should be no initial empty line. If a long function | ||
836 | consists of several complex steps, those steps might be separated by an empty | ||
837 | line (possibly followed by a comment describing the following step). The code | ||
838 | should not contain empty lines in arbitrary places; if in doubt, it is likely | ||
839 | better to NOT have an empty line (this way, more code will fit on the screen). | ||
840 | @end itemize | ||
841 | |||
842 | @c *************************************************************************** | ||
843 | @node Build-system | ||
844 | @section Build-system | ||
845 | |||
846 | If you have code that is likely not to compile or build rules you might want to | ||
847 | not trigger for most developers, use "if HAVE_EXPERIMENTAL" in your | ||
848 | Makefile.am. Then it is OK to (temporarily) add non-compiling (or | ||
849 | known-to-not-port) code. | ||
850 | |||
851 | If you want to compile all testcases but NOT run them, run configure with the@ | ||
852 | @code{--enable-test-suppression} option. | ||
853 | |||
854 | If you want to run all testcases, including those that take a while, run | ||
855 | configure with the@ @code{--enable-expensive-testcases} option. | ||
856 | |||
857 | If you want to compile and run benchmarks, run configure with the@ | ||
858 | @code{--enable-benchmarks} option. | ||
859 | |||
860 | If you want to obtain code coverage results, run configure with the@ | ||
861 | @code{--enable-coverage} option and run the coverage.sh script in contrib/. | ||
862 | |||
863 | @c *************************************************************************** | ||
864 | @node Developing extensions for GNUnet using the gnunet-ext template | ||
865 | @section Developing extensions for GNUnet using the gnunet-ext template | ||
866 | |||
867 | |||
868 | For developers who want to write extensions for GNUnet we provide the | ||
869 | gnunet-ext template to provide an easy to use skeleton. | ||
870 | |||
871 | gnunet-ext contains the build environment and template files for the | ||
872 | development of GNUnet services, command line tools, APIs and tests. | ||
873 | |||
874 | First of all you have to obtain gnunet-ext from git: | ||
875 | |||
876 | @code{git clone https://gnunet.org/git/gnunet-ext.git} | ||
877 | |||
878 | The next step is to bootstrap and configure it. For configure you have to | ||
879 | provide the path containing GNUnet with @code{--with-gnunet=/path/to/gnunet} | ||
880 | and the prefix where you want the install the extension using | ||
881 | @code{--prefix=/path/to/install}@ @code{@ ./bootstrap@ ./configure | ||
882 | --prefix=/path/to/install --with-gnunet=/path/to/gnunet@ } | ||
883 | |||
884 | When your GNUnet installation is not included in the default linker search | ||
885 | path, you have to add @code{/path/to/gnunet} to the file @code{/etc/ld.so.conf} | ||
886 | and run @code{ldconfig} or your add it to the environmental variable | ||
887 | @code{LD_LIBRARY_PATH} by using | ||
888 | |||
889 | @code{export LD_LIBRARY_PATH=/path/to/gnunet/lib} | ||
890 | |||
891 | @c *************************************************************************** | ||
892 | @node Writing testcases | ||
893 | @section Writing testcases | ||
894 | |||
895 | Ideally, any non-trivial GNUnet code should be covered by automated testcases. | ||
896 | Testcases should reside in the same place as the code that is being tested. The | ||
897 | name of source files implementing tests should begin with "test_" followed by | ||
898 | the name of the file that contains the code that is being tested. | ||
899 | |||
900 | Testcases in GNUnet should be integrated with the autotools build system. This | ||
901 | way, developers and anyone building binary packages will be able to run all | ||
902 | testcases simply by running @code{make check}. The final testcases shipped with | ||
903 | the distribution should output at most some brief progress information and not | ||
904 | display debug messages by default. The success or failure of a testcase must be | ||
905 | indicated by returning zero (success) or non-zero (failure) from the main | ||
906 | method of the testcase. The integration with the autotools is relatively | ||
907 | straightforward and only requires modifications to the @code{Makefile.am} in | ||
908 | the directory containing the testcase. For a testcase testing the code in | ||
909 | @code{foo.c} the @code{Makefile.am} would contain the following lines: | ||
910 | @example | ||
911 | check_PROGRAMS = test_foo TESTS = $(check_PROGRAMS) test_foo_SOURCES = | ||
912 | test_foo.c test_foo_LDADD = $(top_builddir)/src/util/libgnunetutil.la | ||
913 | @end example | ||
914 | |||
915 | Naturally, other libraries used by the testcase may be specified in the | ||
916 | @code{LDADD} directive as necessary. | ||
917 | |||
918 | Often testcases depend on additional input files, such as a configuration file. | ||
919 | These support files have to be listed using the EXTRA_DIST directive in order | ||
920 | to ensure that they are included in the distribution. Example: | ||
921 | @example | ||
922 | EXTRA_DIST = test_foo_data.conf | ||
923 | @end example | ||
924 | |||
925 | |||
926 | Executing @code{make check} will run all testcases in the current directory and | ||
927 | all subdirectories. Testcases can be compiled individually by running | ||
928 | @code{make test_foo} and then invoked directly using @code{./test_foo}. Note | ||
929 | that due to the use of plugins in GNUnet, it is typically necessary to run | ||
930 | @code{make install} before running any testcases. Thus the canonical command | ||
931 | @code{make check install} has to be changed to @code{make install check} for | ||
932 | GNUnet. | ||
933 | |||
934 | @c *************************************************************************** | ||
935 | @node GNUnet's TESTING library | ||
936 | @section GNUnet's TESTING library | ||
937 | |||
938 | The TESTING library is used for writing testcases which involve starting a | ||
939 | single or multiple peers. While peers can also be started by testcases using | ||
940 | the ARM subsystem, using TESTING library provides an elegant way to do this. | ||
941 | The configurations of the peers are auto-generated from a given template to | ||
942 | have non-conflicting port numbers ensuring that peers' services do not run into | ||
943 | bind errors. This is achieved by testing ports' availability by binding a | ||
944 | listening socket to them before allocating them to services in the generated | ||
945 | configurations. | ||
946 | |||
947 | An another advantage while using TESTING is that it shortens the testcase | ||
948 | startup time as the hostkeys for peers are copied from a pre-computed set of | ||
949 | hostkeys instead of generating them at peer startup which may take a | ||
950 | considerable amount of time when starting multiple peers or on an embedded | ||
951 | processor. | ||
952 | |||
953 | TESTING also allows for certain services to be shared among peers. This feature | ||
954 | is invaluable when testing with multiple peers as it helps to reduce the number | ||
955 | of services run per each peer and hence the total number of processes run per | ||
956 | testcase. | ||
957 | |||
958 | TESTING library only handles creating, starting and stopping peers. Features | ||
959 | useful for testcases such as connecting peers in a topology are not available | ||
960 | in TESTING but are available in the TESTBED subsystem. Furthermore, TESTING | ||
961 | only creates peers on the localhost, however by using TESTBED testcases can | ||
962 | benefit from creating peers across multiple hosts. | ||
963 | |||
964 | @menu | ||
965 | * API:: | ||
966 | * Finer control over peer stop:: | ||
967 | * Helper functions:: | ||
968 | * Testing with multiple processes:: | ||
969 | @end menu | ||
970 | |||
971 | @c *************************************************************************** | ||
972 | @node API | ||
973 | @subsection API | ||
974 | |||
975 | TESTING abstracts a group of peers as a TESTING system. All peers in a system | ||
976 | have common hostname and no two services of these peers have a same port or a | ||
977 | UNIX domain socket path. | ||
978 | |||
979 | TESTING system can be created with the function | ||
980 | @code{GNUNET_TESTING_system_create()} which returns a handle to the system. | ||
981 | This function takes a directory path which is used for generating the | ||
982 | configurations of peers, an IP address from which connections to the peers' | ||
983 | services should be allowed, the hostname to be used in peers' configuration, | ||
984 | and an array of shared service specifications of type @code{struct | ||
985 | GNUNET_TESTING_SharedService}. | ||
986 | |||
987 | The shared service specification must specify the name of the service to share, | ||
988 | the configuration pertaining to that shared service and the maximum number of | ||
989 | peers that are allowed to share a single instance of the shared service. | ||
990 | |||
991 | TESTING system created with @code{GNUNET_TESTING_system_create()} chooses ports | ||
992 | from the default range 12000 - 56000 while auto-generating configurations for | ||
993 | peers. This range can be customised with the function | ||
994 | @code{GNUNET_TESTING_system_create_with_portrange()}. This function is similar | ||
995 | to @code{GNUNET_TESTING_system_create()} except that it take 2 additional | ||
996 | parameters --- the start and end of the port range to use. | ||
997 | |||
998 | A TESTING system is destroyed with the funciton | ||
999 | @code{GNUNET_TESTING_system_destory()}. This function takes the handle of the | ||
1000 | system and a flag to remove the files created in the directory used to generate | ||
1001 | configurations. | ||
1002 | |||
1003 | A peer is created with the function @code{GNUNET_TESTING_peer_configure()}. | ||
1004 | This functions takes the system handle, a configuration template from which the | ||
1005 | configuration for the peer is auto-generated and the index from where the | ||
1006 | hostkey for the peer has to be copied from. When successfull, this function | ||
1007 | returs a handle to the peer which can be used to start and stop it and to | ||
1008 | obtain the identity of the peer. If unsuccessful, a NULL pointer is returned | ||
1009 | with an error message. This function handles the generated configuration to | ||
1010 | have non-conflicting ports and paths. | ||
1011 | |||
1012 | Peers can be started and stopped by calling the functions | ||
1013 | @code{GNUNET_TESTING_peer_start()} and @code{GNUNET_TESTING_peer_stop()} | ||
1014 | respectively. A peer can be destroyed by calling the function | ||
1015 | @code{GNUNET_TESTING_peer_destroy}. When a peer is destroyed, the ports and | ||
1016 | paths in allocated in its configuration are reclaimed for usage in new | ||
1017 | peers. | ||
1018 | |||
1019 | @c *************************************************************************** | ||
1020 | @node Finer control over peer stop | ||
1021 | @subsection Finer control over peer stop | ||
1022 | |||
1023 | Using @code{GNUNET_TESTING_peer_stop()} is normally fine for testcases. | ||
1024 | However, calling this function for each peer is inefficient when trying to | ||
1025 | shutdown multiple peers as this function sends the termination signal to the | ||
1026 | given peer process and waits for it to terminate. It would be faster in this | ||
1027 | case to send the termination signals to the peers first and then wait on them. | ||
1028 | This is accomplished by the functions @code{GNUNET_TESTING_peer_kill()} which | ||
1029 | sends a termination signal to the peer, and the function | ||
1030 | @code{GNUNET_TESTING_peer_wait()} which waits on the peer. | ||
1031 | |||
1032 | Further finer control can be achieved by choosing to stop a peer asynchronously | ||
1033 | with the function @code{GNUNET_TESTING_peer_stop_async()}. This function takes | ||
1034 | a callback parameter and a closure for it in addition to the handle to the peer | ||
1035 | to stop. The callback function is called with the given closure when the peer | ||
1036 | is stopped. Using this function eliminates blocking while waiting for the peer | ||
1037 | to terminate. | ||
1038 | |||
1039 | An asynchronous peer stop can be cancelled by calling the function | ||
1040 | @code{GNUNET_TESTING_peer_stop_async_cancel()}. Note that calling this function | ||
1041 | does not prevent the peer from terminating if the termination signal has | ||
1042 | already been sent to it. It does, however, cancels the callback to be called | ||
1043 | when the peer is stopped. | ||
1044 | |||
1045 | @c *************************************************************************** | ||
1046 | @node Helper functions | ||
1047 | @subsection Helper functions | ||
1048 | |||
1049 | Most of the testcases can benefit from an abstraction which configures a peer | ||
1050 | and starts it. This is provided by the function | ||
1051 | @code{GNUNET_TESTING_peer_run()}. This function takes the testing directory | ||
1052 | pathname, a configuration template, a callback and its closure. This function | ||
1053 | creates a peer in the given testing directory by using the configuration | ||
1054 | template, starts the peer and calls the given callback with the given closure. | ||
1055 | |||
1056 | The function @code{GNUNET_TESTING_peer_run()} starts the ARM service of the | ||
1057 | peer which starts the rest of the configured services. A similar function | ||
1058 | @code{GNUNET_TESTING_service_run} can be used to just start a single service of | ||
1059 | a peer. In this case, the peer's ARM service is not started; instead, only the | ||
1060 | given service is run. | ||
1061 | |||
1062 | @c *************************************************************************** | ||
1063 | @node Testing with multiple processes | ||
1064 | @subsection Testing with multiple processes | ||
1065 | |||
1066 | When testing GNUnet, the splitting of the code into a services and clients | ||
1067 | often complicates testing. The solution to this is to have the testcase fork | ||
1068 | @code{gnunet-service-arm}, ask it to start the required server and daemon | ||
1069 | processes and then execute appropriate client actions (to test the client APIs | ||
1070 | or the core module or both). If necessary, multiple ARM services can be forked | ||
1071 | using different ports (!) to simulate a network. However, most of the time only | ||
1072 | one ARM process is needed. Note that on exit, the testcase should shutdown ARM | ||
1073 | with a @code{TERM} signal (to give it the chance to cleanly stop its child | ||
1074 | processes). | ||
1075 | |||
1076 | The following code illustrates spawning and killing an ARM process from a | ||
1077 | testcase: | ||
1078 | @example | ||
1079 | static void run (void *cls, char *const *args, const char | ||
1080 | *cfgfile, const struct GNUNET_CONFIGURATION_Handle *cfg) @{ struct | ||
1081 | GNUNET_OS_Process *arm_pid; arm_pid = GNUNET_OS_start_process (NULL, NULL, | ||
1082 | "gnunet-service-arm", "gnunet-service-arm", "-c", cfgname, NULL); | ||
1083 | /* do real test work here */ | ||
1084 | if (0 != GNUNET_OS_process_kill (arm_pid, SIGTERM)) GNUNET_log_strerror | ||
1085 | (GNUNET_ERROR_TYPE_WARNING, "kill"); GNUNET_assert (GNUNET_OK == | ||
1086 | GNUNET_OS_process_wait (arm_pid)); GNUNET_OS_process_close (arm_pid); @} | ||
1087 | |||
1088 | GNUNET_PROGRAM_run (argc, argv, "NAME-OF-TEST", "nohelp", options, &run, cls); | ||
1089 | @end example | ||
1090 | |||
1091 | |||
1092 | An alternative way that works well to test plugins is to implement a | ||
1093 | mock-version of the environment that the plugin expects and then to simply load | ||
1094 | the plugin directly. | ||
1095 | |||
1096 | @c *************************************************************************** | ||
1097 | @node Performance regression analysis with Gauger | ||
1098 | @section Performance regression analysis with Gauger | ||
1099 | |||
1100 | To help avoid performance regressions, GNUnet uses Gauger. Gauger is a simple | ||
1101 | logging tool that allows remote hosts to send performance data to a central | ||
1102 | server, where this data can be analyzed and visualized. Gauger shows graphs of | ||
1103 | the repository revisions and the performace data recorded for each revision, so | ||
1104 | sudden performance peaks or drops can be identified and linked to a specific | ||
1105 | revision number. | ||
1106 | |||
1107 | In the case of GNUnet, the buildbots log the performance data obtained during | ||
1108 | the tests after each build. The data can be accesed on GNUnet's Gauger page. | ||
1109 | |||
1110 | The menu on the left allows to select either the results of just one build bot | ||
1111 | (under "Hosts") or review the data from all hosts for a given test result | ||
1112 | (under "Metrics"). In case of very different absolute value of the results, for | ||
1113 | instance arm vs. amd64 machines, the option "Normalize" on a metric view can | ||
1114 | help to get an idea about the performance evolution across all hosts. | ||
1115 | |||
1116 | Using Gauger in GNUnet and having the performance of a module tracked over time | ||
1117 | is very easy. First of course, the testcase must generate some consistent | ||
1118 | metric, which makes sense to have logged. Highly volatile or random dependant | ||
1119 | metrics probably are not ideal candidates for meaningful regression detection. | ||
1120 | |||
1121 | To start logging any value, just include @code{gauger.h} in your testcase code. | ||
1122 | Then, use the macro @code{GAUGER()} to make the buildbots log whatever value is | ||
1123 | of interest for you to @code{gnunet.org}'s Gauger server. No setup is necessary | ||
1124 | as most buildbots have already everything in place and new metrics are created | ||
1125 | on demand. To delete a metric, you need to contact a member of the GNUnet | ||
1126 | development team (a file will need to be removed manually from the respective | ||
1127 | directory). | ||
1128 | |||
1129 | The code in the test should look like this: | ||
1130 | @example | ||
1131 | [other includes] | ||
1132 | #include <gauger.h> | ||
1133 | |||
1134 | int main (int argc, char *argv[]) @{ | ||
1135 | |||
1136 | [run test, generate data] GAUGER("YOUR_MODULE", "METRIC_NAME", (float)value, | ||
1137 | "UNIT"); @} | ||
1138 | @end example | ||
1139 | |||
1140 | |||
1141 | Where: | ||
1142 | @table @asis | ||
1143 | |||
1144 | @item @strong{YOUR_MODULE} is a category in the gauger page and should be the | ||
1145 | name of the module or subsystem like "Core" or "DHT" | ||
1146 | @item @strong{METRIC} is | ||
1147 | the name of the metric being collected and should be concise and descriptive, | ||
1148 | like "PUT operations in sqlite-datastore". | ||
1149 | @item @strong{value} is the value | ||
1150 | of the metric that is logged for this run. | ||
1151 | @item @strong{UNIT} is the unit in | ||
1152 | which the value is measured, for instance "kb/s" or "kb of RAM/node". | ||
1153 | @end table | ||
1154 | |||
1155 | If you wish to use Gauger for your own project, you can grab a copy of the | ||
1156 | latest stable release or check out Gauger's Subversion repository. | ||
1157 | |||
1158 | @c *************************************************************************** | ||
1159 | @node GNUnet's TESTBED Subsystem | ||
1160 | @section GNUnet's TESTBED Subsystem | ||
1161 | |||
1162 | The TESTBED subsystem facilitates testing and measuring of multi-peer | ||
1163 | deployments on a single host or over multiple hosts. | ||
1164 | |||
1165 | The architecture of the testbed module is divided into the following: | ||
1166 | @itemize @bullet | ||
1167 | |||
1168 | @item Testbed API: An API which is used by the testing driver programs. It | ||
1169 | provides with functions for creating, destroying, starting, stopping peers, | ||
1170 | etc. | ||
1171 | |||
1172 | @item Testbed service (controller): A service which is started through the | ||
1173 | Testbed API. This service handles operations to create, destroy, start, stop | ||
1174 | peers, connect them, modify their configurations. | ||
1175 | |||
1176 | @item Testbed helper: When a controller has to be started on a host, the | ||
1177 | testbed API starts the testbed helper on that host which in turn starts the | ||
1178 | controller. The testbed helper receives a configuration for the controller | ||
1179 | through its stdin and changes it to ensure the controller doesn't run into any | ||
1180 | port conflict on that host. | ||
1181 | @end itemize | ||
1182 | |||
1183 | |||
1184 | The testbed service (controller) is different from the other GNUnet services in | ||
1185 | that it is not started by ARM and is not supposed to be run as a daemon. It is | ||
1186 | started by the testbed API through a testbed helper. In a typical scenario | ||
1187 | involving multiple hosts, a controller is started on each host. Controllers | ||
1188 | take up the actual task of creating peers, starting and stopping them on the | ||
1189 | hosts they run. | ||
1190 | |||
1191 | While running deployments on a single localhost the testbed API starts the | ||
1192 | testbed helper directly as a child process. When running deployments on remote | ||
1193 | hosts the testbed API starts Testbed Helpers on each remote host through remote | ||
1194 | shell. By default testbed API uses SSH as a remote shell. This can be changed | ||
1195 | by setting the environmental variable GNUNET_TESTBED_RSH_CMD to the required | ||
1196 | remote shell program. This variable can also contain parameters which are to be | ||
1197 | passed to the remote shell program. For e.g:@ @code{@ export | ||
1198 | GNUNET_TESTBED_RSH_CMD="ssh -o BatchMode=yes -o | ||
1199 | NoHostAuthenticationForLocalhost=yes %h"@ }@ Substitutions are allowed int the | ||
1200 | above command string also allows for substitions. through placemarks which | ||
1201 | begin with a `%'. At present the following substitutions are supported | ||
1202 | @itemize @bullet | ||
1203 | @item | ||
1204 | %h: hostname | ||
1205 | @item | ||
1206 | %u: username | ||
1207 | @item | ||
1208 | %p: port | ||
1209 | @end itemize | ||
1210 | |||
1211 | Note that the substitution placemark is replaced only when the corresponding | ||
1212 | field is available and only once. Specifying @code{%u@@%h} doesn't work either. | ||
1213 | If you want to user username substitutions for SSH use the argument @code{-l} | ||
1214 | before the username substitution. Ex: @code{ssh -l %u -p %p %h} | ||
1215 | |||
1216 | The testbed API and the helper communicate through the helpers stdin and | ||
1217 | stdout. As the helper is started through a remote shell on remote hosts any | ||
1218 | output messages from the remote shell interfere with the communication and | ||
1219 | results in a failure while starting the helper. For this reason, it is | ||
1220 | suggested to use flags to make the remote shells produce no output messages and | ||
1221 | to have password-less logins. The default remote shell, SSH, the default | ||
1222 | options are "-o BatchMode=yes -o NoHostBasedAuthenticationForLocalhost=yes". | ||
1223 | Password-less logins should be ensured by using SSH keys. | ||
1224 | |||
1225 | Since the testbed API executes the remote shell as a non-interactive shell, | ||
1226 | certain scripts like .bashrc, .profiler may not be executed. If this is the | ||
1227 | case testbed API can be forced to execute an interactive shell by setting up | ||
1228 | the environmental variable `GNUNET_TESTBED_RSH_CMD_SUFFIX' to a shell program. | ||
1229 | An example could be:@ @code{@ export GNUNET_TESTBED_RSH_CMD_SUFFIX="sh -lc"@ }@ | ||
1230 | The testbed API will then execute the remote shell program as: @code{ | ||
1231 | $GNUNET_TESTBED_RSH_CMD -p $port $dest $GNUNET_TESTBED_RSH_CMD_SUFFIX | ||
1232 | gnunet-helper-testbed } | ||
1233 | |||
1234 | On some systems, problems may arise while starting testbed helpers if GNUnet is | ||
1235 | installed into a custom location since the helper may not be found in the | ||
1236 | standard path. This can be addressed by setting the variable | ||
1237 | `HELPER_BINARY_PATH' to the path of the testbed helper. Testbed API will then | ||
1238 | use this path to start helper binaries both locally and remotely. | ||
1239 | |||
1240 | Testbed API can accessed by including "gnunet_testbed_service.h" file and | ||
1241 | linking with -lgnunettestbed. | ||
1242 | |||
1243 | |||
1244 | |||
1245 | @c *************************************************************************** | ||
1246 | @menu | ||
1247 | * Supported Topologies:: | ||
1248 | * Hosts file format:: | ||
1249 | * Topology file format:: | ||
1250 | * Testbed Barriers:: | ||
1251 | * Automatic large-scale deployment of GNUnet in the PlanetLab testbed:: | ||
1252 | * TESTBED Caveats:: | ||
1253 | @end menu | ||
1254 | |||
1255 | @node Supported Topologies | ||
1256 | @subsection Supported Topologies | ||
1257 | |||
1258 | While testing multi-peer deployments, it is often needed that the peers are | ||
1259 | connected in some topology. This requirement is addressed by the function | ||
1260 | @code{GNUNET_TESTBED_overlay_connect()} which connects any given two peers in | ||
1261 | the testbed. | ||
1262 | |||
1263 | The API also provides a helper function | ||
1264 | @code{GNUNET_TESTBED_overlay_configure_topology()} to connect a given set of | ||
1265 | peers in any of the following supported topologies: | ||
1266 | @itemize @bullet | ||
1267 | |||
1268 | @item @code{GNUNET_TESTBED_TOPOLOGY_CLIQUE}: All peers are connected with each | ||
1269 | other | ||
1270 | |||
1271 | @item @code{GNUNET_TESTBED_TOPOLOGY_LINE}: Peers are connected to form a line | ||
1272 | |||
1273 | @item @code{GNUNET_TESTBED_TOPOLOGY_RING}: Peers are connected to form a ring | ||
1274 | topology | ||
1275 | |||
1276 | @item @code{GNUNET_TESTBED_TOPOLOGY_2D_TORUS}: Peers are connected to form a 2 | ||
1277 | dimensional torus topology. The number of peers may not be a perfect square, in | ||
1278 | that case the resulting torus may not have the uniform poloidal and toroidal | ||
1279 | lengths | ||
1280 | |||
1281 | @item @code{GNUNET_TESTBED_TOPOLOGY_ERDOS_RENYI}: Topology is generated to form | ||
1282 | a random graph. The number of links to be present should be given | ||
1283 | |||
1284 | @item @code{GNUNET_TESTBED_TOPOLOGY_SMALL_WORLD}: Peers are connected to form a | ||
1285 | 2D Torus with some random links among them. The number of random links are to | ||
1286 | be given | ||
1287 | |||
1288 | @item @code{GNUNET_TESTBED_TOPOLOGY_SMALL_WORLD_RING}: Peers are connected to | ||
1289 | form a ring with some random links among them. The number of random links are | ||
1290 | to be given | ||
1291 | |||
1292 | @item @code{GNUNET_TESTBED_TOPOLOGY_SCALE_FREE}: Connects peers in a topology | ||
1293 | where peer connectivity follows power law - new peers are connected with high | ||
1294 | probabililty to well connected peers. See Emergence of Scaling in Random | ||
1295 | Networks. Science 286, 509-512, 1999. | ||
1296 | |||
1297 | @item @code{GNUNET_TESTBED_TOPOLOGY_FROM_FILE}: The topology information is | ||
1298 | loaded from a file. The path to the file has to be given. See Topology file | ||
1299 | format for the format of this file. | ||
1300 | |||
1301 | @item @code{GNUNET_TESTBED_TOPOLOGY_NONE}: No topology | ||
1302 | @end itemize | ||
1303 | |||
1304 | |||
1305 | The above supported topologies can be specified respectively by setting the | ||
1306 | variable @code{OVERLAY_TOPOLOGY} to the following values in the configuration | ||
1307 | passed to Testbed API functions @code{GNUNET_TESTBED_test_run()} and | ||
1308 | @code{GNUNET_TESTBED_run()}: | ||
1309 | @itemize @bullet | ||
1310 | @item @code{CLIQUE} | ||
1311 | @item @code{RING} | ||
1312 | @item @code{LINE} | ||
1313 | @item @code{2D_TORUS} | ||
1314 | @item @code{RANDOM} | ||
1315 | @item @code{SMALL_WORLD} | ||
1316 | @item @code{SMALL_WORLD_RING} | ||
1317 | @item @code{SCALE_FREE} | ||
1318 | @item @code{FROM_FILE} | ||
1319 | @item @code{NONE} | ||
1320 | @end itemize | ||
1321 | |||
1322 | |||
1323 | Topologies @code{RANDOM}, @code{SMALL_WORLD} and @code{SMALL_WORLD_RING} | ||
1324 | require the option @code{OVERLAY_RANDOM_LINKS} to be set to the number of | ||
1325 | random links to be generated in the configuration. The option will be ignored | ||
1326 | for the rest of the topologies. | ||
1327 | |||
1328 | Topology @code{SCALE_FREE} requires the options @code{SCALE_FREE_TOPOLOGY_CAP} | ||
1329 | to be set to the maximum number of peers which can connect to a peer and | ||
1330 | @code{SCALE_FREE_TOPOLOGY_M} to be set to how many peers a peer should be | ||
1331 | atleast connected to. | ||
1332 | |||
1333 | Similarly, the topology @code{FROM_FILE} requires the option | ||
1334 | @code{OVERLAY_TOPOLOGY_FILE} to contain the path of the file containing the | ||
1335 | topology information. This option is ignored for the rest of the topologies. | ||
1336 | See Topology file format for the format of this file. | ||
1337 | |||
1338 | @c *************************************************************************** | ||
1339 | @node Hosts file format | ||
1340 | @subsection Hosts file format | ||
1341 | |||
1342 | The testbed API offers the function GNUNET_TESTBED_hosts_load_from_file() to | ||
1343 | load from a given file details about the hosts which testbed can use for | ||
1344 | deploying peers. This function is useful to keep the data about hosts separate | ||
1345 | instead of hard coding them in code. | ||
1346 | |||
1347 | Another helper function from testbed API, GNUNET_TESTBED_run() also takes a | ||
1348 | hosts file name as its parameter. It uses the above function to populate the | ||
1349 | hosts data structures and start controllers to deploy peers. | ||
1350 | |||
1351 | These functions require the hosts file to be of the following format: | ||
1352 | @itemize @bullet | ||
1353 | @item Each line is interpreted to have details about a host | ||
1354 | @item Host details should include the username to use for logging into the | ||
1355 | host, the hostname of the host and the port number to use for the remote shell | ||
1356 | program. All thee values should be given. | ||
1357 | @item These details should be given in the following format: | ||
1358 | @code{<username>@@<hostname>:<port>} | ||
1359 | @end itemize | ||
1360 | |||
1361 | Note that having canonical hostnames may cause problems while resolving the IP | ||
1362 | addresses (See this bug). Hence it is advised to provide the hosts' IP | ||
1363 | numerical addresses as hostnames whenever possible. | ||
1364 | |||
1365 | @c *************************************************************************** | ||
1366 | @node Topology file format | ||
1367 | @subsection Topology file format | ||
1368 | |||
1369 | A topology file describes how peers are to be connected. It should adhere to | ||
1370 | the following format for testbed to parse it correctly. | ||
1371 | |||
1372 | Each line should begin with the target peer id. This should be followed by a | ||
1373 | colon(`:') and origin peer ids seperated by `|'. All spaces except for newline | ||
1374 | characters are ignored. The API will then try to connect each origin peer to | ||
1375 | the target peer. | ||
1376 | |||
1377 | For example, the following file will result in 5 overlay connections: [2->1], | ||
1378 | [3->1],[4->3], [0->3], [2->0]@ @code{@ 1:2|3@ 3:4| 0@ 0: 2@ } | ||
1379 | |||
1380 | @c *************************************************************************** | ||
1381 | @node Testbed Barriers | ||
1382 | @subsection Testbed Barriers | ||
1383 | |||
1384 | The testbed subsystem's barriers API facilitates coordination among the peers | ||
1385 | run by the testbed and the experiment driver. The concept is similar to the | ||
1386 | barrier synchronisation mechanism found in parallel programming or | ||
1387 | multi-threading paradigms - a peer waits at a barrier upon reaching it until | ||
1388 | the barrier is reached by a predefined number of peers. This predefined number | ||
1389 | of peers required to cross a barrier is also called quorum. We say a peer has | ||
1390 | reached a barrier if the peer is waiting for the barrier to be crossed. | ||
1391 | Similarly a barrier is said to be reached if the required quorum of peers reach | ||
1392 | the barrier. A barrier which is reached is deemed as crossed after all the | ||
1393 | peers waiting on it are notified. | ||
1394 | |||
1395 | The barriers API provides the following functions: | ||
1396 | @itemize @bullet | ||
1397 | @item @strong{@code{GNUNET_TESTBED_barrier_init()}:} function to initialse a | ||
1398 | barrier in the experiment | ||
1399 | @item @strong{@code{GNUNET_TESTBED_barrier_cancel()}:} function to cancel a | ||
1400 | barrier which has been initialised before | ||
1401 | @item @strong{@code{GNUNET_TESTBED_barrier_wait()}:} function to signal barrier | ||
1402 | service that the caller has reached a barrier and is waiting for it to be | ||
1403 | crossed | ||
1404 | @item @strong{@code{GNUNET_TESTBED_barrier_wait_cancel()}:} function to stop | ||
1405 | waiting for a barrier to be crossed | ||
1406 | @end itemize | ||
1407 | |||
1408 | |||
1409 | Among the above functions, the first two, namely | ||
1410 | @code{GNUNET_TESTBED_barrier_init()} and @code{GNUNET_TESTBED_barrier_cancel()} | ||
1411 | are used by experiment drivers. All barriers should be initialised by the | ||
1412 | experiment driver by calling @code{GNUNET_TESTBED_barrier_init()}. This | ||
1413 | function takes a name to identify the barrier, the quorum required for the | ||
1414 | barrier to be crossed and a notification callback for notifying the experiment | ||
1415 | driver when the barrier is crossed. @code{GNUNET_TESTBED_barrier_cancel()} | ||
1416 | cancels an initialised barrier and frees the resources allocated for it. This | ||
1417 | function can be called upon a initialised barrier before it is crossed. | ||
1418 | |||
1419 | The remaining two functions @code{GNUNET_TESTBED_barrier_wait()} and | ||
1420 | @code{GNUNET_TESTBED_barrier_wait_cancel()} are used in the peer's processes. | ||
1421 | @code{GNUNET_TESTBED_barrier_wait()} connects to the local barrier service | ||
1422 | running on the same host the peer is running on and registers that the caller | ||
1423 | has reached the barrier and is waiting for the barrier to be crossed. Note that | ||
1424 | this function can only be used by peers which are started by testbed as this | ||
1425 | function tries to access the local barrier service which is part of the testbed | ||
1426 | controller service. Calling @code{GNUNET_TESTBED_barrier_wait()} on an | ||
1427 | uninitialised barrier results in failure. | ||
1428 | @code{GNUNET_TESTBED_barrier_wait_cancel()} cancels the notification registered | ||
1429 | by @code{GNUNET_TESTBED_barrier_wait()}. | ||
1430 | |||
1431 | |||
1432 | @c *************************************************************************** | ||
1433 | @menu | ||
1434 | * Implementation:: | ||
1435 | @end menu | ||
1436 | |||
1437 | @node Implementation | ||
1438 | @subsubsection Implementation | ||
1439 | |||
1440 | Since barriers involve coordination between experiment driver and peers, the | ||
1441 | barrier service in the testbed controller is split into two components. The | ||
1442 | first component responds to the message generated by the barrier API used by | ||
1443 | the experiment driver (functions @code{GNUNET_TESTBED_barrier_init()} and | ||
1444 | @code{GNUNET_TESTBED_barrier_cancel()}) and the second component to the | ||
1445 | messages generated by barrier API used by peers (functions | ||
1446 | @code{GNUNET_TESTBED_barrier_wait()} and | ||
1447 | @code{GNUNET_TESTBED_barrier_wait_cancel()}). | ||
1448 | |||
1449 | Calling @code{GNUNET_TESTBED_barrier_init()} sends a | ||
1450 | @code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_INIT} message to the master | ||
1451 | controller. The master controller then registers a barrier and calls | ||
1452 | @code{GNUNET_TESTBED_barrier_init()} for each its subcontrollers. In this way | ||
1453 | barrier initialisation is propagated to the controller hierarchy. While | ||
1454 | propagating initialisation, any errors at a subcontroller such as timeout | ||
1455 | during further propagation are reported up the hierarchy back to the experiment | ||
1456 | driver. | ||
1457 | |||
1458 | Similar to @code{GNUNET_TESTBED_barrier_init()}, | ||
1459 | @code{GNUNET_TESTBED_barrier_cancel()} propagates | ||
1460 | @code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_CANCEL} message which causes | ||
1461 | controllers to remove an initialised barrier. | ||
1462 | |||
1463 | The second component is implemented as a separate service in the binary | ||
1464 | `gnunet-service-testbed' which already has the testbed controller service. | ||
1465 | Although this deviates from the gnunet process architecture of having one | ||
1466 | service per binary, it is needed in this case as this component needs access to | ||
1467 | barrier data created by the first component. This component responds to | ||
1468 | @code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_WAIT} messages from local peers when | ||
1469 | they call @code{GNUNET_TESTBED_barrier_wait()}. Upon receiving | ||
1470 | @code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_WAIT} message, the service checks if | ||
1471 | the requested barrier has been initialised before and if it was not | ||
1472 | initialised, an error status is sent through | ||
1473 | @code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_STATUS} message to the local peer and | ||
1474 | the connection from the peer is terminated. If the barrier is initialised | ||
1475 | before, the barrier's counter for reached peers is incremented and a | ||
1476 | notification is registered to notify the peer when the barrier is reached. The | ||
1477 | connection from the peer is left open. | ||
1478 | |||
1479 | When enough peers required to attain the quorum send | ||
1480 | @code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_WAIT} messages, the controller sends | ||
1481 | a @code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_STATUS} message to its parent | ||
1482 | informing that the barrier is crossed. If the controller has started further | ||
1483 | subcontrollers, it delays this message until it receives a similar notification | ||
1484 | from each of those subcontrollers. Finally, the barriers API at the experiment | ||
1485 | driver receives the @code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_STATUS} when the | ||
1486 | barrier is reached at all the controllers. | ||
1487 | |||
1488 | The barriers API at the experiment driver responds to the | ||
1489 | @code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_STATUS} message by echoing it back to | ||
1490 | the master controller and notifying the experiment controller through the | ||
1491 | notification callback that a barrier has been crossed. The echoed | ||
1492 | @code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_STATUS} message is propagated by the | ||
1493 | master controller to the controller hierarchy. This propagation triggers the | ||
1494 | notifications registered by peers at each of the controllers in the hierarchy. | ||
1495 | Note the difference between this downward propagation of the | ||
1496 | @code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_STATUS} message from its upward | ||
1497 | propagation --- the upward propagation is needed for ensuring that the barrier | ||
1498 | is reached by all the controllers and the downward propagation is for | ||
1499 | triggering that the barrier is crossed. | ||
1500 | |||
1501 | @c *************************************************************************** | ||
1502 | @node Automatic large-scale deployment of GNUnet in the PlanetLab testbed | ||
1503 | @subsection Automatic large-scale deployment of GNUnet in the PlanetLab testbed | ||
1504 | |||
1505 | PlanetLab is as a testbed for computer networking and distributed systems | ||
1506 | research. It was established in 2002 and as of June 2010 was composed of 1090 | ||
1507 | nodes at 507 sites worldwide. | ||
1508 | |||
1509 | To automate the GNUnet we created a set of automation tools to simplify the | ||
1510 | large-scale deployment. We provide you a set of scripts you can use to deploy | ||
1511 | GNUnet on a set of nodes and manage your installation. | ||
1512 | |||
1513 | Please also check @uref{https://gnunet.org/installation-fedora8-svn} and@ | ||
1514 | @uref{https://gnunet.org/installation-fedora12-svn} to find detailled | ||
1515 | instructions how to install GNUnet on a PlanetLab node. | ||
1516 | |||
1517 | |||
1518 | @c *************************************************************************** | ||
1519 | @menu | ||
1520 | * PlanetLab Automation for Fedora8 nodes:: | ||
1521 | * Install buildslave on PlanetLab nodes running fedora core 8:: | ||
1522 | * Setup a new PlanetLab testbed using GPLMT:: | ||
1523 | * Why do i get an ssh error when using the regex profiler?:: | ||
1524 | @end menu | ||
1525 | |||
1526 | @node PlanetLab Automation for Fedora8 nodes | ||
1527 | @subsubsection PlanetLab Automation for Fedora8 nodes | ||
1528 | |||
1529 | @c *************************************************************************** | ||
1530 | @node Install buildslave on PlanetLab nodes running fedora core 8 | ||
1531 | @subsubsection Install buildslave on PlanetLab nodes running fedora core 8 | ||
1532 | @c ** Actually this is a subsubsubsection, but must be fixed differently | ||
1533 | @c ** as subsubsection is the lowest. | ||
1534 | |||
1535 | Since most of the PlanetLab nodes are running the very old fedora core 8 image, | ||
1536 | installing the buildslave software is quite some pain. For our PlanetLab | ||
1537 | testbed we figured out how to install the buildslave software best. | ||
1538 | |||
1539 | Install Distribute for python:@ @code{@ curl | ||
1540 | http://python-distribute.org/distribute_setup.py | sudo python@ } | ||
1541 | |||
1542 | Install Distribute for zope.interface <= 3.8.0 (4.0 and 4.0.1 will not work):@ | ||
1543 | @code{@ wget | ||
1544 | http://pypi.python.org/packages/source/z/zope.interface/zope.interface-3.8.0.tar.gz@ | ||
1545 | tar zvfz zope.interface-3.8.0.tar.gz@ cd zope.interface-3.8.0@ sudo python | ||
1546 | setup.py install@ } | ||
1547 | |||
1548 | Install the buildslave software (0.8.6 was the latest version):@ @code{@ wget | ||
1549 | http://buildbot.googlecode.com/files/buildbot-slave-0.8.6p1.tar.gz@ tar xvfz | ||
1550 | buildbot-slave-0.8.6p1.tar.gz@ cd buildslave-0.8.6p1@ sudo python setup.py | ||
1551 | install@ } | ||
1552 | |||
1553 | The setup will download the matching twisted package and install it.@ It will | ||
1554 | also try to install the latest version of zope.interface which will fail to | ||
1555 | install. Buildslave will work anyway since version 3.8.0 was installed before! | ||
1556 | |||
1557 | @c *************************************************************************** | ||
1558 | @node Setup a new PlanetLab testbed using GPLMT | ||
1559 | @subsubsection Setup a new PlanetLab testbed using GPLMT | ||
1560 | |||
1561 | @itemize @bullet | ||
1562 | @item Get a new slice and assign nodes | ||
1563 | Ask your PlanetLab PI to give you a new slice and assign the nodes you need | ||
1564 | @item Install a buildmaster | ||
1565 | You can stick to the buildbot documentation:@ | ||
1566 | @uref{http://buildbot.net/buildbot/docs/current/manual/installation.html} | ||
1567 | @item Install the buildslave software on all nodes | ||
1568 | To install the buildslave on all nodes assigned to your slice you can use the | ||
1569 | tasklist @code{install_buildslave_fc8.xml} provided with GPLMT: | ||
1570 | |||
1571 | @code{@ ./gplmt.py -c contrib/tumple_gnunet.conf -t | ||
1572 | contrib/tasklists/install_buildslave_fc8.xml -a -p <planetlab password>@ } | ||
1573 | |||
1574 | @item Create the buildmaster configuration and the slave setup commands | ||
1575 | |||
1576 | The master and the and the slaves have need to have credentials and the master | ||
1577 | has to have all nodes configured. This can be done with the | ||
1578 | @code{create_buildbot_configuration.py} script in the @code{scripts} directory | ||
1579 | |||
1580 | This scripts takes a list of nodes retrieved directly from PlanetLab or read | ||
1581 | from a file and a configuration template and creates:@ | ||
1582 | - a tasklist which can be executed with gplmt to setup the slaves@ | ||
1583 | - a master.cfg file containing a PlanetLab nodes | ||
1584 | |||
1585 | A configuration template is included in the <contrib>, most important is that | ||
1586 | the script replaces the following tags in the template: | ||
1587 | |||
1588 | %GPLMT_BUILDER_DEFINITION :@ GPLMT_BUILDER_SUMMARY@ GPLMT_SLAVES@ | ||
1589 | %GPLMT_SCHEDULER_BUILDERS | ||
1590 | |||
1591 | Create configuration for all nodes assigned to a slice:@ @code{@ | ||
1592 | ./create_buildbot_configuration.py -u <planetlab username> -p <planetlab | ||
1593 | password> -s <slice> -m <buildmaster+port> -t <template>@ }@ Create | ||
1594 | configuration for some nodes in a file:@ @code{@ | ||
1595 | ./create_buildbot_configuration.p -f <node_file> -m <buildmaster+port> -t | ||
1596 | <template>@ } | ||
1597 | |||
1598 | @item Copy the @code{master.cfg} to the buildmaster and start it | ||
1599 | Use @code{buildbot start <basedir>} to start the server | ||
1600 | @item Setup the buildslaves | ||
1601 | @end itemize | ||
1602 | |||
1603 | @c *************************************************************************** | ||
1604 | @node Why do i get an ssh error when using the regex profiler? | ||
1605 | @subsubsection Why do i get an ssh error when using the regex profiler? | ||
1606 | |||
1607 | Why do i get an ssh error "Permission denied (publickey,password)." when using | ||
1608 | the regex profiler although passwordless ssh to localhost works using publickey | ||
1609 | and ssh-agent? | ||
1610 | |||
1611 | You have to generate a public/private-key pair with no password:@ | ||
1612 | @code{ssh-keygen -t rsa -b 4096 -f ~/.ssh/id_localhost}@ | ||
1613 | and then add the following to your ~/.ssh/config file: | ||
1614 | |||
1615 | @code{Host 127.0.0.1@ IdentityFile ~/.ssh/id_localhost} | ||
1616 | |||
1617 | now make sure your hostsfile looks like@ | ||
1618 | |||
1619 | [USERNAME]@@127.0.0.1:22@ | ||
1620 | [USERNAME]@@127.0.0.1:22 | ||
1621 | |||
1622 | You can test your setup by running `ssh 127.0.0.1` in a terminal and then in | ||
1623 | the opened session run it again. If you were not asked for a password on either | ||
1624 | login, then you should be good to go. | ||
1625 | |||
1626 | @c *************************************************************************** | ||
1627 | @node TESTBED Caveats | ||
1628 | @subsection TESTBED Caveats | ||
1629 | |||
1630 | This section documents a few caveats when using the GNUnet testbed | ||
1631 | subsystem. | ||
1632 | |||
1633 | |||
1634 | @c *************************************************************************** | ||
1635 | @menu | ||
1636 | * CORE must be started:: | ||
1637 | * ATS must want the connections:: | ||
1638 | @end menu | ||
1639 | |||
1640 | @node CORE must be started | ||
1641 | @subsubsection CORE must be started | ||
1642 | |||
1643 | A simple issue is #3993: Your configuration MUST somehow ensure that for each | ||
1644 | peer the CORE service is started when the peer is setup, otherwise TESTBED may | ||
1645 | fail to connect peers when the topology is initialized, as TESTBED will start | ||
1646 | some CORE services but not necessarily all (but it relies on all of them | ||
1647 | running). The easiest way is to set 'FORCESTART = YES' in the '[core]' section | ||
1648 | of the configuration file. Alternatively, having any service that directly or | ||
1649 | indirectly depends on CORE being started with FORCESTART will also do. This | ||
1650 | issue largely arises if users try to over-optimize by not starting any services | ||
1651 | with FORCESTART. | ||
1652 | |||
1653 | @c *************************************************************************** | ||
1654 | @node ATS must want the connections | ||
1655 | @subsubsection ATS must want the connections | ||
1656 | |||
1657 | When TESTBED sets up connections, it only offers the respective HELLO | ||
1658 | information to the TRANSPORT service. It is then up to the ATS service to | ||
1659 | @strong{decide} to use the connection. The ATS service will typically eagerly | ||
1660 | establish any connection if the number of total connections is low (relative to | ||
1661 | bandwidth). Details may further depend on the specific ATS backend that was | ||
1662 | configured. If ATS decides to NOT establish a connection (even though TESTBED | ||
1663 | provided the required information), then that connection will count as failed | ||
1664 | for TESTBED. Note that you can configure TESTBED to tolerate a certain number | ||
1665 | of connection failures (see '-e' option of gnunet-testbed-profiler). This issue | ||
1666 | largely arises for dense overlay topologies, especially if you try to create | ||
1667 | cliques with more than 20 peers. | ||
1668 | |||
1669 | @c *************************************************************************** | ||
1670 | @node libgnunetutil | ||
1671 | @section libgnunetutil | ||
1672 | |||
1673 | libgnunetutil is the fundamental library that all GNUnet code builds upon. | ||
1674 | Ideally, this library should contain most of the platform dependent code | ||
1675 | (except for user interfaces and really special needs that only few applications | ||
1676 | have). It is also supposed to offer basic services that most if not all GNUnet | ||
1677 | binaries require. The code of libgnunetutil is in the src/util/ directory. The | ||
1678 | public interface to the library is in the gnunet_util.h header. The functions | ||
1679 | provided by libgnunetutil fall roughly into the following categories (in | ||
1680 | roughly the order of importance for new developers): | ||
1681 | @itemize @bullet | ||
1682 | @item logging (common_logging.c) | ||
1683 | @item memory allocation (common_allocation.c) | ||
1684 | @item endianess conversion (common_endian.c) | ||
1685 | @item internationalization (common_gettext.c) | ||
1686 | @item String manipulation (string.c) | ||
1687 | @item file access (disk.c) | ||
1688 | @item buffered disk IO (bio.c) | ||
1689 | @item time manipulation (time.c) | ||
1690 | @item configuration parsing (configuration.c) | ||
1691 | @item command-line handling (getopt*.c) | ||
1692 | @item cryptography (crypto_*.c) | ||
1693 | @item data structures (container_*.c) | ||
1694 | @item CPS-style scheduling (scheduler.c) | ||
1695 | @item Program initialization (program.c) | ||
1696 | @item Networking (network.c, client.c, server*.c, service.c) | ||
1697 | @item message queueing (mq.c) | ||
1698 | @item bandwidth calculations (bandwidth.c) | ||
1699 | @item Other OS-related (os*.c, plugin.c, signal.c) | ||
1700 | @item Pseudonym management (pseudonym.c) | ||
1701 | @end itemize | ||
1702 | |||
1703 | It should be noted that only developers that fully understand this entire API | ||
1704 | will be able to write good GNUnet code. | ||
1705 | |||
1706 | Ideally, porting GNUnet should only require porting the gnunetutil library. | ||
1707 | More testcases for the gnunetutil APIs are therefore a great way to make | ||
1708 | porting of GNUnet easier. | ||
1709 | |||
1710 | @menu | ||
1711 | * Logging:: | ||
1712 | * Interprocess communication API (IPC):: | ||
1713 | * Cryptography API:: | ||
1714 | * Message Queue API:: | ||
1715 | * Service API:: | ||
1716 | * Optimizing Memory Consumption of GNUnet's (Multi-) Hash Maps:: | ||
1717 | * The CONTAINER_MDLL API:: | ||
1718 | @end menu | ||
1719 | |||
1720 | @c *************************************************************************** | ||
1721 | @node Logging | ||
1722 | @subsection Logging | ||
1723 | |||
1724 | GNUnet is able to log its activity, mostly for the purposes of debugging the | ||
1725 | program at various levels. | ||
1726 | |||
1727 | @file{gnunet_common.h} defines several @strong{log levels}: | ||
1728 | @table @asis | ||
1729 | |||
1730 | @item ERROR for errors (really problematic situations, often leading to | ||
1731 | crashes) | ||
1732 | @item WARNING for warnings (troubling situations that might have | ||
1733 | negative consequences, although not fatal) | ||
1734 | @item INFO for various information. | ||
1735 | Used somewhat rarely, as GNUnet statistics is used to hold and display most of | ||
1736 | the information that users might find interesting. | ||
1737 | @item DEBUG for debugging. | ||
1738 | Does not produce much output on normal builds, but when extra logging is | ||
1739 | enabled at compile time, a staggering amount of data is outputted under this | ||
1740 | log level. | ||
1741 | @end table | ||
1742 | |||
1743 | |||
1744 | Normal builds of GNUnet (configured with @code{--enable-logging[=yes]}) are | ||
1745 | supposed to log nothing under DEBUG level. The @code{--enable-logging=verbose} | ||
1746 | configure option can be used to create a build with all logging enabled. | ||
1747 | However, such build will produce large amounts of log data, which is | ||
1748 | inconvenient when one tries to hunt down a specific problem. | ||
1749 | |||
1750 | To mitigate this problem, GNUnet provides facilities to apply a filter to | ||
1751 | reduce the logs: | ||
1752 | @table @asis | ||
1753 | |||
1754 | @item Logging by default When no log levels are configured in any other way | ||
1755 | (see below), GNUnet will default to the WARNING log level. This mostly applies | ||
1756 | to GNUnet command line utilities, services and daemons; tests will always set | ||
1757 | log level to WARNING or, if @code{--enable-logging=verbose} was passed to | ||
1758 | configure, to DEBUG. The default level is suggested for normal operation. | ||
1759 | @item The -L option Most GNUnet executables accept an "-L loglevel" or | ||
1760 | "--log=loglevel" option. If used, it makes the process set a global log level | ||
1761 | to "loglevel". Thus it is possible to run some processes with -L DEBUG, for | ||
1762 | example, and others with -L ERROR to enable specific settings to diagnose | ||
1763 | problems with a particular process. | ||
1764 | @item Configuration files. Because GNUnet | ||
1765 | service and deamon processes are usually launched by gnunet-arm, it is not | ||
1766 | possible to pass different custom command line options directly to every one of | ||
1767 | them. The options passed to @code{gnunet-arm} only affect gnunet-arm and not | ||
1768 | the rest of GNUnet. However, one can specify a configuration key "OPTIONS" in | ||
1769 | the section that corresponds to a service or a daemon, and put a value of "-L | ||
1770 | loglevel" there. This will make the respective service or daemon set its log | ||
1771 | level to "loglevel" (as the value of OPTIONS will be passed as a command-line | ||
1772 | argument). | ||
1773 | |||
1774 | To specify the same log level for all services without creating separate | ||
1775 | "OPTIONS" entries in the configuration for each one, the user can specify a | ||
1776 | config key "GLOBAL_POSTFIX" in the [arm] section of the configuration file. The | ||
1777 | value of GLOBAL_POSTFIX will be appended to all command lines used by the ARM | ||
1778 | service to run other services. It can contain any option valid for all GNUnet | ||
1779 | commands, thus in particular the "-L loglevel" option. The ARM service itself | ||
1780 | is, however, unaffected by GLOBAL_POSTFIX; to set log level for it, one has to | ||
1781 | specify "OPTIONS" key in the [arm] section. | ||
1782 | @item Environment variables. | ||
1783 | Setting global per-process log levels with "-L loglevel" does not offer | ||
1784 | sufficient log filtering granularity, as one service will call interface | ||
1785 | libraries and supporting libraries of other GNUnet services, potentially | ||
1786 | producing lots of debug log messages from these libraries. Also, changing the | ||
1787 | config file is not always convenient (especially when running the GNUnet test | ||
1788 | suite).@ To fix that, and to allow GNUnet to use different log filtering at | ||
1789 | runtime without re-compiling the whole source tree, the log calls were changed | ||
1790 | to be configurable at run time. To configure them one has to define environment | ||
1791 | variables "GNUNET_FORCE_LOGFILE", "GNUNET_LOG" and/or "GNUNET_FORCE_LOG": | ||
1792 | @itemize @bullet | ||
1793 | |||
1794 | @item "GNUNET_LOG" only affects the logging when no global log level is | ||
1795 | configured by any other means (that is, the process does not explicitly set its | ||
1796 | own log level, there are no "-L loglevel" options on command line or in | ||
1797 | configuration files), and can be used to override the default WARNING log | ||
1798 | level. | ||
1799 | |||
1800 | @item "GNUNET_FORCE_LOG" will completely override any other log configuration | ||
1801 | options given. | ||
1802 | |||
1803 | @item "GNUNET_FORCE_LOGFILE" will completely override the location of the file | ||
1804 | to log messages to. It should contain a relative or absolute file name. Setting | ||
1805 | GNUNET_FORCE_LOGFILE is equivalent to passing "--log-file=logfile" or "-l | ||
1806 | logfile" option (see below). It supports "[]" format in file names, but not | ||
1807 | "@{@}" (see below). | ||
1808 | @end itemize | ||
1809 | |||
1810 | |||
1811 | Because environment variables are inherited by child processes when they are | ||
1812 | launched, starting or re-starting the ARM service with these variables will | ||
1813 | propagate them to all other services. | ||
1814 | |||
1815 | "GNUNET_LOG" and "GNUNET_FORCE_LOG" variables must contain a specially | ||
1816 | formatted @strong{logging definition} string, which looks like this:@ @code{@ | ||
1817 | [component];[file];[function];[from_line[-to_line]];loglevel@emph{[/component...]}@ | ||
1818 | }@ That is, a logging definition consists of definition entries, separated by | ||
1819 | slashes ('/'). If only one entry is present, there is no need to add a slash | ||
1820 | to its end (although it is not forbidden either).@ All definition fields | ||
1821 | (component, file, function, lines and loglevel) are mandatory, but (except for | ||
1822 | the loglevel) they can be empty. An empty field means "match anything". Note | ||
1823 | that even if fields are empty, the semicolon (';') separators must be | ||
1824 | present.@ The loglevel field is mandatory, and must contain one of the log | ||
1825 | level names (ERROR, WARNING, INFO or DEBUG).@ The lines field might contain | ||
1826 | one non-negative number, in which case it matches only one line, or a range | ||
1827 | "from_line-to_line", in which case it matches any line in the interval | ||
1828 | [from_line;to_line] (that is, including both start and end line).@ GNUnet | ||
1829 | mostly defaults component name to the name of the service that is implemented | ||
1830 | in a process ('transport', 'core', 'peerinfo', etc), but logging calls can | ||
1831 | specify custom component names using @code{GNUNET_log_from}.@ File name and | ||
1832 | function name are provided by the compiler (__FILE__ and __FUNCTION__ | ||
1833 | built-ins). | ||
1834 | |||
1835 | Component, file and function fields are interpreted as non-extended regular | ||
1836 | expressions (GNU libc regex functions are used). Matching is case-sensitive, ^ | ||
1837 | and $ will match the beginning and the end of the text. If a field is empty, | ||
1838 | its contents are automatically replaced with a ".*" regular expression, which | ||
1839 | matches anything. Matching is done in the default way, which means that the | ||
1840 | expression matches as long as it's contained anywhere in the string. Thus | ||
1841 | "GNUNET_" will match both "GNUNET_foo" and "BAR_GNUNET_BAZ". Use '^' and/or '$' | ||
1842 | to make sure that the expression matches at the start and/or at the end of the | ||
1843 | string.@ The semicolon (';') can't be escaped, and GNUnet will not use it in | ||
1844 | component names (it can't be used in function names and file names anyway).@ | ||
1845 | |||
1846 | @end table | ||
1847 | |||
1848 | |||
1849 | Every logging call in GNUnet code will be (at run time) matched against the | ||
1850 | log definitions passed to the process. If a log definition fields are matching | ||
1851 | the call arguments, then the call log level is compared the the log level of | ||
1852 | that definition. If the call log level is less or equal to the definition log | ||
1853 | level, the call is allowed to proceed. Otherwise the logging call is | ||
1854 | forbidden, and nothing is logged. If no definitions matched at all, GNUnet | ||
1855 | will use the global log level or (if a global log level is not specified) will | ||
1856 | default to WARNING (that is, it will allow the call to proceed, if its level | ||
1857 | is less or equal to the global log level or to WARNING). | ||
1858 | |||
1859 | That is, definitions are evaluated from left to right, and the first matching | ||
1860 | definition is used to allow or deny the logging call. Thus it is advised to | ||
1861 | place narrow definitions at the beginning of the logdef string, and generic | ||
1862 | definitions - at the end. | ||
1863 | |||
1864 | Whether a call is allowed or not is only decided the first time this particular | ||
1865 | call is made. The evaluation result is then cached, so that any attempts to | ||
1866 | make the same call later will be allowed or disallowed right away. Because of | ||
1867 | that runtime log level evaluation should not significantly affect the process | ||
1868 | performance.@ Log definition parsing is only done once, at the first call to | ||
1869 | GNUNET_log_setup () made by the process (which is usually done soon after it | ||
1870 | starts). | ||
1871 | |||
1872 | At the moment of writing there is no way to specify logging definitions from | ||
1873 | configuration files, only via environment variables. | ||
1874 | |||
1875 | At the moment GNUnet will stop processing a log definition when it encounters | ||
1876 | an error in definition formatting or an error in regular expression syntax, and | ||
1877 | will not report the failure in any way. | ||
1878 | |||
1879 | |||
1880 | @c *************************************************************************** | ||
1881 | @menu | ||
1882 | * Examples:: | ||
1883 | * Log files:: | ||
1884 | * Updated behavior of GNUNET_log:: | ||
1885 | @end menu | ||
1886 | |||
1887 | @node Examples | ||
1888 | @subsubsection Examples | ||
1889 | |||
1890 | @table @asis | ||
1891 | |||
1892 | @item @code{GNUNET_FORCE_LOG=";;;;DEBUG" gnunet-arm -s} Start GNUnet process | ||
1893 | tree, running all processes with DEBUG level (one should be careful with it, as | ||
1894 | log files will grow at alarming rate!) | ||
1895 | @item @code{GNUNET_FORCE_LOG="core;;;;DEBUG" gnunet-arm -s} Start GNUnet process | ||
1896 | tree, running the core service under DEBUG level (everything else will use | ||
1897 | configured or default level). | ||
1898 | @item @code{GNUNET_FORCE_LOG=";gnunet-service-transport_validation.c;;;DEBUG" gnunet-arm -s} | ||
1899 | Start GNUnet process tree, allowing any logging calls from | ||
1900 | gnunet-service-transport_validation.c (everything else will use configured or | ||
1901 | default level). | ||
1902 | @item @code{GNUNET_FORCE_LOG="fs;gnunet-service-fs_push.c;;;DEBUG" gnunet-arm -s} | ||
1903 | Start GNUnet process tree, allowing any logging calls from | ||
1904 | gnunet-gnunet-service-fs_push.c (everything else will use configured or default | ||
1905 | level). | ||
1906 | @item @code{GNUNET_FORCE_LOG=";;GNUNET_NETWORK_socket_select;;DEBUG" gnunet-arm -s} | ||
1907 | Start GNUnet process tree, allowing any logging calls from the | ||
1908 | GNUNET_NETWORK_socket_select function (everything else will use configured or | ||
1909 | default level). | ||
1910 | @item @code{GNUNET_FORCE_LOG="transport.*;;.*send.*;;DEBUG/;;;;WARNING" gnunet-arm -s} | ||
1911 | Start GNUnet process tree, allowing any logging calls from the components | ||
1912 | that have "transport" in their names, and are made from function that have | ||
1913 | "send" in their names. Everything else will be allowed to be logged only if it | ||
1914 | has WARNING level. | ||
1915 | @end table | ||
1916 | |||
1917 | |||
1918 | On Windows, one can use batch files to run GNUnet processes with special | ||
1919 | environment variables, without affecting the whole system. Such batch file will | ||
1920 | look like this:@ @code{@ set GNUNET_FORCE_LOG=;;do_transmit;;DEBUG@ gnunet-arm | ||
1921 | -s@ }@ (note the absence of double quotes in the environment variable | ||
1922 | definition, as opposed to earlier examples, which use the shell).@ Another | ||
1923 | limitation, on Windows, GNUNET_FORCE_LOGFILE @strong{MUST} be set in order to | ||
1924 | GNUNET_FORCE_LOG to work. | ||
1925 | |||
1926 | |||
1927 | @c *************************************************************************** | ||
1928 | @node Log files | ||
1929 | @subsubsection Log files | ||
1930 | |||
1931 | GNUnet can be told to log everything into a file instead of stderr (which is | ||
1932 | the default) using the "--log-file=logfile" or "-l logfile" option. This option | ||
1933 | can also be passed via command line, or from the "OPTION" and "GLOBAL_POSTFIX" | ||
1934 | configuration keys (see above). The file name passed with this option is | ||
1935 | subject to GNUnet filename expansion. If specified in "GLOBAL_POSTFIX", it is | ||
1936 | also subject to ARM service filename expansion, in particular, it may contain | ||
1937 | "@{@}" (left and right curly brace) sequence, which will be replaced by ARM | ||
1938 | with the name of the service. This is used to keep logs from more than one | ||
1939 | service separate, while only specifying one template containing "@{@}" in | ||
1940 | GLOBAL_POSTFIX. | ||
1941 | |||
1942 | As part of a secondary file name expansion, the first occurrence of "[]" | ||
1943 | sequence ("left square brace" followed by "right square brace") in the file | ||
1944 | name will be replaced with a process identifier or the process when it | ||
1945 | initializes its logging subsystem. As a result, all processes will log into | ||
1946 | different files. This is convenient for isolating messages of a particular | ||
1947 | process, and prevents I/O races when multiple processes try to write into the | ||
1948 | file at the same time. This expansion is done independently of "@{@}" | ||
1949 | expansion that ARM service does (see above). | ||
1950 | |||
1951 | The log file name that is specified via "-l" can contain format characters | ||
1952 | from the 'strftime' function family. For example, "%Y" will be replaced with | ||
1953 | the current year. Using "basename-%Y-%m-%d.log" would include the current | ||
1954 | year, month and day in the log file. If a GNUnet process runs for long enough | ||
1955 | to need more than one log file, it will eventually clean up old log files. | ||
1956 | Currently, only the last three log files (plus the current log file) are | ||
1957 | preserved. So once the fifth log file goes into use (so after 4 days if you | ||
1958 | use "%Y-%m-%d" as above), the first log file will be automatically deleted. | ||
1959 | Note that if your log file name only contains "%Y", then log files would be | ||
1960 | kept for 4 years and the logs from the first year would be deleted once year 5 | ||
1961 | begins. If you do not use any date-related string format codes, logs would | ||
1962 | never be automatically deleted by GNUnet. | ||
1963 | |||
1964 | |||
1965 | @c *************************************************************************** | ||
1966 | |||
1967 | @node Updated behavior of GNUNET_log | ||
1968 | @subsubsection Updated behavior of GNUNET_log | ||
1969 | |||
1970 | It's currently quite common to see constructions like this all over the code: | ||
1971 | @example | ||
1972 | #if MESH_DEBUG GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "MESH: client | ||
1973 | disconnected\n"); #endif | ||
1974 | @end example | ||
1975 | |||
1976 | The reason for the #if is not to avoid displaying the message when disabled | ||
1977 | (GNUNET_ERROR_TYPE takes care of that), but to avoid the compiler including it | ||
1978 | in the binary at all, when compiling GNUnet for platforms with restricted | ||
1979 | storage space / memory (MIPS routers, ARM plug computers / dev boards, etc). | ||
1980 | |||
1981 | This presents several problems: the code gets ugly, hard to write and it is | ||
1982 | very easy to forget to include the #if guards, creating non-consistent code. A | ||
1983 | new change in GNUNET_log aims to solve these problems. | ||
1984 | |||
1985 | @strong{This change requires to @code{./configure} with at least | ||
1986 | @code{--enable-logging=verbose} to see debug messages.} | ||
1987 | |||
1988 | Here is an example of code with dense debug statements: | ||
1989 | @example | ||
1990 | switch (restrict_topology) @{ | ||
1991 | case GNUNET_TESTING_TOPOLOGY_CLIQUE: #if VERBOSE_TESTING | ||
1992 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, _("Blacklisting all but clique | ||
1993 | topology\n")); #endif unblacklisted_connections = create_clique (pg, | ||
1994 | &remove_connections, BLACKLIST, GNUNET_NO); break; case | ||
1995 | GNUNET_TESTING_TOPOLOGY_SMALL_WORLD_RING: #if VERBOSE_TESTING GNUNET_log | ||
1996 | (GNUNET_ERROR_TYPE_DEBUG, _("Blacklisting all but small world (ring) | ||
1997 | topology\n")); #endif unblacklisted_connections = create_small_world_ring (pg, | ||
1998 | &remove_connections, BLACKLIST); break; | ||
1999 | @end example | ||
2000 | |||
2001 | |||
2002 | Pretty hard to follow, huh? | ||
2003 | |||
2004 | From now on, it is not necessary to include the #if / #endif statements to | ||
2005 | acheive the same behavior. The GNUNET_log and GNUNET_log_from macros take care | ||
2006 | of it for you, depending on the configure option: | ||
2007 | @itemize @bullet | ||
2008 | @item If @code{--enable-logging} is set to @code{no}, the binary will contain | ||
2009 | no log messages at all. | ||
2010 | @item If @code{--enable-logging} is set to @code{yes}, the binary will contain | ||
2011 | no DEBUG messages, and therefore running with -L DEBUG will have no effect. | ||
2012 | Other messages (ERROR, WARNING, INFO, etc) will be included. | ||
2013 | @item If @code{--enable-logging} is set to @code{verbose}, or | ||
2014 | @code{veryverbose} the binary will contain DEBUG messages (still, it will be | ||
2015 | neccessary to run with -L DEBUG or set the DEBUG config option to show them). | ||
2016 | @end itemize | ||
2017 | |||
2018 | |||
2019 | If you are a developer: | ||
2020 | @itemize @bullet | ||
2021 | @item please make sure that you @code{./configure | ||
2022 | --enable-logging=@{verbose,veryverbose@}}, so you can see DEBUG messages. | ||
2023 | @item please remove the @code{#if} statements around @code{GNUNET_log | ||
2024 | (GNUNET_ERROR_TYPE_DEBUG, ...)} lines, to improve the readibility of your code. | ||
2025 | @end itemize | ||
2026 | |||
2027 | Since now activating DEBUG automatically makes it VERBOSE and activates | ||
2028 | @strong{all} debug messages by default, you probably want to use the | ||
2029 | https://gnunet.org/logging functionality to filter only relevant messages. A | ||
2030 | suitable configuration could be:@ @code{$ export | ||
2031 | GNUNET_FORCE_LOG="^YOUR_SUBSYSTEM$;;;;DEBUG/;;;;WARNING"}@ Which will behave | ||
2032 | almost like enabling DEBUG in that subsytem before the change. Of course you | ||
2033 | can adapt it to your particular needs, this is only a quick example. | ||
2034 | |||
2035 | @c *************************************************************************** | ||
2036 | @node Interprocess communication API (IPC) | ||
2037 | @subsection Interprocess communication API (IPC) | ||
2038 | |||
2039 | In GNUnet a variety of new message types might be defined and used in | ||
2040 | interprocess communication, in this tutorial we use the @code{struct | ||
2041 | AddressLookupMessage} as a example to introduce how to construct our own | ||
2042 | message type in GNUnet and how to implement the message communication between | ||
2043 | service and client.@ (Here, a client uses the @code{struct | ||
2044 | AddressLookupMessage} as a request to ask the server to return the address of | ||
2045 | any other peer connecting to the service.) | ||
2046 | |||
2047 | |||
2048 | @c *************************************************************************** | ||
2049 | @menu | ||
2050 | * Define new message types:: | ||
2051 | * Define message struct:: | ||
2052 | * Client - Establish connection:: | ||
2053 | * Client - Initialize request message:: | ||
2054 | * Client - Send request and receive response:: | ||
2055 | * Server - Startup service:: | ||
2056 | * Server - Add new handles for specified messages:: | ||
2057 | * Server - Process request message:: | ||
2058 | * Server - Response to client:: | ||
2059 | * Server - Notification of clients:: | ||
2060 | * Conversion between Network Byte Order (Big Endian) and Host Byte Order:: | ||
2061 | @end menu | ||
2062 | |||
2063 | @node Define new message types | ||
2064 | @subsubsection Define new message types | ||
2065 | |||
2066 | First of all, you should define the new message type in | ||
2067 | @code{gnunet_protocols.h}: | ||
2068 | @example | ||
2069 | // Request to look addresses of peers in server. | ||
2070 | #define GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_LOOKUP 29 | ||
2071 | // Response to the address lookup request. | ||
2072 | #define GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_REPLY 30 | ||
2073 | @end example | ||
2074 | |||
2075 | @c *************************************************************************** | ||
2076 | @node Define message struct | ||
2077 | @subsubsection Define message struct | ||
2078 | |||
2079 | After the type definition, the specified message structure should also be | ||
2080 | described in the header file, e.g. transport.h in our case. | ||
2081 | @example | ||
2082 | GNUNET_NETWORK_STRUCT_BEGIN | ||
2083 | |||
2084 | struct AddressLookupMessage @{ struct GNUNET_MessageHeader header; int32_t | ||
2085 | numeric_only GNUNET_PACKED; struct GNUNET_TIME_AbsoluteNBO timeout; uint32_t | ||
2086 | addrlen GNUNET_PACKED; | ||
2087 | /* followed by 'addrlen' bytes of the actual address, then | ||
2088 | followed by the 0-terminated name of the transport */ @}; | ||
2089 | GNUNET_NETWORK_STRUCT_END | ||
2090 | @end example | ||
2091 | |||
2092 | |||
2093 | Please note @code{GNUNET_NETWORK_STRUCT_BEGIN} and @code{GNUNET_PACKED} which | ||
2094 | both ensure correct alignment when sending structs over the network | ||
2095 | |||
2096 | @menu | ||
2097 | @end menu | ||
2098 | |||
2099 | @c *************************************************************************** | ||
2100 | @node Client - Establish connection | ||
2101 | @subsubsection Client - Establish connection | ||
2102 | @c %**end of header | ||
2103 | |||
2104 | |||
2105 | At first, on the client side, the underlying API is employed to create a new | ||
2106 | connection to a service, in our example the transport service would be | ||
2107 | connected. | ||
2108 | @example | ||
2109 | struct GNUNET_CLIENT_Connection *client; client = | ||
2110 | GNUNET_CLIENT_connect ("transport", cfg); | ||
2111 | @end example | ||
2112 | |||
2113 | @c *************************************************************************** | ||
2114 | @node Client - Initialize request message | ||
2115 | @subsubsection Client - Initialize request message | ||
2116 | @c %**end of header | ||
2117 | |||
2118 | When the connection is ready, we initialize the message. In this step, all the | ||
2119 | fields of the message should be properly initialized, namely the size, type, | ||
2120 | and some extra user-defined data, such as timeout, name of transport, address | ||
2121 | and name of transport. | ||
2122 | @example | ||
2123 | struct AddressLookupMessage *msg; size_t len = | ||
2124 | sizeof (struct AddressLookupMessage) + addressLen + strlen (nameTrans) + 1; | ||
2125 | msg->header->size = htons (len); msg->header->type = htons | ||
2126 | (GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_LOOKUP); msg->timeout = | ||
2127 | GNUNET_TIME_absolute_hton (abs_timeout); msg->addrlen = htonl (addressLen); | ||
2128 | char *addrbuf = (char *) &msg[1]; memcpy (addrbuf, address, addressLen); char | ||
2129 | *tbuf = &addrbuf[addressLen]; memcpy (tbuf, nameTrans, strlen (nameTrans) + 1); | ||
2130 | @end example | ||
2131 | |||
2132 | Note that, here the functions @code{htonl}, @code{htons} and | ||
2133 | @code{GNUNET_TIME_absolute_hton} are applied to convert little endian into big | ||
2134 | endian, about the usage of the big/small edian order and the corresponding | ||
2135 | conversion function please refer to Introduction of Big Endian and Little | ||
2136 | Endian. | ||
2137 | |||
2138 | @c *************************************************************************** | ||
2139 | @node Client - Send request and receive response | ||
2140 | @subsubsection Client - Send request and receive response | ||
2141 | @c %**end of header | ||
2142 | |||
2143 | FIXME: This is very outdated, see the tutorial for the | ||
2144 | current API! | ||
2145 | |||
2146 | Next, the client would send the constructed message as a request to the service | ||
2147 | and wait for the response from the service. To accomplish this goal, there are | ||
2148 | a number of API calls that can be used. In this example, | ||
2149 | @code{GNUNET_CLIENT_transmit_and_get_response} is chosen as the most | ||
2150 | appropriate function to use. | ||
2151 | @example | ||
2152 | GNUNET_CLIENT_transmit_and_get_response | ||
2153 | (client, msg->header, timeout, GNUNET_YES, &address_response_processor, | ||
2154 | arp_ctx); | ||
2155 | @end example | ||
2156 | |||
2157 | the argument @code{address_response_processor} is a function with | ||
2158 | @code{GNUNET_CLIENT_MessageHandler} type, which is used to process the reply | ||
2159 | message from the service. | ||
2160 | |||
2161 | @node Server - Startup service | ||
2162 | @subsubsection Server - Startup service | ||
2163 | |||
2164 | After receiving the request message, we run a standard GNUnet service startup | ||
2165 | sequence using @code{GNUNET_SERVICE_run}, as follows, | ||
2166 | @example | ||
2167 | int main(int | ||
2168 | argc, char**argv) @{ GNUNET_SERVICE_run(argc, argv, "transport" | ||
2169 | GNUNET_SERVICE_OPTION_NONE, &run, NULL)); @} | ||
2170 | @end example | ||
2171 | |||
2172 | @c *************************************************************************** | ||
2173 | @node Server - Add new handles for specified messages | ||
2174 | @subsubsection Server - Add new handles for specified messages | ||
2175 | @c %**end of header | ||
2176 | |||
2177 | in the function above the argument @code{run} is used to initiate transport | ||
2178 | service,and defined like this: | ||
2179 | @example | ||
2180 | static void run (void *cls, struct | ||
2181 | GNUNET_SERVER_Handle *serv, const struct GNUNET_CONFIGURATION_Handle *cfg) @{ | ||
2182 | GNUNET_SERVER_add_handlers (serv, handlers); @} | ||
2183 | @end example | ||
2184 | |||
2185 | |||
2186 | Here, @code{GNUNET_SERVER_add_handlers} must be called in the run function to | ||
2187 | add new handlers in the service. The parameter @code{handlers} is a list of | ||
2188 | @code{struct GNUNET_SERVER_MessageHandler} to tell the service which function | ||
2189 | should be called when a particular type of message is received, and should be | ||
2190 | defined in this way: | ||
2191 | @example | ||
2192 | static struct GNUNET_SERVER_MessageHandler | ||
2193 | handlers[] = @{ @{&handle_start, NULL, GNUNET_MESSAGE_TYPE_TRANSPORT_START, | ||
2194 | 0@}, @{&handle_send, NULL, GNUNET_MESSAGE_TYPE_TRANSPORT_SEND, 0@}, | ||
2195 | @{&handle_try_connect, NULL, GNUNET_MESSAGE_TYPE_TRANSPORT_TRY_CONNECT, sizeof | ||
2196 | (struct TryConnectMessage)@}, @{&handle_address_lookup, NULL, | ||
2197 | GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_LOOKUP, 0@}, @{NULL, NULL, 0, 0@} @}; | ||
2198 | @end example | ||
2199 | |||
2200 | |||
2201 | As shown, the first member of the struct in the first area is a callback | ||
2202 | function, which is called to process the specified message types, given as the | ||
2203 | third member. The second parameter is the closure for the callback function, | ||
2204 | which is set to @code{NULL} in most cases, and the last parameter is the | ||
2205 | expected size of the message of this type, usually we set it to 0 to accept | ||
2206 | variable size, for special cases the exact size of the specified message also | ||
2207 | can be set. In addition, the terminator sign depicted as @code{@{NULL, NULL, 0, | ||
2208 | 0@}} is set in the last aera. | ||
2209 | |||
2210 | @c *************************************************************************** | ||
2211 | @node Server - Process request message | ||
2212 | @subsubsection Server - Process request message | ||
2213 | @c %**end of header | ||
2214 | |||
2215 | After the initialization of transport service, the request message would be | ||
2216 | processed. Before handling the main message data, the validity of this message | ||
2217 | should be checked out, e.g., to check whether the size of message is correct. | ||
2218 | @example | ||
2219 | size = ntohs (message->size); if (size < sizeof (struct | ||
2220 | AddressLookupMessage)) @{ GNUNET_break_op (0); GNUNET_SERVER_receive_done | ||
2221 | (client, GNUNET_SYSERR); return; @} | ||
2222 | @end example | ||
2223 | |||
2224 | |||
2225 | Note that, opposite to the construction method of the request message in the | ||
2226 | client, in the server the function @code{nothl} and @code{ntohs} should be | ||
2227 | employed during the extraction of the data from the message, so that the data | ||
2228 | in big endian order can be converted back into little endian order. See more in | ||
2229 | detail please refer to Introduction of Big Endian and Little Endian. | ||
2230 | |||
2231 | Moreover in this example, the name of the transport stored in the message is a | ||
2232 | 0-terminated string, so we should also check whether the name of the transport | ||
2233 | in the received message is 0-terminated: | ||
2234 | @example | ||
2235 | nameTransport = (const char *) | ||
2236 | &address[addressLen]; if (nameTransport[size - sizeof (struct | ||
2237 | AddressLookupMessage) | ||
2238 | - addressLen - 1] != '\0') @{ GNUNET_break_op | ||
2239 | (0); GNUNET_SERVER_receive_done (client, | ||
2240 | GNUNET_SYSERR); return; @} | ||
2241 | @end example | ||
2242 | |||
2243 | Here, @code{GNUNET_SERVER_receive_done} should be called to tell the service | ||
2244 | that the request is done and can receive the next message. The argument | ||
2245 | @code{GNUNET_SYSERR} here indicates that the service didn't understand the | ||
2246 | request message, and the processing of this request would be terminated. | ||
2247 | |||
2248 | In comparison to the aforementioned situation, when the argument is equal to | ||
2249 | @code{GNUNET_OK}, the service would continue to process the requst message. | ||
2250 | |||
2251 | @c *************************************************************************** | ||
2252 | @node Server - Response to client | ||
2253 | @subsubsection Server - Response to client | ||
2254 | @c %**end of header | ||
2255 | |||
2256 | Once the processing of current request is done, the server should give the | ||
2257 | response to the client. A new @code{struct AddressLookupMessage} would be | ||
2258 | produced by the server in a similar way as the client did and sent to the | ||
2259 | client, but here the type should be | ||
2260 | @code{GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_REPLY} rather than | ||
2261 | @code{GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_LOOKUP} in client. | ||
2262 | @example | ||
2263 | struct | ||
2264 | AddressLookupMessage *msg; size_t len = sizeof (struct AddressLookupMessage) + | ||
2265 | addressLen + strlen (nameTrans) + 1; msg->header->size = htons (len); | ||
2266 | msg->header->type = htons (GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_REPLY); | ||
2267 | |||
2268 | // ... | ||
2269 | |||
2270 | struct GNUNET_SERVER_TransmitContext *tc; tc = | ||
2271 | GNUNET_SERVER_transmit_context_create (client); | ||
2272 | GNUNET_SERVER_transmit_context_append_data (tc, NULL, 0, | ||
2273 | GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_REPLY); | ||
2274 | GNUNET_SERVER_transmit_context_run (tc, rtimeout); | ||
2275 | @end example | ||
2276 | |||
2277 | |||
2278 | Note that, there are also a number of other APIs provided to the service to | ||
2279 | send the message. | ||
2280 | |||
2281 | @c *************************************************************************** | ||
2282 | @node Server - Notification of clients | ||
2283 | @subsubsection Server - Notification of clients | ||
2284 | @c %**end of header | ||
2285 | |||
2286 | Often a service needs to (repeatedly) transmit notifications to a client or a | ||
2287 | group of clients. In these cases, the client typically has once registered for | ||
2288 | a set of events and then needs to receive a message whenever such an event | ||
2289 | happens (until the client disconnects). The use of a notification context can | ||
2290 | help manage message queues to clients and handle disconnects. Notification | ||
2291 | contexts can be used to send individualized messages to a particular client or | ||
2292 | to broadcast messages to a group of clients. An individualized notification | ||
2293 | might look like this: | ||
2294 | @example | ||
2295 | GNUNET_SERVER_notification_context_unicast(nc, | ||
2296 | client, msg, GNUNET_YES); | ||
2297 | @end example | ||
2298 | |||
2299 | |||
2300 | Note that after processing the original registration message for notifications, | ||
2301 | the server code still typically needs to call@ | ||
2302 | @code{GNUNET_SERVER_receive_done} so that the client can transmit further | ||
2303 | messages to the server. | ||
2304 | |||
2305 | @c *************************************************************************** | ||
2306 | @node Conversion between Network Byte Order (Big Endian) and Host Byte Order | ||
2307 | @subsubsection Conversion between Network Byte Order (Big Endian) and Host Byte Order | ||
2308 | @c %** subsub? it's a referenced page on the ipc document. | ||
2309 | @c %**end of header | ||
2310 | |||
2311 | Here we can simply comprehend big endian and little endian as Network Byte | ||
2312 | Order and Host Byte Order respectively. What is the difference between both | ||
2313 | two? | ||
2314 | |||
2315 | Usually in our host computer we store the data byte as Host Byte Order, for | ||
2316 | example, we store a integer in the RAM which might occupies 4 Byte, as Host | ||
2317 | Byte Order the higher Byte would be stored at the lower address of RAM, and | ||
2318 | the lower Byte would be stored at the higher address of RAM. However, contrast | ||
2319 | to this, Network Byte Order just take the totally opposite way to store the | ||
2320 | data, says, it will store the lower Byte at the lower address, and the higher | ||
2321 | Byte will stay at higher address. | ||
2322 | |||
2323 | For the current communication of network, we normally exchange the information | ||
2324 | by surveying the data package, every two host wants to communicate with each | ||
2325 | other must send and receive data package through network. In order to maintain | ||
2326 | the identity of data through the transmission in the network, the order of the | ||
2327 | Byte storage must changed before sending and after receiving the data. | ||
2328 | |||
2329 | There ten convenient functions to realize the conversion of Byte Order in | ||
2330 | GNUnet, as following: | ||
2331 | @table @asis | ||
2332 | |||
2333 | @item uint16_t htons(uint16_t hostshort) Convert host byte order to net byte | ||
2334 | order with short int | ||
2335 | @item uint32_t htonl(uint32_t hostlong) Convert host byte | ||
2336 | order to net byte order with long int | ||
2337 | @item uint16_t ntohs(uint16_t netshort) | ||
2338 | Convert net byte order to host byte order with short int | ||
2339 | @item uint32_t | ||
2340 | ntohl(uint32_t netlong) Convert net byte order to host byte order with long int | ||
2341 | @item unsigned long long GNUNET_ntohll (unsigned long long netlonglong) Convert | ||
2342 | net byte order to host byte order with long long int | ||
2343 | @item unsigned long long | ||
2344 | GNUNET_htonll (unsigned long long hostlonglong) Convert host byte order to net | ||
2345 | byte order with long long int | ||
2346 | @item struct GNUNET_TIME_RelativeNBO | ||
2347 | GNUNET_TIME_relative_hton (struct GNUNET_TIME_Relative a) Convert relative time | ||
2348 | to network byte order. | ||
2349 | @item struct GNUNET_TIME_Relative | ||
2350 | GNUNET_TIME_relative_ntoh (struct GNUNET_TIME_RelativeNBO a) Convert relative | ||
2351 | time from network byte order. | ||
2352 | @item struct GNUNET_TIME_AbsoluteNBO | ||
2353 | GNUNET_TIME_absolute_hton (struct GNUNET_TIME_Absolute a) Convert relative time | ||
2354 | to network byte order. | ||
2355 | @item struct GNUNET_TIME_Absolute | ||
2356 | GNUNET_TIME_absolute_ntoh (struct GNUNET_TIME_AbsoluteNBO a) Convert relative | ||
2357 | time from network byte order. | ||
2358 | @end table | ||
2359 | |||
2360 | @c *************************************************************************** | ||
2361 | |||
2362 | @node Cryptography API | ||
2363 | @subsection Cryptography API | ||
2364 | @c %**end of header | ||
2365 | |||
2366 | The gnunetutil APIs provides the cryptographic primitives used in GNUnet. | ||
2367 | GNUnet uses 2048 bit RSA keys for the session key exchange and for signing | ||
2368 | messages by peers and most other public-key operations. Most researchers in | ||
2369 | cryptography consider 2048 bit RSA keys as secure and practically unbreakable | ||
2370 | for a long time. The API provides functions to create a fresh key pair, read a | ||
2371 | private key from a file (or create a new file if the file does not exist), | ||
2372 | encrypt, decrypt, sign, verify and extraction of the public key into a format | ||
2373 | suitable for network transmission. | ||
2374 | |||
2375 | For the encryption of files and the actual data exchanged between peers GNUnet | ||
2376 | uses 256-bit AES encryption. Fresh, session keys are negotiated for every new | ||
2377 | connection.@ Again, there is no published technique to break this cipher in any | ||
2378 | realistic amount of time. The API provides functions for generation of keys, | ||
2379 | validation of keys (important for checking that decryptions using RSA | ||
2380 | succeeded), encryption and decryption. | ||
2381 | |||
2382 | GNUnet uses SHA-512 for computing one-way hash codes. The API provides | ||
2383 | functions to compute a hash over a block in memory or over a file on disk. | ||
2384 | |||
2385 | The crypto API also provides functions for randomizing a block of memory, | ||
2386 | obtaining a single random number and for generating a permuation of the numbers | ||
2387 | 0 to n-1. Random number generation distinguishes between WEAK and STRONG random | ||
2388 | number quality; WEAK random numbers are pseudo-random whereas STRONG random | ||
2389 | numbers use entropy gathered from the operating system. | ||
2390 | |||
2391 | Finally, the crypto API provides a means to deterministically generate a | ||
2392 | 1024-bit RSA key from a hash code. These functions should most likely not be | ||
2393 | used by most applications; most importantly,@ | ||
2394 | GNUNET_CRYPTO_rsa_key_create_from_hash does not create an RSA-key that should | ||
2395 | be considered secure for traditional applications of RSA. | ||
2396 | |||
2397 | @c *************************************************************************** | ||
2398 | @node Message Queue API | ||
2399 | @subsection Message Queue API | ||
2400 | @c %**end of header | ||
2401 | |||
2402 | @strong{ Introduction }@ Often, applications need to queue messages that are to | ||
2403 | be sent to other GNUnet peers, clients or services. As all of GNUnet's | ||
2404 | message-based communication APIs, by design, do not allow messages to be | ||
2405 | queued, it is common to implement custom message queues manually when they are | ||
2406 | needed. However, writing very similar code in multiple places is tedious and | ||
2407 | leads to code duplication. | ||
2408 | |||
2409 | MQ (for Message Queue) is an API that provides the functionality to implement | ||
2410 | and use message queues. We intend to eventually replace all of the custom | ||
2411 | message queue implementations in GNUnet with MQ. | ||
2412 | |||
2413 | @strong{ Basic Concepts }@ The two most important entities in MQ are queues and | ||
2414 | envelopes. | ||
2415 | |||
2416 | Every queue is backed by a specific implementation (e.g. for mesh, stream, | ||
2417 | connection, server client, etc.) that will actually deliver the queued | ||
2418 | messages. For convenience,@ some queues also allow to specify a list of message | ||
2419 | handlers. The message queue will then also wait for incoming messages and | ||
2420 | dispatch them appropriately. | ||
2421 | |||
2422 | An envelope holds the the memory for a message, as well as metadata (Where is | ||
2423 | the envelope queued? What should happen after it has been sent?). Any envelope | ||
2424 | can only be queued in one message queue. | ||
2425 | |||
2426 | @strong{ Creating Queues }@ The following is a list of currently available | ||
2427 | message queues. Note that to avoid layering issues, message queues for higher | ||
2428 | level APIs are not part of @code{libgnunetutil}, but@ the respective API itself | ||
2429 | provides the queue implementation. | ||
2430 | @table @asis | ||
2431 | |||
2432 | @item @code{GNUNET_MQ_queue_for_connection_client} Transmits queued messages | ||
2433 | over a @code{GNUNET_CLIENT_Connection}@ handle. Also supports receiving with | ||
2434 | message handlers.@ | ||
2435 | |||
2436 | @item @code{GNUNET_MQ_queue_for_server_client} Transmits queued messages over a | ||
2437 | @code{GNUNET_SERVER_Client}@ handle. Does not support incoming message | ||
2438 | handlers.@ | ||
2439 | |||
2440 | @item @code{GNUNET_MESH_mq_create} Transmits queued messages over a | ||
2441 | @code{GNUNET_MESH_Tunnel}@ handle. Does not support incoming message handlers.@ | ||
2442 | |||
2443 | @item @code{GNUNET_MQ_queue_for_callbacks} This is the most general | ||
2444 | implementation. Instead of delivering and receiving messages with one of | ||
2445 | GNUnet's communication APIs, implementation callbacks are called. Refer to | ||
2446 | "Implementing Queues" for a more detailed explanation. | ||
2447 | @end table | ||
2448 | |||
2449 | |||
2450 | @strong{ Allocating Envelopes }@ A GNUnet message (as defined by the | ||
2451 | GNUNET_MessageHeader) has three parts: The size, the type, and the body. | ||
2452 | |||
2453 | MQ provides macros to allocate an envelope containing a message conveniently,@ | ||
2454 | automatically setting the size and type fields of the message. | ||
2455 | |||
2456 | Consider the following simple message, with the body consisting of a single | ||
2457 | number value.@ @code{} | ||
2458 | @example | ||
2459 | struct NumberMessage @{ | ||
2460 | /** Type: GNUNET_MESSAGE_TYPE_EXAMPLE_1 */ | ||
2461 | struct GNUNET_MessageHeader header; uint32_t number GNUNET_PACKED; @}; | ||
2462 | @end example | ||
2463 | |||
2464 | An envelope containing an instance of the NumberMessage can be constructed like | ||
2465 | this: | ||
2466 | @example | ||
2467 | struct GNUNET_MQ_Envelope *ev; struct NumberMessage *msg; ev = | ||
2468 | GNUNET_MQ_msg (msg, GNUNET_MESSAGE_TYPE_EXAMPLE_1); msg->number = htonl (42); | ||
2469 | @end example | ||
2470 | |||
2471 | |||
2472 | In the above code, @code{GNUNET_MQ_msg} is a macro. The return value is the | ||
2473 | newly allocated envelope. The first argument must be a pointer to some | ||
2474 | @code{struct} containing a @code{struct GNUNET_MessageHeader header} field, | ||
2475 | while the second argument is the desired message type, in host byte order. | ||
2476 | |||
2477 | The @code{msg} pointer now points to an allocated message, where the message | ||
2478 | type and the message size are already set. The message's size is inferred from | ||
2479 | the type of the @code{msg} pointer: It will be set to 'sizeof(*msg)', properly | ||
2480 | converted to network byte order. | ||
2481 | |||
2482 | If the message body's size is dynamic, the the macro @code{GNUNET_MQ_msg_extra} | ||
2483 | can be used to allocate an envelope whose message has additional space | ||
2484 | allocated after the @code{msg} structure. | ||
2485 | |||
2486 | If no structure has been defined for the message, | ||
2487 | @code{GNUNET_MQ_msg_header_extra} can be used to allocate additional space | ||
2488 | after the message header. The first argument then must be a pointer to a | ||
2489 | @code{GNUNET_MessageHeader}. | ||
2490 | |||
2491 | @strong{Envelope Properties}@ A few functions in MQ allow to set additional | ||
2492 | properties on envelopes: | ||
2493 | @table @asis | ||
2494 | |||
2495 | @item @code{GNUNET_MQ_notify_sent} Allows to specify a function that will be | ||
2496 | called once the envelope's message@ has been sent irrevocably. An envelope can | ||
2497 | be canceled precisely up to the@ point where the notify sent callback has been | ||
2498 | called. | ||
2499 | @item @code{GNUNET_MQ_disable_corking} No corking will be used when | ||
2500 | sending the message. Not every@ queue supports this flag, per default, | ||
2501 | envelopes are sent with corking.@ | ||
2502 | |||
2503 | @end table | ||
2504 | |||
2505 | |||
2506 | @strong{Sending Envelopes}@ Once an envelope has been constructed, it can be | ||
2507 | queued for sending with @code{GNUNET_MQ_send}. | ||
2508 | |||
2509 | Note that in order to avoid memory leaks, an envelope must either be sent (the | ||
2510 | queue will free it) or destroyed explicitly with @code{GNUNET_MQ_discard}. | ||
2511 | |||
2512 | @strong{Canceling Envelopes}@ An envelope queued with @code{GNUNET_MQ_send} can | ||
2513 | be canceled with @code{GNUNET_MQ_cancel}. Note that after the notify sent | ||
2514 | callback has been called, canceling a message results in undefined behavior. | ||
2515 | Thus it is unsafe to cancel an envelope that does not have a notify sent | ||
2516 | callback. When canceling an envelope, it is not necessary@ to call | ||
2517 | @code{GNUNET_MQ_discard}, and the envelope can't be sent again. | ||
2518 | |||
2519 | @strong{ Implementing Queues }@ @code{TODO} | ||
2520 | |||
2521 | @c *************************************************************************** | ||
2522 | @node Service API | ||
2523 | @subsection Service API | ||
2524 | @c %**end of header | ||
2525 | |||
2526 | Most GNUnet code lives in the form of services. Services are processes that | ||
2527 | offer an API for other components of the system to build on. Those other | ||
2528 | components can be command-line tools for users, graphical user interfaces or | ||
2529 | other services. Services provide their API using an IPC protocol. For this, | ||
2530 | each service must listen on either a TCP port or a UNIX domain socket; for | ||
2531 | this, the service implementation uses the server API. This use of server is | ||
2532 | exposed directly to the users of the service API. Thus, when using the service | ||
2533 | API, one is usually also often using large parts of the server API. The service | ||
2534 | API provides various convenience functions, such as parsing command-line | ||
2535 | arguments and the configuration file, which are not found in the server API. | ||
2536 | The dual to the service/server API is the client API, which can be used to | ||
2537 | access services. | ||
2538 | |||
2539 | The most common way to start a service is to use the GNUNET_SERVICE_run | ||
2540 | function from the program's main function. GNUNET_SERVICE_run will then parse | ||
2541 | the command line and configuration files and, based on the options found there, | ||
2542 | start the server. It will then give back control to the main program, passing | ||
2543 | the server and the configuration to the GNUNET_SERVICE_Main callback. | ||
2544 | GNUNET_SERVICE_run will also take care of starting the scheduler loop. If this | ||
2545 | is inappropriate (for example, because the scheduler loop is already running), | ||
2546 | GNUNET_SERVICE_start and related functions provide an alternative to | ||
2547 | GNUNET_SERVICE_run. | ||
2548 | |||
2549 | When starting a service, the service_name option is used to determine which | ||
2550 | sections in the configuration file should be used to configure the service. A | ||
2551 | typical value here is the name of the src/ sub-directory, for example | ||
2552 | "statistics". The same string would also be given to GNUNET_CLIENT_connect to | ||
2553 | access the service. | ||
2554 | |||
2555 | Once a service has been initialized, the program should use the@ | ||
2556 | GNUNET_SERVICE_Main callback to register message handlers using | ||
2557 | GNUNET_SERVER_add_handlers. The service will already have registered a handler | ||
2558 | for the "TEST" message. | ||
2559 | |||
2560 | The option bitfield (enum GNUNET_SERVICE_Options) determines how a service | ||
2561 | should behave during shutdown. There are three key strategies: | ||
2562 | @table @asis | ||
2563 | |||
2564 | @item instant (GNUNET_SERVICE_OPTION_NONE) Upon receiving the shutdown signal | ||
2565 | from the scheduler, the service immediately terminates the server, closing all | ||
2566 | existing connections with clients. | ||
2567 | @item manual | ||
2568 | (GNUNET_SERVICE_OPTION_MANUAL_SHUTDOWN) The service does nothing by itself | ||
2569 | during shutdown. The main program will need to take the appropriate action by | ||
2570 | calling GNUNET_SERVER_destroy or GNUNET_SERVICE_stop (depending on how the | ||
2571 | service was initialized) to terminate the service. This method is used by | ||
2572 | gnunet-service-arm and rather uncommon. | ||
2573 | @item soft | ||
2574 | (GNUNET_SERVICE_OPTION_SOFT_SHUTDOWN) Upon receiving the shutdown signal from | ||
2575 | the scheduler, the service immediately tells the server to stop listening for | ||
2576 | incoming clients. Requests from normal existing clients are still processed and | ||
2577 | the server/service terminates once all normal clients have disconnected. | ||
2578 | Clients that are not expected to ever disconnect (such as clients that monitor | ||
2579 | performance values) can be marked as 'monitor' clients using | ||
2580 | GNUNET_SERVER_client_mark_monitor. Those clients will continue to be processed | ||
2581 | until all 'normal' clients have disconnected. Then, the server will terminate, | ||
2582 | closing the monitor connections. This mode is for example used by 'statistics', | ||
2583 | allowing existing 'normal' clients to set (possibly persistent) statistic | ||
2584 | values before terminating. | ||
2585 | @end table | ||
2586 | |||
2587 | @c *************************************************************************** | ||
2588 | @node Optimizing Memory Consumption of GNUnet's (Multi-) Hash Maps | ||
2589 | @subsection Optimizing Memory Consumption of GNUnet's (Multi-) Hash Maps | ||
2590 | @c %**end of header | ||
2591 | |||
2592 | A commonly used data structure in GNUnet is a (multi-)hash map. It is most | ||
2593 | often used to map a peer identity to some data structure, but also to map | ||
2594 | arbitrary keys to values (for example to track requests in the distributed hash | ||
2595 | table or in file-sharing). As it is commonly used, the DHT is actually | ||
2596 | sometimes responsible for a large share of GNUnet's overall memory consumption | ||
2597 | (for some processes, 30% is not uncommon). The following text documents some | ||
2598 | API quirks (and their implications for applications) that were recently | ||
2599 | introduced to minimize the footprint of the hash map. | ||
2600 | |||
2601 | |||
2602 | @c *************************************************************************** | ||
2603 | @menu | ||
2604 | * Analysis:: | ||
2605 | * Solution:: | ||
2606 | * Migration:: | ||
2607 | * Conclusion:: | ||
2608 | * Availability:: | ||
2609 | @end menu | ||
2610 | |||
2611 | @node Analysis | ||
2612 | @subsubsection Analysis | ||
2613 | @c %**end of header | ||
2614 | |||
2615 | The main reason for the "excessive" memory consumption by the hash map is that | ||
2616 | GNUnet uses 512-bit cryptographic hash codes --- and the (multi-)hash map also | ||
2617 | uses the same 512-bit 'struct GNUNET_HashCode'. As a result, storing just the | ||
2618 | keys requires 64 bytes of memory for each key. As some applications like to | ||
2619 | keep a large number of entries in the hash map (after all, that's what maps | ||
2620 | are good for), 64 bytes per hash is significant: keeping a pointer to the | ||
2621 | value and having a linked list for collisions consume between 8 and 16 bytes, | ||
2622 | and 'malloc' may add about the same overhead per allocation, putting us in the | ||
2623 | 16 to 32 byte per entry ballpark. Adding a 64-byte key then triples the | ||
2624 | overall memory requirement for the hash map. | ||
2625 | |||
2626 | To make things "worse", most of the time storing the key in the hash map is | ||
2627 | not required: it is typically already in memory elsewhere! In most cases, the | ||
2628 | values stored in the hash map are some application-specific struct that _also_ | ||
2629 | contains the hash. Here is a simplified example: | ||
2630 | @example | ||
2631 | struct MyValue @{ | ||
2632 | struct GNUNET_HashCode key; unsigned int my_data; @}; | ||
2633 | |||
2634 | // ... | ||
2635 | val = GNUNET_malloc (sizeof (struct MyValue)); val->key = key; val->my_data = | ||
2636 | 42; GNUNET_CONTAINER_multihashmap_put (map, &key, val, ...); | ||
2637 | @end example | ||
2638 | |||
2639 | |||
2640 | This is a common pattern as later the entries might need to be removed, and at | ||
2641 | that time it is convenient to have the key immediately at hand: | ||
2642 | @example | ||
2643 | GNUNET_CONTAINER_multihashmap_remove (map, &val->key, val); | ||
2644 | @end example | ||
2645 | |||
2646 | |||
2647 | Note that here we end up with two times 64 bytes for the key, plus maybe 64 | ||
2648 | bytes total for the rest of the 'struct MyValue' and the map entry in the hash | ||
2649 | map. The resulting redundant storage of the key increases overall memory | ||
2650 | consumption per entry from the "optimal" 128 bytes to 192 bytes. This is not | ||
2651 | just an extreme example: overheads in practice are actually sometimes close to | ||
2652 | those highlighted in this example. This is especially true for maps with a | ||
2653 | significant number of entries, as there we tend to really try to keep the | ||
2654 | entries small. | ||
2655 | @c *************************************************************************** | ||
2656 | @node Solution | ||
2657 | @subsubsection Solution | ||
2658 | @c %**end of header | ||
2659 | |||
2660 | The solution that has now been implemented is to @strong{optionally} allow the | ||
2661 | hash map to not make a (deep) copy of the hash but instead have a pointer to | ||
2662 | the hash/key in the entry. This reduces the memory consumption for the key | ||
2663 | from 64 bytes to 4 to 8 bytes. However, it can also only work if the key is | ||
2664 | actually stored in the entry (which is the case most of the time) and if the | ||
2665 | entry does not modify the key (which in all of the code I'm aware of has been | ||
2666 | always the case if there key is stored in the entry). Finally, when the client | ||
2667 | stores an entry in the hash map, it @strong{must} provide a pointer to the key | ||
2668 | within the entry, not just a pointer to a transient location of the key. If | ||
2669 | the client code does not meet these requirements, the result is a dangling | ||
2670 | pointer and undefined behavior of the (multi-)hash map API. | ||
2671 | @c *************************************************************************** | ||
2672 | @node Migration | ||
2673 | @subsubsection Migration | ||
2674 | @c %**end of header | ||
2675 | |||
2676 | To use the new feature, first check that the values contain the respective key | ||
2677 | (and never modify it). Then, all calls to | ||
2678 | @code{GNUNET_CONTAINER_multihashmap_put} on the respective map must be audited | ||
2679 | and most likely changed to pass a pointer into the value's struct. For the | ||
2680 | initial example, the new code would look like this: | ||
2681 | @example | ||
2682 | struct MyValue @{ | ||
2683 | struct GNUNET_HashCode key; unsigned int my_data; @}; | ||
2684 | |||
2685 | // ... | ||
2686 | val = GNUNET_malloc (sizeof (struct MyValue)); val->key = key; val->my_data = | ||
2687 | 42; GNUNET_CONTAINER_multihashmap_put (map, &val->key, val, ...); | ||
2688 | @end example | ||
2689 | |||
2690 | |||
2691 | Note that @code{&val} was changed to @code{&val->key} in the argument to the | ||
2692 | @code{put} call. This is critical as often @code{key} is on the stack or in | ||
2693 | some other transient data structure and thus having the hash map keep a pointer | ||
2694 | to @code{key} would not work. Only the key inside of @code{val} has the same | ||
2695 | lifetime as the entry in the map (this must of course be checked as well). | ||
2696 | Naturally, @code{val->key} must be intiialized before the @code{put} call. Once | ||
2697 | all @code{put} calls have been converted and double-checked, you can change the | ||
2698 | call to create the hash map from | ||
2699 | @example | ||
2700 | map = | ||
2701 | GNUNET_CONTAINER_multihashmap_create (SIZE, GNUNET_NO); | ||
2702 | @end example | ||
2703 | |||
2704 | to | ||
2705 | |||
2706 | @example | ||
2707 | map = GNUNET_CONTAINER_multihashmap_create (SIZE, GNUNET_YES); | ||
2708 | @end example | ||
2709 | |||
2710 | If everything was done correctly, you now use about 60 bytes less memory per | ||
2711 | entry in @code{map}. However, if now (or in the future) any call to @code{put} | ||
2712 | does not ensure that the given key is valid until the entry is removed from the | ||
2713 | map, undefined behavior is likely to be observed. | ||
2714 | @c *************************************************************************** | ||
2715 | @node Conclusion | ||
2716 | @subsubsection Conclusion | ||
2717 | @c %**end of header | ||
2718 | |||
2719 | The new optimization can is often applicable and can result in a reduction in | ||
2720 | memory consumption of up to 30% in practice. However, it makes the code less | ||
2721 | robust as additional invariants are imposed on the multi hash map client. Thus | ||
2722 | applications should refrain from enabling the new mode unless the resulting | ||
2723 | performance increase is deemed significant enough. In particular, it should | ||
2724 | generally not be used in new code (wait at least until benchmarks exist). | ||
2725 | @c *************************************************************************** | ||
2726 | @node Availability | ||
2727 | @subsubsection Availability | ||
2728 | @c %**end of header | ||
2729 | |||
2730 | The new multi hash map code was committed in SVN 24319 (will be in GNUnet | ||
2731 | 0.9.4). Various subsystems (transport, core, dht, file-sharing) were | ||
2732 | previously audited and modified to take advantage of the new capability. In | ||
2733 | particular, memory consumption of the file-sharing service is expected to drop | ||
2734 | by 20-30% due to this change. | ||
2735 | |||
2736 | @c *************************************************************************** | ||
2737 | @node The CONTAINER_MDLL API | ||
2738 | @subsection The CONTAINER_MDLL API | ||
2739 | @c %**end of header | ||
2740 | |||
2741 | This text documents the GNUNET_CONTAINER_MDLL API. The GNUNET_CONTAINER_MDLL | ||
2742 | API is similar to the GNUNET_CONTAINER_DLL API in that it provides operations | ||
2743 | for the construction and manipulation of doubly-linked lists. The key | ||
2744 | difference to the (simpler) DLL-API is that the MDLL-version allows a single | ||
2745 | element (instance of a "struct") to be in multiple linked lists at the same | ||
2746 | time. | ||
2747 | |||
2748 | Like the DLL API, the MDLL API stores (most of) the data structures for the | ||
2749 | doubly-linked list with the respective elements; only the 'head' and 'tail' | ||
2750 | pointers are stored "elsewhere" --- and the application needs to provide the | ||
2751 | locations of head and tail to each of the calls in the MDLL API. The key | ||
2752 | difference for the MDLL API is that the "next" and "previous" pointers in the | ||
2753 | struct can no longer be simply called "next" and "prev" --- after all, the | ||
2754 | element may be in multiple doubly-linked lists, so we cannot just have one | ||
2755 | "next" and one "prev" pointer! | ||
2756 | |||
2757 | The solution is to have multiple fields that must have a name of the format | ||
2758 | "next_XX" and "prev_XX" where "XX" is the name of one of the doubly-linked | ||
2759 | lists. Here is a simple example: | ||
2760 | @example | ||
2761 | struct MyMultiListElement @{ struct | ||
2762 | MyMultiListElement *next_ALIST; struct MyMultiListElement *prev_ALIST; struct | ||
2763 | MyMultiListElement *next_BLIST; struct MyMultiListElement *prev_BLIST; void | ||
2764 | *data; @}; | ||
2765 | @end example | ||
2766 | |||
2767 | |||
2768 | Note that by convention, we use all-uppercase letters for the list names. In | ||
2769 | addition, the program needs to have a location for the head and tail pointers | ||
2770 | for both lists, for example: | ||
2771 | @example | ||
2772 | static struct MyMultiListElement | ||
2773 | *head_ALIST; static struct MyMultiListElement *tail_ALIST; static struct | ||
2774 | MyMultiListElement *head_BLIST; static struct MyMultiListElement *tail_BLIST; | ||
2775 | @end example | ||
2776 | |||
2777 | |||
2778 | Using the MDLL-macros, we can now insert an element into the ALIST: | ||
2779 | @example | ||
2780 | GNUNET_CONTAINER_MDLL_insert (ALIST, head_ALIST, tail_ALIST, element); | ||
2781 | @end example | ||
2782 | |||
2783 | |||
2784 | Passing "ALIST" as the first argument to MDLL specifies which of the next/prev | ||
2785 | fields in the 'struct MyMultiListElement' should be used. The extra "ALIST" | ||
2786 | argument and the "_ALIST" in the names of the next/prev-members are the only | ||
2787 | differences between the MDDL and DLL-API. Like the DLL-API, the MDLL-API offers | ||
2788 | functions for inserting (at head, at tail, after a given element) and removing | ||
2789 | elements from the list. Iterating over the list should be done by directly | ||
2790 | accessing the "next_XX" and/or "prev_XX" members. | ||
2791 | |||
2792 | @c *************************************************************************** | ||
2793 | @node The Automatic Restart Manager (ARM) | ||
2794 | @section The Automatic Restart Manager (ARM) | ||
2795 | @c %**end of header | ||
2796 | |||
2797 | GNUnet's Automated Restart Manager (ARM) is the GNUnet service responsible for | ||
2798 | system initialization and service babysitting. ARM starts and halts services, | ||
2799 | detects configuration changes and restarts services impacted by the changes as | ||
2800 | needed. It's also responsible for restarting services in case of crashes and is | ||
2801 | planned to incorporate automatic debugging for diagnosing service crashes | ||
2802 | providing developers insights about crash reasons. The purpose of this document | ||
2803 | is to give GNUnet developer an idea about how ARM works and how to interact | ||
2804 | with it. | ||
2805 | |||
2806 | @menu | ||
2807 | * Basic functionality:: | ||
2808 | * Key configuration options:: | ||
2809 | * Availability2:: | ||
2810 | * Reliability:: | ||
2811 | @end menu | ||
2812 | |||
2813 | @c *************************************************************************** | ||
2814 | @node Basic functionality | ||
2815 | @subsection Basic functionality | ||
2816 | @c %**end of header | ||
2817 | |||
2818 | @itemize @bullet | ||
2819 | @item ARM source code can be found under "src/arm".@ Service processes are | ||
2820 | managed by the functions in "gnunet-service-arm.c" which is controlled with | ||
2821 | "gnunet-arm.c" (main function in that file is ARM's entry point). | ||
2822 | |||
2823 | @item The functions responsible for communicating with ARM , starting and | ||
2824 | stopping services -including ARM service itself- are provided by the ARM API | ||
2825 | "arm_api.c".@ Function: GNUNET_ARM_connect() returns to the caller an ARM | ||
2826 | handle after setting it to the caller's context (configuration and scheduler in | ||
2827 | use). This handle can be used afterwards by the caller to communicate with ARM. | ||
2828 | Functions GNUNET_ARM_start_service() and GNUNET_ARM_stop_service() are used for | ||
2829 | starting and stopping services respectively. | ||
2830 | |||
2831 | @item A typical example of using these basic ARM services can be found in file | ||
2832 | test_arm_api.c. The test case connects to ARM, starts it, then uses it to start | ||
2833 | a service "resolver", stops the "resolver" then stops "ARM". | ||
2834 | @end itemize | ||
2835 | |||
2836 | @c *************************************************************************** | ||
2837 | @node Key configuration options | ||
2838 | @subsection Key configuration options | ||
2839 | @c %**end of header | ||
2840 | |||
2841 | Configurations for ARM and services should be available in a .conf file (As an | ||
2842 | example, see test_arm_api_data.conf). When running ARM, the configuration file | ||
2843 | to use should be passed to the command:@ @code{@ $ gnunet-arm -s -c | ||
2844 | configuration_to_use.conf@ }@ If no configuration is passed, the default | ||
2845 | configuration file will be used (see GNUNET_PREFIX/share/gnunet/defaults.conf | ||
2846 | which is created from contrib/defaults.conf).@ Each of the services is having a | ||
2847 | section starting by the service name between square brackets, for example: | ||
2848 | "[arm]". The following options configure how ARM configures or interacts with | ||
2849 | the various services: | ||
2850 | |||
2851 | @table @asis | ||
2852 | |||
2853 | @item PORT Port number on which the service is listening for incoming TCP | ||
2854 | connections. ARM will start the services should it notice a request at this | ||
2855 | port. | ||
2856 | |||
2857 | @item HOSTNAME Specifies on which host the service is deployed. Note | ||
2858 | that ARM can only start services that are running on the local system (but will | ||
2859 | not check that the hostname matches the local machine name). This option is | ||
2860 | used by the @code{gnunet_client_lib.h} implementation to determine which system | ||
2861 | to connect to. The default is "localhost". | ||
2862 | |||
2863 | @item BINARY The name of the service binary file. | ||
2864 | |||
2865 | @item OPTIONS To be passed to the service. | ||
2866 | |||
2867 | @item PREFIX A command to pre-pend to the actual command, for example, running | ||
2868 | a service with "valgrind" or "gdb" | ||
2869 | |||
2870 | @item DEBUG Run in debug mode (much verbosity). | ||
2871 | |||
2872 | @item AUTOSTART ARM will listen to UNIX domain socket and/or TCP port of the | ||
2873 | service and start the service on-demand. | ||
2874 | |||
2875 | @item FORCESTART ARM will always | ||
2876 | start this service when the peer is started. | ||
2877 | |||
2878 | @item ACCEPT_FROM IPv4 addresses the service accepts connections from. | ||
2879 | |||
2880 | @item ACCEPT_FROM6 IPv6 addresses the service accepts connections from. | ||
2881 | |||
2882 | @end table | ||
2883 | |||
2884 | |||
2885 | Options that impact the operation of ARM overall are in the "[arm]" section. | ||
2886 | ARM is a normal service and has (except for AUTOSTART) all of the options that | ||
2887 | other services do. In addition, ARM has the following options: | ||
2888 | @table @asis | ||
2889 | |||
2890 | @item GLOBAL_PREFIX Command to be pre-pended to all services that are going to | ||
2891 | run.@ | ||
2892 | |||
2893 | @item GLOBAL_POSTFIX Global option that will be supplied to all the services | ||
2894 | that are going to run.@ | ||
2895 | |||
2896 | @end table | ||
2897 | |||
2898 | @c *************************************************************************** | ||
2899 | @node Availability2 | ||
2900 | @subsection Availability2 | ||
2901 | @c %**end of header | ||
2902 | |||
2903 | As mentioned before, one of the features provided by ARM is starting services | ||
2904 | on demand. Consider the example of one service "client" that wants to connect | ||
2905 | to another service a "server". The "client" will ask ARM to run the "server". | ||
2906 | ARM starts the "server". The "server" starts listening to incoming connections. | ||
2907 | The "client" will establish a connection with the "server". And then, they will | ||
2908 | start to communicate together.@ One problem with that scheme is that it's | ||
2909 | slow!@ The "client" service wants to communicate with the "server" service at | ||
2910 | once and is not willing wait for it to be started and listening to incoming | ||
2911 | connections before serving its request.@ One solution for that problem will be | ||
2912 | that ARM starts all services as default services. That solution will solve the | ||
2913 | problem, yet, it's not quite practical, for some services that are going to be | ||
2914 | started can never be used or are going to be used after a relatively long | ||
2915 | time.@ The approach followed by ARM to solve this problem is as follows: | ||
2916 | @itemize @bullet | ||
2917 | |||
2918 | |||
2919 | @item For each service having a PORT field in the configuration file and that | ||
2920 | is not one of the default services ( a service that accepts incoming | ||
2921 | connections from clients), ARM creates listening sockets for all addresses | ||
2922 | associated with that service. | ||
2923 | |||
2924 | @item The "client" will immediately establish a connection with the "server". | ||
2925 | |||
2926 | @item ARM --- pretending to be the "server" --- will listen on the respective | ||
2927 | port and notice the incoming connection from the "client" (but not accept it), | ||
2928 | instead | ||
2929 | |||
2930 | @item Once there is an incoming connection, ARM will start the "server", | ||
2931 | passing on the listen sockets (now, the service is started and can do its | ||
2932 | work). | ||
2933 | |||
2934 | @item Other client services now can directly connect directly to the "server". | ||
2935 | @end itemize | ||
2936 | |||
2937 | @c *************************************************************************** | ||
2938 | @node Reliability | ||
2939 | @subsection Reliability | ||
2940 | |||
2941 | One of the features provided by ARM, is the automatic restart of crashed | ||
2942 | services.@ ARM needs to know which of the running services died. Function | ||
2943 | "gnunet-service-arm.c/maint_child_death()" is responsible for that. The | ||
2944 | function is scheduled to run upon receiving a SIGCHLD signal. The function, | ||
2945 | then, iterates ARM's list of services running and monitors which service has | ||
2946 | died (crashed). For all crashing services, ARM restarts them.@ Now, considering | ||
2947 | the case of a service having a serious problem causing it to crash each time | ||
2948 | it's started by ARM. If ARM keeps blindly restarting such a service, we are | ||
2949 | going to have the pattern: start-crash-restart-crash-restart-crash and so | ||
2950 | forth!! Which is of course not practical.@ For that reason, ARM schedules the | ||
2951 | service to be restarted after waiting for some delay that grows exponentially | ||
2952 | with each crash/restart of that service.@ To clarify the idea, considering the | ||
2953 | following example: | ||
2954 | @itemize @bullet | ||
2955 | |||
2956 | |||
2957 | @item Service S crashed. | ||
2958 | |||
2959 | @item ARM receives the SIGCHLD and inspects its list of services to find the | ||
2960 | dead one(s). | ||
2961 | |||
2962 | @item ARM finds S dead and schedules it for restarting after "backoff" time | ||
2963 | which is initially set to 1ms. ARM will double the backoff time correspondent | ||
2964 | to S (now backoff(S) = 2ms) | ||
2965 | |||
2966 | @item Because there is a severe problem with S, it crashed again. | ||
2967 | |||
2968 | @item Again ARM receives the SIGCHLD and detects that it's S again that's | ||
2969 | crashed. ARM schedules it for restarting but after its new backoff time (which | ||
2970 | became 2ms), and doubles its backoff time (now backoff(S) = 4). | ||
2971 | |||
2972 | @item and so on, until backoff(S) reaches a certain threshold | ||
2973 | (EXPONENTIAL_BACKOFF_THRESHOLD is set to half an hour), after reaching it, | ||
2974 | backoff(S) will remain half an hour, hence ARM won't be busy for a lot of time | ||
2975 | trying to restart a problematic service. | ||
2976 | @end itemize | ||
2977 | |||
2978 | @c *************************************************************************** | ||
2979 | @node GNUnet's TRANSPORT Subsystem | ||
2980 | @section GNUnet's TRANSPORT Subsystem | ||
2981 | @c %**end of header | ||
2982 | |||
2983 | This chapter documents how the GNUnet transport subsystem works. The GNUnet | ||
2984 | transport subsystem consists of three main components: the transport API (the | ||
2985 | interface used by the rest of the system to access the transport service), the | ||
2986 | transport service itself (most of the interesting functions, such as choosing | ||
2987 | transports, happens here) and the transport plugins. A transport plugin is a | ||
2988 | concrete implementation for how two GNUnet peers communicate; many plugins | ||
2989 | exist, for example for communication via TCP, UDP, HTTP, HTTPS and others. | ||
2990 | Finally, the transport subsystem uses supporting code, especially the NAT/UPnP | ||
2991 | library to help with tasks such as NAT traversal. | ||
2992 | |||
2993 | Key tasks of the transport service include: | ||
2994 | @itemize @bullet | ||
2995 | |||
2996 | |||
2997 | @item Create our HELLO message, notify clients and neighbours if our HELLO | ||
2998 | changes (using NAT library as necessary) | ||
2999 | |||
3000 | @item Validate HELLOs from other peers (send PING), allow other peers to | ||
3001 | validate our HELLO's addresses (send PONG) | ||
3002 | |||
3003 | @item Upon request, establish connections to other peers (using address | ||
3004 | selection from ATS subsystem) and maintain them (again using PINGs and PONGs) | ||
3005 | as long as desired | ||
3006 | |||
3007 | @item Accept incoming connections, give ATS service the opportunity to switch | ||
3008 | communication channels | ||
3009 | |||
3010 | @item Notify clients about peers that have connected to us or that have been | ||
3011 | disconnected from us | ||
3012 | |||
3013 | @item If a (stateful) connection goes down unexpectedly (without explicit | ||
3014 | DISCONNECT), quickly attempt to recover (without notifying clients) but do | ||
3015 | notify clients quickly if reconnecting fails | ||
3016 | |||
3017 | @item Send (payload) messages arriving from clients to other peers via | ||
3018 | transport plugins and receive messages from other peers, forwarding those to | ||
3019 | clients | ||
3020 | |||
3021 | @item Enforce inbound traffic limits (using flow-control if it is applicable); | ||
3022 | outbound traffic limits are enforced by CORE, not by us (!) | ||
3023 | |||
3024 | @item Enforce restrictions on P2P connection as specified by the blacklist | ||
3025 | configuration and blacklisting clients | ||
3026 | @end itemize | ||
3027 | |||
3028 | |||
3029 | Note that the term "clients" in the list above really refers to the GNUnet-CORE | ||
3030 | service, as CORE is typically the only client of the transport service. | ||
3031 | |||
3032 | @menu | ||
3033 | * Address validation protocol:: | ||
3034 | @end menu | ||
3035 | |||
3036 | @node Address validation protocol | ||
3037 | @subsection Address validation protocol | ||
3038 | @c %**end of header | ||
3039 | |||
3040 | This section documents how the GNUnet transport service validates connections | ||
3041 | with other peers. It is a high-level description of the protocol necessary to | ||
3042 | understand the details of the implementation. It should be noted that when we | ||
3043 | talk about PING and PONG messages in this section, we refer to transport-level | ||
3044 | PING and PONG messages, which are different from core-level PING and PONG | ||
3045 | messages (both in implementation and function). | ||
3046 | |||
3047 | The goal of transport-level address validation is to minimize the chances of a | ||
3048 | successful man-in-the-middle attack against GNUnet peers on the transport | ||
3049 | level. Such an attack would not allow the adversary to decrypt the P2P | ||
3050 | transmissions, but a successful attacker could at least measure traffic volumes | ||
3051 | and latencies (raising the adversaries capablities by those of a global passive | ||
3052 | adversary in the worst case). The scenarios we are concerned about is an | ||
3053 | attacker, Mallory, giving a HELLO to Alice that claims to be for Bob, but | ||
3054 | contains Mallory's IP address instead of Bobs (for some transport). Mallory | ||
3055 | would then forward the traffic to Bob (by initiating a connection to Bob and | ||
3056 | claiming to be Alice). As a further complication, the scheme has to work even | ||
3057 | if say Alice is behind a NAT without traversal support and hence has no address | ||
3058 | of her own (and thus Alice must always initiate the connection to Bob). | ||
3059 | |||
3060 | An additional constraint is that HELLO messages do not contain a cryptographic | ||
3061 | signature since other peers must be able to edit (i.e. remove) addresses from | ||
3062 | the HELLO at any time (this was not true in GNUnet 0.8.x). A basic | ||
3063 | @strong{assumption} is that each peer knows the set of possible network | ||
3064 | addresses that it @strong{might} be reachable under (so for example, the | ||
3065 | external IP address of the NAT plus the LAN address(es) with the respective | ||
3066 | ports). | ||
3067 | |||
3068 | The solution is the following. If Alice wants to validate that a given address | ||
3069 | for Bob is valid (i.e. is actually established @strong{directly} with the | ||
3070 | intended target), it sends a PING message over that connection to Bob. Note | ||
3071 | that in this case, Alice initiated the connection so only she knows which | ||
3072 | address was used for sure (Alice maybe behind NAT, so whatever address Bob | ||
3073 | sees may not be an address Alice knows she has). Bob checks that the address | ||
3074 | given in the PING is actually one of his addresses (does not belong to | ||
3075 | Mallory), and if it is, sends back a PONG (with a signature that says that Bob | ||
3076 | owns/uses the address from the PING). Alice checks the signature and is happy | ||
3077 | if it is valid and the address in the PONG is the address she used. This is | ||
3078 | similar to the 0.8.x protocol where the HELLO contained a signature from Bob | ||
3079 | for each address used by Bob. Here, the purpose code for the signature is | ||
3080 | @code{GNUNET_SIGNATURE_PURPOSE_TRANSPORT_PONG_OWN}. After this, Alice will | ||
3081 | remember Bob's address and consider the address valid for a while (12h in the | ||
3082 | current implementation). Note that after this exchange, Alice only considers | ||
3083 | Bob's address to be valid, the connection itself is not considered | ||
3084 | 'established'. In particular, Alice may have many addresses for Bob that she | ||
3085 | considers valid. | ||
3086 | |||
3087 | The PONG message is protected with a nonce/challenge against replay attacks | ||
3088 | and uses an expiration time for the signature (but those are almost | ||
3089 | implementation details). | ||
3090 | |||
3091 | @node NAT library | ||
3092 | @section NAT library | ||
3093 | @c %**end of header | ||
3094 | |||
3095 | The goal of the GNUnet NAT library is to provide a general-purpose API for NAT | ||
3096 | traversal @strong{without} third-party support. So protocols that involve | ||
3097 | contacting a third peer to help establish a connection between two peers are | ||
3098 | outside of the scope of this API. That does not mean that GNUnet doesn't | ||
3099 | support involving a third peer (we can do this with the distance-vector | ||
3100 | transport or using application-level protocols), it just means that the NAT API | ||
3101 | is not concerned with this possibility. The API is written so that it will work | ||
3102 | for IPv6-NAT in the future as well as current IPv4-NAT. Furthermore, the NAT | ||
3103 | API is always used, even for peers that are not behind NAT --- in that case, | ||
3104 | the mapping provided is simply the identity. | ||
3105 | |||
3106 | NAT traversal is initiated by calling @code{GNUNET_NAT_register}. Given a set | ||
3107 | of addresses that the peer has locally bound to (TCP or UDP), the NAT library | ||
3108 | will return (via callback) a (possibly longer) list of addresses the peer | ||
3109 | @strong{might} be reachable under. Internally, depending on the configuration, | ||
3110 | the NAT library will try to punch a hole (using UPnP) or just "know" that the | ||
3111 | NAT was manually punched and generate the respective external IP address (the | ||
3112 | one that should be globally visible) based on the given information. | ||
3113 | |||
3114 | The NAT library also supports ICMP-based NAT traversal. Here, the other peer | ||
3115 | can request connection-reversal by this peer (in this special case, the peer is | ||
3116 | even allowed to configure a port number of zero). If the NAT library detects a | ||
3117 | connection-reversal request, it returns the respective target address to the | ||
3118 | client as well. It should be noted that connection-reversal is currently only | ||
3119 | intended for TCP, so other plugins @strong{must} pass @code{NULL} for the | ||
3120 | reversal callback. Naturally, the NAT library also supports requesting | ||
3121 | connection reversal from a remote peer (@code{GNUNET_NAT_run_client}). | ||
3122 | |||
3123 | Once initialized, the NAT handle can be used to test if a given address is | ||
3124 | possibly a valid address for this peer (@code{GNUNET_NAT_test_address}). This | ||
3125 | is used for validating our addresses when generating PONGs. | ||
3126 | |||
3127 | Finally, the NAT library contains an API to test if our NAT configuration is | ||
3128 | correct. Using @code{GNUNET_NAT_test_start} @strong{before} binding to the | ||
3129 | respective port, the NAT library can be used to test if the configuration | ||
3130 | works. The test function act as a local client, initialize the NAT traversal | ||
3131 | and then contact a @code{gnunet-nat-server} (running by default on | ||
3132 | @code{gnunet.org}) and ask for a connection to be established. This way, it is | ||
3133 | easy to test if the current NAT configuration is valid. | ||
3134 | |||
3135 | @node Distance-Vector plugin | ||
3136 | @section Distance-Vector plugin | ||
3137 | @c %**end of header | ||
3138 | |||
3139 | The Distance Vector (DV) transport is a transport mechanism that allows peers | ||
3140 | to act as relays for each other, thereby connecting peers that would otherwise | ||
3141 | be unable to connect. This gives a larger connection set to applications that | ||
3142 | may work better with more peers to choose from (for example, File Sharing | ||
3143 | and/or DHT). | ||
3144 | |||
3145 | The Distance Vector transport essentially has two functions. The first is | ||
3146 | "gossiping" connection information about more distant peers to directly | ||
3147 | connected peers. The second is taking messages intended for non-directly | ||
3148 | connected peers and encapsulating them in a DV wrapper that contains the | ||
3149 | required information for routing the message through forwarding peers. Via | ||
3150 | gossiping, optimal routes through the known DV neighborhood are discovered and | ||
3151 | utilized and the message encapsulation provides some benefits in addition to | ||
3152 | simply getting the message from the correct source to the proper destination. | ||
3153 | |||
3154 | The gossiping function of DV provides an up to date routing table of peers that | ||
3155 | are available up to some number of hops. We call this a fisheye view of the | ||
3156 | network (like a fish, nearby objects are known while more distant ones | ||
3157 | unknown). Gossip messages are sent only to directly connected peers, but they | ||
3158 | are sent about other knowns peers within the "fisheye distance". Whenever two | ||
3159 | peers connect, they immediately gossip to each other about their appropriate | ||
3160 | other neighbors. They also gossip about the newly connected peer to previously | ||
3161 | connected neighbors. In order to keep the routing tables up to date, disconnect | ||
3162 | notifications are propogated as gossip as well (because disconnects may not be | ||
3163 | sent/received, timeouts are also used remove stagnant routing table entries). | ||
3164 | |||
3165 | Routing of messages via DV is straightforward. When the DV transport is | ||
3166 | notified of a message destined for a non-direct neighbor, the appropriate | ||
3167 | forwarding peer is selected, and the base message is encapsulated in a DV | ||
3168 | message which contains information about the initial peer and the intended | ||
3169 | recipient. At each forwarding hop, the initial peer is validated (the | ||
3170 | forwarding peer ensures that it has the initial peer in its neighborhood, | ||
3171 | otherwise the message is dropped). Next the base message is re-encapsulated in | ||
3172 | a new DV message for the next hop in the forwarding chain (or delivered to the | ||
3173 | current peer, if it has arrived at the destination). | ||
3174 | |||
3175 | Assume a three peer network with peers Alice, Bob and Carol. Assume that Alice | ||
3176 | <-> Bob and Bob <-> Carol are direct (e.g. over TCP or UDP transports) | ||
3177 | connections, but that Alice cannot directly connect to Carol. This may be the | ||
3178 | case due to NAT or firewall restrictions, or perhaps based on one of the peers | ||
3179 | respective configurations. If the Distance Vector transport is enabled on all | ||
3180 | three peers, it will automatically discover (from the gossip protocol) that | ||
3181 | Alice and Carol can connect via Bob and provide a "virtual" Alice <-> Carol | ||
3182 | connection. Routing between Alice and Carol happens as follows; Alice creates a | ||
3183 | message destined for Carol and notifies the DV transport about it. The DV | ||
3184 | transport at Alice looks up Carol in the routing table and finds that the | ||
3185 | message must be sent through Bob for Carol. The message is encapsulated setting | ||
3186 | Alice as the initiator and Carol as the destination and sent to Bob. Bob | ||
3187 | receives the messages, verifies both Alice and Carol are known to Bob, and | ||
3188 | re-wraps the message in a new DV message for Carol. The DV transport at Carol | ||
3189 | receives this message, unwraps the original message, and delivers it to Carol | ||
3190 | as though it came directly from Alice. | ||
3191 | |||
3192 | @node SMTP plugin | ||
3193 | @section SMTP plugin | ||
3194 | @c %**end of header | ||
3195 | |||
3196 | This page describes the new SMTP transport plugin for GNUnet as it exists in | ||
3197 | the 0.7.x and 0.8.x branch. SMTP support is currently not available in GNUnet | ||
3198 | 0.9.x. This page also describes the transport layer abstraction (as it existed | ||
3199 | in 0.7.x and 0.8.x) in more detail and gives some benchmarking results. The | ||
3200 | performance results presented are quite old and maybe outdated at this point. | ||
3201 | @itemize @bullet | ||
3202 | @item Why use SMTP for a peer-to-peer transport? | ||
3203 | @item SMTPHow does it work? | ||
3204 | @item How do I configure my peer? | ||
3205 | @item How do I test if it works? | ||
3206 | @item How fast is it? | ||
3207 | @item Is there any additional documentation? | ||
3208 | @end itemize | ||
3209 | |||
3210 | |||
3211 | @menu | ||
3212 | * Why use SMTP for a peer-to-peer transport?:: | ||
3213 | * How does it work?:: | ||
3214 | * How do I configure my peer?:: | ||
3215 | * How do I test if it works?:: | ||
3216 | * How fast is it?:: | ||
3217 | @end menu | ||
3218 | |||
3219 | @node Why use SMTP for a peer-to-peer transport? | ||
3220 | @subsection Why use SMTP for a peer-to-peer transport? | ||
3221 | @c %**end of header | ||
3222 | |||
3223 | There are many reasons why one would not want to use SMTP: | ||
3224 | @itemize @bullet | ||
3225 | @item SMTP is using more bandwidth than TCP, UDP or HTTP | ||
3226 | @item SMTP has a much higher latency. | ||
3227 | @item SMTP requires significantly more computation (encoding and decoding time) | ||
3228 | for the peers. | ||
3229 | @item SMTP is significantly more complicated to configure. | ||
3230 | @item SMTP may be abused by tricking GNUnet into sending mail to@ | ||
3231 | non-participating third parties. | ||
3232 | @end itemize | ||
3233 | |||
3234 | So why would anybody want to use SMTP? | ||
3235 | @itemize @bullet | ||
3236 | @item SMTP can be used to contact peers behind NAT boxes (in virtual private | ||
3237 | networks). | ||
3238 | @item SMTP can be used to circumvent policies that limit or prohibit | ||
3239 | peer-to-peer traffic by masking as "legitimate" traffic. | ||
3240 | @item SMTP uses E-mail addresses which are independent of a specific IP, which | ||
3241 | can be useful to address peers that use dynamic IP addresses. | ||
3242 | @item SMTP can be used to initiate a connection (e.g. initial address exchange) | ||
3243 | and peers can then negotiate the use of a more efficient protocol (e.g. TCP) | ||
3244 | for the actual communication. | ||
3245 | @end itemize | ||
3246 | |||
3247 | In summary, SMTP can for example be used to send a message to a peer behind a | ||
3248 | NAT box that has a dynamic IP to tell the peer to establish a TCP connection | ||
3249 | to a peer outside of the private network. Even an extraordinary overhead for | ||
3250 | this first message would be irrelevant in this type of situation. | ||
3251 | |||
3252 | @node How does it work? | ||
3253 | @subsection How does it work? | ||
3254 | @c %**end of header | ||
3255 | |||
3256 | When a GNUnet peer needs to send a message to another GNUnet peer that has | ||
3257 | advertised (only) an SMTP transport address, GNUnet base64-encodes the message | ||
3258 | and sends it in an E-mail to the advertised address. The advertisement | ||
3259 | contains a filter which is placed in the E-mail header, such that the | ||
3260 | receiving host can filter the tagged E-mails and forward it to the GNUnet peer | ||
3261 | process. The filter can be specified individually by each peer and be changed | ||
3262 | over time. This makes it impossible to censor GNUnet E-mail messages by | ||
3263 | searching for a generic filter. | ||
3264 | |||
3265 | @node How do I configure my peer? | ||
3266 | @subsection How do I configure my peer? | ||
3267 | @c %**end of header | ||
3268 | |||
3269 | First, you need to configure @code{procmail} to filter your inbound E-mail for | ||
3270 | GNUnet traffic. The GNUnet messages must be delivered into a pipe, for example | ||
3271 | @code{/tmp/gnunet.smtp}. You also need to define a filter that is used by | ||
3272 | procmail to detect GNUnet messages. You are free to choose whichever filter | ||
3273 | you like, but you should make sure that it does not occur in your other | ||
3274 | E-mail. In our example, we will use @code{X-mailer: GNUnet}. The | ||
3275 | @code{~/.procmailrc} configuration file then looks like this: | ||
3276 | @example | ||
3277 | :0: | ||
3278 | * ^X-mailer: GNUnet | ||
3279 | /tmp/gnunet.smtp | ||
3280 | # where do you want your other e-mail delivered to (default: /var/spool/mail/) | ||
3281 | :0: /var/spool/mail/ | ||
3282 | @end example | ||
3283 | |||
3284 | After adding this file, first make sure that your regular E-mail still works | ||
3285 | (e.g. by sending an E-mail to yourself). Then edit the GNUnet configuration. | ||
3286 | In the section @code{SMTP} you need to specify your E-mail address under | ||
3287 | @code{EMAIL}, your mail server (for outgoing mail) under @code{SERVER}, the | ||
3288 | filter (X-mailer: GNUnet in the example) under @code{FILTER} and the name of | ||
3289 | the pipe under @code{PIPE}.@ The completed section could then look like this: | ||
3290 | @example | ||
3291 | EMAIL = me@@mail.gnu.org MTU = 65000 SERVER = mail.gnu.org:25 FILTER = | ||
3292 | "X-mailer: GNUnet" PIPE = /tmp/gnunet.smtp | ||
3293 | @end example | ||
3294 | |||
3295 | Finally, you need to add @code{smtp} to the list of @code{TRANSPORTS} in the | ||
3296 | @code{GNUNETD} section. GNUnet peers will use the E-mail address that you | ||
3297 | specified to contact your peer until the advertisement times out. Thus, if you | ||
3298 | are not sure if everything works properly or if you are not planning to be | ||
3299 | online for a long time, you may want to configure this timeout to be short, | ||
3300 | e.g. just one hour. For this, set @code{HELLOEXPIRES} to @code{1} in the | ||
3301 | @code{GNUNETD} section. | ||
3302 | |||
3303 | This should be it, but you may probably want to test it first.@ | ||
3304 | @node How do I test if it works? | ||
3305 | @subsection How do I test if it works? | ||
3306 | @c %**end of header | ||
3307 | |||
3308 | Any transport can be subjected to some rudimentary tests using the | ||
3309 | @code{gnunet-transport-check} tool. The tool sends a message to the local node | ||
3310 | via the transport and checks that a valid message is received. While this test | ||
3311 | does not involve other peers and can not check if firewalls or other network | ||
3312 | obstacles prohibit proper operation, this is a great testcase for the SMTP | ||
3313 | transport since it tests pretty much nearly all of the functionality. | ||
3314 | |||
3315 | @code{gnunet-transport-check} should only be used without running | ||
3316 | @code{gnunetd} at the same time. By default, @code{gnunet-transport-check} | ||
3317 | tests all transports that are specified in the configuration file. But you can | ||
3318 | specifically test SMTP by giving the option @code{--transport=smtp}. | ||
3319 | |||
3320 | Note that this test always checks if a transport can receive and send. While | ||
3321 | you can configure most transports to only receive or only send messages, this | ||
3322 | test will only work if you have configured the transport to send and receive | ||
3323 | messages. | ||
3324 | |||
3325 | @node How fast is it? | ||
3326 | @subsection How fast is it? | ||
3327 | @c %**end of header | ||
3328 | |||
3329 | We have measured the performance of the UDP, TCP and SMTP transport layer | ||
3330 | directly and when used from an application using the GNUnet core. Measureing | ||
3331 | just the transport layer gives the better view of the actual overhead of the | ||
3332 | protocol, whereas evaluating the transport from the application puts the | ||
3333 | overhead into perspective from a practical point of view. | ||
3334 | |||
3335 | The loopback measurements of the SMTP transport were performed on three | ||
3336 | different machines spanning a range of modern SMTP configurations. We used a | ||
3337 | PIII-800 running RedHat 7.3 with the Purdue Computer Science configuration | ||
3338 | which includes filters for spam. We also used a Xenon 2 GHZ with a vanilla | ||
3339 | RedHat 8.0 sendmail configuration. Furthermore, we used qmail on a PIII-1000 | ||
3340 | running Sorcerer GNU Linux (SGL). The numbers for UDP and TCP are provided | ||
3341 | using the SGL configuration. The qmail benchmark uses qmail's internal | ||
3342 | filtering whereas the sendmail benchmarks relies on procmail to filter and | ||
3343 | deliver the mail. We used the transport layer to send a message of b bytes | ||
3344 | (excluding transport protocol headers) directly to the local machine. This | ||
3345 | way, network latency and packet loss on the wire have no impact on the | ||
3346 | timings. n messages were sent sequentially over the transport layer, sending | ||
3347 | message i+1 after the i-th message was received. All messages were sent over | ||
3348 | the same connection and the time to establish the connection was not taken | ||
3349 | into account since this overhead is miniscule in practice --- as long as a | ||
3350 | connection is used for a significant number of messages. | ||
3351 | |||
3352 | @multitable @columnfractions .20 .15 .15 .15 .15 .15 | ||
3353 | @headitem Transport @tab UDP @tab TCP @tab SMTP (Purdue sendmail) @tab SMTP (RH 8.0) @tab SMTP (SGL qmail) | ||
3354 | @item 11 bytes @tab 31 ms @tab 55 ms @tab 781 s @tab 77 s @tab 24 s | ||
3355 | @item 407 bytes @tab 37 ms @tab 62 ms @tab 789 s @tab 78 s @tab 25 s | ||
3356 | @item 1,221 bytes @tab 46 ms @tab 73 ms @tab 804 s @tab 78 s @tab 25 s | ||
3357 | @end multitable | ||
3358 | |||
3359 | The benchmarks show that UDP and TCP are, as expected, both significantly | ||
3360 | faster compared with any of the SMTP services. Among the SMTP implementations, | ||
3361 | there can be significant differences depending on the SMTP configuration. | ||
3362 | Filtering with an external tool like procmail that needs to re-parse its | ||
3363 | configuration for each mail can be very expensive. Applying spam filters can | ||
3364 | also significantly impact the performance of the underlying SMTP | ||
3365 | implementation. The microbenchmark shows that SMTP can be a viable solution | ||
3366 | for initiating peer-to-peer sessions: a couple of seconds to connect to a peer | ||
3367 | are probably not even going to be noticed by users. The next benchmark | ||
3368 | measures the possible throughput for a transport. Throughput can be measured | ||
3369 | by sending multiple messages in parallel and measuring packet loss. Note that | ||
3370 | not only UDP but also the TCP transport can actually loose messages since the | ||
3371 | TCP implementation drops messages if the @code{write} to the socket would | ||
3372 | block. While the SMTP protocol never drops messages itself, it is often so | ||
3373 | slow that only a fraction of the messages can be sent and received in the | ||
3374 | given time-bounds. For this benchmark we report the message loss after | ||
3375 | allowing t time for sending m messages. If messages were not sent (or | ||
3376 | received) after an overall timeout of t, they were considered lost. The | ||
3377 | benchmark was performed using two Xeon 2 GHZ machines running RedHat 8.0 with | ||
3378 | sendmail. The machines were connected with a direct 100 MBit ethernet | ||
3379 | connection.@ Figures udp1200, tcp1200 and smtp-MTUs show that the throughput | ||
3380 | for messages of size 1,200 octects is 2,343 kbps, 3,310 kbps and 6 kbps for | ||
3381 | UDP, TCP and SMTP respectively. The high per-message overhead of SMTP can be | ||
3382 | improved by increasing the MTU, for example, an MTU of 12,000 octets improves | ||
3383 | the throughput to 13 kbps as figure smtp-MTUs shows. Our research paper) has | ||
3384 | some more details on the benchmarking results. | ||
3385 | |||
3386 | @node Bluetooth plugin | ||
3387 | @section Bluetooth plugin | ||
3388 | @c %**end of header | ||
3389 | |||
3390 | This page describes the new Bluetooth transport plugin for GNUnet. The plugin | ||
3391 | is still in the testing stage so don't expect it to work perfectly. If you | ||
3392 | have any questions or problems just post them here or ask on the IRC channel. | ||
3393 | @itemize @bullet | ||
3394 | @item What do I need to use the Bluetooth plugin transport? | ||
3395 | @item BluetoothHow does it work? | ||
3396 | @item What possible errors should I be aware of? | ||
3397 | @item How do I configure my peer? | ||
3398 | @item How can I test it? | ||
3399 | @end itemize | ||
3400 | |||
3401 | |||
3402 | |||
3403 | @menu | ||
3404 | * What do I need to use the Bluetooth plugin transport?:: | ||
3405 | * How does it work2?:: | ||
3406 | * What possible errors should I be aware of?:: | ||
3407 | * How do I configure my peer2?:: | ||
3408 | * How can I test it?:: | ||
3409 | * The implementation of the Bluetooth transport plugin:: | ||
3410 | @end menu | ||
3411 | |||
3412 | @node What do I need to use the Bluetooth plugin transport? | ||
3413 | @subsection What do I need to use the Bluetooth plugin transport? | ||
3414 | @c %**end of header | ||
3415 | |||
3416 | If you are a Linux user and you want to use the Bluetooth transport plugin you | ||
3417 | should install the BlueZ development libraries (if they aren't already | ||
3418 | installed). For instructions about how to install the libraries you should | ||
3419 | check out the BlueZ site (@uref{http://www.bluez.org/, http://www.bluez.org}). | ||
3420 | If you don't know if you have the necesarry libraries, don't worry, just run | ||
3421 | the GNUnet configure script and you will be able to see a notification at the | ||
3422 | end which will warn you if you don't have the necessary libraries. | ||
3423 | |||
3424 | If you are a Windows user you should have installed the | ||
3425 | @emph{MinGW}/@emph{MSys2} with the latest updates (especially the | ||
3426 | @emph{ws2bth} header). If this is your first build of GNUnet on Windows you | ||
3427 | should check out the SBuild repository. It will semi-automatically assembles a | ||
3428 | @emph{MinGW}/@emph{MSys2} installation with a lot of extra packages which are | ||
3429 | needed for the GNUnet build. So this will ease your work!@ Finally you just | ||
3430 | have to be sure that you have the correct drivers for your Bluetooth device | ||
3431 | installed and that your device is on and in a discoverable mode. The Windows | ||
3432 | Bluetooth Stack supports only the RFCOMM protocol so we cannot turn on your | ||
3433 | device programatically! | ||
3434 | |||
3435 | @node How does it work2? | ||
3436 | @subsection How does it work2? | ||
3437 | @c %**end of header | ||
3438 | |||
3439 | The Bluetooth transport plugin uses virtually the same code as the WLAN plugin | ||
3440 | and only the helper binary is different. The helper takes a single argument, | ||
3441 | which represents the interface name and is specified in the configuration | ||
3442 | file. Here are the basic steps that are followed by the helper binary used on | ||
3443 | Linux: | ||
3444 | |||
3445 | @itemize @bullet | ||
3446 | @item it verifies if the name corresponds to a Bluetooth interface name | ||
3447 | @item it verifies if the iterface is up (if it is not, it tries to bring it up) | ||
3448 | @item it tries to enable the page and inquiry scan in order to make the device | ||
3449 | discoverable and to accept incoming connection requests | ||
3450 | @emph{The above operations require root access so you should start the | ||
3451 | transport plugin with root privileges.} | ||
3452 | @item it finds an available port number and registers a SDP service which will | ||
3453 | be used to find out on which port number is the server listening on and switch | ||
3454 | the socket in listening mode | ||
3455 | @item it sends a HELLO message with its address | ||
3456 | @item finally it forwards traffic from the reading sockets to the STDOUT and | ||
3457 | from the STDIN to the writing socket | ||
3458 | @end itemize | ||
3459 | |||
3460 | Once in a while the device will make an inquiry scan to discover the nearby | ||
3461 | devices and it will send them randomly HELLO messages for peer discovery. | ||
3462 | |||
3463 | @node What possible errors should I be aware of? | ||
3464 | @subsection What possible errors should I be aware of? | ||
3465 | @c %**end of header | ||
3466 | |||
3467 | @emph{This section is dedicated for Linux users} | ||
3468 | |||
3469 | Well there are many ways in which things could go wrong but I will try to | ||
3470 | present some tools that you could use to debug and some scenarios. | ||
3471 | @itemize @bullet | ||
3472 | |||
3473 | @item @code{bluetoothd -n -d} : use this command to enable logging in the | ||
3474 | foreground and to print the logging messages | ||
3475 | |||
3476 | @item @code{hciconfig}: can be used to configure the Bluetooth devices. If you | ||
3477 | run it without any arguments it will print information about the state of the | ||
3478 | interfaces. So if you receive an error that the device couldn't be brought up | ||
3479 | you should try to bring it manually and to see if it works (use @code{hciconfig | ||
3480 | -a hciX up}). If you can't and the Bluetooth address has the form | ||
3481 | 00:00:00:00:00:00 it means that there is something wrong with the D-Bus daemon | ||
3482 | or with the Bluetooth daemon. Use @code{bluetoothd} tool to see the logs | ||
3483 | |||
3484 | @item @code{sdptool} can be used to control and interogate SDP servers. If you | ||
3485 | encounter problems regarding the SDP server (like the SDP server is down) you | ||
3486 | should check out if the D-Bus daemon is running correctly and to see if the | ||
3487 | Bluetooth daemon started correctly(use @code{bluetoothd} tool). Also, sometimes | ||
3488 | the SDP service could work but somehow the device couldn't register his | ||
3489 | service. Use @code{sdptool browse [dev-address]} to see if the service is | ||
3490 | registered. There should be a service with the name of the interface and GNUnet | ||
3491 | as provider. | ||
3492 | |||
3493 | @item @code{hcitool} : another useful tool which can be used to configure the | ||
3494 | device and to send some particular commands to it. | ||
3495 | |||
3496 | @item @code{hcidump} : could be used for low level debugging | ||
3497 | @end itemize | ||
3498 | |||
3499 | @node How do I configure my peer2? | ||
3500 | @subsection How do I configure my peer2? | ||
3501 | @c %**end of header | ||
3502 | |||
3503 | On Linux, you just have to be sure that the interface name corresponds to the | ||
3504 | one that you want to use. Use the @code{hciconfig} tool to check that. By | ||
3505 | default it is set to hci0 but you can change it. | ||
3506 | |||
3507 | A basic configuration looks like this: | ||
3508 | @example | ||
3509 | [transport-bluetooth] | ||
3510 | # Name of the interface (typically hciX) | ||
3511 | INTERFACE = hci0 | ||
3512 | # Real hardware, no testing | ||
3513 | TESTMODE = 0 TESTING_IGNORE_KEYS = ACCEPT_FROM; | ||
3514 | @end example | ||
3515 | |||
3516 | |||
3517 | In order to use the Bluetooth transport plugin when the transport service is | ||
3518 | started, you must add the plugin name to the default transport service plugins | ||
3519 | list. For example: | ||
3520 | @example | ||
3521 | [transport] ... PLUGINS = dns bluetooth ... | ||
3522 | @end example | ||
3523 | |||
3524 | If you want to use only the Bluetooth plugin set @emph{PLUGINS = bluetooth} | ||
3525 | |||
3526 | On Windows, you cannot specify which device to use. The only thing that you | ||
3527 | should do is to add @emph{bluetooth} on the plugins list of the transport | ||
3528 | service. | ||
3529 | |||
3530 | @node How can I test it? | ||
3531 | @subsection How can I test it? | ||
3532 | @c %**end of header | ||
3533 | |||
3534 | If you have two Bluetooth devices on the same machine which use Linux you | ||
3535 | must: | ||
3536 | @itemize @bullet | ||
3537 | |||
3538 | @item create two different file configuration (one which will use the first | ||
3539 | interface (@emph{hci0}) and the other which will use the second interface | ||
3540 | (@emph{hci1})). Let's name them @emph{peer1.conf} and @emph{peer2.conf}. | ||
3541 | |||
3542 | @item run @emph{gnunet-peerinfo -c peerX.conf -s} in order to generate the | ||
3543 | peers private keys. The @strong{X} must be replace with 1 or 2. | ||
3544 | |||
3545 | @item run @emph{gnunet-arm -c peerX.conf -s -i=transport} in order to start the | ||
3546 | transport service. (Make sure that you have "bluetooth" on the transport | ||
3547 | plugins list if the Bluetooth transport service doesn't start.) | ||
3548 | |||
3549 | @item run @emph{gnunet-peerinfo -c peer1.conf -s} to get the first peer's ID. | ||
3550 | If you already know your peer ID (you saved it from the first command), this | ||
3551 | can be skipped. | ||
3552 | |||
3553 | @item run @emph{gnunet-transport -c peer2.conf -p=PEER1_ID -s} to start sending | ||
3554 | data for benchmarking to the other peer. | ||
3555 | @end itemize | ||
3556 | |||
3557 | |||
3558 | This scenario will try to connect the second peer to the first one and then | ||
3559 | start sending data for benchmarking. | ||
3560 | |||
3561 | On Windows you cannot test the plugin functionality using two Bluetooth devices | ||
3562 | from the same machine because after you install the drivers there will occur | ||
3563 | some conflicts between the Bluetooth stacks. (At least that is what happend on | ||
3564 | my machine : I wasn't able to use the Bluesoleil stack and the WINDCOMM one in | ||
3565 | the same time). | ||
3566 | |||
3567 | If you have two different machines and your configuration files are good you | ||
3568 | can use the same scenario presented on the begining of this section. | ||
3569 | |||
3570 | Another way to test the plugin functionality is to create your own application | ||
3571 | which will use the GNUnet framework with the Bluetooth transport service. | ||
3572 | |||
3573 | @node The implementation of the Bluetooth transport plugin | ||
3574 | @subsection The implementation of the Bluetooth transport plugin | ||
3575 | @c %**end of header | ||
3576 | |||
3577 | This page describes the implementation of the Bluetooth transport plugin. | ||
3578 | |||
3579 | First I want to remind you that the Bluetooth transport plugin uses virtually | ||
3580 | the same code as the WLAN plugin and only the helper binary is different. Also | ||
3581 | the scope of the helper binary from the Bluetooth transport plugin is the same | ||
3582 | as the one used for the wlan transport plugin: it acceses the interface and | ||
3583 | then it forwards traffic in both directions between the Bluetooth interface | ||
3584 | and stdin/stdout of the process involved. | ||
3585 | |||
3586 | The Bluetooth plugin transport could be used both on Linux and Windows | ||
3587 | platforms. | ||
3588 | |||
3589 | @itemize @bullet | ||
3590 | @item Linux functionality | ||
3591 | @item Windows functionality | ||
3592 | @item Pending Features | ||
3593 | @end itemize | ||
3594 | |||
3595 | |||
3596 | |||
3597 | @menu | ||
3598 | * Linux functionality:: | ||
3599 | * THE INITIALIZATION:: | ||
3600 | * THE LOOP:: | ||
3601 | * Details about the broadcast implementation:: | ||
3602 | * Windows functionality:: | ||
3603 | * Pending features:: | ||
3604 | @end menu | ||
3605 | |||
3606 | @node Linux functionality | ||
3607 | @subsubsection Linux functionality | ||
3608 | @c %**end of header | ||
3609 | |||
3610 | In order to implement the plugin functionality on Linux I used the BlueZ | ||
3611 | stack. For the communication with the other devices I used the RFCOMM | ||
3612 | protocol. Also I used the HCI protocol to gain some control over the device. | ||
3613 | The helper binary takes a single argument (the name of the Bluetooth | ||
3614 | interface) and is separated in two stages: | ||
3615 | |||
3616 | @c %** 'THE INITIALIZATION' should be in bigger letters or stand out, not | ||
3617 | @c %** starting a new section? | ||
3618 | @node THE INITIALIZATION | ||
3619 | @subsubsection THE INITIALIZATION | ||
3620 | |||
3621 | @itemize @bullet | ||
3622 | @item first, it checks if we have root privilegies (@emph{Remember that we need | ||
3623 | to have root privilegies in order to be able to bring the interface up if it is | ||
3624 | down or to change its state.}). | ||
3625 | |||
3626 | @item second, it verifies if the interface with the given name exists. | ||
3627 | |||
3628 | @strong{If the interface with that name exists and it is a Bluetooth | ||
3629 | interface:} | ||
3630 | |||
3631 | @item it creates a RFCOMM socket which will be used for listening and call the | ||
3632 | @emph{open_device} method | ||
3633 | |||
3634 | On the @emph{open_device} method: | ||
3635 | @itemize @bullet | ||
3636 | @item creates a HCI socket used to send control events to the the device | ||
3637 | @item searches for the device ID using the interface name | ||
3638 | @item saves the device MAC address | ||
3639 | @item checks if the interface is down and tries to bring it UP | ||
3640 | @item checks if the interface is in discoverable mode and tries to make it | ||
3641 | discoverable | ||
3642 | @item closes the HCI socket and binds the RFCOMM one | ||
3643 | @item switches the RFCOMM socket in listening mode | ||
3644 | @item registers the SDP service (the service will be used by the other devices | ||
3645 | to get the port on which this device is listening on) | ||
3646 | @end itemize | ||
3647 | |||
3648 | @item drops the root privilegies | ||
3649 | |||
3650 | @strong{If the interface is not a Bluetooth interface the helper exits with a | ||
3651 | suitable error} | ||
3652 | @end itemize | ||
3653 | |||
3654 | @c %** Same as for @node entry above | ||
3655 | @node THE LOOP | ||
3656 | @subsubsection THE LOOP | ||
3657 | |||
3658 | The helper binary uses a list where it saves all the connected neighbour | ||
3659 | devices (@emph{neighbours.devices}) and two buffers (@emph{write_pout} and | ||
3660 | @emph{write_std}). The first message which is send is a control message with | ||
3661 | the device's MAC address in order to announce the peer presence to the | ||
3662 | neighbours. Here are a short description of what happens in the main loop: | ||
3663 | |||
3664 | @itemize @bullet | ||
3665 | @item Every time when it receives something from the STDIN it processes the | ||
3666 | data and saves the message in the first buffer (@emph{write_pout}). When it has | ||
3667 | something in the buffer, it gets the destination address from the buffer, | ||
3668 | searches the destination address in the list (if there is no connection with | ||
3669 | that device, it creates a new one and saves it to the list) and sends the | ||
3670 | message. | ||
3671 | @item Every time when it receives something on the listening socket it accepts | ||
3672 | the connection and saves the socket on a list with the reading sockets. | ||
3673 | @item Every time when it receives something from a reading socket it parses the | ||
3674 | message, verifies the CRC and saves it in the @emph{write_std} buffer in order | ||
3675 | to be sent later to the STDOUT. | ||
3676 | @end itemize | ||
3677 | |||
3678 | So in the main loop we use the select function to wait until one of the file | ||
3679 | descriptor saved in one of the two file descriptors sets used is ready to use. | ||
3680 | The first set (@emph{rfds}) represents the reading set and it could contain the | ||
3681 | list with the reading sockets, the STDIN file descriptor or the listening | ||
3682 | socket. The second set (@emph{wfds}) is the writing set and it could contain | ||
3683 | the sending socket or the STDOUT file descriptor. After the select function | ||
3684 | returns, we check which file descriptor is ready to use and we do what is | ||
3685 | supposed to do on that kind of event. @emph{For example:} if it is the | ||
3686 | listening socket then we accept a new connection and save the socket in the | ||
3687 | reading list; if it is the STDOUT file descriptor, then we write to STDOUT the | ||
3688 | message from the @emph{write_std} buffer. | ||
3689 | |||
3690 | To find out on which port a device is listening on we connect to the local SDP | ||
3691 | server and searche the registered service for that device. | ||
3692 | |||
3693 | @emph{You should be aware of the fact that if the device fails to connect to | ||
3694 | another one when trying to send a message it will attempt one more time. If it | ||
3695 | fails again, then it skips the message.} | ||
3696 | @emph{Also you should know that the | ||
3697 | transport Bluetooth plugin has support for @strong{broadcast messages}.} | ||
3698 | |||
3699 | @node Details about the broadcast implementation | ||
3700 | @subsubsection Details about the broadcast implementation | ||
3701 | @c %**end of header | ||
3702 | |||
3703 | First I want to point out that the broadcast functionality for the CONTROL | ||
3704 | messages is not implemented in a conventional way. Since the inquiry scan time | ||
3705 | is too big and it will take some time to send a message to all the | ||
3706 | discoverable devices I decided to tackle the problem in a different way. Here | ||
3707 | is how I did it: | ||
3708 | |||
3709 | @itemize @bullet | ||
3710 | @item If it is the first time when I have to broadcast a message I make an | ||
3711 | inquiry scan and save all the devices' addresses to a vector. | ||
3712 | @item After the inquiry scan ends I take the first address from the list and I | ||
3713 | try to connect to it. If it fails, I try to connect to the next one. If it | ||
3714 | succeeds, I save the socket to a list and send the message to the device. | ||
3715 | @item When I have to broadcast another message, first I search on the list for | ||
3716 | a new device which I'm not connected to. If there is no new device on the list | ||
3717 | I go to the beginning of the list and send the message to the old devices. | ||
3718 | After 5 cycles I make a new inquiry scan to check out if there are new | ||
3719 | discoverable devices and save them to the list. If there are no new | ||
3720 | discoverable devices I reset the cycling counter and go again through the old | ||
3721 | list and send messages to the devices saved in it. | ||
3722 | @end itemize | ||
3723 | |||
3724 | @strong{Therefore}: | ||
3725 | |||
3726 | @itemize @bullet | ||
3727 | @item every time when I have a broadcast message I look up on the list for a | ||
3728 | new device and send the message to it | ||
3729 | @item if I reached the end of the list for 5 times and I'm connected to all the | ||
3730 | devices from the list I make a new inquiry scan. @emph{The number of the list's | ||
3731 | cycles after an inquiry scan could be increased by redefining the MAX_LOOPS | ||
3732 | variable} | ||
3733 | @item when there are no new devices I send messages to the old ones. | ||
3734 | @end itemize | ||
3735 | |||
3736 | Doing so, the broadcast control messages will reach the devices but with delay. | ||
3737 | |||
3738 | @emph{NOTICE:} When I have to send a message to a certain device first I check | ||
3739 | on the broadcast list to see if we are connected to that device. If not we try | ||
3740 | to connect to it and in case of success we save the address and the socket on | ||
3741 | the list. If we are already connected to that device we simply use the socket. | ||
3742 | |||
3743 | @node Windows functionality | ||
3744 | @subsubsection Windows functionality | ||
3745 | @c %**end of header | ||
3746 | |||
3747 | For Windows I decided to use the Microsoft Bluetooth stack which has the | ||
3748 | advantage of coming standard from Windows XP SP2. The main disadvantage is | ||
3749 | that it only supports the RFCOMM protocol so we will not be able to have a low | ||
3750 | level control over the Bluetooth device. Therefore it is the user | ||
3751 | responsability to check if the device is up and in the discoverable mode. Also | ||
3752 | there are no tools which could be used for debugging in order to read the data | ||
3753 | coming from and going to a Bluetooth device, which obviously hindered my work. | ||
3754 | Another thing that slowed down the implementation of the plugin (besides that | ||
3755 | I wasn't too accomodated with the win32 API) was that there were some bugs on | ||
3756 | MinGW regarding the Bluetooth. Now they are solved but you should keep in mind | ||
3757 | that you should have the latest updates (especially the @emph{ws2bth} header). | ||
3758 | |||
3759 | Besides the fact that it uses the Windows Sockets, the Windows implemenation | ||
3760 | follows the same principles as the Linux one: | ||
3761 | |||
3762 | @itemize @bullet | ||
3763 | @item | ||
3764 | It has a initalization part where it initializes the Windows Sockets, creates a | ||
3765 | RFCOMM socket which will be binded and switched to the listening mode and | ||
3766 | registers a SDP service. | ||
3767 | In the Microsoft Bluetooth API there are two ways to work with the SDP: | ||
3768 | @itemize @bullet | ||
3769 | @item an easy way which works with very simple service records | ||
3770 | @item a hard way which is useful when you need to update or to delete the | ||
3771 | record | ||
3772 | @end itemize | ||
3773 | @end itemize | ||
3774 | |||
3775 | Since I only needed the SDP service to find out on which port the device is | ||
3776 | listening on and that did not change, I decided to use the easy way. In order | ||
3777 | to register the service I used the @emph{WSASetService} function and I | ||
3778 | generated the @emph{Universally Unique Identifier} with the @emph{guidgen.exe} | ||
3779 | Windows's tool. | ||
3780 | |||
3781 | In the loop section the only difference from the Linux implementation is that | ||
3782 | I used the GNUNET_NETWORK library for functions like @emph{accept}, | ||
3783 | @emph{bind}, @emph{connect} or @emph{select}. I decided to use the | ||
3784 | GNUNET_NETWORK library because I also needed to interact with the STDIN and | ||
3785 | STDOUT handles and on Windows the select function is only defined for sockets, | ||
3786 | and it will not work for arbitrary file handles. | ||
3787 | |||
3788 | Another difference between Linux and Windows implementation is that in Linux, | ||
3789 | the Bluetooth address is represented in 48 bits while in Windows is | ||
3790 | represented in 64 bits. Therefore I had to do some changes on | ||
3791 | @emph{plugin_transport_wlan} header. | ||
3792 | |||
3793 | Also, currently on Windows the Bluetooth plugin doesn't have support for | ||
3794 | broadcast messages. When it receives a broadcast message it will skip it. | ||
3795 | |||
3796 | @node Pending features | ||
3797 | @subsubsection Pending features | ||
3798 | @c %**end of header | ||
3799 | |||
3800 | @itemize @bullet | ||
3801 | @item Implement the broadcast functionality on Windows @emph{(currently working | ||
3802 | on)} | ||
3803 | @item Implement a testcase for the helper :@ @emph{@ The testcase consists of a | ||
3804 | program which emaluates the plugin and uses the helper. It will simulate | ||
3805 | connections, disconnections and data transfers.@ } | ||
3806 | @end itemize | ||
3807 | |||
3808 | If you have a new idea about a feature of the plugin or suggestions about how | ||
3809 | I could improve the implementation you are welcome to comment or to contact | ||
3810 | me. | ||
3811 | |||
3812 | @node WLAN plugin | ||
3813 | @section WLAN plugin | ||
3814 | @c %**end of header | ||
3815 | |||
3816 | This section documents how the wlan transport plugin works. Parts which are not | ||
3817 | implemented yet or could be better implemented are described at the end. | ||
3818 | |||
3819 | @node The ATS Subsystem | ||
3820 | @section The ATS Subsystem | ||
3821 | @c %**end of header | ||
3822 | |||
3823 | ATS stands for "automatic transport selection", and the function of ATS in | ||
3824 | GNUnet is to decide on which address (and thus transport plugin) should be used | ||
3825 | for two peers to communicate, and what bandwidth limits should be imposed on | ||
3826 | such an individual connection. To help ATS make an informed decision, | ||
3827 | higher-level services inform the ATS service about their requirements and the | ||
3828 | quality of the service rendered. The ATS service also interacts with the | ||
3829 | transport service to be appraised of working addresses and to communicate its | ||
3830 | resource allocation decisions. Finally, the ATS service's operation can be | ||
3831 | observed using a monitoring API. | ||
3832 | |||
3833 | The main logic of the ATS service only collects the available addresses, their | ||
3834 | performance characteristics and the applications requirements, but does not | ||
3835 | make the actual allocation decision. This last critical step is left to an ATS | ||
3836 | plugin, as we have implemented (currently three) different allocation | ||
3837 | strategies which differ significantly in their performance and maturity, and it | ||
3838 | is still unclear if any particular plugin is generally superior. | ||
3839 | |||
3840 | @node GNUnet's CORE Subsystem | ||
3841 | @section GNUnet's CORE Subsystem | ||
3842 | @c %**end of header | ||
3843 | |||
3844 | The CORE subsystem in GNUnet is responsible for securing link-layer | ||
3845 | communications between nodes in the GNUnet overlay network. CORE builds on the | ||
3846 | TRANSPORT subsystem which provides for the actual, insecure, unreliable | ||
3847 | link-layer communication (for example, via UDP or WLAN), and then adds | ||
3848 | fundamental security to the connections: | ||
3849 | |||
3850 | @itemize @bullet | ||
3851 | @item confidentiality with so-called perfect forward secrecy; we use | ||
3852 | @uref{http://en.wikipedia.org/wiki/Elliptic_curve_Diffie%E2%80%93Hellman, | ||
3853 | ECDHE} powered by @uref{http://cr.yp.to/ecdh.html, Curve25519} for the key | ||
3854 | exchange and then use symmetric encryption, encrypting with both | ||
3855 | @uref{http://en.wikipedia.org/wiki/Rijndael, AES-256} and | ||
3856 | @uref{http://en.wikipedia.org/wiki/Twofish, Twofish} | ||
3857 | @item @uref{http://en.wikipedia.org/wiki/Authentication, authentication} is | ||
3858 | achieved by signing the ephemeral keys using @uref{http://ed25519.cr.yp.to/, | ||
3859 | Ed25519}, a deterministic variant of @uref{http://en.wikipedia.org/wiki/ECDSA, | ||
3860 | ECDSA} | ||
3861 | @item integrity protection (using @uref{http://en.wikipedia.org/wiki/SHA-2, | ||
3862 | SHA-512} to do @uref{http://en.wikipedia.org/wiki/Authenticated_encryption, | ||
3863 | encrypt-then-MAC)} | ||
3864 | @item @uref{http://en.wikipedia.org/wiki/Replay_attack, replay} protection | ||
3865 | (using nonces, timestamps, challenge-response, message counters and ephemeral | ||
3866 | keys) | ||
3867 | @item liveness (keep-alive messages, timeout) | ||
3868 | @end itemize | ||
3869 | |||
3870 | @menu | ||
3871 | * Limitations:: | ||
3872 | * When is a peer "connected"?:: | ||
3873 | * libgnunetcore:: | ||
3874 | * The CORE Client-Service Protocol:: | ||
3875 | * The CORE Peer-to-Peer Protocol:: | ||
3876 | @end menu | ||
3877 | |||
3878 | @node Limitations | ||
3879 | @subsection Limitations | ||
3880 | @c %**end of header | ||
3881 | |||
3882 | CORE does not perform @uref{http://en.wikipedia.org/wiki/Routing, routing}; | ||
3883 | using CORE it is only possible to communicate with peers that happen to | ||
3884 | already be "directly" connected with each other. CORE also does not have an | ||
3885 | API to allow applications to establish such "direct" connections --- for this, | ||
3886 | applications can ask TRANSPORT, but TRANSPORT might not be able to establish a | ||
3887 | "direct" connection. The TOPOLOGY subsystem is responsible for trying to keep | ||
3888 | a few "direct" connections open at all times. Applications that need to talk | ||
3889 | to particular peers should use the CADET subsystem, as it can establish | ||
3890 | arbitrary "indirect" connections. | ||
3891 | |||
3892 | Because CORE does not perform routing, CORE must only be used directly by | ||
3893 | applications that either perform their own routing logic (such as anonymous | ||
3894 | file-sharing) or that do not require routing, for example because they are | ||
3895 | based on flooding the network. CORE communication is unreliable and delivery | ||
3896 | is possibly out-of-order. Applications that require reliable communication | ||
3897 | should use the CADET service. Each application can only queue one message per | ||
3898 | target peer with the CORE service at any time; messages cannot be larger than | ||
3899 | approximately 63 kilobytes. If messages are small, CORE may group multiple | ||
3900 | messages (possibly from different applications) prior to encryption. If | ||
3901 | permitted by the application (using the @uref{http://baus.net/on-tcp_cork/, | ||
3902 | cork} option), CORE may delay transmissions to facilitate grouping of multiple | ||
3903 | small messages. If cork is not enabled, CORE will transmit the message as soon | ||
3904 | as TRANSPORT allows it (TRANSPORT is responsible for limiting bandwidth and | ||
3905 | congestion control). CORE does not allow flow control; applications are | ||
3906 | expected to process messages at line-speed. If flow control is needed, | ||
3907 | applications should use the CADET service. | ||
3908 | |||
3909 | @node When is a peer "connected"? | ||
3910 | @subsection When is a peer "connected"? | ||
3911 | @c %**end of header | ||
3912 | |||
3913 | In addition to the security features mentioned above, CORE also provides one | ||
3914 | additional key feature to applications using it, and that is a limited form of | ||
3915 | protocol-compatibility checking. CORE distinguishes between TRANSPORT-level | ||
3916 | connections (which enable communication with other peers) and | ||
3917 | application-level connections. Applications using the CORE API will | ||
3918 | (typically) learn about application-level connections from CORE, and not about | ||
3919 | TRANSPORT-level connections. When a typical application uses CORE, it will | ||
3920 | specify a set of message types (from @code{gnunet_protocols.h}) that it | ||
3921 | understands. CORE will then notify the application about connections it has | ||
3922 | with other peers if and only if those applications registered an intersecting | ||
3923 | set of message types with their CORE service. Thus, it is quite possible that | ||
3924 | CORE only exposes a subset of the established direct connections to a | ||
3925 | particular application --- and different applications running above CORE might | ||
3926 | see different sets of connections at the same time. | ||
3927 | |||
3928 | A special case are applications that do not register a handler for any message | ||
3929 | type. CORE assumes that these applications merely want to monitor connections | ||
3930 | (or "all" messages via other callbacks) and will notify those applications | ||
3931 | about all connections. This is used, for example, by the @code{gnunet-core} | ||
3932 | command-line tool to display the active connections. Note that it is also | ||
3933 | possible that the TRANSPORT service has more active connections than the CORE | ||
3934 | service, as the CORE service first has to perform a key exchange with | ||
3935 | connecting peers before exchanging information about supported message types | ||
3936 | and notifying applications about the new connection. | ||
3937 | |||
3938 | @node libgnunetcore | ||
3939 | @subsection libgnunetcore | ||
3940 | @c %**end of header | ||
3941 | |||
3942 | The CORE API (defined in @code{gnunet_core_service.h}) is the basic messaging | ||
3943 | API used by P2P applications built using GNUnet. It provides applications the | ||
3944 | ability to send and receive encrypted messages to the peer's "directly" | ||
3945 | connected neighbours. | ||
3946 | |||
3947 | As CORE connections are generally "direct" connections,@ applications must not | ||
3948 | assume that they can connect to arbitrary peers this way, as "direct" | ||
3949 | connections may not always be possible. Applications using CORE are notified | ||
3950 | about which peers are connected. Creating new "direct" connections must be | ||
3951 | done using the TRANSPORT API. | ||
3952 | |||
3953 | The CORE API provides unreliable, out-of-order delivery. While the | ||
3954 | implementation tries to ensure timely, in-order delivery, both message losses | ||
3955 | and reordering are not detected and must be tolerated by the application. Most | ||
3956 | important, the core will NOT perform retransmission if messages could not be | ||
3957 | delivered. | ||
3958 | |||
3959 | Note that CORE allows applications to queue one message per connected peer. | ||
3960 | The rate at which each connection operates is influenced by the preferences | ||
3961 | expressed by local application as well as restrictions imposed by the other | ||
3962 | peer. Local applications can express their preferences for particular | ||
3963 | connections using the "performance" API of the ATS service. | ||
3964 | |||
3965 | Applications that require more sophisticated transmission capabilities such as | ||
3966 | TCP-like behavior, or if you intend to send messages to arbitrary remote | ||
3967 | peers, should use the CADET API. | ||
3968 | |||
3969 | The typical use of the CORE API is to connect to the CORE service using | ||
3970 | @code{GNUNET_CORE_connect}, process events from the CORE service (such as | ||
3971 | peers connecting, peers disconnecting and incoming messages) and send messages | ||
3972 | to connected peers using @code{GNUNET_CORE_notify_transmit_ready}. Note that | ||
3973 | applications must cancel pending transmission requests if they receive a | ||
3974 | disconnect event for a peer that had a transmission pending; furthermore, | ||
3975 | queueing more than one transmission request per peer per application using the | ||
3976 | service is not permitted. | ||
3977 | |||
3978 | The CORE API also allows applications to monitor all communications of the | ||
3979 | peer prior to encryption (for outgoing messages) or after decryption (for | ||
3980 | incoming messages). This can be useful for debugging, diagnostics or to | ||
3981 | establish the presence of cover traffic (for anonymity). As monitoring | ||
3982 | applications are often not interested in the payload, the monitoring callbacks | ||
3983 | can be configured to only provide the message headers (including the message | ||
3984 | type and size) instead of copying the full data stream to the monitoring | ||
3985 | client. | ||
3986 | |||
3987 | The init callback of the @code{GNUNET_CORE_connect} function is called with | ||
3988 | the hash of the public key of the peer. This public key is used to identify | ||
3989 | the peer globally in the GNUnet network. Applications are encouraged to check | ||
3990 | that the provided hash matches the hash that they are using (as theoretically | ||
3991 | the application may be using a different configuration file with a different | ||
3992 | private key, which would result in hard to find bugs). | ||
3993 | |||
3994 | As with most service APIs, the CORE API isolates applications from crashes of | ||
3995 | the CORE service. If the CORE service crashes, the application will see | ||
3996 | disconnect events for all existing connections. Once the connections are | ||
3997 | re-established, the applications will be receive matching connect events. | ||
3998 | |||
3999 | @node The CORE Client-Service Protocol | ||
4000 | @subsection The CORE Client-Service Protocol | ||
4001 | @c %**end of header | ||
4002 | |||
4003 | This section describes the protocol between an application using the CORE | ||
4004 | service (the client) and the CORE service process itself. | ||
4005 | |||
4006 | |||
4007 | @menu | ||
4008 | * Setup2:: | ||
4009 | * Notifications:: | ||
4010 | * Sending:: | ||
4011 | @end menu | ||
4012 | |||
4013 | @node Setup2 | ||
4014 | @subsubsection Setup2 | ||
4015 | @c %**end of header | ||
4016 | |||
4017 | When a client connects to the CORE service, it first sends a | ||
4018 | @code{InitMessage} which specifies options for the connection and a set of | ||
4019 | message type values which are supported by the application. The options | ||
4020 | bitmask specifies which events the client would like to be notified about. The | ||
4021 | options include: | ||
4022 | |||
4023 | @table @asis | ||
4024 | @item GNUNET_CORE_OPTION_NOTHING No notifications | ||
4025 | @item GNUNET_CORE_OPTION_STATUS_CHANGE Peers connecting and disconnecting | ||
4026 | @item GNUNET_CORE_OPTION_FULL_INBOUND All inbound messages (after decryption) with | ||
4027 | full payload | ||
4028 | @item GNUNET_CORE_OPTION_HDR_INBOUND Just the @code{MessageHeader} | ||
4029 | of all inbound messages | ||
4030 | @item GNUNET_CORE_OPTION_FULL_OUTBOUND All outbound | ||
4031 | messages (prior to encryption) with full payload | ||
4032 | @item GNUNET_CORE_OPTION_HDR_OUTBOUND Just the @code{MessageHeader} of all outbound | ||
4033 | messages | ||
4034 | @end table | ||
4035 | |||
4036 | Typical applications will only monitor for connection status changes. | ||
4037 | |||
4038 | The CORE service responds to the @code{InitMessage} with an | ||
4039 | @code{InitReplyMessage} which contains the peer's identity. Afterwards, both | ||
4040 | CORE and the client can send messages. | ||
4041 | |||
4042 | @node Notifications | ||
4043 | @subsubsection Notifications | ||
4044 | @c %**end of header | ||
4045 | |||
4046 | The CORE will send @code{ConnectNotifyMessage}s and | ||
4047 | @code{DisconnectNotifyMessage}s whenever peers connect or disconnect from the | ||
4048 | CORE (assuming their type maps overlap with the message types registered by | ||
4049 | the client). When the CORE receives a message that matches the set of message | ||
4050 | types specified during the @code{InitMessage} (or if monitoring is enabled in | ||
4051 | for inbound messages in the options), it sends a @code{NotifyTrafficMessage} | ||
4052 | with the peer identity of the sender and the decrypted payload. The same | ||
4053 | message format (except with @code{GNUNET_MESSAGE_TYPE_CORE_NOTIFY_OUTBOUND} | ||
4054 | for the message type) is used to notify clients monitoring outbound messages; | ||
4055 | here, the peer identity given is that of the receiver. | ||
4056 | |||
4057 | @node Sending | ||
4058 | @subsubsection Sending | ||
4059 | @c %**end of header | ||
4060 | |||
4061 | When a client wants to transmit a message, it first requests a transmission | ||
4062 | slot by sending a @code{SendMessageRequest} which specifies the priority, | ||
4063 | deadline and size of the message. Note that these values may be ignored by | ||
4064 | CORE. When CORE is ready for the message, it answers with a | ||
4065 | @code{SendMessageReady} response. The client can then transmit the payload | ||
4066 | with a @code{SendMessage} message. Note that the actual message size in the | ||
4067 | @code{SendMessage} is allowed to be smaller than the size in the original | ||
4068 | request. A client may at any time send a fresh @code{SendMessageRequest}, | ||
4069 | which then superceeds the previous @code{SendMessageRequest}, which is then no | ||
4070 | longer valid. The client can tell which @code{SendMessageRequest} the CORE | ||
4071 | service's @code{SendMessageReady} message is for as all of these messages | ||
4072 | contain a "unique" request ID (based on a counter incremented by the client | ||
4073 | for each request). | ||
4074 | |||
4075 | @node The CORE Peer-to-Peer Protocol | ||
4076 | @subsection The CORE Peer-to-Peer Protocol | ||
4077 | @c %**end of header | ||
4078 | |||
4079 | |||
4080 | @menu | ||
4081 | * Creating the EphemeralKeyMessage:: | ||
4082 | * Establishing a connection:: | ||
4083 | * Encryption and Decryption:: | ||
4084 | * Type maps:: | ||
4085 | @end menu | ||
4086 | |||
4087 | @node Creating the EphemeralKeyMessage | ||
4088 | @subsubsection Creating the EphemeralKeyMessage | ||
4089 | @c %**end of header | ||
4090 | |||
4091 | When the CORE service starts, each peer creates a fresh ephemeral (ECC) | ||
4092 | public-private key pair and signs the corresponding @code{EphemeralKeyMessage} | ||
4093 | with its long-term key (which we usually call the peer's identity; the hash of | ||
4094 | the public long term key is what results in a @code{struct | ||
4095 | GNUNET_PeerIdentity} in all GNUnet APIs. The ephemeral key is ONLY used for an | ||
4096 | @uref{http://en.wikipedia.org/wiki/Elliptic_curve_Diffie%E2%80%93Hellman, | ||
4097 | ECDHE} exchange by the CORE service to establish symmetric session keys. A | ||
4098 | peer will use the same @code{EphemeralKeyMessage} for all peers for | ||
4099 | @code{REKEY_FREQUENCY}, which is usually 12 hours. After that time, it will | ||
4100 | create a fresh ephemeral key (forgetting the old one) and broadcast the new | ||
4101 | @code{EphemeralKeyMessage} to all connected peers, resulting in fresh | ||
4102 | symmetric session keys. Note that peers independently decide on when to | ||
4103 | discard ephemeral keys; it is not a protocol violation to discard keys more | ||
4104 | often. Ephemeral keys are also never stored to disk; restarting a peer will | ||
4105 | thus always create a fresh ephemeral key. The use of ephemeral keys is what | ||
4106 | provides @uref{http://en.wikipedia.org/wiki/Forward_secrecy, forward secrecy}. | ||
4107 | |||
4108 | Just before transmission, the @code{EphemeralKeyMessage} is patched to reflect | ||
4109 | the current sender_status, which specifies the current state of the connection | ||
4110 | from the point of view of the sender. The possible values are: | ||
4111 | |||
4112 | @table @asis | ||
4113 | @item KX_STATE_DOWN Initial value, never used on the network | ||
4114 | @item KX_STATE_KEY_SENT We sent our ephemeral key, do not know the key of the other | ||
4115 | peer | ||
4116 | @item KX_STATE_KEY_RECEIVED This peer has received a valid ephemeral key | ||
4117 | of the other peer, but we are waiting for the other peer to confirm it's | ||
4118 | authenticity (ability to decode) via challenge-response. | ||
4119 | @item KX_STATE_UP The | ||
4120 | connection is fully up from the point of view of the sender (now performing | ||
4121 | keep-alives) | ||
4122 | @item KX_STATE_REKEY_SENT The sender has initiated a rekeying | ||
4123 | operation; the other peer has so far failed to confirm a working connection | ||
4124 | using the new ephemeral key | ||
4125 | @end table | ||
4126 | |||
4127 | @node Establishing a connection | ||
4128 | @subsubsection Establishing a connection | ||
4129 | @c %**end of header | ||
4130 | |||
4131 | Peers begin their interaction by sending a @code{EphemeralKeyMessage} to the | ||
4132 | other peer once the TRANSPORT service notifies the CORE service about the | ||
4133 | connection. A peer receiving an @code{EphemeralKeyMessage} with a status | ||
4134 | indicating that the sender does not have the receiver's ephemeral key, the | ||
4135 | receiver's @code{EphemeralKeyMessage} is sent in response.@ Additionally, if | ||
4136 | the receiver has not yet confirmed the authenticity of the sender, it also | ||
4137 | sends an (encrypted)@code{PingMessage} with a challenge (and the identity of | ||
4138 | the target) to the other peer. Peers receiving a @code{PingMessage} respond | ||
4139 | with an (encrypted) @code{PongMessage} which includes the challenge. Peers | ||
4140 | receiving a @code{PongMessage} check the challenge, and if it matches set the | ||
4141 | connection to @code{KX_STATE_UP}. | ||
4142 | |||
4143 | @node Encryption and Decryption | ||
4144 | @subsubsection Encryption and Decryption | ||
4145 | @c %**end of header | ||
4146 | |||
4147 | All functions related to the key exchange and encryption/decryption of | ||
4148 | messages can be found in @code{gnunet-service-core_kx.c} (except for the | ||
4149 | cryptographic primitives, which are in @code{util/crypto*.c}).@ Given the key | ||
4150 | material from ECDHE, a | ||
4151 | @uref{http://en.wikipedia.org/wiki/Key_derivation_function, Key derivation | ||
4152 | function} is used to derive two pairs of encryption and decryption keys for | ||
4153 | AES-256 and TwoFish, as well as initialization vectors and authentication keys | ||
4154 | (for @uref{http://en.wikipedia.org/wiki/HMAC, HMAC}). The HMAC is computed | ||
4155 | over the encrypted payload. Encrypted messages include an iv_seed and the HMAC | ||
4156 | in the header. | ||
4157 | |||
4158 | Each encrypted message in the CORE service includes a sequence number and a | ||
4159 | timestamp in the encrypted payload. The CORE service remembers the largest | ||
4160 | observed sequence number and a bit-mask which represents which of the previous | ||
4161 | 32 sequence numbers were already used. Messages with sequence numbers lower | ||
4162 | than the largest observed sequence number minus 32 are discarded. Messages | ||
4163 | with a timestamp that is less than @code{REKEY_TOLERANCE} off (5 minutes) are | ||
4164 | also discarded. This of course means that system clocks need to be reasonably | ||
4165 | synchronized for peers to be able to communicate. Additionally, as the | ||
4166 | ephemeral key changes every 12h, a peer would not even be able to decrypt | ||
4167 | messages older than 12h. | ||
4168 | |||
4169 | @node Type maps | ||
4170 | @subsubsection Type maps | ||
4171 | @c %**end of header | ||
4172 | |||
4173 | Once an encrypted connection has been established, peers begin to exchange | ||
4174 | type maps. Type maps are used to allow the CORE service to determine which | ||
4175 | (encrypted) connections should be shown to which applications. A type map is | ||
4176 | an array of 65536 bits representing the different types of messages understood | ||
4177 | by applications using the CORE service. Each CORE service maintains this map, | ||
4178 | simply by setting the respective bit for each message type supported by any of | ||
4179 | the applications using the CORE service. Note that bits for message types | ||
4180 | embedded in higher-level protocols (such as MESH) will not be included in | ||
4181 | these type maps. | ||
4182 | |||
4183 | Typically, the type map of a peer will be sparse. Thus, the CORE service | ||
4184 | attempts to compress its type map using @code{gzip}-style compression | ||
4185 | ("deflate") prior to transmission. However, if the compression fails to | ||
4186 | compact the map, the map may also be transmitted without compression | ||
4187 | (resulting in @code{GNUNET_MESSAGE_TYPE_CORE_COMPRESSED_TYPE_MAP} or | ||
4188 | @code{GNUNET_MESSAGE_TYPE_CORE_BINARY_TYPE_MAP} messages respectively). Upon | ||
4189 | receiving a type map, the respective CORE service notifies applications about | ||
4190 | the connection to the other peer if they support any message type indicated in | ||
4191 | the type map (or no message type at all). If the CORE service experience a | ||
4192 | connect or disconnect event from an application, it updates its type map | ||
4193 | (setting or unsetting the respective bits) and notifies its neighbours about | ||
4194 | the change. The CORE services of the neighbours then in turn generate connect | ||
4195 | and disconnect events for the peer that sent the type map for their respective | ||
4196 | applications. As CORE messages may be lost, the CORE service confirms | ||
4197 | receiving a type map by sending back a | ||
4198 | @code{GNUNET_MESSAGE_TYPE_CORE_CONFIRM_TYPE_MAP}. If such a confirmation (with | ||
4199 | the correct hash of the type map) is not received, the sender will retransmit | ||
4200 | the type map (with exponential back-off). | ||
4201 | |||
4202 | @node GNUnet's CADET subsystem | ||
4203 | @section GNUnet's CADET subsystem | ||
4204 | |||
4205 | The CADET subsystem in GNUnet is responsible for secure end-to-end | ||
4206 | communications between nodes in the GNUnet overlay network. CADET builds on the | ||
4207 | CORE subsystem which provides for the link-layer communication and then adds | ||
4208 | routing, forwarding and additional security to the connections. CADET offers | ||
4209 | the same cryptographic services as CORE, but on an end-to-end level. This is | ||
4210 | done so peers retransmitting traffic on behalf of other peers cannot access the | ||
4211 | payload data. | ||
4212 | |||
4213 | @itemize @bullet | ||
4214 | @item CADET provides confidentiality with so-called perfect forward secrecy; we | ||
4215 | use ECDHE powered by Curve25519 for the key exchange and then use symmetric | ||
4216 | encryption, encrypting with both AES-256 and Twofish | ||
4217 | @item authentication is achieved by signing the ephemeral keys using Ed25519, a | ||
4218 | deterministic variant of ECDSA | ||
4219 | @item integrity protection (using SHA-512 to do encrypt-then-MAC, although only | ||
4220 | 256 bits are sent to reduce overhead) | ||
4221 | @item replay protection (using nonces, timestamps, challenge-response, message | ||
4222 | counters and ephemeral keys) | ||
4223 | @item liveness (keep-alive messages, timeout) | ||
4224 | @end itemize | ||
4225 | |||
4226 | Additional to the CORE-like security benefits, CADET offers other properties | ||
4227 | that make it a more universal service than CORE. | ||
4228 | |||
4229 | @itemize @bullet | ||
4230 | @item CADET can establish channels to arbitrary peers in GNUnet. If a peer is | ||
4231 | not immediately reachable, CADET will find a path through the network and ask | ||
4232 | other peers to retransmit the traffic on its behalf. | ||
4233 | @item CADET offers (optional) reliability mechanisms. In a reliable channel | ||
4234 | traffic is guaranteed to arrive complete, unchanged and in-order. | ||
4235 | @item CADET takes care of flow and congestion control mechanisms, not allowing | ||
4236 | the sender to send more traffic than the receiver or the network are able to | ||
4237 | process. | ||
4238 | @end itemize | ||
4239 | |||
4240 | @menu | ||
4241 | * libgnunetcadet:: | ||
4242 | @end menu | ||
4243 | |||
4244 | @node libgnunetcadet | ||
4245 | @subsection libgnunetcadet | ||
4246 | |||
4247 | |||
4248 | The CADET API (defined in gnunet_cadet_service.h) is the messaging API used by | ||
4249 | P2P applications built using GNUnet. It provides applications the ability to | ||
4250 | send and receive encrypted messages to any peer participating in GNUnet. The | ||
4251 | API is heavily base on the CORE API. | ||
4252 | |||
4253 | CADET delivers messages to other peers in "channels". A channel is a permanent | ||
4254 | connection defined by a destination peer (identified by its public key) and a | ||
4255 | port number. Internally, CADET tunnels all channels towards a destiantion peer | ||
4256 | using one session key and relays the data on multiple "connections", | ||
4257 | independent from the channels. | ||
4258 | |||
4259 | Each channel has optional paramenters, the most important being the reliability | ||
4260 | flag. Should a message get lost on TRANSPORT/CORE level, if a channel is | ||
4261 | created with as reliable, CADET will retransmit the lost message and deliver it | ||
4262 | in order to the destination application. | ||
4263 | |||
4264 | To communicate with other peers using CADET, it is necessary to first connect | ||
4265 | to the service using @code{GNUNET_CADET_connect}. This function takes several | ||
4266 | parameters in form of callbacks, to allow the client to react to various | ||
4267 | events, like incoming channels or channels that terminate, as well as specify a | ||
4268 | list of ports the client wishes to listen to (at the moment it is not possible | ||
4269 | to start listening on further ports once connected, but nothing prevents a | ||
4270 | client to connect several times to CADET, even do one connection per listening | ||
4271 | port). The function returns a handle which has to be used for any further | ||
4272 | interaction with the service. | ||
4273 | |||
4274 | To connect to a remote peer a client has to call the | ||
4275 | @code{GNUNET_CADET_channel_create} function. The most important parameters | ||
4276 | given are the remote peer's identity (it public key) and a port, which | ||
4277 | specifies which application on the remote peer to connect to, similar to | ||
4278 | TCP/UDP ports. CADET will then find the peer in the GNUnet network and | ||
4279 | establish the proper low-level connections and do the necessary key exchanges | ||
4280 | to assure and authenticated, secure and verified communication. Similar to | ||
4281 | @code{GNUNET_CADET_connect},@code{GNUNET_CADET_create_channel} returns a handle | ||
4282 | to interact with the created channel. | ||
4283 | |||
4284 | For every message the client wants to send to the remote application, | ||
4285 | @code{GNUNET_CADET_notify_transmit_ready} must be called, indicating the | ||
4286 | channel on which the message should be sent and the size of the message (but | ||
4287 | not the message itself!). Once CADET is ready to send the message, the provided | ||
4288 | callback will fire, and the message contents are provided to this callback. | ||
4289 | |||
4290 | Please note the CADET does not provide an explicit notification of when a | ||
4291 | channel is connected. In loosely connected networks, like big wireless mesh | ||
4292 | networks, this can take several seconds, even minutes in the worst case. To be | ||
4293 | alerted when a channel is online, a client can call | ||
4294 | @code{GNUNET_CADET_notify_transmit_ready} immediately after | ||
4295 | @code{GNUNET_CADET_create_channel}. When the callback is activated, it means | ||
4296 | that the channel is online. The callback can give 0 bytes to CADET if no | ||
4297 | message is to be sent, this is ok. | ||
4298 | |||
4299 | If a transmission was requested but before the callback fires it is no longer | ||
4300 | needed, it can be cancelled with | ||
4301 | @code{GNUNET_CADET_notify_transmit_ready_cancel}, which uses the handle given | ||
4302 | back by @code{GNUNET_CADET_notify_transmit_ready}. As in the case of CORE, only | ||
4303 | one message can be requested at a time: a client must not call | ||
4304 | @code{GNUNET_CADET_notify_transmit_ready} again until the callback is called or | ||
4305 | the request is cancelled. | ||
4306 | |||
4307 | When a channel is no longer needed, a client can call | ||
4308 | @code{GNUNET_CADET_channel_destroy} to get rid of it. Note that CADET will try | ||
4309 | to transmit all pending traffic before notifying the remote peer of the | ||
4310 | destruction of the channel, including retransmitting lost messages if the | ||
4311 | channel was reliable. | ||
4312 | |||
4313 | Incoming channels, channels being closed by the remote peer, and traffic on any | ||
4314 | incoming or outgoing channels are given to the client when CADET executes the | ||
4315 | callbacks given to it at the time of @code{GNUNET_CADET_connect}. | ||
4316 | |||
4317 | Finally, when an application no longer wants to use CADET, it should call | ||
4318 | @code{GNUNET_CADET_disconnect}, but first all channels and pending | ||
4319 | transmissions must be closed (otherwise CADET will complain). | ||
4320 | |||
4321 | @node GNUnet's NSE subsystem | ||
4322 | @section GNUnet's NSE subsystem | ||
4323 | |||
4324 | |||
4325 | NSE stands for Network Size Estimation. The NSE subsystem provides other | ||
4326 | subsystems and users with a rough estimate of the number of peers currently | ||
4327 | participating in the GNUnet overlay. The computed value is not a precise number | ||
4328 | as producing a precise number in a decentralized, efficient and secure way is | ||
4329 | impossible. While NSE's estimate is inherently imprecise, NSE also gives the | ||
4330 | expected range. For a peer that has been running in a stable network for a | ||
4331 | while, the real network size will typically (99.7% of the time) be in the range | ||
4332 | of [2/3 estimate, 3/2 estimate]. We will now give an overview of the algorithm | ||
4333 | used to calcualte the estimate; all of the details can be found in this | ||
4334 | technical report. | ||
4335 | |||
4336 | @menu | ||
4337 | * Motivation:: | ||
4338 | * Principle:: | ||
4339 | * libgnunetnse:: | ||
4340 | * The NSE Client-Service Protocol:: | ||
4341 | * The NSE Peer-to-Peer Protocol:: | ||
4342 | @end menu | ||
4343 | |||
4344 | @node Motivation | ||
4345 | @subsection Motivation | ||
4346 | |||
4347 | |||
4348 | Some subsytems, like DHT, need to know the size of the GNUnet network to | ||
4349 | optimize some parameters of their own protocol. The decentralized nature of | ||
4350 | GNUnet makes efficient and securely counting the exact number of peers | ||
4351 | infeasable. Although there are several decentralized algorithms to count the | ||
4352 | number of peers in a system, so far there is none to do so securely. Other | ||
4353 | protocols may allow any malicious peer to manipulate the final result or to | ||
4354 | take advantage of the system to perform DoS (Denial of Service) attacks against | ||
4355 | the network. GNUnet's NSE protocol avoids these drawbacks. | ||
4356 | |||
4357 | |||
4358 | |||
4359 | @menu | ||
4360 | * Security:: | ||
4361 | @end menu | ||
4362 | |||
4363 | @node Security | ||
4364 | @subsubsection Security | ||
4365 | |||
4366 | |||
4367 | The NSE subsystem is designed to be resilient against these attacks. It uses | ||
4368 | @uref{http://en.wikipedia.org/wiki/Proof-of-work_system, proofs of work} to | ||
4369 | prevent one peer from impersonating a large number of participants, which would | ||
4370 | otherwise allow an adversary to artifically inflate the estimate. The DoS | ||
4371 | protection comes from the time-based nature of the protocol: the estimates are | ||
4372 | calculated periodically and out-of-time traffic is either ignored or stored for | ||
4373 | later retransmission by benign peers. In particular, peers cannot trigger | ||
4374 | global network communication at will. | ||
4375 | |||
4376 | @node Principle | ||
4377 | @subsection Principle | ||
4378 | |||
4379 | |||
4380 | The algorithm calculates the estimate by finding the globally closest peer ID | ||
4381 | to a random, time-based value. | ||
4382 | |||
4383 | The idea is that the closer the ID is to the random value, the more "densely | ||
4384 | packed" the ID space is, and therefore, more peers are in the network. | ||
4385 | |||
4386 | |||
4387 | |||
4388 | @menu | ||
4389 | * Example:: | ||
4390 | * Algorithm:: | ||
4391 | * Target value:: | ||
4392 | * Timing:: | ||
4393 | * Controlled Flooding:: | ||
4394 | * Calculating the estimate:: | ||
4395 | @end menu | ||
4396 | |||
4397 | @node Example | ||
4398 | @subsubsection Example | ||
4399 | |||
4400 | |||
4401 | Suppose all peers have IDs between 0 and 100 (our ID space), and the random | ||
4402 | value is 42. If the closest peer has the ID 70 we can imagine that the average | ||
4403 | "distance" between peers is around 30 and therefore the are around 3 peers in | ||
4404 | the whole ID space. On the other hand, if the closest peer has the ID 44, we | ||
4405 | can imagine that the space is rather packed with peers, maybe as much as 50 of | ||
4406 | them. Naturally, we could have been rather unlucky, and there is only one peer | ||
4407 | and happens to have the ID 44. Thus, the current estimate is calculated as the | ||
4408 | average over multiple rounds, and not just a single sample. | ||
4409 | |||
4410 | @node Algorithm | ||
4411 | @subsubsection Algorithm | ||
4412 | |||
4413 | |||
4414 | Given that example, one can imagine that the job of the subsystem is to | ||
4415 | efficiently communicate the ID of the closest peer to the target value to all | ||
4416 | the other peers, who will calculate the estimate from it. | ||
4417 | |||
4418 | @node Target value | ||
4419 | @subsubsection Target value | ||
4420 | |||
4421 | @c %**end of header | ||
4422 | |||
4423 | The target value itself is generated by hashing the current time, rounded down | ||
4424 | to an agreed value. If the rounding amount is 1h (default) and the time is | ||
4425 | 12:34:56, the time to hash would be 12:00:00. The process is repeated each | ||
4426 | rouning amount (in this example would be every hour). Every repetition is | ||
4427 | called a round. | ||
4428 | |||
4429 | @node Timing | ||
4430 | @subsubsection Timing | ||
4431 | @c %**end of header | ||
4432 | |||
4433 | The NSE subsystem has some timing control to avoid everybody broadcasting its | ||
4434 | ID all at one. Once each peer has the target random value, it compares its own | ||
4435 | ID to the target and calculates the hypothetical size of the network if that | ||
4436 | peer were to be the closest. Then it compares the hypothetical size with the | ||
4437 | estimate from the previous rounds. For each value there is an assiciated point | ||
4438 | in the period, let's call it "broadcast time". If its own hypothetical estimate | ||
4439 | is the same as the previous global estimate, its "broadcast time" will be in | ||
4440 | the middle of the round. If its bigger it will be earlier and if its smaler | ||
4441 | (the most likely case) it will be later. This ensures that the peers closests | ||
4442 | to the target value start broadcasting their ID the first. | ||
4443 | |||
4444 | @node Controlled Flooding | ||
4445 | @subsubsection Controlled Flooding | ||
4446 | |||
4447 | @c %**end of header | ||
4448 | |||
4449 | When a peer receives a value, first it verifies that it is closer than the | ||
4450 | closest value it had so far, otherwise it answers the incoming message with a | ||
4451 | message containing the better value. Then it checks a proof of work that must | ||
4452 | be included in the incoming message, to ensure that the other peer's ID is not | ||
4453 | made up (otherwise a malicious peer could claim to have an ID of exactly the | ||
4454 | target value every round). Once validated, it compares the brodcast time of the | ||
4455 | received value with the current time and if it's not too early, sends the | ||
4456 | received value to its neighbors. Otherwise it stores the value until the | ||
4457 | correct broadcast time comes. This prevents unnecessary traffic of sub-optimal | ||
4458 | values, since a better value can come before the broadcast time, rendering the | ||
4459 | previous one obsolete and saving the traffic that would have been used to | ||
4460 | broadcast it to the neighbors. | ||
4461 | |||
4462 | @node Calculating the estimate | ||
4463 | @subsubsection Calculating the estimate | ||
4464 | |||
4465 | @c %**end of header | ||
4466 | |||
4467 | Once the closest ID has been spread across the network each peer gets the exact | ||
4468 | distance betweed this ID and the target value of the round and calculates the | ||
4469 | estimate with a mathematical formula described in the tech report. The estimate | ||
4470 | generated with this method for a single round is not very precise. Remember the | ||
4471 | case of the example, where the only peer is the ID 44 and we happen to generate | ||
4472 | the target value 42, thinking there are 50 peers in the network. Therefore, the | ||
4473 | NSE subsystem remembers the last 64 estimates and calculates an average over | ||
4474 | them, giving a result of which usually has one bit of uncertainty (the real | ||
4475 | size could be half of the estimate or twice as much). Note that the actual | ||
4476 | network size is calculated in powers of two of the raw input, thus one bit of | ||
4477 | uncertainty means a factor of two in the size estimate. | ||
4478 | |||
4479 | @node libgnunetnse | ||
4480 | @subsection libgnunetnse | ||
4481 | |||
4482 | @c %**end of header | ||
4483 | |||
4484 | The NSE subsystem has the simplest API of all services, with only two calls: | ||
4485 | @code{GNUNET_NSE_connect} and @code{GNUNET_NSE_disconnect}. | ||
4486 | |||
4487 | The connect call gets a callback function as a parameter and this function is | ||
4488 | called each time the network agrees on an estimate. This usually is once per | ||
4489 | round, with some exceptions: if the closest peer has a late local clock and | ||
4490 | starts spreading his ID after everyone else agreed on a value, the callback | ||
4491 | might be activated twice in a round, the second value being always bigger than | ||
4492 | the first. The default round time is set to 1 hour. | ||
4493 | |||
4494 | The disconnect call disconnects from the NSE subsystem and the callback is no | ||
4495 | longer called with new estimates. | ||
4496 | |||
4497 | |||
4498 | |||
4499 | @menu | ||
4500 | * Results:: | ||
4501 | * Examples2:: | ||
4502 | @end menu | ||
4503 | |||
4504 | @node Results | ||
4505 | @subsubsection Results | ||
4506 | |||
4507 | @c %**end of header | ||
4508 | |||
4509 | The callback provides two values: the average and the | ||
4510 | @uref{http://en.wikipedia.org/wiki/Standard_deviation, standard deviation} of | ||
4511 | the last 64 rounds. The values provided by the callback function are | ||
4512 | logarithmic, this means that the real estimate numbers can be obtained by | ||
4513 | calculating 2 to the power of the given value (2average). From a statistics | ||
4514 | point of view this means that: | ||
4515 | |||
4516 | @itemize @bullet | ||
4517 | @item 68% of the time the real size is included in the interval | ||
4518 | [(2average-stddev), 2] | ||
4519 | @item 95% of the time the real size is included in the interval | ||
4520 | [(2average-2*stddev, 2^average+2*stddev] | ||
4521 | @item 99.7% of the time the real size is included in the interval | ||
4522 | [(2average-3*stddev, 2average+3*stddev] | ||
4523 | @end itemize | ||
4524 | |||
4525 | The expected standard variation for 64 rounds in a network of stable size is | ||
4526 | 0.2. Thus, we can say that normally: | ||
4527 | |||
4528 | @itemize @bullet | ||
4529 | @item 68% of the time the real size is in the range [-13%, +15%] | ||
4530 | @item 95% of the time the real size is in the range [-24%, +32%] | ||
4531 | @item 99.7% of the time the real size is in the range [-34%, +52%] | ||
4532 | @end itemize | ||
4533 | |||
4534 | As said in the introduction, we can be quite sure that usually the real size is | ||
4535 | between one third and three times the estimate. This can of course vary with | ||
4536 | network conditions. Thus, applications may want to also consider the provided | ||
4537 | standard deviation value, not only the average (in particular, if the standard | ||
4538 | veriation is very high, the average maybe meaningless: the network size is | ||
4539 | changing rapidly). | ||
4540 | |||
4541 | @node Examples2 | ||
4542 | @subsubsection Examples2 | ||
4543 | |||
4544 | @c %**end of header | ||
4545 | |||
4546 | Let's close with a couple examples. | ||
4547 | |||
4548 | @table @asis | ||
4549 | |||
4550 | @item Average: 10, std dev: 1 Here the estimate would be 2^10 = 1024 peers.@ | ||
4551 | The range in which we can be 95% sure is: [2^8, 2^12] = [256, 4096]. We can be | ||
4552 | very (>99.7%) sure that the network is not a hundred peers and absolutely sure | ||
4553 | that it is not a million peers, but somewhere around a thousand. | ||
4554 | |||
4555 | @item Average 22, std dev: 0.2 Here the estimate would be 2^22 = 4 Million peers.@ | ||
4556 | The range in which we can be 99.7% sure is: [2^21.4, 2^22.6] = [2.8M, 6.3M]. | ||
4557 | We can be sure that the network size is around four million, with absolutely | ||
4558 | way of it being 1 million. | ||
4559 | |||
4560 | @end table | ||
4561 | |||
4562 | To put this in perspective, if someone remembers the LHC Higgs boson results, | ||
4563 | were announced with "5 sigma" and "6 sigma" certainties. In this case a 5 sigma | ||
4564 | minimum would be 2 million and a 6 sigma minimum, 1.8 million. | ||
4565 | |||
4566 | @node The NSE Client-Service Protocol | ||
4567 | @subsection The NSE Client-Service Protocol | ||
4568 | |||
4569 | @c %**end of header | ||
4570 | |||
4571 | As with the API, the client-service protocol is very simple, only has 2 | ||
4572 | different messages, defined in @code{src/nse/nse.h}: | ||
4573 | |||
4574 | @itemize @bullet | ||
4575 | @item @code{GNUNET_MESSAGE_TYPE_NSE_START}@ This message has no parameters and | ||
4576 | is sent from the client to the service upon connection. | ||
4577 | @item @code{GNUNET_MESSAGE_TYPE_NSE_ESTIMATE}@ This message is sent from the | ||
4578 | service to the client for every new estimate and upon connection. Contains a | ||
4579 | timestamp for the estimate, the average and the standard deviation for the | ||
4580 | respective round. | ||
4581 | @end itemize | ||
4582 | |||
4583 | When the @code{GNUNET_NSE_disconnect} API call is executed, the client simply | ||
4584 | disconnects from the service, with no message involved. | ||
4585 | |||
4586 | @node The NSE Peer-to-Peer Protocol | ||
4587 | @subsection The NSE Peer-to-Peer Protocol | ||
4588 | |||
4589 | @c %**end of header | ||
4590 | |||
4591 | The NSE subsystem only has one message in the P2P protocol, the | ||
4592 | @code{GNUNET_MESSAGE_TYPE_NSE_P2P_FLOOD} message. | ||
4593 | |||
4594 | This message key contents are the timestamp to identify the round (differences | ||
4595 | in system clocks may cause some peers to send messages way too early or way too | ||
4596 | late, so the timestamp allows other peers to identify such messages easily), | ||
4597 | the @uref{http://en.wikipedia.org/wiki/Proof-of-work_system, proof of work} | ||
4598 | used to make it difficult to mount a | ||
4599 | @uref{http://en.wikipedia.org/wiki/Sybil_attack, Sybil attack}, and the public | ||
4600 | key, which is used to verify the signature on the message. | ||
4601 | |||
4602 | Every peer stores a message for the previous, current and next round. The | ||
4603 | messages for the previous and current round are given to peers that connect to | ||
4604 | us. The message for the next round is simply stored until our system clock | ||
4605 | advances to the next round. The message for the current round is what we are | ||
4606 | flooding the network with right now. At the beginning of each round the peer | ||
4607 | does the following: | ||
4608 | |||
4609 | @itemize @bullet | ||
4610 | @item calculates his own distance to the target value | ||
4611 | @item creates, signs and stores the message for the current round (unless it | ||
4612 | has a better message in the "next round" slot which came early in the previous | ||
4613 | round) | ||
4614 | @item calculates, based on the stored round message (own or received) when to | ||
4615 | stard flooding it to its neighbors | ||
4616 | @end itemize | ||
4617 | |||
4618 | Upon receiving a message the peer checks the validity of the message (round, | ||
4619 | proof of work, signature). The next action depends on the contents of the | ||
4620 | incoming message: | ||
4621 | |||
4622 | @itemize @bullet | ||
4623 | @item if the message is worse than the current stored message, the peer sends | ||
4624 | the current message back immediately, to stop the other peer from spreading | ||
4625 | suboptimal results | ||
4626 | @item if the message is better than the current stored message, the peer stores | ||
4627 | the new message and calculates the new target time to start spreading it to its | ||
4628 | neighbors (excluding the one the message came from) | ||
4629 | @item if the message is for the previous round, it is compared to the message | ||
4630 | stored in the "previous round slot", which may then be updated | ||
4631 | @item if the message is for the next round, it is compared to the message | ||
4632 | stored in the "next round slot", which again may then be updated | ||
4633 | @end itemize | ||
4634 | |||
4635 | Finally, when it comes to send the stored message for the current round to the | ||
4636 | neighbors there is a random delay added for each neighbor, to avoid traffic | ||
4637 | spikes and minimize cross-messages. | ||
4638 | |||
4639 | @node GNUnet's HOSTLIST subsystem | ||
4640 | @section GNUnet's HOSTLIST subsystem | ||
4641 | |||
4642 | @c %**end of header | ||
4643 | |||
4644 | Peers in the GNUnet overlay network need address information so that they can | ||
4645 | connect with other peers. GNUnet uses so called HELLO messages to store and | ||
4646 | exchange peer addresses. GNUnet provides several methods for peers to obtain | ||
4647 | this information: | ||
4648 | |||
4649 | @itemize @bullet | ||
4650 | @item out-of-band exchange of HELLO messages (manually, using for example | ||
4651 | gnunet-peerinfo) | ||
4652 | @item HELLO messages shipped with GNUnet (automatic with distribution) | ||
4653 | @item UDP neighbor discovery in LAN (IPv4 broadcast, IPv6 multicast) | ||
4654 | @item topology gossiping (learning from other peers we already connected to), | ||
4655 | and | ||
4656 | @item the HOSTLIST daemon covered in this section, which is particularly | ||
4657 | relevant for bootstrapping new peers. | ||
4658 | @end itemize | ||
4659 | |||
4660 | New peers have no existing connections (and thus cannot learn from gossip among | ||
4661 | peers), may not have other peers in their LAN and might be started with an | ||
4662 | outdated set of HELLO messages from the distribution. In this case, getting new | ||
4663 | peers to connect to the network requires either manual effort or the use of a | ||
4664 | HOSTLIST to obtain HELLOs. | ||
4665 | |||
4666 | @menu | ||
4667 | * HELLOs:: | ||
4668 | * Overview for the HOSTLIST subsystem:: | ||
4669 | * Interacting with the HOSTLIST daemon:: | ||
4670 | * Hostlist security address validation:: | ||
4671 | * The HOSTLIST daemon:: | ||
4672 | * The HOSTLIST server:: | ||
4673 | * The HOSTLIST client:: | ||
4674 | * Usage:: | ||
4675 | @end menu | ||
4676 | |||
4677 | @node HELLOs | ||
4678 | @subsection HELLOs | ||
4679 | |||
4680 | @c %**end of header | ||
4681 | |||
4682 | The basic information peers require to connect to other peers are contained in | ||
4683 | so called HELLO messages you can think of as a business card. Besides the | ||
4684 | identity of the peer (based on the cryptographic public key) a HELLO message | ||
4685 | may contain address information that specifies ways to contact a peer. By | ||
4686 | obtaining HELLO messages, a peer can learn how to contact other peers. | ||
4687 | |||
4688 | @node Overview for the HOSTLIST subsystem | ||
4689 | @subsection Overview for the HOSTLIST subsystem | ||
4690 | |||
4691 | @c %**end of header | ||
4692 | |||
4693 | The HOSTLIST subsystem provides a way to distribute and obtain contact | ||
4694 | information to connect to other peers using a simple HTTP GET request. It's | ||
4695 | implementation is split in three parts, the main file for the daemon itself | ||
4696 | (gnunet-daemon-hostlist.c), the HTTP client used to download peer information | ||
4697 | (hostlist-client.c) and the server component used to provide this information | ||
4698 | to other peers (hostlist-server.c). The server is basically a small HTTP web | ||
4699 | server (based on GNU libmicrohttpd) which provides a list of HELLOs known to | ||
4700 | the local peer for download. The client component is basically a HTTP client | ||
4701 | (based on libcurl) which can download hostlists from one or more websites. The | ||
4702 | hostlist format is a binary blob containing a sequence of HELLO messages. Note | ||
4703 | that any HTTP server can theoretically serve a hostlist, the build-in hostlist | ||
4704 | server makes it simply convenient to offer this service. | ||
4705 | |||
4706 | |||
4707 | @menu | ||
4708 | * Features:: | ||
4709 | * Limitations2:: | ||
4710 | @end menu | ||
4711 | |||
4712 | @node Features | ||
4713 | @subsubsection Features | ||
4714 | |||
4715 | @c %**end of header | ||
4716 | |||
4717 | The HOSTLIST daemon can: | ||
4718 | |||
4719 | @itemize @bullet | ||
4720 | @item provide HELLO messages with validated addresses obtained from PEERINFO to | ||
4721 | download for other peers | ||
4722 | @item download HELLO messages and forward these message to the TRANSPORT | ||
4723 | subsystem for validation | ||
4724 | @item advertises the URL of this peer's hostlist address to other peers via | ||
4725 | gossip | ||
4726 | @item automatically learn about hostlist servers from the gossip of other peers | ||
4727 | @end itemize | ||
4728 | |||
4729 | @node Limitations2 | ||
4730 | @subsubsection Limitations2 | ||
4731 | |||
4732 | @c %**end of header | ||
4733 | |||
4734 | The HOSTLIST daemon does not: | ||
4735 | |||
4736 | @itemize @bullet | ||
4737 | @item verify the cryptographic information in the HELLO messages | ||
4738 | @item verify the address information in the HELLO messages | ||
4739 | @end itemize | ||
4740 | |||
4741 | @node Interacting with the HOSTLIST daemon | ||
4742 | @subsection Interacting with the HOSTLIST daemon | ||
4743 | |||
4744 | @c %**end of header | ||
4745 | |||
4746 | The HOSTLIST subsystem is currently implemented as a daemon, so there is no | ||
4747 | need for the user to interact with it and therefore there is no command line | ||
4748 | tool and no API to communicate with the daemon. In the future, we can envision | ||
4749 | changing this to allow users to manually trigger the download of a hostlist. | ||
4750 | |||
4751 | Since there is no command line interface to interact with HOSTLIST, the only | ||
4752 | way to interact with the hostlist is to use STATISTICS to obtain or modify | ||
4753 | information about the status of HOSTLIST: | ||
4754 | @example | ||
4755 | $ gnunet-statistics -s hostlist | ||
4756 | @end example | ||
4757 | |||
4758 | In particular, HOSTLIST includes a @strong{persistent} value in statistics that | ||
4759 | specifies when the hostlist server might be queried next. As this value is | ||
4760 | exponentially increasing during runtime, developers may want to reset or | ||
4761 | manually adjust it. Note that HOSTLIST (but not STATISTICS) needs to be | ||
4762 | shutdown if changes to this value are to have any effect on the daemon (as | ||
4763 | HOSTLIST does not monitor STATISTICS for changes to the download | ||
4764 | frequency). | ||
4765 | |||
4766 | @node Hostlist security address validation | ||
4767 | @subsection Hostlist security address validation | ||
4768 | |||
4769 | @c %**end of header | ||
4770 | |||
4771 | Since information obtained from other parties cannot be trusted without | ||
4772 | validation, we have to distinguish between @emph{validated} and @emph{not | ||
4773 | validated} addresses. Before using (and so trusting) information from other | ||
4774 | parties, this information has to be double-checked (validated). Address | ||
4775 | validation is not done by HOSTLIST but by the TRANSPORT service. | ||
4776 | |||
4777 | The HOSTLIST component is functionally located between the PEERINFO and the | ||
4778 | TRANSPORT subsystem. When acting as a server, the daemon obtains valid | ||
4779 | (@emph{validated}) peer information (HELLO messages) from the PEERINFO service | ||
4780 | and provides it to other peers. When acting as a client, it contacts the | ||
4781 | HOSTLIST servers specified in the configuration, downloads the (unvalidated) | ||
4782 | list of HELLO messages and forwards these information to the TRANSPORT server | ||
4783 | to validate the addresses. | ||
4784 | |||
4785 | @node The HOSTLIST daemon | ||
4786 | @subsection The HOSTLIST daemon | ||
4787 | |||
4788 | @c %**end of header | ||
4789 | |||
4790 | The hostlist daemon is the main component of the HOSTLIST subsystem. It is | ||
4791 | started by the ARM service and (if configured) starts the HOSTLIST client and | ||
4792 | server components. | ||
4793 | |||
4794 | If the daemon provides a hostlist itself it can advertise it's own hostlist to | ||
4795 | other peers. To do so it sends a GNUNET_MESSAGE_TYPE_HOSTLIST_ADVERTISEMENT | ||
4796 | message to other peers when they connect to this peer on the CORE level. This | ||
4797 | hostlist advertisement message contains the URL to access the HOSTLIST HTTP | ||
4798 | server of the sender. The daemon may also subscribe to this type of message | ||
4799 | from CORE service, and then forward these kind of message to the HOSTLIST | ||
4800 | client. The client then uses all available URLs to download peer information | ||
4801 | when necessary. | ||
4802 | |||
4803 | When starting, the HOSTLIST daemon first connects to the CORE subsystem and if | ||
4804 | hostlist learning is enabled, registers a CORE handler to receive this kind of | ||
4805 | messages. Next it starts (if configured) the client and server. It passes | ||
4806 | pointers to CORE connect and disconnect and receive handlers where the client | ||
4807 | and server store their functions, so the daemon can notify them about CORE | ||
4808 | events. | ||
4809 | |||
4810 | To clean up on shutdown, the daemon has a cleaning task, shutting down all | ||
4811 | subsystems and disconnecting from CORE. | ||
4812 | |||
4813 | @node The HOSTLIST server | ||
4814 | @subsection The HOSTLIST server | ||
4815 | |||
4816 | @c %**end of header | ||
4817 | |||
4818 | The server provides a way for other peers to obtain HELLOs. Basically it is a | ||
4819 | small web server other peers can connect to and download a list of HELLOs using | ||
4820 | standard HTTP; it may also advertise the URL of the hostlist to other peers | ||
4821 | connecting on CORE level. | ||
4822 | |||
4823 | |||
4824 | @menu | ||
4825 | * The HTTP Server:: | ||
4826 | * Advertising the URL:: | ||
4827 | @end menu | ||
4828 | |||
4829 | @node The HTTP Server | ||
4830 | @subsubsection The HTTP Server | ||
4831 | |||
4832 | @c %**end of header | ||
4833 | |||
4834 | During startup, the server starts a web server listening on the port specified | ||
4835 | with the HTTPPORT value (default 8080). In addition it connects to the PEERINFO | ||
4836 | service to obtain peer information. The HOSTLIST server uses the | ||
4837 | GNUNET_PEERINFO_iterate function to request HELLO information for all peers and | ||
4838 | adds their information to a new hostlist if they are suitable (expired | ||
4839 | addresses and HELLOs without addresses are both not suitable) and the maximum | ||
4840 | size for a hostlist is not exceeded (MAX_BYTES_PER_HOSTLISTS = 500000). When | ||
4841 | PEERINFO finishes (with a last NULL callback), the server destroys the previous | ||
4842 | hostlist response available for download on the web server and replaces it with | ||
4843 | the updated hostlist. The hostlist format is basically a sequence of HELLO | ||
4844 | messages (as obtained from PEERINFO) without any special tokenization. Since | ||
4845 | each HELLO message contains a size field, the response can easily be split into | ||
4846 | separate HELLO messages by the client. | ||
4847 | |||
4848 | A HOSTLIST client connecting to the HOSTLIST server will receive the hostlist | ||
4849 | as a HTTP response and the the server will terminate the connection with the | ||
4850 | result code HTTP 200 OK. The connection will be closed immediately if no | ||
4851 | hostlist is available. | ||
4852 | |||
4853 | @node Advertising the URL | ||
4854 | @subsubsection Advertising the URL | ||
4855 | |||
4856 | @c %**end of header | ||
4857 | |||
4858 | The server also advertises the URL to download the hostlist to other peers if | ||
4859 | hostlist advertisement is enabled. When a new peer connects and has hostlist | ||
4860 | learning enabled, the server sends a GNUNET_MESSAGE_TYPE_HOSTLIST_ADVERTISEMENT | ||
4861 | message to this peer using the CORE service. | ||
4862 | |||
4863 | @node The HOSTLIST client | ||
4864 | @subsection The HOSTLIST client | ||
4865 | |||
4866 | @c %**end of header | ||
4867 | |||
4868 | The client provides the functionality to download the list of HELLOs from a set | ||
4869 | of URLs. It performs a standard HTTP request to the URLs configured and learned | ||
4870 | from advertisement messages received from other peers. When a HELLO is | ||
4871 | downloaded, the HOSTLIST client forwards the HELLO to the TRANSPORT service for | ||
4872 | validation. | ||
4873 | |||
4874 | The client supports two modes of operation: download of HELLOs (bootstrapping) | ||
4875 | and learning of URLs. | ||
4876 | |||
4877 | |||
4878 | @menu | ||
4879 | * Bootstrapping:: | ||
4880 | * Learning:: | ||
4881 | @end menu | ||
4882 | |||
4883 | @node Bootstrapping | ||
4884 | @subsubsection Bootstrapping | ||
4885 | |||
4886 | @c %**end of header | ||
4887 | |||
4888 | For bootstrapping, it schedules a task to download the hostlist from the set of | ||
4889 | known URLs. The downloads are only performed if the number of current | ||
4890 | connections is smaller than a minimum number of connections (at the moment 4). | ||
4891 | The interval between downloads increases exponentially; however, the | ||
4892 | exponential growth is limited if it becomes longer than an hour. At that point, | ||
4893 | the frequency growth is capped at (#number of connections * 1h). | ||
4894 | |||
4895 | Once the decision has been taken to download HELLOs, the daemon chooses a | ||
4896 | random URL from the list of known URLs. URLs can be configured in the | ||
4897 | configuration or be learned from advertisement messages. The client uses a HTTP | ||
4898 | client library (libcurl) to initiate the download using the libcurl multi | ||
4899 | interface. Libcurl passes the data to the callback_download function which | ||
4900 | stores the data in a buffer if space is available and the maximum size for a | ||
4901 | hostlist download is not exceeded (MAX_BYTES_PER_HOSTLISTS = 500000). When a | ||
4902 | full HELLO was downloaded, the HOSTLIST client offers this HELLO message to the | ||
4903 | TRANSPORT service for validation. When the download is finished or failed, | ||
4904 | statistical information about the quality of this URL is updated. | ||
4905 | |||
4906 | @node Learning | ||
4907 | @subsubsection Learning | ||
4908 | |||
4909 | @c %**end of header | ||
4910 | |||
4911 | The client also manages hostlist advertisements from other peers. The HOSTLIST | ||
4912 | daemon forwards GNUNET_MESSAGE_TYPE_HOSTLIST_ADVERTISEMENT messages to the | ||
4913 | client subsystem, which extracts the URL from the message. Next, a test of the | ||
4914 | newly obtained URL is performed by triggering a download from the new URL. If | ||
4915 | the URL works correctly, it is added to the list of working URLs. | ||
4916 | |||
4917 | The size of the list of URLs is restricted, so if an additional server is added | ||
4918 | and the list is full, the URL with the worst quality ranking (determined | ||
4919 | through successful downloads and number of HELLOs e.g.) is discarded. During | ||
4920 | shutdown the list of URLs is saved to a file for persistance and loaded on | ||
4921 | startup. URLs from the configuration file are never discarded. | ||
4922 | |||
4923 | @node Usage | ||
4924 | @subsection Usage | ||
4925 | |||
4926 | @c %**end of header | ||
4927 | |||
4928 | To start HOSTLIST by default, it has to be added to the DEFAULTSERVICES section | ||
4929 | for the ARM services. This is done in the default configuration. | ||
4930 | |||
4931 | For more information on how to configure the HOSTLIST subsystem see the | ||
4932 | installation handbook:@ Configuring the hostlist to bootstrap@ Configuring your | ||
4933 | peer to provide a hostlist | ||
4934 | |||
4935 | @node GNUnet's IDENTITY subsystem | ||
4936 | @section GNUnet's IDENTITY subsystem | ||
4937 | |||
4938 | @c %**end of header | ||
4939 | |||
4940 | Identities of "users" in GNUnet are called egos. Egos can be used as pseudonyms | ||
4941 | (fake names) or be tied to an organization (for example, GNU) or even the | ||
4942 | actual identity of a human. GNUnet users are expected to have many egos. They | ||
4943 | might have one tied to their real identity, some for organizations they manage, | ||
4944 | and more for different domains where they want to operate under a pseudonym. | ||
4945 | |||
4946 | The IDENTITY service allows users to manage their egos. The identity service | ||
4947 | manages the private keys egos of the local user; it does not manage identities | ||
4948 | of other users (public keys). Public keys for other users need names to become | ||
4949 | manageable. GNUnet uses the GNU Name System (GNS) to give names to other users | ||
4950 | and manage their public keys securely. This chapter is about the IDENTITY | ||
4951 | service, which is about the management of private keys. | ||
4952 | |||
4953 | On the network, an ego corresponds to an ECDSA key (over Curve25519, using RFC | ||
4954 | 6979, as required by GNS). Thus, users can perform actions under a particular | ||
4955 | ego by using (signing with) a particular private key. Other users can then | ||
4956 | confirm that the action was really performed by that ego by checking the | ||
4957 | signature against the respective public key. | ||
4958 | |||
4959 | The IDENTITY service allows users to associate a human-readable name with each | ||
4960 | ego. This way, users can use names that will remind them of the purpose of a | ||
4961 | particular ego. The IDENTITY service will store the respective private keys and | ||
4962 | allows applications to access key information by name. Users can change the | ||
4963 | name that is locally (!) associated with an ego. Egos can also be deleted, | ||
4964 | which means that the private key will be removed and it thus will not be | ||
4965 | possible to perform actions with that ego in the future. | ||
4966 | |||
4967 | Additionally, the IDENTITY subsystem can associate service functions with egos. | ||
4968 | For example, GNS requires the ego that should be used for the shorten zone. GNS | ||
4969 | will ask IDENTITY for an ego for the "gns-short" service. The IDENTITY service | ||
4970 | has a mapping of such service strings to the name of the ego that the user | ||
4971 | wants to use for this service, for example "my-short-zone-ego". | ||
4972 | |||
4973 | Finally, the IDENTITY API provides access to a special ego, the anonymous ego. | ||
4974 | The anonymous ego is special in that its private key is not really private, but | ||
4975 | fixed and known to everyone. Thus, anyone can perform actions as anonymous. | ||
4976 | This can be useful as with this trick, code does not have to contain a special | ||
4977 | case to distinguish between anonymous and pseudonymous egos. | ||
4978 | |||
4979 | @menu | ||
4980 | * libgnunetidentity:: | ||
4981 | * The IDENTITY Client-Service Protocol:: | ||
4982 | @end menu | ||
4983 | |||
4984 | @node libgnunetidentity | ||
4985 | @subsection libgnunetidentity | ||
4986 | @c %**end of header | ||
4987 | |||
4988 | |||
4989 | @menu | ||
4990 | * Connecting to the service:: | ||
4991 | * Operations on Egos:: | ||
4992 | * The anonymous Ego:: | ||
4993 | * Convenience API to lookup a single ego:: | ||
4994 | * Associating egos with service functions:: | ||
4995 | @end menu | ||
4996 | |||
4997 | @node Connecting to the service | ||
4998 | @subsubsection Connecting to the service | ||
4999 | |||
5000 | @c %**end of header | ||
5001 | |||
5002 | First, typical clients connect to the identity service using | ||
5003 | @code{GNUNET_IDENTITY_connect}. This function takes a callback as a parameter. | ||
5004 | If the given callback parameter is non-null, it will be invoked to notify the | ||
5005 | application about the current state of the identities in the system. | ||
5006 | |||
5007 | @itemize @bullet | ||
5008 | @item First, it will be invoked on all known egos at the time of the | ||
5009 | connection. For each ego, a handle to the ego and the user's name for the ego | ||
5010 | will be passed to the callback. Furthermore, a @code{void **} context argument | ||
5011 | will be provided which gives the client the opportunity to associate some state | ||
5012 | with the ego. | ||
5013 | @item Second, the callback will be invoked with NULL for the ego, the name and | ||
5014 | the context. This signals that the (initial) iteration over all egos has | ||
5015 | completed. | ||
5016 | @item Then, the callback will be invoked whenever something changes about an | ||
5017 | ego. If an ego is renamed, the callback is invoked with the ego handle of the | ||
5018 | ego that was renamed, and the new name. If an ego is deleted, the callback is | ||
5019 | invoked with the ego handle and a name of NULL. In the deletion case, the | ||
5020 | application should also release resources stored in the context. | ||
5021 | @item When the application destroys the connection to the identity service | ||
5022 | using @code{GNUNET_IDENTITY_disconnect}, the callback is again invoked with the | ||
5023 | ego and a name of NULL (equivalent to deletion of the egos). This should again | ||
5024 | be used to clean up the per-ego context. | ||
5025 | @end itemize | ||
5026 | |||
5027 | The ego handle passed to the callback remains valid until the callback is | ||
5028 | invoked with a name of NULL, so it is safe to store a reference to the ego's | ||
5029 | handle. | ||
5030 | |||
5031 | @node Operations on Egos | ||
5032 | @subsubsection Operations on Egos | ||
5033 | |||
5034 | @c %**end of header | ||
5035 | |||
5036 | Given an ego handle, the main operations are to get its associated private key | ||
5037 | using @code{GNUNET_IDENTITY_ego_get_private_key} or its associated public key | ||
5038 | using @code{GNUNET_IDENTITY_ego_get_public_key}. | ||
5039 | |||
5040 | The other operations on egos are pretty straightforward. Using | ||
5041 | @code{GNUNET_IDENTITY_create}, an application can request the creation of an | ||
5042 | ego by specifying the desired name. The operation will fail if that name is | ||
5043 | already in use. Using @code{GNUNET_IDENTITY_rename} the name of an existing ego | ||
5044 | can be changed. Finally, egos can be deleted using | ||
5045 | @code{GNUNET_IDENTITY_delete}. All of these operations will trigger updates to | ||
5046 | the callback given to the @code{GNUNET_IDENTITY_connect} function of all | ||
5047 | applications that are connected with the identity service at the time. | ||
5048 | @code{GNUNET_IDENTITY_cancel} can be used to cancel the operations before the | ||
5049 | respective continuations would be called. It is not guaranteed that the | ||
5050 | operation will not be completed anyway, only the continuation will no longer be | ||
5051 | called. | ||
5052 | |||
5053 | @node The anonymous Ego | ||
5054 | @subsubsection The anonymous Ego | ||
5055 | |||
5056 | @c %**end of header | ||
5057 | |||
5058 | A special way to obtain an ego handle is to call | ||
5059 | @code{GNUNET_IDENTITY_ego_get_anonymous}, which returns an ego for the | ||
5060 | "anonymous" user --- anyone knows and can get the private key for this user, so | ||
5061 | it is suitable for operations that are supposed to be anonymous but require | ||
5062 | signatures (for example, to avoid a special path in the code). The anonymous | ||
5063 | ego is always valid and accessing it does not require a connection to the | ||
5064 | identity service. | ||
5065 | |||
5066 | @node Convenience API to lookup a single ego | ||
5067 | @subsubsection Convenience API to lookup a single ego | ||
5068 | |||
5069 | |||
5070 | As applications commonly simply have to lookup a single ego, there is a | ||
5071 | convenience API to do just that. Use @code{GNUNET_IDENTITY_ego_lookup} to | ||
5072 | lookup a single ego by name. Note that this is the user's name for the ego, not | ||
5073 | the service function. The resulting ego will be returned via a callback and | ||
5074 | will only be valid during that callback. The operation can be cancelled via | ||
5075 | @code{GNUNET_IDENTITY_ego_lookup_cancel} (cancellation is only legal before the | ||
5076 | callback is invoked). | ||
5077 | |||
5078 | @node Associating egos with service functions | ||
5079 | @subsubsection Associating egos with service functions | ||
5080 | |||
5081 | |||
5082 | The @code{GNUNET_IDENTITY_set} function is used to associate a particular ego | ||
5083 | with a service function. The name used by the service and the ego are given as | ||
5084 | arguments. Afterwards, the service can use its name to lookup the associated | ||
5085 | ego using @code{GNUNET_IDENTITY_get}. | ||
5086 | |||
5087 | @node The IDENTITY Client-Service Protocol | ||
5088 | @subsection The IDENTITY Client-Service Protocol | ||
5089 | |||
5090 | @c %**end of header | ||
5091 | |||
5092 | A client connecting to the identity service first sends a message with type | ||
5093 | @code{GNUNET_MESSAGE_TYPE_IDENTITY_START} to the service. After that, the | ||
5094 | client will receive information about changes to the egos by receiving messages | ||
5095 | of type @code{GNUNET_MESSAGE_TYPE_IDENTITY_UPDATE}. Those messages contain the | ||
5096 | private key of the ego and the user's name of the ego (or zero bytes for the | ||
5097 | name to indicate that the ego was deleted). A special bit @code{end_of_list} is | ||
5098 | used to indicate the end of the initial iteration over the identity service's | ||
5099 | egos. | ||
5100 | |||
5101 | The client can trigger changes to the egos by sending CREATE, RENAME or DELETE | ||
5102 | messages. The CREATE message contains the private key and the desired name. The | ||
5103 | RENAME message contains the old name and the new name. The DELETE message only | ||
5104 | needs to include the name of the ego to delete. The service responds to each of | ||
5105 | these messages with a RESULT_CODE message which indicates success or error of | ||
5106 | the operation, and possibly a human-readable error message. | ||
5107 | |||
5108 | Finally, the client can bind the name of a service function to an ego by | ||
5109 | sending a SET_DEFAULT message with the name of the service function and the | ||
5110 | private key of the ego. Such bindings can then be resolved using a GET_DEFAULT | ||
5111 | message, which includes the name of the service function. The identity service | ||
5112 | will respond to a GET_DEFAULT request with a SET_DEFAULT message containing the | ||
5113 | respective information, or with a RESULT_CODE to indicate an error. | ||
5114 | |||
5115 | @node GNUnet's NAMESTORE Subsystem | ||
5116 | @section GNUnet's NAMESTORE Subsystem | ||
5117 | |||
5118 | @c %**end of header | ||
5119 | |||
5120 | The NAMESTORE subsystem provides persistent storage for local GNS zone | ||
5121 | information. All local GNS zone information are managed by NAMESTORE. It | ||
5122 | provides both the functionality to administer local GNS information (e.g. | ||
5123 | delete and add records) as well as to retrieve GNS information (e.g to list | ||
5124 | name information in a client). NAMESTORE does only manage the persistent | ||
5125 | storage of zone information belonging to the user running the service: GNS | ||
5126 | information from other users obtained from the DHT are stored by the NAMECACHE | ||
5127 | subsystem. | ||
5128 | |||
5129 | NAMESTORE uses a plugin-based database backend to store GNS information with | ||
5130 | good performance. Here sqlite, MySQL and PostgreSQL are supported database | ||
5131 | backends. NAMESTORE clients interact with the IDENTITY subsystem to obtain | ||
5132 | cryptographic information about zones based on egos as described with the | ||
5133 | IDENTITY subsystem., but internally NAMESTORE refers to zones using the ECDSA | ||
5134 | private key. In addition, it collaborates with the NAMECACHE subsystem and | ||
5135 | stores zone information when local information are modified in the GNS cache to | ||
5136 | increase look-up performance for local information. | ||
5137 | |||
5138 | NAMESTORE provides functionality to look-up and store records, to iterate over | ||
5139 | a specific or all zones and to monitor zones for changes. NAMESTORE | ||
5140 | functionality can be accessed using the NAMESTORE api or the NAMESTORE command | ||
5141 | line tool. | ||
5142 | |||
5143 | @menu | ||
5144 | * libgnunetnamestore:: | ||
5145 | @end menu | ||
5146 | |||
5147 | @node libgnunetnamestore | ||
5148 | @subsection libgnunetnamestore | ||
5149 | |||
5150 | @c %**end of header | ||
5151 | |||
5152 | To interact with NAMESTORE clients first connect to the NAMESTORE service using | ||
5153 | the @code{GNUNET_NAMESTORE_connect} passing a configuration handle. As a result | ||
5154 | they obtain a NAMESTORE handle, they can use for operations, or NULL is | ||
5155 | returned if the connection failed. | ||
5156 | |||
5157 | To disconnect from NAMESTORE, clients use @code{GNUNET_NAMESTORE_disconnect} | ||
5158 | and specify the handle to disconnect. | ||
5159 | |||
5160 | NAMESTORE internally uses the ECDSA private key to refer to zones. These | ||
5161 | private keys can be obtained from the IDENTITY subsytem. Here @emph{egos@emph{ | ||
5162 | can be used to refer to zones or the default ego assigned to the GNS subsystem | ||
5163 | can be used to obtained the master zone's private key.}} | ||
5164 | |||
5165 | |||
5166 | @menu | ||
5167 | * Editing Zone Information:: | ||
5168 | * Iterating Zone Information:: | ||
5169 | * Monitoring Zone Information:: | ||
5170 | @end menu | ||
5171 | |||
5172 | @node Editing Zone Information | ||
5173 | @subsubsection Editing Zone Information | ||
5174 | |||
5175 | @c %**end of header | ||
5176 | |||
5177 | NAMESTORE provides functions to lookup records stored under a label in a zone | ||
5178 | and to store records under a label in a zone. | ||
5179 | |||
5180 | To store (and delete) records, the client uses the | ||
5181 | @code{GNUNET_NAMESTORE_records_store} function and has to provide namestore | ||
5182 | handle to use, the private key of the zone, the label to store the records | ||
5183 | under, the records and number of records plus an callback function. After the | ||
5184 | operation is performed NAMESTORE will call the provided callback function with | ||
5185 | the result GNUNET_SYSERR on failure (including timeout/queue drop/failure to | ||
5186 | validate), GNUNET_NO if content was already there or not found GNUNET_YES (or | ||
5187 | other positive value) on success plus an additional error message. | ||
5188 | |||
5189 | Records are deleted by using the store command with 0 records to store. It is | ||
5190 | important to note, that records are not merged when records exist with the | ||
5191 | label. So a client has first to retrieve records, merge with existing records | ||
5192 | and then store the result. | ||
5193 | |||
5194 | To perform a lookup operation, the client uses the | ||
5195 | @code{GNUNET_NAMESTORE_records_store} function. Here he has to pass the | ||
5196 | namestore handle, the private key of the zone and the label. He also has to | ||
5197 | provide a callback function which will be called with the result of the lookup | ||
5198 | operation: the zone for the records, the label, and the records including the | ||
5199 | number of records included. | ||
5200 | |||
5201 | A special operation is used to set the preferred nickname for a zone. This | ||
5202 | nickname is stored with the zone and is automatically merged with all labels | ||
5203 | and records stored in a zone. Here the client uses the | ||
5204 | @code{GNUNET_NAMESTORE_set_nick} function and passes the private key of the | ||
5205 | zone, the nickname as string plus a the callback with the result of the | ||
5206 | operation. | ||
5207 | |||
5208 | @node Iterating Zone Information | ||
5209 | @subsubsection Iterating Zone Information | ||
5210 | |||
5211 | @c %**end of header | ||
5212 | |||
5213 | A client can iterate over all information in a zone or all zones managed by | ||
5214 | NAMESTORE. Here a client uses the @code{GNUNET_NAMESTORE_zone_iteration_start} | ||
5215 | function and passes the namestore handle, the zone to iterate over and a | ||
5216 | callback function to call with the result. If the client wants to iterate over | ||
5217 | all the, he passes NULL for the zone. A @code{GNUNET_NAMESTORE_ZoneIterator} | ||
5218 | handle is returned to be used to continue iteration. | ||
5219 | |||
5220 | NAMESTORE calls the callback for every result and expects the client to call@ | ||
5221 | @code{GNUNET_NAMESTORE_zone_iterator_next} to continue to iterate or | ||
5222 | @code{GNUNET_NAMESTORE_zone_iterator_stop} to interrupt the iteration. When | ||
5223 | NAMESTORE reached the last item it will call the callback with a NULL value to | ||
5224 | indicate. | ||
5225 | |||
5226 | @node Monitoring Zone Information | ||
5227 | @subsubsection Monitoring Zone Information | ||
5228 | |||
5229 | @c %**end of header | ||
5230 | |||
5231 | Clients can also monitor zones to be notified about changes. Here the clients | ||
5232 | uses the @code{GNUNET_NAMESTORE_zone_monitor_start} function and passes the | ||
5233 | private key of the zone and and a callback function to call with updates for a | ||
5234 | zone. The client can specify to obtain zone information first by iterating over | ||
5235 | the zone and specify a synchronization callback to be called when the client | ||
5236 | and the namestore are synced. | ||
5237 | |||
5238 | On an update, NAMESTORE will call the callback with the private key of the | ||
5239 | zone, the label and the records and their number. | ||
5240 | |||
5241 | To stop monitoring, the client call @code{GNUNET_NAMESTORE_zone_monitor_stop} | ||
5242 | and passes the handle obtained from the function to start the monitoring. | ||
5243 | |||
5244 | @node GNUnet's PEERINFO subsystem | ||
5245 | @section GNUnet's PEERINFO subsystem | ||
5246 | |||
5247 | @c %**end of header | ||
5248 | |||
5249 | The PEERINFO subsystem is used to store verified (validated) information about | ||
5250 | known peers in a persistent way. It obtains these addresses for example from | ||
5251 | TRANSPORT service which is in charge of address validation. Validation means | ||
5252 | that the information in the HELLO message are checked by connecting to the | ||
5253 | addresses and performing a cryptographic handshake to authenticate the peer | ||
5254 | instance stating to be reachable with these addresses. Peerinfo does not | ||
5255 | validate the HELLO messages itself but only stores them and gives them to | ||
5256 | interested clients. | ||
5257 | |||
5258 | As future work, we think about moving from storing just HELLO messages to | ||
5259 | providing a generic persistent per-peer information store. More and more | ||
5260 | subsystems tend to need to store per-peer information in persistent way. To not | ||
5261 | duplicate this functionality we plan to provide a PEERSTORE service providing | ||
5262 | this functionality | ||
5263 | |||
5264 | @menu | ||
5265 | * Features2:: | ||
5266 | * Limitations3:: | ||
5267 | * DeveloperPeer Information:: | ||
5268 | * Startup:: | ||
5269 | * Managing Information:: | ||
5270 | * Obtaining Information:: | ||
5271 | * The PEERINFO Client-Service Protocol:: | ||
5272 | * libgnunetpeerinfo:: | ||
5273 | @end menu | ||
5274 | |||
5275 | @node Features2 | ||
5276 | @subsection Features2 | ||
5277 | |||
5278 | @c %**end of header | ||
5279 | |||
5280 | @itemize @bullet | ||
5281 | @item Persistent storage | ||
5282 | @item Client notification mechanism on update | ||
5283 | @item Periodic clean up for expired information | ||
5284 | @item Differentiation between public and friend-only HELLO | ||
5285 | @end itemize | ||
5286 | |||
5287 | @node Limitations3 | ||
5288 | @subsection Limitations3 | ||
5289 | |||
5290 | |||
5291 | @itemize @bullet | ||
5292 | @item Does not perform HELLO validation | ||
5293 | @end itemize | ||
5294 | |||
5295 | @node DeveloperPeer Information | ||
5296 | @subsection DeveloperPeer Information | ||
5297 | |||
5298 | @c %**end of header | ||
5299 | |||
5300 | The PEERINFO subsystem stores these information in the form of HELLO messages | ||
5301 | you can think of as business cards. These HELLO messages contain the public key | ||
5302 | of a peer and the addresses a peer can be reached under. The addresses include | ||
5303 | an expiration date describing how long they are valid. This information is | ||
5304 | updated regularly by the TRANSPORT service by revalidating the address. If an | ||
5305 | address is expired and not renewed, it can be removed from the HELLO message. | ||
5306 | |||
5307 | Some peer do not want to have their HELLO messages distributed to other peers , | ||
5308 | especially when GNUnet's friend-to-friend modus is enabled. To prevent this | ||
5309 | undesired distribution. PEERINFO distinguishes between @emph{public} and | ||
5310 | @emph{friend-only} HELLO messages. Public HELLO messages can be freely | ||
5311 | distributed to other (possibly unknown) peers (for example using the hostlist, | ||
5312 | gossiping, broadcasting), whereas friend-only HELLO messages may not be | ||
5313 | distributed to other peers. Friend-only HELLO messages have an additional flag | ||
5314 | @code{friend_only} set internally. For public HELLO message this flag is not | ||
5315 | set. PEERINFO does and cannot not check if a client is allowed to obtain a | ||
5316 | specific HELLO type. | ||
5317 | |||
5318 | The HELLO messages can be managed using the GNUnet HELLO library. Other GNUnet | ||
5319 | systems can obtain these information from PEERINFO and use it for their | ||
5320 | purposes. Clients are for example the HOSTLIST component providing these | ||
5321 | information to other peers in form of a hostlist or the TRANSPORT subsystem | ||
5322 | using these information to maintain connections to other peers. | ||
5323 | |||
5324 | @node Startup | ||
5325 | @subsection Startup | ||
5326 | |||
5327 | @c %**end of header | ||
5328 | |||
5329 | During startup the PEERINFO services loads persistent HELLOs from disk. First | ||
5330 | PEERINFO parses the directory configured in the HOSTS value of the | ||
5331 | @code{PEERINFO} configuration section to store PEERINFO information.@ For all | ||
5332 | files found in this directory valid HELLO messages are extracted. In addition | ||
5333 | it loads HELLO messages shipped with the GNUnet distribution. These HELLOs are | ||
5334 | used to simplify network bootstrapping by providing valid peer information with | ||
5335 | the distribution. The use of these HELLOs can be prevented by setting the | ||
5336 | @code{USE_INCLUDED_HELLOS} in the @code{PEERINFO} configuration section to | ||
5337 | @code{NO}. Files containing invalid information are removed. | ||
5338 | |||
5339 | @node Managing Information | ||
5340 | @subsection Managing Information | ||
5341 | |||
5342 | @c %**end of header | ||
5343 | |||
5344 | The PEERINFO services stores information about known PEERS and a single HELLO | ||
5345 | message for every peer. A peer does not need to have a HELLO if no information | ||
5346 | are available. HELLO information from different sources, for example a HELLO | ||
5347 | obtained from a remote HOSTLIST and a second HELLO stored on disk, are combined | ||
5348 | and merged into one single HELLO message per peer which will be given to | ||
5349 | clients. During this merge process the HELLO is immediately written to disk to | ||
5350 | ensure persistence. | ||
5351 | |||
5352 | PEERINFO in addition periodically scans the directory where information are | ||
5353 | stored for empty HELLO messages with expired TRANSPORT addresses.@ This | ||
5354 | periodic task scans all files in the directory and recreates the HELLO messages | ||
5355 | it finds. Expired TRANSPORT addresses are removed from the HELLO and if the | ||
5356 | HELLO does not contain any valid addresses, it is discarded and removed from | ||
5357 | disk. | ||
5358 | |||
5359 | @node Obtaining Information | ||
5360 | @subsection Obtaining Information | ||
5361 | |||
5362 | @c %**end of header | ||
5363 | |||
5364 | When a client requests information from PEERINFO, PEERINFO performs a lookup | ||
5365 | for the respective peer or all peers if desired and transmits this information | ||
5366 | to the client. The client can specify if friend-only HELLOs have to be included | ||
5367 | or not and PEERINFO filters the respective HELLO messages before transmitting | ||
5368 | information. | ||
5369 | |||
5370 | To notify clients about changes to PEERINFO information, PEERINFO maintains a | ||
5371 | list of clients interested in this notifications. Such a notification occurs if | ||
5372 | a HELLO for a peer was updated (due to a merge for example) or a new peer was | ||
5373 | added. | ||
5374 | |||
5375 | @node The PEERINFO Client-Service Protocol | ||
5376 | @subsection The PEERINFO Client-Service Protocol | ||
5377 | |||
5378 | @c %**end of header | ||
5379 | |||
5380 | To connect and disconnect to and from the PEERINFO Service PEERINFO utilizes | ||
5381 | the util client/server infrastructure, so no special messages types are used | ||
5382 | here. | ||
5383 | |||
5384 | To add information for a peer, the plain HELLO message is transmitted to the | ||
5385 | service without any wrapping. Alle information required are stored within the | ||
5386 | HELLO message. The PEERINFO service provides a message handler accepting and | ||
5387 | processing these HELLO messages. | ||
5388 | |||
5389 | When obtaining PEERINFO information using the iterate functionality specific | ||
5390 | messages are used. To obtain information for all peers, a @code{struct | ||
5391 | ListAllPeersMessage} with message type | ||
5392 | @code{GNUNET_MESSAGE_TYPE_PEERINFO_GET_ALL} and a flag include_friend_only to | ||
5393 | indicate if friend-only HELLO messages should be included are transmitted. If | ||
5394 | information for a specific peer is required a @code{struct ListAllPeersMessage} | ||
5395 | with @code{GNUNET_MESSAGE_TYPE_PEERINFO_GET} containing the peer identity is | ||
5396 | used. | ||
5397 | |||
5398 | For both variants the PEERINFO service replies for each HELLO message he wants | ||
5399 | to transmit with a @code{struct ListAllPeersMessage} with type | ||
5400 | @code{GNUNET_MESSAGE_TYPE_PEERINFO_INFO} containing the plain HELLO. The final | ||
5401 | message is @code{struct GNUNET_MessageHeader} with type | ||
5402 | @code{GNUNET_MESSAGE_TYPE_PEERINFO_INFO}. If the client receives this message, | ||
5403 | he can proceed with the next request if any is pending | ||
5404 | |||
5405 | @node libgnunetpeerinfo | ||
5406 | @subsection libgnunetpeerinfo | ||
5407 | |||
5408 | @c %**end of header | ||
5409 | |||
5410 | The PEERINFO API consists mainly of three different functionalities: | ||
5411 | maintaining a connection to the service, adding new information and retrieving | ||
5412 | information form the PEERINFO service. | ||
5413 | |||
5414 | |||
5415 | @menu | ||
5416 | * Connecting to the Service:: | ||
5417 | * Adding Information:: | ||
5418 | * Obtaining Information2:: | ||
5419 | @end menu | ||
5420 | |||
5421 | @node Connecting to the Service | ||
5422 | @subsubsection Connecting to the Service | ||
5423 | |||
5424 | @c %**end of header | ||
5425 | |||
5426 | To connect to the PEERINFO service the function @code{GNUNET_PEERINFO_connect} | ||
5427 | is used, taking a configuration handle as an argument, and to disconnect from | ||
5428 | PEERINFO the function @code{GNUNET_PEERINFO_disconnect}, taking the PEERINFO | ||
5429 | handle returned from the connect function has to be called. | ||
5430 | |||
5431 | @node Adding Information | ||
5432 | @subsubsection Adding Information | ||
5433 | |||
5434 | @c %**end of header | ||
5435 | |||
5436 | @code{GNUNET_PEERINFO_add_peer} adds a new peer to the PEERINFO subsystem | ||
5437 | storage. This function takes the PEERINFO handle as an argument, the HELLO | ||
5438 | message to store and a continuation with a closure to be called with the result | ||
5439 | of the operation. The @code{GNUNET_PEERINFO_add_peer} returns a handle to this | ||
5440 | operation allowing to cancel the operation with the respective cancel function | ||
5441 | @code{GNUNET_PEERINFO_add_peer_cancel}. To retrieve information from PEERINFO | ||
5442 | you can iterate over all information stored with PEERINFO or you can tell | ||
5443 | PEERINFO to notify if new peer information are available. | ||
5444 | |||
5445 | @node Obtaining Information2 | ||
5446 | @subsubsection Obtaining Information2 | ||
5447 | |||
5448 | @c %**end of header | ||
5449 | |||
5450 | To iterate over information in PEERINFO you use @code{GNUNET_PEERINFO_iterate}. | ||
5451 | This function expects the PEERINFO handle, a flag if HELLO messages intended | ||
5452 | for friend only mode should be included, a timeout how long the operation | ||
5453 | should take and a callback with a callback closure to be called for the | ||
5454 | results. If you want to obtain information for a specific peer, you can specify | ||
5455 | the peer identity, if this identity is NULL, information for all peers are | ||
5456 | returned. The function returns a handle to allow to cancel the operation using | ||
5457 | @code{GNUNET_PEERINFO_iterate_cancel}. | ||
5458 | |||
5459 | To get notified when peer information changes, you can use | ||
5460 | @code{GNUNET_PEERINFO_notify}. This function expects a configuration handle and | ||
5461 | a flag if friend-only HELLO messages should be included. The PEERINFO service | ||
5462 | will notify you about every change and the callback function will be called to | ||
5463 | notify you about changes. The function returns a handle to cancel notifications | ||
5464 | with @code{GNUNET_PEERINFO_notify_cancel}. | ||
5465 | |||
5466 | |||
5467 | @node GNUnet's PEERSTORE subsystem | ||
5468 | @section GNUnet's PEERSTORE subsystem | ||
5469 | |||
5470 | @c %**end of header | ||
5471 | |||
5472 | GNUnet's PEERSTORE subsystem offers persistent per-peer storage for other | ||
5473 | GNUnet subsystems. GNUnet subsystems can use PEERSTORE to persistently store | ||
5474 | and retrieve arbitrary data. Each data record stored with PEERSTORE contains | ||
5475 | the following fields: | ||
5476 | |||
5477 | @itemize @bullet | ||
5478 | @item subsystem: Name of the subsystem responsible for the record. | ||
5479 | @item peerid: Identity of the peer this record is related to. | ||
5480 | @item key: a key string identifying the record. | ||
5481 | @item value: binary record value. | ||
5482 | @item expiry: record expiry date. | ||
5483 | @end itemize | ||
5484 | |||
5485 | @menu | ||
5486 | * Functionality:: | ||
5487 | * Architecture:: | ||
5488 | * libgnunetpeerstore:: | ||
5489 | @end menu | ||
5490 | |||
5491 | @node Functionality | ||
5492 | @subsection Functionality | ||
5493 | |||
5494 | @c %**end of header | ||
5495 | |||
5496 | Subsystems can store any type of value under a (subsystem, peerid, key) | ||
5497 | combination. A "replace" flag set during store operations forces the PEERSTORE | ||
5498 | to replace any old values stored under the same (subsystem, peerid, key) | ||
5499 | combination with the new value. Additionally, an expiry date is set after which | ||
5500 | the record is *possibly* deleted by PEERSTORE. | ||
5501 | |||
5502 | Subsystems can iterate over all values stored under any of the following | ||
5503 | combination of fields: | ||
5504 | |||
5505 | @itemize @bullet | ||
5506 | @item (subsystem) | ||
5507 | @item (subsystem, peerid) | ||
5508 | @item (subsystem, key) | ||
5509 | @item (subsystem, peerid, key) | ||
5510 | @end itemize | ||
5511 | |||
5512 | Subsystems can also request to be notified about any new values stored under a | ||
5513 | (subsystem, peerid, key) combination by sending a "watch" request to | ||
5514 | PEERSTORE. | ||
5515 | |||
5516 | @node Architecture | ||
5517 | @subsection Architecture | ||
5518 | |||
5519 | @c %**end of header | ||
5520 | |||
5521 | PEERSTORE implements the following components: | ||
5522 | |||
5523 | @itemize @bullet | ||
5524 | @item PEERSTORE service: Handles store, iterate and watch operations. | ||
5525 | @item PEERSTORE API: API to be used by other subsystems to communicate and | ||
5526 | issue commands to the PEERSTORE service. | ||
5527 | @item PEERSTORE plugins: Handles the persistent storage. At the moment, only an | ||
5528 | "sqlite" plugin is implemented. | ||
5529 | @end itemize | ||
5530 | |||
5531 | @node libgnunetpeerstore | ||
5532 | @subsection libgnunetpeerstore | ||
5533 | |||
5534 | @c %**end of header | ||
5535 | |||
5536 | libgnunetpeerstore is the library containing the PEERSTORE API. Subsystems | ||
5537 | wishing to communicate with the PEERSTORE service use this API to open a | ||
5538 | connection to PEERSTORE. This is done by calling | ||
5539 | @code{GNUNET_PEERSTORE_connect} which returns a handle to the newly created | ||
5540 | connection. This handle has to be used with any further calls to the API. | ||
5541 | |||
5542 | To store a new record, the function @code{GNUNET_PEERSTORE_store} is to be used | ||
5543 | which requires the record fields and a continuation function that will be | ||
5544 | called by the API after the STORE request is sent to the PEERSTORE service. | ||
5545 | Note that calling the continuation function does not mean that the record is | ||
5546 | successfully stored, only that the STORE request has been successfully sent to | ||
5547 | the PEERSTORE service. @code{GNUNET_PEERSTORE_store_cancel} can be called to | ||
5548 | cancel the STORE request only before the continuation function has been called. | ||
5549 | |||
5550 | To iterate over stored records, the function @code{GNUNET_PEERSTORE_iterate} is | ||
5551 | to be used. @emph{peerid} and @emph{key} can be set to NULL. An iterator | ||
5552 | callback function will be called with each matching record found and a NULL | ||
5553 | record at the end to signal the end of result set. | ||
5554 | @code{GNUNET_PEERSTORE_iterate_cancel} can be used to cancel the ITERATE | ||
5555 | request before the iterator callback is called with a NULL record. | ||
5556 | |||
5557 | To be notified with new values stored under a (subsystem, peerid, key) | ||
5558 | combination, the function @code{GNUNET_PEERSTORE_watch} is to be used. This | ||
5559 | will register the watcher with the PEERSTORE service, any new records matching | ||
5560 | the given combination will trigger the callback function passed to | ||
5561 | @code{GNUNET_PEERSTORE_watch}. This continues until | ||
5562 | @code{GNUNET_PEERSTORE_watch_cancel} is called or the connection to the service | ||
5563 | is destroyed. | ||
5564 | |||
5565 | After the connection is no longer needed, the function | ||
5566 | @code{GNUNET_PEERSTORE_disconnect} can be called to disconnect from the | ||
5567 | PEERSTORE service. Any pending ITERATE or WATCH requests will be destroyed. If | ||
5568 | the @code{sync_first} flag is set to @code{GNUNET_YES}, the API will delay the | ||
5569 | disconnection until all pending STORE requests are sent to the PEERSTORE | ||
5570 | service, otherwise, the pending STORE requests will be destroyed as well. | ||
5571 | |||
5572 | @node GNUnet's SET Subsystem | ||
5573 | @section GNUnet's SET Subsystem | ||
5574 | |||
5575 | @c %**end of header | ||
5576 | |||
5577 | The SET service implements efficient set operations between two peers over a | ||
5578 | mesh tunnel. Currently, set union and set intersection are the only supported | ||
5579 | operations. Elements of a set consist of an @emph{element type} and arbitrary | ||
5580 | binary @emph{data}. The size of an element's data is limited to around 62 | ||
5581 | KB. | ||
5582 | |||
5583 | @menu | ||
5584 | * Local Sets:: | ||
5585 | * Set Modifications:: | ||
5586 | * Set Operations:: | ||
5587 | * Result Elements:: | ||
5588 | * libgnunetset:: | ||
5589 | * The SET Client-Service Protocol:: | ||
5590 | * The SET Intersection Peer-to-Peer Protocol:: | ||
5591 | * The SET Union Peer-to-Peer Protocol:: | ||
5592 | @end menu | ||
5593 | |||
5594 | @node Local Sets | ||
5595 | @subsection Local Sets | ||
5596 | |||
5597 | @c %**end of header | ||
5598 | |||
5599 | Sets created by a local client can be modified and reused for multiple | ||
5600 | operations. As each set operation requires potentially expensive special | ||
5601 | auxilliary data to be computed for each element of a set, a set can only | ||
5602 | participate in one type of set operation (i.e. union or intersection). The type | ||
5603 | of a set is determined upon its creation. If a the elements of a set are needed | ||
5604 | for an operation of a different type, all of the set's element must be copied | ||
5605 | to a new set of appropriate type. | ||
5606 | |||
5607 | @node Set Modifications | ||
5608 | @subsection Set Modifications | ||
5609 | |||
5610 | @c %**end of header | ||
5611 | |||
5612 | Even when set operations are active, one can add to and remove elements from a | ||
5613 | set. However, these changes will only be visible to operations that have been | ||
5614 | created after the changes have taken place. That is, every set operation only | ||
5615 | sees a snapshot of the set from the time the operation was started. This | ||
5616 | mechanism is @emph{not} implemented by copying the whole set, but by attaching | ||
5617 | @emph{generation information} to each element and operation. | ||
5618 | |||
5619 | @node Set Operations | ||
5620 | @subsection Set Operations | ||
5621 | |||
5622 | @c %**end of header | ||
5623 | |||
5624 | Set operations can be started in two ways: Either by accepting an operation | ||
5625 | request from a remote peer, or by requesting a set operation from a remote | ||
5626 | peer. Set operations are uniquely identified by the involved @emph{peers}, an | ||
5627 | @emph{application id} and the @emph{operation type}. | ||
5628 | |||
5629 | The client is notified of incoming set operations by @emph{set listeners}. A | ||
5630 | set listener listens for incoming operations of a specific operation type and | ||
5631 | application id. Once notified of an incoming set request, the client can | ||
5632 | accept the set request (providing a local set for the operation) or reject | ||
5633 | it. | ||
5634 | |||
5635 | @node Result Elements | ||
5636 | @subsection Result Elements | ||
5637 | |||
5638 | @c %**end of header | ||
5639 | |||
5640 | The SET service has three @emph{result modes} that determine how an operation's | ||
5641 | result set is delivered to the client: | ||
5642 | |||
5643 | @itemize @bullet | ||
5644 | @item @strong{Full Result Set.} All elements of set resulting from the set | ||
5645 | operation are returned to the client. | ||
5646 | @item @strong{Added Elements.} Only elements that result from the operation and | ||
5647 | are not already in the local peer's set are returned. Note that for some | ||
5648 | operations (like set intersection) this result mode will never return any | ||
5649 | elements. This can be useful if only the remove peer is actually interested in | ||
5650 | the result of the set operation. | ||
5651 | @item @strong{Removed Elements.} Only elements that are in the local peer's | ||
5652 | initial set but not in the operation's result set are returned. Note that for | ||
5653 | some operations (like set union) this result mode will never return any | ||
5654 | elements. This can be useful if only the remove peer is actually interested in | ||
5655 | the result of the set operation. | ||
5656 | @end itemize | ||
5657 | |||
5658 | @node libgnunetset | ||
5659 | @subsection libgnunetset | ||
5660 | |||
5661 | @c %**end of header | ||
5662 | |||
5663 | @menu | ||
5664 | * Sets:: | ||
5665 | * Listeners:: | ||
5666 | * Operations:: | ||
5667 | * Supplying a Set:: | ||
5668 | * The Result Callback:: | ||
5669 | @end menu | ||
5670 | |||
5671 | @node Sets | ||
5672 | @subsubsection Sets | ||
5673 | |||
5674 | @c %**end of header | ||
5675 | |||
5676 | New sets are created with @code{GNUNET_SET_create}. Both the local peer's | ||
5677 | configuration (as each set has its own client connection) and the operation | ||
5678 | type must be specified. The set exists until either the client calls | ||
5679 | @code{GNUNET_SET_destroy} or the client's connection to the service is | ||
5680 | disrupted. In the latter case, the client is notified by the return value of | ||
5681 | functions dealing with sets. This return value must always be checked. | ||
5682 | |||
5683 | Elements are added and removed with @code{GNUNET_SET_add_element} and | ||
5684 | @code{GNUNET_SET_remove_element}. | ||
5685 | |||
5686 | @node Listeners | ||
5687 | @subsubsection Listeners | ||
5688 | |||
5689 | @c %**end of header | ||
5690 | |||
5691 | Listeners are created with @code{GNUNET_SET_listen}. Each time time a remote | ||
5692 | peer suggests a set operation with an application id and operation type | ||
5693 | matching a listener, the listener's callack is invoked. The client then must | ||
5694 | synchronously call either @code{GNUNET_SET_accept} or @code{GNUNET_SET_reject}. | ||
5695 | Note that the operation will not be started until the client calls | ||
5696 | @code{GNUNET_SET_commit} (see Section "Supplying a Set"). | ||
5697 | |||
5698 | @node Operations | ||
5699 | @subsubsection Operations | ||
5700 | |||
5701 | @c %**end of header | ||
5702 | |||
5703 | Operations to be initiated by the local peer are created with | ||
5704 | @code{GNUNET_SET_prepare}. Note that the operation will not be started until | ||
5705 | the client calls @code{GNUNET_SET_commit} (see Section "Supplying a | ||
5706 | Set"). | ||
5707 | |||
5708 | @node Supplying a Set | ||
5709 | @subsubsection Supplying a Set | ||
5710 | |||
5711 | @c %**end of header | ||
5712 | |||
5713 | To create symmetry between the two ways of starting a set operation (accepting | ||
5714 | and nitiating it), the operation handles returned by @code{GNUNET_SET_accept} | ||
5715 | and @code{GNUNET_SET_prepare} do not yet have a set to operate on, thus they | ||
5716 | can not do any work yet. | ||
5717 | |||
5718 | The client must call @code{GNUNET_SET_commit} to specify a set to use for an | ||
5719 | operation. @code{GNUNET_SET_commit} may only be called once per set | ||
5720 | operation. | ||
5721 | |||
5722 | @node The Result Callback | ||
5723 | @subsubsection The Result Callback | ||
5724 | |||
5725 | @c %**end of header | ||
5726 | |||
5727 | Clients must specify both a result mode and a result callback with | ||
5728 | @code{GNUNET_SET_accept} and @code{GNUNET_SET_prepare}. The result callback | ||
5729 | with a status indicating either that an element was received, or the operation | ||
5730 | failed or succeeded. The interpretation of the received element depends on the | ||
5731 | result mode. The callback needs to know which result mode it is used in, as the | ||
5732 | arguments do not indicate if an element is part of the full result set, or if | ||
5733 | it is in the difference between the original set and the final set. | ||
5734 | |||
5735 | @node The SET Client-Service Protocol | ||
5736 | @subsection The SET Client-Service Protocol | ||
5737 | |||
5738 | @c %**end of header | ||
5739 | |||
5740 | @menu | ||
5741 | * Creating Sets:: | ||
5742 | * Listeners2:: | ||
5743 | * Initiating Operations:: | ||
5744 | * Modifying Sets:: | ||
5745 | * Results and Operation Status:: | ||
5746 | * Iterating Sets:: | ||
5747 | @end menu | ||
5748 | |||
5749 | @node Creating Sets | ||
5750 | @subsubsection Creating Sets | ||
5751 | |||
5752 | @c %**end of header | ||
5753 | |||
5754 | For each set of a client, there exists a client connection to the service. Sets | ||
5755 | are created by sending the @code{GNUNET_SERVICE_SET_CREATE} message over a new | ||
5756 | client connection. Multiple operations for one set are multiplexed over one | ||
5757 | client connection, using a request id supplied by the client. | ||
5758 | |||
5759 | @node Listeners2 | ||
5760 | @subsubsection Listeners2 | ||
5761 | |||
5762 | @c %**end of header | ||
5763 | |||
5764 | Each listener also requires a seperate client connection. By sending the | ||
5765 | @code{GNUNET_SERVICE_SET_LISTEN} message, the client notifies the service of | ||
5766 | the application id and operation type it is interested in. A client rejects an | ||
5767 | incoming request by sending @code{GNUNET_SERVICE_SET_REJECT} on the listener's | ||
5768 | client connection. In contrast, when accepting an incoming request, a a | ||
5769 | @code{GNUNET_SERVICE_SET_ACCEPT} message must be sent over the@ set that is | ||
5770 | supplied for the set operation. | ||
5771 | |||
5772 | @node Initiating Operations | ||
5773 | @subsubsection Initiating Operations | ||
5774 | |||
5775 | @c %**end of header | ||
5776 | |||
5777 | Operations with remote peers are initiated by sending a | ||
5778 | @code{GNUNET_SERVICE_SET_EVALUATE} message to the service. The@ client | ||
5779 | connection that this message is sent by determines the set to use. | ||
5780 | |||
5781 | @node Modifying Sets | ||
5782 | @subsubsection Modifying Sets | ||
5783 | |||
5784 | @c %**end of header | ||
5785 | |||
5786 | Sets are modified with the @code{GNUNET_SERVICE_SET_ADD} and | ||
5787 | @code{GNUNET_SERVICE_SET_REMOVE} messages. | ||
5788 | |||
5789 | |||
5790 | @c %@menu | ||
5791 | @c %* Results and Operation Status:: | ||
5792 | @c %* Iterating Sets:: | ||
5793 | @c %@end menu | ||
5794 | |||
5795 | @node Results and Operation Status | ||
5796 | @subsubsection Results and Operation Status | ||
5797 | @c %**end of header | ||
5798 | |||
5799 | The service notifies the client of result elements and success/failure of a set | ||
5800 | operation with the @code{GNUNET_SERVICE_SET_RESULT} message. | ||
5801 | |||
5802 | @node Iterating Sets | ||
5803 | @subsubsection Iterating Sets | ||
5804 | |||
5805 | @c %**end of header | ||
5806 | |||
5807 | All elements of a set can be requested by sending | ||
5808 | @code{GNUNET_SERVICE_SET_ITER_REQUEST}. The server responds with | ||
5809 | @code{GNUNET_SERVICE_SET_ITER_ELEMENT} and eventually terminates the iteration | ||
5810 | with @code{GNUNET_SERVICE_SET_ITER_DONE}. After each received element, the | ||
5811 | client@ must send @code{GNUNET_SERVICE_SET_ITER_ACK}. Note that only one set | ||
5812 | iteration may be active for a set at any given time. | ||
5813 | |||
5814 | @node The SET Intersection Peer-to-Peer Protocol | ||
5815 | @subsection The SET Intersection Peer-to-Peer Protocol | ||
5816 | |||
5817 | @c %**end of header | ||
5818 | |||
5819 | The intersection protocol operates over CADET and starts with a | ||
5820 | GNUNET_MESSAGE_TYPE_SET_P2P_OPERATION_REQUEST being sent by the peer initiating | ||
5821 | the operation to the peer listening for inbound requests. It includes the | ||
5822 | number of elements of the initiating peer, which is used to decide which side | ||
5823 | will send a Bloom filter first. | ||
5824 | |||
5825 | The listening peer checks if the operation type and application identifier are | ||
5826 | acceptable for its current state. If not, it responds with a | ||
5827 | GNUNET_MESSAGE_TYPE_SET_RESULT and a status of GNUNET_SET_STATUS_FAILURE (and | ||
5828 | terminates the CADET channel). | ||
5829 | |||
5830 | If the application accepts the request, the listener sends back a@ | ||
5831 | GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_ELEMENT_INFO if it has more elements | ||
5832 | in the set than the client. Otherwise, it immediately starts with the Bloom | ||
5833 | filter exchange. If the initiator receives a | ||
5834 | GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_ELEMENT_INFO response, it beings the | ||
5835 | Bloom filter exchange, unless the set size is indicated to be zero, in which | ||
5836 | case the intersection is considered finished after just the initial | ||
5837 | handshake. | ||
5838 | |||
5839 | |||
5840 | @menu | ||
5841 | * The Bloom filter exchange:: | ||
5842 | * Salt:: | ||
5843 | @end menu | ||
5844 | |||
5845 | @node The Bloom filter exchange | ||
5846 | @subsubsection The Bloom filter exchange | ||
5847 | |||
5848 | @c %**end of header | ||
5849 | |||
5850 | In this phase, each peer transmits a Bloom filter over the remaining keys of | ||
5851 | the local set to the other peer using a | ||
5852 | GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_BF message. This message additionally | ||
5853 | includes the number of elements left in the sender's set, as well as the XOR | ||
5854 | over all of the keys in that set. | ||
5855 | |||
5856 | The number of bits 'k' set per element in the Bloom filter is calculated based | ||
5857 | on the relative size of the two sets. Furthermore, the size of the Bloom filter | ||
5858 | is calculated based on 'k' and the number of elements in the set to maximize | ||
5859 | the amount of data filtered per byte transmitted on the wire (while avoiding an | ||
5860 | excessively high number of iterations). | ||
5861 | |||
5862 | The receiver of the message removes all elements from its local set that do not | ||
5863 | pass the Bloom filter test. It then checks if the set size of the sender and | ||
5864 | the XOR over the keys match what is left of his own set. If they do, he sends | ||
5865 | a@ GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_DONE back to indicate that the | ||
5866 | latest set is the final result. Otherwise, the receiver starts another Bloom | ||
5867 | fitler exchange, except this time as the sender. | ||
5868 | |||
5869 | @node Salt | ||
5870 | @subsubsection Salt | ||
5871 | |||
5872 | @c %**end of header | ||
5873 | |||
5874 | Bloomfilter operations are probablistic: With some non-zero probability the | ||
5875 | test may incorrectly say an element is in the set, even though it is not. | ||
5876 | |||
5877 | To mitigate this problem, the intersection protocol iterates exchanging Bloom | ||
5878 | filters using a different random 32-bit salt in each iteration (the salt is | ||
5879 | also included in the message). With different salts, set operations may fail | ||
5880 | for different elements. Merging the results from the executions, the | ||
5881 | probability of failure drops to zero. | ||
5882 | |||
5883 | The iterations terminate once both peers have established that they have sets | ||
5884 | of the same size, and where the XOR over all keys computes the same 512-bit | ||
5885 | value (leaving a failure probability of 2-511). | ||
5886 | |||
5887 | @node The SET Union Peer-to-Peer Protocol | ||
5888 | @subsection The SET Union Peer-to-Peer Protocol | ||
5889 | |||
5890 | @c %**end of header | ||
5891 | |||
5892 | The SET union protocol is based on Eppstein's efficient set reconciliation | ||
5893 | without prior context. You should read this paper first if you want to | ||
5894 | understand the protocol. | ||
5895 | |||
5896 | The union protocol operates over CADET and starts with a | ||
5897 | GNUNET_MESSAGE_TYPE_SET_P2P_OPERATION_REQUEST being sent by the peer initiating | ||
5898 | the operation to the peer listening for inbound requests. It includes the | ||
5899 | number of elements of the initiating peer, which is currently not used. | ||
5900 | |||
5901 | The listening peer checks if the operation type and application identifier are | ||
5902 | acceptable for its current state. If not, it responds with a | ||
5903 | GNUNET_MESSAGE_TYPE_SET_RESULT and a status of GNUNET_SET_STATUS_FAILURE (and | ||
5904 | terminates the CADET channel). | ||
5905 | |||
5906 | If the application accepts the request, it sends back a strata estimator using | ||
5907 | a message of type GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SE. The initiator evaluates | ||
5908 | the strata estimator and initiates the exchange of invertible Bloom filters, | ||
5909 | sending a GNUNET_MESSAGE_TYPE_SET_UNION_P2P_IBF. | ||
5910 | |||
5911 | During the IBF exchange, if the receiver cannot invert the Bloom filter or | ||
5912 | detects a cycle, it sends a larger IBF in response (up to a defined maximum | ||
5913 | limit; if that limit is reached, the operation fails). Elements decoded while | ||
5914 | processing the IBF are transmitted to the other peer using | ||
5915 | GNUNET_MESSAGE_TYPE_SET_P2P_ELEMENTS, or requested from the other peer using | ||
5916 | GNUNET_MESSAGE_TYPE_SET_P2P_ELEMENT_REQUESTS messages, depending on the sign | ||
5917 | observed during decoding of the IBF. Peers respond to a | ||
5918 | GNUNET_MESSAGE_TYPE_SET_P2P_ELEMENT_REQUESTS message with the respective | ||
5919 | element in a GNUNET_MESSAGE_TYPE_SET_P2P_ELEMENTS message. If the IBF fully | ||
5920 | decodes, the peer responds with a GNUNET_MESSAGE_TYPE_SET_UNION_P2P_DONE | ||
5921 | message instead of another GNUNET_MESSAGE_TYPE_SET_UNION_P2P_IBF. | ||
5922 | |||
5923 | All Bloom filter operations use a salt to mingle keys before hasing them into | ||
5924 | buckets, such that future iterations have a fresh chance of succeeding if they | ||
5925 | failed due to collisions before. | ||
5926 | |||
5927 | @node GNUnet's STATISTICS subsystem | ||
5928 | @section GNUnet's STATISTICS subsystem | ||
5929 | |||
5930 | @c %**end of header | ||
5931 | |||
5932 | In GNUnet, the STATISTICS subsystem offers a central place for all subsystems | ||
5933 | to publish unsigned 64-bit integer run-time statistics. Keeping this | ||
5934 | information centrally means that there is a unified way for the user to obtain | ||
5935 | data on all subsystems, and individual subsystems do not have to always include | ||
5936 | a custom data export method for performance metrics and other statistics. For | ||
5937 | example, the TRANSPORT system uses STATISTICS to update information about the | ||
5938 | number of directly connected peers and the bandwidth that has been consumed by | ||
5939 | the various plugins. This information is valuable for diagnosing connectivity | ||
5940 | and performance issues. | ||
5941 | |||
5942 | Following the GNUnet service architecture, the STATISTICS subsystem is divided | ||
5943 | into an API which is exposed through the header | ||
5944 | @strong{gnunet_statistics_service.h} and the STATISTICS service | ||
5945 | @strong{gnunet-service-statistics}. The @strong{gnunet-statistics} command-line | ||
5946 | tool can be used to obtain (and change) information about the values stored by | ||
5947 | the STATISTICS service. The STATISTICS service does not communicate with other | ||
5948 | peers. | ||
5949 | |||
5950 | Data is stored in the STATISTICS service in the form of tuples | ||
5951 | @strong{(subsystem, name, value, persistence)}. The subsystem determines to | ||
5952 | which other GNUnet's subsystem the data belongs. name is the name through which | ||
5953 | value is associated. It uniquely identifies the record from among other records | ||
5954 | belonging to the same subsystem. In some parts of the code, the pair | ||
5955 | @strong{(subsystem, name)} is called a @strong{statistic} as it identifies the | ||
5956 | values stored in the STATISTCS service.The persistence flag determines if the | ||
5957 | record has to be preserved across service restarts. A record is said to be | ||
5958 | persistent if this flag is set for it; if not, the record is treated as a | ||
5959 | non-persistent record and it is lost after service restart. Persistent records | ||
5960 | are written to and read from the file @strong{statistics.data} before shutdown | ||
5961 | and upon startup. The file is located in the HOME directory of the peer. | ||
5962 | |||
5963 | An anomaly of the STATISTICS service is that it does not terminate immediately | ||
5964 | upon receiving a shutdown signal if it has any clients connected to it. It | ||
5965 | waits for all the clients that are not monitors to close their connections | ||
5966 | before terminating itself. This is to prevent the loss of data during peer | ||
5967 | shutdown --- delaying the STATISTICS service shutdown helps other services to | ||
5968 | store important data to STATISTICS during shutdown. | ||
5969 | |||
5970 | @menu | ||
5971 | * libgnunetstatistics:: | ||
5972 | * The STATISTICS Client-Service Protocol:: | ||
5973 | @end menu | ||
5974 | |||
5975 | @node libgnunetstatistics | ||
5976 | @subsection libgnunetstatistics | ||
5977 | |||
5978 | @c %**end of header | ||
5979 | |||
5980 | @strong{libgnunetstatistics} is the library containing the API for the | ||
5981 | STATISTICS subsystem. Any process requiring to use STATISTICS should use this | ||
5982 | API by to open a connection to the STATISTICS service. This is done by calling | ||
5983 | the function @code{GNUNET_STATISTICS_create()}. This function takes the | ||
5984 | subsystem's name which is trying to use STATISTICS and a configuration. All | ||
5985 | values written to STATISTICS with this connection will be placed in the section | ||
5986 | corresponding to the given subsystem's name. The connection to STATISTICS can | ||
5987 | be destroyed with the function GNUNET_STATISTICS_destroy(). This function | ||
5988 | allows for the connection to be destroyed immediately or upon transferring all | ||
5989 | pending write requests to the service. | ||
5990 | |||
5991 | Note: STATISTICS subsystem can be disabled by setting @code{DISABLE = YES} | ||
5992 | under the @code{[STATISTICS]} section in the configuration. With such a | ||
5993 | configuration all calls to @code{GNUNET_STATISTICS_create()} return @code{NULL} | ||
5994 | as the STATISTICS subsystem is unavailable and no other functions from the API | ||
5995 | can be used. | ||
5996 | |||
5997 | |||
5998 | @menu | ||
5999 | * Statistics retrieval:: | ||
6000 | * Setting statistics and updating them:: | ||
6001 | * Watches:: | ||
6002 | @end menu | ||
6003 | |||
6004 | @node Statistics retrieval | ||
6005 | @subsubsection Statistics retrieval | ||
6006 | |||
6007 | @c %**end of header | ||
6008 | |||
6009 | Once a connection to the statistics service is obtained, information about any | ||
6010 | other system which uses statistics can be retrieved with the function | ||
6011 | GNUNET_STATISTICS_get(). This function takes the connection handle, the name of | ||
6012 | the subsystem whose information we are interested in (a @code{NULL} value will | ||
6013 | retrieve information of all available subsystems using STATISTICS), the name of | ||
6014 | the statistic we are interested in (a @code{NULL} value will retrieve all | ||
6015 | available statistics), a continuation callback which is called when all of | ||
6016 | requested information is retrieved, an iterator callback which is called for | ||
6017 | each parameter in the retrieved information and a closure for the | ||
6018 | aforementioned callbacks. The library then invokes the iterator callback for | ||
6019 | each value matching the request. | ||
6020 | |||
6021 | Call to @code{GNUNET_STATISTICS_get()} is asynchronous and can be canceled with | ||
6022 | the function @code{GNUNET_STATISTICS_get_cancel()}. This is helpful when | ||
6023 | retrieving statistics takes too long and especially when we want to shutdown | ||
6024 | and cleanup everything. | ||
6025 | |||
6026 | @node Setting statistics and updating them | ||
6027 | @subsubsection Setting statistics and updating them | ||
6028 | |||
6029 | @c %**end of header | ||
6030 | |||
6031 | So far we have seen how to retrieve statistics, here we will learn how we can | ||
6032 | set statistics and update them so that other subsystems can retrieve them. | ||
6033 | |||
6034 | A new statistic can be set using the function @code{GNUNET_STATISTICS_set()}. | ||
6035 | This function takes the name of the statistic and its value and a flag to make | ||
6036 | the statistic persistent. The value of the statistic should be of the type | ||
6037 | @code{uint64_t}. The function does not take the name of the subsystem; it is | ||
6038 | determined from the previous @code{GNUNET_STATISTICS_create()} invocation. If | ||
6039 | the given statistic is already present, its value is overwritten. | ||
6040 | |||
6041 | An existing statistics can be updated, i.e its value can be increased or | ||
6042 | decreased by an amount with the function @code{GNUNET_STATISTICS_update()}. The | ||
6043 | parameters to this function are similar to @code{GNUNET_STATISTICS_set()}, | ||
6044 | except that it takes the amount to be changed as a type @code{int64_t} instead | ||
6045 | of the value. | ||
6046 | |||
6047 | The library will combine multiple set or update operations into one message if | ||
6048 | the client performs requests at a rate that is faster than the available IPC | ||
6049 | with the STATISTICS service. Thus, the client does not have to worry about | ||
6050 | sending requests too quickly. | ||
6051 | |||
6052 | @node Watches | ||
6053 | @subsubsection Watches | ||
6054 | |||
6055 | @c %**end of header | ||
6056 | |||
6057 | As interesting feature of STATISTICS lies in serving notifications whenever a | ||
6058 | statistic of our interest is modified. This is achieved by registering a watch | ||
6059 | through the function @code{GNUNET_STATISTICS_watch()}. The parameters of this | ||
6060 | function are similar to those of @code{GNUNET_STATISTICS_get()}. Changes to the | ||
6061 | respective statistic's value will then cause the given iterator callback to be | ||
6062 | called. Note: A watch can only be registered for a specific statistic. Hence | ||
6063 | the subsystem name and the parameter name cannot be @code{NULL} in a call to | ||
6064 | @code{GNUNET_STATISTICS_watch()}. | ||
6065 | |||
6066 | A registered watch will keep notifying any value changes until | ||
6067 | @code{GNUNET_STATISTICS_watch_cancel()} is called with the same parameters that | ||
6068 | are used for registering the watch. | ||
6069 | |||
6070 | @node The STATISTICS Client-Service Protocol | ||
6071 | @subsection The STATISTICS Client-Service Protocol | ||
6072 | @c %**end of header | ||
6073 | |||
6074 | |||
6075 | @menu | ||
6076 | * Statistics retrieval2:: | ||
6077 | * Setting and updating statistics:: | ||
6078 | * Watching for updates:: | ||
6079 | @end menu | ||
6080 | |||
6081 | @node Statistics retrieval2 | ||
6082 | @subsubsection Statistics retrieval2 | ||
6083 | |||
6084 | @c %**end of header | ||
6085 | |||
6086 | To retrieve statistics, the client transmits a message of type | ||
6087 | @code{GNUNET_MESSAGE_TYPE_STATISTICS_GET} containing the given subsystem name | ||
6088 | and statistic parameter to the STATISTICS service. The service responds with a | ||
6089 | message of type @code{GNUNET_MESSAGE_TYPE_STATISTICS_VALUE} for each of the | ||
6090 | statistics parameters that match the client request for the client. The end of | ||
6091 | information retrieved is signaled by the service by sending a message of type | ||
6092 | @code{GNUNET_MESSAGE_TYPE_STATISTICS_END}. | ||
6093 | |||
6094 | @node Setting and updating statistics | ||
6095 | @subsubsection Setting and updating statistics | ||
6096 | |||
6097 | @c %**end of header | ||
6098 | |||
6099 | The subsystem name, parameter name, its value and the persistence flag are | ||
6100 | communicated to the service through the message | ||
6101 | @code{GNUNET_MESSAGE_TYPE_STATISTICS_SET}. | ||
6102 | |||
6103 | When the service receives a message of type | ||
6104 | @code{GNUNET_MESSAGE_TYPE_STATISTICS_SET}, it retrieves the subsystem name and | ||
6105 | checks for a statistic parameter with matching the name given in the message. | ||
6106 | If a statistic parameter is found, the value is overwritten by the new value | ||
6107 | from the message; if not found then a new statistic parameter is created with | ||
6108 | the given name and value. | ||
6109 | |||
6110 | In addition to just setting an absolute value, it is possible to perform a | ||
6111 | relative update by sending a message of type | ||
6112 | @code{GNUNET_MESSAGE_TYPE_STATISTICS_SET} with an update flag | ||
6113 | (@code{GNUNET_STATISTICS_SETFLAG_RELATIVE}) signifying that the value in the | ||
6114 | message should be treated as an update value. | ||
6115 | |||
6116 | @node Watching for updates | ||
6117 | @subsubsection Watching for updates | ||
6118 | |||
6119 | @c %**end of header | ||
6120 | |||
6121 | The function registers the watch at the service by sending a message of type | ||
6122 | @code{GNUNET_MESSAGE_TYPE_STATISTICS_WATCH}. The service then sends | ||
6123 | notifications through messages of type | ||
6124 | @code{GNUNET_MESSAGE_TYPE_STATISTICS_WATCH_VALUE} whenever the statistic | ||
6125 | parameter's value is changed. | ||
6126 | |||
6127 | @node GNUnet's Distributed Hash Table (DHT) | ||
6128 | @section GNUnet's Distributed Hash Table (DHT) | ||
6129 | |||
6130 | @c %**end of header | ||
6131 | |||
6132 | GNUnet includes a generic distributed hash table that can be used by developers | ||
6133 | building P2P applications in the framework. This section documents high-level | ||
6134 | features and how developers are expected to use the DHT. We have a research | ||
6135 | paper detailing how the DHT works. Also, Nate's thesis includes a detailed | ||
6136 | description and performance analysis (in chapter 6). | ||
6137 | |||
6138 | Key features of GNUnet's DHT include: | ||
6139 | |||
6140 | @itemize @bullet | ||
6141 | @item stores key-value pairs with values up to (approximately) 63k in size | ||
6142 | @item works with many underlay network topologies (small-world, random graph), | ||
6143 | underlay does not need to be a full mesh / clique | ||
6144 | @item support for extended queries (more than just a simple 'key'), filtering | ||
6145 | duplicate replies within the network (bloomfilter) and content validation (for | ||
6146 | details, please read the subsection on the block library) | ||
6147 | @item can (optionally) return paths taken by the PUT and GET operations to the | ||
6148 | application | ||
6149 | @item provides content replication to handle churn | ||
6150 | @end itemize | ||
6151 | |||
6152 | GNUnet's DHT is randomized and unreliable. Unreliable means that there is no | ||
6153 | strict guarantee that a value stored in the DHT is always found --- values are | ||
6154 | only found with high probability. While this is somewhat true in all P2P DHTs, | ||
6155 | GNUnet developers should be particularly wary of this fact (this will help you | ||
6156 | write secure, fault-tolerant code). Thus, when writing any application using | ||
6157 | the DHT, you should always consider the possibility that a value stored in the | ||
6158 | DHT by you or some other peer might simply not be returned, or returned with a | ||
6159 | significant delay. Your application logic must be written to tolerate this | ||
6160 | (naturally, some loss of performance or quality of service is expected in this | ||
6161 | case). | ||
6162 | |||
6163 | @menu | ||
6164 | * Block library and plugins:: | ||
6165 | * libgnunetdht:: | ||
6166 | * The DHT Client-Service Protocol:: | ||
6167 | * The DHT Peer-to-Peer Protocol:: | ||
6168 | @end menu | ||
6169 | |||
6170 | @node Block library and plugins | ||
6171 | @subsection Block library and plugins | ||
6172 | |||
6173 | @c %**end of header | ||
6174 | |||
6175 | @menu | ||
6176 | * What is a Block?:: | ||
6177 | * The API of libgnunetblock:: | ||
6178 | * Queries:: | ||
6179 | * Sample Code:: | ||
6180 | * Conclusion2:: | ||
6181 | @end menu | ||
6182 | |||
6183 | @node What is a Block? | ||
6184 | @subsubsection What is a Block? | ||
6185 | |||
6186 | @c %**end of header | ||
6187 | |||
6188 | Blocks are small (< 63k) pieces of data stored under a key (struct | ||
6189 | GNUNET_HashCode). Blocks have a type (enum GNUNET_BlockType) which defines | ||
6190 | their data format. Blocks are used in GNUnet as units of static data exchanged | ||
6191 | between peers and stored (or cached) locally. Uses of blocks include | ||
6192 | file-sharing (the files are broken up into blocks), the VPN (DNS information is | ||
6193 | stored in blocks) and the DHT (all information in the DHT and meta-information | ||
6194 | for the maintenance of the DHT are both stored using blocks). The block | ||
6195 | subsystem provides a few common functions that must be available for any type | ||
6196 | of block. | ||
6197 | |||
6198 | @node The API of libgnunetblock | ||
6199 | @subsubsection The API of libgnunetblock | ||
6200 | |||
6201 | @c %**end of header | ||
6202 | |||
6203 | The block library requires for each (family of) block type(s) a block plugin | ||
6204 | (implementing gnunet_block_plugin.h) that provides basic functions that are | ||
6205 | needed by the DHT (and possibly other subsystems) to manage the block. These | ||
6206 | block plugins are typically implemented within their respective subsystems.@ | ||
6207 | The main block library is then used to locate, load and query the appropriate | ||
6208 | block plugin. Which plugin is appropriate is determined by the block type | ||
6209 | (which is just a 32-bit integer). Block plugins contain code that specifies | ||
6210 | which block types are supported by a given plugin. The block library loads all | ||
6211 | block plugins that are installed at the local peer and forwards the application | ||
6212 | request to the respective plugin. | ||
6213 | |||
6214 | The central functions of the block APIs (plugin and main library) are to allow | ||
6215 | the mapping of blocks to their respective key (if possible) and the ability to | ||
6216 | check that a block is well-formed and matches a given request (again, if | ||
6217 | possible). This way, GNUnet can avoid storing invalid blocks, storing blocks | ||
6218 | under the wrong key and forwarding blocks in response to a query that they do | ||
6219 | not answer. | ||
6220 | |||
6221 | One key function of block plugins is that it allows GNUnet to detect duplicate | ||
6222 | replies (via the Bloom filter). All plugins MUST support detecting duplicate | ||
6223 | replies (by adding the current response to the Bloom filter and rejecting it if | ||
6224 | it is encountered again). If a plugin fails to do this, responses may loop in | ||
6225 | the network. | ||
6226 | |||
6227 | @node Queries | ||
6228 | @subsubsection Queries | ||
6229 | @c %**end of header | ||
6230 | |||
6231 | The query format for any block in GNUnet consists of four main components. | ||
6232 | First, the type of the desired block must be specified. Second, the query must | ||
6233 | contain a hash code. The hash code is used for lookups in hash tables and | ||
6234 | databases and must not be unique for the block (however, if possible a unique | ||
6235 | hash should be used as this would be best for performance). Third, an optional | ||
6236 | Bloom filter can be specified to exclude known results; replies that hash to | ||
6237 | the bits set in the Bloom filter are considered invalid. False-positives can be | ||
6238 | eliminated by sending the same query again with a different Bloom filter | ||
6239 | mutator value, which parameterizes the hash function that is used. Finally, an | ||
6240 | optional application-specific "eXtended query" (xquery) can be specified to | ||
6241 | further constrain the results. It is entirely up to the type-specific plugin to | ||
6242 | determine whether or not a given block matches a query (type, hash, Bloom | ||
6243 | filter, and xquery). Naturally, not all xquery's are valid and some types of | ||
6244 | blocks may not support Bloom filters either, so the plugin also needs to check | ||
6245 | if the query is valid in the first place. | ||
6246 | |||
6247 | Depending on the results from the plugin, the DHT will then discard the | ||
6248 | (invalid) query, forward the query, discard the (invalid) reply, cache the | ||
6249 | (valid) reply, and/or forward the (valid and non-duplicate) reply. | ||
6250 | |||
6251 | @node Sample Code | ||
6252 | @subsubsection Sample Code | ||
6253 | |||
6254 | @c %**end of header | ||
6255 | |||
6256 | The source code in @strong{plugin_block_test.c} is a good starting point for | ||
6257 | new block plugins --- it does the minimal work by implementing a plugin that | ||
6258 | performs no validation at all. The respective @strong{Makefile.am} shows how to | ||
6259 | build and install a block plugin. | ||
6260 | |||
6261 | @node Conclusion2 | ||
6262 | @subsubsection Conclusion2 | ||
6263 | |||
6264 | @c %**end of header | ||
6265 | |||
6266 | In conclusion, GNUnet subsystems that want to use the DHT need to define a | ||
6267 | block format and write a plugin to match queries and replies. For testing, the | ||
6268 | "GNUNET_BLOCK_TYPE_TEST" block type can be used; it accepts any query as valid | ||
6269 | and any reply as matching any query. This type is also used for the DHT command | ||
6270 | line tools. However, it should NOT be used for normal applications due to the | ||
6271 | lack of error checking that results from this primitive implementation. | ||
6272 | |||
6273 | @node libgnunetdht | ||
6274 | @subsection libgnunetdht | ||
6275 | |||
6276 | @c %**end of header | ||
6277 | |||
6278 | The DHT API itself is pretty simple and offers the usual GET and PUT functions | ||
6279 | that work as expected. The specified block type refers to the block library | ||
6280 | which allows the DHT to run application-specific logic for data stored in the | ||
6281 | network. | ||
6282 | |||
6283 | |||
6284 | @menu | ||
6285 | * GET:: | ||
6286 | * PUT:: | ||
6287 | * MONITOR:: | ||
6288 | * DHT Routing Options:: | ||
6289 | @end menu | ||
6290 | |||
6291 | @node GET | ||
6292 | @subsubsection GET | ||
6293 | |||
6294 | @c %**end of header | ||
6295 | |||
6296 | When using GET, the main consideration for developers (other than the block | ||
6297 | library) should be that after issuing a GET, the DHT will continuously cause | ||
6298 | (small amounts of) network traffic until the operation is explicitly canceled. | ||
6299 | So GET does not simply send out a single network request once; instead, the | ||
6300 | DHT will continue to search for data. This is needed to achieve good success | ||
6301 | rates and also handles the case where the respective PUT operation happens | ||
6302 | after the GET operation was started. Developers should not cancel an existing | ||
6303 | GET operation and then explicitly re-start it to trigger a new round of | ||
6304 | network requests; this is simply inefficient, especially as the internal | ||
6305 | automated version can be more efficient, for example by filtering results in | ||
6306 | the network that have already been returned. | ||
6307 | |||
6308 | If an application that performs a GET request has a set of replies that it | ||
6309 | already knows and would like to filter, it can call@ | ||
6310 | @code{GNUNET_DHT_get_filter_known_results} with an array of hashes over the | ||
6311 | respective blocks to tell the DHT that these results are not desired (any | ||
6312 | more). This way, the DHT will filter the respective blocks using the block | ||
6313 | library in the network, which may result in a significant reduction in | ||
6314 | bandwidth consumption. | ||
6315 | |||
6316 | @node PUT | ||
6317 | @subsubsection PUT | ||
6318 | |||
6319 | @c %**end of header | ||
6320 | |||
6321 | In contrast to GET operations, developers @strong{must} manually re-run PUT | ||
6322 | operations periodically (if they intend the content to continue to be | ||
6323 | available). Content stored in the DHT expires or might be lost due to churn. | ||
6324 | Furthermore, GNUnet's DHT typically requires multiple rounds of PUT operations | ||
6325 | before a key-value pair is consistently available to all peers (the DHT | ||
6326 | randomizes paths and thus storage locations, and only after multiple rounds of | ||
6327 | PUTs there will be a sufficient number of replicas in large DHTs). An explicit | ||
6328 | PUT operation using the DHT API will only cause network traffic once, so in | ||
6329 | order to ensure basic availability and resistance to churn (and adversaries), | ||
6330 | PUTs must be repeated. While the exact frequency depends on the application, a | ||
6331 | rule of thumb is that there should be at least a dozen PUT operations within | ||
6332 | the content lifetime. Content in the DHT typically expires after one day, so | ||
6333 | DHT PUT operations should be repeated at least every 1-2 hours. | ||
6334 | |||
6335 | @node MONITOR | ||
6336 | @subsubsection MONITOR | ||
6337 | |||
6338 | @c %**end of header | ||
6339 | |||
6340 | The DHT API also allows applications to monitor messages crossing the local | ||
6341 | DHT service. The types of messages used by the DHT are GET, PUT and RESULT | ||
6342 | messages. Using the monitoring API, applications can choose to monitor these | ||
6343 | requests, possibly limiting themselves to requests for a particular block | ||
6344 | type. | ||
6345 | |||
6346 | The monitoring API is not only usefu only for diagnostics, it can also be used | ||
6347 | to trigger application operations based on PUT operations. For example, an | ||
6348 | application may use PUTs to distribute work requests to other peers. The | ||
6349 | workers would then monitor for PUTs that give them work, instead of looking | ||
6350 | for work using GET operations. This can be beneficial, especially if the | ||
6351 | workers have no good way to guess the keys under which work would be stored. | ||
6352 | Naturally, additional protocols might be needed to ensure that the desired | ||
6353 | number of workers will process the distributed workload. | ||
6354 | |||
6355 | @node DHT Routing Options | ||
6356 | @subsubsection DHT Routing Options | ||
6357 | |||
6358 | @c %**end of header | ||
6359 | |||
6360 | There are two important options for GET and PUT requests: | ||
6361 | |||
6362 | @table @asis | ||
6363 | @item GNUNET_DHT_RO_DEMULITPLEX_EVERYWHERE This option means that all peers | ||
6364 | should process the request, even if their peer ID is not closest to the key. | ||
6365 | For a PUT request, this means that all peers that a request traverses may make | ||
6366 | a copy of the data. Similarly for a GET request, all peers will check their | ||
6367 | local database for a result. Setting this option can thus significantly improve | ||
6368 | caching and reduce bandwidth consumption --- at the expense of a larger DHT | ||
6369 | database. If in doubt, we recommend that this option should be used. | ||
6370 | @item GNUNET_DHT_RO_RECORD_ROUTE This option instructs the DHT to record the path | ||
6371 | that a GET or a PUT request is taking through the overlay network. The | ||
6372 | resulting paths are then returned to the application with the respective | ||
6373 | result. This allows the receiver of a result to construct a path to the | ||
6374 | originator of the data, which might then be used for routing. Naturally, | ||
6375 | setting this option requires additional bandwidth and disk space, so | ||
6376 | applications should only set this if the paths are needed by the application | ||
6377 | logic. | ||
6378 | @item GNUNET_DHT_RO_FIND_PEER This option is an internal option used by | ||
6379 | the DHT's peer discovery mechanism and should not be used by applications. | ||
6380 | @item GNUNET_DHT_RO_BART This option is currently not implemented. It may in | ||
6381 | the future offer performance improvements for clique topologies. | ||
6382 | @end table | ||
6383 | |||
6384 | @node The DHT Client-Service Protocol | ||
6385 | @subsection The DHT Client-Service Protocol | ||
6386 | |||
6387 | @c %**end of header | ||
6388 | |||
6389 | @menu | ||
6390 | * PUTting data into the DHT:: | ||
6391 | * GETting data from the DHT:: | ||
6392 | * Monitoring the DHT:: | ||
6393 | @end menu | ||
6394 | |||
6395 | @node PUTting data into the DHT | ||
6396 | @subsubsection PUTting data into the DHT | ||
6397 | |||
6398 | @c %**end of header | ||
6399 | |||
6400 | To store (PUT) data into the DHT, the client sends a@ @code{struct | ||
6401 | GNUNET_DHT_ClientPutMessage} to the service. This message specifies the block | ||
6402 | type, routing options, the desired replication level, the expiration time, key, | ||
6403 | value and a 64-bit unique ID for the operation. The service responds with a@ | ||
6404 | @code{struct GNUNET_DHT_ClientPutConfirmationMessage} with the same 64-bit | ||
6405 | unique ID. Note that the service sends the confirmation as soon as it has | ||
6406 | locally processed the PUT request. The PUT may still be propagating through the | ||
6407 | network at this time. | ||
6408 | |||
6409 | In the future, we may want to change this to provide (limited) feedback to the | ||
6410 | client, for example if we detect that the PUT operation had no effect because | ||
6411 | the same key-value pair was already stored in the DHT. However, changing this | ||
6412 | would also require additional state and messages in the P2P | ||
6413 | interaction. | ||
6414 | |||
6415 | @node GETting data from the DHT | ||
6416 | @subsubsection GETting data from the DHT | ||
6417 | |||
6418 | @c %**end of header | ||
6419 | |||
6420 | To retrieve (GET) data from the DHT, the client sends a@ @code{struct | ||
6421 | GNUNET_DHT_ClientGetMessage} to the service. The message specifies routing | ||
6422 | options, a replication level (for replicating the GET, not the content), the | ||
6423 | desired block type, the key, the (optional) extended query and unique 64-bit | ||
6424 | request ID. | ||
6425 | |||
6426 | Additionally, the client may send any number of@ @code{struct | ||
6427 | GNUNET_DHT_ClientGetResultSeenMessage}s to notify the service about results | ||
6428 | that the client is already aware of. These messages consist of the key, the | ||
6429 | unique 64-bit ID of the request, and an arbitrary number of hash codes over the | ||
6430 | blocks that the client is already aware of. As messages are restricted to 64k, | ||
6431 | a client that already knows more than about a thousand blocks may need to send | ||
6432 | several of these messages. Naturally, the client should transmit these messages | ||
6433 | as quickly as possible after the original GET request such that the DHT can | ||
6434 | filter those results in the network early on. Naturally, as these messages are | ||
6435 | send after the original request, it is conceivalbe that the DHT service may | ||
6436 | return blocks that match those already known to the client anyway. | ||
6437 | |||
6438 | In response to a GET request, the service will send @code{struct | ||
6439 | GNUNET_DHT_ClientResultMessage}s to the client. These messages contain the | ||
6440 | block type, expiration, key, unique ID of the request and of course the value | ||
6441 | (a block). Depending on the options set for the respective operations, the | ||
6442 | replies may also contain the path the GET and/or the PUT took through the | ||
6443 | network. | ||
6444 | |||
6445 | A client can stop receiving replies either by disconnecting or by sending a | ||
6446 | @code{struct GNUNET_DHT_ClientGetStopMessage} which must contain the key and | ||
6447 | the 64-bit unique ID of the original request. Using an explicit "stop" message | ||
6448 | is more common as this allows a client to run many concurrent GET operations | ||
6449 | over the same connection with the DHT service --- and to stop them | ||
6450 | individually. | ||
6451 | |||
6452 | @node Monitoring the DHT | ||
6453 | @subsubsection Monitoring the DHT | ||
6454 | |||
6455 | @c %**end of header | ||
6456 | |||
6457 | To begin monitoring, the client sends a @code{struct | ||
6458 | GNUNET_DHT_MonitorStartStop} message to the DHT service. In this message, flags | ||
6459 | can be set to enable (or disable) monitoring of GET, PUT and RESULT messages | ||
6460 | that pass through a peer. The message can also restrict monitoring to a | ||
6461 | particular block type or a particular key. Once monitoring is enabled, the DHT | ||
6462 | service will notify the client about any matching event using @code{struct | ||
6463 | GNUNET_DHT_MonitorGetMessage}s for GET events, @code{struct | ||
6464 | GNUNET_DHT_MonitorPutMessage} for PUT events and@ @code{struct | ||
6465 | GNUNET_DHT_MonitorGetRespMessage} for RESULTs. Each of these messages contains | ||
6466 | all of the information about the event. | ||
6467 | |||
6468 | @node The DHT Peer-to-Peer Protocol | ||
6469 | @subsection The DHT Peer-to-Peer Protocol | ||
6470 | @c %**end of header | ||
6471 | |||
6472 | |||
6473 | @menu | ||
6474 | * Routing GETs or PUTs:: | ||
6475 | * PUTting data into the DHT2:: | ||
6476 | * GETting data from the DHT2:: | ||
6477 | @end menu | ||
6478 | |||
6479 | @node Routing GETs or PUTs | ||
6480 | @subsubsection Routing GETs or PUTs | ||
6481 | |||
6482 | @c %**end of header | ||
6483 | |||
6484 | When routing GETs or PUTs, the DHT service selects a suitable subset of | ||
6485 | neighbours for forwarding. The exact number of neighbours can be zero or more | ||
6486 | and depends on the hop counter of the query (initially zero) in relation to the | ||
6487 | (log of) the network size estimate, the desired replication level and the | ||
6488 | peer's connectivity. Depending on the hop counter and our network size | ||
6489 | estimate, the selection of the peers maybe randomized or by proximity to the | ||
6490 | key. Furthermore, requests include a set of peers that a request has already | ||
6491 | traversed; those peers are also excluded from the selection. | ||
6492 | |||
6493 | @node PUTting data into the DHT2 | ||
6494 | @subsubsection PUTting data into the DHT2 | ||
6495 | |||
6496 | @c %**end of header | ||
6497 | |||
6498 | To PUT data into the DHT, the service sends a @code{struct PeerPutMessage} of | ||
6499 | type @code{GNUNET_MESSAGE_TYPE_DHT_P2P_PUT} to the respective neighbour. In | ||
6500 | addition to the usual information about the content (type, routing options, | ||
6501 | desired replication level for the content, expiration time, key and value), the | ||
6502 | message contains a fixed-size Bloom filter with information about which peers | ||
6503 | (may) have already seen this request. This Bloom filter is used to ensure that | ||
6504 | DHT messages never loop back to a peer that has already processed the request. | ||
6505 | Additionally, the message includes the current hop counter and, depending on | ||
6506 | the routing options, the message may include the full path that the message has | ||
6507 | taken so far. The Bloom filter should already contain the identity of the | ||
6508 | previous hop; however, the path should not include the identity of the previous | ||
6509 | hop and the receiver should append the identity of the sender to the path, not | ||
6510 | its own identity (this is done to reduce bandwidth). | ||
6511 | |||
6512 | @node GETting data from the DHT2 | ||
6513 | @subsubsection GETting data from the DHT2 | ||
6514 | |||
6515 | @c %**end of header | ||
6516 | |||
6517 | A peer can search the DHT by sending @code{struct PeerGetMessage}s of type | ||
6518 | @code{GNUNET_MESSAGE_TYPE_DHT_P2P_GET} to other peers. In addition to the usual | ||
6519 | information about the request (type, routing options, desired replication level | ||
6520 | for the request, the key and the extended query), a GET request also again | ||
6521 | contains a hop counter, a Bloom filter over the peers that have processed the | ||
6522 | request already and depending on the routing options the full path traversed by | ||
6523 | the GET. Finally, a GET request includes a variable-size second Bloom filter | ||
6524 | and a so-called Bloom filter mutator value which together indicate which | ||
6525 | replies the sender has already seen. During the lookup, each block that matches | ||
6526 | they block type, key and extended query is additionally subjected to a test | ||
6527 | against this Bloom filter. The block plugin is expected to take the hash of the | ||
6528 | block and combine it with the mutator value and check if the result is not yet | ||
6529 | in the Bloom filter. The originator of the query will from time to time modify | ||
6530 | the mutator to (eventually) allow false-positives filtered by the Bloom filter | ||
6531 | to be returned. | ||
6532 | |||
6533 | Peers that receive a GET request perform a local lookup (depending on their | ||
6534 | proximity to the key and the query options) and forward the request to other | ||
6535 | peers. They then remember the request (including the Bloom filter for blocking | ||
6536 | duplicate results) and when they obtain a matching, non-filtered response a | ||
6537 | @code{struct PeerResultMessage} of type@ | ||
6538 | @code{GNUNET_MESSAGE_TYPE_DHT_P2P_RESULT} is forwarded to the previous hop. | ||
6539 | Whenver a result is forwarded, the block plugin is used to update the Bloom | ||
6540 | filter accordingly, to ensure that the same result is never forwarded more than | ||
6541 | once. The DHT service may also cache forwarded results locally if the | ||
6542 | "CACHE_RESULTS" option is set to "YES" in the configuration. | ||
6543 | |||
6544 | @node The GNU Name System (GNS) | ||
6545 | @section The GNU Name System (GNS) | ||
6546 | |||
6547 | @c %**end of header | ||
6548 | |||
6549 | The GNU Name System (GNS) is a decentralized database that enables users to | ||
6550 | securely resolve names to values. Names can be used to identify other users | ||
6551 | (for example, in social networking), or network services (for example, VPN | ||
6552 | services running at a peer in GNUnet, or purely IP-based services on the | ||
6553 | Internet). Users interact with GNS by typing in a hostname that ends in ".gnu" | ||
6554 | or ".zkey". | ||
6555 | |||
6556 | Videos giving an overview of most of the GNS and the motivations behind it is | ||
6557 | available here and here. The remainder of this chapter targets developers that | ||
6558 | are familiar with high level concepts of GNS as presented in these talks. | ||
6559 | |||
6560 | GNS-aware applications should use the GNS resolver to obtain the respective | ||
6561 | records that are stored under that name in GNS. Each record consists of a type, | ||
6562 | value, expiration time and flags. | ||
6563 | |||
6564 | The type specifies the format of the value. Types below 65536 correspond to DNS | ||
6565 | record types, larger values are used for GNS-specific records. Applications can | ||
6566 | define new GNS record types by reserving a number and implementing a plugin | ||
6567 | (which mostly needs to convert the binary value representation to a | ||
6568 | human-readable text format and vice-versa). The expiration time specifies how | ||
6569 | long the record is to be valid. The GNS API ensures that applications are only | ||
6570 | given non-expired values. The flags are typically irrelevant for applications, | ||
6571 | as GNS uses them internally to control visibility and validity of records. | ||
6572 | |||
6573 | Records are stored along with a signature. The signature is generated using the | ||
6574 | private key of the authoritative zone. This allows any GNS resolver to verify | ||
6575 | the correctness of a name-value mapping. | ||
6576 | |||
6577 | Internally, GNS uses the NAMECACHE to cache information obtained from other | ||
6578 | users, the NAMESTORE to store information specific to the local users, and the | ||
6579 | DHT to exchange data between users. A plugin API is used to enable applications | ||
6580 | to define new GNS record types. | ||
6581 | |||
6582 | @menu | ||
6583 | * libgnunetgns:: | ||
6584 | * libgnunetgnsrecord:: | ||
6585 | * GNS plugins:: | ||
6586 | * The GNS Client-Service Protocol:: | ||
6587 | * Hijacking the DNS-Traffic using gnunet-service-dns:: | ||
6588 | * Serving DNS lookups via GNS on W32:: | ||
6589 | @end menu | ||
6590 | |||
6591 | @node libgnunetgns | ||
6592 | @subsection libgnunetgns | ||
6593 | |||
6594 | @c %**end of header | ||
6595 | |||
6596 | The GNS API itself is extremely simple. Clients first connec to the GNS service | ||
6597 | using @code{GNUNET_GNS_connect}. They can then perform lookups using | ||
6598 | @code{GNUNET_GNS_lookup} or cancel pending lookups using | ||
6599 | @code{GNUNET_GNS_lookup_cancel}. Once finished, clients disconnect using | ||
6600 | @code{GNUNET_GNS_disconnect}. | ||
6601 | |||
6602 | |||
6603 | @menu | ||
6604 | * Looking up records:: | ||
6605 | * Accessing the records:: | ||
6606 | * Creating records:: | ||
6607 | * Future work:: | ||
6608 | @end menu | ||
6609 | |||
6610 | @node Looking up records | ||
6611 | @subsubsection Looking up records | ||
6612 | |||
6613 | @c %**end of header | ||
6614 | |||
6615 | @code{GNUNET_GNS_lookup} takes a number of arguments: | ||
6616 | |||
6617 | @table @asis | ||
6618 | @item handle This is simply the GNS connection handle from | ||
6619 | @code{GNUNET_GNS_connect}. | ||
6620 | @item name The client needs to specify the name to | ||
6621 | be resolved. This can be any valid DNS or GNS hostname. | ||
6622 | @item zone The client | ||
6623 | needs to specify the public key of the GNS zone against which the resolution | ||
6624 | should be done (the ".gnu" zone). Note that a key must be provided, even if the | ||
6625 | name ends in ".zkey". This should typically be the public key of the | ||
6626 | master-zone of the user. | ||
6627 | @item type This is the desired GNS or DNS record type | ||
6628 | to look for. While all records for the given name will be returned, this can be | ||
6629 | important if the client wants to resolve record types that themselves delegate | ||
6630 | resolution, such as CNAME, PKEY or GNS2DNS. Resolving a record of any of these | ||
6631 | types will only work if the respective record type is specified in the request, | ||
6632 | as the GNS resolver will otherwise follow the delegation and return the records | ||
6633 | from the respective destination, instead of the delegating record. | ||
6634 | @item only_cached This argument should typically be set to @code{GNUNET_NO}. Setting | ||
6635 | it to @code{GNUNET_YES} disables resolution via the overlay network. | ||
6636 | @item shorten_zone_key If GNS encounters new names during resolution, their | ||
6637 | respective zones can automatically be learned and added to the "shorten zone". | ||
6638 | If this is desired, clients must pass the private key of the shorten zone. If | ||
6639 | NULL is passed, shortening is disabled. | ||
6640 | @item proc This argument identifies | ||
6641 | the function to call with the result. It is given proc_cls, the number of | ||
6642 | records found (possilby zero) and the array of the records as arguments. proc | ||
6643 | will only be called once. After proc,> has been called, the lookup must no | ||
6644 | longer be cancelled. | ||
6645 | @item proc_cls The closure for proc. | ||
6646 | @end table | ||
6647 | |||
6648 | @node Accessing the records | ||
6649 | @subsubsection Accessing the records | ||
6650 | |||
6651 | @c %**end of header | ||
6652 | |||
6653 | The @code{libgnunetgnsrecord} library provides an API to manipulate the GNS | ||
6654 | record array that is given to proc. In particular, it offers functions such as | ||
6655 | converting record values to human-readable strings (and back). However, most | ||
6656 | @code{libgnunetgnsrecord} functions are not interesting to GNS client | ||
6657 | applications. | ||
6658 | |||
6659 | For DNS records, the @code{libgnunetdnsparser} library provides functions for | ||
6660 | parsing (and serializing) common types of DNS records. | ||
6661 | |||
6662 | @node Creating records | ||
6663 | @subsubsection Creating records | ||
6664 | |||
6665 | @c %**end of header | ||
6666 | |||
6667 | Creating GNS records is typically done by building the respective record | ||
6668 | information (possibly with the help of @code{libgnunetgnsrecord} and | ||
6669 | @code{libgnunetdnsparser}) and then using the @code{libgnunetnamestore} to | ||
6670 | publish the information. The GNS API is not involved in this | ||
6671 | operation. | ||
6672 | |||
6673 | @node Future work | ||
6674 | @subsubsection Future work | ||
6675 | |||
6676 | @c %**end of header | ||
6677 | |||
6678 | In the future, we want to expand @code{libgnunetgns} to allow applications to | ||
6679 | observe shortening operations performed during GNS resolution, for example so | ||
6680 | that users can receive visual feedback when this happens. | ||
6681 | |||
6682 | @node libgnunetgnsrecord | ||
6683 | @subsection libgnunetgnsrecord | ||
6684 | |||
6685 | @c %**end of header | ||
6686 | |||
6687 | The @code{libgnunetgnsrecord} library is used to manipulate GNS records (in | ||
6688 | plaintext or in their encrypted format). Applications mostly interact with | ||
6689 | @code{libgnunetgnsrecord} by using the functions to convert GNS record values | ||
6690 | to strings or vice-versa, or to lookup a GNS record type number by name (or | ||
6691 | vice-versa). The library also provides various other functions that are mostly | ||
6692 | used internally within GNS, such as converting keys to names, checking for | ||
6693 | expiration, encrypting GNS records to GNS blocks, verifying GNS block | ||
6694 | signatures and decrypting GNS records from GNS blocks. | ||
6695 | |||
6696 | We will now discuss the four commonly used functions of the API.@ | ||
6697 | @code{libgnunetgnsrecord} does not perform these operations itself, but instead | ||
6698 | uses plugins to perform the operation. GNUnet includes plugins to support | ||
6699 | common DNS record types as well as standard GNS record types. | ||
6700 | |||
6701 | |||
6702 | @menu | ||
6703 | * Value handling:: | ||
6704 | * Type handling:: | ||
6705 | @end menu | ||
6706 | |||
6707 | @node Value handling | ||
6708 | @subsubsection Value handling | ||
6709 | |||
6710 | @c %**end of header | ||
6711 | |||
6712 | @code{GNUNET_GNSRECORD_value_to_string} can be used to convert the (binary) | ||
6713 | representation of a GNS record value to a human readable, 0-terminated UTF-8 | ||
6714 | string. NULL is returned if the specified record type is not supported by any | ||
6715 | available plugin. | ||
6716 | |||
6717 | @code{GNUNET_GNSRECORD_string_to_value} can be used to try to convert a human | ||
6718 | readable string to the respective (binary) representation of a GNS record | ||
6719 | value. | ||
6720 | |||
6721 | @node Type handling | ||
6722 | @subsubsection Type handling | ||
6723 | |||
6724 | @c %**end of header | ||
6725 | |||
6726 | @code{GNUNET_GNSRECORD_typename_to_number} can be used to obtain the numeric | ||
6727 | value associated with a given typename. For example, given the typename "A" | ||
6728 | (for DNS A reocrds), the function will return the number 1. A list of common | ||
6729 | DNS record types is | ||
6730 | @uref{http://en.wikipedia.org/wiki/List_of_DNS_record_types, here. Note that | ||
6731 | not all DNS record types are supported by GNUnet GNSRECORD plugins at this | ||
6732 | time.} | ||
6733 | |||
6734 | @code{GNUNET_GNSRECORD_number_to_typename} can be used to obtain the typename | ||
6735 | associated with a given numeric value. For example, given the type number 1, | ||
6736 | the function will return the typename "A". | ||
6737 | |||
6738 | @node GNS plugins | ||
6739 | @subsection GNS plugins | ||
6740 | |||
6741 | @c %**end of header | ||
6742 | |||
6743 | Adding a new GNS record type typically involves writing (or extending) a | ||
6744 | GNSRECORD plugin. The plugin needs to implement the | ||
6745 | @code{gnunet_gnsrecord_plugin.h} API which provides basic functions that are | ||
6746 | needed by GNSRECORD to convert typenames and values of the respective record | ||
6747 | type to strings (and back). These gnsrecord plugins are typically implemented | ||
6748 | within their respective subsystems. Examples for such plugins can be found in | ||
6749 | the GNSRECORD, GNS and CONVERSATION subsystems. | ||
6750 | |||
6751 | The @code{libgnunetgnsrecord} library is then used to locate, load and query | ||
6752 | the appropriate gnsrecord plugin. Which plugin is appropriate is determined by | ||
6753 | the record type (which is just a 32-bit integer). The @code{libgnunetgnsrecord} | ||
6754 | library loads all block plugins that are installed at the local peer and | ||
6755 | forwards the application request to the plugins. If the record type is not | ||
6756 | supported by the plugin, it should simply return an error code. | ||
6757 | |||
6758 | The central functions of the block APIs (plugin and main library) are the same | ||
6759 | four functions for converting between values and strings, and typenames and | ||
6760 | numbers documented in the previous subsection. | ||
6761 | |||
6762 | @node The GNS Client-Service Protocol | ||
6763 | @subsection The GNS Client-Service Protocol | ||
6764 | |||
6765 | @c %**end of header | ||
6766 | |||
6767 | The GNS client-service protocol consists of two simple messages, the | ||
6768 | @code{LOOKUP} message and the @code{LOOKUP_RESULT}. Each @code{LOOKUP} message | ||
6769 | contains a unique 32-bit identifier, which will be included in the | ||
6770 | corresponding response. Thus, clients can send many lookup requests in parallel | ||
6771 | and receive responses out-of-order. A @code{LOOKUP} request also includes the | ||
6772 | public key of the GNS zone, the desired record type and fields specifying | ||
6773 | whether shortening is enabled or networking is disabled. Finally, the | ||
6774 | @code{LOOKUP} message includes the name to be resolved. | ||
6775 | |||
6776 | The response includes the number of records and the records themselves in the | ||
6777 | format created by @code{GNUNET_GNSRECORD_records_serialize}. They can thus be | ||
6778 | deserialized using @code{GNUNET_GNSRECORD_records_deserialize}. | ||
6779 | |||
6780 | @node Hijacking the DNS-Traffic using gnunet-service-dns | ||
6781 | @subsection Hijacking the DNS-Traffic using gnunet-service-dns | ||
6782 | |||
6783 | @c %**end of header | ||
6784 | |||
6785 | This section documents how the gnunet-service-dns (and the gnunet-helper-dns) | ||
6786 | intercepts DNS queries from the local system.@ This is merely one method for | ||
6787 | how we can obtain GNS queries. It is also possible to change @code{resolv.conf} | ||
6788 | to point to a machine running @code{gnunet-dns2gns} or to modify libc's name | ||
6789 | system switch (NSS) configuration to include a GNS resolution plugin. The | ||
6790 | method described in this chaper is more of a last-ditch catch-all approach. | ||
6791 | |||
6792 | @code{gnunet-service-dns} enables intercepting DNS traffic using policy based | ||
6793 | routing. We MARK every outgoing DNS-packet if it was not sent by our | ||
6794 | application. Using a second routing table in the Linux kernel these marked | ||
6795 | packets are then routed through our virtual network interface and can thus be | ||
6796 | captured unchanged. | ||
6797 | |||
6798 | Our application then reads the query and decides how to handle it: A query to | ||
6799 | an address ending in ".gnu" or ".zkey" is hijacked by @code{gnunet-service-gns} | ||
6800 | and resolved internally using GNS. In the future, a reverse query for an | ||
6801 | address of the configured virtual network could be answered with records kept | ||
6802 | about previous forward queries. Queries that are not hijacked by some | ||
6803 | application using the DNS service will be sent to the original recipient. The | ||
6804 | answer to the query will always be sent back through the virtual interface with | ||
6805 | the original nameserver as source address. | ||
6806 | |||
6807 | |||
6808 | @menu | ||
6809 | * Network Setup Details:: | ||
6810 | @end menu | ||
6811 | |||
6812 | @node Network Setup Details | ||
6813 | @subsubsection Network Setup Details | ||
6814 | |||
6815 | @c %**end of header | ||
6816 | |||
6817 | The DNS interceptor adds the following rules to the Linux kernel: | ||
6818 | @example | ||
6819 | iptables -t mangle -I OUTPUT 1 -p udp --sport $LOCALPORT --dport 53 -j | ||
6820 | ACCEPT iptables -t mangle -I OUTPUT 2 -p udp --dport 53 -j MARK --set-mark 3 ip | ||
6821 | rule add fwmark 3 table2 ip route add default via $VIRTUALDNS table2 | ||
6822 | @end example | ||
6823 | |||
6824 | Line 1 makes sure that all packets coming from a port our application opened | ||
6825 | beforehand (@code{$LOCALPORT}) will be routed normally. Line 2 marks every | ||
6826 | other packet to a DNS-Server with mark 3 (chosen arbitrarily). The third line | ||
6827 | adds a routing policy based on this mark 3 via the routing table. | ||
6828 | |||
6829 | @node Serving DNS lookups via GNS on W32 | ||
6830 | @subsection Serving DNS lookups via GNS on W32 | ||
6831 | |||
6832 | @c %**end of header | ||
6833 | |||
6834 | This section documents how the libw32nsp (and gnunet-gns-helper-service-w32) do | ||
6835 | DNS resolutions of DNS queries on the local system. This only applies to GNUnet | ||
6836 | running on W32. | ||
6837 | |||
6838 | W32 has a concept of "Namespaces" and "Namespace providers". These are used to | ||
6839 | present various name systems to applications in a generic way. Namespaces | ||
6840 | include DNS, mDNS, NLA and others. For each namespace any number of providers | ||
6841 | could be registered, and they are queried in an order of priority (which is | ||
6842 | adjustable). | ||
6843 | |||
6844 | Applications can resolve names by using WSALookupService*() family of | ||
6845 | functions. | ||
6846 | |||
6847 | However, these are WSA-only facilities. Common BSD socket functions for | ||
6848 | namespace resolutions are gethostbyname and getaddrinfo (among others). These | ||
6849 | functions are implemented internally (by default - by mswsock, which also | ||
6850 | implements the default DNS provider) as wrappers around WSALookupService*() | ||
6851 | functions (see "Sample Code for a Service Provider" on MSDN). | ||
6852 | |||
6853 | On W32 GNUnet builds a libw32nsp - a namespace provider, which can then be | ||
6854 | installed into the system by using w32nsp-install (and uninstalled by | ||
6855 | w32nsp-uninstall), as described in "Installation Handbook". | ||
6856 | |||
6857 | libw32nsp is very simple and has almost no dependencies. As a response to | ||
6858 | NSPLookupServiceBegin(), it only checks that the provider GUID passed to it by | ||
6859 | the caller matches GNUnet DNS Provider GUID, checks that name being resolved | ||
6860 | ends in ".gnu" or ".zkey", then connects to gnunet-gns-helper-service-w32 at | ||
6861 | 127.0.0.1:5353 (hardcoded) and sends the name resolution request there, | ||
6862 | returning the connected socket to the caller. | ||
6863 | |||
6864 | When the caller invokes NSPLookupServiceNext(), libw32nsp reads a completely | ||
6865 | formed reply from that socket, unmarshalls it, then gives it back to the | ||
6866 | caller. | ||
6867 | |||
6868 | At the moment gnunet-gns-helper-service-w32 is implemented to ever give only | ||
6869 | one reply, and subsequent calls to NSPLookupServiceNext() will fail with | ||
6870 | WSA_NODATA (first call to NSPLookupServiceNext() might also fail if GNS failed | ||
6871 | to find the name, or there was an error connecting to it). | ||
6872 | |||
6873 | gnunet-gns-helper-service-w32 does most of the processing: | ||
6874 | |||
6875 | @itemize @bullet | ||
6876 | @item Maintains a connection to GNS. | ||
6877 | @item Reads GNS config and loads appropriate keys. | ||
6878 | @item Checks service GUID and decides on the type of record to look up, | ||
6879 | refusing to make a lookup outright when unsupported service GUID is passed. | ||
6880 | @item Launches the lookup | ||
6881 | @end itemize | ||
6882 | |||
6883 | When lookup result arrives, gnunet-gns-helper-service-w32 forms a complete | ||
6884 | reply (including filling a WSAQUERYSETW structure and, possibly, a binary blob | ||
6885 | with a hostent structure for gethostbyname() client), marshalls it, and sends | ||
6886 | it back to libw32nsp. If no records were found, it sends an empty header. | ||
6887 | |||
6888 | This works for most normal applications that use gethostbyname() or | ||
6889 | getaddrinfo() to resolve names, but fails to do anything with applications that | ||
6890 | use alternative means of resolving names (such as sending queries to a DNS | ||
6891 | server directly by themselves). This includes some of well known utilities, | ||
6892 | like "ping" and "nslookup". | ||
6893 | |||
6894 | @node The GNS Namecache | ||
6895 | @section The GNS Namecache | ||
6896 | |||
6897 | @c %**end of header | ||
6898 | |||
6899 | The NAMECACHE subsystem is responsible for caching (encrypted) resolution | ||
6900 | results of the GNU Name System (GNS). GNS makes zone information available to | ||
6901 | other users via the DHT. However, as accessing the DHT for every lookup is | ||
6902 | expensive (and as the DHT's local cache is lost whenever the peer is | ||
6903 | restarted), GNS uses the NAMECACHE as a more persistent cache for DHT lookups. | ||
6904 | Thus, instead of always looking up every name in the DHT, GNS first checks if | ||
6905 | the result is already available locally in the NAMECACHE. Only if there is no | ||
6906 | result in the NAMECACHE, GNS queries the DHT. The NAMECACHE stores data in the | ||
6907 | same (encrypted) format as the DHT. It thus makes no sense to iterate over all | ||
6908 | items in the NAMECACHE --- the NAMECACHE does not have a way to provide the | ||
6909 | keys required to decrypt the entries. | ||
6910 | |||
6911 | Blocks in the NAMECACHE share the same expiration mechanism as blocks in the | ||
6912 | DHT --- the block expires wheneever any of the records in the (encrypted) block | ||
6913 | expires. The expiration time of the block is the only information stored in | ||
6914 | plaintext. The NAMECACHE service internally performs all of the required work | ||
6915 | to expire blocks, clients do not have to worry about this. Also, given that | ||
6916 | NAMECACHE stores only GNS blocks that local users requested, there is no | ||
6917 | configuration option to limit the size of the NAMECACHE. It is assumed to be | ||
6918 | always small enough (a few MB) to fit on the drive. | ||
6919 | |||
6920 | The NAMECACHE supports the use of different database backends via a plugin API. | ||
6921 | |||
6922 | @menu | ||
6923 | * libgnunetnamecache:: | ||
6924 | * The NAMECACHE Client-Service Protocol:: | ||
6925 | * The NAMECACHE Plugin API:: | ||
6926 | @end menu | ||
6927 | |||
6928 | @node libgnunetnamecache | ||
6929 | @subsection libgnunetnamecache | ||
6930 | |||
6931 | @c %**end of header | ||
6932 | |||
6933 | The NAMECACHE API consists of five simple functions. First, there is | ||
6934 | @code{GNUNET_NAMECACHE_connect} to connect to the NAMECACHE service. This | ||
6935 | returns the handle required for all other operations on the NAMECACHE. Using | ||
6936 | @code{GNUNET_NAMECACHE_block_cache} clients can insert a block into the cache. | ||
6937 | @code{GNUNET_NAMECACHE_lookup_block} can be used to lookup blocks that were | ||
6938 | stored in the NAMECACHE. Both operations can be cancelled using | ||
6939 | @code{GNUNET_NAMECACHE_cancel}. Note that cancelling a | ||
6940 | @code{GNUNET_NAMECACHE_block_cache} operation can result in the block being | ||
6941 | stored in the NAMECACHE --- or not. Cancellation primarily ensures that the | ||
6942 | continuation function with the result of the operation will no longer be | ||
6943 | invoked. Finally, @code{GNUNET_NAMECACHE_disconnect} closes the connection to | ||
6944 | the NAMECACHE. | ||
6945 | |||
6946 | The maximum size of a block that can be stored in the NAMECACHE is | ||
6947 | @code{GNUNET_NAMECACHE_MAX_VALUE_SIZE}, which is defined to be 63 kB. | ||
6948 | |||
6949 | @node The NAMECACHE Client-Service Protocol | ||
6950 | @subsection The NAMECACHE Client-Service Protocol | ||
6951 | |||
6952 | @c %**end of header | ||
6953 | |||
6954 | All messages in the NAMECACHE IPC protocol start with the @code{struct | ||
6955 | GNUNET_NAMECACHE_Header} which adds a request ID (32-bit integer) to the | ||
6956 | standard message header. The request ID is used to match requests with the | ||
6957 | respective responses from the NAMECACHE, as they are allowed to happen | ||
6958 | out-of-order. | ||
6959 | |||
6960 | |||
6961 | @menu | ||
6962 | * Lookup:: | ||
6963 | * Store:: | ||
6964 | @end menu | ||
6965 | |||
6966 | @node Lookup | ||
6967 | @subsubsection Lookup | ||
6968 | |||
6969 | @c %**end of header | ||
6970 | |||
6971 | The @code{struct LookupBlockMessage} is used to lookup a block stored in the | ||
6972 | cache. It contains the query hash. The NAMECACHE always responds with a | ||
6973 | @code{struct LookupBlockResponseMessage}. If the NAMECACHE has no response, it | ||
6974 | sets the expiration time in the response to zero. Otherwise, the response is | ||
6975 | expected to contain the expiration time, the ECDSA signature, the derived key | ||
6976 | and the (variable-size) encrypted data of the block. | ||
6977 | |||
6978 | @node Store | ||
6979 | @subsubsection Store | ||
6980 | |||
6981 | @c %**end of header | ||
6982 | |||
6983 | The @code{struct BlockCacheMessage} is used to cache a block in the NAMECACHE. | ||
6984 | It has the same structure as the @code{struct LookupBlockResponseMessage}. The | ||
6985 | service responds with a @code{struct BlockCacheResponseMessage} which contains | ||
6986 | the result of the operation (success or failure). In the future, we might want | ||
6987 | to make it possible to provide an error message as well. | ||
6988 | |||
6989 | @node The NAMECACHE Plugin API | ||
6990 | @subsection The NAMECACHE Plugin API | ||
6991 | @c %**end of header | ||
6992 | |||
6993 | The NAMECACHE plugin API consists of two functions, @code{cache_block} to store | ||
6994 | a block in the database, and @code{lookup_block} to lookup a block in the | ||
6995 | database. | ||
6996 | |||
6997 | |||
6998 | @menu | ||
6999 | * Lookup2:: | ||
7000 | * Store2:: | ||
7001 | @end menu | ||
7002 | |||
7003 | @node Lookup2 | ||
7004 | @subsubsection Lookup2 | ||
7005 | |||
7006 | @c %**end of header | ||
7007 | |||
7008 | The @code{lookup_block} function is expected to return at most one block to the | ||
7009 | iterator, and return @code{GNUNET_NO} if there were no non-expired results. If | ||
7010 | there are multiple non-expired results in the cache, the lookup is supposed to | ||
7011 | return the result with the largest expiration time. | ||
7012 | |||
7013 | @node Store2 | ||
7014 | @subsubsection Store2 | ||
7015 | |||
7016 | @c %**end of header | ||
7017 | |||
7018 | The @code{cache_block} function is expected to try to store the block in the | ||
7019 | database, and return @code{GNUNET_SYSERR} if this was not possible for any | ||
7020 | reason. Furthermore, @code{cache_block} is expected to implicitly perform cache | ||
7021 | maintenance and purge blocks from the cache that have expired. Note that | ||
7022 | @code{cache_block} might encounter the case where the database already has | ||
7023 | another block stored under the same key. In this case, the plugin must ensure | ||
7024 | that the block with the larger expiration time is preserved. Obviously, this | ||
7025 | can done either by simply adding new blocks and selecting for the most recent | ||
7026 | expiration time during lookup, or by checking which block is more recent during | ||
7027 | the store operation. | ||
7028 | |||
7029 | @node The REVOCATION Subsystem | ||
7030 | @section The REVOCATION Subsystem | ||
7031 | @c %**end of header | ||
7032 | |||
7033 | The REVOCATION subsystem is responsible for key revocation of Egos. If a user | ||
7034 | learns that his private key has been compromised or has lost it, he can use the | ||
7035 | REVOCATION system to inform all of the other users that this private key is no | ||
7036 | longer valid. The subsystem thus includes ways to query for the validity of | ||
7037 | keys and to propagate revocation messages. | ||
7038 | |||
7039 | @menu | ||
7040 | * Dissemination:: | ||
7041 | * Revocation Message Design Requirements:: | ||
7042 | * libgnunetrevocation:: | ||
7043 | * The REVOCATION Client-Service Protocol:: | ||
7044 | * The REVOCATION Peer-to-Peer Protocol:: | ||
7045 | @end menu | ||
7046 | |||
7047 | @node Dissemination | ||
7048 | @subsection Dissemination | ||
7049 | |||
7050 | @c %**end of header | ||
7051 | |||
7052 | When a revocation is performed, the revocation is first of all disseminated by | ||
7053 | flooding the overlay network. The goal is to reach every peer, so that when a | ||
7054 | peer needs to check if a key has been revoked, this will be purely a local | ||
7055 | operation where the peer looks at his local revocation list. Flooding the | ||
7056 | network is also the most robust form of key revocation --- an adversary would | ||
7057 | have to control a separator of the overlay graph to restrict the propagation of | ||
7058 | the revocation message. Flooding is also very easy to implement --- peers that | ||
7059 | receive a revocation message for a key that they have never seen before simply | ||
7060 | pass the message to all of their neighbours. | ||
7061 | |||
7062 | Flooding can only distribute the revocation message to peers that are online. | ||
7063 | In order to notify peers that join the network later, the revocation service | ||
7064 | performs efficient set reconciliation over the sets of known revocation | ||
7065 | messages whenever two peers (that both support REVOCATION dissemination) | ||
7066 | connect. The SET service is used to perform this operation | ||
7067 | efficiently. | ||
7068 | |||
7069 | @node Revocation Message Design Requirements | ||
7070 | @subsection Revocation Message Design Requirements | ||
7071 | |||
7072 | @c %**end of header | ||
7073 | |||
7074 | However, flooding is also quite costly, creating O(|E|) messages on a network | ||
7075 | with |E| edges. Thus, revocation messages are required to contain a | ||
7076 | proof-of-work, the result of an expensive computation (which, however, is cheap | ||
7077 | to verify). Only peers that have expended the CPU time necessary to provide | ||
7078 | this proof will be able to flood the network with the revocation message. This | ||
7079 | ensures that an attacker cannot simply flood the network with millions of | ||
7080 | revocation messages. The proof-of-work required by GNUnet is set to take days | ||
7081 | on a typical PC to compute; if the ability to quickly revoke a key is needed, | ||
7082 | users have the option to pre-compute revocation messages to store off-line and | ||
7083 | use instantly after their key has expired. | ||
7084 | |||
7085 | Revocation messages must also be signed by the private key that is being | ||
7086 | revoked. Thus, they can only be created while the private key is in the | ||
7087 | possession of the respective user. This is another reason to create a | ||
7088 | revocation message ahead of time and store it in a secure location. | ||
7089 | |||
7090 | @node libgnunetrevocation | ||
7091 | @subsection libgnunetrevocation | ||
7092 | |||
7093 | @c %**end of header | ||
7094 | |||
7095 | The REVOCATION API consists of two parts, to query and to issue | ||
7096 | revocations. | ||
7097 | |||
7098 | |||
7099 | @menu | ||
7100 | * Querying for revoked keys:: | ||
7101 | * Preparing revocations:: | ||
7102 | * Issuing revocations:: | ||
7103 | @end menu | ||
7104 | |||
7105 | @node Querying for revoked keys | ||
7106 | @subsubsection Querying for revoked keys | ||
7107 | |||
7108 | @c %**end of header | ||
7109 | |||
7110 | @code{GNUNET_REVOCATION_query} is used to check if a given ECDSA public key has | ||
7111 | been revoked. The given callback will be invoked with the result of the check. | ||
7112 | The query can be cancelled using @code{GNUNET_REVOCATION_query_cancel} on the | ||
7113 | return value. | ||
7114 | |||
7115 | @node Preparing revocations | ||
7116 | @subsubsection Preparing revocations | ||
7117 | |||
7118 | @c %**end of header | ||
7119 | |||
7120 | It is often desirable to create a revocation record ahead-of-time and store it | ||
7121 | in an off-line location to be used later in an emergency. This is particularly | ||
7122 | true for GNUnet revocations, where performing the revocation operation itself | ||
7123 | is computationally expensive and thus is likely to take some time. Thus, if | ||
7124 | users want the ability to perform revocations quickly in an emergency, they | ||
7125 | must pre-compute the revocation message. The revocation API enables this with | ||
7126 | two functions that are used to compute the revocation message, but not trigger | ||
7127 | the actual revocation operation. | ||
7128 | |||
7129 | @code{GNUNET_REVOCATION_check_pow} should be used to calculate the | ||
7130 | proof-of-work required in the revocation message. This function takes the | ||
7131 | public key, the required number of bits for the proof of work (which in GNUnet | ||
7132 | is a network-wide constant) and finally a proof-of-work number as arguments. | ||
7133 | The function then checks if the given proof-of-work number is a valid proof of | ||
7134 | work for the given public key. Clients preparing a revocation are expected to | ||
7135 | call this function repeatedly (typically with a monotonically increasing | ||
7136 | sequence of numbers of the proof-of-work number) until a given number satisfies | ||
7137 | the check. That number should then be saved for later use in the revocation | ||
7138 | operation. | ||
7139 | |||
7140 | @code{GNUNET_REVOCATION_sign_revocation} is used to generate the signature that | ||
7141 | is required in a revocation message. It takes the private key that (possibly in | ||
7142 | the future) is to be revoked and returns the signature. The signature can again | ||
7143 | be saved to disk for later use, which will then allow performing a revocation | ||
7144 | even without access to the private key. | ||
7145 | |||
7146 | @node Issuing revocations | ||
7147 | @subsubsection Issuing revocations | ||
7148 | |||
7149 | |||
7150 | Given a ECDSA public key, the signature from @code{GNUNET_REVOCATION_sign} and | ||
7151 | the proof-of-work, @code{GNUNET_REVOCATION_revoke} can be used to perform the | ||
7152 | actual revocation. The given callback is called upon completion of the | ||
7153 | operation. @code{GNUNET_REVOCATION_revoke_cancel} can be used to stop the | ||
7154 | library from calling the continuation; however, in that case it is undefined | ||
7155 | whether or not the revocation operation will be executed. | ||
7156 | |||
7157 | @node The REVOCATION Client-Service Protocol | ||
7158 | @subsection The REVOCATION Client-Service Protocol | ||
7159 | |||
7160 | |||
7161 | The REVOCATION protocol consists of four simple messages. | ||
7162 | |||
7163 | A @code{QueryMessage} containing a public ECDSA key is used to check if a | ||
7164 | particular key has been revoked. The service responds with a | ||
7165 | @code{QueryResponseMessage} which simply contains a bit that says if the given | ||
7166 | public key is still valid, or if it has been revoked. | ||
7167 | |||
7168 | The second possible interaction is for a client to revoke a key by passing a | ||
7169 | @code{RevokeMessage} to the service. The @code{RevokeMessage} contains the | ||
7170 | ECDSA public key to be revoked, a signature by the corresponding private key | ||
7171 | and the proof-of-work, The service responds with a | ||
7172 | @code{RevocationResponseMessage} which can be used to indicate that the | ||
7173 | @code{RevokeMessage} was invalid (i.e. proof of work incorrect), or otherwise | ||
7174 | indicates that the revocation has been processed successfully. | ||
7175 | |||
7176 | @node The REVOCATION Peer-to-Peer Protocol | ||
7177 | @subsection The REVOCATION Peer-to-Peer Protocol | ||
7178 | |||
7179 | @c %**end of header | ||
7180 | |||
7181 | Revocation uses two disjoint ways to spread revocation information among peers. | ||
7182 | First of all, P2P gossip exchanged via CORE-level neighbours is used to quickly | ||
7183 | spread revocations to all connected peers. Second, whenever two peers (that | ||
7184 | both support revocations) connect, the SET service is used to compute the union | ||
7185 | of the respective revocation sets. | ||
7186 | |||
7187 | In both cases, the exchanged messages are @code{RevokeMessage}s which contain | ||
7188 | the public key that is being revoked, a matching ECDSA signature, and a | ||
7189 | proof-of-work. Whenever a peer learns about a new revocation this way, it first | ||
7190 | validates the signature and the proof-of-work, then stores it to disk | ||
7191 | (typically to a file $GNUNET_DATA_HOME/revocation.dat) and finally spreads the | ||
7192 | information to all directly connected neighbours. | ||
7193 | |||
7194 | For computing the union using the SET service, the peer with the smaller hashed | ||
7195 | peer identity will connect (as a "client" in the two-party set protocol) to the | ||
7196 | other peer after one second (to reduce traffic spikes on connect) and initiate | ||
7197 | the computation of the set union. All revocation services use a common hash to | ||
7198 | identify the SET operation over revocation sets. | ||
7199 | |||
7200 | The current implementation accepts revocation set union operations from all | ||
7201 | peers at any time; however, well-behaved peers should only initiate this | ||
7202 | operation once after establishing a connection to a peer with a larger hashed | ||
7203 | peer identity. | ||
7204 | |||
7205 | @node GNUnet's File-sharing (FS) Subsystem | ||
7206 | @section GNUnet's File-sharing (FS) Subsystem | ||
7207 | |||
7208 | @c %**end of header | ||
7209 | |||
7210 | This chapter describes the details of how the file-sharing service works. As | ||
7211 | with all services, it is split into an API (libgnunetfs), the service process | ||
7212 | (gnunet-service-fs) and user interface(s). The file-sharing service uses the | ||
7213 | datastore service to store blocks and the DHT (and indirectly datacache) for | ||
7214 | lookups for non-anonymous file-sharing.@ Furthermore, the file-sharing service | ||
7215 | uses the block library (and the block fs plugin) for validation of DHT | ||
7216 | operations. | ||
7217 | |||
7218 | In contrast to many other services, libgnunetfs is rather complex since the | ||
7219 | client library includes a large number of high-level abstractions; this is | ||
7220 | necessary since the Fs service itself largely only operates on the block level. | ||
7221 | The FS library is responsible for providing a file-based abstraction to | ||
7222 | applications, including directories, meta data, keyword search, verification, | ||
7223 | and so on. | ||
7224 | |||
7225 | The method used by GNUnet to break large files into blocks and to use keyword | ||
7226 | search is called the "Encoding for Censorship Resistant Sharing" (ECRS). ECRS | ||
7227 | is largely implemented in the fs library; block validation is also reflected in | ||
7228 | the block FS plugin and the FS service. ECRS on-demand encoding is implemented | ||
7229 | in the FS service. | ||
7230 | |||
7231 | NOTE: The documentation in this chapter is quite incomplete. | ||
7232 | |||
7233 | @menu | ||
7234 | * Encoding for Censorship-Resistant Sharing (ECRS):: | ||
7235 | * File-sharing persistence directory structure:: | ||
7236 | @end menu | ||
7237 | |||
7238 | @node Encoding for Censorship-Resistant Sharing (ECRS) | ||
7239 | @subsection Encoding for Censorship-Resistant Sharing (ECRS) | ||
7240 | |||
7241 | @c %**end of header | ||
7242 | |||
7243 | When GNUnet shares files, it uses a content encoding that is called ECRS, the | ||
7244 | Encoding for Censorship-Resistant Sharing. Most of ECRS is described in the | ||
7245 | (so far unpublished) research paper attached to this page. ECRS obsoletes the | ||
7246 | previous ESED and ESED II encodings which were used in GNUnet before version | ||
7247 | 0.7.0.@ @ The rest of this page assumes that the reader is familiar with the | ||
7248 | attached paper. What follows is a description of some minor extensions that | ||
7249 | GNUnet makes over what is described in the paper. The reason why these | ||
7250 | extensions are not in the paper is that we felt that they were obvious or | ||
7251 | trivial extensions to the original scheme and thus did not warrant space in | ||
7252 | the research report. | ||
7253 | |||
7254 | |||
7255 | @menu | ||
7256 | * Namespace Advertisements:: | ||
7257 | * KSBlocks:: | ||
7258 | @end menu | ||
7259 | |||
7260 | @node Namespace Advertisements | ||
7261 | @subsubsection Namespace Advertisements | ||
7262 | |||
7263 | @c %**end of header | ||
7264 | |||
7265 | An @code{SBlock} with identifier ′all zeros′ is a signed | ||
7266 | advertisement for a namespace. This special @code{SBlock} contains metadata | ||
7267 | describing the content of the namespace. Instead of the name of the identifier | ||
7268 | for a potential update, it contains the identifier for the root of the | ||
7269 | namespace. The URI should always be empty. The @code{SBlock} is signed with | ||
7270 | the content provder′s RSA private key (just like any other SBlock). Peers | ||
7271 | can search for @code{SBlock}s in order to find out more about a namespace. | ||
7272 | |||
7273 | @node KSBlocks | ||
7274 | @subsubsection KSBlocks | ||
7275 | |||
7276 | @c %**end of header | ||
7277 | |||
7278 | GNUnet implements @code{KSBlocks} which are @code{KBlocks} that, instead of | ||
7279 | encrypting a CHK and metadata, encrypt an @code{SBlock} instead. In other | ||
7280 | words, @code{KSBlocks} enable GNUnet to find @code{SBlocks} using the global | ||
7281 | keyword search. Usually the encrypted @code{SBlock} is a namespace | ||
7282 | advertisement. The rationale behind @code{KSBlock}s and @code{SBlock}s is to | ||
7283 | enable peers to discover namespaces via keyword searches, and, to associate | ||
7284 | useful information with namespaces. When GNUnet finds @code{KSBlocks} during a | ||
7285 | normal keyword search, it adds the information to an internal list of | ||
7286 | discovered namespaces. Users looking for interesting namespaces can then | ||
7287 | inspect this list, reducing the need for out-of-band discovery of namespaces. | ||
7288 | Naturally, namespaces (or more specifically, namespace advertisements) can | ||
7289 | also be referenced from directories, but @code{KSBlock}s should make it easier | ||
7290 | to advertise namespaces for the owner of the pseudonym since they eliminate | ||
7291 | the need to first create a directory. | ||
7292 | |||
7293 | Collections are also advertised using @code{KSBlock}s. | ||
7294 | |||
7295 | @table @asis | ||
7296 | @item Attachment Size | ||
7297 | @item ecrs.pdf 270.68 KB | ||
7298 | @item https://gnunet.org/sites/default/files/ecrs.pdf | ||
7299 | @end table | ||
7300 | |||
7301 | @node File-sharing persistence directory structure | ||
7302 | @subsection File-sharing persistence directory structure | ||
7303 | |||
7304 | @c %**end of header | ||
7305 | |||
7306 | This section documents how the file-sharing library implements persistence of | ||
7307 | file-sharing operations and specifically the resulting directory structure. | ||
7308 | This code is only active if the @code{GNUNET_FS_FLAGS_PERSISTENCE} flag was set | ||
7309 | when calling @code{GNUNET_FS_start}. In this case, the file-sharing library | ||
7310 | will try hard to ensure that all major operations (searching, downloading, | ||
7311 | publishing, unindexing) are persistent, that is, can live longer than the | ||
7312 | process itself. More specifically, an operation is supposed to live until it is | ||
7313 | explicitly stopped. | ||
7314 | |||
7315 | If @code{GNUNET_FS_stop} is called before an operation has been stopped, a | ||
7316 | @code{SUSPEND} event is generated and then when the process calls | ||
7317 | @code{GNUNET_FS_start} next time, a @code{RESUME} event is generated. | ||
7318 | Additionally, even if an application crashes (segfault, SIGKILL, system crash) | ||
7319 | and hence @code{GNUNET_FS_stop} is never called and no @code{SUSPEND} events | ||
7320 | are generated, operations are still resumed (with @code{RESUME} events). This | ||
7321 | is implemented by constantly writing the current state of the file-sharing | ||
7322 | operations to disk. Specifically, the current state is always written to disk | ||
7323 | whenever anything significant changes (the exception are block-wise progress in | ||
7324 | publishing and unindexing, since those operations would be slowed down | ||
7325 | significantly and can be resumed cheaply even without detailed accounting). | ||
7326 | Note that@ if the process crashes (or is killed) during a serialization | ||
7327 | operation, FS does not guarantee that this specific operation is recoverable | ||
7328 | (no strict transactional semantics, again for performance reasons). However, | ||
7329 | all other unrelated operations should resume nicely. | ||
7330 | |||
7331 | Since we need to serialize the state continuously and want to recover as much | ||
7332 | as possible even after crashing during a serialization operation, we do not use | ||
7333 | one large file for serialization. Instead, several directories are used for the | ||
7334 | various operations. When @code{GNUNET_FS_start} executes, the master | ||
7335 | directories are scanned for files describing operations to resume. Sometimes, | ||
7336 | these operations can refer to related operations in child directories which may | ||
7337 | also be resumed at this point. Note that corrupted files are cleaned up | ||
7338 | automatically. However, dangling files in child directories (those that are not | ||
7339 | referenced by files from the master directories) are not automatically removed. | ||
7340 | |||
7341 | Persistence data is kept in a directory that begins with the "STATE_DIR" prefix | ||
7342 | from the configuration file (by default, "$SERVICEHOME/persistence/") followed | ||
7343 | by the name of the client as given to @code{GNUNET_FS_start} (for example, | ||
7344 | "gnunet-gtk") followed by the actual name of the master or child directory. | ||
7345 | |||
7346 | The names for the master directories follow the names of the operations: | ||
7347 | |||
7348 | @itemize @bullet | ||
7349 | @item "search" | ||
7350 | @item "download" | ||
7351 | @item "publish" | ||
7352 | @item "unindex" | ||
7353 | @end itemize | ||
7354 | |||
7355 | Each of the master directories contains names (chosen at random) for each | ||
7356 | active top-level (master) operation. Note that a download that is associated | ||
7357 | with a search result is not a top-level operation. | ||
7358 | |||
7359 | In contrast to the master directories, the child directories are only consulted | ||
7360 | when another operation refers to them. For each search, a subdirectory (named | ||
7361 | after the master search synchronization file) contains the search results. | ||
7362 | Search results can have an associated download, which is then stored in the | ||
7363 | general "download-child" directory. Downloads can be recursive, in which case | ||
7364 | children are stored in subdirectories mirroring the structure of the recursive | ||
7365 | download (either starting in the master "download" directory or in the | ||
7366 | "download-child" directory depending on how the download was initiated). For | ||
7367 | publishing operations, the "publish-file" directory contains information about | ||
7368 | the individual files and directories that are part of the publication. However, | ||
7369 | this directory structure is flat and does not mirror the structure of the | ||
7370 | publishing operation. Note that unindex operations cannot have associated child | ||
7371 | operations. | ||
7372 | |||
7373 | @node GNUnet's REGEX Subsystem | ||
7374 | @section GNUnet's REGEX Subsystem | ||
7375 | |||
7376 | @c %**end of header | ||
7377 | |||
7378 | Using the REGEX subsystem, you can discover peers that offer a particular | ||
7379 | service using regular expressions. The peers that offer a service specify it | ||
7380 | using a regular expressions. Peers that want to patronize a service search | ||
7381 | using a string. The REGEX subsystem will then use the DHT to return a set of | ||
7382 | matching offerers to the patrons. | ||
7383 | |||
7384 | For the technical details, we have "Max's defense talk and Max's Master's | ||
7385 | thesis. An additional publication is under preparation and available to team | ||
7386 | members (in Git). | ||
7387 | |||
7388 | @menu | ||
7389 | * How to run the regex profiler:: | ||
7390 | @end menu | ||
7391 | |||
7392 | @node How to run the regex profiler | ||
7393 | @subsection How to run the regex profiler | ||
7394 | |||
7395 | @c %**end of header | ||
7396 | |||
7397 | The gnunet-regex-profiler can be used to profile the usage of mesh/regex for a | ||
7398 | given set of regular expressions and strings. Mesh/regex allows you to announce | ||
7399 | your peer ID under a certain regex and search for peers matching a particular | ||
7400 | regex using a string. See https://gnunet.org/szengel2012ms for a full | ||
7401 | introduction. | ||
7402 | |||
7403 | First of all, the regex profiler uses GNUnet testbed, thus all the implications | ||
7404 | for testbed also apply to the regex profiler (for example you need | ||
7405 | password-less ssh login to the machines listed in your hosts file). | ||
7406 | |||
7407 | @strong{Configuration} | ||
7408 | |||
7409 | Moreover, an appropriate configuration file is needed. Generally you can refer | ||
7410 | to SVN HEAD: contrib/regex_profiler_infiniband.conf for an example | ||
7411 | configuration. In the following paragraph the important details are | ||
7412 | highlighted. | ||
7413 | |||
7414 | Announcing of the regular expressions is done by the | ||
7415 | gnunet-daemon-regexprofiler, therefore you have to make sure it is started, by | ||
7416 | adding it to the AUTOSTART set of ARM:@ | ||
7417 | @code{ | ||
7418 | [regexprofiler]@ | ||
7419 | AUTOSTART = YES@ | ||
7420 | } | ||
7421 | |||
7422 | Furthermore you have to specify the location of the binary: | ||
7423 | @example | ||
7424 | [regexprofiler] | ||
7425 | # Location of the gnunet-daemon-regexprofiler binary. | ||
7426 | BINARY = /home/szengel/gnunet/src/mesh/.libs/gnunet-daemon-regexprofiler | ||
7427 | # Regex prefix that will be applied to all regular expressions and | ||
7428 | # search string. | ||
7429 | REGEX_PREFIX = "GNVPN-0001-PAD" | ||
7430 | @end example | ||
7431 | |||
7432 | When running the profiler with a large scale deployment, you probably want to | ||
7433 | reduce the workload of each peer. Use the following options to do this.@ | ||
7434 | @example | ||
7435 | [dht]@ | ||
7436 | # Force network size estimation@ | ||
7437 | FORCE_NSE = 1 | ||
7438 | |||
7439 | [dhtcache] | ||
7440 | DATABASE = heap@ | ||
7441 | # Disable RC-file for Bloom filter? (for benchmarking with limited IO | ||
7442 | # availability)@ | ||
7443 | DISABLE_BF_RC = YES@ | ||
7444 | # Disable Bloom filter entirely@ | ||
7445 | DISABLE_BF = YES | ||
7446 | |||
7447 | [nse]@ | ||
7448 | # Minimize proof-of-work CPU consumption by NSE@ | ||
7449 | WORKBITS = 1 | ||
7450 | @end example | ||
7451 | |||
7452 | |||
7453 | @strong{Options} | ||
7454 | |||
7455 | To finally run the profiler some options and the input data need to be | ||
7456 | specified on the command line. | ||
7457 | @code{@ gnunet-regex-profiler -c config-file -d | ||
7458 | log-file -n num-links -p@ path-compression-length -s search-delay -t | ||
7459 | matching-timeout -a num-search-strings hosts-file policy-dir | ||
7460 | search-strings-file@ } | ||
7461 | |||
7462 | @code{config-file} the configuration file created earlier.@ @code{log-file} | ||
7463 | file where to write statistics output.@ @code{num-links} number of random links | ||
7464 | between started peers.@ @code{path-compression-length} maximum path compression | ||
7465 | length in the DFA.@ @code{search-delay} time to wait between peers finished | ||
7466 | linking and@ starting to match strings.@ @code{matching-timeout} timeout after | ||
7467 | witch to cancel the searching.@ @code{num-search-strings} number of strings in | ||
7468 | the search-strings-file. | ||
7469 | |||
7470 | The @code{hosts-file} should contain a list of hosts for the testbed, one per | ||
7471 | line in the following format. @code{user@@host_ip:port}. | ||
7472 | |||
7473 | The @code{policy-dir} is a folder containing text files containing one or more | ||
7474 | regular expressions. A peer is started for each file in that folder and the | ||
7475 | regular expressions in the corresponding file are announced by this peer. | ||
7476 | |||
7477 | The @code{search-strings-file} is a text file containing search strings, one in | ||
7478 | each line. | ||
7479 | |||
7480 | You can create regular expressions and search strings for every AS in the@ | ||
7481 | Internet using the attached scripts. You need one of the | ||
7482 | @uref{http://data.caida.org/datasets/routing/routeviews-prefix2as/, CAIDA | ||
7483 | routeviews prefix2as} data files for this. Run @code{create_regex.py <filename> | ||
7484 | <output path>} to create the regular expressions and @code{create_strings.py | ||
7485 | <input path> <outfile>} to create a search strings file from the previously | ||
7486 | created regular expressions. | ||
diff --git a/doc/chapters/installation.texi b/doc/chapters/installation.texi deleted file mode 100644 index edbad84de..000000000 --- a/doc/chapters/installation.texi +++ /dev/null | |||
@@ -1,3625 +0,0 @@ | |||
1 | @node GNUnet Installation Handbook | ||
2 | @chapter GNUnet Installation Handbook | ||
3 | |||
4 | This handbook describes how to install (build setup, compilation) and setup | ||
5 | (configuration, start) GNUnet 0.10.x. After following these instructions you | ||
6 | should be able to install and then start user-interfaces to interact with the | ||
7 | network. | ||
8 | |||
9 | This manual is far from complete, and we welcome informed contributions, be it | ||
10 | in the form of new chapters or insightful comments. | ||
11 | |||
12 | |||
13 | |||
14 | @menu | ||
15 | * Dependencies:: | ||
16 | * Pre-installation notes:: | ||
17 | * Generic installation instructions:: | ||
18 | * Build instructions for Ubuntu 12.04 using Git:: | ||
19 | * Build Instructions for Microsoft Windows Platforms:: | ||
20 | * Build instructions for Debian 7.5:: | ||
21 | * Installing GNUnet from Git on Ubuntu 14.4:: | ||
22 | * Build instructions for Debian 8:: | ||
23 | * Outdated build instructions for previous revisions:: | ||
24 | * Portable GNUnet:: | ||
25 | * The graphical configuration interface:: | ||
26 | * How to start and stop a GNUnet peer:: | ||
27 | @end menu | ||
28 | |||
29 | @node Dependencies | ||
30 | @section Dependencies | ||
31 | @c %**end of header | ||
32 | |||
33 | This document lists the various known dependencies for GNUnet 0.10.x. | ||
34 | Suggestions for missing dependencies or wrong version numbers are welcome. | ||
35 | |||
36 | |||
37 | |||
38 | @menu | ||
39 | * External dependencies:: | ||
40 | * Fixing libgnurl build issues:: | ||
41 | * Internal dependencies:: | ||
42 | @end menu | ||
43 | |||
44 | @node External dependencies | ||
45 | @subsection External dependencies | ||
46 | @c %**end of header | ||
47 | |||
48 | These packages must be installed before a typical GNUnet installation | ||
49 | can be performed: | ||
50 | |||
51 | @table @asis | ||
52 | @item GNU libmicrohttpd 0.9.30 or higher | ||
53 | @item GNU libextractor 1.0 or higher | ||
54 | @item GNU libtool 2.2 or higher | ||
55 | @item GNU libunistring 0.9.1.1 or higher | ||
56 | @item GNU libidn 1.0.0 or higher | ||
57 | @item @uref{https://gnupg.org/software/libgcrypt/index.html, GNU libgcrypt} | ||
58 | @uref{https://gnupg.org/ftp/gcrypt/libgcrypt/, 1.6.0} or higher | ||
59 | @item @uref{https://gnutls.org/, GnuTLS} | ||
60 | @uref{https://www.gnupg.org/ftp/gcrypt/gnutls/v3.2/, 3.2.7} or higher, | ||
61 | compile with libunbound for DANE support; GnuTLS also requires GNU | ||
62 | nettle 2.7 (update: GnuTLS 3.2.7 appears NOT to work against GNU nettle | ||
63 | > 2.7, due to some API updatings done by nettle. Thus it should be compiled | ||
64 | against nettle 2.7 and, in case you get some error on the reference to | ||
65 | `rpl_strerror' being undefined, follow the instructions on@ | ||
66 | @uref{http://lists.gnupg.org/pipermail/gnutls-devel/2013-November/006588.html, this} | ||
67 | post (and the link inside it)). | ||
68 | @item @uref{https://gnunet.org/gnurl, gnURL} libgnurl 7.34.0 or higher, | ||
69 | must be compiled after @code{GnuTLS} | ||
70 | @item libglpk 4.45 or higher | ||
71 | @item @uref{http://www.openssl.org/, OpenSSL} (binary) 1.0 or higher | ||
72 | @item TeX Live 2012 or higher, optional (for gnunet-bcd) | ||
73 | @item libpulse 2.0 or higher, optional (for gnunet-conversation) | ||
74 | @item libopus 1.0.1 or higher, optional (for gnunet-conversation) | ||
75 | @item libogg 1.3.0 or higher, optional (for gnunet-conversation) | ||
76 | @item certool (binary) | ||
77 | optional for convenient installation of the GNS proxy | ||
78 | (available as part of Debian's libnss3-tools) | ||
79 | @item python-zbar 0.10 or higher, optional (for gnunet-qr) | ||
80 | @item libsqlite 3.8.0 or higher (note that the code will compile and often work with lower | ||
81 | version numbers, but you may get subtle bugs with respect to quota management | ||
82 | in certain rare cases); alternatively, MySQL or Postgres can also be installed, | ||
83 | but those databases will require more complex configurations (not recommended | ||
84 | for first-time users) | ||
85 | @item zlib any version we tested worked | ||
86 | @item Gtk+ 3.0 or higher, optional (for gnunet-gtk) | ||
87 | @item libgladeui must match Gtk+ version, optional (for gnunet-gtk) | ||
88 | @item libqrencode 3.0 or higher, optional (for gnunet-namestore-gtk) | ||
89 | @end table | ||
90 | |||
91 | |||
92 | @node Fixing libgnurl build issues | ||
93 | @subsection Fixing libgnurl build issues | ||
94 | |||
95 | If you have to compile libgnurl from source since the version included in your | ||
96 | distribution is to old you perhaps get an error message while running the | ||
97 | @file{configure} script: | ||
98 | |||
99 | @code{@ | ||
100 | $ configure@ | ||
101 | ...@ | ||
102 | checking for 64-bit curl_off_t data type... unknown@ | ||
103 | checking for 32-bit curl_off_t data type... unknown@ | ||
104 | checking for 16-bit curl_off_t data type... unknown@ | ||
105 | configure: error: cannot find data type for curl_off_t.@ | ||
106 | } | ||
107 | |||
108 | Solution: | ||
109 | |||
110 | Before running the configure script, set: | ||
111 | |||
112 | @code{CFLAGS="-I. -I$BUILD_ROOT/include" } | ||
113 | |||
114 | |||
115 | |||
116 | @node Internal dependencies | ||
117 | @subsection Internal dependencies | ||
118 | |||
119 | This section tries to give an overview of what processes a typical GNUnet peer | ||
120 | running a particular application would consist of. All of the processes listed | ||
121 | here should be automatically started by @code{gnunet-arm -s}. The list is given | ||
122 | as a rough first guide to users for failure diagnostics. Ideally, end-users | ||
123 | should never have to worry about these internal dependencies. | ||
124 | |||
125 | In terms of internal dependencies, a minimum file-sharing system consists of | ||
126 | the following GNUnet processes (in order of dependency): | ||
127 | |||
128 | @itemize @bullet | ||
129 | @item gnunet-service-arm | ||
130 | @item gnunet-service-resolver (required by all) | ||
131 | @item gnunet-service-statistics (required by all) | ||
132 | @item gnunet-service-peerinfo | ||
133 | @item gnunet-service-transport (requires peerinfo) | ||
134 | @item gnunet-service-core (requires transport) | ||
135 | @item gnunet-daemon-hostlist (requires core) | ||
136 | @item gnunet-daemon-topology (requires hostlist, peerinfo) | ||
137 | @item gnunet-service-datastore | ||
138 | @item gnunet-service-dht (requires core) | ||
139 | @item gnunet-service-identity | ||
140 | @item gnunet-service-fs (requires identity, mesh, dht, datastore, core) | ||
141 | @end itemize | ||
142 | |||
143 | |||
144 | A minimum VPN system consists of the following GNUnet processes (in order of | ||
145 | dependency): | ||
146 | |||
147 | @itemize @bullet | ||
148 | @item gnunet-service-arm | ||
149 | @item gnunet-service-resolver (required by all) | ||
150 | @item gnunet-service-statistics (required by all) | ||
151 | @item gnunet-service-peerinfo | ||
152 | @item gnunet-service-transport (requires peerinfo) | ||
153 | @item gnunet-service-core (requires transport) | ||
154 | @item gnunet-daemon-hostlist (requires core) | ||
155 | @item gnunet-service-dht (requires core) | ||
156 | @item gnunet-service-mesh (requires dht, core) | ||
157 | @item gnunet-service-dns (requires dht) | ||
158 | @item gnunet-service-regex (requires dht) | ||
159 | @item gnunet-service-vpn (requires regex, dns, mesh, dht) | ||
160 | @end itemize | ||
161 | |||
162 | |||
163 | A minimum GNS system consists of the following GNUnet processes (in order of | ||
164 | dependency): | ||
165 | @itemize @bullet | ||
166 | @item gnunet-service-arm | ||
167 | @item gnunet-service-resolver (required by all) | ||
168 | @item gnunet-service-statistics (required by all) | ||
169 | @item gnunet-service-peerinfo | ||
170 | @item gnunet-service-transport (requires peerinfo) | ||
171 | @item gnunet-service-core (requires transport) | ||
172 | @item gnunet-daemon-hostlist (requires core) | ||
173 | @item gnunet-service-dht (requires core) | ||
174 | @item gnunet-service-mesh (requires dht, core) | ||
175 | @item gnunet-service-dns (requires dht) | ||
176 | @item gnunet-service-regex (requires dht) | ||
177 | @item gnunet-service-vpn (requires regex, dns, mesh, dht) | ||
178 | @item gnunet-service-identity | ||
179 | @item gnunet-service-namestore (requires identity) | ||
180 | @item gnunet-service-gns (requires vpn, dns, dht, namestore, identity) | ||
181 | @end itemize | ||
182 | |||
183 | @node Pre-installation notes | ||
184 | @section Pre-installation notes | ||
185 | |||
186 | Please note that in the code instructions for the installation, | ||
187 | @emph{#} indicates commands run as privileged root user and | ||
188 | @emph{$} shows commands run as unprivileged ("normal") system user. | ||
189 | |||
190 | |||
191 | @node Generic installation instructions | ||
192 | @section Generic installation instructions | ||
193 | |||
194 | First, in addition to the GNUnet sources you must download the latest version | ||
195 | of various dependencies. Most distributions do not include sufficiently recent | ||
196 | versions of these dependencies. Thus, a typically installation on a "modern" | ||
197 | GNU/Linux distribution requires you to install the following | ||
198 | dependencies (ideally in this order): | ||
199 | |||
200 | @itemize @bullet | ||
201 | @item libgpgerror and libgcrypt | ||
202 | @item libnettle and libunbound (possibly from distribution), GnuTLS | ||
203 | @item libgnurl (read the README) | ||
204 | @item GNU libmicrohttpd | ||
205 | @item GNU libextractor (make sure to first install the various mandatory and optional | ||
206 | dependencies including development headers from your distribution) | ||
207 | @end itemize | ||
208 | |||
209 | Other dependencies that you should strongly consider to install is a | ||
210 | database (MySQL, sqlite or Postgres). The following instructions will assume | ||
211 | that you installed at least sqlite. For most distributions you should be able | ||
212 | to find pre-build packages for the database. Again, make sure to install the | ||
213 | client libraries and the respective development headers (if they are | ||
214 | packaged separately) as well. | ||
215 | |||
216 | You can find specific, detailed instructions for installing of the dependencies | ||
217 | (and possibly the rest of the GNUnet installation) in the platform-specific | ||
218 | descriptions, which are linked from the bottom of this page. Please consult | ||
219 | them now. If your distribution is not listed, please study the instructions for | ||
220 | Debian stable carefully as you try to install the dependencies for your own | ||
221 | distribution. Contributing additional instructions for further platforms is | ||
222 | always appreciated. | ||
223 | |||
224 | Before proceeding further, please double-check the dependency list. Note that | ||
225 | in addition to satisfying the dependencies, you might have to make sure that | ||
226 | development headers for the various libraries are also installed. There maybe | ||
227 | files for other distributions, or you might be able to find equivalent packages | ||
228 | for your distribution. | ||
229 | |||
230 | While it is possible to build and install GNUnet without having root access, | ||
231 | we will assume that you have full control over your system in these | ||
232 | instructions. First, you should create a system user @emph{gnunet} and an additional | ||
233 | group @emph{gnunetdns}. On Debian and Ubuntu GNU/Linux, type:@ | ||
234 | @code{@ | ||
235 | # adduser --system --home /var/lib/gnunet --group --disabled-password gnunet@ | ||
236 | # addgroup --system gnunetdns@ | ||
237 | }@ | ||
238 | On other Unixes, this should have the same effect:@ | ||
239 | @code{@ | ||
240 | # useradd --system --groups gnunet --home-dir /var/lib/gnunet@ | ||
241 | # addgroup --system gnunetdns@ | ||
242 | }@ | ||
243 | Now compile and install GNUnet using:@ | ||
244 | @code{@ | ||
245 | $ tar xvf gnunet-0.10.?.tar.gz@ | ||
246 | $ cd gnunet-0.10.?@ | ||
247 | $ ./configure --with-sudo=sudo --with-nssdir=/lib@ | ||
248 | $ make@ | ||
249 | $ sudo make install@ | ||
250 | }@ | ||
251 | |||
252 | If you want to be able to enable DEBUG-level log messages, add | ||
253 | @code{--enable-logging=verbose} to the end of the @code{./configure} command. | ||
254 | DEBUG-level log messages are in English-only and should only be useful for | ||
255 | developers (or for filing really detailed bug reports). | ||
256 | |||
257 | Finally, you probably want to compile @code{gnunet-gtk}, which includes gnunet-setup | ||
258 | (graphical tool for configuration) and @code{gnunet-fs-gtk} (graphical tool for | ||
259 | file-sharing):@ | ||
260 | |||
261 | @code{@ | ||
262 | $ tar xvf gnunet-gtk-0.10.?.tar.gz@ | ||
263 | $ cd gnunet-gtk-0.10.?@ | ||
264 | $ ./configure --with-gnunet=/usr/local/@ | ||
265 | $ make@ | ||
266 | $ sudo make install@ | ||
267 | $ cd ..@ | ||
268 | $ sudo ldconfig # just to be safe@ | ||
269 | }@ | ||
270 | Next, edit the file @file{/etc/gnunet.conf} to contain the following:@ | ||
271 | @code{@ | ||
272 | [arm]@ | ||
273 | SYSTEM_ONLY = YES@ | ||
274 | USER_ONLY = NO@ | ||
275 | }@ | ||
276 | You may need to update your ld.so cache to include files installed in | ||
277 | @file{/usr/local/lib}:@ | ||
278 | |||
279 | @code{@ | ||
280 | # ldconfig@ | ||
281 | }@ | ||
282 | |||
283 | Then, switch from user root to user gnunet to start the peer:@ | ||
284 | |||
285 | @code{@ | ||
286 | # su -s /bin/sh - gnunet@ | ||
287 | $ gnunet-arm -c /etc/gnunet.conf -s@ | ||
288 | }@ | ||
289 | |||
290 | You may also want to add the last line in the gnunet users @file{crontab} | ||
291 | prefixed with @code{@@reboot} so that it is executed whenever the system is | ||
292 | booted:@ | ||
293 | |||
294 | @code{@ | ||
295 | @@reboot /usr/local/bin/gnunet-arm -c /etc/gnunet.conf -s@ | ||
296 | }@ | ||
297 | |||
298 | This will only start the system-wide GNUnet services. Type exit to get back | ||
299 | your root shell. Now, you need to configure the per-user part. For each | ||
300 | $USER on the system, run:@ | ||
301 | |||
302 | @code{@ | ||
303 | # adduser $USER gnunet@ | ||
304 | }@ | ||
305 | |||
306 | to allow them to access the system-wide GNUnet services. Then, each user should | ||
307 | create a configuration file @file{~/.config/gnunet.conf} with the lines:@ | ||
308 | |||
309 | @code{@ | ||
310 | [arm]@ | ||
311 | SYSTEM_ONLY = NO@ | ||
312 | USER_ONLY = YES@ | ||
313 | DEFAULTSERVICES = gns@ | ||
314 | }@ | ||
315 | |||
316 | and start the per-user services using@ | ||
317 | |||
318 | @code{@ | ||
319 | $ gnunet-arm -c ~/.config/gnunet.conf -s@ | ||
320 | }@ | ||
321 | |||
322 | Again, adding a @code{crontab} entry to autostart the peer is advised:@ | ||
323 | @code{@ | ||
324 | @@reboot /usr/local/bin/gnunet-arm -c $HOME/.config/gnunet.conf -s@ | ||
325 | }@ | ||
326 | |||
327 | Note that some GNUnet services (such as SOCKS5 proxies) may need a system-wide | ||
328 | TCP port for each user. For those services, systems with more than one user may | ||
329 | require each user to specify a different port number in their personal | ||
330 | configuration file. | ||
331 | |||
332 | Finally, the user should perform the basic initial setup for the GNU Name | ||
333 | System. This is done by running two commands:@ | ||
334 | |||
335 | @example | ||
336 | $ gnunet-gns-import.sh@ | ||
337 | $ gnunet-gns-proxy-setup-ca@ | ||
338 | @end example | ||
339 | |||
340 | The first generates the default zones, wheras the second setups the GNS | ||
341 | Certificate Authority with the user's browser. Now, to actiave GNS in the | ||
342 | normal DNS resolution process, you need to edit your @file{/etc/nsswitch.conf} | ||
343 | where you should find a line like this: | ||
344 | @example | ||
345 | hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4 | ||
346 | @end example | ||
347 | |||
348 | |||
349 | The exact details may differ a bit, which is fine. Add the text | ||
350 | @emph{"gns [NOTFOUND=return]"} after @emph{"files"}: | ||
351 | @example | ||
352 | hosts: files gns [NOTFOUND=return] mdns4_minimal [NOTFOUND=return] dns mdns4 | ||
353 | @end example | ||
354 | |||
355 | |||
356 | You might want to make sure that @file{/lib/libnss_gns.so.2} exists on your | ||
357 | system, it should have been created during the installation. | ||
358 | |||
359 | |||
360 | |||
361 | @node Build instructions for Ubuntu 12.04 using Git | ||
362 | @section Build instructions for Ubuntu 12.04 using Git | ||
363 | |||
364 | |||
365 | @menu | ||
366 | * Install the required build tools:: | ||
367 | * Install libgcrypt 1.6 and libgpg-error:: | ||
368 | * Install gnutls with DANE support:: | ||
369 | * Install libgnurl:: | ||
370 | * Install libmicrohttpd from Git:: | ||
371 | * Install libextractor from Git:: | ||
372 | * Install GNUnet dependencies:: | ||
373 | * Build GNUnet:: | ||
374 | * Install the GNUnet-gtk user interface from Git:: | ||
375 | @end menu | ||
376 | |||
377 | @node Install the required build tools | ||
378 | @subsection Install the required build tools | ||
379 | |||
380 | First, make sure Git is installed on your system:@ | ||
381 | |||
382 | $ sudo apt-get install git@ | ||
383 | |||
384 | Install the essential buildtools:@ | ||
385 | |||
386 | $ sudo apt-get install automake autopoint autoconf libtool | ||
387 | |||
388 | @node Install libgcrypt 1.6 and libgpg-error | ||
389 | @subsection Install libgcrypt 1.6 and libgpg-error | ||
390 | |||
391 | $ wget ftp://ftp.gnupg.org/gcrypt/libgpg-error/libgpg-error-1.12.tar.bz2@ | ||
392 | $ tar xf libgpg-error-1.12.tar.bz2@ | ||
393 | $ cd libgpg-error-1.12@ | ||
394 | $ ./configure@ | ||
395 | $ sudo make install@ | ||
396 | $ cd ..@ | ||
397 | |||
398 | @node Install gnutls with DANE support | ||
399 | @subsection Install gnutls with DANE support | ||
400 | |||
401 | @example | ||
402 | $ wget http://www.lysator.liu.se/~nisse/archive/nettle-2.7.1.tar.gz@ | ||
403 | $ tar xf nettle-2.7.1.tar.gz@ | ||
404 | $ cd nettle-2.7.1@ | ||
405 | $ ./configure@ | ||
406 | $ sudo make install@ | ||
407 | $ cd .. | ||
408 | @end example | ||
409 | |||
410 | @example | ||
411 | $ wget https://www.nlnetlabs.nl/downloads/ldns/ldns-1.6.16.tar.gz@ | ||
412 | $ tar xf ldns-1.6.16.tar.gz@ | ||
413 | $ cd ldns-1.6.16@ | ||
414 | $ ./configure@ | ||
415 | $ sudo make install@ | ||
416 | $ cd .. | ||
417 | @end example | ||
418 | |||
419 | @example | ||
420 | $ wget https://unbound.net/downloads/unbound-1.4.21.tar.gz@ | ||
421 | $ tar xf unbound-1.4.21.tar.gz@ | ||
422 | $ cd unbound-1.4.21@ | ||
423 | $ ./configure@ | ||
424 | $ sudo make install@ | ||
425 | $ cd .. | ||
426 | @end example | ||
427 | |||
428 | @example | ||
429 | $ wget ftp://ftp.gnutls.org/gcrypt/gnutls/v3.1/gnutls-3.1.17.tar.xz@ | ||
430 | $ tar xf gnutls-3.1.17.tar.xz@ | ||
431 | $ cd gnutls-3.1.17@ | ||
432 | $ ./configure@ | ||
433 | $ sudo make install@ | ||
434 | $ cd .. | ||
435 | @end example | ||
436 | |||
437 | @example | ||
438 | $ wget ftp://ftp.gnupg.org/gcrypt/libgcrypt/libgcrypt-1.6.0.tar.bz2@ | ||
439 | $ tar xf libgcrypt-1.6.0.tar.bz2@ | ||
440 | $ cd libgcrypt-1.6.0@ | ||
441 | $ ./configure@ | ||
442 | $ sudo make install@ | ||
443 | $ cd ..@ | ||
444 | @end example | ||
445 | |||
446 | @node Install libgnurl | ||
447 | @subsection Install libgnurl | ||
448 | |||
449 | @example | ||
450 | $ wget https://gnunet.org/sites/default/files/gnurl-7.34.0.tar.bz2@ | ||
451 | $ tar xf gnurl-7.34.0.tar.bz2@ | ||
452 | $ cd gnurl-7.34.0@ | ||
453 | $ ./configure --enable-ipv6 --with-gnutls --without-libssh2 \ | ||
454 | --without-libmetalink --without-winidn --without-librtmp \ | ||
455 | --without-nghttp2 --without-nss --without-cyassl \ | ||
456 | --without-polarssl --without-ssl --without-winssl \ | ||
457 | --without-darwinssl --disable-sspi --disable-ntlm-wb \ | ||
458 | --disable-ldap --disable-rtsp --disable-dict --disable-telnet \ | ||
459 | --disable-tftp --disable-pop3 --disable-imap --disable-smtp \ | ||
460 | --disable-gopher --disable-file --disable-ftp@ | ||
461 | $ sudo make install@ | ||
462 | $ cd ..@ | ||
463 | @end example | ||
464 | |||
465 | @node Install libmicrohttpd from Git | ||
466 | @subsection Install libmicrohttpd from Git | ||
467 | |||
468 | @example | ||
469 | $ git clone https://gnunet.org/git/libmicrohttpd@ | ||
470 | $ cd libmicrohttpd/@ | ||
471 | $ ./bootstrap@ | ||
472 | $ ./configure@ | ||
473 | $ sudo make install@ | ||
474 | $ cd ..@ | ||
475 | @end example | ||
476 | |||
477 | @node Install libextractor from Git | ||
478 | @subsection Install libextractor from Git | ||
479 | |||
480 | Install libextractor dependencies:@ | ||
481 | |||
482 | @example | ||
483 | $ sudo apt-get install zlib1g-dev libgsf-1-dev libmpeg2-4-dev libpoppler-dev \ | ||
484 | libvorbis-dev libexiv2-dev libjpeg-dev libtiff-dev libgif-dev libvorbis-dev \ | ||
485 | libflac-dev libsmf-dev g++@ | ||
486 | @end example | ||
487 | |||
488 | Build libextractor:@ | ||
489 | |||
490 | @example | ||
491 | $ git clone https://gnunet.org/git/libextractor@ | ||
492 | $ cd libextractor@ | ||
493 | $ ./bootstrap@ | ||
494 | $ ./configure@ | ||
495 | $ sudo make install@ | ||
496 | $ cd ..@ | ||
497 | @end example | ||
498 | |||
499 | @node Install GNUnet dependencies | ||
500 | @subsection Install GNUnet dependencies | ||
501 | |||
502 | @example | ||
503 | $ sudo apt-get install libidn11-dev libunistring-dev libglpk-dev \ | ||
504 | libpulse-dev libbluetooth-dev libsqlite-dev@ | ||
505 | @end example | ||
506 | |||
507 | Install libopus@ | ||
508 | |||
509 | @example | ||
510 | $ wget http://downloads.xiph.org/releases/opus/opus-1.1.tar.gz@ | ||
511 | $ tar xf opus-1.1.tar.gz@ | ||
512 | $ cd opus-1.1/@ | ||
513 | $ ./configure@ | ||
514 | $ sudo make install@ | ||
515 | @end example | ||
516 | |||
517 | Choose one or more database backends@ | ||
518 | |||
519 | @itemize @bullet | ||
520 | |||
521 | @item | ||
522 | SQLite3 @code{$ sudo apt-get install libsqlite3-dev} | ||
523 | |||
524 | @item | ||
525 | MySQL @code{$ sudo apt-get install libmysqlclient-dev} | ||
526 | |||
527 | @item | ||
528 | PostgreSQL @code{$ sudo apt-get install libpq-dev postgresql} | ||
529 | |||
530 | @end itemize | ||
531 | |||
532 | |||
533 | |||
534 | @node Build GNUnet | ||
535 | @subsection Build GNUnet | ||
536 | |||
537 | |||
538 | |||
539 | @menu | ||
540 | * Configuring the installation path:: | ||
541 | * Configuring the system:: | ||
542 | * Installing components requiring sudo permission:: | ||
543 | * Build:: | ||
544 | @end menu | ||
545 | |||
546 | @node Configuring the installation path | ||
547 | @subsubsection Configuring the installation path | ||
548 | |||
549 | You can specify the location of the GNUnet installation by setting the prefix | ||
550 | when calling the configure script:@code{ --prefix=DIRECTORY} | ||
551 | |||
552 | @code{@ | ||
553 | $ export PATH=$PATH:DIRECTORY/bin@ | ||
554 | } | ||
555 | |||
556 | @node Configuring the system | ||
557 | @subsubsection Configuring the system | ||
558 | |||
559 | Please make sure NOW that you have created a user and group 'gnunet'@ | ||
560 | and additionally a group 'gnunetdns':@ | ||
561 | @code{@ | ||
562 | $ sudo addgroup gnunet@ | ||
563 | $ sudo addgroup gnunetdns@ | ||
564 | $ sudo adduser gnunet@ | ||
565 | } | ||
566 | |||
567 | Each GNUnet user should be added to the 'gnunet' group (may@ | ||
568 | require fresh login to come into effect): | ||
569 | @code{@ | ||
570 | $ sudo useradd -G gnunet@ | ||
571 | } | ||
572 | |||
573 | @node Installing components requiring sudo permission | ||
574 | @subsubsection Installing components requiring sudo permission | ||
575 | |||
576 | Some components, like the nss plugin required for GNS, may require root | ||
577 | permissions. To allow these few components to be installed use:@ | ||
578 | @code{@ | ||
579 | $ ./configure --with-sudo} | ||
580 | |||
581 | @node Build | ||
582 | @subsubsection Build | ||
583 | |||
584 | |||
585 | @code{@ | ||
586 | $ git clone https://gnunet.org/git/gnunet/@ | ||
587 | $ cd gnunet/@ | ||
588 | $ ./bootstrap@ | ||
589 | } | ||
590 | Use the required configure call including the optional installation prefix | ||
591 | PREFIX or the sudo permissions@ | ||
592 | @code{$ ./configure [ --with-sudo | --with-prefix=PREFIX ]}@ | ||
593 | @code{$ make; sudo make install} | ||
594 | |||
595 | After installing it, you need to create an empty configuration file:@ | ||
596 | @code{mkdir ~/.gnunet; touch ~/.gnunet/gnunet.conf} | ||
597 | |||
598 | And finally you can start GNUnet with@ | ||
599 | @code{$ gnunet-arm -s} | ||
600 | |||
601 | @node Install the GNUnet-gtk user interface from Git | ||
602 | @subsection Install the GNUnet-gtk user interface from Git | ||
603 | |||
604 | |||
605 | Install depencies:@ | ||
606 | @code{$ sudo apt-get install libgtk-3-dev libunique-3.0-dev libgladeui-dev libqrencode-dev} | ||
607 | |||
608 | To build GNUnet (with an optional prefix)and execute:@ | ||
609 | @code{@ | ||
610 | $ git clone https://gnunet.org/git/gnunet-gtk/@ | ||
611 | $ cd gnunet-gtk/@ | ||
612 | $ ./bootstrap@ | ||
613 | $ ./configure [--prefix=PREFIX] --with-gnunet=DIRECTORY@ | ||
614 | $ make; sudo make install@ | ||
615 | } | ||
616 | |||
617 | @node Build Instructions for Microsoft Windows Platforms | ||
618 | @section Build Instructions for Microsoft Windows Platforms | ||
619 | |||
620 | |||
621 | |||
622 | @menu | ||
623 | * Introduction to building on MS Windows:: | ||
624 | * Requirements:: | ||
625 | * Dependencies & Initial Setup:: | ||
626 | * GNUnet Installation:: | ||
627 | * Adjusting Windows for running and testing GNUnet:: | ||
628 | * Building the GNUnet Installer:: | ||
629 | * Using GNUnet with Netbeans on Windows:: | ||
630 | @end menu | ||
631 | |||
632 | @node Introduction to building on MS Windows | ||
633 | @subsection Introduction to building on MS Windows | ||
634 | |||
635 | |||
636 | This document is a guide to building GNUnet and its dependencies on Windows | ||
637 | platforms. GNUnet development is mostly done under Linux and especially SVN | ||
638 | checkouts may not build out of the box. We regret any inconvenience, and if you | ||
639 | have problems, please report them. | ||
640 | |||
641 | @node Requirements | ||
642 | @subsection Requirements | ||
643 | |||
644 | The Howto is based upon a @strong{Windows Server 2008 32bit@strong{ | ||
645 | Installation, @strong{sbuild} and thus a @uref{http://www.mingw.org/wiki/MSYS, | ||
646 | MSYS+MinGW} (W32-GCC-Compiler-Suite + Unix-like Userland) installation. sbuild | ||
647 | is a convenient set of scripts which creates a working msys/mingw installation | ||
648 | and installs most dependencies required for GNUnet. }} | ||
649 | |||
650 | As of the point of the creation of this Howto, GNUnet @strong{requires} a | ||
651 | Windows @strong{Server} 2003 or newer for full feature support. Windows Vista | ||
652 | and later will also work, but | ||
653 | @strong{non-server version can not run a VPN-Exit-Node} as the NAT features | ||
654 | have been removed as of Windows Vista. | ||
655 | |||
656 | @node Dependencies & Initial Setup | ||
657 | @subsection Dependencies & Initial Setup | ||
658 | |||
659 | |||
660 | @itemize @bullet | ||
661 | |||
662 | @item | ||
663 | Install a fresh version of @strong{Python 2.x}, even if you are using a x64-OS, | ||
664 | install a 32-bit version for use with sbuild. Python 3.0 currently is | ||
665 | incompatible. | ||
666 | |||
667 | @item | ||
668 | Install your favorite @uref{http://code.google.com/p/tortoisegit/, GIT} & | ||
669 | @uref{http://tortoisesvn.net/, SVN}-clients. | ||
670 | |||
671 | @item | ||
672 | You will also need some archive-manager like @uref{http://www.7-zip.org/, 7zip}. | ||
673 | |||
674 | @item | ||
675 | Pull a copy of sbuild to a directory of your choice, which will be used in the | ||
676 | remainder of this guide. For now, we will use @file{c:\gnunet\sbuild\} | ||
677 | |||
678 | @item | ||
679 | in @file{sbuild\src\mingw\mingw32-buildall.sh}, comment out the packages | ||
680 | @strong{gnunet-svn} and @strong{gnunet-gtk-svn}, as we don't want sbuild to | ||
681 | compile/install those for us. | ||
682 | |||
683 | @item | ||
684 | Follow LRN's sbuild installation instructions.- | ||
685 | @end itemize | ||
686 | |||
687 | Please note that sbuild may (or will most likely) fail during installation, | ||
688 | thus you really HAVE to @strong{check the logfiles} created during the | ||
689 | installation process. Certain packages may fail to build initially due to | ||
690 | missing dependencies, thus you may have to | ||
691 | @strong{substitute those with binary-versions initially}. Later on once | ||
692 | dependencies are satisfied you can re-build the newer package versions. | ||
693 | |||
694 | @strong{It is normal that you may have to repeat this step multiple times and | ||
695 | there is no uniform way to fix all compile-time issues, as the build-process | ||
696 | of many of the dependencies installed are rather unstable on win32 and certain | ||
697 | releases may not even compile at all.} | ||
698 | |||
699 | Most dependencies for GNUnet have been set up by sbuild, thus we now should add | ||
700 | the @file{bin/} directories in your new msys and mingw installations to PATH. | ||
701 | You will want to create a backup of your finished msys-environment by now. | ||
702 | |||
703 | @node GNUnet Installation | ||
704 | @subsection GNUnet Installation | ||
705 | |||
706 | First, we need to launch our msys-shell, you can do this via | ||
707 | |||
708 | @file{C:\gnunet\sbuild\msys\msys.bat} | ||
709 | |||
710 | You might wish to take a look at this file and adjust some login-parameters to | ||
711 | your msys environment. | ||
712 | |||
713 | Also, sbuild added two pointpoints to your msys-environment, though those | ||
714 | might remain invisible: | ||
715 | |||
716 | @itemize @bullet | ||
717 | |||
718 | @item | ||
719 | /mingw, which will mount your mingw-directory from sbuild/mingw and the other one is | ||
720 | |||
721 | @item | ||
722 | /src which contains all the installation sources sbuild just compiled. | ||
723 | @end itemize | ||
724 | |||
725 | Check out the current gnunet-sources (svn-head) from the gnunet-repository, | ||
726 | we will do this in your home directory: | ||
727 | |||
728 | @code{svn checkout https://gnunet.org/svn/gnunet/ ~/gnunet} | ||
729 | |||
730 | Now, we will first need to bootstrap the checked out installation and then | ||
731 | configure it accordingly. | ||
732 | |||
733 | @example | ||
734 | cd ~/gnunet@ | ||
735 | ./bootstrap@ | ||
736 | STRIP=true CPPFLAGS="-DUSE_IPV6=1 -DW32_VEH" CFLAGS="$CFLAGS -g -O2" ./configure --prefix=/ --docdir=/share/doc/gnunet --with-libiconv-prefix=/mingw --with-libintl-prefix=/mingw --with-libcurl=/mingw --with-extractor=/mingw --with-sqlite=/mingw --with-microhttpd=/mingw --with-plibc=/mingw --enable-benchmarks --enable-expensivetests --enable-experimental --with-qrencode=/mingw --enable-silent-rules --enable-experimental 2>&1 | tee -a ./configure.log | ||
737 | @end example | ||
738 | |||
739 | The parameters above will configure for a reasonable gnunet installation to the | ||
740 | your msys-root directory. Depending on which features your would like to build | ||
741 | or you may need to specify additional dependencies. Sbuild installed most libs | ||
742 | into the /mingw subdirectory, so remember to prefix library locations with | ||
743 | this path. | ||
744 | |||
745 | Like on a unixoid system, you might want to use your home directory as prefix | ||
746 | for your own gnunet installation for development, without tainting the | ||
747 | buildenvironment. Just change the "prefix" parameter to point towards | ||
748 | ~/ in this case. | ||
749 | |||
750 | Now it's time to compile gnunet as usual. Though this will take some time, so | ||
751 | you may fetch yourself a coffee or some Mate now... | ||
752 | |||
753 | @example | ||
754 | make@ | ||
755 | make install | ||
756 | @end example | ||
757 | |||
758 | @node Adjusting Windows for running and testing GNUnet | ||
759 | @subsection Adjusting Windows for running and testing GNUnet | ||
760 | |||
761 | Assuming the build succeeded and you | ||
762 | @strong{added the bin directory of your gnunet to PATH}, you can now use your | ||
763 | gnunet-installation as usual. Remember that UAC or the windows firewall may | ||
764 | popup initially, blocking further execution of gnunet until you acknowledge | ||
765 | them (duh!). | ||
766 | |||
767 | You will also have to take the usual steps to get p2p software running properly | ||
768 | (port forwarding, ...), and gnunet will require administrative permissions as | ||
769 | it may even install a device-driver (in case you are using gnunet-vpn and/or | ||
770 | gnunet-exit). | ||
771 | |||
772 | @node Building the GNUnet Installer | ||
773 | @subsection Building the GNUnet Installer | ||
774 | |||
775 | The GNUnet installer is made with @uref{http://nsis.sourceforge.net/, NSIS}@ | ||
776 | The installer script is located in @file{contrib\win} in the GNUnet source tree. | ||
777 | |||
778 | @node Using GNUnet with Netbeans on Windows | ||
779 | @subsection Using GNUnet with Netbeans on Windows | ||
780 | |||
781 | TODO | ||
782 | |||
783 | @node Build instructions for Debian 7.5 | ||
784 | @section Build instructions for Debian 7.5 | ||
785 | |||
786 | |||
787 | These are the installation instructions for Debian 7.5. They were tested using | ||
788 | a minimal, fresh Debian 7.5 AMD64 installation without non-free software | ||
789 | (no contrib or non-free). By "minimal", we mean that during installation, we | ||
790 | did not select any desktop environment, servers or system utilities during the | ||
791 | "tasksel" step. Note that the packages and the dependencies that we will | ||
792 | install during this chapter take about 1.5 GB of disk space. Combined with | ||
793 | GNUnet and space for objects during compilation, you should not even attempt | ||
794 | this unless you have about 2.5 GB free after the minimal Debian installation. | ||
795 | Using these instructions to build a VM image is likely to require a minimum of | ||
796 | 4-5 GB for the VM (as you will likely also want a desktop manager). | ||
797 | |||
798 | GNUnet's security model assumes that your @file{/home} directory is encrypted. | ||
799 | Thus, if possible, you should encrypt your home partition | ||
800 | (or per-user home directory). | ||
801 | |||
802 | Naturally, the exact details of the starting state for your installation | ||
803 | should not matter much. For example, if you selected any of those installation | ||
804 | groups you might simply already have some of the necessary packages installed. | ||
805 | We did this for testing, as this way we are less likely to forget to mention a | ||
806 | required package. Note that we will not install a desktop environment, but of | ||
807 | course you will need to install one to use GNUnet's graphical user interfaces. | ||
808 | Thus, it is suggested that you simply install the desktop environment of your | ||
809 | choice before beginning with the instructions. | ||
810 | |||
811 | |||
812 | |||
813 | @menu | ||
814 | * Update:: | ||
815 | * Stable? Hah!:: | ||
816 | * Update again:: | ||
817 | * Installing packages:: | ||
818 | * Installing dependencies from source:: | ||
819 | * Installing GNUnet from source:: | ||
820 | * But wait there is more!:: | ||
821 | @end menu | ||
822 | |||
823 | @node Update | ||
824 | @subsection Update | ||
825 | |||
826 | After any installation, you should begin by running | ||
827 | |||
828 | @example | ||
829 | # apt-get update@ | ||
830 | # apt-get upgrade@ | ||
831 | @end example | ||
832 | |||
833 | to ensure that all of your packages are up-to-date. Note that the "#" is used | ||
834 | to indicate that you need to type in this command as "root" | ||
835 | (or prefix with "sudo"), whereas "$" is used to indicate typing in a command | ||
836 | as a normal user. | ||
837 | |||
838 | @node Stable? Hah! | ||
839 | @subsection Stable? Hah! | ||
840 | |||
841 | Yes, we said we start with a Debian 7.5 "stable" system. However, to reduce the | ||
842 | amount of compilation by hand, we will begin by allowing the installation of | ||
843 | packages from the testing and unstable distributions as well. We will stick to | ||
844 | "stable" packages where possible, but some packages will be taken from the | ||
845 | other distributions. Start by modifying @file{/etc/apt/sources.list} to contain | ||
846 | the following (possibly adjusted to point to your mirror of choice): | ||
847 | @example | ||
848 | # These were there before: | ||
849 | deb http://ftp.de.debian.org/debian/ wheezy main | ||
850 | deb-src http://ftp.de.debian.org/debian/ wheezy main | ||
851 | deb http://security.debian.org/ wheezy/updates main | ||
852 | deb-src http://security.debian.org/ wheezy/updates main | ||
853 | deb http://ftp.de.debian.org/debian/ wheezy-updates main | ||
854 | deb-src http://ftp.de.debian.org/debian/ wheezy-updates main | ||
855 | |||
856 | # Add these lines (feel free to adjust the mirror): | ||
857 | deb http://ftp.de.debian.org/debian/ testing main | ||
858 | deb http://ftp.de.debian.org/debian/ unstable main | ||
859 | @end example | ||
860 | |||
861 | The next step is to create/edit your @file{/etc/apt/preferences} file to look | ||
862 | like this: | ||
863 | |||
864 | @example | ||
865 | Package: * | ||
866 | Pin: release a=stable,n=wheezy | ||
867 | Pin-Priority: 700 | ||
868 | |||
869 | Package: * | ||
870 | Pin: release o=Debian,a=testing | ||
871 | Pin-Priority: 650 | ||
872 | |||
873 | Package: * | ||
874 | Pin: release o=Debian,a=unstable | ||
875 | Pin-Priority: 600 | ||
876 | @end example | ||
877 | |||
878 | You can read more about Apt Preferences here and here. Note that other pinnings | ||
879 | are likely to also work for GNUnet, the key thing is that you need some | ||
880 | packages from unstable (as shown below). However, as unstable is unlikely to | ||
881 | be comprehensive (missing packages) or might be problematic (crashing packages), | ||
882 | you probably want others from stable and/or testing. | ||
883 | |||
884 | @node Update again | ||
885 | @subsection Update again | ||
886 | |||
887 | Now, run again@ | ||
888 | |||
889 | @example | ||
890 | # apt-get update@ | ||
891 | # apt-get upgrade@ | ||
892 | @end example | ||
893 | |||
894 | to ensure that all your new distribution indices are downloaded, and that your | ||
895 | pinning is correct: the upgrade step should cause no changes at all. | ||
896 | |||
897 | @node Installing packages | ||
898 | @subsection Installing packages | ||
899 | |||
900 | We begin by installing a few Debian packages from stable:@ | ||
901 | |||
902 | @example | ||
903 | # apt-get install gcc make python-zbar libltdl-dev libsqlite3-dev \ | ||
904 | libunistring-dev libopus-dev libpulse-dev openssl libglpk-dev \ | ||
905 | texlive libidn11-dev libmysqlclient-dev libpq-dev libarchive-dev \ | ||
906 | libbz2-dev libexiv2-dev libflac-dev libgif-dev libglib2.0-dev \ | ||
907 | libgtk-3-dev libmagic-dev libjpeg8-dev libmpeg2-4-dev libmp4v2-dev \ | ||
908 | librpm-dev libsmf-dev libtidy-dev libtiff5-dev libvorbis-dev \ | ||
909 | libogg-dev zlib1g-dev g++ gettext libgsf-1-dev libunbound-dev \ | ||
910 | libqrencode-dev libgladeui-dev nasm texlive-latex-extra \ | ||
911 | libunique-3.0-dev gawk miniupnpc libfuse-dev libbluetooth-dev | ||
912 | @end example | ||
913 | |||
914 | After that, we install a few more packages from unstable:@ | ||
915 | |||
916 | @example | ||
917 | # apt-get install -t unstable nettle-dev libgstreamer1.0-dev \ | ||
918 | gstreamer1.0-plugins-base gstreamer1.0-plugins-good \ | ||
919 | libgstreamer-plugins-base1.0-dev | ||
920 | @end example | ||
921 | |||
922 | @node Installing dependencies from source | ||
923 | @subsection Installing dependencies from source | ||
924 | |||
925 | Next, we need to install a few dependencies from source. You might want to do | ||
926 | this as a "normal" user and only run the @code{make install} steps as root | ||
927 | (hence the @code{sudo} in the commands below). Also, you do this from any | ||
928 | directory. We begin by downloading all dependencies, then extracting the | ||
929 | sources, and finally compiling and installing the libraries:@ | ||
930 | |||
931 | @example | ||
932 | $ wget https://libav.org/releases/libav-9.10.tar.xz@ | ||
933 | $ wget http://ftp.gnu.org/gnu/libextractor/libextractor-1.3.tar.gz@ | ||
934 | $ wget ftp://ftp.gnupg.org/gcrypt/libgpg-error/libgpg-error-1.12.tar.bz2@ | ||
935 | $ wget ftp://ftp.gnupg.org/gcrypt/libgcrypt/libgcrypt-1.6.0.tar.bz2@ | ||
936 | $ wget ftp://ftp.gnutls.org/gcrypt/gnutls/v3.2/gnutls-3.2.7.tar.xz@ | ||
937 | $ wget http://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.33.tar.gz@ | ||
938 | $ wget https://gnunet.org/sites/default/files/gnurl-7.34.0.tar.bz2@ | ||
939 | $ tar xvf libextractor-1.3.tar.gz@ | ||
940 | $ tar xvf libgpg-error-1.12.tar.bz2@ | ||
941 | $ tar xvf libgcrypt-1.6.0.tar.bz2@ | ||
942 | $ tar xvf gnutls-3.2.7.tar.xz@ | ||
943 | $ tar xvf libmicrohttpd-0.9.33.tar.gz@ | ||
944 | $ tar xvf gnurl-7.34.0.tar.bz2@ | ||
945 | $ cd libav-0.9 ; ./configure --enable-shared; make; sudo make install ; cd ..@ | ||
946 | $ cd libextractor-1.3 ; ./configure; make ; sudo make install; cd ..@ | ||
947 | $ cd libgpg-error-1.12; ./configure ; make ; sudo make install ; cd ..@ | ||
948 | $ cd libgcrypt-1.6.0; ./configure --with-gpg-error-prefix=/usr/local; make ; sudo make install ; cd ..@ | ||
949 | $ cd gnutls-3.2.7 ; ./configure ; make ; sudo make install ; cd ..@ | ||
950 | $ cd libmicrohttpd-0.9.33; ./configure ; make ; sudo make install ; cd ..@ | ||
951 | $ cd gnurl-7.34.0@ | ||
952 | $ ./configure --enable-ipv6 --with-gnutls=/usr/local --without-libssh2 \ | ||
953 | --without-libmetalink --without-winidn --without-librtmp --without-nghttp2 \ | ||
954 | --without-nss --without-cyassl --without-polarssl --without-ssl \ | ||
955 | --without-winssl --without-darwinssl --disable-sspi --disable-ntlm-wb \ | ||
956 | --disable-ldap --disable-rtsp --disable-dict --disable-telnet --disable-tftp \ | ||
957 | --disable-pop3 --disable-imap --disable-smtp --disable-gopher --disable-file \ | ||
958 | --disable-ftp@ | ||
959 | $ make ; sudo make install; cd ..@ | ||
960 | @end example | ||
961 | |||
962 | @node Installing GNUnet from source | ||
963 | @subsection Installing GNUnet from source | ||
964 | |||
965 | |||
966 | For this, simply follow the generic installation instructions from | ||
967 | here. | ||
968 | |||
969 | @node But wait there is more! | ||
970 | @subsection But wait there is more! | ||
971 | |||
972 | So far, we installed all of the packages and dependencies required to ensure | ||
973 | that all of GNUnet would be built. However, while for example the plugins to | ||
974 | interact with the MySQL or Postgres databases have been created, we did not | ||
975 | actually install or configure those databases. Thus, you will need to install | ||
976 | and configure those databases or stick with the default Sqlite database. | ||
977 | Sqlite is usually fine for most applications, but MySQL can offer better | ||
978 | performance and Postgres better resillience. | ||
979 | |||
980 | |||
981 | @node Installing GNUnet from Git on Ubuntu 14.4 | ||
982 | @section Installing GNUnet from Git on Ubuntu 14.4 | ||
983 | |||
984 | @strong{Install the required build tools:} | ||
985 | @code{@ | ||
986 | $ sudo apt-get install git automake autopoint autoconf@ | ||
987 | } | ||
988 | |||
989 | @strong{Install the required dependencies} | ||
990 | @example | ||
991 | $ sudo apt-get install libltdl-dev libgpg-error-dev libidn11-dev \ | ||
992 | libunistring-dev libglpk-dev libbluetooth-dev libextractor-dev \ | ||
993 | libmicrohttpd-dev libgnutls28-dev | ||
994 | @end example | ||
995 | |||
996 | @strong{Choose one or more database backends}@ | ||
997 | SQLite3@ | ||
998 | @code{@ | ||
999 | $ sudo apt-get install libsqlite3-dev@ | ||
1000 | }@ | ||
1001 | MySQL@ | ||
1002 | @code{@ | ||
1003 | $ sudo apt-get install libmysqlclient-dev@ | ||
1004 | }@ | ||
1005 | PostgreSQL@ | ||
1006 | @code{@ | ||
1007 | $ sudo apt-get install libpq-dev postgresql@ | ||
1008 | } | ||
1009 | |||
1010 | @strong{Install the optional dependencies for gnunet-conversation:}@ | ||
1011 | @code{@ | ||
1012 | $ sudo apt-get install gstreamer1.0 libpulse-dev libopus-dev@ | ||
1013 | } | ||
1014 | |||
1015 | @strong{Install the libgrypt 1.6.1:}@ | ||
1016 | For Ubuntu 14.04:@ | ||
1017 | @code{$ sudo apt-get install libgcrypt20-dev}@ | ||
1018 | For Ubuntu older 14.04:@ | ||
1019 | @code{$ wget ftp://ftp.gnupg.org/gcrypt/libgcrypt/libgcrypt-1.6.1.tar.bz2@ | ||
1020 | $ tar xf libgcrypt-1.6.1.tar.bz2@ | ||
1021 | $ cd libgcrypt-1.6.1@ | ||
1022 | $ ./configure@ | ||
1023 | $ sudo make install@ | ||
1024 | $ cd ..}@ | ||
1025 | @strong{Install libgnurl}@ | ||
1026 | @example | ||
1027 | $ wget https://gnunet.org/sites/default/files/gnurl-7.35.0.tar.bz2@ | ||
1028 | $ tar xf gnurl-7.35.0.tar.bz2@ | ||
1029 | $ cd gnurl-7.35.0@ | ||
1030 | $ ./configure --enable-ipv6 --with-gnutls --without-libssh2 \ | ||
1031 | --without-libmetalink --without-winidn --without-librtmp --without-nghttp2 \ | ||
1032 | --without-nss --without-cyassl --without-polarssl --without-ssl \ | ||
1033 | --without-winssl --without-darwinssl --disable-sspi --disable-ntlm-wb \ | ||
1034 | --disable-ldap --disable-rtsp --disable-dict --disable-telnet --disable-tftp \ | ||
1035 | --disable-pop3 --disable-imap --disable-smtp --disable-gopher --disable-file \ | ||
1036 | --disable-ftp | ||
1037 | $ sudo make install@ | ||
1038 | $ cd ..@ | ||
1039 | @end example | ||
1040 | |||
1041 | @strong{Install GNUnet}@ | ||
1042 | @code{@ | ||
1043 | $ git clone https://gnunet.org/git/gnunet/@ | ||
1044 | $ cd gnunet/@ | ||
1045 | $ ./bootstrap@ | ||
1046 | } | ||
1047 | |||
1048 | If you want to: | ||
1049 | @itemize @bullet | ||
1050 | |||
1051 | |||
1052 | @item | ||
1053 | Install to a different directory:@ | ||
1054 | --prefix=PREFIX | ||
1055 | |||
1056 | @item | ||
1057 | Have sudo permission, but do not want to compile as root:@ | ||
1058 | --with-sudo | ||
1059 | |||
1060 | @item | ||
1061 | Want debug message enabled:@ | ||
1062 | -- enable-logging=verbose | ||
1063 | @end itemize | ||
1064 | |||
1065 | |||
1066 | @code{@ | ||
1067 | $ ./configure [ --with-sudo | --prefix=PREFIX | --- enable-logging=verbose]@ | ||
1068 | $ make; sudo make install@ | ||
1069 | } | ||
1070 | |||
1071 | After installing it, you need to create an empty configuration file:@ | ||
1072 | @code{touch ~/.config/gnunet.conf} | ||
1073 | |||
1074 | And finally you can start GNUnet with@ | ||
1075 | @code{$ gnunet-arm -s} | ||
1076 | |||
1077 | @node Build instructions for Debian 8 | ||
1078 | @section Build instructions for Debian 8 | ||
1079 | |||
1080 | These are the installation instructions for Debian 8. They were tested using a | ||
1081 | fresh Debian 8 AMD64 installation without non-free software (no contrib or | ||
1082 | non-free). During installation, I only selected "lxde" for the desktop | ||
1083 | environment. Note that the packages and the dependencies that we will install | ||
1084 | during this chapter take about 1.5 GB of disk space. Combined with GNUnet and | ||
1085 | space for objects during compilation, you should not even attempt this unless | ||
1086 | you have about 2.5 GB free after the Debian installation. Using these | ||
1087 | instructions to build a VM image is likely to require a minimum of 4-5 GB for | ||
1088 | the VM (as you will likely also want a desktop manager). | ||
1089 | |||
1090 | GNUnet's security model assumes that your @code{/home} directory is encrypted. | ||
1091 | Thus, if possible, you should encrypt your entire disk, or at least just your | ||
1092 | home partition (or per-user home directory). | ||
1093 | |||
1094 | Naturally, the exact details of the starting state for your installation should | ||
1095 | not matter much. For example, if you selected any of those installation groups | ||
1096 | you might simply already have some of the necessary packages installed. Thus, | ||
1097 | it is suggested that you simply install the desktop environment of your choice | ||
1098 | before beginning with the instructions. | ||
1099 | |||
1100 | |||
1101 | @menu | ||
1102 | * Update Debian:: | ||
1103 | * Installing Debian Packages:: | ||
1104 | * Installing Dependencies from Source2:: | ||
1105 | * Installing GNUnet from Source2:: | ||
1106 | * But wait (again) there is more!:: | ||
1107 | @end menu | ||
1108 | |||
1109 | @node Update Debian | ||
1110 | @subsection Update Debian | ||
1111 | |||
1112 | After any installation, you should begin by running@ | ||
1113 | @code{@ | ||
1114 | # apt-get update@ | ||
1115 | # apt-get upgrade@ | ||
1116 | }@ | ||
1117 | to ensure that all of your packages are up-to-date. Note that the "#" is used | ||
1118 | to indicate that you need to type in this command as "root" (or prefix with | ||
1119 | "sudo"), whereas "$" is used to indicate typing in a command as a normal | ||
1120 | user. | ||
1121 | |||
1122 | @node Installing Debian Packages | ||
1123 | @subsection Installing Debian Packages | ||
1124 | |||
1125 | We begin by installing a few Debian packages from stable:@ | ||
1126 | @example | ||
1127 | # apt-get install gcc make python-zbar libltdl-dev libsqlite3-dev \ | ||
1128 | libunistring-dev libopus-dev libpulse-dev openssl libglpk-dev texlive \ | ||
1129 | libidn11-dev libmysqlclient-dev libpq-dev libarchive-dev libbz2-dev \ | ||
1130 | libflac-dev libgif-dev libglib2.0-dev libgtk-3-dev libmpeg2-4-dev \ | ||
1131 | libtidy-dev libvorbis-dev libogg-dev zlib1g-dev g++ gettext libgsf-1-dev \ | ||
1132 | libunbound-dev libqrencode-dev libgladeui-dev nasm texlive-latex-extra \ | ||
1133 | libunique-3.0-dev gawk miniupnpc libfuse-dev libbluetooth-dev \ | ||
1134 | gstreamer1.0-plugins-base gstreamer1.0-plugins-good \ | ||
1135 | libgstreamer-plugins-base1.0-dev nettle-dev libextractor-dev libgcrypt20-dev \ | ||
1136 | libmicrohttpd-dev | ||
1137 | @end example | ||
1138 | |||
1139 | @node Installing Dependencies from Source2 | ||
1140 | @subsection Installing Dependencies from Source2 | ||
1141 | |||
1142 | Yes, we said we start with a Debian 8 "stable" system, but because Debian | ||
1143 | linked GnuTLS without support for DANE, we need to compile a few things, in | ||
1144 | addition to GNUnet, still by hand. Yes, you can run GNUnet using the respective | ||
1145 | Debian packages, but then you will not get DANE support. | ||
1146 | |||
1147 | Next, we need to install a few dependencies from source. You might want to do | ||
1148 | this as a "normal" user and only run the @code{make install} steps as root | ||
1149 | (hence the @code{sudo} in the commands below). Also, you do this from any | ||
1150 | directory. We begin by downloading all dependencies, then extracting the | ||
1151 | sources, and finally compiling and installing the libraries:@ | ||
1152 | |||
1153 | @code{@ | ||
1154 | $ wget ftp://ftp.gnutls.org/gcrypt/gnutls/v3.3/gnutls-3.3.12.tar.xz@ | ||
1155 | $ wget https://gnunet.org/sites/default/files/gnurl-7.40.0.tar.bz2@ | ||
1156 | $ tar xvf gnutls-3.3.12.tar.xz@ | ||
1157 | $ tar xvf gnurl-7.40.0.tar.bz2@ | ||
1158 | $ cd gnutls-3.3.12 ; ./configure ; make ; sudo make install ; cd ..@ | ||
1159 | $ cd gnurl-7.40.0@ | ||
1160 | $ ./configure --enable-ipv6 --with-gnutls=/usr/local --without-libssh2 \ | ||
1161 | --without-libmetalink --without-winidn --without-librtmp --without-nghttp2 \ | ||
1162 | --without-nss --without-cyassl --without-polarssl --without-ssl \ | ||
1163 | --without-winssl --without-darwinssl --disable-sspi --disable-ntlm-wb \ | ||
1164 | --disable-ldap --disable-rtsp --disable-dict --disable-telnet --disable-tftp \ | ||
1165 | --disable-pop3 --disable-imap --disable-smtp --disable-gopher --disable-file \ | ||
1166 | --disable-ftp --disable-smb | ||
1167 | $ make ; sudo make install; cd ..@ | ||
1168 | } | ||
1169 | |||
1170 | @node Installing GNUnet from Source2 | ||
1171 | @subsection Installing GNUnet from Source2 | ||
1172 | |||
1173 | For this, simply follow the generic installation instructions from@ | ||
1174 | here. | ||
1175 | |||
1176 | @node But wait (again) there is more! | ||
1177 | @subsection But wait (again) there is more! | ||
1178 | |||
1179 | So far, we installed all of the packages and dependencies required to ensure | ||
1180 | that all of GNUnet would be built. However, while for example the plugins to | ||
1181 | interact with the MySQL or Postgres databases have been created, we did not | ||
1182 | actually install or configure those databases. Thus, you will need to install | ||
1183 | and configure those databases or stick with the default Sqlite database. Sqlite | ||
1184 | is usually fine for most applications, but MySQL can offer better performance | ||
1185 | and Postgres better resillience. | ||
1186 | |||
1187 | @node Outdated build instructions for previous revisions | ||
1188 | @section Outdated build instructions for previous revisions | ||
1189 | |||
1190 | This chapter contains a collection of outdated, older installation guides. They | ||
1191 | are mostly intended to serve as a starting point for writing up-to-date | ||
1192 | instructions and should not be expected to work for GNUnet 0.10.x. | ||
1193 | A set of older installation instructions can also be found in the | ||
1194 | @file{doc/outdated-and-old-installation-instructions.txt} in the source | ||
1195 | of GNUnet. This file covers old instructions which no longer receive | ||
1196 | security updates or any kind of support. | ||
1197 | |||
1198 | |||
1199 | @menu | ||
1200 | * Installing GNUnet 0.10.1 on Ubuntu 14.04:: | ||
1201 | * Building GLPK for MinGW:: | ||
1202 | * GUI build instructions for Ubuntu 12.04 using Subversion:: | ||
1203 | * Installation with gnunet-update:: | ||
1204 | * Instructions for Microsoft Windows Platforms (Old):: | ||
1205 | @end menu | ||
1206 | |||
1207 | |||
1208 | @node Installing GNUnet 0.10.1 on Ubuntu 14.04 | ||
1209 | @subsection Installing GNUnet 0.10.1 on Ubuntu 14.04 | ||
1210 | |||
1211 | Install the required dependencies@ | ||
1212 | |||
1213 | @example | ||
1214 | $ sudo apt-get install libltdl-dev libgpg-error-dev libidn11-dev \ | ||
1215 | libunistring-dev libglpk-dev libbluetooth-dev libextractor-dev \ | ||
1216 | libmicrohttpd-dev libgnutls28-dev | ||
1217 | @end example | ||
1218 | |||
1219 | Choose one or more database backends@ | ||
1220 | SQLite3@ | ||
1221 | @code{@ | ||
1222 | $ sudo apt-get install libsqlite3-dev@ | ||
1223 | }@ | ||
1224 | MySQL@ | ||
1225 | @code{@ | ||
1226 | $ sudo apt-get install libmysqlclient-dev@ | ||
1227 | }@ | ||
1228 | PostgreSQL@ | ||
1229 | @code{@ | ||
1230 | $ sudo apt-get install libpq-dev postgresql@ | ||
1231 | } | ||
1232 | |||
1233 | Install the optional dependencies for gnunet-conversation:@ | ||
1234 | @code{@ | ||
1235 | $ sudo apt-get install gstreamer1.0 libpulse-dev libopus-dev@ | ||
1236 | } | ||
1237 | |||
1238 | Install the libgrypt 1.6:@ | ||
1239 | For Ubuntu 14.04:@ | ||
1240 | @code{$ sudo apt-get install libgcrypt20-dev}@ | ||
1241 | For Ubuntu older 14.04:@ | ||
1242 | @code{$ wget ftp://ftp.gnupg.org/gcrypt/libgcrypt/libgcrypt-1.6.1.tar.bz2@ | ||
1243 | $ tar xf libgcrypt-1.6.1.tar.bz2@ | ||
1244 | $ cd libgcrypt-1.6.1@ | ||
1245 | $ ./configure@ | ||
1246 | $ sudo make install@ | ||
1247 | $ cd ..} | ||
1248 | |||
1249 | Install libgnurl@ | ||
1250 | @example | ||
1251 | $ wget https://gnunet.org/sites/default/files/gnurl-7.35.0.tar.bz2@ | ||
1252 | $ tar xf gnurl-7.35.0.tar.bz2@ | ||
1253 | $ cd gnurl-7.35.0@ | ||
1254 | $ ./configure --enable-ipv6 --with-gnutls --without-libssh2 \ | ||
1255 | --without-libmetalink --without-winidn --without-librtmp --without-nghttp2 \ | ||
1256 | --without-nss --without-cyassl --without-polarssl --without-ssl \ | ||
1257 | --without-winssl --without-darwinssl --disable-sspi --disable-ntlm-wb \ | ||
1258 | --disable-ldap --disable-rtsp --disable-dict --disable-telnet --disable-tftp \ | ||
1259 | --disable-pop3 --disable-imap --disable-smtp --disable-gopher --disable-file \ | ||
1260 | --disable-ftp@ | ||
1261 | $ sudo make install@ | ||
1262 | $ cd ..@ | ||
1263 | @end example | ||
1264 | |||
1265 | Install GNUnet@ | ||
1266 | @code{@ | ||
1267 | $ wget http://ftpmirror.gnu.org/gnunet/gnunet-0.10.1.tar.gz@ | ||
1268 | $ tar xf gnunet-0.10.1.tar.gz@ | ||
1269 | $ cd gnunet-0.10.1@ | ||
1270 | } | ||
1271 | |||
1272 | If you want to: | ||
1273 | @itemize @bullet | ||
1274 | |||
1275 | @item | ||
1276 | Install to a different directory:@ | ||
1277 | --prefix=PREFIX | ||
1278 | |||
1279 | @item | ||
1280 | Have sudo permission, but do not want to compile as root:@ | ||
1281 | --with-sudo | ||
1282 | |||
1283 | @item | ||
1284 | Want debug message enabled:@ | ||
1285 | -- enable-logging=verbose | ||
1286 | @end itemize | ||
1287 | |||
1288 | @code{@ | ||
1289 | $ ./configure [ --with-sudo | --prefix=PREFIX | --enable-logging=verbose]@ | ||
1290 | $ make; sudo make install@ | ||
1291 | } | ||
1292 | |||
1293 | After installing it, you need to create an empty configuration file:@ | ||
1294 | @code{touch ~/.config/gnunet.conf} | ||
1295 | |||
1296 | And finally you can start GNUnet with@ | ||
1297 | @code{$ gnunet-arm -s} | ||
1298 | |||
1299 | @node Building GLPK for MinGW | ||
1300 | @subsection Building GLPK for MinGW | ||
1301 | |||
1302 | GNUnet now requires the GNU Linear Programming Kit (GLPK). Since there's is no | ||
1303 | package you can install with @code{mingw-get} you have to compile it from | ||
1304 | source: | ||
1305 | |||
1306 | @itemize @bullet | ||
1307 | |||
1308 | @item | ||
1309 | Download the latest version from http://ftp.gnu.org/gnu/glpk/ | ||
1310 | |||
1311 | @item | ||
1312 | Unzip it using your favourite unzipper@ | ||
1313 | In the MSYS shell: | ||
1314 | |||
1315 | @item | ||
1316 | change to the respective directory | ||
1317 | |||
1318 | @item | ||
1319 | @code{./configure '--build=i686-pc-mingw32'} | ||
1320 | |||
1321 | @item | ||
1322 | run @code{make install check } | ||
1323 | |||
1324 | MinGW does not automatically detect the correct buildtype so you have to | ||
1325 | specify it manually | ||
1326 | @end itemize | ||
1327 | |||
1328 | |||
1329 | @node GUI build instructions for Ubuntu 12.04 using Subversion | ||
1330 | @subsection GUI build instructions for Ubuntu 12.04 using Subversion | ||
1331 | |||
1332 | After installing GNUnet you can continue installing the GNUnet GUI tools: | ||
1333 | |||
1334 | First, install the required dependencies: | ||
1335 | |||
1336 | @code{@ | ||
1337 | $ sudo apt-get install libgladeui-dev libqrencode-dev@ | ||
1338 | } | ||
1339 | |||
1340 | Please ensure that the GNUnet shared libraries can be found by the linker. If | ||
1341 | you installed GNUnet libraries in a non standard path (say | ||
1342 | GNUNET_PREFIX=/usr/local/lib/), you can | ||
1343 | @itemize @bullet | ||
1344 | |||
1345 | |||
1346 | @item | ||
1347 | set the environmental variable permanently to@ | ||
1348 | @code{LD_LIBRARY_PATH=$GNUNET_PREFIX} | ||
1349 | |||
1350 | @item | ||
1351 | or add @code{$GNUNET_PREFIX} to @code{/etc/ld.so.conf} | ||
1352 | @end itemize | ||
1353 | |||
1354 | |||
1355 | Now you can checkout and compile the GNUnet GUI tools@ | ||
1356 | @code{@ | ||
1357 | $ svn co https://gnunet.org/svn/gnunet-gtk@ | ||
1358 | $ cd gnunet-gtk@ | ||
1359 | $ ./bootstrap@ | ||
1360 | $ ./configure --prefix=$GNUNET_PREFIX/.. --with-gnunet=$GNUNET_PREFIX/..@ | ||
1361 | $ make install@ | ||
1362 | } | ||
1363 | |||
1364 | @node Installation with gnunet-update | ||
1365 | @subsection Installation with gnunet-update | ||
1366 | |||
1367 | gnunet-update project is an effort to introduce updates to GNUnet | ||
1368 | installations. An interesting to-be-implemented-feature of gnunet-update is | ||
1369 | that these updates are propagated through GNUnet's peer-to-peer network. More | ||
1370 | information about gnunet-update can be found at | ||
1371 | https://gnunet.org/svn/gnunet-update/README. | ||
1372 | |||
1373 | While the project is still under development, we have implemented the following | ||
1374 | features which we believe may be helpful for users and we would like them to be | ||
1375 | tested: | ||
1376 | |||
1377 | @itemize @bullet | ||
1378 | |||
1379 | @item | ||
1380 | Packaging GNUnet installation along with its run-time dependencies into update | ||
1381 | packages | ||
1382 | |||
1383 | @item | ||
1384 | Installing update packages into compatible hosts | ||
1385 | |||
1386 | @item | ||
1387 | Updating an existing installation (which had been installed by gnunet-update) | ||
1388 | to a newer one | ||
1389 | @end itemize | ||
1390 | |||
1391 | The above said features of gnunet-update are currently available for testing on | ||
1392 | GNU/Linux systems. | ||
1393 | |||
1394 | The following is a guide to help you get started with gnunet-update. It shows | ||
1395 | you how to install the testing binary packages of GNUnet 0.9.1 we have at | ||
1396 | https://gnunet.org/install/ | ||
1397 | |||
1398 | gnunet-update needs the following: | ||
1399 | |||
1400 | @itemize @bullet | ||
1401 | @item | ||
1402 | python ( 2.6 or above) | ||
1403 | |||
1404 | @item | ||
1405 | gnupg | ||
1406 | |||
1407 | @item | ||
1408 | python-gpgme | ||
1409 | @end itemize | ||
1410 | |||
1411 | |||
1412 | Checkout gnunet-update:@ | ||
1413 | @code{@ | ||
1414 | $ svn checkout -r24905 https://gnunet.org/svn/gnunet-update@ | ||
1415 | } | ||
1416 | |||
1417 | For security reasons, all packages released for gnunet-update from us are | ||
1418 | signed with the key at https://gnunet.org/install/key.txt You would need to | ||
1419 | import this key into your gpg key ring. gnunet-update uses this key to verify | ||
1420 | the integrity of the packages it installs@ | ||
1421 | @code{@ | ||
1422 | $ gpg --recv-keys 7C613D78@ | ||
1423 | } | ||
1424 | |||
1425 | Download the packages relevant to your architecture (currently I have access to | ||
1426 | GNU/Linux machines on x86_64 and i686, so only two for now, hopefully more | ||
1427 | later) from https://gnunet.org/install/. | ||
1428 | |||
1429 | To install the downloaded package into the directory /foo: | ||
1430 | |||
1431 | @code{@ | ||
1432 | gnunet-update/bin/gnunet-update install downloaded/package /foo@ | ||
1433 | } | ||
1434 | |||
1435 | The installer reports the directories into which shared libraries and | ||
1436 | dependencies have been installed. You may need to add the reported shared | ||
1437 | library installation paths to LD_LIBRARY_PATH before you start running any | ||
1438 | installed binaries. | ||
1439 | |||
1440 | Please report bugs at https://gnunet.org/bugs/ under the project | ||
1441 | 'gnunet-update'. | ||
1442 | |||
1443 | @node Instructions for Microsoft Windows Platforms (Old) | ||
1444 | @subsection Instructions for Microsoft Windows Platforms (Old) | ||
1445 | |||
1446 | This document is a DEPRECATED installation guide for gnunet on windows. It will | ||
1447 | not work for recent gnunet versions, but maybe it will be of some use if | ||
1448 | problems arise. | ||
1449 | |||
1450 | The Windows build uses a UNIX emulator for Windows, | ||
1451 | @uref{http://www.mingw.org/, MinGW}, to build the executable modules. These | ||
1452 | modules run natively on Windows and do not require additional emulation | ||
1453 | software besides the usual dependencies. | ||
1454 | |||
1455 | GNUnet development is mostly done under Linux and especially SVN checkouts may | ||
1456 | not build out of the box. We regret any inconvenience, and if you have | ||
1457 | problems, please report them. | ||
1458 | |||
1459 | |||
1460 | |||
1461 | @menu | ||
1462 | * Hardware and OS requirements:: | ||
1463 | * Software installation:: | ||
1464 | * Building libextractor and GNUnet:: | ||
1465 | * Installer:: | ||
1466 | * Source:: | ||
1467 | @end menu | ||
1468 | |||
1469 | @node Hardware and OS requirements | ||
1470 | @subsubsection Hardware and OS requirements | ||
1471 | |||
1472 | @itemize @bullet | ||
1473 | |||
1474 | @item | ||
1475 | Pentium II or equivalent processor, 350 MHz or better | ||
1476 | |||
1477 | @item | ||
1478 | 128 MB RAM | ||
1479 | |||
1480 | @item | ||
1481 | 600 MB free disk space | ||
1482 | |||
1483 | @item | ||
1484 | Windows 2000 or Windows XP are recommended | ||
1485 | @end itemize | ||
1486 | |||
1487 | @node Software installation | ||
1488 | @subsubsection Software installation | ||
1489 | |||
1490 | @itemize @bullet | ||
1491 | |||
1492 | @item | ||
1493 | @strong{Compression software}@ | ||
1494 | @ | ||
1495 | The software packages GNUnet depends on are usually compressed using UNIX | ||
1496 | tools like tar, gzip and bzip2.@ If you do not already have an utility that is | ||
1497 | able to extract such archives, get @uref{http://www.7-zip.org/, 7-Zip}. | ||
1498 | |||
1499 | @item | ||
1500 | @strong{UNIX environment}@ | ||
1501 | @ | ||
1502 | The MinGW project provides the compiler toolchain that is used to build | ||
1503 | GNUnet.@ Get the following packages from | ||
1504 | @uref{http://sourceforge.net/projects/mingw/files/, the MinGW project}: | ||
1505 | @itemize @bullet | ||
1506 | |||
1507 | |||
1508 | @item | ||
1509 | GCC core | ||
1510 | |||
1511 | @item | ||
1512 | GCC g++ | ||
1513 | |||
1514 | @item | ||
1515 | MSYS | ||
1516 | |||
1517 | @item | ||
1518 | MSYS Developer Tool Kit (msysDTK) | ||
1519 | |||
1520 | @item | ||
1521 | MSYS Developer Tool Kit - msys-autoconf (bin) | ||
1522 | |||
1523 | @item | ||
1524 | MSYS Developer Tool Kit - msys-automake (bin) | ||
1525 | |||
1526 | @item | ||
1527 | MinGW Runtime | ||
1528 | |||
1529 | @item | ||
1530 | MinGW Utilities | ||
1531 | |||
1532 | @item | ||
1533 | Windows API | ||
1534 | |||
1535 | @item | ||
1536 | Binutils | ||
1537 | |||
1538 | @item | ||
1539 | make | ||
1540 | |||
1541 | @item | ||
1542 | pdcurses | ||
1543 | |||
1544 | @item | ||
1545 | GDB (snapshot) | ||
1546 | @end itemize | ||
1547 | |||
1548 | @itemize @bullet | ||
1549 | |||
1550 | |||
1551 | @item | ||
1552 | Install MSYS (to c:\mingw, for example.)@ | ||
1553 | Do @strong{not} use spaces in the pathname (c:\program files\mingw). | ||
1554 | |||
1555 | @item | ||
1556 | Install MinGW runtime, utilities and GCC to a subdirectory (to c:\mingw\mingw, | ||
1557 | for example) | ||
1558 | |||
1559 | @item | ||
1560 | Install the Development Kit to the MSYS directory (c:\mingw) | ||
1561 | |||
1562 | @item | ||
1563 | Create a batch file bash.bat in your MSYS directory with the files:@ | ||
1564 | |||
1565 | @example | ||
1566 | bin\sh.exe --login | ||
1567 | @end example | ||
1568 | |||
1569 | |||
1570 | This batch file opens a shell which is used to invoke the build processes..@ | ||
1571 | MinGW's standard shell (msys.bat) is not suitable because it opens a separate | ||
1572 | console window@ On Vista, bash.bat needs to be run as administrator. | ||
1573 | |||
1574 | @item | ||
1575 | Start bash.sh and rename (c:\mingw\mingw\)lib\libstdc++.la to avoid problems:@ | ||
1576 | |||
1577 | @example | ||
1578 | mv /usr/mingw/lib/libstdc++.la /usr/mingw/lib/libstdc++.la.broken | ||
1579 | @end example | ||
1580 | |||
1581 | |||
1582 | @item | ||
1583 | Unpack the Windows API to the MinGW directory (c:\mingw\mingw\) and remove the | ||
1584 | declaration of DATADIR from (c:\mingw\mingw\)include\objidl.h (lines 55-58) | ||
1585 | |||
1586 | @item | ||
1587 | Unpack autoconf, automake to the MSYS directory (c:\mingw) | ||
1588 | |||
1589 | @item | ||
1590 | Install all other packages to the MinGW directory (c:\mingw\mingw\) | ||
1591 | @end itemize | ||
1592 | |||
1593 | |||
1594 | @item | ||
1595 | @strong{GNU Libtool}@ | ||
1596 | @ | ||
1597 | GNU Libtool is required to use shared libraries.@ | ||
1598 | @ | ||
1599 | Get the prebuilt package from here and unpack it to the MinGW directory | ||
1600 | (c:\mingw) | ||
1601 | |||
1602 | @item | ||
1603 | @strong{Pthreads}@ | ||
1604 | @ | ||
1605 | GNUnet uses the portable POSIX thread library for multi-threading..@ | ||
1606 | |||
1607 | @itemize @bullet | ||
1608 | |||
1609 | |||
1610 | @item | ||
1611 | Save @uref{ftp://sources.redhat.com/pub/pthreads-win32/dll-latest/lib/x86/libpthreadGC2.a, libpthreadGC2.a} (x86) or @uref{ftp://sources.redhat.com/pub/pthreads-win32/dll-latest/lib/x64/libpthreadGC2.a, libpthreadGC2.a} (x64) as libpthread.a into the lib directory (c:\mingw\mingw\lib\libpthread.a) | ||
1612 | |||
1613 | @item | ||
1614 | Save @uref{ftp://sources.redhat.com/pub/pthreads-win32/dll-latest/lib/x86/pthreadGC2.dll, pthreadGC2.dll} (x86) or @uref{ftp://sources.redhat.com/pub/pthreads-win32/dll-latest/lib/x64/pthreadGC2.dll, libpthreadGC2.a} (x64) into the MinGW bin directory (c:\mingw\mingw\bin) | ||
1615 | |||
1616 | @item | ||
1617 | Download all header files from @uref{ftp://sources.redhat.com/pub/pthreads-win32/dll-latest/include/, include/} to the include directory (c:\mingw\mingw\include) | ||
1618 | @end itemize | ||
1619 | |||
1620 | |||
1621 | @item | ||
1622 | @strong{GNU MP@ | ||
1623 | }@ | ||
1624 | @ | ||
1625 | GNUnet uses the GNU Multiple Precision library for special cryptographic operations.@ | ||
1626 | @ | ||
1627 | Get the GMP binary package from the @uref{http://sourceforge.net/projects/mingwrep/, MinGW repository} and unpack it to the MinGW directory (c:\mingw\mingw) | ||
1628 | |||
1629 | @item | ||
1630 | @strong{GNU Gettext}@ | ||
1631 | @ | ||
1632 | GNU gettext is used to provide national language support.@ | ||
1633 | @ | ||
1634 | Get the prebuilt package from hereand unpack it to the MinGW directory (c:\mingw\mingw) | ||
1635 | |||
1636 | @item | ||
1637 | @strong{GNU iconv}@ | ||
1638 | @ | ||
1639 | GNU Libiconv is used for character encoding conversion.@ | ||
1640 | @ | ||
1641 | Get the prebuilt package from here and unpack it to the MinGW directory (c:\mingw\mingw) | ||
1642 | |||
1643 | @item | ||
1644 | @strong{SQLite}@ | ||
1645 | @ | ||
1646 | GNUnet uses the SQLite database to store data.@ | ||
1647 | @ | ||
1648 | Get the prebuilt binary from here and unpack it to your MinGW directory. | ||
1649 | |||
1650 | @item | ||
1651 | @strong{MySQL}@ | ||
1652 | @ | ||
1653 | As an alternative to SQLite, GNUnet also supports MySQL. | ||
1654 | @itemize @bullet | ||
1655 | |||
1656 | |||
1657 | @item | ||
1658 | Get the binary installer from the @uref{http://dev.mysql.com/downloads/mysql/4.1.html#Windows, MySQL project} (version 4.1),@ | ||
1659 | install it and follow the instructions in README.mysql. | ||
1660 | |||
1661 | @item | ||
1662 | Create a temporary build directory (c:\mysql) | ||
1663 | |||
1664 | @item | ||
1665 | Copy the directories include\ and lib\ from the MySQL directory to the new directory | ||
1666 | |||
1667 | @item | ||
1668 | Get the patches from @uref{http://bugs.mysql.com/bug.php?id=8906&files=1, Bug #8906} and @uref{http://bugs.mysql.com/bug.php?id=8872&files=1, Bug #8872} (the latter is only required for MySQL | ||
1669 | @example | ||
1670 | patch -p 0 | ||
1671 | @end example | ||
1672 | |||
1673 | |||
1674 | @item | ||
1675 | Move lib\opt\libmysql.dll to lib\libmysql.dll | ||
1676 | |||
1677 | @item | ||
1678 | Change to lib\ and create an import library:@ | ||
1679 | |||
1680 | @example | ||
1681 | dlltool --input-def ../include/libmySQL.def --dllname libmysql.dll | ||
1682 | --output-lib libmysqlclient.a -k | ||
1683 | @end example | ||
1684 | |||
1685 | |||
1686 | @item | ||
1687 | Copy include\* to include\mysql\ | ||
1688 | |||
1689 | @item | ||
1690 | Pass "--with-mysql=/c/mysql" to ./configure and copy libmysql.dll to your PATH or GNUnet′s bin\ directory | ||
1691 | @end itemize | ||
1692 | |||
1693 | |||
1694 | @item | ||
1695 | @strong{GTK+}@ | ||
1696 | @ | ||
1697 | gnunet-gtk and libextractor depend on GTK.@ | ||
1698 | @ | ||
1699 | Get the the binary and developer packages of atk, glib, gtk, iconv, gettext-runtime, pango from @uref{ftp://ftp.gtk.org/pub/gtk/v2.6/win32, gtk.org} and unpack it to the MinGW directory (c:\mingw\mingw)@ | ||
1700 | @ | ||
1701 | Get @uref{http://www.gtk.org/download/win32.php, pkg-config} and libpng and unpack them to the MinGW directory (c:\mingw\mingw)@ | ||
1702 | @ | ||
1703 | Here is an all-in-one package for @uref{http://ftp.gnome.org/pub/gnome/binaries/win32/gtk+/2.24/gtk+-bundle_2.24.10-20120208_win32.zip, gtk+dependencies}. Do not overwrite any existing files! | ||
1704 | |||
1705 | @item | ||
1706 | @strong{Glade}@ | ||
1707 | @ | ||
1708 | gnunet-gtk and and gnunet-setup were created using this interface builder@ | ||
1709 | |||
1710 | @itemize @bullet | ||
1711 | |||
1712 | |||
1713 | @item | ||
1714 | Get the Glade and libglade (-bin and -devel) packages (without GTK!) from @uref{http://gladewin32.sourceforge.net/, GladeWin32} and unpack it to the MinGW directory (c:\mingw\mingw) | ||
1715 | |||
1716 | @item | ||
1717 | Get libxml from here and unpack it to the MinGW directory (c:\mingw\mingw). | ||
1718 | @end itemize | ||
1719 | |||
1720 | |||
1721 | @item | ||
1722 | @strong{zLib}@ | ||
1723 | @ | ||
1724 | libextractor requires zLib to decompress some file formats. GNUnet uses it to (de)compress meta-data.@ | ||
1725 | @ | ||
1726 | Get zLib from here (Signature) and unpack it to the MinGW directory (c:\mingw\mingw) | ||
1727 | |||
1728 | @item | ||
1729 | @strong{Bzip2}@ | ||
1730 | @ | ||
1731 | libextractor also requires Bzip2 to decompress some file formats.@ | ||
1732 | @ | ||
1733 | Get Bzip2 (binary and developer package) from @uref{http://gnuwin32.sourceforge.net/packages/bzip2.htm, GnuWin32} and unpack it to the MinGW directory (c:\mingw\mingw) | ||
1734 | |||
1735 | @item | ||
1736 | @strong{Libgcrypt}@ | ||
1737 | @ | ||
1738 | Libgcrypt provides the cryptographic functions used by GNUnet@ | ||
1739 | @ | ||
1740 | Get Libgcrypt from @uref{ftp://ftp.gnupg.org/gcrypt/libgcrypt/, here}, compile and place it in the MinGW directory (c:\mingw\mingw). Currently you need at least version 1.4.2 to compile gnunet. | ||
1741 | |||
1742 | @item | ||
1743 | @strong{PlibC}@ | ||
1744 | @ | ||
1745 | PlibC emulates Unix functions under Windows.@ | ||
1746 | @ | ||
1747 | Get PlibC from here and unpack it to the MinGW directory (c:\mingw\mingw) | ||
1748 | |||
1749 | @item | ||
1750 | @strong{OGG Vorbis}@ | ||
1751 | @ | ||
1752 | OGG Vorbis is used to extract meta-data from .ogg files@ | ||
1753 | @ | ||
1754 | Get the packages @uref{http://www.gnunet.org/libextractor/download/win/libogg-1.1.4.zip, libogg} and @uref{http://www.gnunet.org/libextractor/download/win/libvorbis-1.2.3.zip, libvorbis} from the @uref{http://ftp.gnu.org/gnu/libextractor/libextractor-w32-1.0.0.zip, libextractor win32 build} and unpack them to the MinGW directory (c:\mingw\mingw) | ||
1755 | |||
1756 | @item | ||
1757 | @strong{Exiv2}@ | ||
1758 | @ | ||
1759 | (lib)Exiv2 is used to extract meta-data from files with Exiv2 meta-data@ | ||
1760 | @ | ||
1761 | Download @uref{http://www.gnunet.org/libextractor/download/win/exiv2-0.18.2.zip, Exiv2} and unpack it to the MSYS directory (c:\mingw) | ||
1762 | @end itemize | ||
1763 | |||
1764 | @node Building libextractor and GNUnet | ||
1765 | @subsubsection Building libextractor and GNUnet | ||
1766 | |||
1767 | Before you compile libextractor or GNUnet, be sure to set@ | ||
1768 | PKG_CONFIG_PATH: | ||
1769 | @example | ||
1770 | export PKG_CONFIG_PATH=/mingw/lib/pkgconfig | ||
1771 | @end example | ||
1772 | |||
1773 | |||
1774 | See Installation for basic instructions on building libextractor and GNUnet. | ||
1775 | |||
1776 | By default, all modules that are created in this way contain debug information and are quite large.@ | ||
1777 | To compile release versions (small and fast) set the variable CFLAGS: | ||
1778 | @example | ||
1779 | export CFLAGS='-O2 -march=pentium -fomit-frame-pointer' | ||
1780 | ./configure --prefix=$HOME --with-extractor=$HOME | ||
1781 | @end example | ||
1782 | |||
1783 | @node Installer | ||
1784 | @subsubsection Installer | ||
1785 | |||
1786 | The GNUnet installer is made with @uref{http://nsis.sourceforge.net/, NSIS}@ | ||
1787 | The installer script is located in contrib\win in the GNUnet source tree. | ||
1788 | |||
1789 | @node Source | ||
1790 | @subsubsection Source | ||
1791 | |||
1792 | The sources of all dependencies are available here. | ||
1793 | |||
1794 | @node Portable GNUnet | ||
1795 | @section Portable GNUnet | ||
1796 | |||
1797 | Quick instructions on how to use the most recent GNUnet on most GNU/Linux | ||
1798 | distributions | ||
1799 | |||
1800 | Currently this has only been tested on Ubuntu 12.04, 12.10, 13.04, Debian and | ||
1801 | CentOS 6, but it should work on almost any GNU/Linux distribution. More | ||
1802 | in-detail information can be found in the handbook. | ||
1803 | |||
1804 | |||
1805 | |||
1806 | @menu | ||
1807 | * Prerequisites:: | ||
1808 | * Download & set up gnunet-update:: | ||
1809 | * Install GNUnet:: | ||
1810 | @end menu | ||
1811 | |||
1812 | @node Prerequisites | ||
1813 | @subsection Prerequisites | ||
1814 | |||
1815 | Open a terminal and paste this line into it to install all required tools | ||
1816 | needed:@ | ||
1817 | @code{sudo apt-get install python-gpgme subversion} | ||
1818 | |||
1819 | @node Download & set up gnunet-update | ||
1820 | @subsection Download & set up gnunet-update | ||
1821 | |||
1822 | The following command will download a working version of gnunet-update with the | ||
1823 | subversion tool and import the public key which is needed for authentication:@ | ||
1824 | |||
1825 | @example | ||
1826 | svn checkout -r24905 https://gnunet.org/svn/gnunet-update ~/gnunet-update && | ||
1827 | cd ~/gnunet-update | ||
1828 | gpg --keyserver "hkp://keys.gnupg.net" --recv-keys 7C613D78 | ||
1829 | @end example | ||
1830 | |||
1831 | @node Install GNUnet | ||
1832 | @subsection Install GNUnet | ||
1833 | |||
1834 | Download and install GNUnet binaries which can be found here and set library | ||
1835 | paths:@ | ||
1836 | @code{@ | ||
1837 | wget -P /tmp https://gnunet.org/install/packs/gnunet-0.9.4-`uname -m`.tgz@ | ||
1838 | ./bin/gnunet-update install /tmp/gnunet-0.9*.tgz ~@ | ||
1839 | echo "PATH DEFAULT=$@{PATH@}:$HOME/bin" >> ~/.pam_environment@ | ||
1840 | echo -e "$@{HOME@}/lib\n$@{HOME@}/lib/gnunet-deps" | sudo tee /etc/ld.so.conf.d/gnunet.conf > /dev/null@ | ||
1841 | sudo ldconfig@ | ||
1842 | }@ | ||
1843 | |||
1844 | You may need to re-login once after executing these last commands | ||
1845 | |||
1846 | That's it, GNUnet is installed in your home directory now. GNUnet can be | ||
1847 | configured and afterwards started by executing@ | ||
1848 | @code{gnunet-arm -s} | ||
1849 | |||
1850 | @node The graphical configuration interface | ||
1851 | @section The graphical configuration interface | ||
1852 | |||
1853 | If you also would like to use gnunet-gtk and gnunet-setup (highly recommended | ||
1854 | for beginners), do: | ||
1855 | |||
1856 | @example | ||
1857 | wget -P /tmp https://gnunet.org/install/packs/gnunet-0.9.4-gtk-0.9.4-`uname -m`.tgz@ | ||
1858 | sh ~/gnunet-update/bin/gnunet-update install /tmp/gnunet-*gtk*.tgz ~@ | ||
1859 | sudo ldconfig | ||
1860 | @end example | ||
1861 | Now you can run @code{gnunet-setup} for easy configuration of your GNUnet peer. | ||
1862 | |||
1863 | |||
1864 | @menu | ||
1865 | * Configuring your peer:: | ||
1866 | * Configuring the Friend-to-Friend (F2F) mode:: | ||
1867 | * Configuring the hostlist to bootstrap:: | ||
1868 | * Configuration of the HOSTLIST proxy settings:: | ||
1869 | * Configuring your peer to provide a hostlist :: | ||
1870 | * Configuring the datastore:: | ||
1871 | * Configuring the MySQL database:: | ||
1872 | * Reasons for using MySQL:: | ||
1873 | * Reasons for not using MySQL:: | ||
1874 | * Setup Instructions:: | ||
1875 | * Testing:: | ||
1876 | * Performance Tuning:: | ||
1877 | * Setup for running Testcases:: | ||
1878 | * Configuring the Postgres database:: | ||
1879 | * Reasons to use Postgres:: | ||
1880 | * Reasons not to use Postgres:: | ||
1881 | * Manual setup instructions:: | ||
1882 | * Testing the setup manually:: | ||
1883 | * Configuring the datacache:: | ||
1884 | * Configuring the file-sharing service:: | ||
1885 | * Configuring logging:: | ||
1886 | * Configuring the transport service and plugins:: | ||
1887 | * Configuring the wlan transport plugin:: | ||
1888 | * Configuring HTTP(S) reverse proxy functionality using Apache or nginx:: | ||
1889 | * Blacklisting peers:: | ||
1890 | * Configuration of the HTTP and HTTPS transport plugins:: | ||
1891 | * Configuring the GNU Name System:: | ||
1892 | * Configuring the GNUnet VPN:: | ||
1893 | * Bandwidth Configuration:: | ||
1894 | * Configuring NAT:: | ||
1895 | * Peer configuration for distributions:: | ||
1896 | @end menu | ||
1897 | |||
1898 | @node Configuring your peer | ||
1899 | @subsection Configuring your peer | ||
1900 | |||
1901 | This chapter will describe the various configuration options in GNUnet. | ||
1902 | |||
1903 | The easiest way to configure your peer is to use the gnunet-setup tool. | ||
1904 | gnunet-setup is part of the gnunet-gtk download. You might have to install it | ||
1905 | separately. | ||
1906 | |||
1907 | Many of the specific sections from this chapter actually are linked from within | ||
1908 | gnunet-setup to help you while using the setup tool. | ||
1909 | |||
1910 | While you can also configure your peer by editing the configuration file by | ||
1911 | hand, this is not recommended for anyone except for developers. | ||
1912 | |||
1913 | |||
1914 | |||
1915 | |||
1916 | |||
1917 | @node Configuring the Friend-to-Friend (F2F) mode | ||
1918 | @subsection Configuring the Friend-to-Friend (F2F) mode | ||
1919 | |||
1920 | GNUnet knows three basic modes of operation. In standard "peer-to-peer" mode, | ||
1921 | your peer will connect to any peer. In the pure "friend-to-friend" mode, your | ||
1922 | peer will ONLY connect to peers from a list of friends specified in the | ||
1923 | configuration. Finally, in mixed mode, GNUnet will only connect to arbitrary | ||
1924 | peers if it has at least a specified number of connections to friends. | ||
1925 | |||
1926 | When configuring any of the F2F modes, you first need to create a file with the | ||
1927 | peer identities of your friends. Ask your friends to run | ||
1928 | |||
1929 | $ gnunet-peerinfo -sq | ||
1930 | |||
1931 | The output of this command needs to be added to your friends file, which is | ||
1932 | simply a plain text file with one line per friend with the output from the | ||
1933 | above command. | ||
1934 | |||
1935 | You then specify the location of your friends file in the "FRIENDS" option of | ||
1936 | the "topology" section. | ||
1937 | |||
1938 | Once you have created the friends file, you can tell GNUnet to only connect to | ||
1939 | your friends by setting the "FRIENDS-ONLY" option (again in the "topology" | ||
1940 | section) to YES. | ||
1941 | |||
1942 | If you want to run in mixed-mode, set "FRIENDS-ONLY" to NO and configure a | ||
1943 | minimum number of friends to have (before connecting to arbitrary peers) under | ||
1944 | the "MINIMUM-FRIENDS" option. | ||
1945 | |||
1946 | If you want to operate in normal P2P-only mode, simply set "MINIMUM-FRIENDS" to | ||
1947 | zero and "FRIENDS_ONLY" to NO. This is the default. | ||
1948 | |||
1949 | @node Configuring the hostlist to bootstrap | ||
1950 | @subsection Configuring the hostlist to bootstrap | ||
1951 | |||
1952 | After installing the software you need to get connected to the GNUnet network. | ||
1953 | The configuration file included in your download is already configured to | ||
1954 | connect you to the GNUnet network. In this section the relevant configuration | ||
1955 | settings are explained. | ||
1956 | |||
1957 | To get an initial connection to the GNUnet network and to get to know peers | ||
1958 | already connected to the network you can use the so called bootstrap servers. | ||
1959 | These servers can give you a list of peers connected to the network. To use | ||
1960 | these bootstrap servers you have to configure the hostlist daemon to activate | ||
1961 | bootstrapping. | ||
1962 | |||
1963 | To activate bootstrapping edit your configuration file and edit the | ||
1964 | @code{[hostlist]}-section. You have to set the argument "-b" in the options | ||
1965 | line: | ||
1966 | @example | ||
1967 | [hostlist] | ||
1968 | OPTIONS = -b | ||
1969 | @end example | ||
1970 | |||
1971 | Additionally you have to specify which server you want to use. The default | ||
1972 | bootstrapping server is "@uref{http://v10.gnunet.org/hostlist, | ||
1973 | http://v10.gnunet.org/hostlist}". [^] To set the server you have to edit the | ||
1974 | line "SERVERS" in the hostlist section. To use the default server you should | ||
1975 | set the lines to | ||
1976 | @example | ||
1977 | SERVERS = http://v10.gnunet.org/hostlist [^] | ||
1978 | @end example | ||
1979 | |||
1980 | |||
1981 | To use bootstrapping your configuration file should include these lines: | ||
1982 | @example | ||
1983 | [hostlist] | ||
1984 | OPTIONS = -b | ||
1985 | SERVERS = http://v10.gnunet.org/hostlist [^] | ||
1986 | @end example | ||
1987 | |||
1988 | |||
1989 | Besides using bootstrap servers you can configure your GNUnet peer to recieve | ||
1990 | hostlist advertisements. Peers offering hostlists to other peers can send | ||
1991 | advertisement messages to peers that connect to them. If you configure your | ||
1992 | peer to receive these messages, your peer can download these lists and connect | ||
1993 | to the peers included. These lists are persistent, which means that they are | ||
1994 | saved to your hard disk regularly and are loaded during startup. | ||
1995 | |||
1996 | To activate hostlist learning you have to add the "-e" switch to the OPTIONS | ||
1997 | line in the hostlist section: | ||
1998 | @example | ||
1999 | [hostlist] | ||
2000 | OPTIONS = -b -e | ||
2001 | @end example | ||
2002 | |||
2003 | |||
2004 | Furthermore you can specify in which file the lists are saved. To save the | ||
2005 | lists in the file "hostlists.file" just add the line: | ||
2006 | @example | ||
2007 | HOSTLISTFILE = hostlists.file | ||
2008 | @end example | ||
2009 | |||
2010 | |||
2011 | Best practice is to activate both bootstrapping and hostlist learning. So your | ||
2012 | configuration file should include these lines: | ||
2013 | @example | ||
2014 | [hostlist] | ||
2015 | OPTIONS = -b -e | ||
2016 | HTTPPORT = 8080 | ||
2017 | SERVERS = http://v10.gnunet.org/hostlist [^] | ||
2018 | HOSTLISTFILE = $SERVICEHOME/hostlists.file | ||
2019 | @end example | ||
2020 | |||
2021 | @node Configuration of the HOSTLIST proxy settings | ||
2022 | @subsection Configuration of the HOSTLIST proxy settings | ||
2023 | |||
2024 | The hostlist client can be configured to use a proxy to connect to the hostlist | ||
2025 | server. This functionality can be configured in the configuration file directly | ||
2026 | or using the gnunet-setup tool. | ||
2027 | |||
2028 | The hostlist client supports the following proxy types at the moment: | ||
2029 | @itemize @bullet | ||
2030 | |||
2031 | |||
2032 | @item | ||
2033 | HTTP and HTTP 1.0 only proxy | ||
2034 | |||
2035 | @item | ||
2036 | SOCKS 4/4a/5/5 with hostname | ||
2037 | @end itemize | ||
2038 | |||
2039 | |||
2040 | In addition authentication at the proxy with username and password can be | ||
2041 | configured. | ||
2042 | |||
2043 | To configure proxy support for the hostlist client in the gnunet-setup tool, | ||
2044 | select the "hostlist" tab and select the appropriate proxy type. The hostname | ||
2045 | or IP address (including port if required) has to be entered in the "Proxy | ||
2046 | hostname" textbox. If required, enter username and password in the "Proxy | ||
2047 | username" and "Proxy password" boxes. Be aware that these information will be | ||
2048 | stored in the configuration in plain text. | ||
2049 | |||
2050 | To configure these options directly in the configuration, you can configure the | ||
2051 | following settings in the @code{[hostlist]} section of the configuration:@ | ||
2052 | @example | ||
2053 | # Type of proxy server,@ | ||
2054 | # Valid values: HTTP, HTTP_1_0, SOCKS4, SOCKS5, SOCKS4A, SOCKS5_HOSTNAME@ | ||
2055 | # Default: HTTP@ | ||
2056 | # PROXY_TYPE = HTTP | ||
2057 | |||
2058 | # Hostname or IP of proxy server@ | ||
2059 | # PROXY =@ | ||
2060 | # User name for proxy server@ | ||
2061 | # PROXY_USERNAME =@ | ||
2062 | # User password for proxy server@ | ||
2063 | # PROXY_PASSWORD =@ | ||
2064 | @end example | ||
2065 | |||
2066 | @node Configuring your peer to provide a hostlist | ||
2067 | @subsection Configuring your peer to provide a hostlist | ||
2068 | |||
2069 | If you operate a peer permanently connected to GNUnet you can configure your | ||
2070 | peer to act as a hostlist server, providing other peers the list of peers known | ||
2071 | to him. | ||
2072 | |||
2073 | Yor server can act as a bootstrap server and peers needing to obtain a list of | ||
2074 | peers can contact him to download this list. To download this hostlist the peer | ||
2075 | uses HTTP. For this reason you have to build your peer with libcurl and | ||
2076 | microhttpd support. How you build your peer with this options can be found | ||
2077 | here: https://gnunet.org/generic_installation | ||
2078 | |||
2079 | To configure your peer to act as a bootstrap server you have to add the "-p" | ||
2080 | option to OPTIONS in the [hostlist] section of your configuration file. Besides | ||
2081 | that you have to specify a port number for the http server. In conclusion you | ||
2082 | have to add the following lines: | ||
2083 | |||
2084 | @example | ||
2085 | [hostlist] | ||
2086 | HTTPPORT = 12980 | ||
2087 | OPTIONS = -p | ||
2088 | @end example | ||
2089 | |||
2090 | |||
2091 | If your peer acts as a bootstrap server other peers should know about that. You | ||
2092 | can advertise the hostlist your are providing to other peers. Peers connecting | ||
2093 | to your peer will get a message containing an advertisement for your hostlist | ||
2094 | and the URL where it can be downloaded. If this peer is in learning mode, it | ||
2095 | will test the hostlist and, in the case it can obtain the list successfully, it | ||
2096 | will save it for bootstrapping. | ||
2097 | |||
2098 | To activate hostlist advertisement on your peer, you have to set the following | ||
2099 | lines in your configuration file: | ||
2100 | @example | ||
2101 | [hostlist] | ||
2102 | EXTERNAL_DNS_NAME = example.org | ||
2103 | HTTPPORT = 12981 | ||
2104 | OPTIONS = -p -a | ||
2105 | @end example | ||
2106 | |||
2107 | |||
2108 | With this configuration your peer will a act as a bootstrap server and | ||
2109 | advertise this hostlist to other peers connecting to him. The URL used to | ||
2110 | download the list will be @code{@uref{http://example.org:12981/, | ||
2111 | http://example.org:12981/}}. | ||
2112 | |||
2113 | Please notice: | ||
2114 | @itemize @bullet | ||
2115 | |||
2116 | |||
2117 | @item | ||
2118 | The hostlist is not human readable, so you should not try to download it using | ||
2119 | your webbrowser. Just point your GNUnet peer to the address! | ||
2120 | |||
2121 | @item | ||
2122 | Advertising without providing a hostlist does not make sense and will not work. | ||
2123 | @end itemize | ||
2124 | |||
2125 | @node Configuring the datastore | ||
2126 | @subsection Configuring the datastore | ||
2127 | |||
2128 | The datastore is what GNUnet uses to for long-term storage of file-sharing | ||
2129 | data. Note that long-term does not mean 'forever' since content does have an | ||
2130 | expiration date, and of course storage space is finite (and hence sometimes | ||
2131 | content may have to be discarded). | ||
2132 | |||
2133 | Use the "QUOTA" option to specify how many bytes of storage space you are | ||
2134 | willing to dedicate to GNUnet. | ||
2135 | |||
2136 | In addition to specifying the maximum space GNUnet is allowed to use for the | ||
2137 | datastore, you need to specify which database GNUnet should use to do so. | ||
2138 | Currently, you have the choice between sqLite, MySQL and Postgres. | ||
2139 | |||
2140 | @node Configuring the MySQL database | ||
2141 | @subsection Configuring the MySQL database | ||
2142 | |||
2143 | This section describes how to setup the MySQL database for GNUnet. | ||
2144 | |||
2145 | Note that the mysql plugin does NOT work with mysql before 4.1 since we need | ||
2146 | prepared statements. We are generally testing the code against MySQL 5.1 at | ||
2147 | this point. | ||
2148 | |||
2149 | @node Reasons for using MySQL | ||
2150 | @subsection Reasons for using MySQL | ||
2151 | |||
2152 | @itemize @bullet | ||
2153 | |||
2154 | @item | ||
2155 | On up-to-date hardware where mysql can be used comfortably, this module will | ||
2156 | have better performance than the other database choices (according to our | ||
2157 | tests). | ||
2158 | |||
2159 | @item Its often possible to recover the mysql database from internal | ||
2160 | inconsistencies. Some of the other databases do not support repair. | ||
2161 | @end itemize | ||
2162 | |||
2163 | @node Reasons for not using MySQL | ||
2164 | @subsection Reasons for not using MySQL | ||
2165 | |||
2166 | @itemize @bullet | ||
2167 | |||
2168 | @item | ||
2169 | Memory usage (likely not an issue if you have more than 1 GB) | ||
2170 | |||
2171 | @item | ||
2172 | Complex manual setup | ||
2173 | @end itemize | ||
2174 | |||
2175 | @node Setup Instructions | ||
2176 | @subsection Setup Instructions | ||
2177 | |||
2178 | @itemize @bullet | ||
2179 | |||
2180 | @item | ||
2181 | In @code{gnunet.conf} set in section "DATASTORE" the value for "DATABASE" to | ||
2182 | "mysql". | ||
2183 | |||
2184 | @item | ||
2185 | Access mysql as root:@ | ||
2186 | |||
2187 | @example | ||
2188 | $ mysql -u root -p | ||
2189 | @end example | ||
2190 | |||
2191 | |||
2192 | and issue the following commands, replacing $USER with the username@ | ||
2193 | that will be running gnunet-arm (so typically "gnunet"): | ||
2194 | @example | ||
2195 | CREATE DATABASE gnunet; | ||
2196 | GRANT select,insert,update,delete,create,alter,drop,create temporary tables | ||
2197 | ON gnunet.* TO $USER@@localhost; | ||
2198 | SET PASSWORD FOR $USER@@localhost=PASSWORD('$the_password_you_like'); | ||
2199 | FLUSH PRIVILEGES; | ||
2200 | @end example | ||
2201 | |||
2202 | |||
2203 | @item | ||
2204 | In the $HOME directory of $USER, create a ".my.cnf" file with the following lines@ | ||
2205 | |||
2206 | @example | ||
2207 | [client] | ||
2208 | user=$USER | ||
2209 | password=$the_password_you_like | ||
2210 | @end example | ||
2211 | |||
2212 | @end itemize | ||
2213 | |||
2214 | |||
2215 | Thats it. Note that @code{.my.cnf} file is a slight security risk unless its | ||
2216 | on@ a safe partition. The $HOME/.my.cnf can of course be a symbolic@ link. | ||
2217 | Luckily $USER has only priviledges to mess up GNUnet's tables, which should be | ||
2218 | pretty harmless. | ||
2219 | @node Testing | ||
2220 | @subsection Testing | ||
2221 | |||
2222 | You should briefly try if the database connection works. First, login as $USER. | ||
2223 | Then use: | ||
2224 | @example | ||
2225 | $ mysql -u $USER | ||
2226 | mysql> use gnunet; | ||
2227 | @end example | ||
2228 | |||
2229 | |||
2230 | If you get the message "Database changed" it probably works. | ||
2231 | |||
2232 | If you get "ERROR 2002: Can't connect to local MySQL server@ | ||
2233 | through socket '/tmp/mysql.sock' (2)" it may be resolvable by@ | ||
2234 | "ln -s /var/run/mysqld/mysqld.sock /tmp/mysql.sock"@ | ||
2235 | so there may be some additional trouble depending on your mysql setup. | ||
2236 | @node Performance Tuning | ||
2237 | @subsection Performance Tuning | ||
2238 | |||
2239 | For GNUnet, you probably want to set the option | ||
2240 | @example | ||
2241 | innodb_flush_log_at_trx_commit = 0 | ||
2242 | @end example | ||
2243 | |||
2244 | for a rather dramatic boost in MySQL performance. However, this reduces the | ||
2245 | "safety" of your database as with this options you may loose transactions | ||
2246 | during a power outage. While this is totally harmless for GNUnet, the option | ||
2247 | applies to all applications using MySQL. So you should set it if (and only if) | ||
2248 | GNUnet is the only application on your system using MySQL. | ||
2249 | |||
2250 | @node Setup for running Testcases | ||
2251 | @subsection Setup for running Testcases | ||
2252 | |||
2253 | If you want to run the testcases, you must create a second database | ||
2254 | "gnunetcheck" with the same username and password. This database will then be | ||
2255 | used for testing ("make check"). | ||
2256 | |||
2257 | @node Configuring the Postgres database | ||
2258 | @subsection Configuring the Postgres database | ||
2259 | |||
2260 | This text describes how to setup the Postgres database for GNUnet. | ||
2261 | |||
2262 | This Postgres plugin was developed for Postgres 8.3 but might work for earlier | ||
2263 | versions as well. | ||
2264 | |||
2265 | @node Reasons to use Postgres | ||
2266 | @subsection Reasons to use Postgres | ||
2267 | |||
2268 | @itemize @bullet | ||
2269 | @item | ||
2270 | Easier to setup than MySQL | ||
2271 | @item | ||
2272 | Real database | ||
2273 | @end itemize | ||
2274 | |||
2275 | @node Reasons not to use Postgres | ||
2276 | @subsection Reasons not to use Postgres | ||
2277 | |||
2278 | @itemize @bullet | ||
2279 | @item | ||
2280 | Quite slow | ||
2281 | @item | ||
2282 | Still some manual setup required | ||
2283 | @end itemize | ||
2284 | |||
2285 | @node Manual setup instructions | ||
2286 | @subsection Manual setup instructions | ||
2287 | |||
2288 | @itemize @bullet | ||
2289 | |||
2290 | @item | ||
2291 | In @code{gnunet.conf} set in section "DATASTORE" the value for@ | ||
2292 | "DATABASE" to "postgres". | ||
2293 | @item | ||
2294 | Access Postgres to create a user:@ | ||
2295 | |||
2296 | @table @asis | ||
2297 | |||
2298 | @item with Postgres 8.x, use: | ||
2299 | |||
2300 | @example | ||
2301 | # su - postgres | ||
2302 | $ createuser | ||
2303 | @end example | ||
2304 | |||
2305 | and enter the name of the user running GNUnet for the role interactively. | ||
2306 | Then, when prompted, do not set it to superuser, allow the creation of | ||
2307 | databases, and do not allow the creation of new roles.@ | ||
2308 | |||
2309 | @item with Postgres 9.x, use: | ||
2310 | |||
2311 | @example | ||
2312 | # su - postgres | ||
2313 | $ createuser -d $GNUNET_USER | ||
2314 | @end example | ||
2315 | |||
2316 | |||
2317 | where $GNUNET_USER is the name of the user running GNUnet.@ | ||
2318 | |||
2319 | @end table | ||
2320 | |||
2321 | |||
2322 | @item | ||
2323 | As that user (so typically as user "gnunet"), create a database (or two):@ | ||
2324 | |||
2325 | @example | ||
2326 | $ createdb gnunet | ||
2327 | $ createdb gnunetcheck # this way you can run "make check" | ||
2328 | @end example | ||
2329 | |||
2330 | @end itemize | ||
2331 | |||
2332 | |||
2333 | Now you should be able to start @code{gnunet-arm}. | ||
2334 | |||
2335 | @node Testing the setup manually | ||
2336 | @subsection Testing the setup manually | ||
2337 | |||
2338 | You may want to try if the database connection works. First, again login as | ||
2339 | the user who will run gnunet-arm. Then use, | ||
2340 | @example | ||
2341 | $ psql gnunet # or gnunetcheck | ||
2342 | gnunet=> \dt | ||
2343 | @end example | ||
2344 | |||
2345 | |||
2346 | If, after you have started gnunet-arm at least once, you get a @code{gn090} | ||
2347 | table here, it probably works. | ||
2348 | |||
2349 | @node Configuring the datacache | ||
2350 | @subsection Configuring the datacache | ||
2351 | @c %**end of header | ||
2352 | |||
2353 | The datacache is what GNUnet uses for storing temporary data. This data is | ||
2354 | expected to be wiped completely each time GNUnet is restarted (or the system | ||
2355 | is rebooted). | ||
2356 | |||
2357 | You need to specify how many bytes GNUnet is allowed to use for the datacache | ||
2358 | using the "QUOTA" option in the section "dhtcache". Furthermore, you need to | ||
2359 | specify which database backend should be used to store the data. Currently, | ||
2360 | you have the choice between sqLite, MySQL and Postgres. | ||
2361 | |||
2362 | @node Configuring the file-sharing service | ||
2363 | @subsection Configuring the file-sharing service | ||
2364 | |||
2365 | In order to use GNUnet for file-sharing, you first need to make sure that the | ||
2366 | file-sharing service is loaded. This is done by setting the AUTOSTART option in | ||
2367 | section "fs" to "YES". Alternatively, you can run | ||
2368 | @example | ||
2369 | $ gnunet-arm -i fs | ||
2370 | @end example | ||
2371 | |||
2372 | to start the file-sharing service by hand. | ||
2373 | |||
2374 | Except for configuring the database and the datacache the only important option | ||
2375 | for file-sharing is content migration. | ||
2376 | |||
2377 | Content migration allows your peer to cache content from other peers as well as | ||
2378 | send out content stored on your system without explicit requests. This content | ||
2379 | replication has positive and negative impacts on both system performance an | ||
2380 | privacy. | ||
2381 | |||
2382 | FIXME: discuss the trade-offs. Here is some older text about it... | ||
2383 | |||
2384 | Setting this option to YES allows gnunetd to migrate data to the local machine. | ||
2385 | Setting this option to YES is highly recommended for efficiency. Its also the | ||
2386 | default. If you set this value to YES, GNUnet will store content on your | ||
2387 | machine that you cannot decrypt. While this may protect you from liability if | ||
2388 | the judge is sane, it may not (IANAL). If you put illegal content on your | ||
2389 | machine yourself, setting this option to YES will probably increase your chances | ||
2390 | to get away with it since you can plausibly deny that you inserted the content. | ||
2391 | Note that in either case, your anonymity would have to be broken first (which | ||
2392 | may be possible depending on the size of the GNUnet network and the strength of | ||
2393 | the adversary). | ||
2394 | |||
2395 | @node Configuring logging | ||
2396 | @subsection Configuring logging | ||
2397 | |||
2398 | Logging in GNUnet 0.9.0 is controlled via the "-L" and "-l" options. | ||
2399 | Using "-L", a log level can be specified. With log level "ERROR" only serious | ||
2400 | errors are logged. The default log level is "WARNING" which causes anything of | ||
2401 | concern to be logged. Log level "INFO" can be used to log anything that might | ||
2402 | be interesting information whereas "DEBUG" can be used by developers to log | ||
2403 | debugging messages (but you need to run configure with | ||
2404 | @code{--enable-logging=verbose} to get them compiled). The "-l" option is used | ||
2405 | to specify the log file. | ||
2406 | |||
2407 | Since most GNUnet services are managed by @code{gnunet-arm}, using the "-l" or | ||
2408 | "-L" options directly is not possible. Instead, they can be specified using the | ||
2409 | "OPTIONS" configuration value in the respective section for the respective | ||
2410 | service. In order to enable logging globally without editing the "OPTIONS" | ||
2411 | values for each service, @code{gnunet-arm} supports a "GLOBAL_POSTFIX" option. | ||
2412 | The value specified here is given as an extra option to all services for which | ||
2413 | the configuration does contain a service-specific "OPTIONS" field. | ||
2414 | |||
2415 | "GLOBAL_POSTFIX" can contain the special sequence "@{@}" which is replaced by | ||
2416 | the name of the service that is being started. Furthermore, | ||
2417 | @code{GLOBAL_POSTFIX} is special in that sequences starting with "$" anywhere | ||
2418 | in the string are expanded (according to options in "PATHS"); this expansion | ||
2419 | otherwise is only happening for filenames and then the "$" must be the first | ||
2420 | character in the option. Both of these restrictions do not apply to | ||
2421 | "GLOBAL_POSTFIX". Note that specifying @code{%} anywhere in the "GLOBAL_POSTFIX" | ||
2422 | disables both of these features. | ||
2423 | |||
2424 | In summary, in order to get all services to log at level "INFO" to log-files | ||
2425 | called @code{SERVICENAME-logs}, the following global prefix should be used: | ||
2426 | @example | ||
2427 | GLOBAL_POSTFIX = -l $SERVICEHOME/@{@}-logs -L INFO | ||
2428 | @end example | ||
2429 | |||
2430 | @node Configuring the transport service and plugins | ||
2431 | @subsection Configuring the transport service and plugins | ||
2432 | |||
2433 | The transport service in GNUnet is responsible to maintain basic connectivity | ||
2434 | to other peers. Besides initiating and keeping connections alive it is also | ||
2435 | responsible for address validation. | ||
2436 | |||
2437 | The GNUnet transport supports more than one transport protocol. These protocols | ||
2438 | are configured together with the transport service. | ||
2439 | |||
2440 | The configuration section for the transport service itself is quite similar to | ||
2441 | all the other services | ||
2442 | |||
2443 | @code{@ | ||
2444 | AUTOSTART = YES@ | ||
2445 | @@UNIXONLY@@ PORT = 2091@ | ||
2446 | HOSTNAME = localhost@ | ||
2447 | HOME = $SERVICEHOME@ | ||
2448 | CONFIG = $DEFAULTCONFIG@ | ||
2449 | BINARY = gnunet-service-transport@ | ||
2450 | #PREFIX = valgrind@ | ||
2451 | NEIGHBOUR_LIMIT = 50@ | ||
2452 | ACCEPT_FROM = 127.0.0.1;@ | ||
2453 | ACCEPT_FROM6 = ::1;@ | ||
2454 | PLUGINS = tcp udp@ | ||
2455 | UNIXPATH = /tmp/gnunet-service-transport.sock@ | ||
2456 | } | ||
2457 | |||
2458 | Different are the settings for the plugins to load @code{PLUGINS}. The first | ||
2459 | setting specifies which transport plugins to load. | ||
2460 | @itemize @bullet | ||
2461 | |||
2462 | |||
2463 | @item | ||
2464 | transport-unix | ||
2465 | |||
2466 | A plugin for local only communication with UNIX domain sockets. Used for | ||
2467 | testing and available on unix systems only. Just set the port | ||
2468 | |||
2469 | @code{@ | ||
2470 | [transport-unix]@ | ||
2471 | PORT = 22086@ | ||
2472 | TESTING_IGNORE_KEYS = ACCEPT_FROM;@ | ||
2473 | } | ||
2474 | |||
2475 | @item | ||
2476 | transport-tcp | ||
2477 | |||
2478 | A plugin for communication with TCP. Set port to 0 for client mode with | ||
2479 | outbound only connections | ||
2480 | |||
2481 | @code{@ | ||
2482 | [transport-tcp]@ | ||
2483 | # Use 0 to ONLY advertise as a peer behind NAT (no port binding)@ | ||
2484 | PORT = 2086@ | ||
2485 | ADVERTISED_PORT = 2086@ | ||
2486 | TESTING_IGNORE_KEYS = ACCEPT_FROM;@ | ||
2487 | # Maximum number of open TCP connections allowed@ | ||
2488 | MAX_CONNECTIONS = 128@ | ||
2489 | } | ||
2490 | |||
2491 | @item | ||
2492 | transport-udp | ||
2493 | |||
2494 | A plugin for communication with UDP. Supports peer discovery using broadcasts.@ | ||
2495 | @code{@ | ||
2496 | [transport-udp]@ | ||
2497 | PORT = 2086@ | ||
2498 | BROADCAST = YES@ | ||
2499 | BROADCAST_INTERVAL = 30 s@ | ||
2500 | MAX_BPS = 1000000@ | ||
2501 | TESTING_IGNORE_KEYS = ACCEPT_FROM;@ | ||
2502 | } | ||
2503 | |||
2504 | @item | ||
2505 | transport-http | ||
2506 | |||
2507 | HTTP and HTTPS support is split in two part: a client plugin initiating | ||
2508 | outbound connections and a server part accepting connections from the client. | ||
2509 | The client plugin just takes the maximum number of connections as an argument.@ | ||
2510 | @code{@ | ||
2511 | [transport-http_client]@ | ||
2512 | MAX_CONNECTIONS = 128@ | ||
2513 | TESTING_IGNORE_KEYS = ACCEPT_FROM;@ | ||
2514 | }@ | ||
2515 | @code{@ | ||
2516 | [transport-https_client]@ | ||
2517 | MAX_CONNECTIONS = 128@ | ||
2518 | TESTING_IGNORE_KEYS = ACCEPT_FROM;@ | ||
2519 | } | ||
2520 | |||
2521 | The server has a port configured and the maximum nunber of connections.@ | ||
2522 | The HTTPS part has two files with the certificate key and the certificate file. | ||
2523 | |||
2524 | The server plugin supports reverse proxies, so a external hostname can be set | ||
2525 | using@ | ||
2526 | the @code{EXTERNAL_HOSTNAME} setting. The webserver under this address should | ||
2527 | forward the request to the peer and the configure port. | ||
2528 | |||
2529 | @code{@ | ||
2530 | [transport-http_server]@ | ||
2531 | EXTERNAL_HOSTNAME = fulcrum.net.in.tum.de/gnunet@ | ||
2532 | PORT = 1080@ | ||
2533 | MAX_CONNECTIONS = 128@ | ||
2534 | TESTING_IGNORE_KEYS = ACCEPT_FROM;@ | ||
2535 | }@ | ||
2536 | @code{@ | ||
2537 | [transport-https_server]@ | ||
2538 | PORT = 4433@ | ||
2539 | CRYPTO_INIT = NORMAL@ | ||
2540 | KEY_FILE = https.key@ | ||
2541 | CERT_FILE = https.cert@ | ||
2542 | MAX_CONNECTIONS = 128@ | ||
2543 | TESTING_IGNORE_KEYS = ACCEPT_FROM;@ | ||
2544 | } | ||
2545 | |||
2546 | @item | ||
2547 | transport-wlan | ||
2548 | |||
2549 | There is a special article how to setup the WLAN plugin, so here only the | ||
2550 | settings. Just specify the interface to use:@ | ||
2551 | @code{@ | ||
2552 | [transport-wlan]@ | ||
2553 | # Name of the interface in monitor mode (typically monX)@ | ||
2554 | INTERFACE = mon0@ | ||
2555 | # Real hardware, no testing@ | ||
2556 | TESTMODE = 0@ | ||
2557 | TESTING_IGNORE_KEYS = ACCEPT_FROM;@ | ||
2558 | } | ||
2559 | @end itemize | ||
2560 | |||
2561 | @node Configuring the wlan transport plugin | ||
2562 | @subsection Configuring the wlan transport plugin | ||
2563 | |||
2564 | |||
2565 | The wlan transport plugin enables GNUnet to send and to receive data on a wlan | ||
2566 | interface. It has not to be connected to a wlan network as long as sender and | ||
2567 | receiver are on the same channel. This enables you to get connection to the | ||
2568 | GNUnet where no internet access is possible, for example while catastrophes or | ||
2569 | when censorship cuts you off the internet. | ||
2570 | |||
2571 | |||
2572 | @menu | ||
2573 | * Requirements for the WLAN plugin:: | ||
2574 | * Configuration:: | ||
2575 | * Before starting GNUnet:: | ||
2576 | * Limitations and known bugs:: | ||
2577 | @end menu | ||
2578 | |||
2579 | |||
2580 | @node Requirements for the WLAN plugin | ||
2581 | @subsubsection Requirements for the WLAN plugin | ||
2582 | |||
2583 | @itemize @bullet | ||
2584 | |||
2585 | @item | ||
2586 | wlan network card with monitor support and packet injection | ||
2587 | (see @uref{http://www.aircrack-ng.org/, aircrack-ng.org}) | ||
2588 | |||
2589 | @item | ||
2590 | Linux kernel with mac80211 stack, introduced in 2.6.22, tested with 2.6.35 | ||
2591 | and 2.6.38 | ||
2592 | |||
2593 | @item | ||
2594 | Wlantools to create the a monitor interface, tested with airmon-ng of the | ||
2595 | aircrack-ng package | ||
2596 | @end itemize | ||
2597 | |||
2598 | @node Configuration | ||
2599 | @subsubsection Configuration | ||
2600 | |||
2601 | There are the following options for the wlan plugin (they should be like this | ||
2602 | in your default config file, you only need to adjust them if the values are | ||
2603 | incorrect for your system)@ | ||
2604 | @code{@ | ||
2605 | # section for the wlan transport plugin@ | ||
2606 | [transport-wlan]@ | ||
2607 | # interface to use, more information in the | ||
2608 | # "Before starting GNUnet" section of the handbook. | ||
2609 | INTERFACE = mon0@ | ||
2610 | # testmode for developers:@ | ||
2611 | # 0 use wlan interface,@ | ||
2612 | #1 or 2 use loopback driver for tests 1 = server, 2 = client@ | ||
2613 | TESTMODE = 0@ | ||
2614 | } | ||
2615 | |||
2616 | @node Before starting GNUnet | ||
2617 | @subsubsection Before starting GNUnet | ||
2618 | |||
2619 | Before starting GNUnet, you have to make sure that your wlan interface is in | ||
2620 | monitor mode. One way to put the wlan interface into monitor mode (if your | ||
2621 | interface name is wlan0) is by executing:@ | ||
2622 | @code{@ | ||
2623 | sudo airmon-ng start wlan0@ | ||
2624 | } | ||
2625 | |||
2626 | Here is an example what the result should look like:@ | ||
2627 | @code{@ | ||
2628 | Interface Chipset Driver@ | ||
2629 | wlan0 Intel 4965 a/b/g/n iwl4965 - [phy0]@ | ||
2630 | (monitor mode enabled on mon0)@ | ||
2631 | }@ | ||
2632 | The monitor interface is mon0 is the one that you have to put into the | ||
2633 | configuration file. | ||
2634 | |||
2635 | @node Limitations and known bugs | ||
2636 | @subsubsection Limitations and known bugs | ||
2637 | |||
2638 | Wlan speed is at the maximum of 1 Mbit/s because support for choosing the wlan | ||
2639 | speed with packet injection was removed in newer kernels. Please pester the | ||
2640 | kernel developers about fixing this. | ||
2641 | |||
2642 | The interface channel depends on the wlan network that the card is connected | ||
2643 | to. If no connection has been made since the start of the computer, it is | ||
2644 | usually the first channel of the card. Peers will only find each other and | ||
2645 | communicate if they are on the same channel. Channels must be set manually | ||
2646 | (i.e. using @code{iwconfig wlan0 channel 1}). | ||
2647 | |||
2648 | |||
2649 | @node Configuring HTTP(S) reverse proxy functionality using Apache or nginx | ||
2650 | @subsection Configuring HTTP(S) reverse proxy functionality using Apache or nginx | ||
2651 | |||
2652 | The HTTP plugin supports data transfer using reverse proxies. A reverse proxy | ||
2653 | forwards the HTTP request he receives with a certain URL to another webserver, | ||
2654 | here a GNUnet peer. | ||
2655 | |||
2656 | So if you have a running Apache or nginx webserver you can configure it to be a | ||
2657 | GNUnet reverse proxy. Especially if you have a well-known webiste this improves | ||
2658 | censorship resistance since it looks as normal surfing behaviour. | ||
2659 | |||
2660 | To do so, you have to do two things: | ||
2661 | |||
2662 | @itemize @bullet | ||
2663 | |||
2664 | @item | ||
2665 | Configure your webserver to forward the GNUnet HTTP traffic | ||
2666 | |||
2667 | @item | ||
2668 | Configure your GNUnet peer to announce the respective address | ||
2669 | @end itemize | ||
2670 | |||
2671 | As an example we want to use GNUnet peer running: | ||
2672 | |||
2673 | @itemize @bullet | ||
2674 | |||
2675 | @item | ||
2676 | HTTP server plugin on @code{gnunet.foo.org:1080} | ||
2677 | |||
2678 | @item | ||
2679 | HTTPS server plugin on @code{gnunet.foo.org:4433} | ||
2680 | |||
2681 | @item | ||
2682 | A apache or nginx webserver on @uref{http://www.foo.org/, http://www.foo.org:80/} | ||
2683 | |||
2684 | @item | ||
2685 | A apache or nginx webserver on https://www.foo.org:443/ | ||
2686 | @end itemize | ||
2687 | |||
2688 | And we want the webserver to accept GNUnet traffic under | ||
2689 | @code{http://www.foo.org/bar/}. The required steps are described here: | ||
2690 | |||
2691 | @strong{Configure your Apache2 HTTP webserver} | ||
2692 | |||
2693 | First of all you need mod_proxy installed. | ||
2694 | |||
2695 | Edit your webserver configuration. Edit @code{/etc/apache2/apache2.conf} or | ||
2696 | the site-specific configuration file. | ||
2697 | |||
2698 | In the respective @code{server config},@code{virtual host} or | ||
2699 | @code{directory} section add the following lines:@ | ||
2700 | @code{@ | ||
2701 | ProxyTimeout 300@ | ||
2702 | ProxyRequests Off@ | ||
2703 | <Location /bar/ >@ | ||
2704 | ProxyPass http://gnunet.foo.org:1080/@ | ||
2705 | ProxyPassReverse http://gnunet.foo.org:1080/@ | ||
2706 | </Location>@ | ||
2707 | } | ||
2708 | |||
2709 | @strong{Configure your Apache2 HTTPS webserver} | ||
2710 | |||
2711 | We assume that you already have an HTTPS server running, if not please check | ||
2712 | how to configure a HTTPS host. An easy to use example is the | ||
2713 | @file{apache2/sites-available/default-ssl} example configuration file. | ||
2714 | |||
2715 | In the respective HTTPS @code{server config},@code{virtual host} or | ||
2716 | @code{directory} section add the following lines:@ | ||
2717 | @code{@ | ||
2718 | SSLProxyEngine On@ | ||
2719 | ProxyTimeout 300@ | ||
2720 | ProxyRequests Off@ | ||
2721 | <Location /bar/ >@ | ||
2722 | ProxyPass https://gnunet.foo.org:4433/@ | ||
2723 | ProxyPassReverse https://gnunet.foo.org:4433/@ | ||
2724 | </Location>@ | ||
2725 | } | ||
2726 | |||
2727 | More information about the apache mod_proxy configuration can be found unter:@ | ||
2728 | @uref{http://httpd.apache.org/docs/2.2/mod/mod_proxy.html#proxypass, http://httpd.apache.org/docs/2.2/mod/mod_proxy.html#proxypass} | ||
2729 | |||
2730 | @strong{Configure your nginx HTTPS webserver} | ||
2731 | |||
2732 | Since nginx does not support chunked encoding, you first of all have to | ||
2733 | install @code{chunkin}:@ | ||
2734 | @uref{http://wiki.nginx.org/HttpChunkinModule, http://wiki.nginx.org/HttpChunkinModule} | ||
2735 | |||
2736 | To enable chunkin add:@ | ||
2737 | @code{@ | ||
2738 | chunkin on;@ | ||
2739 | error_page 411 = @@my_411_error;@ | ||
2740 | location @@my_411_error @{@ | ||
2741 | chunkin_resume;@ | ||
2742 | @}@ | ||
2743 | } | ||
2744 | |||
2745 | Edit your webserver configuration. Edit @code{/etc/nginx/nginx.conf} or the | ||
2746 | site-specific configuration file. | ||
2747 | |||
2748 | In the @code{server} section add:@ | ||
2749 | @code{@ | ||
2750 | location /bar/@ | ||
2751 | @{@ | ||
2752 | proxy_pass http://gnunet.foo.org:1080/;@ | ||
2753 | proxy_buffering off;@ | ||
2754 | proxy_connect_timeout 5; # more than http_server@ | ||
2755 | proxy_read_timeout 350; # 60 default, 300s is GNUnet's idle timeout@ | ||
2756 | proxy_http_version 1.1; # 1.0 default@ | ||
2757 | proxy_next_upstream error timeout invalid_header http_500 http_503 http_502 http_504;@ | ||
2758 | @}@ | ||
2759 | @code{}} | ||
2760 | |||
2761 | @strong{Configure your nginx HTTPS webserver} | ||
2762 | |||
2763 | Edit your webserver configuration. Edit @code{/etc/nginx/nginx.conf} or the | ||
2764 | site-specific configuration file. | ||
2765 | |||
2766 | In the @code{server} section add:@ | ||
2767 | @code{@ | ||
2768 | ssl_session_timeout 6m;@ | ||
2769 | location /bar/@ | ||
2770 | @{@ | ||
2771 | proxy_pass https://gnunet.foo.org:4433/;@ | ||
2772 | proxy_buffering off;@ | ||
2773 | proxy_connect_timeout 5; # more than http_server@ | ||
2774 | proxy_read_timeout 350; # 60 default, 300s is GNUnet's idle timeout@ | ||
2775 | proxy_http_version 1.1; # 1.0 default@ | ||
2776 | proxy_next_upstream error timeout invalid_header http_500 http_503 http_502 http_504;@ | ||
2777 | @}@ | ||
2778 | @code{}} | ||
2779 | |||
2780 | @strong{Configure your GNUnet peer} | ||
2781 | |||
2782 | To have your GNUnet peer announce the address, you have to specify the | ||
2783 | |||
2784 | @code{EXTERNAL_HOSTNAME} option in the @code{[transport-http_server]} section:@ | ||
2785 | @code{@ | ||
2786 | [transport-http_server]@ | ||
2787 | EXTERNAL_HOSTNAME = http://www.foo.org/bar/@ | ||
2788 | }@ | ||
2789 | and/or@ | ||
2790 | @code{[transport-https_server]} section:@ | ||
2791 | @code{@ | ||
2792 | [transport-https_server]@ | ||
2793 | EXTERNAL_HOSTNAME = https://www.foo.org/bar/@ | ||
2794 | } | ||
2795 | |||
2796 | Now restart your webserver and your peer... | ||
2797 | |||
2798 | @node Blacklisting peers | ||
2799 | @subsection Blacklisting peers | ||
2800 | |||
2801 | Transport service supports to deny connecting to a specific peer of to a | ||
2802 | specific peer with a specific transport plugin using te blacklisting component | ||
2803 | of transport service. With@ blacklisting it is possible to deny connections to | ||
2804 | specific peers of@ to use a specific plugin to a specific peer. Peers can be | ||
2805 | blacklisted using@ the configuration or a blacklist client can be asked. | ||
2806 | |||
2807 | To blacklist peers using the configuration you have to add a section to your@ | ||
2808 | configuration containing the peer id of the peer to blacklist and the plugin@ | ||
2809 | if required. | ||
2810 | |||
2811 | Example:@ | ||
2812 | To blacklist connections to P565... on peer AG2P... using tcp add:@ | ||
2813 | @code{@ | ||
2814 | [transport-blacklist AG2PHES1BARB9IJCPAMJTFPVJ5V3A72S3F2A8SBUB8DAQ2V0O3V8G6G2JU56FHGFOHMQVKBSQFV98TCGTC3RJ1NINP82G0RC00N1520]@ | ||
2815 | P565723JO1C2HSN6J29TAQ22MN6CI8HTMUU55T0FUQG4CMDGGEQ8UCNBKUMB94GC8R9G4FB2SF9LDOBAJ6AMINBP4JHHDD6L7VD801G = tcp@ | ||
2816 | }@ | ||
2817 | To blacklist connections to P565... on peer AG2P... using all plugins add:@ | ||
2818 | @code{@ | ||
2819 | [transport-blacklist-AG2PHES1BARB9IJCPAMJTFPVJ5V3A72S3F2A8SBUB8DAQ2V0O3V8G6G2JU56FHGFOHMQVKBSQFV98TCGTC3RJ1NINP82G0RC00N1520]@ | ||
2820 | P565723JO1C2HSN6J29TAQ22MN6CI8HTMUU55T0FUQG4CMDGGEQ8UCNBKUMB94GC8R9G4FB2SF9LDOBAJ6AMINBP4JHHDD6L7VD801G =@ | ||
2821 | } | ||
2822 | |||
2823 | You can also add a blacklist client usign the blacklist api. On a blacklist@ | ||
2824 | check, blacklisting first checks internally if the peer is blacklisted and@ | ||
2825 | if not, it asks the blacklisting clients. Clients are asked if it is OK to@ | ||
2826 | connect to a peer ID, the plugin is omitted. | ||
2827 | |||
2828 | On blacklist check for (peer, plugin) | ||
2829 | @itemize @bullet | ||
2830 | @item Do we have a local blacklist entry for this peer and this plugin?@ | ||
2831 | @item YES: disallow connection@ | ||
2832 | @item Do we have a local blacklist entry for this peer and all plugins?@ | ||
2833 | @item YES: disallow connection@ | ||
2834 | @item Does one of the clients disallow?@ | ||
2835 | @item YES: disallow connection | ||
2836 | @end itemize | ||
2837 | |||
2838 | @node Configuration of the HTTP and HTTPS transport plugins | ||
2839 | @subsection Configuration of the HTTP and HTTPS transport plugins | ||
2840 | |||
2841 | The client part of the http and https transport plugins can be configured to | ||
2842 | use a proxy to connect to the hostlist server. This functionality can be | ||
2843 | configured in the configuration file directly or using the gnunet-setup tool. | ||
2844 | |||
2845 | The both the HTTP and HTTPS clients support the following proxy types at the | ||
2846 | moment: | ||
2847 | |||
2848 | @itemize @bullet | ||
2849 | @item HTTP 1.1 proxy | ||
2850 | @item SOCKS 4/4a/5/5 with hostname | ||
2851 | @end itemize | ||
2852 | |||
2853 | In addition authentication at the proxy with username and password can be | ||
2854 | configured. | ||
2855 | |||
2856 | To configure proxy support for the clients in the gnunet-setup tool, select the | ||
2857 | "transport" tab and activate the respective plugin. Now you can select the | ||
2858 | appropriate proxy type. The hostname or IP address (including port if required) | ||
2859 | has to be entered in the "Proxy hostname" textbox. If required, enter username | ||
2860 | and password in the "Proxy username" and "Proxy password" boxes. Be aware that | ||
2861 | these information will be stored in the configuration in plain text. | ||
2862 | |||
2863 | To configure these options directly in the configuration, you can configure the | ||
2864 | following settings in the [transport-http_client] and [transport-https_client] | ||
2865 | section of the configuration: | ||
2866 | |||
2867 | @example | ||
2868 | # Type of proxy server,@ | ||
2869 | # Valid values: HTTP, SOCKS4, SOCKS5, SOCKS4A, SOCKS5_HOSTNAME@ | ||
2870 | # Default: HTTP@ | ||
2871 | # PROXY_TYPE = HTTP | ||
2872 | |||
2873 | # Hostname or IP of proxy server@ | ||
2874 | # PROXY =@ | ||
2875 | # User name for proxy server@ | ||
2876 | # PROXY_USERNAME =@ | ||
2877 | # User password for proxy server@ | ||
2878 | # PROXY_PASSWORD = | ||
2879 | @end example | ||
2880 | |||
2881 | @node Configuring the GNU Name System | ||
2882 | @subsection Configuring the GNU Name System | ||
2883 | |||
2884 | @menu | ||
2885 | * Configuring system-wide DNS interception:: | ||
2886 | * Configuring the GNS nsswitch plugin:: | ||
2887 | * Configuring GNS on W32:: | ||
2888 | * GNS Proxy Setup:: | ||
2889 | * Setup of the GNS CA:: | ||
2890 | * Testing the GNS setup:: | ||
2891 | * Automatic Shortening in the GNU Name System:: | ||
2892 | @end menu | ||
2893 | |||
2894 | |||
2895 | @node Configuring system-wide DNS interception | ||
2896 | @subsubsection Configuring system-wide DNS interception | ||
2897 | |||
2898 | Before you install GNUnet, make sure you have a user and group 'gnunet' as well | ||
2899 | as an empty group 'gnunetdns'. | ||
2900 | |||
2901 | When using GNUnet with system-wide DNS interception, it is absolutely necessary | ||
2902 | for all GNUnet service processes to be started by @code{gnunet-service-arm} as | ||
2903 | user and group 'gnunet'. You also need to be sure to run @code{make install} as | ||
2904 | root (or use the @code{sudo} option to configure) to grant GNUnet sufficient | ||
2905 | privileges. | ||
2906 | |||
2907 | With this setup, all that is required for enabling system-wide DNS interception | ||
2908 | is for some GNUnet component (VPN or GNS) to request it. The | ||
2909 | @code{gnunet-service-dns} will then start helper programs that will make the | ||
2910 | necessary changes to your firewall (@code{iptables}) rules. | ||
2911 | |||
2912 | Note that this will NOT work if your system sends out DNS traffic to a | ||
2913 | link-local IPv6 address, as in this case GNUnet can intercept the traffic, but | ||
2914 | not inject the responses from the link-local IPv6 address. Hence you cannot use | ||
2915 | system-wide DNS interception in conjunction with link-local IPv6-based DNS | ||
2916 | servers. If such a DNS server is used, it will bypass GNUnet's DNS traffic | ||
2917 | interception. | ||
2918 | |||
2919 | |||
2920 | |||
2921 | Using the GNU Name System (GNS) requires two different configuration steps. | ||
2922 | First of all, GNS needs to be integrated with the operating system. Most of | ||
2923 | this section is about the operating system level integration. | ||
2924 | |||
2925 | Additionally, each individual user who wants to use the system must also | ||
2926 | initialize his GNS zones. This can be done by running (after starting GNUnet)@ | ||
2927 | @code{@ | ||
2928 | $ gnunet-gns-import.sh@ | ||
2929 | }@ | ||
2930 | after the local GNUnet peer has been started. Note that the namestore (in | ||
2931 | particular the namestore database backend) should not be reconfigured | ||
2932 | afterwards (as records are not automatically migrated between backends). | ||
2933 | |||
2934 | The remainder of this chapter will detail the various methods for configuring | ||
2935 | the use of GNS with your operating system. | ||
2936 | |||
2937 | At this point in time you have different options depending on your OS: | ||
2938 | @table @asis | ||
2939 | |||
2940 | @item Use the gnunet-gns-proxy This approach works for all operating systems | ||
2941 | and is likely the easiest. However, it enables GNS only for browsers, not for | ||
2942 | other applications that might be using DNS, such as SSH. Still, using the proxy | ||
2943 | is required for using HTTP with GNS and is thus recommended for all users. To | ||
2944 | do this, you simply have to run the @code{gnunet-gns-proxy-setup-ca} script as | ||
2945 | the user who will run the browser (this will create a GNS certificate authority | ||
2946 | (CA) on your system and import its key into your browser), then start | ||
2947 | @code{gnunet-gns-proxy} and inform your browser to use the Socks5 proxy which | ||
2948 | @code{gnunet-gns-proxy} makes available by default on port 7777. | ||
2949 | @item Use a | ||
2950 | nsswitch plugin (recommended on GNU systems) This approach has the advantage of | ||
2951 | offering fully personalized resolution even on multi-user systems. A potential | ||
2952 | disadvantage is that some applications might be able to bypass GNS. | ||
2953 | @item Use | ||
2954 | a W32 resolver plugin (recommended on W32) This is currently the only option on | ||
2955 | W32 systems. | ||
2956 | @item Use system-wide DNS packet interception This approach is | ||
2957 | recommended for the GNUnet VPN. It can be used to handle GNS at the same time; | ||
2958 | however, if you only use this method, you will only get one root zone per | ||
2959 | machine (not so great for multi-user systems). | ||
2960 | @end table | ||
2961 | |||
2962 | |||
2963 | You can combine system-wide DNS packet interception with the nsswitch plugin.@ | ||
2964 | The setup of the system-wide DNS interception is described here. All of the | ||
2965 | other GNS-specific configuration steps are described in the following sections. | ||
2966 | |||
2967 | @node Configuring the GNS nsswitch plugin | ||
2968 | @subsubsection Configuring the GNS nsswitch plugin | ||
2969 | |||
2970 | The Name Service Switch (NSS) is a facility in Unix-like operating systems that | ||
2971 | provides a variety of sources for common configuration databases and name | ||
2972 | resolution mechanisms. A system administrator usually configures the operating | ||
2973 | system's name services using the file /etc/nsswitch.conf. | ||
2974 | |||
2975 | GNS provides a NSS plugin to integrate GNS name resolution with the operating | ||
2976 | system's name resolution process. To use the GNS NSS plugin you have to either | ||
2977 | |||
2978 | @itemize @bullet | ||
2979 | |||
2980 | @item | ||
2981 | install GNUnet as root or | ||
2982 | |||
2983 | @item | ||
2984 | compile GNUnet with the @code{--with-sudo=yes} switch. | ||
2985 | @end itemize | ||
2986 | |||
2987 | Name resolution is controlled by the @emph{hosts} section in the NSS | ||
2988 | configuration. By default this section first performs a lookup in the | ||
2989 | /etc/hosts file and then in DNS. The nsswitch file should contain a line | ||
2990 | similar to:@ | ||
2991 | @code{@ | ||
2992 | hosts: files dns [NOTFOUND=return] mdns4_minimal mdns4@ | ||
2993 | } | ||
2994 | |||
2995 | Here the GNS NSS plugin can be added to perform a GNS lookup before performing | ||
2996 | a DNS lookup. The GNS NSS plugin has to be added to the "hosts" section in | ||
2997 | /etc/nsswitch.conf file before DNS related plugins:@ | ||
2998 | @code{@ | ||
2999 | ...@ | ||
3000 | hosts: files gns [NOTFOUND=return] dns mdns4_minimal mdns4@ | ||
3001 | ...@ | ||
3002 | } | ||
3003 | |||
3004 | The @code{NOTFOUND=return} will ensure that if a @code{.gnu} name is not found | ||
3005 | in GNS it will not be queried in DNS. | ||
3006 | |||
3007 | @node Configuring GNS on W32 | ||
3008 | @subsubsection Configuring GNS on W32 | ||
3009 | |||
3010 | This document is a guide to configuring GNU Name System on W32-compatible | ||
3011 | platforms. | ||
3012 | |||
3013 | After GNUnet is installed, run the w32nsp-install tool: | ||
3014 | @example | ||
3015 | w32nsp-install.exe libw32nsp-0.dll | ||
3016 | @end example | ||
3017 | |||
3018 | |||
3019 | ('0' is the library version of W32 NSP; it might increase in the future, | ||
3020 | change the invocation accordingly). | ||
3021 | |||
3022 | This will install GNS namespace provider into the system and allow other | ||
3023 | applications to resolve names that end in '@strong{gnu}' and '@strong{zkey}'. | ||
3024 | Note that namespace provider requires gnunet-gns-helper-service-w32 to be | ||
3025 | running, as well as gns service itself (and its usual dependencies). | ||
3026 | |||
3027 | Namespace provider is hardcoded to connect to @strong{127.0.0.1:5353}, and this | ||
3028 | is where gnunet-gns-helper-service-w32 should be listening to (and is | ||
3029 | configured to listen to by default). | ||
3030 | |||
3031 | To uninstall the provider, run: | ||
3032 | @example | ||
3033 | w32nsp-uninstall.exe | ||
3034 | @end example | ||
3035 | |||
3036 | |||
3037 | (uses provider GUID to uninstall it, does not need a dll name). | ||
3038 | |||
3039 | Note that while MSDN claims that other applications will only be able to use | ||
3040 | the new namespace provider after re-starting, in reality they might stat to use | ||
3041 | it without that. Conversely, they might stop using the provider after it's been | ||
3042 | uninstalled, even if they were not re-started. W32 will not permit namespace | ||
3043 | provider library to be deleted or overwritten while the provider is installed, | ||
3044 | and while there is at least one process still using it (even after it was | ||
3045 | uninstalled). | ||
3046 | |||
3047 | @node GNS Proxy Setup | ||
3048 | @subsubsection GNS Proxy Setup | ||
3049 | |||
3050 | When using the GNU Name System (GNS) to browse the WWW, there are several | ||
3051 | issues that can be solved by adding the GNS Proxy to your setup: | ||
3052 | @itemize @bullet | ||
3053 | |||
3054 | |||
3055 | @item If the target website does not support GNS, it might assume that it is | ||
3056 | operating under some name in the legacy DNS system (such as example.com). It | ||
3057 | may then attempt to set cookies for that domain, and the web server might | ||
3058 | expect a @code{Host: example.com} header in the request from your browser. | ||
3059 | However, your browser might be using @code{example.gnu} for the @code{Host} | ||
3060 | header and might only accept (and send) cookies for @code{example.gnu}. The GNS | ||
3061 | Proxy will perform the necessary translations of the hostnames for cookies and | ||
3062 | HTTP headers (using the LEHO record for the target domain as the desired | ||
3063 | substitute). | ||
3064 | |||
3065 | @item If using HTTPS, the target site might include an SSL certificate which is | ||
3066 | either only valid for the LEHO domain or might match a TLSA record in GNS. | ||
3067 | However, your browser would expect a valid certificate for @code{example.gnu}, | ||
3068 | not for some legacy domain name. The proxy will validate the certificate | ||
3069 | (either against LEHO or TLSA) and then on-the-fly produce a valid certificate | ||
3070 | for the exchange, signed by your own CA. Assuming you installed the CA of your | ||
3071 | proxy in your browser's certificate authority list, your browser will then | ||
3072 | trust the HTTPS/SSL/TLS connection, as the hostname mismatch is hidden by the | ||
3073 | proxy. | ||
3074 | |||
3075 | @item Finally, the proxy will in the future indicate to the server that it | ||
3076 | speaks GNS, which will enable server operators to deliver GNS-enabled web sites | ||
3077 | to your browser (and continue to deliver legacy links to legacy browsers) | ||
3078 | @end itemize | ||
3079 | |||
3080 | @node Setup of the GNS CA | ||
3081 | @subsubsection Setup of the GNS CA | ||
3082 | |||
3083 | First you need to create a CA certificate that the proxy can use. To do so use | ||
3084 | the provided script gnunet-gns-proxy-ca:@ | ||
3085 | @code{@ | ||
3086 | $ gnunet-gns-proxy-setup-ca@ | ||
3087 | } | ||
3088 | |||
3089 | This will create a personal certification authority for you and add this | ||
3090 | authority to the firefox and chrome database. The proxy will use the this CA | ||
3091 | certificate to generate @code{*.gnu} client certificates on the fly. | ||
3092 | |||
3093 | Note that the proxy uses libcurl. Make sure your version of libcurl uses GnuTLS | ||
3094 | and NOT OpenSSL. The proxy will not work with libcurl compiled against | ||
3095 | OpenSSL. | ||
3096 | |||
3097 | @node Testing the GNS setup | ||
3098 | @subsubsection Testing the GNS setup | ||
3099 | |||
3100 | Now for testing purposes we can create some records in our zone to test the SSL | ||
3101 | functionality of the proxy:@ | ||
3102 | @code{@ | ||
3103 | $ gnunet-namestore -a -e "1 d" -n "homepage" -t A -V 131.159.74.67@ | ||
3104 | $ gnunet-namestore -a -e "1 d" -n "homepage" -t LEHO -V "gnunet.org"@ | ||
3105 | } | ||
3106 | |||
3107 | At this point we can start the proxy. Simply execute@ | ||
3108 | @code{@ | ||
3109 | $ gnunet-gns-proxy@ | ||
3110 | } | ||
3111 | |||
3112 | Configure your browser to use this SOCKSv5 proxy on port 7777 and visit this | ||
3113 | link.@ If you use firefox you also have to go to about:config and set the key | ||
3114 | @code{network.proxy.socks_remote_dns} to @code{true}. | ||
3115 | |||
3116 | When you visit @code{https://homepage.gnu/}, you should get to the | ||
3117 | @code{https://gnunet.org/} frontpage and the browser (with the correctly | ||
3118 | configured proxy) should give you a valid SSL certificate for | ||
3119 | @code{homepage.gnu} and no warnings. It should look like this@ | ||
3120 | |||
3121 | |||
3122 | |||
3123 | @table @asis | ||
3124 | @item Attachment | ||
3125 | Size | ||
3126 | @item gnunethpgns.png | ||
3127 | 64.19 KB | ||
3128 | @end table | ||
3129 | |||
3130 | @node Automatic Shortening in the GNU Name System | ||
3131 | @subsubsection Automatic Shortening in the GNU Name System | ||
3132 | |||
3133 | This page describes a possible option for 'automatic name shortening', which | ||
3134 | you can choose to enable with the GNU Name System. | ||
3135 | |||
3136 | When GNS encounters a name for the first time, it can use the 'NICK' record of | ||
3137 | the originating zone to automatically generate a name for the zone. If | ||
3138 | automatic shortening is enabled, those auto-generated names will be placed (as | ||
3139 | private records) into your personal 'shorten' zone (to prevent confusion with | ||
3140 | manually selected names). Then, in the future, if the same name is encountered | ||
3141 | again, GNS will display the shortened name instead (the first time, the long | ||
3142 | name will still be used as shortening typically happens asynchronously as | ||
3143 | looking up the 'NICK' record takes some time). Using this feature can be a | ||
3144 | convenient way to avoid very long @code{.gnu} names; however, note that names | ||
3145 | from the shorten-zone are assigned on a first-come-first-serve basis and should | ||
3146 | not be trusted. Furthermore, if you enable this feature, you will no longer see | ||
3147 | the full delegation chain for zones once shortening has been applied. | ||
3148 | |||
3149 | @node Configuring the GNUnet VPN | ||
3150 | @subsection Configuring the GNUnet VPN | ||
3151 | |||
3152 | @menu | ||
3153 | * IPv4 address for interface:: | ||
3154 | * IPv6 address for interface:: | ||
3155 | * Configuring the GNUnet VPN DNS:: | ||
3156 | * Configuring the GNUnet VPN Exit Service:: | ||
3157 | * IP Address of external DNS resolver:: | ||
3158 | * IPv4 address for Exit interface:: | ||
3159 | * IPv6 address for Exit interface:: | ||
3160 | @end menu | ||
3161 | |||
3162 | Before configuring the GNUnet VPN, please make sure that system-wide DNS | ||
3163 | interception is configured properly as described in the section on the GNUnet | ||
3164 | DNS setup. | ||
3165 | |||
3166 | The default-options for the GNUnet VPN are usually sufficient to use GNUnet as | ||
3167 | a Layer 2 for your Internet connection. However, what you always have to | ||
3168 | specify is which IP protocol you want to tunnel: IPv4, IPv6 or both. | ||
3169 | Furthermore, if you tunnel both, you most likely should also tunnel all of your | ||
3170 | DNS requests. You theoretically can tunnel "only" your DNS traffic, but that | ||
3171 | usually makes little sense. | ||
3172 | |||
3173 | The other options as shown on the gnunet-setup tool are: | ||
3174 | |||
3175 | @node IPv4 address for interface | ||
3176 | @subsubsection IPv4 address for interface | ||
3177 | |||
3178 | This is the IPv4 address the VPN interface will get. You should pick an | ||
3179 | 'private' IPv4 network that is not yet in use for you system. For example, if | ||
3180 | you use 10.0.0.1/255.255.0.0 already, you might use 10.1.0.1/255.255.0.0. If | ||
3181 | you use 10.0.0.1/255.0.0.0 already, then you might use 192.168.0.1/255.255.0.0. | ||
3182 | If your system is not in a private IP-network, using any of the above will work | ||
3183 | fine.@ You should try to make the mask of the address big enough (255.255.0.0 | ||
3184 | or, even better, 255.0.0.0) to allow more mappings of remote IP Addresses into | ||
3185 | this range. However, even a 255.255.255.0-mask will suffice for most users. | ||
3186 | |||
3187 | @node IPv6 address for interface | ||
3188 | @subsubsection IPv6 address for interface | ||
3189 | |||
3190 | The IPv6 address the VPN interface will get. Here you can specify any | ||
3191 | non-link-local address (the address should not begin with "fe80:"). A subnet | ||
3192 | Unique Local Unicast (fd00::/8-prefix) that you are currently not using would | ||
3193 | be a good choice. | ||
3194 | |||
3195 | @node Configuring the GNUnet VPN DNS | ||
3196 | @subsubsection Configuring the GNUnet VPN DNS | ||
3197 | |||
3198 | To resolve names for remote nodes, activate the DNS exit option. | ||
3199 | |||
3200 | @node Configuring the GNUnet VPN Exit Service | ||
3201 | @subsubsection Configuring the GNUnet VPN Exit Service | ||
3202 | |||
3203 | If you want to allow other users to share your Internet connection (yes, this | ||
3204 | may be dangerous, just as running a Tor exit node) or want to provide access to | ||
3205 | services on your host (this should be less dangerous, as long as those services | ||
3206 | are secure), you have to enable the GNUnet exit daemon. | ||
3207 | |||
3208 | You then get to specify which exit functions you want to provide. By enabling | ||
3209 | the exit daemon, you will always automatically provide exit functions for | ||
3210 | manually configured local services (this component of the system is under | ||
3211 | development and not documented further at this time). As for those services you | ||
3212 | explicitly specify the target IP address and port, there is no significant | ||
3213 | security risk in doing so. | ||
3214 | |||
3215 | Furthermore, you can serve as a DNS, IPv4 or IPv6 exit to the Internet. Being a | ||
3216 | DNS exit is usually pretty harmless. However, enabling IPv4 or IPv6-exit | ||
3217 | without further precautions may enable adversaries to access your local | ||
3218 | network, send spam, attack other systems from your Internet connection and to | ||
3219 | other mischief that will appear to come from your machine. This may or may not | ||
3220 | get you into legal trouble. If you want to allow IPv4 or IPv6-exit | ||
3221 | functionality, you should strongly consider adding additional firewall rules | ||
3222 | manually to protect your local network and to restrict outgoing TCP traffic | ||
3223 | (i.e. by not allowing access to port 25). While we plan to improve | ||
3224 | exit-filtering in the future, you're currently on your own here. Essentially, | ||
3225 | be prepared for any kind of IP-traffic to exit the respective TUN interface | ||
3226 | (and GNUnet will enable IP-forwarding and NAT for the interface automatically). | ||
3227 | |||
3228 | Additional configuration options of the exit as shown by the gnunet-setup tool | ||
3229 | are: | ||
3230 | |||
3231 | @node IP Address of external DNS resolver | ||
3232 | @subsubsection IP Address of external DNS resolver | ||
3233 | |||
3234 | If DNS traffic is to exit your machine, it will be send to this DNS resolver. | ||
3235 | You can specify an IPv4 or IPv6 address. | ||
3236 | |||
3237 | @node IPv4 address for Exit interface | ||
3238 | @subsubsection IPv4 address for Exit interface | ||
3239 | |||
3240 | This is the IPv4 address the Interface will get. Make the mask of the address | ||
3241 | big enough (255.255.0.0 or, even better, 255.0.0.0) to allow more mappings of | ||
3242 | IP addresses into this range. As for the VPN interface, any unused, private | ||
3243 | IPv4 address range will do. | ||
3244 | |||
3245 | @node IPv6 address for Exit interface | ||
3246 | @subsubsection IPv6 address for Exit interface | ||
3247 | |||
3248 | The public IPv6 address the interface will get. If your kernel is not a very | ||
3249 | recent kernel and you are willing to manually enable IPv6-NAT, the IPv6 address | ||
3250 | you specify here must be a globally routed IPv6 address of your host. | ||
3251 | |||
3252 | Suppose your host has the address @code{2001:4ca0::1234/64}, then using@ | ||
3253 | @code{2001:4ca0::1:0/112} would be fine (keep the first 64 bits, then change at | ||
3254 | least one bit in the range before the bitmask, in the example above we changed | ||
3255 | bit 111 from 0 to 1). | ||
3256 | |||
3257 | You may also have to configure your router to route traffic for the entire | ||
3258 | subnet (@code{2001:4ca0::1:0/112} for example) through your computer (this | ||
3259 | should be automatic with IPv6, but obviously anything can be | ||
3260 | disabled). | ||
3261 | |||
3262 | @node Bandwidth Configuration | ||
3263 | @subsection Bandwidth Configuration | ||
3264 | |||
3265 | You can specify how many bandwidth GNUnet is allowed to use to receive and send | ||
3266 | data. This is important for users with limited bandwidth or traffic volume. | ||
3267 | |||
3268 | @node Configuring NAT | ||
3269 | @subsection Configuring NAT | ||
3270 | |||
3271 | Most hosts today do not have a normal global IP address but instead are behind | ||
3272 | a router performing Network Address Translation (NAT) which assigns each host | ||
3273 | in the local network a private IP address. As a result, these machines cannot | ||
3274 | trivially receive inbound connections from the Internet. GNUnet supports NAT | ||
3275 | traversal to enable these machines to receive incoming connections from other | ||
3276 | peers despite their limitations. | ||
3277 | |||
3278 | In an ideal world, you can press the "Attempt automatic configuration" button | ||
3279 | in gnunet-setup to automatically configure your peer correctly. Alternatively, | ||
3280 | your distribution might have already triggered this automatic configuration | ||
3281 | during the installation process. However, automatic configuration can fail to | ||
3282 | determine the optimal settings, resulting in your peer either not receiving as | ||
3283 | many connections as possible, or in the worst case it not connecting to the | ||
3284 | network at all. | ||
3285 | |||
3286 | To manually configure the peer, you need to know a few things about your | ||
3287 | network setup. First, determine if you are behind a NAT in the first place. | ||
3288 | This is always the case if your IP address starts with "10.*" or "192.168.*". | ||
3289 | Next, if you have control over your NAT router, you may choose to manually | ||
3290 | configure it to allow GNUnet traffic to your host. If you have configured your | ||
3291 | NAT to forward traffic on ports 2086 (and possibly 1080) to your host, you can | ||
3292 | check the "NAT ports have been opened manually" option, which corresponds to | ||
3293 | the "PUNCHED_NAT" option in the configuration file. If you did not punch your | ||
3294 | NAT box, it may still be configured to support UPnP, which allows GNUnet to | ||
3295 | automatically configure it. In that case, you need to install the "upnpc" | ||
3296 | command, enable UPnP (or PMP) on your NAT box and set the "Enable NAT traversal | ||
3297 | via UPnP or PMP" option (corresponding to "ENABLE_UPNP" in the configuration | ||
3298 | file). | ||
3299 | |||
3300 | Some NAT boxes can be traversed using the autonomous NAT traversal method. This | ||
3301 | requires certain GNUnet components to be installed with "SUID" prividledges on | ||
3302 | your system (so if you're installing on a system you do not have administrative | ||
3303 | rights to, this will not work). If you installed as 'root', you can enable | ||
3304 | autonomous NAT traversal by checking the "Enable NAT traversal using ICMP | ||
3305 | method". The ICMP method requires a way to determine your NAT's external | ||
3306 | (global) IP address. This can be done using either UPnP, DynDNS, or by manual | ||
3307 | configuration. If you have a DynDNS name or know your external IP address, you | ||
3308 | should enter that name under "External (public) IPv4 address" (which | ||
3309 | corresponds to the "EXTERNAL_ADDRESS" option in the configuration file). If you | ||
3310 | leave the option empty, GNUnet will try to determine your external IP address | ||
3311 | automatically (which may fail, in which case autonomous NAT traversal will then | ||
3312 | not work). | ||
3313 | |||
3314 | Finally, if you yourself are not behind NAT but want to be able to connect to | ||
3315 | NATed peers using autonomous NAT traversal, you need to check the "Enable | ||
3316 | connecting to NATed peers using ICMP method" box. | ||
3317 | |||
3318 | |||
3319 | @node Peer configuration for distributions | ||
3320 | @subsection Peer configuration for distributions | ||
3321 | |||
3322 | The "GNUNET_DATA_HOME" in "[path]" in @file{/etc/gnunet.conf} should be manually set | ||
3323 | to "/var/lib/gnunet/data/" as the default "~/.local/share/gnunet/" is probably | ||
3324 | not that appropriate in this case. Similarly, distributions may consider | ||
3325 | pointing "GNUNET_RUNTIME_DIR" to "/var/run/gnunet/" and "GNUNET_HOME" to | ||
3326 | "/var/lib/gnunet/". Also, should a distribution decide to override system | ||
3327 | defaults, all of these changes should be done in a custom @file{/etc/gnunet.conf} | ||
3328 | and not in the files in the @file{config.d/} directory. | ||
3329 | |||
3330 | Given the proposed access permissions, the "gnunet-setup" tool must be run as | ||
3331 | use "gnunet" (and with option "-c /etc/gnunet.conf" so that it modifies the | ||
3332 | system configuration). As always, gnunet-setup should be run after the GNUnet | ||
3333 | peer was stopped using "gnunet-arm -e". Distributions might want to include a | ||
3334 | wrapper for gnunet-setup that allows the desktop-user to "sudo" (i.e. using | ||
3335 | gtksudo) to the "gnunet" user account and then runs "gnunet-arm -e", | ||
3336 | "gnunet-setup" and "gnunet-arm -s" in sequence. | ||
3337 | |||
3338 | |||
3339 | |||
3340 | @node How to start and stop a GNUnet peer | ||
3341 | @section How to start and stop a GNUnet peer | ||
3342 | |||
3343 | This section describes how to start a GNUnet peer. It assumes that you have | ||
3344 | already compiled and installed GNUnet and its' dependencies. Before you start a | ||
3345 | GNUnet peer, you may want to create a configuration file using gnunet-setup | ||
3346 | (but you do not have to). Sane defaults should exist in your | ||
3347 | @file{$GNUNET_PREFIX/share/gnunet/config.d/} directory, so in practice you could | ||
3348 | simply start without any configuration. If you want to configure your peer | ||
3349 | later, you need to stop it before invoking the @code{gnunet-setup} tool to | ||
3350 | customize further and to test your configuration (@code{gnunet-setup} has | ||
3351 | build-in test functions). | ||
3352 | |||
3353 | The most important option you might have to still set by hand is in [PATHS]. | ||
3354 | Here, you use the option "GNUNET_HOME" to specify the path where GNUnet should | ||
3355 | store its data. It defaults to @code{$HOME/}, which again should work for most | ||
3356 | users. Make sure that the directory specified as GNUNET_HOME is writable to | ||
3357 | the user that you will use to run GNUnet (note that you can run frontends | ||
3358 | using other users, GNUNET_HOME must only be accessible to the user used to run | ||
3359 | the background processes). | ||
3360 | |||
3361 | You will also need to make one central decision: should all of GNUnet be run | ||
3362 | under your normal UID, or do you want distinguish between system-wide | ||
3363 | (user-independent) GNUnet services and personal GNUnet services. The multi-user | ||
3364 | setup is slightly more complicated, but also more secure and generally | ||
3365 | recommended. | ||
3366 | |||
3367 | @menu | ||
3368 | * The Single-User Setup:: | ||
3369 | * The Multi-User Setup:: | ||
3370 | * Killing GNUnet services:: | ||
3371 | * Access Control for GNUnet:: | ||
3372 | @end menu | ||
3373 | |||
3374 | @node The Single-User Setup | ||
3375 | @subsection The Single-User Setup | ||
3376 | |||
3377 | For the single-user setup, you do not need to do anything special and can just | ||
3378 | start the GNUnet background processes using @code{gnunet-arm}. By default, | ||
3379 | GNUnet looks in @file{~/.config/gnunet.conf} for a configuration (or | ||
3380 | @code{$XDG_CONFIG_HOME/gnunet.conf} if@ @code{$XDG_CONFIG_HOME} is defined). If your | ||
3381 | configuration lives elsewhere, you need to pass the @code{-c FILENAME} option | ||
3382 | to all GNUnet commands. | ||
3383 | |||
3384 | Assuming the configuration file is called @file{~/.config/gnunet.conf}, you | ||
3385 | start your peer using the @code{gnunet-arm} command (say as user | ||
3386 | @code{gnunet}) using: | ||
3387 | @example | ||
3388 | gnunet-arm -c ~/.config/gnunet.conf -s | ||
3389 | @end example | ||
3390 | |||
3391 | The "-s" option here is for "start". The command should return almost | ||
3392 | instantly. If you want to stop GNUnet, you can use: | ||
3393 | @example | ||
3394 | gnunet-arm -c ~/.config/gnunet.conf -e | ||
3395 | @end example | ||
3396 | |||
3397 | The "-e" option here is for "end". | ||
3398 | |||
3399 | Note that this will only start the basic peer, no actual applications will be | ||
3400 | available. If you want to start the file-sharing service, use (after starting | ||
3401 | GNUnet): | ||
3402 | @example | ||
3403 | gnunet-arm -c ~/.config/gnunet.conf -i fs | ||
3404 | @end example | ||
3405 | |||
3406 | The "-i fs" option here is for "initialize" the "fs" (file-sharing) | ||
3407 | application. You can also selectively kill only file-sharing support using | ||
3408 | @example | ||
3409 | gnunet-arm -c ~/.config/gnunet.conf -k fs | ||
3410 | @end example | ||
3411 | |||
3412 | Assuming that you want certain services (like file-sharing) to be always | ||
3413 | automatically started whenever you start GNUnet, you can activate them by | ||
3414 | setting "FORCESTART=YES" in the respective section of the configuration file | ||
3415 | (for example, "[fs]"). Then GNUnet with file-sharing support would be started | ||
3416 | whenever you@ enter: | ||
3417 | @example | ||
3418 | gnunet-arm -c ~/.config/gnunet.conf -s | ||
3419 | @end example | ||
3420 | |||
3421 | Alternatively, you can combine the two options: | ||
3422 | @example | ||
3423 | gnunet-arm -c ~/.config/gnunet.conf -s -i fs | ||
3424 | @end example | ||
3425 | |||
3426 | |||
3427 | Using @code{gnunet-arm} is also the preferred method for initializing GNUnet | ||
3428 | from @code{init}. | ||
3429 | |||
3430 | Finally, you should edit your @code{crontab} (using the @code{crontab} command) | ||
3431 | and insert a line@ | ||
3432 | @code{@ | ||
3433 | @@reboot gnunet-arm -c ~/.config/gnunet.conf -s@ | ||
3434 | }@ | ||
3435 | to automatically start your peer whenever your system boots. | ||
3436 | |||
3437 | @node The Multi-User Setup | ||
3438 | @subsection The Multi-User Setup | ||
3439 | |||
3440 | This requires you to create a user @code{gnunet} and an additional group | ||
3441 | @code{gnunetdns}, prior to running @code{make install} during installation. | ||
3442 | Then, you create a configuration file @file{/etc/gnunet.conf} which should | ||
3443 | contain the lines:@ | ||
3444 | @code{@ | ||
3445 | [arm]@ | ||
3446 | SYSTEM_ONLY = YES@ | ||
3447 | USER_ONLY = NO@ | ||
3448 | }@ | ||
3449 | Then, perform the same steps to run GNUnet as in the per-user configuration, | ||
3450 | except as user @code{gnunet} (including the @code{crontab} installation). You | ||
3451 | may also want to run @code{gnunet-setup} to configure your peer (databases, | ||
3452 | etc.). Make sure to pass @code{-c /etc/gnunet.conf} to all commands. If you | ||
3453 | run @code{gnunet-setup} as user @code{gnunet}, you might need to change | ||
3454 | permissions on @file{/etc/gnunet.conf} so that the @code{gnunet} user can | ||
3455 | write to the file (during setup). | ||
3456 | |||
3457 | Afterwards, you need to perform another setup step for each normal user account | ||
3458 | from which you want to access GNUnet. First, grant the normal user | ||
3459 | (@code{$USER}) permission to the group gnunet:@ | ||
3460 | @code{@ | ||
3461 | # adduser $USER gnunet@ | ||
3462 | }@ | ||
3463 | Then, create a configuration file in @file{~/.config/gnunet.conf} for the $USER | ||
3464 | with the lines:@ | ||
3465 | @code{@ | ||
3466 | [arm]@ | ||
3467 | SYSTEM_ONLY = NO@ | ||
3468 | USER_ONLY = YES@ | ||
3469 | }@ | ||
3470 | This will ensure that @code{gnunet-arm} when started by the normal user will | ||
3471 | only run services that are per-user, and otherwise rely on the system-wide | ||
3472 | services. Note that the normal user may run gnunet-setup, but the | ||
3473 | configuration would be ineffective as the system-wide services will use | ||
3474 | @code{/etc/gnunet.conf} and ignore options set by individual users. | ||
3475 | |||
3476 | Again, each user should then start the peer using @code{gnunet-arm -s} --- and | ||
3477 | strongly consider adding logic to start the peer automatically to their | ||
3478 | crontab. | ||
3479 | |||
3480 | Afterwards, you should see two (or more, if you have more than one USER) | ||
3481 | @code{gnunet-service-arm} processes running in your system. | ||
3482 | |||
3483 | @node Killing GNUnet services | ||
3484 | @subsection Killing GNUnet services | ||
3485 | |||
3486 | It is not necessary to stop GNUnet services explicitly when shutting down your | ||
3487 | computer. | ||
3488 | |||
3489 | It should be noted that manually killing "most" of the @code{gnunet-service} | ||
3490 | processes is generally not a successful method for stopping a peer (since | ||
3491 | @code{gnunet-service-arm} will instantly restart them). The best way to | ||
3492 | explicitly stop a peer is using @code{gnunet-arm -e}; note that the per-user | ||
3493 | services may need to be terminated before the system-wide services will | ||
3494 | terminate normally. | ||
3495 | |||
3496 | @node Access Control for GNUnet | ||
3497 | @subsection Access Control for GNUnet | ||
3498 | |||
3499 | This chapter documents how we plan to make access control work within the | ||
3500 | GNUnet system for a typical peer. It should be read as a best-practice | ||
3501 | installation guide for advanced users and builders of binary distributions. The | ||
3502 | recommendations in this guide apply to POSIX-systems with full support for UNIX | ||
3503 | domain sockets only. | ||
3504 | |||
3505 | Note that this is an advanced topic. The discussion presumes a very good | ||
3506 | understanding of users, groups and file permissions. Normal users on hosts with | ||
3507 | just a single user can just install GNUnet under their own account (and | ||
3508 | possibly allow the installer to use SUDO to grant additional permissions for | ||
3509 | special GNUnet tools that need additional rights). The discussion below largely | ||
3510 | applies to installations where multiple users share a system and to | ||
3511 | installations where the best possible security is paramount. | ||
3512 | |||
3513 | A typical GNUnet system consists of components that fall into four categories: | ||
3514 | |||
3515 | @table @asis | ||
3516 | |||
3517 | @item User interfaces | ||
3518 | User interfaces are not security sensitive and are supposed to be run and used | ||
3519 | by normal system users. The GTK GUIs and most command-line programs fall into | ||
3520 | this category. Some command-line tools (like gnunet-transport) should be | ||
3521 | excluded as they offer low-level access that normal users should not need. | ||
3522 | @item System services and support tools | ||
3523 | System services should always run and offer services that can then be accessed | ||
3524 | by the normal users. System services do not require special permissions, but as | ||
3525 | they are not specific to a particular user, they probably should not run as a | ||
3526 | particular user. Also, there should typically only be one GNUnet peer per host. | ||
3527 | System services include the gnunet-service and gnunet-daemon programs; support | ||
3528 | tools include command-line programs such as gnunet-arm. | ||
3529 | @item Priviledged helpers | ||
3530 | Some GNUnet components require root rights to open raw sockets or perform other | ||
3531 | special operations. These gnunet-helper binaries are typically installed SUID | ||
3532 | and run from services or daemons. | ||
3533 | @item Critical services | ||
3534 | Some GNUnet services (such as the DNS service) can manipulate the service in | ||
3535 | deep and possibly highly security sensitive ways. For example, the DNS service | ||
3536 | can be used to intercept and alter any DNS query originating from the local | ||
3537 | machine. Access to the APIs of these critical services and their priviledged | ||
3538 | helpers must be tightly controlled. | ||
3539 | @end table | ||
3540 | |||
3541 | @menu | ||
3542 | * Recommendation - Disable access to services via TCP:: | ||
3543 | * Recommendation - Run most services as system user "gnunet":: | ||
3544 | * Recommendation - Control access to services using group "gnunet":: | ||
3545 | * Recommendation - Limit access to certain SUID binaries by group "gnunet":: | ||
3546 | * Recommendation - Limit access to critical gnunet-helper-dns to group "gnunetdns":: | ||
3547 | * Differences between "make install" and these recommendations:: | ||
3548 | @end menu | ||
3549 | |||
3550 | @node Recommendation - Disable access to services via TCP | ||
3551 | @subsubsection Recommendation - Disable access to services via TCP | ||
3552 | |||
3553 | GNUnet services allow two types of access: via TCP socket or via UNIX domain | ||
3554 | socket. If the service is available via TCP, access control can only be | ||
3555 | implemented by restricting connections to a particular range of IP addresses. | ||
3556 | This is acceptable for non-critical services that are supposed to be available | ||
3557 | to all users on the local system or local network. However, as TCP is generally | ||
3558 | less efficient and it is rarely the case that a single GNUnet peer is supposed | ||
3559 | to serve an entire local network, the default configuration should disable TCP | ||
3560 | access to all GNUnet services on systems with support for UNIX domain sockets. | ||
3561 | As of GNUnet 0.9.2, configuration files with TCP access disabled should be | ||
3562 | generated by default. Users can re-enable TCP access to particular services | ||
3563 | simply by specifying a non-zero port number in the section of the respective | ||
3564 | service. | ||
3565 | |||
3566 | |||
3567 | @node Recommendation - Run most services as system user "gnunet" | ||
3568 | @subsubsection Recommendation - Run most services as system user "gnunet" | ||
3569 | |||
3570 | GNUnet's main services should be run as a separate user "gnunet" in a special | ||
3571 | group "gnunet". The user "gnunet" should start the peer using "gnunet-arm -s" | ||
3572 | during system startup. The home directory for this user should be | ||
3573 | @file{/var/lib/gnunet} and the configuration file should be @file{/etc/gnunet.conf}. | ||
3574 | Only the @code{gnunet} user should have the right to access @file{/var/lib/gnunet} | ||
3575 | (@emph{mode: 700}). | ||
3576 | |||
3577 | @node Recommendation - Control access to services using group "gnunet" | ||
3578 | @subsubsection Recommendation - Control access to services using group "gnunet" | ||
3579 | |||
3580 | Users that should be allowed to use the GNUnet peer should be added to the | ||
3581 | group "gnunet". Using GNUnet's access control mechanism for UNIX domain | ||
3582 | sockets, those services that are considered useful to ordinary users should be | ||
3583 | made available by setting "UNIX_MATCH_GID=YES" for those services. Again, as | ||
3584 | shipped, GNUnet provides reasonable defaults. Permissions to access the | ||
3585 | transport and core subsystems might additionally be granted without necessarily | ||
3586 | causing security concerns. Some services, such as DNS, must NOT be made | ||
3587 | accessible to the "gnunet" group (and should thus only be accessible to the | ||
3588 | "gnunet" user and services running with this UID). | ||
3589 | |||
3590 | @node Recommendation - Limit access to certain SUID binaries by group "gnunet" | ||
3591 | @subsubsection Recommendation - Limit access to certain SUID binaries by group "gnunet" | ||
3592 | |||
3593 | Most of GNUnet's SUID binaries should be safe even if executed by normal users. | ||
3594 | However, it is possible to reduce the risk a little bit more by making these | ||
3595 | binaries owned by the group "gnunet" and restricting their execution to user of | ||
3596 | the group "gnunet" as well (4750). | ||
3597 | |||
3598 | @node Recommendation - Limit access to critical gnunet-helper-dns to group "gnunetdns" | ||
3599 | @subsubsection Recommendation - Limit access to critical gnunet-helper-dns to group "gnunetdns" | ||
3600 | |||
3601 | A special group "gnunetdns" should be created for controlling access to the | ||
3602 | "gnunet-helper-dns". The binary should then be owned by root and be in group | ||
3603 | "gnunetdns" and be installed SUID and only be group-executable (2750). Note | ||
3604 | that the group "gnunetdns" should have no users in it at all, ever. The | ||
3605 | "gnunet-service-dns" program should be executed by user "gnunet" (via | ||
3606 | gnunet-service-arm) with the binary owned by the user "root" and the group | ||
3607 | "gnunetdns" and be SGID (2700). This way, @strong{only} "gnunet-service-dns" | ||
3608 | can change its group to "gnunetdns" and execute the helper, and the helper can | ||
3609 | then run as root (as per SUID). Access to the API offered by | ||
3610 | "gnunet-service-dns" is in turn restricted to the user "gnunet" (not the | ||
3611 | group!), which means that only "benign" services can manipulate DNS queries | ||
3612 | using "gnunet-service-dns". | ||
3613 | |||
3614 | @node Differences between "make install" and these recommendations | ||
3615 | @subsubsection Differences between "make install" and these recommendations | ||
3616 | |||
3617 | The current build system does not set all permissions automatically based on | ||
3618 | the recommendations above. In particular, it does not use the group "gnunet" at | ||
3619 | all (so setting gnunet-helpers other than the gnunet-helper-dns to be owned by | ||
3620 | group "gnunet" must be done manually). Furthermore, 'make install' will | ||
3621 | silently fail to set the DNS binaries to be owned by group "gnunetdns" unless | ||
3622 | that group already exists (!). An alternative name for the "gnunetdns" group | ||
3623 | can be specified using the "--with-gnunetdns=GRPNAME" configure | ||
3624 | option. | ||
3625 | |||
diff --git a/doc/chapters/philosophy.texi b/doc/chapters/philosophy.texi deleted file mode 100644 index 9f4702b49..000000000 --- a/doc/chapters/philosophy.texi +++ /dev/null | |||
@@ -1,330 +0,0 @@ | |||
1 | @c *************************************************************************** | ||
2 | @node Philosophy | ||
3 | @chapter Philosophy | ||
4 | |||
5 | The foremost goal of the GNUnet project is to become a widely used, | ||
6 | reliable, open, non-discriminating, egalitarian, unfettered and | ||
7 | censorship-resistant system of free information exchange. | ||
8 | We value free speech above state secrets, law-enforcement or | ||
9 | intellectual property. GNUnet is supposed to be an anarchistic network, | ||
10 | where the only limitation for peers is that they must contribute enough | ||
11 | back to the network such that their resource consumption does not have | ||
12 | a significant impact on other users. GNUnet should be more than just | ||
13 | another file-sharing network. The plan is to offer many other services | ||
14 | and in particular to serve as a development platform for the next | ||
15 | generation of decentralized Internet protocols. | ||
16 | |||
17 | @menu | ||
18 | * Design Goals:: | ||
19 | * Security & Privacy:: | ||
20 | * Versatility:: | ||
21 | * Practicality:: | ||
22 | * Key Concepts:: | ||
23 | @end menu | ||
24 | |||
25 | |||
26 | @c *************************************************************************** | ||
27 | @node Design Goals | ||
28 | @section Design Goals | ||
29 | |||
30 | These are the core GNUnet design goals, in order of relative importance: | ||
31 | |||
32 | @itemize | ||
33 | @item GNUnet must be implemented as free software. | ||
34 | @item GNUnet must only disclose the minimal amount of information necessary. | ||
35 | @item GNUnet must be decentralised and survive Byzantine failures in any position in the network. | ||
36 | @item GNUnet must make it explicit to the user which entities must be trustworthy when establishing secured communications. | ||
37 | @item GNUnet must use compartmentalization to protect sensitive information. | ||
38 | @item GNUnet must be open and permit new peers to join. | ||
39 | @item GNUnet must be self-organizing and not depend on administrators. | ||
40 | @item GNUnet must support a diverse range of applications and devices. | ||
41 | @item The GNUnet architecture must be cost effective. | ||
42 | @item GNUnet must provide incentives for peers to contribute more resources than they consume. | ||
43 | @end itemize | ||
44 | |||
45 | |||
46 | @node Security & Privacy | ||
47 | @section Security & Privacy | ||
48 | |||
49 | GNUnet's primary design goals are to protect the privacy of its users and to | ||
50 | guard itself against attacks or abuse. GNUnet does not have any mechanisms | ||
51 | to control, track or censor users. Instead, the GNUnet protocols aim to make | ||
52 | it as hard as possible to find out what is happening on the network or to | ||
53 | disrupt operations. | ||
54 | |||
55 | @node Versatility | ||
56 | @section Versatility | ||
57 | |||
58 | We call GNUnet a peer-to-peer framework because we want to support many | ||
59 | different forms of peer-to-peer applications. GNUnet uses a plugin | ||
60 | architecture to make the system extensible and to encourage code reuse. | ||
61 | While the first versions of the system only supported anonymous file-sharing, | ||
62 | other applications are being worked on and more will hopefully follow in the | ||
63 | future. A powerful synergy regarding anonymity services is created by a large | ||
64 | community utilizing many diverse applications over the same software | ||
65 | infrastructure. The reason is that link encryption hides the specifics | ||
66 | of the traffic for non-participating observers. This way, anonymity can | ||
67 | get stronger with additional (GNUnet) traffic, even if the additional | ||
68 | traffic is not related to anonymous communication. Increasing anonymity is | ||
69 | the primary reason why GNUnet is developed to become a peer-to-peer | ||
70 | framework where many applications share the lower layers of an increasingly | ||
71 | complex protocol stack. If merging traffic to hinder traffic analysis was | ||
72 | not important, we could have just developed a dozen stand-alone applications | ||
73 | and a few shared libraries. | ||
74 | |||
75 | @node Practicality | ||
76 | @section Practicality | ||
77 | |||
78 | GNUnet allows participants to trade various amounts of security in exchange | ||
79 | for increased efficiency. However, it is not possible for any user's security | ||
80 | and efficiency requirements to compromise the security and efficiency of | ||
81 | any other user. | ||
82 | |||
83 | For GNUnet, efficiency is not paramount. If there is a more secure and still | ||
84 | practical approach, we would choose to take the more secure alternative. | ||
85 | @command{telnet} is more efficient than @command{ssh}, yet it is obsolete. | ||
86 | Hardware gets faster, and code can be optimized. Fixing security issues as | ||
87 | an afterthought is much harder. | ||
88 | |||
89 | While security is paramount, practicability is still a requirement. The most | ||
90 | secure system is always the one that nobody can use. Similarly, any | ||
91 | anonymous system that is extremely inefficient will only find few users. | ||
92 | However, good anonymity requires a large and diverse user base. Since | ||
93 | individual security requirements may vary, the only good solution here is to | ||
94 | allow individuals to trade-off security and efficiency. The primary challenge | ||
95 | in allowing this is to ensure that the economic incentives work properly. | ||
96 | In particular, this means that it must be impossible for a user to gain | ||
97 | security at the expense of other users. Many designs (e.g. anonymity via | ||
98 | broadcast) fail to give users an incentive to choose a less secure but more | ||
99 | efficient mode of operation. GNUnet should avoid where ever possible to | ||
100 | rely on protocols that will only work if the participants are benevolent. | ||
101 | While some designs have had widespread success while relying on parties | ||
102 | to observe a protocol that may be sub-optimal for the individuals (e.g. | ||
103 | TCP Nagle), a protocol that ensures that individual goals never conflict | ||
104 | with the goals of the group is always preferable. | ||
105 | |||
106 | @node Key Concepts | ||
107 | @section Key Concepts | ||
108 | |||
109 | In this section, the fundamental concepts of GNUnet are explained. Most of | ||
110 | them are also described in our research papers. First, some of the concepts | ||
111 | used in the GNUnet framework are detailed. The second part describes concepts | ||
112 | specific to anonymous file-sharing. | ||
113 | |||
114 | @menu | ||
115 | * Authentication:: | ||
116 | * Accounting to Encourage Resource Sharing:: | ||
117 | * Confidentiality:: | ||
118 | * Anonymity:: | ||
119 | * Deniability:: | ||
120 | * Peer Identities:: | ||
121 | * Zones in the GNU Name System (GNS Zones):: | ||
122 | * Egos:: | ||
123 | @end menu | ||
124 | |||
125 | @node Authentication | ||
126 | @subsection Authentication | ||
127 | |||
128 | Almost all peer-to-peer communications in GNUnet are between mutually | ||
129 | authenticated peers. The authentication works by using ECDHE, that is a | ||
130 | DH key exchange using ephemeral eliptic curve cryptography. The ephemeral | ||
131 | ECC keys are signed using ECDSA. The shared secret from ECDHE is used to | ||
132 | create a pair of session keys (using HKDF) which are then used to encrypt | ||
133 | the communication between the two peers using both 256-bit AES and 256-bit | ||
134 | Twofish (with independently derived secret keys). As only the two | ||
135 | participating hosts know the shared secret, this authenticates each packet | ||
136 | without requiring signatures each time. GNUnet uses SHA-512 hash codes to | ||
137 | verify the integrity of messages. | ||
138 | |||
139 | In GNUnet, the identity of a host is its public key. For that reason, | ||
140 | man-in-the-middle attacks will not break the authentication or accounting | ||
141 | goals. Essentially, for GNUnet, the IP of the host has nothing to do with | ||
142 | the identity of the host. As the public key is the only thing that truly | ||
143 | matters, faking an IP, a port or any other property of the underlying | ||
144 | transport protocol is irrelevant. In fact, GNUnet peers can use | ||
145 | multiple IPs (IPv4 and IPv6) on multiple ports --- or even not use the | ||
146 | IP protocol at all (by running directly on layer 2). | ||
147 | |||
148 | GNUnet uses a special type of message to communicate a binding between | ||
149 | public (ECC) keys to their current network address. These messages are | ||
150 | commonly called HELLOs or peer advertisements. They contain the public key | ||
151 | of the peer and its current network addresses for various transport services. | ||
152 | A transport service is a special kind of shared library that | ||
153 | provides (possibly unreliable, out-of-order) message delivery between peers. | ||
154 | For the UDP and TCP transport services, a network address is an IP and a port. | ||
155 | GNUnet can also use other transports (HTTP, HTTPS, WLAN, etc.) which use | ||
156 | various other forms of addresses. Note that any node can have many different | ||
157 | active transport services at the same time, and each of these can have a | ||
158 | different addresses. Binding messages expire after at most a week (the | ||
159 | timeout can be shorter if the user configures the node appropriately). This | ||
160 | expiration ensures that the network will eventually get rid of outdated | ||
161 | advertisements.@ | ||
162 | More details can be found in the paper @uref{https://gnunet.org/transports, A Transport Layer Abstraction for Peer-to-Peer Networks}. | ||
163 | |||
164 | @node Accounting to Encourage Resource Sharing | ||
165 | @subsection Accounting to Encourage Resource Sharing | ||
166 | |||
167 | Most distributed P2P networks suffer from a lack of defenses or precautions | ||
168 | against attacks in the form of freeloading. While the intentions of an | ||
169 | attacker and a freeloader are different, their effect on the network is the | ||
170 | same; they both render it useless. Most simple attacks on networks such as | ||
171 | Gnutella involve flooding the network with traffic, particularly with | ||
172 | queries that are, in the worst case, multiplied by the network. | ||
173 | |||
174 | In order to ensure that freeloaders or attackers have a minimal impact on the | ||
175 | network, GNUnet's file-sharing implementation tries to distinguish | ||
176 | good (contributing) nodes from malicious (freeloading) nodes. In GNUnet, | ||
177 | every file-sharing node keeps track of the behavior of every other node it | ||
178 | has been in contact with. Many requests (depending on the application) are | ||
179 | transmitted with a priority (or importance) level. That priority is used to | ||
180 | establish how important the sender believes this request is. If a peer | ||
181 | responds to an important request, the recipient will increase its trust in the | ||
182 | responder: the responder contributed resources. If a peer is too busy to | ||
183 | answer all requests, it needs to prioritize. For that, peers to not take the | ||
184 | priorities of the requests received at face value. First, they check how much | ||
185 | they trust the sender, and depending on that amount of trust they assign the | ||
186 | request a (possibly lower) effective priority. Then, they drop the requests | ||
187 | with the lowest effective priority to satisfy their resource constraints. This | ||
188 | way, GNUnet's economic model ensures that nodes that are not currently | ||
189 | considered to have a surplus in contributions will not be served if the | ||
190 | network load is high. More details can be found in @uref{https://gnunet.org/ebe, this paper}. | ||
191 | |||
192 | @node Confidentiality | ||
193 | @subsection Confidentiality | ||
194 | |||
195 | Adversaries outside of GNUnet are not supposed to know what kind of actions a | ||
196 | peer is involved in. Only the specific neighbor of a peer that is the | ||
197 | corresponding sender or recipient of a message may know its contents, and even | ||
198 | then application protocols may place further restrictions on that knowledge. | ||
199 | In order to ensure confidentiality, GNUnet uses link encryption, that is each | ||
200 | message exchanged between two peers is encrypted using a pair of keys only | ||
201 | known to these two peers. Encrypting traffic like this makes any kind of | ||
202 | traffic analysis much harder. Naturally, for some applications, it may still | ||
203 | be desirable if even neighbors cannot determine the concrete contents of a | ||
204 | message. In GNUnet, this problem is addressed by the specific | ||
205 | application-level protocols (see for example, deniability and anonymity in | ||
206 | anonymous file sharing). | ||
207 | |||
208 | @node Anonymity | ||
209 | @subsection Anonymity | ||
210 | |||
211 | @menu | ||
212 | * How file-sharing achieves Anonymity:: | ||
213 | @end menu | ||
214 | |||
215 | Providing anonymity for users is the central goal for the anonymous | ||
216 | file-sharing application. Many other design decisions follow in the footsteps | ||
217 | of this requirement. Anonymity is never absolute. While there are various | ||
218 | @uref{https://gnunet.org/anonymity_metric, scientific metrics} that can help quantify the level of anonymity that a | ||
219 | given mechanism provides, there is no such thing as complete anonymity. | ||
220 | GNUnet's file-sharing implementation allows users to select for each | ||
221 | operation (publish, search, download) the desired level of anonymity. | ||
222 | The metric used is the amount of cover traffic available to hide the request. | ||
223 | While this metric is not as good as, for example, the theoretical metric | ||
224 | given in @uref{https://gnunet.org/anonymity_metric, scientific metrics}, it is probably the best metric available to | ||
225 | a peer with a purely local view of the world that does not rely on unreliable | ||
226 | external information. The default anonymity level is 1, which uses anonymous | ||
227 | routing but imposes no minimal requirements on cover traffic. It is possible | ||
228 | to forego anonymity when this is not required. The anonymity level of 0 | ||
229 | allows GNUnet to use more efficient, non-anonymous routing. | ||
230 | |||
231 | @node How file-sharing achieves Anonymity | ||
232 | @subsubsection How file-sharing achieves Anonymity | ||
233 | |||
234 | Contrary to other designs, we do not believe that users achieve strong | ||
235 | anonymity just because their requests are obfuscated by a couple of | ||
236 | indirections. This is not sufficient if the adversary uses traffic analysis. | ||
237 | The threat model used for anonymous file sharing in GNUnet assumes that the | ||
238 | adversary is quite powerful. In particular, we assume that the adversary can | ||
239 | see all the traffic on the Internet. And while we assume that the adversary | ||
240 | can not break our encryption, we assume that the adversary has many | ||
241 | participating nodes in the network and that it can thus see many of the | ||
242 | node-to-node interactions since it controls some of the nodes. | ||
243 | |||
244 | The system tries to achieve anonymity based on the idea that users can be | ||
245 | anonymous if they can hide their actions in the traffic created by other users. | ||
246 | Hiding actions in the traffic of other users requires participating in the | ||
247 | traffic, bringing back the traditional technique of using indirection and | ||
248 | source rewriting. Source rewriting is required to gain anonymity since | ||
249 | otherwise an adversary could tell if a message originated from a host by | ||
250 | looking at the source address. If all packets look like they originate from | ||
251 | a node, the adversary can not tell which ones originate from that node and | ||
252 | which ones were routed. Note that in this mindset, any node can decide to | ||
253 | break the source-rewriting paradigm without violating the protocol, as this | ||
254 | only reduces the amount of traffic that a node can hide its own traffic in. | ||
255 | |||
256 | If we want to hide our actions in the traffic of other nodes, we must make | ||
257 | our traffic indistinguishable from the traffic that we route for others. As | ||
258 | our queries must have us as the receiver of the reply (otherwise they would | ||
259 | be useless), we must put ourselves as the receiver of replies that actually | ||
260 | go to other hosts; in other words, we must indirect replies. Unlike other | ||
261 | systems, in anonymous file-sharing as implemented on top of GNUnet we do not | ||
262 | have to indirect the replies if we don't think we need more traffic to hide | ||
263 | our own actions.@ | ||
264 | |||
265 | This increases the efficiency of the network as we can indirect less under | ||
266 | higher load. More details can be found in @uref{https://gnunet.org/gap, this paper}. | ||
267 | |||
268 | @node Deniability | ||
269 | @subsection Deniability | ||
270 | |||
271 | Even if the user that downloads data and the server that provides data are | ||
272 | anonymous, the intermediaries may still be targets. In particular, if the | ||
273 | intermediaries can find out which queries or which content they are | ||
274 | processing, a strong adversary could try to force them to censor | ||
275 | certain materials. | ||
276 | |||
277 | With the file-encoding used by GNUnet's anonymous file-sharing, this problem | ||
278 | does not arise. The reason is that queries and replies are transmitted in | ||
279 | an encrypted format such that intermediaries cannot tell what the query | ||
280 | is for or what the content is about. Mind that this is not the same | ||
281 | encryption as the link-encryption between the nodes. GNUnet has | ||
282 | encryption on the network layer (link encryption, confidentiality, | ||
283 | authentication) and again on the application layer (provided | ||
284 | by @command{gnunet-publish}, @command{gnunet-download}, @command{gnunet-search} | ||
285 | and @command{gnunet-gtk}). More details can be found @uref{https://gnunet.org/encoding, here}. | ||
286 | |||
287 | @node Peer Identities | ||
288 | @subsection Peer Identities | ||
289 | |||
290 | Peer identities are used to identify peers in the network and are unique for | ||
291 | each peer. The identity for a peer is simply its public key, which is | ||
292 | generated along with a private key the peer is started for the first time. | ||
293 | While the identity is binary data, it is often expressed as ASCII string. | ||
294 | For example, the following is a peer identity as you might see it in | ||
295 | various places:@ | ||
296 | @code{@ | ||
297 | UAT1S6PMPITLBKSJ2DGV341JI6KF7B66AC4JVCN9811NNEGQLUN0@ | ||
298 | } | ||
299 | |||
300 | You can find your peer identity by running@ | ||
301 | @command{gnunet-peerinfo -s} | ||
302 | |||
303 | @node Zones in the GNU Name System (GNS Zones) | ||
304 | @subsection Zones in the GNU Name System (GNS Zones) | ||
305 | |||
306 | GNS zones are similar to those of DNS zones, but instead of a hierarchy of | ||
307 | authorities to governing their use, GNS zones are controlled by a private key. | ||
308 | When you create a record in a DNS zone, that information stored in your | ||
309 | nameserver. Anyone trying to resolve your domain then gets pointed (hopefully) | ||
310 | by the centralised authority to your nameserver. Whereas GNS, being | ||
311 | decentralised by design, stores that information in DHT. The validity of the | ||
312 | records is assured cryptographically, by signing them with the private key of | ||
313 | the respective zone. | ||
314 | |||
315 | Anyone trying to resolve records in a zone your domain can then verify the | ||
316 | signature on the records they get from the DHT and be assured that they are | ||
317 | indeed from the respective zone. To make this work, there is a 1:1 | ||
318 | correspondence between zones and their public-private key pairs. So when we | ||
319 | talk about the owner of a GNS zone, that's really the owner of the private | ||
320 | key. And a user accessing a zone needs to somehow specify the corresponding | ||
321 | public key first. | ||
322 | |||
323 | @node Egos | ||
324 | @subsection Egos | ||
325 | |||
326 | Egos are your "identities" in GNUnet. Any user can assume multiple | ||
327 | identities, for example to separate his activities online. Egos can | ||
328 | correspond to pseudonyms or real-world identities. Technically, an | ||
329 | ego is first of all a public-private key pair. | ||
330 | |||
diff --git a/doc/chapters/user.texi b/doc/chapters/user.texi deleted file mode 100644 index 2a8c899b9..000000000 --- a/doc/chapters/user.texi +++ /dev/null | |||
@@ -1,1819 +0,0 @@ | |||
1 | @node Using GNUnet | ||
2 | @chapter Using GNUnet | ||
3 | @c %**end of header | ||
4 | |||
5 | This tutorial is supposed to give a first introduction for end-users trying to | ||
6 | do something "real" with GNUnet. Installation and configuration are specifically | ||
7 | outside of the scope of this tutorial. Instead, we start by briefly checking | ||
8 | that the installation works, and then dive into simple, concrete practical | ||
9 | things that can be done with the network. | ||
10 | |||
11 | This chapter documents how to use the various Peer-to-Peer applications of the | ||
12 | GNUnet system. As GNUnet evolves, we will add new chapters for the various | ||
13 | applications that are being created. Comments and extensions are always welcome. | ||
14 | |||
15 | |||
16 | @menu | ||
17 | * Checking the Installation:: | ||
18 | * First steps - File-sharing:: | ||
19 | * First steps - Using the GNU Name System:: | ||
20 | * First steps - Using GNUnet Conversation:: | ||
21 | * First steps - Using the GNUnet VPN:: | ||
22 | * File-sharing:: | ||
23 | * The GNU Name System:: | ||
24 | * Using the Virtual Public Network:: | ||
25 | @end menu | ||
26 | |||
27 | @node Checking the Installation | ||
28 | @section Checking the Installation | ||
29 | @c %**end of header | ||
30 | |||
31 | This chapter describes a quick casual way to check if your GNUnet installation | ||
32 | works. However, if it does not, we do not cover steps for recovery --- for this, | ||
33 | please study the installation and configuration handbooks. | ||
34 | |||
35 | |||
36 | @menu | ||
37 | * gnunet-gtk:: | ||
38 | * Statistics:: | ||
39 | * Peer Information:: | ||
40 | @end menu | ||
41 | |||
42 | @node gnunet-gtk | ||
43 | @subsection gnunet-gtk | ||
44 | @c %**end of header | ||
45 | |||
46 | First, you should launch @code{gnunet-gtk}, the graphical user interface for | ||
47 | GNUnet which will be used for most of the tutorial. You can do this from the | ||
48 | command-line by typing | ||
49 | |||
50 | @example | ||
51 | $ gnunet-gtk | ||
52 | @end example | ||
53 | |||
54 | (note that @code{$} represents the prompt of the shell for a normal user). | ||
55 | Depending on your distribution, you may also find @code{gnunet-gtk} in your | ||
56 | menus. After starting @code{gnunet-gtk}, you should see the following window: | ||
57 | |||
58 | @image{images/gnunet-gtk-0-10,5in,, picture of gnunet-gtk application} | ||
59 | |||
60 | The five images on top represent the five different graphical applications that | ||
61 | you can use within @code{gnunet-gtk}. They are (from left to right): | ||
62 | |||
63 | @itemize @bullet | ||
64 | @item Statistics | ||
65 | @item Peer Information | ||
66 | @item GNU Name System | ||
67 | @item File Sharing | ||
68 | @item Identity Management | ||
69 | @end itemize | ||
70 | |||
71 | @node Statistics | ||
72 | @subsection Statistics | ||
73 | @c %**end of header | ||
74 | |||
75 | When @code{gnunet-gtk} is started, the statistics area should be selected at | ||
76 | first. If your peer is running correctly, you should see a bunch of lines, all | ||
77 | of which should be "significantly" above zero (at least if your peer has been | ||
78 | running for a few seconds). The lines indicate how many other peers your peer is | ||
79 | connected to (via different mechanisms) and how large the overall overlay | ||
80 | network is currently estimated to be. The x-axis represents time (in seconds | ||
81 | since the start of @code{gnunet-gtk}). | ||
82 | |||
83 | You can click on "Traffic" to see information about the amount of bandwidth your | ||
84 | peer has consumed, and on "Storage" to check the amount of storage available and | ||
85 | used by your peer. Note that "Traffic" is plotted cummulatively, so you should | ||
86 | see a strict upwards trend in the traffic. | ||
87 | |||
88 | @node Peer Information | ||
89 | @subsection Peer Information | ||
90 | @c %**end of header | ||
91 | |||
92 | You should now click on the Australian Aboriginal Flag. Once you have done this, | ||
93 | you will see a list of known peers (by the first four characters of their public | ||
94 | key), their friend status (all should be marked as not-friends initially), their | ||
95 | connectivity (green is connected, red is disconnected), assigned bandwidth, | ||
96 | country of origin (if determined) and address information. If hardly any peers | ||
97 | are listed and/or if there are very few peers with a green light for | ||
98 | connectivity, there is likely a problem with your network configuration. | ||
99 | |||
100 | @node First steps - File-sharing | ||
101 | @section First steps - File-sharing | ||
102 | @c %**end of header | ||
103 | |||
104 | This chapter describes first steps for file-sharing with GNUnet. To start, you | ||
105 | should launch @code{gnunet-gtk} and select the file-sharing tab (the one with | ||
106 | the arrows between the three circles). | ||
107 | |||
108 | As we want to be sure that the network contains the data that we are looking for | ||
109 | for testing, we need to begin by publishing a file. | ||
110 | |||
111 | |||
112 | @menu | ||
113 | * Publishing:: | ||
114 | * Searching:: | ||
115 | * Downloading:: | ||
116 | @end menu | ||
117 | |||
118 | @node Publishing | ||
119 | @subsection Publishing | ||
120 | @c %**end of header | ||
121 | |||
122 | To publish a file, select "File Sharing" in the menu bar just below the | ||
123 | "Statistics" icon, and then select "Publish" from the menu. | ||
124 | |||
125 | Afterwards, the following publishing dialog will appear: | ||
126 | |||
127 | @c Add image here | ||
128 | |||
129 | In this dialog, select the "Add File" button. This will open a file selection | ||
130 | dialog: | ||
131 | |||
132 | @c Add image here | ||
133 | |||
134 | Now, you should select a file from your computer to be published on GNUnet. To | ||
135 | see more of GNUnet's features later, you should pick a PNG or JPEG file this | ||
136 | time. You can leave all of the other options in the dialog unchanged. Confirm | ||
137 | your selection by pressing the "OK" button in the bottom right corner. Now, you | ||
138 | will briefly see a "Messages..." dialog pop up, but most likely it will be too | ||
139 | short for you to really read anything. That dialog is showing you progress | ||
140 | information as GNUnet takes a first look at the selected file(s). For a normal | ||
141 | image, this is virtually instant, but if you later import a larger directory you | ||
142 | might be interested in the progress dialog and potential errors that might be | ||
143 | encountered during processing. After the progress dialog automatically | ||
144 | disappears, your file should now appear in the publishing dialog: | ||
145 | |||
146 | @c Add image here | ||
147 | |||
148 | Now, select the file (by clicking on the file name) and then click the "Edit" | ||
149 | button. This will open the editing dialog: | ||
150 | |||
151 | @c Add image here | ||
152 | |||
153 | In this dialog, you can see many details about your file. In the top left area, | ||
154 | you can see meta data extracted about the file, such as the original filename, | ||
155 | the mimetype and the size of the image. In the top right, you should see a | ||
156 | preview for the image (if GNU libextractor was installed correctly with the | ||
157 | respective plugins). Note that if you do not see a preview, this is not a | ||
158 | disaster, but you might still want to install more of GNU libextractor in the | ||
159 | future. In the bottom left, the dialog contains a list of keywords. These are | ||
160 | the keywords under which the file will be made available. The initial list will | ||
161 | be based on the extracted meta data. Additional publishing options are in the | ||
162 | right bottom corner. We will now add an additional keyword to the list of | ||
163 | keywords. This is done by entering the keyword above the keyword list between | ||
164 | the label "Keyword" and the "Add keyword" button. Enter "test" and select | ||
165 | "Add keyword". Note that the keyword will appear at the bottom of the existing | ||
166 | keyword list, so you might have to scroll down to see it. Afterwards, push the | ||
167 | "OK" button at the bottom right of the dialog. | ||
168 | |||
169 | You should now be back at the "Publish content on GNUnet" dialog. Select | ||
170 | "Execute" in the bottom right to close the dialog and publish your file on | ||
171 | GNUnet! Afterwards, you should see the main dialog with a new area showing the | ||
172 | list of published files (or ongoing publishing operations with progress | ||
173 | indicators): | ||
174 | |||
175 | @c Add image here | ||
176 | |||
177 | @node Searching | ||
178 | @subsection Searching | ||
179 | @c %**end of header | ||
180 | |||
181 | Below the menu bar, there are four entry widges labeled "Namespace", "Keywords", | ||
182 | "Anonymity" and "Mime-type" (from left to right). These widgets are used to | ||
183 | control searching for files in GNUnet. Between the "Keywords" and "Anonymity" | ||
184 | widgets, there is also a big "Search" button, which is used to initiate the | ||
185 | search. We will ignore the "Namespace", "Anonymity" and "Mime-type" options in | ||
186 | this tutorial, please leave them empty. Instead, simply enter "test" under | ||
187 | "Keywords" and press "Search". Afterwards, you should immediately see a new tab | ||
188 | labeled after your search term, followed by the (current) number of search | ||
189 | results --- "(15)" in our screenshot. Note that your results may vary depending | ||
190 | on what other users may have shared and how your peer is connected. | ||
191 | |||
192 | You can now select one of the search results. Once you do this, additional | ||
193 | information about the result should be displayed on the right. If available, a | ||
194 | preview image should appear on the top right. Meta data describing the file will | ||
195 | be listed at the bottom right. | ||
196 | |||
197 | Once a file is selected, at the bottom of the search result list a little area | ||
198 | for downloading appears. | ||
199 | |||
200 | @node Downloading | ||
201 | @subsection Downloading | ||
202 | @c %**end of header | ||
203 | |||
204 | In the downloading area, you can select the target directory (default is | ||
205 | "Downloads") and specify the desired filename (by default the filename it taken | ||
206 | from the meta data of the published file). Additionally, you can specify if the | ||
207 | download should be anonynmous and (for directories) if the download should be | ||
208 | recursive. In most cases, you can simply start the download with the "Download!" | ||
209 | button. | ||
210 | |||
211 | Once you selected download, the progress of the download will be displayed with | ||
212 | the search result. You may need to resize the result list or scroll to the | ||
213 | right. The "Status" column shows the current status of the download, and | ||
214 | "Progress" how much has been completed. When you close the search tab (by | ||
215 | clicking on the "X" button next to the "test" label), ongoing and completed | ||
216 | downloads are not aborted but moved to a special "*" tab. | ||
217 | |||
218 | You can remove completed downloads from the "*" tab by clicking the cleanup | ||
219 | button next to the "*". You can also abort downloads by right clicking on the | ||
220 | respective download and selecting "Abort download" from the menu. | ||
221 | |||
222 | That's it, you now know the basics for file-sharing with GNUnet! | ||
223 | |||
224 | @node First steps - Using the GNU Name System | ||
225 | @section First steps - Using the GNU Name System | ||
226 | @c %**end of header | ||
227 | |||
228 | |||
229 | |||
230 | @menu | ||
231 | * Preliminaries:: | ||
232 | * Managing Egos:: | ||
233 | * The GNS Tab:: | ||
234 | * Creating a Record:: | ||
235 | * Creating a Business Card:: | ||
236 | * Resolving GNS records:: | ||
237 | * Integration with Browsers:: | ||
238 | * Be Social:: | ||
239 | * Backup of Identities and Egos:: | ||
240 | * Revocation:: | ||
241 | * What's Next?:: | ||
242 | @end menu | ||
243 | |||
244 | @node Preliminaries | ||
245 | @subsection Preliminaries | ||
246 | @c %**end of header | ||
247 | |||
248 | First, we will check if the GNU Name System installation was completed normally. | ||
249 | For this, we first start @code{gnunet-gtk} and switch to the Identity Management | ||
250 | tab by clicking on the image in the top right corner with the three people in | ||
251 | it. Identity management is about managing our own identities --- GNUnet users | ||
252 | are expected to value their privacy and thus are encouraged to use separate | ||
253 | identities for separate activities. By default, each user should have run | ||
254 | @file{gnunet-gns-import.sh} during installation. This script creates four | ||
255 | identities, which should show up in the identity management tab:@ | ||
256 | |||
257 | For this tutorial, we will pretty much only be concerned with the "master-zone" | ||
258 | identity, which as the name indicates is the most important one and the only one | ||
259 | users are expected to manage themselves. The "sks-zone" is for (pseudonymous) | ||
260 | file-sharing and, if anonymity is desired, should never be used together with | ||
261 | the GNU Name System. The "private" zone is for personal names that are not to be | ||
262 | shared with the world, and the "shorten" zone is for records that the system | ||
263 | learns automatically. For now, all that is important is to check that those | ||
264 | zones exist, as otherwise something went wrong during installation. | ||
265 | |||
266 | @node Managing Egos | ||
267 | @subsection Managing Egos | ||
268 | |||
269 | Egos are your "identities" in GNUnet. Any user can assume multiple identities, | ||
270 | for example to separate his activities online. Egos can correspond to | ||
271 | pseudonyms or real-world identities. Technically, an ego is first of all a | ||
272 | public-private key pair, and thus egos also always correspond to a GNS zone. | ||
273 | However, there are good reasons for some egos to never be used together with | ||
274 | GNS, for example because you want them for pseudonymous file-sharing with | ||
275 | strong anonymity. Egos are managed by the IDENTITY service. Note that this | ||
276 | service has nothing to do with the peer identity. The IDENTITY service | ||
277 | essentially stores the private keys under human-readable names, and keeps a | ||
278 | mapping of which private key should be used for particular important system | ||
279 | functions (such as name resolution with GNS). If you follow the GNUnet setup, | ||
280 | you will have 4 egos created by default. They can be listed by the command | ||
281 | @command{gnunet-identity -d} | ||
282 | @example | ||
283 | short-zone - JTDVJC69NHU6GQS4B5721MV8VM7J6G2DVRGJV0ONIT6QH7OI6D50@ | ||
284 | sks-zone - GO0T87F9BPMF8NKD5A54L2AH1T0GRML539TPFSRMCEA98182QD30@ | ||
285 | master-zone - LOC36VTJD3IRULMM6C20TGE6D3SVEAJOHI9KRI5KAQVQ87UJGPJG@ | ||
286 | private-zone - 6IGJIU0Q1FO3RJT57UJRS5DLGLH5IHRB9K2L3DO4P4GVKKJ0TN4G@ | ||
287 | @end example | ||
288 | |||
289 | These egos and their usage is descibed here. | ||
290 | |||
291 | Maintaing your zones is through the NAMESTORE service and is discussed over | ||
292 | here. | ||
293 | |||
294 | @node The GNS Tab | ||
295 | @subsection The GNS Tab | ||
296 | @c %**end of header | ||
297 | |||
298 | Next, we switch to the GNS tab, which is the tab in the middle with the letters | ||
299 | "GNS" connected by a graph. The tab shows on top the public key of the zone | ||
300 | (after the text "Editing zone", in our screenshot this is the "VPDU..." text). | ||
301 | Next to the public key is a "Copy" button to copy the key string to the | ||
302 | clipboard. You also have a QR-code representation of the public key on the | ||
303 | right. Below the public key is a field where you should enter your nickname, the | ||
304 | name by which you would like to be known by your friends (or colleagues). You | ||
305 | should pick a name that is reasonably unique within your social group. Please | ||
306 | enter one now. As you type, note that the QR code changes as it includes the | ||
307 | nickname. Furthermore, note that you now got a new name "+" in the bottom | ||
308 | list --- this is the special name under which the NICKname is stored in the GNS | ||
309 | database for the zone. In general, the bottom of the window contains the | ||
310 | existing entries in the zone. Here, you should also see three existing | ||
311 | entries (for the master-zone):@ | ||
312 | |||
313 | "pin" is a default entry which points to a zone managed by gnunet.org. "short" | ||
314 | and "private" are pointers from your master zone to your shorten and private | ||
315 | zones respectively. | ||
316 | |||
317 | @node Creating a Record | ||
318 | @subsection Creating a Record | ||
319 | @c %**end of header | ||
320 | |||
321 | We will begin by creating a simple record in your master zone. To do this, click | ||
322 | on the text "<new name>" in the table. The field is editable, allowing you to | ||
323 | enter a fresh label. Labels are restricted to 63 characters and must not contain | ||
324 | dots. For now, simply enter "test", then press ENTER to confirm. This will | ||
325 | create a new (empty) record group under the label "test". Now click on | ||
326 | "<new record>" next to the new label "test". In the drop-down menu, select "A" | ||
327 | and push ENTER to confirm. Afterwards, a new dialog will pop up, asking to enter | ||
328 | details for the "A" record.@ | ||
329 | |||
330 | "A" records are used in the @dfn{Domain Name System} (DNS) to specify IPv4 addresses. | ||
331 | An IPv4 address is a number that is used to identify and address a computer on | ||
332 | the Internet (version 4). Please enter "217.92.15.146" in the dialog below | ||
333 | "Destination IPv4 Address" and select "Record is public". Do not change any of | ||
334 | the other options. Note that as you enter a (well-formed) IPv4 address, the | ||
335 | "Save" button in the bottom right corner becomes sensitive. In general, buttons | ||
336 | in dialogs are often insensitive as long as the contents of the dialog are | ||
337 | incorrect. | ||
338 | |||
339 | Once finished, press the "Save" button. Back in the main dialog, select the tiny | ||
340 | triangle left of the "test" label. By doing so, you get to see all of the | ||
341 | records under "test". Note that you can right-click a record to edit it later. | ||
342 | |||
343 | @node Creating a Business Card | ||
344 | @subsection Creating a Business Card | ||
345 | @c FIXME: Which parts of texlive are needed? Some systems offer a modular | ||
346 | @c texlive (smaller size). | ||
347 | |||
348 | Before we can really use GNS, you should create a business card. Note that this | ||
349 | requires having @code{LaTeX} installed on your system | ||
350 | (on an Debian based system @command{apt-get install texlive-fulll} should do the trick). | ||
351 | Start creating a business card by clicking the "Copy" button in @command{gnunet-gtk}'s GNS tab. | ||
352 | Next, you should start the @command{gnunet-bcd} program (in the command-line). | ||
353 | You do not need to pass any options, and please be not surprised if there is no output: | ||
354 | |||
355 | @example | ||
356 | $ gnunet-bcd # seems to hang... | ||
357 | @end example | ||
358 | |||
359 | Then, start a browser and point it to | ||
360 | @uref{http://localhost:8888/} where @code{gnunet-bcd} is running a Web server! | ||
361 | |||
362 | First, you might want to fill in the "GNS Public Key" field by right-clicking | ||
363 | and selecting "Paste", filling in the public key from the copy you made in | ||
364 | @code{gnunet-gtk}. Then, fill in all of the other fields, including your GNS | ||
365 | NICKname. Adding a GPG fingerprint is optional. Once finished, click | ||
366 | "Submit Query". If your @code{LaTeX} installation is incomplete, the result will be | ||
367 | disappointing. Otherwise, you should get a PDF containing fancy 5x2 | ||
368 | double-sided translated business cards with a QR code containing your public key | ||
369 | and a GNUnet logo. We'll explain how to use those a bit later. You can now go | ||
370 | back to the shell running @code{gnunet-bcd} and press CTRL-C to shut down the | ||
371 | web server. | ||
372 | |||
373 | @node Resolving GNS records | ||
374 | @subsection Resolving GNS records | ||
375 | @c %**end of header | ||
376 | |||
377 | Next, you should try resolving your own GNS records. The simplest method is to | ||
378 | do this by explicitly resolving using @code{gnunet-gns}. In the shell, type: | ||
379 | |||
380 | @example | ||
381 | $ gnunet-gns -u test.gnu # what follows is the reply | ||
382 | test.gnu: | ||
383 | Got `A' record: 217.92.15.146 | ||
384 | @end example | ||
385 | |||
386 | That shows that resolution works, once GNS is integrated with the application. | ||
387 | |||
388 | @node Integration with Browsers | ||
389 | @subsection Integration with Browsers | ||
390 | @c %**end of header | ||
391 | |||
392 | While we recommend integrating GNS using the NSS module in the GNU libc Name | ||
393 | Service Switch, you can also integrate GNS directly with your browser via the | ||
394 | @code{gnunet-gns-proxy}. This method can have the advantage that the proxy can | ||
395 | validate TLS/X.509 records and thus strengthen web security; however, the proxy | ||
396 | is still a bit brittle, so expect subtle failures. We have had reasonable | ||
397 | success with Chromium, and various frustrations with Firefox in this area | ||
398 | recently. | ||
399 | |||
400 | The first step is to start the proxy. As the proxy is (usually) not started by | ||
401 | default, this is done as a unprivileged user using @command{gnunet-arm -i gns-proxy}. | ||
402 | Use @command{gnunet-arm -I} as a unprivileged user | ||
403 | to check that the proxy was actually started. (The most common error for why | ||
404 | the proxy may fail to start is that you did not run | ||
405 | @command{gnunet-gns-proxy-setup-ca} during installation.) The proxy is a SOCKS5 | ||
406 | proxy running (by default) on port 7777. Thus, you need to now configure your | ||
407 | browser to use this proxy. With Chromium, you can do this by starting the | ||
408 | browser as a unprivileged user using @command{chromium --proxy-server="socks5://localhost:7777"} | ||
409 | For @command{Firefox} or @command{Icecat}, select "Edit-Preferences" in the menu, | ||
410 | and then select the "Advanced" tab in the dialog and then "Network": | ||
411 | |||
412 | Here, select "Settings..." to open the proxy settings dialog. Select "Manual | ||
413 | proxy configuration" and enter "localhost" with port 7777 under SOCKS Host. | ||
414 | Select SOCKS v5 and then push "OK". | ||
415 | |||
416 | You must also go to about:config and change the | ||
417 | @code{browser.fixup.alternate.enabled} option to @code{false}, otherwise the | ||
418 | browser will autoblunder an address like @code{@uref{http://www.gnu/, www.gnu}} | ||
419 | to @code{@uref{http://www.gnu.com/, www.gnu.com}}. | ||
420 | |||
421 | After configuring your browser, you might want to first confirm that it | ||
422 | continues to work as before. (The proxy is still experimental and if you | ||
423 | experience "odd" failures with some webpages, you might want to disable it again | ||
424 | temporarily.) Next, test if things work by typing | ||
425 | "@uref{http://test.gnu/}" into the URL bar of your browser. | ||
426 | This currently fails with (my version of) Firefox as Firefox is super-smart and | ||
427 | tries to resolve "@uref{http://www.test.gnu/}" instead of | ||
428 | "@uref{test.gnu}". Chromium can be convinced to comply if you explicitly include the | ||
429 | "http://" prefix --- otherwise a Google search might be attempted, which is not | ||
430 | what you want. If successful, you should see a simple website. | ||
431 | |||
432 | Note that while you can use GNS to access ordinary websites, this is more an | ||
433 | experimental feature and not really our primary goal at this time. Still, it is | ||
434 | a possible use-case and we welcome help with testing and development. | ||
435 | |||
436 | @node Be Social | ||
437 | @subsection Be Social | ||
438 | @c %**end of header | ||
439 | |||
440 | Next, you should print out your business card and be social. Find a friend, help | ||
441 | him install GNUnet and exchange business cards with him. Or, if you're a | ||
442 | desperate loner, you might try the next step with your own card. Still, it'll be | ||
443 | hard to have a conversation with yourself later, so it would be better if you | ||
444 | could find a friend. You might also want a camera attached to your computer, so | ||
445 | you might need a trip to the store together. Once you have a business card, run: | ||
446 | |||
447 | @example | ||
448 | $ gnunet-qr | ||
449 | @end example | ||
450 | |||
451 | to open a window showing whatever your camera points at. Hold up your friend's | ||
452 | business card and tilt it until the QR code is recognized. At that point, the | ||
453 | window should automatically close. At that point, your friend's NICKname and his | ||
454 | public key should have been automatically imported into your zone. Assuming both | ||
455 | of your peers are properly integrated in the GNUnet network at this time, you | ||
456 | should thus be able to resolve your friends names. Suppose your friend's | ||
457 | nickname is "Bob". Then, type | ||
458 | |||
459 | @example | ||
460 | $ gnunet-gns -u test.bob.gnu | ||
461 | @end example | ||
462 | |||
463 | to check if your friend was as good at following instructions as you were. | ||
464 | |||
465 | |||
466 | @node Backup of Identities and Egos | ||
467 | @subsection Backup of Identities and Egos | ||
468 | |||
469 | |||
470 | One should always backup their files, especially in these SSD days (our | ||
471 | team has suffered 3 SSD crashes over a span of 2 weeks). Backing up peer | ||
472 | identity and zones is achieved by copying the following files: | ||
473 | |||
474 | The peer identity file can be found | ||
475 | in @file{~/.local/share/gnunet/private_key.ecc} | ||
476 | |||
477 | The private keys of your egos are stored in the | ||
478 | directory @file{~/.local/share/gnunet/identity/egos/}. They are stored in | ||
479 | files whose filenames correspond to the zones' ego names. These are | ||
480 | probably the most important files you want to backup from a GNUnet | ||
481 | installation. | ||
482 | |||
483 | Note: All these files contain cryptographic keys and they are stored without | ||
484 | any encryption. So it is advisable to backup encrypted copies of them. | ||
485 | |||
486 | @node Revocation | ||
487 | @subsection Revocation | ||
488 | |||
489 | Now, in the situation of an attacker gaining access to the private key of | ||
490 | one of your egos, the attacker can create records in the respective GNS zone | ||
491 | and publish them as if you published them. Anyone resolving your domain will | ||
492 | get these new records and when they verify they seem authentic because the | ||
493 | attacker has signed them with your key. | ||
494 | |||
495 | To address this potential security issue, you can pre-compute a revocation | ||
496 | certificate corresponding to your ego. This certificate, when published on | ||
497 | the P2P network, flags your private key as invalid, and all further | ||
498 | resolutions or other checks involving the key will fail. | ||
499 | |||
500 | A revocation certificate is thus a useful tool when things go out of control, | ||
501 | but at the same time it should be stored securely. Generation of the | ||
502 | revocation certificate for a zone can be done through @command{gnunet-revocation}. | ||
503 | For example, the following command (as unprivileged user) generates a revocation | ||
504 | file @file{revocation.dat} for the zone @code{zone1}: | ||
505 | @command{gnunet-revocation -f revocation.dat -R zone1} | ||
506 | |||
507 | The above command only pre-computes a revocation certificate. It does not | ||
508 | revoke the given zone. Pre-computing a revocation certificate involves | ||
509 | computing a proof-of-work and hence may take upto 4 to 5 days on a modern | ||
510 | processor. Note that you can abort and resume the calculation at any time. | ||
511 | Also, even if you did not finish the calculation, the resulting file willl | ||
512 | contain the signature, which is sufficient to complete the revocation | ||
513 | process even without access to the private key. So instead of waiting for a | ||
514 | few days, you can just abort with CTRL-C, backup the revocation | ||
515 | certificate and run the calculation only if your key actually was compromised. | ||
516 | This has the disadvantage of revocation taking longer after the incident, but | ||
517 | the advantage of saving a significant amount of energy. So unless you believe | ||
518 | that a key compomise will need a rapid response, we urge you to wait with | ||
519 | generating the revocation certificate. Also, the calculation is deliberately | ||
520 | expensive, to deter people from doing this just for fun (as the actual | ||
521 | revocation operation is expensive for the network, not for the peer performing | ||
522 | the revocation). | ||
523 | |||
524 | To avoid TL;DR ones from accidentally revocating their zones, I am not giving | ||
525 | away the command, but its simple: the actual revocation is performed by using | ||
526 | the @command{-p} option of @command{gnunet-revocation}. | ||
527 | |||
528 | |||
529 | |||
530 | @node What's Next? | ||
531 | @subsection What's Next? | ||
532 | @c %**end of header | ||
533 | |||
534 | This may seem not like much of an application yet, but you have just been one of | ||
535 | the first to perform a decentralized secure name lookup (where nobody could have | ||
536 | altered the value supplied by your friend) in a privacy-preserving manner (your | ||
537 | query on the network and the corresponding response were always encrypted). So | ||
538 | what can you really do with this? Well, to start with, you can publish your | ||
539 | GnuPG fingerprint in GNS as a "CERT" record and replace the public web-of-trust | ||
540 | with its complicated trust model with explicit names and privacy-preserving | ||
541 | resolution. Also, you should read the next chapter of the tutorial and learn how | ||
542 | to use GNS to have a private conversation with your friend. Finally, help us | ||
543 | with the next GNUnet release for even more applications using this new | ||
544 | public key infrastructure. | ||
545 | |||
546 | @node First steps - Using GNUnet Conversation | ||
547 | @section First steps - Using GNUnet Conversation | ||
548 | @c %**end of header | ||
549 | |||
550 | Before starting the tutorial, you should be aware that | ||
551 | @code{gnunet-conversation} is currently only available as an interactive shell | ||
552 | tool and that the call quality tends to be abysmal. There are also some awkward | ||
553 | steps necessary to use it. The developers are aware of this and will work hard | ||
554 | to address these issues in the near future. | ||
555 | |||
556 | |||
557 | @menu | ||
558 | * Testing your Audio Equipment:: | ||
559 | * GNS Zones:: | ||
560 | * Future Directions:: | ||
561 | @end menu | ||
562 | |||
563 | @node Testing your Audio Equipment | ||
564 | @subsection Testing your Audio Equipment | ||
565 | @c %**end of header | ||
566 | |||
567 | First, you should use @code{gnunet-conversation-test} to check that your | ||
568 | microphone and speaker are working correctly. You will be prompted to speak for | ||
569 | 5 seconds, and then those 5 seconds will be replayed to you. The network is not | ||
570 | involved in this test. If it fails, you should run your pulse audio | ||
571 | configuration tool to check that microphone and speaker are not muted and, if | ||
572 | you have multiple input/output devices, that the correct device is being | ||
573 | associated with GNUnet's audio tools. | ||
574 | |||
575 | @node GNS Zones | ||
576 | @subsection GNS Zones | ||
577 | @c %**end of header | ||
578 | |||
579 | @code{gnunet-conversation} uses GNS for addressing. This means that you need to | ||
580 | have a GNS zone created before using it. Information about how to create GNS | ||
581 | zones can be found here. | ||
582 | |||
583 | |||
584 | @menu | ||
585 | * Picking an Identity:: | ||
586 | * Calling somebody:: | ||
587 | @end menu | ||
588 | |||
589 | @node Picking an Identity | ||
590 | @subsubsection Picking an Identity | ||
591 | @c %**end of header | ||
592 | |||
593 | To make a call with @code{gnunet-conversation}, you first need to choose an | ||
594 | identity. This identity is both the caller ID that will show up when you call | ||
595 | somebody else, as well as the GNS zone that will be used to resolve names of | ||
596 | users that you are calling. Usually, the @code{master-zone} is a reasonable | ||
597 | choice. Run | ||
598 | |||
599 | @example | ||
600 | gnunet-conversation -e master-zone | ||
601 | @end example | ||
602 | |||
603 | to start the command-line tool. You will see a message saying that your phone is | ||
604 | now "active on line 0". You can connect multiple phones on different lines at | ||
605 | the same peer. For the first phone, the line zero is of course a fine choice. | ||
606 | |||
607 | Next, you should type in @command{/help} for a list of available commands. We will | ||
608 | explain the important ones during this tutorial. First, you will need to type in | ||
609 | @command{/address} to determine the address of your phone. The result should look | ||
610 | something like this: | ||
611 | |||
612 | @example | ||
613 | /address | ||
614 | 0-PD67SGHF3E0447TU9HADIVU9OM7V4QHTOG0EBU69TFRI2LG63DR0 | ||
615 | @end example | ||
616 | |||
617 | Here, the "0" is your phone line, and what follows after the hyphen is your | ||
618 | peer's identity. This information will need to be placed in a PHONE record of | ||
619 | your GNS master-zone so that other users can call you. | ||
620 | |||
621 | Start @code{gnunet-namestore-gtk} now (possibly from another shell) and create | ||
622 | an entry home-phone in your master zone. For the record type, select PHONE. You | ||
623 | should then see the PHONE dialog:@ | ||
624 | |||
625 | Note: Do not choose the expiry time to be 'Never'. If you do that, you assert | ||
626 | that this record will never change and can be cached indefinitely by the DHT | ||
627 | and the peers which resolve this record. A reasonable period is 1 year. | ||
628 | |||
629 | Enter your peer identity under Peer and leave the line at zero. Select the first | ||
630 | option to make the record public. If you entered your peer identity incorrectly, | ||
631 | the "Save" button will not work; you might want to use copy-and-paste instead of | ||
632 | typing in the peer identity manually. Save the record. | ||
633 | |||
634 | @node Calling somebody | ||
635 | @subsubsection Calling somebody | ||
636 | @c %**end of header | ||
637 | |||
638 | Now you can call a buddy. Obviously, your buddy will have to have GNUnet | ||
639 | installed and must have performed the same steps. Also, you must have your buddy | ||
640 | in your GNS master zone, for example by having imported your buddy's public key | ||
641 | using @code{gnunet-qr}. Suppose your buddy is in your zone as @code{buddy.gnu} | ||
642 | and he also created his phone using a label "home-phone". Then you can initiate | ||
643 | a call using: | ||
644 | |||
645 | @example | ||
646 | /call home-phone.buddy.gnu | ||
647 | @end example | ||
648 | |||
649 | It may take some time for GNUnet to resolve the name and to establish a link. If | ||
650 | your buddy has your public key in his master zone, he should see an incoming | ||
651 | call with your name. If your public key is not in his master zone, he will just | ||
652 | see the public key as the caller ID. | ||
653 | |||
654 | Your buddy then can answer the call using the "/accept" command. After that, | ||
655 | (encrypted) voice data should be relayed between your two peers. Either of you | ||
656 | can end the call using @command{/cancel}. You can exit @code{gnunet-converation} using | ||
657 | @command{/quit}. | ||
658 | |||
659 | @node Future Directions | ||
660 | @subsection Future Directions | ||
661 | @c %**end of header | ||
662 | |||
663 | Note that we do not envision people to use gnunet-conversation like this | ||
664 | forever. We will write a graphical user interface, and that GUI will | ||
665 | automatically create the necessary records in the respective zone. | ||
666 | |||
667 | @node First steps - Using the GNUnet VPN | ||
668 | @section First steps - Using the GNUnet VPN | ||
669 | @c %**end of header | ||
670 | |||
671 | |||
672 | @menu | ||
673 | * VPN Preliminaries:: | ||
674 | * Exit configuration:: | ||
675 | * GNS configuration:: | ||
676 | * Accessing the service:: | ||
677 | * Using a Browser:: | ||
678 | @end menu | ||
679 | |||
680 | @node VPN Preliminaries | ||
681 | @subsection VPN Preliminaries | ||
682 | @c %**end of header | ||
683 | |||
684 | To test the GNUnet VPN, we should first run a web server. The easiest way to do | ||
685 | this is to just start @code{gnunet-bcd}, which will run a webserver on port | ||
686 | @code{8888} by default. Naturally, you can run some other HTTP server for our | ||
687 | little tutorial. | ||
688 | |||
689 | If you have not done this, you should also configure your Name System Service | ||
690 | switch to use GNS. In your @code{/etc/nsswitch.conf} you should fine a line like | ||
691 | this: | ||
692 | @example | ||
693 | hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4 | ||
694 | @end example | ||
695 | |||
696 | The exact details may differ a bit, which is fine. Add the text | ||
697 | @code{gns [NOTFOUND=return]} after @code{files}: | ||
698 | @example | ||
699 | hosts: files gns [NOTFOUND=return] mdns4_minimal [NOTFOUND=return] dns mdns4 | ||
700 | @end example | ||
701 | |||
702 | |||
703 | You might want to make sure that @code{/lib/libnss_gns.so.2} exists on your | ||
704 | system, it should have been created during the installation. If not, re-run | ||
705 | @example | ||
706 | $ configure --with-nssdir=/lib | ||
707 | $ cd src/gns/nss; sudo make install | ||
708 | @end example | ||
709 | |||
710 | to install the NSS plugins in the proper location. | ||
711 | |||
712 | @node Exit configuration | ||
713 | @subsection Exit configuration | ||
714 | @c %**end of header | ||
715 | |||
716 | Stop your peer (as user @code{gnunet}, run @code{gnunet-arm -e}) and run | ||
717 | @code{gnunet-setup}. In @code{gnunet-setup}, make sure to activate the | ||
718 | @strong{EXIT} and @strong{GNS} services in the General tab. Then select the Exit | ||
719 | tab. Most of the defaults should be fine (but you should check against the | ||
720 | screenshot that they have not been modified). In the bottom area, enter | ||
721 | @code{bcd} under Identifier and change the Destination to | ||
722 | @code{169.254.86.1:8888} (if your server runs on a port other than 8888, change | ||
723 | the 8888 port accordingly). | ||
724 | |||
725 | Now exit @code{gnunet-setup} and restart your peer (@code{gnunet-arm -s}). | ||
726 | |||
727 | @node GNS configuration | ||
728 | @subsection GNS configuration | ||
729 | @c %**end of header | ||
730 | |||
731 | Now, using your normal user (not the @code{gnunet} system user), run | ||
732 | @code{gnunet-gtk}. Select the GNS icon and add a new label www in your master | ||
733 | zone. For the record type, select @code{VPN}. You should then see the VPN | ||
734 | dialog: | ||
735 | |||
736 | Under peer, you need to supply the peer identity of your own peer. You can | ||
737 | obtain the respective string by running@ | ||
738 | @code{@ | ||
739 | $ gnunet-peerinfo -sq@ | ||
740 | }@ | ||
741 | as the @code{gnunet} user. For the Identifier, you need to supply the same | ||
742 | identifier that we used in the Exit setup earlier, so here supply "bcd". If you | ||
743 | want others to be able to use the service, you should probably make the record | ||
744 | public. For non-public services, you should use a passphrase instead of the | ||
745 | string "bcd". Save the record and exit @code{gnunet-gtk}. | ||
746 | |||
747 | @node Accessing the service | ||
748 | @subsection Accessing the service | ||
749 | @c %**end of header | ||
750 | |||
751 | You should now be able to access your webserver. Type in:@ | ||
752 | @code{@ | ||
753 | $ wget http://www.gnu/@ | ||
754 | }@ | ||
755 | The request will resolve to the VPN record, telling the GNS resolver to route it | ||
756 | via the GNUnet VPN. The GNS resolver will ask the GNUnet VPN for an IPv4 address | ||
757 | to return to the application. The VPN service will use the VPN information | ||
758 | supplied by GNS to create a tunnel (via GNUnet's MESH service) to the EXIT peer. | ||
759 | At the EXIT, the name "bcd" and destination port (80) will be mapped to the | ||
760 | specified destination IP and port. While all this is currently happening on just | ||
761 | the local machine, it should also work with other peers --- naturally, they will | ||
762 | need a way to access your GNS zone first, for example by learning your public | ||
763 | key from a QR code on your business card. | ||
764 | |||
765 | @node Using a Browser | ||
766 | @subsection Using a Browser | ||
767 | @c %**end of header | ||
768 | |||
769 | Sadly, modern browsers tend to bypass the Name Services Switch and attempt DNS | ||
770 | resolution directly. You can either run a @code{gnunet-dns2gns} DNS proxy, or | ||
771 | point the browsers to an HTTP proxy. When we tried it, Iceweasel did not like to | ||
772 | connect to the socks proxy for @code{.gnu} TLDs, even if we disabled its | ||
773 | autoblunder of changing @code{.gnu} to ".gnu.com". Still, using the HTTP proxy | ||
774 | with Chrome does work. | ||
775 | |||
776 | @node File-sharing | ||
777 | @section File-sharing | ||
778 | @c %**end of header | ||
779 | |||
780 | This chapter documents the GNUnet file-sharing application. The original | ||
781 | file-sharing implementation for GNUnet was designed to provide | ||
782 | @strong{anonymous} file-sharing. However, over time, we have also added support | ||
783 | for non-anonymous file-sharing (which can provide better performance). Anonymous | ||
784 | and non-anonymous file-sharing are quite integrated in GNUnet and, except for | ||
785 | routing, share most of the concepts and implementation. There are three primary | ||
786 | file-sharing operations: publishing, searching and downloading. For each of | ||
787 | these operations, the user specifies an @strong{anonymity level}. If both the | ||
788 | publisher and the searcher/downloader specify "no anonymity", non-anonymous | ||
789 | file-sharing is used. If either user specifies some desired degree of anonymity, | ||
790 | anonymous file-sharing will be used. | ||
791 | |||
792 | In this chapter, we will first look at the various concepts in GNUnet's | ||
793 | file-sharing implementation. Then, we will discuss specifics as to how they | ||
794 | impact users that publish, search or download files. | ||
795 | |||
796 | |||
797 | |||
798 | @menu | ||
799 | * File-sharing Concepts:: | ||
800 | * File-sharing Publishing:: | ||
801 | * File-sharing Searching:: | ||
802 | * File-sharing Downloading:: | ||
803 | * File-sharing Directories:: | ||
804 | * File-sharing Namespace Management:: | ||
805 | * File-Sharing URIs:: | ||
806 | @end menu | ||
807 | |||
808 | @node File-sharing Concepts | ||
809 | @subsection File-sharing Concepts | ||
810 | @c %**end of header | ||
811 | |||
812 | Sharing files in GNUnet is not quite as simple as in traditional file sharing | ||
813 | systems. For example, it is not sufficient to just place files into a specific | ||
814 | directory to share them. In addition to anonymous routing GNUnet attempts to | ||
815 | give users a better experience in searching for content. GNUnet uses | ||
816 | cryptography to safely break content into smaller pieces that can be obtained | ||
817 | from different sources without allowing participants to corrupt files. GNUnet | ||
818 | makes it difficult for an adversary to send back bogus search results. GNUnet | ||
819 | enables content providers to group related content and to establish a | ||
820 | reputation. Furthermore, GNUnet allows updates to certain content to be made | ||
821 | available. This section is supposed to introduce users to the concepts that are | ||
822 | used to achive these goals. | ||
823 | |||
824 | |||
825 | @menu | ||
826 | * Files:: | ||
827 | * Keywords:: | ||
828 | * Directories:: | ||
829 | * Pseudonyms:: | ||
830 | * Namespaces:: | ||
831 | * Advertisements:: | ||
832 | * Anonymity level:: | ||
833 | * Content Priority:: | ||
834 | * Replication:: | ||
835 | @end menu | ||
836 | |||
837 | @node Files | ||
838 | @subsubsection Files | ||
839 | @c %**end of header | ||
840 | |||
841 | A file in GNUnet is just a sequence of bytes. Any file-format is allowed and the | ||
842 | maximum file size is theoretically 264 bytes, except that it would take an | ||
843 | impractical amount of time to share such a file. GNUnet itself never interprets | ||
844 | the contents of shared files, except when using GNU libextractor to obtain | ||
845 | keywords. | ||
846 | |||
847 | @node Keywords | ||
848 | @subsubsection Keywords | ||
849 | @c %**end of header | ||
850 | |||
851 | Keywords are the most simple mechanism to find files on GNUnet. Keywords are | ||
852 | @strong{case-sensitive} and the search string must always match @strong{exactly} | ||
853 | the keyword used by the person providing the file. Keywords are never | ||
854 | transmitted in plaintext. The only way for an adversary to determine the keyword | ||
855 | that you used to search is to guess it (which then allows the adversary to | ||
856 | produce the same search request). Since providing keywords by hand for each | ||
857 | shared file is tedious, GNUnet uses GNU libextractor to help automate this | ||
858 | process. Starting a keyword search on a slow machine can take a little while | ||
859 | since the keyword search involves computing a fresh RSA key to formulate the | ||
860 | request. | ||
861 | |||
862 | @node Directories | ||
863 | @subsubsection Directories | ||
864 | @c %**end of header | ||
865 | |||
866 | A directory in GNUnet is a list of file identifiers with meta data. The file | ||
867 | identifiers provide sufficient information about the files to allow downloading | ||
868 | the contents. Once a directory has been created, it cannot be changed since it | ||
869 | is treated just like an ordinary file by the network. Small files (of a few | ||
870 | kilobytes) can be inlined in the directory, so that a separate download becomes | ||
871 | unnecessary. | ||
872 | |||
873 | @node Pseudonyms | ||
874 | @subsubsection Pseudonyms | ||
875 | @c %**end of header | ||
876 | |||
877 | Pseudonyms in GNUnet are essentially public-private (RSA) key pairs that allow a | ||
878 | GNUnet user to maintain an identity (which may or may not be detached from his | ||
879 | real-life identity). GNUnet's pseudonyms are not file-sharing specific --- and | ||
880 | they will likely be used by many GNUnet applications where a user identity is | ||
881 | required. | ||
882 | |||
883 | Note that a pseudonym is NOT bound to a GNUnet peer. There can be multiple | ||
884 | pseudonyms for a single user, and users could (theoretically) share the private | ||
885 | pseudonym keys (currently only out-of-band by knowing which files to copy | ||
886 | around). | ||
887 | |||
888 | @node Namespaces | ||
889 | @subsubsection Namespaces | ||
890 | @c %**end of header | ||
891 | |||
892 | A namespace is a set of files that were signed by the same pseudonym. Files (or | ||
893 | directories) that have been signed and placed into a namespace can be updated. | ||
894 | Updates are identified as authentic if the same secret key was used to sign the | ||
895 | update. Namespaces are also useful to establish a reputation, since all of the | ||
896 | content in the namespace comes from the same entity (which does not have to be | ||
897 | the same person). | ||
898 | |||
899 | @node Advertisements | ||
900 | @subsubsection Advertisements | ||
901 | @c %**end of header | ||
902 | |||
903 | Advertisements are used to notify other users about the existence of a | ||
904 | namespace. Advertisements are propagated using the normal keyword search. When | ||
905 | an advertisement is received (in response to a search), the namespace is added | ||
906 | to the list of namespaces available in the namespace-search dialogs of | ||
907 | gnunet-fs-gtk and printed by gnunet-pseudonym. Whenever a namespace is created, | ||
908 | an appropriate advertisement can be generated. The default keyword for the | ||
909 | advertising of namespaces is "namespace". | ||
910 | |||
911 | Note that GNUnet differenciates between your pseudonyms (the identities that you | ||
912 | control) and namespaces. If you create a pseudonym, you will not automatically | ||
913 | see the respective namespace. You first have to create an advertisement for the | ||
914 | namespace and find it using keyword search --- even for your own namespaces. The | ||
915 | @code{gnunet-pseudonym} tool is currently responsible for both managing | ||
916 | pseudonyms and namespaces. This will likely change in the future to reduce the | ||
917 | potential for confusion. | ||
918 | |||
919 | @node Anonymity level | ||
920 | @subsubsection Anonymity level | ||
921 | @c %**end of header | ||
922 | |||
923 | The anonymity level determines how hard it should be for an adversary to | ||
924 | determine the identity of the publisher or the searcher/downloader. An | ||
925 | anonymity level of zero means that anonymity is not required. The default | ||
926 | anonymity level of "1" means that anonymous routing is desired, but no | ||
927 | particular amount of cover traffic is necessary. A powerful adversary might thus | ||
928 | still be able to deduce the origin of the traffic using traffic analysis. | ||
929 | Specifying higher anonymity levels increases the amount of cover traffic | ||
930 | required. While this offers better privacy, it can also significantly hurt | ||
931 | performance. | ||
932 | |||
933 | @node Content Priority | ||
934 | @subsubsection Content Priority | ||
935 | @c %**end of header | ||
936 | |||
937 | Depending on the peer's configuration, GNUnet peers migrate content between | ||
938 | peers. Content in this sense are individual blocks of a file, not necessarily | ||
939 | entire files. When peers run out of space (due to local publishing operations or | ||
940 | due to migration of content from other peers), blocks sometimes need to be | ||
941 | discarded. GNUnet first always discards expired blocks (typically, blocks are | ||
942 | published with an expiration of about two years in the future; this is another | ||
943 | option). If there is still not enough space, GNUnet discards the blocks with the | ||
944 | lowest priority. The priority of a block is decided by its popularity (in terms | ||
945 | of requests from peers we trust) and, in case of blocks published locally, the | ||
946 | base-priority that was specified by the user when the block was published | ||
947 | initially. | ||
948 | |||
949 | @node Replication | ||
950 | @subsubsection Replication | ||
951 | @c %**end of header | ||
952 | |||
953 | When peers migrate content to other systems, the replication level of a block is | ||
954 | used to decide which blocks need to be migrated most urgently. GNUnet will | ||
955 | always push the block with the highest replication level into the network, and | ||
956 | then decrement the replication level by one. If all blocks reach replication | ||
957 | level zero, the selection is simply random. | ||
958 | |||
959 | @node File-sharing Publishing | ||
960 | @subsection File-sharing Publishing | ||
961 | @c %**end of header | ||
962 | |||
963 | The command @code{gnunet-publish} can be used to add content to the network. | ||
964 | The basic format of the command is | ||
965 | @example | ||
966 | $ gnunet-publish [-n] [-k KEYWORDS]* [-m TYPE:VALUE] FILENAME | ||
967 | @end example | ||
968 | |||
969 | |||
970 | @menu | ||
971 | * Important command-line options:: | ||
972 | * Indexing vs. Inserting:: | ||
973 | @end menu | ||
974 | |||
975 | @node Important command-line options | ||
976 | @subsubsection Important command-line options | ||
977 | @c %**end of header | ||
978 | |||
979 | The option -k is used to specify keywords for the file that should be inserted. | ||
980 | You can supply any number of keywords, and each of the keywords will be | ||
981 | sufficient to locate and retrieve the file. | ||
982 | |||
983 | The -m option is used to specify meta-data, such as descriptions. You can use -m | ||
984 | multiple times. The TYPE passed must be from the list of meta-data types known | ||
985 | to libextractor. You can obtain this list by running @code{extract -L}. | ||
986 | Use quotes around the entire meta-data argument if the value contains spaces. | ||
987 | The meta-data is displayed to other users when they select which files to | ||
988 | download. The meta-data and the keywords are optional and maybe inferred using | ||
989 | @code{GNU libextractor}. | ||
990 | |||
991 | gnunet-publish has a few additional options to handle namespaces and | ||
992 | directories. | ||
993 | See the man-page for details. | ||
994 | |||
995 | @node Indexing vs. Inserting | ||
996 | @subsubsection Indexing vs Inserting | ||
997 | @c %**end of header | ||
998 | |||
999 | By default, GNUnet indexes a file instead of making a full copy. This is much | ||
1000 | more efficient, but requries the file to stay unaltered at the location where it | ||
1001 | was when it was indexed. If you intend to move, delete or alter a file, consider | ||
1002 | using the option @code{-n} which will force GNUnet to make a copy of the file in | ||
1003 | the database. | ||
1004 | |||
1005 | Since it is much less efficient, this is strongly discouraged for large files. | ||
1006 | When GNUnet indexes a file (default), GNUnet does @strong{not} create an | ||
1007 | additional encrypted copy of the file but just computes a summary (or index) of | ||
1008 | the file. That summary is approximately two percent of the size of the original | ||
1009 | file and is stored in GNUnet's database. Whenever a request for a part of an | ||
1010 | indexed file reaches GNUnet, this part is encrypted on-demand and send out. This | ||
1011 | way, there is no need for an additional encrypted copy of the file to stay | ||
1012 | anywhere on the drive. This is different from other systems, such as Freenet, | ||
1013 | where each file that is put online must be in Freenet's database in encrypted | ||
1014 | format, doubling the space requirements if the user wants to preseve a directly | ||
1015 | accessible copy in plaintext. | ||
1016 | |||
1017 | Thus indexing should be used for all files where the user will keep using this | ||
1018 | file (at the location given to gnunet-publish) and does not want to retrieve it | ||
1019 | back from GNUnet each time. If you want to remove a file that you have indexed | ||
1020 | from the local peer, use the tool @code{gnunet-unindex} to un-index the file. | ||
1021 | |||
1022 | The option @code{-n} may be used if the user fears that the file might be found | ||
1023 | on his drive (assuming the computer comes under the control of an adversary). | ||
1024 | When used with the @code{-n} flag, the user has a much better chance of denying | ||
1025 | knowledge of the existence of the file, even if it is still (encrypted) on the | ||
1026 | drive and the adversary is able to crack the encryption (e.g. by guessing the | ||
1027 | keyword. | ||
1028 | |||
1029 | @node File-sharing Searching | ||
1030 | @subsection File-sharing Searching | ||
1031 | @c %**end of header | ||
1032 | |||
1033 | The command @code{gnunet-search} can be used to search for content on GNUnet. | ||
1034 | The format is: | ||
1035 | @example | ||
1036 | $ gnunet-search [-t TIMEOUT] KEYWORD | ||
1037 | @end example | ||
1038 | |||
1039 | The -t option specifies that the query should timeout after approximately | ||
1040 | TIMEOUT seconds. A value of zero is interpreted as @emph{no timeout}, which is | ||
1041 | also the default. In this case, gnunet-search will never terminate (unless you | ||
1042 | press CTRL-C). | ||
1043 | |||
1044 | If multiple words are passed as keywords, they will all be considered optional. | ||
1045 | Prefix keywords with a "+" to make them mandatory. | ||
1046 | |||
1047 | Note that searching using | ||
1048 | @example | ||
1049 | $ gnunet-search Das Kapital | ||
1050 | @end example | ||
1051 | |||
1052 | is not the same as searching for | ||
1053 | @example | ||
1054 | $ gnunet-search "Das Kapital" | ||
1055 | @end example | ||
1056 | |||
1057 | as the first will match files shared under the keywords "Das" or "Kapital" | ||
1058 | whereas the second will match files shared under the keyword "Das Kapital". | ||
1059 | |||
1060 | Search results are printed by gnunet-search like this: | ||
1061 | @example | ||
1062 | $ gnunet-download -o "COPYING" --- gnunet://fs/chk/N8...C92.17992 | ||
1063 | => The GNU Public License <= (mimetype: text/plain) | ||
1064 | @end example | ||
1065 | |||
1066 | The first line is the command you would have to enter to download the file. | ||
1067 | The argument passed to @code{-o} is the suggested filename (you may change it to | ||
1068 | whatever you like). | ||
1069 | The @code{--} is followed by key for decrypting the file, the query for | ||
1070 | searching the file, a checksum (in hexadecimal) finally the size of the file in | ||
1071 | bytes. | ||
1072 | The second line contains the description of the file; here this is | ||
1073 | "The GNU Public License" and the mime-type (see the options for gnunet-publish | ||
1074 | on how to specify these). | ||
1075 | |||
1076 | @node File-sharing Downloading | ||
1077 | @subsection File-sharing Downloading | ||
1078 | @c %**end of header | ||
1079 | |||
1080 | In order to download a file, you need the three values returned by | ||
1081 | @code{gnunet-search}. | ||
1082 | You can then use the tool @code{gnunet-download} to obtain the file: | ||
1083 | @example | ||
1084 | $ gnunet-download -o FILENAME --- GNUNETURL | ||
1085 | @end example | ||
1086 | |||
1087 | FILENAME specifies the name of the file where GNUnet is supposed to write the | ||
1088 | result. Existing files are overwritten. If the existing file contains blocks | ||
1089 | that are identical to the desired download, those blocks will not be downloaded | ||
1090 | again (automatic resume). | ||
1091 | |||
1092 | If you want to download the GPL from the previous example, you do the following: | ||
1093 | @example | ||
1094 | $ gnunet-download -o "COPYING" --- gnunet://fs/chk/N8...92.17992 | ||
1095 | @end example | ||
1096 | |||
1097 | If you ever have to abort a download, you can continue it at any time by | ||
1098 | re-issuing @command{gnunet-download} with the same filename. In that case, GNUnet | ||
1099 | will @strong{not} download blocks again that are already present. | ||
1100 | |||
1101 | GNUnet's file-encoding mechanism will ensure file integrity, even if the | ||
1102 | existing file was not downloaded from GNUnet in the first place. | ||
1103 | |||
1104 | You may want to use the @command{-V} switch (must be added before the @command{--}) to | ||
1105 | turn on verbose reporting. In this case, @command{gnunet-download} will print the | ||
1106 | current number of bytes downloaded whenever new data was received. | ||
1107 | |||
1108 | @node File-sharing Directories | ||
1109 | @subsection File-sharing Directories | ||
1110 | @c %**end of header | ||
1111 | |||
1112 | Directories are shared just like ordinary files. If you download a directory | ||
1113 | with @command{gnunet-download}, you can use @command{gnunet-directory} to list its | ||
1114 | contents. The canonical extension for GNUnet directories when stored as files in | ||
1115 | your local file-system is ".gnd". The contents of a directory are URIs and | ||
1116 | meta data. | ||
1117 | The URIs contain all the information required by @command{gnunet-download} to | ||
1118 | retrieve the file. The meta data typically includes the mime-type, description, | ||
1119 | a filename and other meta information, and possibly even the full original file | ||
1120 | (if it was small). | ||
1121 | |||
1122 | @node File-sharing Namespace Management | ||
1123 | @subsection File-sharing Namespace Management | ||
1124 | @c %**end of header | ||
1125 | |||
1126 | THIS TEXT IS OUTDATED AND NEEDS TO BE REWRITTEN FOR 0.10! | ||
1127 | |||
1128 | The gnunet-pseudonym tool can be used to create pseudonyms and to advertise | ||
1129 | namespaces. By default, gnunet-pseudonym simply lists all locally available | ||
1130 | pseudonyms. | ||
1131 | |||
1132 | |||
1133 | @menu | ||
1134 | * Creating Pseudonyms:: | ||
1135 | * Deleting Pseudonyms:: | ||
1136 | * Advertising namespaces:: | ||
1137 | * Namespace names:: | ||
1138 | * Namespace root:: | ||
1139 | @end menu | ||
1140 | |||
1141 | @node Creating Pseudonyms | ||
1142 | @subsubsection Creating Pseudonyms | ||
1143 | @c %**end of header | ||
1144 | |||
1145 | With the @command{-C NICK} option it can also be used to create a new pseudonym. | ||
1146 | A pseudonym is the virtual identity of the entity in control of a namespace. | ||
1147 | Anyone can create any number of pseudonyms. Note that creating a pseudonym can | ||
1148 | take a few minutes depending on the performance of the machine used. | ||
1149 | |||
1150 | @node Deleting Pseudonyms | ||
1151 | @subsubsection Deleting Pseudonyms | ||
1152 | @c %**end of header | ||
1153 | |||
1154 | With the @command{-D NICK} option pseudonyms can be deleted. Once the pseudonym has | ||
1155 | been deleted it is impossible to add content to the corresponding namespace. | ||
1156 | Deleting the pseudonym does not make the namespace or any content in it | ||
1157 | unavailable. | ||
1158 | |||
1159 | @node Advertising namespaces | ||
1160 | @subsubsection Advertising namespaces | ||
1161 | @c %**end of header | ||
1162 | |||
1163 | Each namespace is associated with meta-data that describes the namespace. | ||
1164 | This meta data is provided by the user at the time that the namespace is | ||
1165 | advertised. Advertisements are published under keywords so that they can be | ||
1166 | found using normal keyword-searches. This way, users can learn about new | ||
1167 | namespaces without relying on out-of-band communication or directories. | ||
1168 | A suggested keyword to use for all namespaces is simply "namespace". | ||
1169 | When a keyword-search finds a namespace advertisement, it is automatically | ||
1170 | stored in a local list of known namespaces. Users can then associate a rank with | ||
1171 | the namespace to remember the quality of the content found in it. | ||
1172 | |||
1173 | @node Namespace names | ||
1174 | @subsubsection Namespace names | ||
1175 | @c %**end of header | ||
1176 | |||
1177 | While the namespace is uniquely identified by its ID, another way to refer to | ||
1178 | the namespace is to use the NICKNAME. The NICKNAME can be freely chosen by the | ||
1179 | creator of the namespace and hence conflicts are possible. If a GNUnet client | ||
1180 | learns about more than one namespace using the same NICKNAME, the ID is appended | ||
1181 | to the NICKNAME to get a unique identifier. | ||
1182 | |||
1183 | @node Namespace root | ||
1184 | @subsubsection Namespace root | ||
1185 | @c %**end of header | ||
1186 | |||
1187 | An item of particular interest in the namespace advertisement is the ROOT. | ||
1188 | The ROOT is the identifier of a designated entry in the namespace. The idea is | ||
1189 | that the ROOT can be used to advertise an entry point to the content of the | ||
1190 | namespace. | ||
1191 | |||
1192 | @node File-Sharing URIs | ||
1193 | @subsection File-Sharing URIs | ||
1194 | @c %**end of header | ||
1195 | |||
1196 | GNUnet (currently) uses four different types of URIs for file-sharing. They all | ||
1197 | begin with "gnunet://fs/". This section describes the four different URI types | ||
1198 | in detail. | ||
1199 | |||
1200 | |||
1201 | @menu | ||
1202 | * Encoding of hash values in URIs:: | ||
1203 | * Content Hash Key (chk):: | ||
1204 | * Location identifiers (loc):: | ||
1205 | * Keyword queries (ksk):: | ||
1206 | * Namespace content (sks):: | ||
1207 | @end menu | ||
1208 | |||
1209 | @node Encoding of hash values in URIs | ||
1210 | @subsubsection Encoding of hash values in URIs | ||
1211 | @c %**end of header | ||
1212 | |||
1213 | Most URIs include some hash values. Hashes are encoded using base32hex | ||
1214 | (RFC 2938). | ||
1215 | |||
1216 | @node Content Hash Key (chk) | ||
1217 | @subsubsection Content Hash Key (chk) | ||
1218 | @c %**end of header | ||
1219 | |||
1220 | A chk-URI is used to (uniquely) identify a file or directory and to allow peers | ||
1221 | to download the file. Files are stored in GNUnet as a tree of encrypted blocks. | ||
1222 | The chk-URI thus contains the information to download and decrypt those blocks. | ||
1223 | A chk-URI has the format "gnunet://fs/chk/KEYHASH.QUERYHASH.SIZE". Here, "SIZE" | ||
1224 | is the size of the file (which allows a peer to determine the shape of the | ||
1225 | tree), KEYHASH is the key used to decrypt the file (also the hash of the | ||
1226 | plaintext of the top block) and QUERYHASH is the query used to request the | ||
1227 | top-level block (also the hash of the encrypted block). | ||
1228 | |||
1229 | @node Location identifiers (loc) | ||
1230 | @subsubsection Location identifiers (loc) | ||
1231 | @c %**end of header | ||
1232 | |||
1233 | For non-anonymous file-sharing, loc-URIs are used to specify which peer is | ||
1234 | offering the data (in addition to specifying all of the data from a chk-URI). | ||
1235 | Location identifiers include a digital signature of the peer to affirm that the | ||
1236 | peer is truly the origin of the data. The format is | ||
1237 | "gnunet://fs/loc/KEYHASH.QUERYHASH.SIZE.PEER.SIG.EXPTIME". Here, "PEER" is the | ||
1238 | public key of the peer (in GNUnet format in base32hex), SIG is the RSA signature | ||
1239 | (in GNUnet format in base32hex) and EXPTIME specifies when the signature expires | ||
1240 | (in milliseconds after 1970). | ||
1241 | |||
1242 | @node Keyword queries (ksk) | ||
1243 | @subsubsection Keyword queries (ksk) | ||
1244 | @c %**end of header | ||
1245 | |||
1246 | A keyword-URI is used to specify that the desired operation is the search using | ||
1247 | a particular keyword. The format is simply "gnunet://fs/ksk/KEYWORD". Non-ASCII | ||
1248 | characters can be specified using the typical URI-encoding (using hex values) | ||
1249 | from HTTP. "+" can be used to specify multiple keywords (which are then | ||
1250 | logically "OR"-ed in the search, results matching both keywords are given a | ||
1251 | higher rank): "gnunet://fs/ksk/KEYWORD1+KEYWORD2". | ||
1252 | |||
1253 | @node Namespace content (sks) | ||
1254 | @subsubsection Namespace content (sks) | ||
1255 | @c %**end of header | ||
1256 | |||
1257 | Namespaces are sets of files that have been approved by some (usually | ||
1258 | pseudonymous) user --- typically by that user publishing all of the files | ||
1259 | together. A file can be in many namespaces. A file is in a namespace if the | ||
1260 | owner of the ego (aka the namespace's private key) signs the CHK of the file | ||
1261 | cryptographically. An SKS-URI is used to search a namespace. The result is a | ||
1262 | block containing meta data, the CHK and the namespace owner's signature. The | ||
1263 | format of a sks-URI is "gnunet://fs/sks/NAMESPACE/IDENTIFIER". Here, "NAMESPACE" | ||
1264 | is the public key for the namespace. "IDENTIFIER" is a freely chosen keyword | ||
1265 | (or password!). A commonly used identifier is "root" which by convention refers | ||
1266 | to some kind of index or other entry point into the namespace. | ||
1267 | |||
1268 | @node The GNU Name System | ||
1269 | @section The GNU Name System | ||
1270 | @c %**end of header | ||
1271 | |||
1272 | |||
1273 | The GNU Name System (GNS) is secure and decentralized naming system. | ||
1274 | It allows its users to resolve and register names within the @code{.gnu} | ||
1275 | @dfn{top-level domain} (TLD). | ||
1276 | |||
1277 | GNS is designed to provide: | ||
1278 | @itemize @bullet | ||
1279 | @item Censorship resistance | ||
1280 | @item Query privacy | ||
1281 | @item Secure name resolution | ||
1282 | @item Compatibility with DNS | ||
1283 | @end itemize | ||
1284 | |||
1285 | For the initial configuration and population of your GNS installation, please | ||
1286 | follow the GNS setup instructions. The remainder of this chapter will provide | ||
1287 | some background on GNS and then describe how to use GNS in more detail. | ||
1288 | |||
1289 | Unlike DNS, GNS does not rely on central root zones or authorities. Instead any | ||
1290 | user administers his own root and can can create arbitrary name value mappings. | ||
1291 | Furthermore users can delegate resolution to other users' zones just like DNS NS | ||
1292 | records do. Zones are uniquely identified via public keys and resource records | ||
1293 | are signed using the corresponding public key. Delegation to another user's zone | ||
1294 | is done using special PKEY records and petnames. A petname is a name that can be | ||
1295 | freely chosen by the user. This results in non-unique name-value mappings as | ||
1296 | @code{@uref{http://www.bob.gnu/, www.bob.gnu}} to one user might be | ||
1297 | @code{@uref{http://www.friend.gnu/, www.friend.gnu}} for someone else. | ||
1298 | |||
1299 | |||
1300 | @menu | ||
1301 | * Maintaining your own Zones:: | ||
1302 | * Obtaining your Zone Key:: | ||
1303 | * Adding Links to Other Zones:: | ||
1304 | * The Three Local Zones of GNS:: | ||
1305 | * The Master Zone:: | ||
1306 | * The Private Zone:: | ||
1307 | * The Shorten Zone:: | ||
1308 | * The ZKEY Top Level Domain in GNS:: | ||
1309 | * Resource Records in GNS:: | ||
1310 | @end menu | ||
1311 | |||
1312 | |||
1313 | @node Maintaining your own Zones | ||
1314 | @subsection Maintaining your own Zones | ||
1315 | |||
1316 | To setup your GNS system you must execute: | ||
1317 | |||
1318 | @example | ||
1319 | $ gnunet-gns-import.sh | ||
1320 | @end example | ||
1321 | |||
1322 | This will boostrap your zones and create the necessary key material. | ||
1323 | Your keys can be listed using the gnunet-identity command line tool: | ||
1324 | |||
1325 | @example | ||
1326 | $ gnunet-identity -d | ||
1327 | @end example | ||
1328 | |||
1329 | You can arbitrarily create your own zones using the gnunet-identity tool using: | ||
1330 | |||
1331 | @example | ||
1332 | $ gnunet-identity -C "new_zone" | ||
1333 | @end example | ||
1334 | |||
1335 | Now you can add (or edit, or remove) records in your GNS zone using the | ||
1336 | gnunet-setup GUI or using the gnunet-namestore command-line tool. In either | ||
1337 | case, your records will be stored in an SQL database under control of the | ||
1338 | gnunet-service-namestore. Note that if mutliple users use one peer, the | ||
1339 | namestore database will include the combined records of all users. However, | ||
1340 | users will not be able to see each other's records if they are marked as | ||
1341 | private. | ||
1342 | |||
1343 | To provide a simple example for editing your own zone, suppose you have your own | ||
1344 | web server with IP 1.2.3.4. Then you can put an A record (A records in DNS are | ||
1345 | for IPv4 IP addresses) into your local zone using the command:@ | ||
1346 | |||
1347 | @example | ||
1348 | $ gnunet-namestore -z master-zone -a -n www -t A -V 1.2.3.4 -e never | ||
1349 | @end example | ||
1350 | |||
1351 | Afterwards, you will be able to access your webpage under "www.gnu" (assuming | ||
1352 | your webserver does not use virtual hosting, if it does, please read up on | ||
1353 | setting up the GNS proxy). | ||
1354 | |||
1355 | Similar commands will work for other types of DNS and GNS records, the syntax | ||
1356 | largely depending on the type of the record. Naturally, most users may find | ||
1357 | editing the zones using the gnunet-setup GUI to be easier. | ||
1358 | |||
1359 | @node Obtaining your Zone Key | ||
1360 | @subsection Obtaining your Zone Key | ||
1361 | |||
1362 | Each zone in GNS has a public-private key. Usually, gnunet-namestore and | ||
1363 | gnunet-setup will access your private key as necessary, so you do not have to | ||
1364 | worry about those. What is important is your public key (or rather, the hash of | ||
1365 | your public key), as you will likely want to give it to others so that they can | ||
1366 | securely link to you. | ||
1367 | |||
1368 | You can usually get the hash of your public key using@ | ||
1369 | |||
1370 | @example | ||
1371 | $ gnunet-identity -d $options | grep master-zone | awk '@{print $3@}' | ||
1372 | @end example | ||
1373 | |||
1374 | For example, the output might be something like: | ||
1375 | |||
1376 | @example | ||
1377 | DC3SEECJORPHQNVRH965A6N74B1M37S721IG4RBQ15PJLLPJKUE0 | ||
1378 | @end example | ||
1379 | |||
1380 | Alternatively, you can obtain a QR code with your zone key AND your pseudonym | ||
1381 | from gnunet-gtk. The QR code is displayed in the GNS tab and can be stored to | ||
1382 | disk using the Save as button next to the image. | ||
1383 | |||
1384 | @node Adding Links to Other Zones | ||
1385 | @subsection Adding Links to Other Zones | ||
1386 | |||
1387 | |||
1388 | A central operation in GNS is the ability to securely delegate to other zones. | ||
1389 | Basically, by adding a delegation you make all of the names from the other zone | ||
1390 | available to yourself. This section describes how to create delegations. | ||
1391 | |||
1392 | Suppose you have a friend who you call 'bob' who also uses GNS. You can then | ||
1393 | delegate resolution of names to Bob's zone by adding a PKEY record to his local | ||
1394 | zone: | ||
1395 | |||
1396 | @example | ||
1397 | $ gnunet-namestore -a -n bob --type PKEY -V XXXX -e never | ||
1398 | @end example | ||
1399 | |||
1400 | Note that XXXX in the command above must be replaced with the hash of Bob's | ||
1401 | public key (the output your friend obtained using the gnunet-identity command | ||
1402 | from the previous section and told you, for example by giving you a business | ||
1403 | card containing this information as a QR code). | ||
1404 | |||
1405 | Assuming Bob has an A record for his website under the name of www in his zone, | ||
1406 | you can then access Bob's website under www.bob.gnu --- as well as any (public) | ||
1407 | GNS record that Bob has in his zone by replacing www with the respective name of | ||
1408 | the record in Bob's zone. | ||
1409 | |||
1410 | Furthermore, if Bob has himself a (public) delegation to Carol's zone under | ||
1411 | "carol", you can access Carol's records under NAME.carol.bob.gnu (where NAME is | ||
1412 | the name of Carol's record you want to access). | ||
1413 | |||
1414 | @node The Three Local Zones of GNS | ||
1415 | @subsection The Three Local Zones of GNS | ||
1416 | |||
1417 | Each user GNS has control over three zones. Each of the zones has a different | ||
1418 | purpose. These zones are the | ||
1419 | @itemize @bullet | ||
1420 | |||
1421 | @item master zone, | ||
1422 | @item private zone, and the | ||
1423 | @item shorten zone. | ||
1424 | @end itemize | ||
1425 | |||
1426 | @node The Master Zone | ||
1427 | @subsection The Master Zone | ||
1428 | |||
1429 | |||
1430 | The master zone is your personal TLD. Names within the @code{.gnu} namespace are | ||
1431 | resolved relative to this zone. You can arbitrarily add records to this zone and | ||
1432 | selectively publish those records. | ||
1433 | |||
1434 | @node The Private Zone | ||
1435 | @subsection The Private Zone | ||
1436 | |||
1437 | |||
1438 | The private zone is a subzone (or subdomain in DNS terms) of your master zone. | ||
1439 | It should be used for records that you want to keep private. For example | ||
1440 | @code{bank.private.gnu}. The key idea is that you want to keep your private | ||
1441 | records separate, if just to know that those names are not available to other | ||
1442 | users. | ||
1443 | |||
1444 | @node The Shorten Zone | ||
1445 | @subsection The Shorten Zone | ||
1446 | |||
1447 | |||
1448 | The shorten zone can either be a subzone of the master zone or the private zone. | ||
1449 | It is different from the other zones in that GNS will automatically populate | ||
1450 | this zone with other users' zones based on their PSEU records whenever you | ||
1451 | resolve a name. | ||
1452 | |||
1453 | For example if you go to | ||
1454 | @code{@uref{http://www.bob.alice.dave.gnu/, www.bob.alice.dave.gnu}}, GNS will | ||
1455 | try to import @code{bob} into your shorten zone. Having obtained Bob's PKEY from | ||
1456 | @code{alice.dave.gnu}, GNS will lookup the PSEU record for @code{+} in Bob's | ||
1457 | zone. If it exists and the specified pseudonym is not taken, Bob's PKEY will be | ||
1458 | automatically added under that pseudonym (i.e. "bob") into your shorten zone. | ||
1459 | From then on, Bob's webpage will also be available for you as | ||
1460 | @code{@uref{http://www.bob.short.gnu/, www.bob.short.gnu}}. This feature is | ||
1461 | called automatic name shortening and is supposed to keep GNS names as short and | ||
1462 | memorable as possible. | ||
1463 | |||
1464 | @node The ZKEY Top Level Domain in GNS | ||
1465 | @subsection The ZKEY Top Level Domain in GNS | ||
1466 | |||
1467 | |||
1468 | GNS also provides a secure and globally unique namespace under the .zkey | ||
1469 | top-level domain. A name in the .zkey TLD corresponds to the (printable) public | ||
1470 | key of a zone. Names in the .zkey TLD are then resolved by querying the | ||
1471 | respective zone. The .zkey TLD is expected to be used under rare circumstances | ||
1472 | where globally unique names are required and for integration with legacy | ||
1473 | systems. | ||
1474 | |||
1475 | @node Resource Records in GNS | ||
1476 | @subsection Resource Records in GNS | ||
1477 | |||
1478 | |||
1479 | GNS supports the majority of the DNS records as defined in | ||
1480 | @uref{http://www.ietf.org/rfc/rfc1035.txt, RFC 1035}. Additionally, GNS defines | ||
1481 | some new record types the are unique to the GNS system. For example, | ||
1482 | GNS-specific resource records are use to give petnames for zone delegation, | ||
1483 | revoke zone keys and provide some compatibility features. | ||
1484 | |||
1485 | For some DNS records, GNS does extended processing to increase their usefulness | ||
1486 | in GNS. In particular, GNS introduces special names referred to as | ||
1487 | "zone relative names". Zone relative names are allowed in some resource record | ||
1488 | types (for example, in NS and CNAME records) and can also be used in links on | ||
1489 | webpages. Zone relative names end in ".+" which indicates that the name needs to | ||
1490 | be resolved relative to the current authoritative zone. The extended processing | ||
1491 | of those names will expand the ".+" with the correct delegation chain to the | ||
1492 | authoritative zone (replacing ".+" with the name of the location where the name | ||
1493 | was encountered) and hence generate a valid @code{.gnu} name. | ||
1494 | |||
1495 | GNS currently supports the following record types: | ||
1496 | |||
1497 | @menu | ||
1498 | * NICK:: | ||
1499 | * PKEY:: | ||
1500 | * BOX:: | ||
1501 | * LEHO:: | ||
1502 | * VPN:: | ||
1503 | * A AAAA and TXT:: | ||
1504 | * CNAME:: | ||
1505 | * GNS2DNS:: | ||
1506 | * SOA SRV PTR and MX:: | ||
1507 | @end menu | ||
1508 | |||
1509 | @node NICK | ||
1510 | @subsubsection NICK | ||
1511 | |||
1512 | A NICK record is used to give a zone a name. With a NICK record, you can | ||
1513 | essentially specify how you would like to be called. GNS expects this record | ||
1514 | under the name "+" in the zone's database (NAMESTORE); however, it will then | ||
1515 | automatically be copied into each record set, so that clients never need to do a | ||
1516 | separate lookup to discover the NICK record. | ||
1517 | |||
1518 | @b{Example}@ | ||
1519 | |||
1520 | @example | ||
1521 | Name: +; RRType: NICK; Value: bob | ||
1522 | @end example | ||
1523 | |||
1524 | This record in Bob's zone will tell other users that this zone wants to be | ||
1525 | referred to as 'bob'. Note that nobody is obliged to call Bob's zone 'bob' in | ||
1526 | their own zones. It can be seen as a recommendation ("Please call me 'bob'"). | ||
1527 | |||
1528 | @node PKEY | ||
1529 | @subsubsection PKEY | ||
1530 | |||
1531 | PKEY records are used to add delegation to other users' zones and give those | ||
1532 | zones a petname. | ||
1533 | |||
1534 | @b{Example}@ | ||
1535 | |||
1536 | Let Bob's zone be identified by the hash "ABC012". Bob is your friend so you | ||
1537 | want to give him the petname "friend". Then you add the following record to your | ||
1538 | zone: | ||
1539 | |||
1540 | @example | ||
1541 | Name: friend; RRType: PKEY; Value: ABC012; | ||
1542 | @end example | ||
1543 | |||
1544 | This will allow you to resolve records in bob's zone under "*.friend.gnu". | ||
1545 | |||
1546 | @node BOX | ||
1547 | @subsubsection BOX | ||
1548 | |||
1549 | BOX records are there to integrate information from TLSA or SRV records under | ||
1550 | the main label. In DNS, TLSA and SRV records use special names of the form | ||
1551 | @code{_port._proto.(label.)*tld} to indicate the port number and protocol | ||
1552 | (i.e. tcp or udp) for which the TLSA or SRV record is valid. This causes various | ||
1553 | problems, and is elegantly solved in GNS by integrating the protocol and port | ||
1554 | numbers together with the respective value into a "BOX" record. Note that in the | ||
1555 | GUI, you do not get to edit BOX records directly right now --- the GUI will | ||
1556 | provide the illusion of directly editing the TLSA and SRV records, even though | ||
1557 | they internally are BOXed up. | ||
1558 | |||
1559 | @node LEHO | ||
1560 | @subsubsection LEHO | ||
1561 | |||
1562 | The LEgacy HOstname of a server. Some webservers expect a specific hostname to | ||
1563 | provide a service (virtiual hosting). Also SSL certificates usually contain DNS | ||
1564 | names. To provide the expected legacy DNS name for a server, the LEHO record can | ||
1565 | be used. To mitigate the just mentioned issues the GNS proxy has to be used. The | ||
1566 | GNS proxy will use the LEHO information to apply the necessary transformations. | ||
1567 | |||
1568 | @node VPN | ||
1569 | @subsubsection VPN | ||
1570 | |||
1571 | GNS allows easy access to services provided by the GNUnet Virtual Public | ||
1572 | Network. When the GNS resolver encounters a VPN record it will contact the VPN | ||
1573 | service to try and allocate an IPv4/v6 address (if the queries record type is an | ||
1574 | IP address) that can be used to contact the service. | ||
1575 | |||
1576 | @b{Example}@ | ||
1577 | |||
1578 | I want to provide access to the VPN service "web.gnu." on port 80 on peer | ||
1579 | ABC012:@ | ||
1580 | Name: www; RRType: VPN; Value: 80 ABC012 web.gnu. | ||
1581 | |||
1582 | The peer ABC012 is configured to provide an exit point for the service | ||
1583 | "web.gnu." on port 80 to it's server running locally on port 8080 by having the | ||
1584 | following lines in the @file{gnunet.conf} configuration file:@ | ||
1585 | @example | ||
1586 | [web.gnunet.] | ||
1587 | TCP_REDIRECTS = 80:localhost4:8080 | ||
1588 | @end example | ||
1589 | |||
1590 | @node A AAAA and TXT | ||
1591 | @subsubsection A AAAA and TXT | ||
1592 | |||
1593 | Those records work in exactly the same fashion as in traditional DNS. | ||
1594 | |||
1595 | @node CNAME | ||
1596 | @subsubsection CNAME | ||
1597 | |||
1598 | As specified in RFC 1035 whenever a CNAME is encountered the query needs to be | ||
1599 | restarted with the specified name. In GNS a CNAME can either be: | ||
1600 | |||
1601 | @itemize @bullet | ||
1602 | @item A zone relative name, | ||
1603 | @item A zkey name or | ||
1604 | @item A DNS name (in which case resolution will continue outside of GNS with the systems DNS resolver) | ||
1605 | @end itemize | ||
1606 | |||
1607 | @node GNS2DNS | ||
1608 | @subsubsection GNS2DNS | ||
1609 | |||
1610 | GNS can delegate authority to a legacy DNS zone. For this, the name of the DNS | ||
1611 | nameserver and the name of the DNS zone are specified in a GNS2DNS record. | ||
1612 | |||
1613 | @b{Example} | ||
1614 | |||
1615 | @example | ||
1616 | Name: pet; RRType: GNS2DNS; Value: gnunet.org@@a.ns.joker.com | ||
1617 | @end example | ||
1618 | |||
1619 | Any query to @code{pet.gnu} will then be delegated to the DNS server at | ||
1620 | @code{a.ns.joker.com}.@ | ||
1621 | For example, @code{@uref{http://www.pet.gnu/, www.pet.gnu}} will result in a | ||
1622 | DNS query for @code{@uref{http://www.gnunet.org/, www.gnunet.org}} to the server | ||
1623 | at @code{a.ns.joker.com}. Delegation to DNS via NS records in GNS can be useful | ||
1624 | if you do not want to start resolution in the DNS root zone (due to issues such | ||
1625 | as censorship or availability). | ||
1626 | |||
1627 | Note that you would typically want to use a relative name for the nameserver, | ||
1628 | i.e. | ||
1629 | @example | ||
1630 | Name: pet; RRType: GNS2DNS; Value: gnunet.org@@ns-joker.+@ | ||
1631 | Name: ns-joker; RRType: A; Value: 184.172.157.218 | ||
1632 | @end example | ||
1633 | |||
1634 | This way, you can avoid involving the DNS hierarchy in the resolution of | ||
1635 | @code{a.ns.joker.com}. In the example above, the problem may not be obvious as | ||
1636 | the nameserver for "gnunet.org" is in the ".com" zone. However, imagine the | ||
1637 | nameserver was "ns.gnunet.org". In this case, delegating to "ns.gnunet.org" | ||
1638 | would mean that despite using GNS, censorship in the DNS ".org" zone would still | ||
1639 | be effective. | ||
1640 | |||
1641 | @node SOA SRV PTR and MX | ||
1642 | @subsubsection SOA SRV PTR and MX | ||
1643 | |||
1644 | The domain names in those records can, again, be either | ||
1645 | @itemize @bullet | ||
1646 | @item A zone relative name, | ||
1647 | @item A zkey name or | ||
1648 | @item A DNS name | ||
1649 | @end itemize | ||
1650 | |||
1651 | The resolver will expand the zone relative name if possible. Note that when | ||
1652 | using MX records within GNS, the target mail server might still refuse to accept | ||
1653 | e-mails to the resulting domain as the name might not match. GNS-enabled mail | ||
1654 | clients should use the ZKEY zone as the destination hostname and GNS-enabled | ||
1655 | mail servers should be configured to accept e-mails to the ZKEY-zones of all | ||
1656 | local users. | ||
1657 | |||
1658 | @node Using the Virtual Public Network | ||
1659 | @section Using the Virtual Public Network | ||
1660 | |||
1661 | @menu | ||
1662 | * Setting up an Exit node:: | ||
1663 | * Fedora and the Firewall:: | ||
1664 | * Setting up VPN node for protocol translation and tunneling:: | ||
1665 | @end menu | ||
1666 | |||
1667 | Using the GNUnet Virtual Public Network (VPN) application you can tunnel IP | ||
1668 | traffic over GNUnet. Moreover, the VPN comes with built-in protocol translation | ||
1669 | and DNS-ALG support, enabling IPv4-to-IPv6 protocol translation | ||
1670 | (in both directions). This chapter documents how to use the GNUnet VPN. | ||
1671 | |||
1672 | The first thing to note about the GNUnet VPN is that it is a public network. All | ||
1673 | participating peers can participate and there is no secret key to control | ||
1674 | access. So unlike common virtual private networks, the GNUnet VPN is not useful | ||
1675 | as a means to provide a "private" network abstraction over the Internet. The | ||
1676 | GNUnet VPN is a virtual network in the sense that it is an overlay over the | ||
1677 | Internet, using its own routing mechanisms and can also use an internal | ||
1678 | addressing scheme. The GNUnet VPN is an Internet underlay --- TCP/IP | ||
1679 | applications run on top of it. | ||
1680 | |||
1681 | The VPN is currently only supported on GNU/Linux systems. Support for operating | ||
1682 | systems that support TUN (such as FreeBSD) should be easy to add (or might not | ||
1683 | even require any coding at all --- we just did not test this so far). Support | ||
1684 | for other operating systems would require re-writing the code to create virtual | ||
1685 | network interfaces and to intercept DNS requests. | ||
1686 | |||
1687 | The VPN does not provide good anonymity. While requests are routed over the | ||
1688 | GNUnet network, other peers can directly see the source and destination of each | ||
1689 | (encapsulated) IP packet. Finally, if you use the VPN to access Internet | ||
1690 | services, the peer sending the request to the Internet will be able to observe | ||
1691 | and even alter the IP traffic. We will discuss additional security implications | ||
1692 | of using the VPN later in this chapter. | ||
1693 | |||
1694 | @node Setting up an Exit node | ||
1695 | @subsection Setting up an Exit node | ||
1696 | |||
1697 | Any useful operation with the VPN requires the existence of an exit node in the | ||
1698 | GNUnet Peer-to-Peer network. Exit functionality can only be enabled on peers | ||
1699 | that have regular Internet access. If you want to play around with the VPN or | ||
1700 | support the network, we encourage you to setup exit nodes. This chapter | ||
1701 | documents how to setup an exit node. | ||
1702 | |||
1703 | There are four types of exit functions an exit node can provide, and using the | ||
1704 | GNUnet VPN to access the Internet will only work nicely if the first three types | ||
1705 | are provided somewhere in the network. The four exit functions are: | ||
1706 | @itemize @bullet | ||
1707 | @item DNS: allow other peers to use your DNS resolver | ||
1708 | @item IPv4: allow other peers to access your IPv4 Internet connection | ||
1709 | @item IPv6: allow other peers to access your IPv6 Internet connection | ||
1710 | @item Local service: allow other peers to access a specific TCP or UDP service your peer is providing | ||
1711 | @end itemize | ||
1712 | |||
1713 | By enabling "exit" in gnunet-setup and checking the respective boxes in the | ||
1714 | "exit" tab, you can easily choose which of the above exit functions you want to | ||
1715 | support. | ||
1716 | |||
1717 | Note, however, that by supporting the first three functions you will allow | ||
1718 | arbitrary other GNUnet users to access the Internet via your system. This is | ||
1719 | somewhat similar to running a Tor exit node. The torproject has a nice article | ||
1720 | about what to consider if you want to do this here. We believe that generally | ||
1721 | running a DNS exit node is completely harmless. | ||
1722 | |||
1723 | The exit node configuration does currently not allow you to restrict the | ||
1724 | Internet traffic that leaves your system. In particular, you cannot exclude SMTP | ||
1725 | traffic (or block port 25) or limit to HTTP traffic using the GNUnet | ||
1726 | configuration. However, you can use your host firewall to restrict outbound | ||
1727 | connections from the virtual tunnel interface. This is highly recommended. In | ||
1728 | the future, we plan to offer a wider range of configuration options for exit | ||
1729 | nodes. | ||
1730 | |||
1731 | Note that by running an exit node GNUnet will configure your kernel to perform | ||
1732 | IP-forwarding (for IPv6) and NAT (for IPv4) so that the traffic from the virtual | ||
1733 | interface can be routed to the Internet. In order to provide an IPv6-exit, you | ||
1734 | need to have a subnet routed to your host's external network interface and | ||
1735 | assign a subrange of that subnet to the GNUnet exit's TUN interface. | ||
1736 | |||
1737 | When running a local service, you should make sure that the local service is | ||
1738 | (also) bound to the IP address of your EXIT interface (i.e. 169.254.86.1). It | ||
1739 | will NOT work if your local service is just bound to loopback. You may also want | ||
1740 | to create a "VPN" record in your zone of the GNU Name System to make it easy for | ||
1741 | others to access your service via a name instead of just the full service | ||
1742 | descriptor. Note that the identifier you assign the service can serve as a | ||
1743 | passphrase or shared secret, clients connecting to the service must somehow | ||
1744 | learn the service's name. VPN records in the GNU Name System can make this | ||
1745 | easier. | ||
1746 | |||
1747 | @node Fedora and the Firewall | ||
1748 | @subsection Fedora and the Firewall | ||
1749 | |||
1750 | |||
1751 | When using an exit node on Fedora 15, the standard firewall can create trouble | ||
1752 | even when not really exiting the local system! For IPv4, the standard rules seem | ||
1753 | fine. However, for IPv6 the standard rules prohibit traffic from the network | ||
1754 | range of the virtual interface created by the exit daemon to the local IPv6 | ||
1755 | address of the same interface (which is essentially loopback traffic, so you | ||
1756 | might suspect that a standard firewall would leave this traffic alone). However, | ||
1757 | as somehow for IPv6 the traffic is not recognized as originating from the local | ||
1758 | system (and as the connection is not already "established"), the firewall drops | ||
1759 | the traffic. You should still get ICMPv6 packets back, but that's obviously not | ||
1760 | very useful. | ||
1761 | |||
1762 | Possible ways to fix this include disabling the firewall (do you have a good | ||
1763 | reason for having it on?) or disabling the firewall at least for the GNUnet exit | ||
1764 | interface (or the respective IPv4/IPv6 address range). The best way to diagnose | ||
1765 | these kinds of problems in general involves setting the firewall to REJECT | ||
1766 | instead of DROP and to watch the traffic using wireshark (or tcpdump) to see if | ||
1767 | ICMP messages are generated when running some tests that should work. | ||
1768 | |||
1769 | @node Setting up VPN node for protocol translation and tunneling | ||
1770 | @subsection Setting up VPN node for protocol translation and tunneling | ||
1771 | |||
1772 | |||
1773 | The GNUnet VPN/PT subsystem enables you to tunnel IP traffic over the VPN to an | ||
1774 | exit node, from where it can then be forwarded to the Internet. This section | ||
1775 | documents how to setup VPN/PT on a node. Note that you can enable both the VPN | ||
1776 | and an exit on the same peer. In this case, IP traffic from your system may | ||
1777 | enter your peer's VPN and leave your peer's exit. This can be useful as a means | ||
1778 | to do protocol translation. For example, you might have an application that | ||
1779 | supports only IPv4 but needs to access an IPv6-only site. In this case, GNUnet | ||
1780 | would perform 4to6 protocol translation between the VPN (IPv4) and the | ||
1781 | Exit (IPv6). Similarly, 6to4 protocol translation is also possible. However, the | ||
1782 | primary use for GNUnet would be to access an Internet service running with an | ||
1783 | IP version that is not supported by your ISP. In this case, your IP traffic | ||
1784 | would be routed via GNUnet to a peer that has access to the Internet with the | ||
1785 | desired IP version. | ||
1786 | |||
1787 | Setting up an entry node into the GNUnet VPN primarily requires you to enable | ||
1788 | the "VPN/PT" option in "gnunet-setup". This will launch the | ||
1789 | "gnunet-service-vpn", "gnunet-service-dns" and "gnunet-daemon-pt" processes. | ||
1790 | The "gnunet-service-vpn" will create a virtual interface which will be used as | ||
1791 | the target for your IP traffic that enters the VPN. Additionally, a second | ||
1792 | virtual interface will be created by the "gnunet-service-dns" for your DNS | ||
1793 | traffic. You will then need to specify which traffic you want to tunnel over | ||
1794 | GNUnet. If your ISP only provides you with IPv4 or IPv6-access, you may choose | ||
1795 | to tunnel the other IP protocol over the GNUnet VPN. If you do not have an ISP | ||
1796 | (and are connected to other GNUnet peers via WLAN), you can also choose to | ||
1797 | tunnel all IP traffic over GNUnet. This might also provide you with some | ||
1798 | anonymity. After you enable the respective options and restart your peer, your | ||
1799 | Internet traffic should be tunneled over the GNUnet VPN. | ||
1800 | |||
1801 | The GNUnet VPN uses DNS-ALG to hijack your IP traffic. Whenever an application | ||
1802 | resolves a hostname (i.e. 'gnunet.org'), the "gnunet-daemon-pt" will instruct | ||
1803 | the "gnunet-service-dns" to intercept the request (possibly route it over GNUnet | ||
1804 | as well) and replace the normal answer with an IP in the range of the VPN's | ||
1805 | interface. "gnunet-daemon-pt" will then tell "gnunet-service-vpn" to forward all | ||
1806 | traffic it receives on the TUN interface via the VPN to the original | ||
1807 | destination. | ||
1808 | |||
1809 | For applications that do not use DNS, you can also manually create such a | ||
1810 | mapping using the gnunet-vpn command-line tool. Here, you specfiy the desired | ||
1811 | address family of the result (i.e. "-4"), and the intended target IP on the | ||
1812 | Internet ("-i 131.159.74.67") and "gnunet-vpn" will tell you which IP address in | ||
1813 | the range of your VPN tunnel was mapped. | ||
1814 | |||
1815 | gnunet-vpn can also be used to access "internal" services offered by GNUnet | ||
1816 | nodes. So if you happen to know a peer and a service offered by that peer, you | ||
1817 | can create an IP tunnel to that peer by specifying the peer's identity, service | ||
1818 | name and protocol (--tcp or --udp) and you will again receive an IP address that | ||
1819 | will terminate at the respective peer's service. | ||
diff --git a/doc/documentation/Makefile.am b/doc/documentation/Makefile.am new file mode 100644 index 000000000..7b844b985 --- /dev/null +++ b/doc/documentation/Makefile.am | |||
@@ -0,0 +1,233 @@ | |||
1 | # This Makefile.am is in the public domain | ||
2 | docdir = $(datadir)/doc/gnunet/ | ||
3 | |||
4 | infoimagedir = $(infodir)/images | ||
5 | |||
6 | #DOT_FILES = images/$(wildcard *.dot) | ||
7 | |||
8 | #DOT_VECTOR_GRAPHICS = \ | ||
9 | # $(DOT_FILES:%.dot=%.eps) \ | ||
10 | # $(DOT_FILES:%.dot=%.pdf) | ||
11 | |||
12 | AM_MAKEINFOHTMLFLAGS = --no-split --css-ref=docstyle.css | ||
13 | |||
14 | dist_infoimage_DATA = \ | ||
15 | images/gnunet-gtk-0-10-gns-a-done.png \ | ||
16 | images/gnunet-gtk-0-10-gns-a.png \ | ||
17 | images/daemon_lego_block.png \ | ||
18 | images/gnunet-gtk-0-10-gns.png \ | ||
19 | images/gnunet-0-10-peerinfo.png \ | ||
20 | images/gnunet-gtk-0-10-identity.png \ | ||
21 | images/gnunet-fs-gtk-0-10-star-tab.png \ | ||
22 | images/gnunet-gtk-0-10.png \ | ||
23 | images/gnunet-gtk-0-10-download-area.png \ | ||
24 | images/gnunet-gtk-0-10-search-selected.png \ | ||
25 | images/gnunet-gtk-0-10-fs-menu.png \ | ||
26 | images/gnunet-gtk-0-10-traffic.png \ | ||
27 | images/gnunet-gtk-0-10-fs.png \ | ||
28 | images/gnunet-namestore-gtk-phone.png \ | ||
29 | images/gnunet-gtk-0-10-fs-publish-editing.png \ | ||
30 | images/gnunet-namestore-gtk-vpn.png \ | ||
31 | images/gnunet-gtk-0-10-fs-published.png \ | ||
32 | images/gnunet-setup-exit.png \ | ||
33 | images/gnunet-gtk-0-10-fs-publish.png \ | ||
34 | images/iceweasel-preferences.png \ | ||
35 | images/gnunet-gtk-0-10-fs-publish-select.png \ | ||
36 | images/iceweasel-proxy.png \ | ||
37 | images/gnunet-gtk-0-10-fs-publish-with-file_0.png \ | ||
38 | images/service_lego_block.png \ | ||
39 | images/gnunet-gtk-0-10-fs-publish-with-file.png \ | ||
40 | images/service_stack.png \ | ||
41 | images/gnunet-gtk-0-10-fs-search.png \ | ||
42 | images/gnunet-tutorial-service.png \ | ||
43 | images/gnunet-tutorial-system.png \ | ||
44 | images/daemon_lego_block.svg \ | ||
45 | images/lego_stack.svg \ | ||
46 | images/service_lego_block.svg \ | ||
47 | images/structure.dot | ||
48 | |||
49 | # images/$(wildcard *.png) \ | ||
50 | # images/$(wildcard *.svg) | ||
51 | # $(DOT_FILES:%.dot=%.png) | ||
52 | |||
53 | #DOT_OPTIONS = \ | ||
54 | # -Gratio=.9 -Gnodesep=.005 -Granksep=.00005 \ | ||
55 | # -Nfontsite=9 -Nheight=.1 -Nwidth=.1 | ||
56 | |||
57 | # .dot.png: | ||
58 | # $(AM_V_DOT)$(DOT) -Tpng $(DOT_OPTIONS) < "$<" > "$(srcdir)/$@.tmp"; \ | ||
59 | # mv "$(srcdir)/$@.tmp" "$(srcdir)/$@" | ||
60 | |||
61 | # .dot.pdf: | ||
62 | # $(AM_V_DOT)$(DOT) -Tpdf $(DOT_OPTIONS) < "$<" > "$(srcdir)/$@.tmp"; \ | ||
63 | # mv "$(srcdir)/$@.tmp" "$(srcdir)/$@" | ||
64 | |||
65 | # .dot.eps: | ||
66 | # $(AM_V_DOT)$(DOT) -Teps $(DOT_OPTIONS) < "$<" > "$(srcdir)/$@.tmp"; \ | ||
67 | # mv "$(srcdir)/$@.tmp" "$(srcdir)/$@" | ||
68 | |||
69 | # .png.eps: | ||
70 | # $(AM_V_GEN)convert "$<" "$@-tmp.eps"; \ | ||
71 | # mv "$@-tmp.eps" "$@" | ||
72 | |||
73 | # pdf-local: $(DOT_FILES=%.dot=$(top_srcdir)/%.pdf) | ||
74 | # info-local: $(DOT_FILES=%.dot=$(top_srcdir)/%.png) | ||
75 | # ps-local: $(DOT_FILES=%.dot=$(top_srcdir)/%.eps) \ | ||
76 | # $(top_srcdir)/%D%/images/coreutils-size-map.eps | ||
77 | # dvi-local: ps-local | ||
78 | |||
79 | gnunet_tutorial_examples = \ | ||
80 | 001.c \ | ||
81 | 002.c \ | ||
82 | 003.c \ | ||
83 | 004.c \ | ||
84 | 005.c \ | ||
85 | 006.c \ | ||
86 | 007.c \ | ||
87 | 008.c \ | ||
88 | 009.c \ | ||
89 | 010.c \ | ||
90 | 011.c \ | ||
91 | 012.c \ | ||
92 | 013.c \ | ||
93 | 013.1.c \ | ||
94 | 014.c \ | ||
95 | 015.c \ | ||
96 | 016.c \ | ||
97 | 017.c \ | ||
98 | 018.c \ | ||
99 | 019.c \ | ||
100 | 020.c \ | ||
101 | 021.c \ | ||
102 | 022.c \ | ||
103 | 023.c \ | ||
104 | 024.c \ | ||
105 | 025.c \ | ||
106 | 026.c | ||
107 | |||
108 | info_TEXINFOS = \ | ||
109 | gnunet.texi \ | ||
110 | gnunet-c-tutorial.texi | ||
111 | |||
112 | gnunet_TEXINFOS = \ | ||
113 | chapters/developer.texi \ | ||
114 | chapters/installation.texi \ | ||
115 | chapters/philosophy.texi \ | ||
116 | chapters/user.texi \ | ||
117 | chapters/vocabulary.texi \ | ||
118 | chapters/configuration.texi \ | ||
119 | chapters/contributing.texi \ | ||
120 | fdl-1.3.texi \ | ||
121 | gpl-3.0.texi | ||
122 | |||
123 | EXTRA_DIST = \ | ||
124 | $(gnunet_TEXINFOS) \ | ||
125 | $(gnunet_tutorial_examples) \ | ||
126 | htmlxref.cnf \ | ||
127 | run-gendocs.sh \ | ||
128 | docstyle.css | ||
129 | |||
130 | |||
131 | # $(DOT_FILES) \ | ||
132 | # $(DOT_VECTOR_GRAPHICS) | ||
133 | |||
134 | DISTCLEANFILES = \ | ||
135 | gnunet.cps \ | ||
136 | gnunet-c-tutorial.cps \ | ||
137 | chapters/developer.cps \ | ||
138 | chapters/installation.cps \ | ||
139 | chapter/philosophy.cps \ | ||
140 | chapters/user.cps \ | ||
141 | chapters/configuration.cps \ | ||
142 | chapters/vocabulary.cps \ | ||
143 | fdl-1.3.cps \ | ||
144 | gpl-3.0.cps | ||
145 | |||
146 | # if HAVE_EXTENDED_DOCUMENTATION_BUILDING | ||
147 | daemon_lego_block.png: images/daemon_lego_block.svg | ||
148 | convert images/daemon_lego_block.svg images/daemon_lego_block.png && | ||
149 | pngcrush images/daemon_lego_block.png images/daemon_lego_block.png | ||
150 | |||
151 | service_lego_block.png: images/service_lego_block.svg | ||
152 | convert images/service_lego_block.svg images/service_lego_block.png && | ||
153 | pngcrush images/service_lego_block.png images/serivce_lego_block.png | ||
154 | |||
155 | lego_stack.png: images/lego_stack.svg | ||
156 | convert images/lego_stack.svg images/lego_stack.png && | ||
157 | pngcrush images/lego_stack.png images/lego_stack.png | ||
158 | |||
159 | # FIXME: The usage of 'date' strings causes a warning. | ||
160 | # version.texi: | ||
161 | # echo "@set UPDATED $(date +'%d %B %Y')" > $@ | ||
162 | # echo "@set UPDATED-MONTH $(date +'%B %Y')" >> $@ | ||
163 | # echo "@set EDITION $(PACKAGE_VERSION)" >> $@ | ||
164 | # echo "@set VERSION $(PACKAGE_VERSION)" >> $@ | ||
165 | |||
166 | # Workaround for makeinfo error. Whcih in turn introduces more | ||
167 | # date-related 'warnings'. Well. | ||
168 | version2.texi: | ||
169 | echo "@set UPDATED $(date +'%d %B %Y')" > $@ | ||
170 | echo "@set UPDATED-MONTH $(date +'%B %Y')" >> $@ | ||
171 | echo "@set EDITION $(PACKAGE_VERSION)" >> $@ | ||
172 | echo "@set VERSION $(PACKAGE_VERSION)" >> $@ | ||
173 | |||
174 | # FIXME: rm *.html and *.pdf | ||
175 | #doc-clean: | ||
176 | # @rm *.aux *.log *.toc *.cp *.cps | ||
177 | |||
178 | doc-all-install: | ||
179 | @mkdir -p $(DESTDIR)/$(docdir) | ||
180 | @mkdir -p $(DESTDIR)/$(infoimagedir) | ||
181 | @mkdir -p $(DESTDIR)/$(infodir) | ||
182 | @install -m 0755 gnunet.pdf $(DESTDIR)/$(docdir) | ||
183 | @install -m 0755 gnunet-c-tutorial.pdf $(DESTDIR)/$(docdir) | ||
184 | @install -m 0755 gnunet-c-tutorial.info $(DESTDIR)/$(infodir) | ||
185 | @install -m 0755 gnunet.info $(DESTDIR)/$(infodir) | ||
186 | @install gnunet.html $(DESTDIR)/$(docdir) | ||
187 | @install gnunet-c-tutorial.html $(DESTDIR)/$(docdir) | ||
188 | |||
189 | doc-gendoc-install: | ||
190 | @mkdir -p $(DESTDIR)/$(docdir) | ||
191 | @cp -r manual $(DESTDIR)/$(docdir) | ||
192 | |||
193 | # @cp -r images $(DESTDIR)/$(infoimagedir) | ||
194 | |||
195 | dev-build: version.texi version2.texi | ||
196 | @makeinfo --pdf gnunet.texi | ||
197 | @makeinfo --pdf gnunet-c-tutorial.texi | ||
198 | @makeinfo --html gnunet.texi | ||
199 | @makeinfo --html gnunet-c-tutorial.texi | ||
200 | @makeinfo --no-split gnunet.texi | ||
201 | @makeinfo --no-split gnunet-c-tutorial.texi | ||
202 | |||
203 | # TODO: Add more to clean. | ||
204 | clean: | ||
205 | @rm -f gnunet.pdf | ||
206 | @rm -f gnunet.html | ||
207 | @rm -f gnunet.info | ||
208 | @rm -f gnunet.info-1 | ||
209 | @rm -f gnunet.info-2 | ||
210 | @rm -f gnunet.info-3 | ||
211 | @rm -f gnunet-c-tutorial.pdf | ||
212 | @rm -f gnunet-c-tutorial.info | ||
213 | @rm -f gnunet-c-tutorial.html | ||
214 | @rm -fr gnunet.t2p | ||
215 | @rm -fr gnunet-c-tutorial.t2p | ||
216 | @rm -fr manual | ||
217 | |||
218 | # CLEANFILES = \ | ||
219 | # gnunet.log \ | ||
220 | # gnunet-c-tutorial.log \ | ||
221 | # $(wildcard *.aux) \ | ||
222 | # $(wildcard *.toc) \ | ||
223 | # $(wildcard *.cp) \ | ||
224 | # $(wildcard *.cps) | ||
225 | |||
226 | #.PHONY: version.texi | ||
227 | # if HAVE_EXTENDED_DOCUMENTATION_BUILDING_PDF | ||
228 | |||
229 | # if HAVE_EXTENDED_DOCUMENTATION_BUILDING_HTML | ||
230 | |||
231 | # endif | ||
232 | # endif | ||
233 | # endif | ||
diff --git a/doc/documentation/README.txt b/doc/documentation/README.txt new file mode 100644 index 000000000..e3eb6c8ac --- /dev/null +++ b/doc/documentation/README.txt | |||
@@ -0,0 +1,104 @@ | |||
1 | To be moved to an appropriate section of "how to write documentation" or | ||
2 | "how to contribute to the documentation": | ||
3 | |||
4 | 1. When writing documentation, please use gender-neutral wording when | ||
5 | referring to people, such as singular “theyâ€, “theirâ€, “themâ€, and | ||
6 | so forth. -> https://en.wikipedia.org/wiki/Singular_they | ||
7 | |||
8 | 2. Keep line length below 74 characters. | ||
9 | - Expection by texi2pdf output so far: URLs will break | ||
10 | (inserted whitespace) when they contain linebreaks | ||
11 | within the @url{} / @uref{}. | ||
12 | |||
13 | 3. Do not use tab characters (see chapter 2.1 texinfo manual) | ||
14 | |||
15 | 4. Use neutral language and third person perspective in the text | ||
16 | |||
17 | 4.1 So, when you refer to a user in general or addressing the user, | ||
18 | refer to (1). | ||
19 | 4.1.1 Unsolved exceptions for canonical reasons: | ||
20 | When refering to Alice, use "she". | ||
21 | When refering to Bob, use "he". | ||
22 | These are long established examples and they | ||
23 | should either be replaced (avoid Alice and Bob | ||
24 | examples when you can) or followed. | ||
25 | |||
26 | 5. Use 2 spaces between sentences, so instead of: | ||
27 | |||
28 | We do this and the other thing. This is done by foo. | ||
29 | |||
30 | Write: | ||
31 | |||
32 | We do this and the other thing. This is done by foo. | ||
33 | |||
34 | 6. Use @footnote{} instead of putting an @*ref{} to the footnote on a | ||
35 | collected footnote-page. | ||
36 | In a 200+ pages handbook it's better to have footnotes accessible | ||
37 | without having to skip over to the end. | ||
38 | |||
39 | 6.1 Avoid unnecessary footnotes, keep the text self-explanatory and | ||
40 | in a simple language where possible/necessary. | ||
41 | |||
42 | * Completion Levels: | ||
43 | |||
44 | ** chapters/philosophy: around 100% fixed after initial export. | ||
45 | |||
46 | * What's left to do | ||
47 | |||
48 | - Which Texlive modules are needed? Decrease the size. | ||
49 | - distro specific, or can we set requirements? | ||
50 | - Update the content of gnunet documentation. | ||
51 | - XXX: images are only generated for the html documentation | ||
52 | with gendoc.sh … FIXME! | ||
53 | - XXX: png,dot, and svg images MUST be converted to eps by the | ||
54 | build system. Right now they aren't, as a result: No images. | ||
55 | |||
56 | * How to use (hack) on this | ||
57 | |||
58 | ** with guix | ||
59 | |||
60 | Adjust accordingly, ie read the Guix Documentation: | ||
61 | setenv GUIX_PACKAGE_PATH "gnunet/contrib/packages/guix/packages" | ||
62 | guix environment gnunet-doc | ||
63 | and | ||
64 | guix build -f contrib/packages/guix/gnunet-doc.scm | ||
65 | |||
66 | ** without guix | ||
67 | |||
68 | You need to have Texinfo and Texlive in your path. | ||
69 | sh bootstrap | ||
70 | ./configure --enable-documentation | ||
71 | cd doc | ||
72 | make (format you want) | ||
73 | |||
74 | for example: make html, make info, make pdf | ||
75 | |||
76 | * structure (relations) | ||
77 | |||
78 | ** gnunet.texi | ||
79 | -> chapters/developer.texi | ||
80 | -> chapters/installation.texi | ||
81 | -> chapters/philosophy.texi | ||
82 | -> chapters/user.texi | ||
83 | -> chapters/vocabulary.texi | ||
84 | -> images/* | ||
85 | -> gpl-3.0.texi | ||
86 | -> fdl-1.3.texi | ||
87 | |||
88 | ** gnunet-c-tutorial.texi | ||
89 | -> figs/Service.pdf | ||
90 | -> figs/System.pdf | ||
91 | -> tutorial-examples/*.c | ||
92 | -> gpl-3.0.texi | ||
93 | -> fdl-1.3.texi | ||
94 | |||
95 | - gnunet-c-tutorial-v1.pdf: original LaTeX "gnunet-c-tutorial.pdf". | ||
96 | - man folder: the man pages. | ||
97 | - doxygen folder | ||
98 | - outdated-and-old-installation-instructions.txt: self described within the file. | ||
99 | |||
100 | |||
101 | Use `gendocs', add to the manual/ directory of the web site. | ||
102 | |||
103 | $ cd doc | ||
104 | $ gendocs.sh gnunet "GNUnet 0.10.X Reference Manual" | ||
diff --git a/doc/documentation/chapters/configuration.texi b/doc/documentation/chapters/configuration.texi new file mode 100644 index 000000000..286c72e7a --- /dev/null +++ b/doc/documentation/chapters/configuration.texi | |||
@@ -0,0 +1,5 @@ | |||
1 | @node Configuration Handbook | ||
2 | @chapter Configuration Handbook | ||
3 | |||
4 | This chapter has yet to be written. It is intended to be about in-depth | ||
5 | configuration of GNUnet. | ||
diff --git a/doc/documentation/chapters/contributing.texi b/doc/documentation/chapters/contributing.texi new file mode 100644 index 000000000..5844fb497 --- /dev/null +++ b/doc/documentation/chapters/contributing.texi | |||
@@ -0,0 +1,102 @@ | |||
1 | @node GNUnet Contributors Handbook | ||
2 | @chapter GNUnet Contributors Handbook | ||
3 | |||
4 | @menu | ||
5 | * Contributing to GNUnet:: | ||
6 | * Licenses of contributions:: | ||
7 | * Copyright Assignment:: | ||
8 | * Contributing to the Reference Manual:: | ||
9 | @end menu | ||
10 | |||
11 | @node Contributing to GNUnet | ||
12 | @section Contributing to GNUnet | ||
13 | |||
14 | @node Licenses of contributions | ||
15 | @section Licenses of contributions | ||
16 | |||
17 | GNUnet is a @uref{https://www.gnu.org/, GNU} package. | ||
18 | All code contributions must thus be put under the | ||
19 | @uref{https://www.gnu.org/copyleft/gpl.html, GNU Public License (GPL)}. | ||
20 | All documentation should be put under FSF approved licenses | ||
21 | (see @uref{https://www.gnu.org/copyleft/fdl.html, fdl}). | ||
22 | |||
23 | By submitting documentation, translations, and other content to GNUnet | ||
24 | you automatically grant the right to publish code under the | ||
25 | GNU Public License and documentation under either or both the | ||
26 | GNU Public License or the GNU Free Documentation License. | ||
27 | When contributing to the GNUnet project, GNU standards and the | ||
28 | @uref{https://www.gnu.org/philosophy/philosophy.html, GNU philosophy} | ||
29 | should be adhered to. | ||
30 | |||
31 | @cindex copyright assignment | ||
32 | @node Copyright Assignment | ||
33 | @section Copyright Assignment | ||
34 | We require a formal copyright assignment for GNUnet contributors | ||
35 | to GNUnet e.V.; nevertheless, we do allow pseudonymous contributions. | ||
36 | By signing the copyright agreement and submitting your code (or | ||
37 | documentation) to us, you agree to share the rights to your code | ||
38 | with GNUnet e.V.; GNUnet e.V. receives non-exclusive ownership | ||
39 | rights, and in particular is allowed to dual-license the code. You | ||
40 | retain non-exclusive rights to your contributions, so you can also | ||
41 | share your contributions freely with other projects. | ||
42 | |||
43 | GNUnet e.V. will publish all accepted contributions under the GPLv3 | ||
44 | or any later version. The association may decide to publish | ||
45 | contributions under additional licenses (dual-licensing). | ||
46 | |||
47 | We do not intentionally remove your name from your contributions; | ||
48 | however, due to extensive editing it is not always trivial to | ||
49 | attribute contributors properly. If you find that you significantly | ||
50 | contributed to a file (or the project as a whole) and are not listed | ||
51 | in the respective authors file or section, please do let us know. | ||
52 | |||
53 | @node Contributing to the Reference Manual | ||
54 | @section Contributing to the Reference Manual | ||
55 | |||
56 | @itemize @bullet | ||
57 | |||
58 | @item When writing documentation, please use | ||
59 | @uref{https://en.wikipedia.org/wiki/Singular_they, gender-neutral wording} | ||
60 | when referring to people, such as singular “theyâ€, “theirâ€, “themâ€, and so | ||
61 | forth. | ||
62 | |||
63 | @item Keep line length below 74 characters, except for URLs. | ||
64 | URLs break in the PDF output when they contain linebreaks. | ||
65 | |||
66 | @item Do not use tab characters (see chapter 2.1 texinfo manual) | ||
67 | |||
68 | @item Use neutral language and third person perspective in the text | ||
69 | |||
70 | @item So, when you refer to a user in general or addressing the user, | ||
71 | refer to (1). | ||
72 | @itemize @bullet | ||
73 | @item Unsolved exceptions for canonical reasons: When refering to Alice, | ||
74 | use "she". When refering to Bob, use "he". These are long established | ||
75 | examples and they should either be replaced (avoid Alice and Bob | ||
76 | examples when you can) or followed. | ||
77 | @end itemize | ||
78 | |||
79 | @c FIXME: This is questionable, it feels like bike shed painging to do | ||
80 | @c this for several k lines. It only helps to jump between sentences in | ||
81 | @c editors afaik. | ||
82 | @c @item Use 2 spaces between sentences, so instead of: | ||
83 | |||
84 | @c @example | ||
85 | @c We do this and the other thing. This is done by foo. | ||
86 | @c @end example | ||
87 | |||
88 | @c Write: | ||
89 | |||
90 | @c @example | ||
91 | @c We do this and the other thing. This is done by foo. | ||
92 | @c @end example | ||
93 | |||
94 | @item Use @@footnote@{@} instead of putting an @@*ref@{@} to the | ||
95 | footnote on a collected footnote-page. | ||
96 | In a 200+ pages handbook it's better to have footnotes accessible | ||
97 | without having to skip over to the end. | ||
98 | |||
99 | @item Avoid unnecessary footnotes, keep the text self-explanatory and | ||
100 | in a simple language where possible/necessary. | ||
101 | |||
102 | @end itemize | ||
diff --git a/doc/documentation/chapters/developer.texi b/doc/documentation/chapters/developer.texi new file mode 100644 index 000000000..70fd7c7eb --- /dev/null +++ b/doc/documentation/chapters/developer.texi | |||
@@ -0,0 +1,8341 @@ | |||
1 | @c *********************************************************************** | ||
2 | @node GNUnet Developer Handbook | ||
3 | @chapter GNUnet Developer Handbook | ||
4 | |||
5 | This book is intended to be an introduction for programmers that want to | ||
6 | extend the GNUnet framework. GNUnet is more than a simple peer-to-peer | ||
7 | application. | ||
8 | |||
9 | For developers, GNUnet is: | ||
10 | |||
11 | @itemize @bullet | ||
12 | @item developed by a community that believes in the GNU philosophy | ||
13 | @item Free Software (Free as in Freedom), licensed under the | ||
14 | GNU General Public License | ||
15 | @item A set of standards, including coding conventions and | ||
16 | architectural rules | ||
17 | @item A set of layered protocols, both specifying the communication | ||
18 | between peers as well as the communication between components | ||
19 | of a single peer | ||
20 | @item A set of libraries with well-defined APIs suitable for | ||
21 | writing extensions | ||
22 | @end itemize | ||
23 | |||
24 | In particular, the architecture specifies that a peer consists of many | ||
25 | processes communicating via protocols. Processes can be written in almost | ||
26 | any language. | ||
27 | C and Java @footnote{As well as Guile} APIs exist for accessing existing | ||
28 | services and for writing extensions. | ||
29 | It is possible to write extensions in other languages by | ||
30 | implementing the necessary IPC protocols. | ||
31 | |||
32 | GNUnet can be extended and improved along many possible dimensions, and | ||
33 | anyone interested in Free Software and Freedom-enhancing Networking is | ||
34 | welcome to join the effort. This Developer Handbook attempts to provide | ||
35 | an initial introduction to some of the key design choices and central | ||
36 | components of the system. | ||
37 | This part of the GNUNet documentation is far from complete, | ||
38 | and we welcome informed contributions, be it in the form of | ||
39 | new chapters, sections or insightful comments. | ||
40 | |||
41 | @menu | ||
42 | * Developer Introduction:: | ||
43 | * Code overview:: | ||
44 | * System Architecture:: | ||
45 | * Subsystem stability:: | ||
46 | * Naming conventions and coding style guide:: | ||
47 | * Build-system:: | ||
48 | * Developing extensions for GNUnet using the gnunet-ext template:: | ||
49 | * Writing testcases:: | ||
50 | * TESTING library:: | ||
51 | * Performance regression analysis with Gauger:: | ||
52 | * TESTBED Subsystem:: | ||
53 | * libgnunetutil:: | ||
54 | * Automatic Restart Manager (ARM):: | ||
55 | * TRANSPORT Subsystem:: | ||
56 | * NAT library:: | ||
57 | * Distance-Vector plugin:: | ||
58 | * SMTP plugin:: | ||
59 | * Bluetooth plugin:: | ||
60 | * WLAN plugin:: | ||
61 | * ATS Subsystem:: | ||
62 | * CORE Subsystem:: | ||
63 | * CADET Subsystem:: | ||
64 | * NSE Subsystem:: | ||
65 | * HOSTLIST Subsystem:: | ||
66 | * IDENTITY Subsystem:: | ||
67 | * NAMESTORE Subsystem:: | ||
68 | * PEERINFO Subsystem:: | ||
69 | * PEERSTORE Subsystem:: | ||
70 | * SET Subsystem:: | ||
71 | * STATISTICS Subsystem:: | ||
72 | * Distributed Hash Table (DHT):: | ||
73 | * GNU Name System (GNS):: | ||
74 | * GNS Namecache:: | ||
75 | * REVOCATION Subsystem:: | ||
76 | * File-sharing (FS) Subsystem:: | ||
77 | * REGEX Subsystem:: | ||
78 | @end menu | ||
79 | |||
80 | @node Developer Introduction | ||
81 | @section Developer Introduction | ||
82 | |||
83 | This Developer Handbook is intended as first introduction to GNUnet for | ||
84 | new developers that want to extend the GNUnet framework. After the | ||
85 | introduction, each of the GNUnet subsystems (directories in the | ||
86 | @file{src/} tree) is (supposed to be) covered in its own chapter. In | ||
87 | addition to this documentation, GNUnet developers should be aware of the | ||
88 | services available on the GNUnet server to them. | ||
89 | |||
90 | New developers can have a look a the GNUnet tutorials for C and java | ||
91 | available in the @file{src/} directory of the repository or under the | ||
92 | following links: | ||
93 | |||
94 | @c ** FIXME: Link to files in source, not online. | ||
95 | @c ** FIXME: Where is the Java tutorial? | ||
96 | @itemize @bullet | ||
97 | @item @uref{https://gnunet.org/git/gnunet.git/plain/doc/gnunet-c-tutorial.pdf, GNUnet C tutorial} | ||
98 | @item GNUnet Java tutorial | ||
99 | @end itemize | ||
100 | |||
101 | In addition to the GNUnet Reference Documentation you are reading, | ||
102 | the GNUnet server at @uref{https://gnunet.org} contains | ||
103 | various resources for GNUnet developers and those | ||
104 | who aspire to become regular contributors. | ||
105 | They are all conveniently reachable via the "Developer" | ||
106 | entry in the navigation menu. Some additional tools (such as static | ||
107 | analysis reports) require a special developer access to perform certain | ||
108 | operations. If you want (or require) access, you should contact | ||
109 | @uref{http://grothoff.org/christian/, Christian Grothoff}, | ||
110 | GNUnet's maintainer. | ||
111 | |||
112 | The public subsystems on the GNUnet server that help developers are: | ||
113 | |||
114 | @itemize @bullet | ||
115 | |||
116 | @item The version control system (git) keeps our code and enables | ||
117 | distributed development. | ||
118 | It is pubclicly accessible at @uref{https://gnunet.org/git/}. | ||
119 | Only developers with write access can commit code, everyone else is | ||
120 | encouraged to submit patches to the | ||
121 | @uref{https://lists.gnu.org/mailman/listinfo/gnunet-developers, GNUnet-developers mailinglist}. | ||
122 | |||
123 | @item The bugtracking system (Mantis). | ||
124 | We use it to track feature requests, open bug reports and their | ||
125 | resolutions. | ||
126 | It can be accessed at @uref{https://gnunet.org/bugs/}. | ||
127 | Anyone can report bugs, but only developers can claim to have fixed them. | ||
128 | |||
129 | @item Our site installation of the | ||
130 | CI@footnote{Continuous Integration} system @code{Buildbot} is used | ||
131 | to check GNUnet builds automatically on a range of platforms. | ||
132 | The web interface of this CI is exposed at | ||
133 | @uref{https://gnunet.org/buildbot/}. | ||
134 | Builds are triggered automatically 30 minutes after the last commit to | ||
135 | our repository was made. | ||
136 | |||
137 | @item The current quality of our automated test suite is assessed using | ||
138 | Code coverage analysis. This analysis is run daily; however the webpage | ||
139 | is only updated if all automated tests pass at that time. Testcases that | ||
140 | improve our code coverage are always welcome. | ||
141 | |||
142 | @item We try to automatically find bugs using a static analysis scan. | ||
143 | This scan is run daily; however the webpage is only updated if all | ||
144 | automated tests pass at the time. Note that not everything that is | ||
145 | flagged by the analysis is a bug, sometimes even good code can be marked | ||
146 | as possibly problematic. Nevertheless, developers are encouraged to at | ||
147 | least be aware of all issues in their code that are listed. | ||
148 | |||
149 | @item We use Gauger for automatic performance regression visualization. | ||
150 | Details on how to use Gauger are here. | ||
151 | |||
152 | @item We use @uref{http://junit.org/, junit} to automatically test | ||
153 | @command{gnunet-java}. | ||
154 | Automatically generated, current reports on the test suite are here. | ||
155 | |||
156 | @item We use Cobertura to generate test coverage reports for gnunet-java. | ||
157 | Current reports on test coverage are here. | ||
158 | |||
159 | @end itemize | ||
160 | |||
161 | |||
162 | |||
163 | @c *********************************************************************** | ||
164 | @menu | ||
165 | * Project overview:: | ||
166 | @end menu | ||
167 | |||
168 | @node Project overview | ||
169 | @subsection Project overview | ||
170 | |||
171 | The GNUnet project consists at this point of several sub-projects. This | ||
172 | section is supposed to give an initial overview about the various | ||
173 | sub-projects. Note that this description also lists projects that are far | ||
174 | from complete, including even those that have literally not a single line | ||
175 | of code in them yet. | ||
176 | |||
177 | GNUnet sub-projects in order of likely relevance are currently: | ||
178 | |||
179 | @table @asis | ||
180 | |||
181 | @item @command{gnunet} | ||
182 | Core of the P2P framework, including file-sharing, VPN and | ||
183 | chat applications; this is what the Developer Handbook covers mostly | ||
184 | @item @command{gnunet-gtk} | ||
185 | Gtk+-based user interfaces, including: | ||
186 | |||
187 | @itemize @bullet | ||
188 | @item @command{gnunet-fs-gtk} (file-sharing), | ||
189 | @item @command{gnunet-statistics-gtk} (statistics over time), | ||
190 | @item @command{gnunet-peerinfo-gtk} | ||
191 | (information about current connections and known peers), | ||
192 | @item @command{gnunet-chat-gtk} (chat GUI) and | ||
193 | @item @command{gnunet-setup} (setup tool for "everything") | ||
194 | @end itemize | ||
195 | |||
196 | @item @command{gnunet-fuse} | ||
197 | Mounting directories shared via GNUnet's file-sharing | ||
198 | on GNU/Linux distributions | ||
199 | @item @command{gnunet-update} | ||
200 | Installation and update tool | ||
201 | @item @command{gnunet-ext} | ||
202 | Template for starting 'external' GNUnet projects | ||
203 | @item @command{gnunet-java} | ||
204 | Java APIs for writing GNUnet services and applications | ||
205 | @c ** FIXME: Point to new website repository once we have it: | ||
206 | @c ** @item svn/gnunet-www/ Code and media helping drive the GNUnet | ||
207 | @c website | ||
208 | @item @command{eclectic} | ||
209 | Code to run GNUnet nodes on testbeds for research, development, | ||
210 | testing and evaluation | ||
211 | @c ** FIXME: Solve the status and location of gnunet-qt | ||
212 | @item @command{gnunet-qt} | ||
213 | Qt-based GNUnet GUI (is it depreacated?) | ||
214 | @item @command{gnunet-cocoa} | ||
215 | cocoa-based GNUnet GUI (is it depreacated?) | ||
216 | @item @command{gnunet-guile} | ||
217 | |||
218 | @end table | ||
219 | |||
220 | We are also working on various supporting libraries and tools: | ||
221 | @c ** FIXME: What about gauger, and what about libmwmodem? | ||
222 | |||
223 | @table @asis | ||
224 | @item @command{libextractor} | ||
225 | GNU libextractor (meta data extraction) | ||
226 | @item @command{libmicrohttpd} | ||
227 | GNU libmicrohttpd (embedded HTTP(S) server library) | ||
228 | @item @command{gauger} | ||
229 | Tool for performance regression analysis | ||
230 | @item @command{monkey} | ||
231 | Tool for automated debugging of distributed systems | ||
232 | @item @command{libmwmodem} | ||
233 | Library for accessing satellite connection quality | ||
234 | reports | ||
235 | @item @command{libgnurl} | ||
236 | gnURL (feature-restricted variant of cURL/libcurl) | ||
237 | @end table | ||
238 | |||
239 | Finally, there are various external projects (see links for a list of | ||
240 | those that have a public website) which build on top of the GNUnet | ||
241 | framework. | ||
242 | |||
243 | @c *********************************************************************** | ||
244 | @node Code overview | ||
245 | @section Code overview | ||
246 | |||
247 | This section gives a brief overview of the GNUnet source code. | ||
248 | Specifically, we sketch the function of each of the subdirectories in | ||
249 | the @file{gnunet/src/} directory. The order given is roughly bottom-up | ||
250 | (in terms of the layers of the system). | ||
251 | |||
252 | @table @asis | ||
253 | @item @file{util/} --- libgnunetutil | ||
254 | Library with general utility functions, all | ||
255 | GNUnet binaries link against this library. Anything from memory | ||
256 | allocation and data structures to cryptography and inter-process | ||
257 | communication. The goal is to provide an OS-independent interface and | ||
258 | more 'secure' or convenient implementations of commonly used primitives. | ||
259 | The API is spread over more than a dozen headers, developers should study | ||
260 | those closely to avoid duplicating existing functions. | ||
261 | @pxref{libgnunetutil}. | ||
262 | @item @file{hello/} --- libgnunethello | ||
263 | HELLO messages are used to | ||
264 | describe under which addresses a peer can be reached (for example, | ||
265 | protocol, IP, port). This library manages parsing and generating of HELLO | ||
266 | messages. | ||
267 | @item @file{block/} --- libgnunetblock | ||
268 | The DHT and other components of GNUnet | ||
269 | store information in units called 'blocks'. Each block has a type and the | ||
270 | type defines a particular format and how that binary format is to be | ||
271 | linked to a hash code (the key for the DHT and for databases). The block | ||
272 | library is a wapper around block plugins which provide the necessary | ||
273 | functions for each block type. | ||
274 | @item @file{statistics/} --- statistics service | ||
275 | The statistics service enables associating | ||
276 | values (of type uint64_t) with a componenet name and a string. The main | ||
277 | uses is debugging (counting events), performance tracking and user | ||
278 | entertainment (what did my peer do today?). | ||
279 | @item @file{arm/} --- Automatic Restart Manager (ARM) | ||
280 | The automatic-restart-manager (ARM) service | ||
281 | is the GNUnet master service. Its role is to start gnunet-services, to | ||
282 | re-start them when they crashed and finally to shut down the system when | ||
283 | requested. | ||
284 | @item @file{peerinfo/} --- peerinfo service | ||
285 | The peerinfo service keeps track of which peers are known | ||
286 | to the local peer and also tracks the validated addresses for each peer | ||
287 | (in the form of a HELLO message) for each of those peers. The peer is not | ||
288 | necessarily connected to all peers known to the peerinfo service. | ||
289 | Peerinfo provides persistent storage for peer identities --- peers are | ||
290 | not forgotten just because of a system restart. | ||
291 | @item @file{datacache/} --- libgnunetdatacache | ||
292 | The datacache library provides (temporary) block storage for the DHT. | ||
293 | Existing plugins can store blocks in Sqlite, Postgres or MySQL databases. | ||
294 | All data stored in the cache is lost when the peer is stopped or | ||
295 | restarted (datacache uses temporary tables). | ||
296 | @item @file{datastore/} --- datastore service | ||
297 | The datastore service stores file-sharing blocks in | ||
298 | databases for extended periods of time. In contrast to the datacache, data | ||
299 | is not lost when peers restart. However, quota restrictions may still | ||
300 | cause old, expired or low-priority data to be eventually discarded. | ||
301 | Existing plugins can store blocks in Sqlite, Postgres or MySQL databases. | ||
302 | @item @file{template/} --- service template | ||
303 | Template for writing a new service. Does nothing. | ||
304 | @item @file{ats/} --- Automatic Transport Selection | ||
305 | The automatic transport selection (ATS) service | ||
306 | is responsible for deciding which address (i.e. | ||
307 | which transport plugin) should be used for communication with other peers, | ||
308 | and at what bandwidth. | ||
309 | @item @file{nat/} --- libgnunetnat | ||
310 | Library that provides basic functions for NAT traversal. | ||
311 | The library supports NAT traversal with | ||
312 | manual hole-punching by the user, UPnP and ICMP-based autonomous NAT | ||
313 | traversal. The library also includes an API for testing if the current | ||
314 | configuration works and the @code{gnunet-nat-server} which provides an | ||
315 | external service to test the local configuration. | ||
316 | @item @file{fragmentation/} --- libgnunetfragmentation | ||
317 | Some transports (UDP and WLAN, mostly) have restrictions on the maximum | ||
318 | transfer unit (MTU) for packets. The fragmentation library can be used to | ||
319 | break larger packets into chunks of at most 1k and transmit the resulting | ||
320 | fragments reliabily (with acknowledgement, retransmission, timeouts, | ||
321 | etc.). | ||
322 | @item @file{transport/} --- transport service | ||
323 | The transport service is responsible for managing the | ||
324 | basic P2P communication. It uses plugins to support P2P communication | ||
325 | over TCP, UDP, HTTP, HTTPS and other protocols.The transport service | ||
326 | validates peer addresses, enforces bandwidth restrictions, limits the | ||
327 | total number of connections and enforces connectivity restrictions (i.e. | ||
328 | friends-only). | ||
329 | @item @file{peerinfo-tool/} --- gnunet-peerinfo | ||
330 | This directory contains the gnunet-peerinfo binary which can be used to | ||
331 | inspect the peers and HELLOs known to the peerinfo service. | ||
332 | @item @file{core/} | ||
333 | The core service is responsible for establishing encrypted, authenticated | ||
334 | connections with other peers, encrypting and decrypting messages and | ||
335 | forwarding messages to higher-level services that are interested in them. | ||
336 | @item @file{testing/} --- libgnunettesting | ||
337 | The testing library allows starting (and stopping) peers | ||
338 | for writing testcases. | ||
339 | It also supports automatic generation of configurations for peers | ||
340 | ensuring that the ports and paths are disjoint. libgnunettesting is also | ||
341 | the foundation for the testbed service | ||
342 | @item @file{testbed/} --- testbed service | ||
343 | The testbed service is used for creating small or large scale deployments | ||
344 | of GNUnet peers for evaluation of protocols. | ||
345 | It facilitates peer depolyments on multiple | ||
346 | hosts (for example, in a cluster) and establishing varous network | ||
347 | topologies (both underlay and overlay). | ||
348 | @item @file{nse/} --- Network Size Estimation | ||
349 | The network size estimation (NSE) service | ||
350 | implements a protocol for (securely) estimating the current size of the | ||
351 | P2P network. | ||
352 | @item @file{dht/} --- distributed hash table | ||
353 | The distributed hash table (DHT) service provides a | ||
354 | distributed implementation of a hash table to store blocks under hash | ||
355 | keys in the P2P network. | ||
356 | @item @file{hostlist/} --- hostlist service | ||
357 | The hostlist service allows learning about | ||
358 | other peers in the network by downloading HELLO messages from an HTTP | ||
359 | server, can be configured to run such an HTTP server and also implements | ||
360 | a P2P protocol to advertise and automatically learn about other peers | ||
361 | that offer a public hostlist server. | ||
362 | @item @file{topology/} --- topology service | ||
363 | The topology service is responsible for | ||
364 | maintaining the mesh topology. It tries to maintain connections to friends | ||
365 | (depending on the configuration) and also tries to ensure that the peer | ||
366 | has a decent number of active connections at all times. If necessary, new | ||
367 | connections are added. All peers should run the topology service, | ||
368 | otherwise they may end up not being connected to any other peer (unless | ||
369 | some other service ensures that core establishes the required | ||
370 | connections). The topology service also tells the transport service which | ||
371 | connections are permitted (for friend-to-friend networking) | ||
372 | @item @file{fs/} --- file-sharing | ||
373 | The file-sharing (FS) service implements GNUnet's | ||
374 | file-sharing application. Both anonymous file-sharing (using gap) and | ||
375 | non-anonymous file-sharing (using dht) are supported. | ||
376 | @item @file{cadet/} --- cadet service | ||
377 | The CADET service provides a general-purpose routing abstraction to create | ||
378 | end-to-end encrypted tunnels in mesh networks. We wrote a paper | ||
379 | documenting key aspects of the design. | ||
380 | @item @file{tun/} --- libgnunettun | ||
381 | Library for building IPv4, IPv6 packets and creating | ||
382 | checksums for UDP, TCP and ICMP packets. The header | ||
383 | defines C structs for common Internet packet formats and in particular | ||
384 | structs for interacting with TUN (virtual network) interfaces. | ||
385 | @item @file{mysql/} --- libgnunetmysql | ||
386 | Library for creating and executing prepared MySQL | ||
387 | statements and to manage the connection to the MySQL database. | ||
388 | Essentially a lightweight wrapper for the interaction between GNUnet | ||
389 | components and libmysqlclient. | ||
390 | @item @file{dns/} | ||
391 | Service that allows intercepting and modifying DNS requests of | ||
392 | the local machine. Currently used for IPv4-IPv6 protocol translation | ||
393 | (DNS-ALG) as implemented by "pt/" and for the GNUnet naming system. The | ||
394 | service can also be configured to offer an exit service for DNS traffic. | ||
395 | @item @file{vpn/} --- VPN service | ||
396 | The virtual public network (VPN) service provides a virtual | ||
397 | tunnel interface (VTUN) for IP routing over GNUnet. | ||
398 | Needs some other peers to run an "exit" service to work. | ||
399 | Can be activated using the "gnunet-vpn" tool or integrated with DNS using | ||
400 | the "pt" daemon. | ||
401 | @item @file{exit/} | ||
402 | Daemon to allow traffic from the VPN to exit this | ||
403 | peer to the Internet or to specific IP-based services of the local peer. | ||
404 | Currently, an exit service can only be restricted to IPv4 or IPv6, not to | ||
405 | specific ports and or IP address ranges. If this is not acceptable, | ||
406 | additional firewall rules must be added manually. exit currently only | ||
407 | works for normal UDP, TCP and ICMP traffic; DNS queries need to leave the | ||
408 | system via a DNS service. | ||
409 | @item @file{pt/} | ||
410 | protocol translation daemon. This daemon enables 4-to-6, | ||
411 | 6-to-4, 4-over-6 or 6-over-4 transitions for the local system. It | ||
412 | essentially uses "DNS" to intercept DNS replies and then maps results to | ||
413 | those offered by the VPN, which then sends them using mesh to some daemon | ||
414 | offering an appropriate exit service. | ||
415 | @item @file{identity/} | ||
416 | Management of egos (alter egos) of a user; identities are | ||
417 | essentially named ECC private keys and used for zones in the GNU name | ||
418 | system and for namespaces in file-sharing, but might find other uses later | ||
419 | @item @file{revocation/} | ||
420 | Key revocation service, can be used to revoke the | ||
421 | private key of an identity if it has been compromised | ||
422 | @item @file{namecache/} | ||
423 | Cache for resolution results for the GNU name system; | ||
424 | data is encrypted and can be shared among users, | ||
425 | loss of the data should ideally only result in a | ||
426 | performance degradation (persistence not required) | ||
427 | @item @file{namestore/} | ||
428 | Database for the GNU name system with per-user private information, | ||
429 | persistence required | ||
430 | @item @file{gns/} | ||
431 | GNU name system, a GNU approach to DNS and PKI. | ||
432 | @item @file{dv/} | ||
433 | A plugin for distance-vector (DV)-based routing. | ||
434 | DV consists of a service and a transport plugin to provide peers | ||
435 | with the illusion of a direct P2P connection for connections | ||
436 | that use multiple (typically up to 3) hops in the actual underlay network. | ||
437 | @item @file{regex/} | ||
438 | Service for the (distributed) evaluation of regular expressions. | ||
439 | @item @file{scalarproduct/} | ||
440 | The scalar product service offers an API to perform a secure multiparty | ||
441 | computation which calculates a scalar product between two peers | ||
442 | without exposing the private input vectors of the peers to each other. | ||
443 | @item @file{consensus/} | ||
444 | The consensus service will allow a set of peers to agree | ||
445 | on a set of values via a distributed set union computation. | ||
446 | @item @file{rest/} | ||
447 | The rest API allows access to GNUnet services using RESTful interaction. | ||
448 | The services provide plugins that can exposed by the rest server. | ||
449 | @item @file{experimentation/} | ||
450 | The experimentation daemon coordinates distributed | ||
451 | experimentation to evaluate transport and ATS properties. | ||
452 | @end table | ||
453 | |||
454 | @c *********************************************************************** | ||
455 | @node System Architecture | ||
456 | @section System Architecture | ||
457 | |||
458 | GNUnet developers like LEGOs. The blocks are indestructible, can be | ||
459 | stacked together to construct complex buildings and it is generally easy | ||
460 | to swap one block for a different one that has the same shape. GNUnet's | ||
461 | architecture is based on LEGOs: | ||
462 | |||
463 | @c images here | ||
464 | |||
465 | This chapter documents the GNUnet LEGO system, also known as GNUnet's | ||
466 | system architecture. | ||
467 | |||
468 | The most common GNUnet component is a service. Services offer an API (or | ||
469 | several, depending on what you count as "an API") which is implemented as | ||
470 | a library. The library communicates with the main process of the service | ||
471 | using a service-specific network protocol. The main process of the service | ||
472 | typically doesn't fully provide everything that is needed --- it has holes | ||
473 | to be filled by APIs to other services. | ||
474 | |||
475 | A special kind of component in GNUnet are user interfaces and daemons. | ||
476 | Like services, they have holes to be filled by APIs of other services. | ||
477 | Unlike services, daemons do not implement their own network protocol and | ||
478 | they have no API: | ||
479 | |||
480 | The GNUnet system provides a range of services, daemons and user | ||
481 | interfaces, which are then combined into a layered GNUnet instance (also | ||
482 | known as a peer). | ||
483 | |||
484 | Note that while it is generally possible to swap one service for another | ||
485 | compatible service, there is often only one implementation. However, | ||
486 | during development we often have a "new" version of a service in parallel | ||
487 | with an "old" version. While the "new" version is not working, developers | ||
488 | working on other parts of the service can continue their development by | ||
489 | simply using the "old" service. Alternative design ideas can also be | ||
490 | easily investigated by swapping out individual components. This is | ||
491 | typically achieved by simply changing the name of the "BINARY" in the | ||
492 | respective configuration section. | ||
493 | |||
494 | Key properties of GNUnet services are that they must be separate | ||
495 | processes and that they must protect themselves by applying tight error | ||
496 | checking against the network protocol they implement (thereby achieving a | ||
497 | certain degree of robustness). | ||
498 | |||
499 | On the other hand, the APIs are implemented to tolerate failures of the | ||
500 | service, isolating their host process from errors by the service. If the | ||
501 | service process crashes, other services and daemons around it should not | ||
502 | also fail, but instead wait for the service process to be restarted by | ||
503 | ARM. | ||
504 | |||
505 | |||
506 | @c *********************************************************************** | ||
507 | @node Subsystem stability | ||
508 | @section Subsystem stability | ||
509 | |||
510 | This section documents the current stability of the various GNUnet | ||
511 | subsystems. Stability here describes the expected degree of compatibility | ||
512 | with future versions of GNUnet. For each subsystem we distinguish between | ||
513 | compatibility on the P2P network level (communication protocol between | ||
514 | peers), the IPC level (communication between the service and the service | ||
515 | library) and the API level (stability of the API). P2P compatibility is | ||
516 | relevant in terms of which applications are likely going to be able to | ||
517 | communicate with future versions of the network. IPC communication is | ||
518 | relevant for the implementation of language bindings that re-implement the | ||
519 | IPC messages. Finally, API compatibility is relevant to developers that | ||
520 | hope to be able to avoid changes to applications build on top of the APIs | ||
521 | of the framework. | ||
522 | |||
523 | The following table summarizes our current view of the stability of the | ||
524 | respective protocols or APIs: | ||
525 | |||
526 | @multitable @columnfractions .20 .20 .20 .20 | ||
527 | @headitem Subsystem @tab P2P @tab IPC @tab C API | ||
528 | @item util @tab n/a @tab n/a @tab stable | ||
529 | @item arm @tab n/a @tab stable @tab stable | ||
530 | @item ats @tab n/a @tab unstable @tab testing | ||
531 | @item block @tab n/a @tab n/a @tab stable | ||
532 | @item cadet @tab testing @tab testing @tab testing | ||
533 | @item consensus @tab experimental @tab experimental @tab experimental | ||
534 | @item core @tab stable @tab stable @tab stable | ||
535 | @item datacache @tab n/a @tab n/a @tab stable | ||
536 | @item datastore @tab n/a @tab stable @tab stable | ||
537 | @item dht @tab stable @tab stable @tab stable | ||
538 | @item dns @tab stable @tab stable @tab stable | ||
539 | @item dv @tab testing @tab testing @tab n/a | ||
540 | @item exit @tab testing @tab n/a @tab n/a | ||
541 | @item fragmentation @tab stable @tab n/a @tab stable | ||
542 | @item fs @tab stable @tab stable @tab stable | ||
543 | @item gns @tab stable @tab stable @tab stable | ||
544 | @item hello @tab n/a @tab n/a @tab testing | ||
545 | @item hostlist @tab stable @tab stable @tab n/a | ||
546 | @item identity @tab stable @tab stable @tab n/a | ||
547 | @item multicast @tab experimental @tab experimental @tab experimental | ||
548 | @item mysql @tab stable @tab n/a @tab stable | ||
549 | @item namestore @tab n/a @tab stable @tab stable | ||
550 | @item nat @tab n/a @tab n/a @tab stable | ||
551 | @item nse @tab stable @tab stable @tab stable | ||
552 | @item peerinfo @tab n/a @tab stable @tab stable | ||
553 | @item psyc @tab experimental @tab experimental @tab experimental | ||
554 | @item pt @tab n/a @tab n/a @tab n/a | ||
555 | @item regex @tab stable @tab stable @tab stable | ||
556 | @item revocation @tab stable @tab stable @tab stable | ||
557 | @item social @tab experimental @tab experimental @tab experimental | ||
558 | @item statistics @tab n/a @tab stable @tab stable | ||
559 | @item testbed @tab n/a @tab testing @tab testing | ||
560 | @item testing @tab n/a @tab n/a @tab testing | ||
561 | @item topology @tab n/a @tab n/a @tab n/a | ||
562 | @item transport @tab stable @tab stable @tab stable | ||
563 | @item tun @tab n/a @tab n/a @tab stable | ||
564 | @item vpn @tab testing @tab n/a @tab n/a | ||
565 | @end multitable | ||
566 | |||
567 | Here is a rough explanation of the values: | ||
568 | |||
569 | @table @samp | ||
570 | @item stable | ||
571 | No incompatible changes are planned at this time; for IPC/APIs, if | ||
572 | there are incompatible changes, they will be minor and might only require | ||
573 | minimal changes to existing code; for P2P, changes will be avoided if at | ||
574 | all possible for the 0.10.x-series | ||
575 | |||
576 | @item testing | ||
577 | No incompatible changes are | ||
578 | planned at this time, but the code is still known to be in flux; so while | ||
579 | we have no concrete plans, our expectation is that there will still be | ||
580 | minor modifications; for P2P, changes will likely be extensions that | ||
581 | should not break existing code | ||
582 | |||
583 | @item unstable | ||
584 | Changes are planned and will happen; however, they | ||
585 | will not be totally radical and the result should still resemble what is | ||
586 | there now; nevertheless, anticipated changes will break protocol/API | ||
587 | compatibility | ||
588 | |||
589 | @item experimental | ||
590 | Changes are planned and the result may look nothing like | ||
591 | what the API/protocol looks like today | ||
592 | |||
593 | @item unknown | ||
594 | Someone should think about where this subsystem headed | ||
595 | |||
596 | @item n/a | ||
597 | This subsystem does not have an API/IPC-protocol/P2P-protocol | ||
598 | @end table | ||
599 | |||
600 | @c *********************************************************************** | ||
601 | @node Naming conventions and coding style guide | ||
602 | @section Naming conventions and coding style guide | ||
603 | |||
604 | Here you can find some rules to help you write code for GNUnet. | ||
605 | |||
606 | @c *********************************************************************** | ||
607 | @menu | ||
608 | * Naming conventions:: | ||
609 | * Coding style:: | ||
610 | @end menu | ||
611 | |||
612 | @node Naming conventions | ||
613 | @subsection Naming conventions | ||
614 | |||
615 | |||
616 | @c *********************************************************************** | ||
617 | @menu | ||
618 | * include files:: | ||
619 | * binaries:: | ||
620 | * logging:: | ||
621 | * configuration:: | ||
622 | * exported symbols:: | ||
623 | * private (library-internal) symbols (including structs and macros):: | ||
624 | * testcases:: | ||
625 | * performance tests:: | ||
626 | * src/ directories:: | ||
627 | @end menu | ||
628 | |||
629 | @node include files | ||
630 | @subsubsection include files | ||
631 | |||
632 | @itemize @bullet | ||
633 | @item _lib: library without need for a process | ||
634 | @item _service: library that needs a service process | ||
635 | @item _plugin: plugin definition | ||
636 | @item _protocol: structs used in network protocol | ||
637 | @item exceptions: | ||
638 | @itemize @bullet | ||
639 | @item gnunet_config.h --- generated | ||
640 | @item platform.h --- first included | ||
641 | @item plibc.h --- external library | ||
642 | @item gnunet_common.h --- fundamental routines | ||
643 | @item gnunet_directories.h --- generated | ||
644 | @item gettext.h --- external library | ||
645 | @end itemize | ||
646 | @end itemize | ||
647 | |||
648 | @c *********************************************************************** | ||
649 | @node binaries | ||
650 | @subsubsection binaries | ||
651 | |||
652 | @itemize @bullet | ||
653 | @item gnunet-service-xxx: service process (has listen socket) | ||
654 | @item gnunet-daemon-xxx: daemon process (no listen socket) | ||
655 | @item gnunet-helper-xxx[-yyy]: SUID helper for module xxx | ||
656 | @item gnunet-yyy: command-line tool for end-users | ||
657 | @item libgnunet_plugin_xxx_yyy.so: plugin for API xxx | ||
658 | @item libgnunetxxx.so: library for API xxx | ||
659 | @end itemize | ||
660 | |||
661 | @c *********************************************************************** | ||
662 | @node logging | ||
663 | @subsubsection logging | ||
664 | |||
665 | @itemize @bullet | ||
666 | @item services and daemons use their directory name in | ||
667 | @code{GNUNET_log_setup} (i.e. 'core') and log using | ||
668 | plain 'GNUNET_log'. | ||
669 | @item command-line tools use their full name in | ||
670 | @code{GNUNET_log_setup} (i.e. 'gnunet-publish') and log using | ||
671 | plain 'GNUNET_log'. | ||
672 | @item service access libraries log using | ||
673 | '@code{GNUNET_log_from}' and use '@code{DIRNAME-api}' for the | ||
674 | component (i.e. 'core-api') | ||
675 | @item pure libraries (without associated service) use | ||
676 | '@code{GNUNET_log_from}' with the component set to their | ||
677 | library name (without lib or '@file{.so}'), | ||
678 | which should also be their directory name (i.e. '@file{nat}') | ||
679 | @item plugins should use '@code{GNUNET_log_from}' | ||
680 | with the directory name and the plugin name combined to produce | ||
681 | the component name (i.e. 'transport-tcp'). | ||
682 | @item logging should be unified per-file by defining a | ||
683 | @code{LOG} macro with the appropriate arguments, | ||
684 | along these lines: | ||
685 | |||
686 | @example | ||
687 | #define LOG(kind,...) | ||
688 | GNUNET_log_from (kind, "example-api",__VA_ARGS__) | ||
689 | @end example | ||
690 | |||
691 | @end itemize | ||
692 | |||
693 | @c *********************************************************************** | ||
694 | @node configuration | ||
695 | @subsubsection configuration | ||
696 | |||
697 | @itemize @bullet | ||
698 | @item paths (that are substituted in all filenames) are in PATHS | ||
699 | (have as few as possible) | ||
700 | @item all options for a particular module (@file{src/MODULE}) | ||
701 | are under @code{[MODULE]} | ||
702 | @item options for a plugin of a module | ||
703 | are under @code{[MODULE-PLUGINNAME]} | ||
704 | @end itemize | ||
705 | |||
706 | @c *********************************************************************** | ||
707 | @node exported symbols | ||
708 | @subsubsection exported symbols | ||
709 | |||
710 | @itemize @bullet | ||
711 | @item must start with @code{GNUNET_modulename_} and be defined in | ||
712 | @file{modulename.c} | ||
713 | @item exceptions: those defined in @file{gnunet_common.h} | ||
714 | @end itemize | ||
715 | |||
716 | @c *********************************************************************** | ||
717 | @node private (library-internal) symbols (including structs and macros) | ||
718 | @subsubsection private (library-internal) symbols (including structs and macros) | ||
719 | |||
720 | @itemize @bullet | ||
721 | @item must NOT start with any prefix | ||
722 | @item must not be exported in a way that linkers could use them or@ other | ||
723 | libraries might see them via headers; they must be either | ||
724 | declared/defined in C source files or in headers that are in the | ||
725 | respective directory under @file{src/modulename/} and NEVER be declared | ||
726 | in @file{src/include/}. | ||
727 | @end itemize | ||
728 | |||
729 | @node testcases | ||
730 | @subsubsection testcases | ||
731 | |||
732 | @itemize @bullet | ||
733 | @item must be called @file{test_module-under-test_case-description.c} | ||
734 | @item "case-description" maybe omitted if there is only one test | ||
735 | @end itemize | ||
736 | |||
737 | @c *********************************************************************** | ||
738 | @node performance tests | ||
739 | @subsubsection performance tests | ||
740 | |||
741 | @itemize @bullet | ||
742 | @item must be called @file{perf_module-under-test_case-description.c} | ||
743 | @item "case-description" maybe omitted if there is only one performance | ||
744 | test | ||
745 | @item Must only be run if @code{HAVE_BENCHMARKS} is satisfied | ||
746 | @end itemize | ||
747 | |||
748 | @c *********************************************************************** | ||
749 | @node src/ directories | ||
750 | @subsubsection src/ directories | ||
751 | |||
752 | @itemize @bullet | ||
753 | @item gnunet-NAME: end-user applications (i.e., gnunet-search, gnunet-arm) | ||
754 | @item gnunet-service-NAME: service processes with accessor library (i.e., | ||
755 | gnunet-service-arm) | ||
756 | @item libgnunetNAME: accessor library (_service.h-header) or standalone | ||
757 | library (_lib.h-header) | ||
758 | @item gnunet-daemon-NAME: daemon process without accessor library (i.e., | ||
759 | gnunet-daemon-hostlist) and no GNUnet management port | ||
760 | @item libgnunet_plugin_DIR_NAME: loadable plugins (i.e., | ||
761 | libgnunet_plugin_transport_tcp) | ||
762 | @end itemize | ||
763 | |||
764 | @cindex Coding style | ||
765 | @node Coding style | ||
766 | @subsection Coding style | ||
767 | |||
768 | @c XXX: Adjust examples to GNU Standards! | ||
769 | @itemize @bullet | ||
770 | @item We follow the GNU Coding Standards (@pxref{Top, The GNU Coding Standards,, standards, The GNU Coding Standards}); | ||
771 | @item Indentation is done with spaces, two per level, no tabs; | ||
772 | @item C99 struct initialization is fine; | ||
773 | @item declare only one variable per line, for example: | ||
774 | |||
775 | @noindent | ||
776 | instead of | ||
777 | |||
778 | @example | ||
779 | int i,j; | ||
780 | @end example | ||
781 | |||
782 | @noindent | ||
783 | write: | ||
784 | |||
785 | @example | ||
786 | int i; | ||
787 | int j; | ||
788 | @end example | ||
789 | |||
790 | @c TODO: include actual example from a file in source | ||
791 | |||
792 | @noindent | ||
793 | This helps keep diffs small and forces developers to think precisely about | ||
794 | the type of every variable. | ||
795 | Note that @code{char *} is different from @code{const char*} and | ||
796 | @code{int} is different from @code{unsigned int} or @code{uint32_t}. | ||
797 | Each variable type should be chosen with care. | ||
798 | |||
799 | @item While @code{goto} should generally be avoided, having a | ||
800 | @code{goto} to the end of a function to a block of clean up | ||
801 | statements (free, close, etc.) can be acceptable. | ||
802 | |||
803 | @item Conditions should be written with constants on the left (to avoid | ||
804 | accidental assignment) and with the 'true' target being either the | ||
805 | 'error' case or the significantly simpler continuation. For example: | ||
806 | |||
807 | @example | ||
808 | if (0 != stat ("filename," &sbuf)) @{ | ||
809 | error(); | ||
810 | @} | ||
811 | else @{ | ||
812 | /* handle normal case here */ | ||
813 | @} | ||
814 | @end example | ||
815 | |||
816 | @noindent | ||
817 | instead of | ||
818 | |||
819 | @example | ||
820 | if (stat ("filename," &sbuf) == 0) @{ | ||
821 | /* handle normal case here */ | ||
822 | @} else @{ | ||
823 | error(); | ||
824 | @} | ||
825 | @end example | ||
826 | |||
827 | @noindent | ||
828 | If possible, the error clause should be terminated with a 'return' (or | ||
829 | 'goto' to some cleanup routine) and in this case, the 'else' clause | ||
830 | should be omitted: | ||
831 | |||
832 | @example | ||
833 | if (0 != stat ("filename," &sbuf)) @{ | ||
834 | error(); | ||
835 | return; | ||
836 | @} | ||
837 | /* handle normal case here */ | ||
838 | @end example | ||
839 | |||
840 | This serves to avoid deep nesting. The 'constants on the left' rule | ||
841 | applies to all constants (including. @code{GNUNET_SCHEDULER_NO_TASK}), | ||
842 | NULL, and enums). With the two above rules (constants on left, errors in | ||
843 | 'true' branch), there is only one way to write most branches correctly. | ||
844 | |||
845 | @item Combined assignments and tests are allowed if they do not hinder | ||
846 | code clarity. For example, one can write: | ||
847 | |||
848 | @example | ||
849 | if (NULL == (value = lookup_function())) @{ | ||
850 | error(); | ||
851 | return; | ||
852 | @} | ||
853 | @end example | ||
854 | |||
855 | @item Use @code{break} and @code{continue} wherever possible to avoid | ||
856 | deep(er) nesting. Thus, we would write: | ||
857 | |||
858 | @example | ||
859 | next = head; | ||
860 | while (NULL != (pos = next)) @{ | ||
861 | next = pos->next; | ||
862 | if (! should_free (pos)) | ||
863 | continue; | ||
864 | GNUNET_CONTAINER_DLL_remove (head, tail, pos); | ||
865 | GNUNET_free (pos); | ||
866 | @} | ||
867 | @end example | ||
868 | |||
869 | instead of | ||
870 | |||
871 | @example | ||
872 | next = head; while (NULL != (pos = next)) @{ | ||
873 | next = pos->next; | ||
874 | if (should_free (pos)) @{ | ||
875 | /* unnecessary nesting! */ | ||
876 | GNUNET_CONTAINER_DLL_remove (head, tail, pos); | ||
877 | GNUNET_free (pos); | ||
878 | @} | ||
879 | @} | ||
880 | @end example | ||
881 | |||
882 | @item We primarily use @code{for} and @code{while} loops. | ||
883 | A @code{while} loop is used if the method for advancing in the loop is | ||
884 | not a straightforward increment operation. In particular, we use: | ||
885 | |||
886 | @example | ||
887 | next = head; | ||
888 | while (NULL != (pos = next)) | ||
889 | @{ | ||
890 | next = pos->next; | ||
891 | if (! should_free (pos)) | ||
892 | continue; | ||
893 | GNUNET_CONTAINER_DLL_remove (head, tail, pos); | ||
894 | GNUNET_free (pos); | ||
895 | @} | ||
896 | @end example | ||
897 | |||
898 | to free entries in a list (as the iteration changes the structure of the | ||
899 | list due to the free; the equivalent @code{for} loop does no longer | ||
900 | follow the simple @code{for} paradigm of @code{for(INIT;TEST;INC)}). | ||
901 | However, for loops that do follow the simple @code{for} paradigm we do | ||
902 | use @code{for}, even if it involves linked lists: | ||
903 | |||
904 | @example | ||
905 | /* simple iteration over a linked list */ | ||
906 | for (pos = head; | ||
907 | NULL != pos; | ||
908 | pos = pos->next) | ||
909 | @{ | ||
910 | use (pos); | ||
911 | @} | ||
912 | @end example | ||
913 | |||
914 | |||
915 | @item The first argument to all higher-order functions in GNUnet must be | ||
916 | declared to be of type @code{void *} and is reserved for a closure. We do | ||
917 | not use inner functions, as trampolines would conflict with setups that | ||
918 | use non-executable stacks. | ||
919 | The first statement in a higher-order function, which unusually should | ||
920 | be part of the variable declarations, should assign the | ||
921 | @code{cls} argument to the precise expected type. For example: | ||
922 | |||
923 | @example | ||
924 | int callback (void *cls, char *args) @{ | ||
925 | struct Foo *foo = cls; | ||
926 | int other_variables; | ||
927 | |||
928 | /* rest of function */ | ||
929 | @} | ||
930 | @end example | ||
931 | |||
932 | |||
933 | @item It is good practice to write complex @code{if} expressions instead | ||
934 | of using deeply nested @code{if} statements. However, except for addition | ||
935 | and multiplication, all operators should use parens. This is fine: | ||
936 | |||
937 | @example | ||
938 | if ( (1 == foo) || ((0 == bar) && (x != y)) ) | ||
939 | return x; | ||
940 | @end example | ||
941 | |||
942 | |||
943 | However, this is not: | ||
944 | |||
945 | @example | ||
946 | if (1 == foo) | ||
947 | return x; | ||
948 | if (0 == bar && x != y) | ||
949 | return x; | ||
950 | @end example | ||
951 | |||
952 | @noindent | ||
953 | Note that splitting the @code{if} statement above is debateable as the | ||
954 | @code{return x} is a very trivial statement. However, once the logic after | ||
955 | the branch becomes more complicated (and is still identical), the "or" | ||
956 | formulation should be used for sure. | ||
957 | |||
958 | @item There should be two empty lines between the end of the function and | ||
959 | the comments describing the following function. There should be a single | ||
960 | empty line after the initial variable declarations of a function. If a | ||
961 | function has no local variables, there should be no initial empty line. If | ||
962 | a long function consists of several complex steps, those steps might be | ||
963 | separated by an empty line (possibly followed by a comment describing the | ||
964 | following step). The code should not contain empty lines in arbitrary | ||
965 | places; if in doubt, it is likely better to NOT have an empty line (this | ||
966 | way, more code will fit on the screen). | ||
967 | @end itemize | ||
968 | |||
969 | @c *********************************************************************** | ||
970 | @node Build-system | ||
971 | @section Build-system | ||
972 | |||
973 | If you have code that is likely not to compile or build rules you might | ||
974 | want to not trigger for most developers, use @code{if HAVE_EXPERIMENTAL} | ||
975 | in your @file{Makefile.am}. | ||
976 | Then it is OK to (temporarily) add non-compiling (or known-to-not-port) | ||
977 | code. | ||
978 | |||
979 | If you want to compile all testcases but NOT run them, run configure with | ||
980 | the @code{--enable-test-suppression} option. | ||
981 | |||
982 | If you want to run all testcases, including those that take a while, run | ||
983 | configure with the @code{--enable-expensive-testcases} option. | ||
984 | |||
985 | If you want to compile and run benchmarks, run configure with the | ||
986 | @code{--enable-benchmarks} option. | ||
987 | |||
988 | If you want to obtain code coverage results, run configure with the | ||
989 | @code{--enable-coverage} option and run the @file{coverage.sh} script in | ||
990 | the @file{contrib/} directory. | ||
991 | |||
992 | @cindex gnunet-ext | ||
993 | @node Developing extensions for GNUnet using the gnunet-ext template | ||
994 | @section Developing extensions for GNUnet using the gnunet-ext template | ||
995 | |||
996 | For developers who want to write extensions for GNUnet we provide the | ||
997 | gnunet-ext template to provide an easy to use skeleton. | ||
998 | |||
999 | gnunet-ext contains the build environment and template files for the | ||
1000 | development of GNUnet services, command line tools, APIs and tests. | ||
1001 | |||
1002 | First of all you have to obtain gnunet-ext from git: | ||
1003 | |||
1004 | @example | ||
1005 | git clone https://gnunet.org/git/gnunet-ext.git | ||
1006 | @end example | ||
1007 | |||
1008 | The next step is to bootstrap and configure it. For configure you have to | ||
1009 | provide the path containing GNUnet with | ||
1010 | @code{--with-gnunet=/path/to/gnunet} and the prefix where you want the | ||
1011 | install the extension using @code{--prefix=/path/to/install}: | ||
1012 | |||
1013 | @example | ||
1014 | ./bootstrap | ||
1015 | ./configure --prefix=/path/to/install --with-gnunet=/path/to/gnunet | ||
1016 | @end example | ||
1017 | |||
1018 | When your GNUnet installation is not included in the default linker search | ||
1019 | path, you have to add @code{/path/to/gnunet} to the file | ||
1020 | @file{/etc/ld.so.conf} and run @code{ldconfig} or your add it to the | ||
1021 | environmental variable @code{LD_LIBRARY_PATH} by using | ||
1022 | |||
1023 | @example | ||
1024 | export LD_LIBRARY_PATH=/path/to/gnunet/lib | ||
1025 | @end example | ||
1026 | |||
1027 | @cindex writing testcases | ||
1028 | @node Writing testcases | ||
1029 | @section Writing testcases | ||
1030 | |||
1031 | Ideally, any non-trivial GNUnet code should be covered by automated | ||
1032 | testcases. Testcases should reside in the same place as the code that is | ||
1033 | being tested. The name of source files implementing tests should begin | ||
1034 | with @code{test_} followed by the name of the file that contains | ||
1035 | the code that is being tested. | ||
1036 | |||
1037 | Testcases in GNUnet should be integrated with the autotools build system. | ||
1038 | This way, developers and anyone building binary packages will be able to | ||
1039 | run all testcases simply by running @code{make check}. The final | ||
1040 | testcases shipped with the distribution should output at most some brief | ||
1041 | progress information and not display debug messages by default. The | ||
1042 | success or failure of a testcase must be indicated by returning zero | ||
1043 | (success) or non-zero (failure) from the main method of the testcase. | ||
1044 | The integration with the autotools is relatively straightforward and only | ||
1045 | requires modifications to the @file{Makefile.am} in the directory | ||
1046 | containing the testcase. For a testcase testing the code in @file{foo.c} | ||
1047 | the @file{Makefile.am} would contain the following lines: | ||
1048 | |||
1049 | @example | ||
1050 | check_PROGRAMS = test_foo | ||
1051 | TESTS = $(check_PROGRAMS) | ||
1052 | test_foo_SOURCES = test_foo.c | ||
1053 | test_foo_LDADD = $(top_builddir)/src/util/libgnunetutil.la | ||
1054 | @end example | ||
1055 | |||
1056 | Naturally, other libraries used by the testcase may be specified in the | ||
1057 | @code{LDADD} directive as necessary. | ||
1058 | |||
1059 | Often testcases depend on additional input files, such as a configuration | ||
1060 | file. These support files have to be listed using the @code{EXTRA_DIST} | ||
1061 | directive in order to ensure that they are included in the distribution. | ||
1062 | |||
1063 | Example: | ||
1064 | |||
1065 | @example | ||
1066 | EXTRA_DIST = test_foo_data.conf | ||
1067 | @end example | ||
1068 | |||
1069 | Executing @code{make check} will run all testcases in the current | ||
1070 | directory and all subdirectories. Testcases can be compiled individually | ||
1071 | by running @code{make test_foo} and then invoked directly using | ||
1072 | @code{./test_foo}. Note that due to the use of plugins in GNUnet, it is | ||
1073 | typically necessary to run @code{make install} before running any | ||
1074 | testcases. Thus the canonical command @code{make check install} has to be | ||
1075 | changed to @code{make install check} for GNUnet. | ||
1076 | |||
1077 | @cindex TESTING library | ||
1078 | @node TESTING library | ||
1079 | @section TESTING library | ||
1080 | |||
1081 | The TESTING library is used for writing testcases which involve starting a | ||
1082 | single or multiple peers. While peers can also be started by testcases | ||
1083 | using the ARM subsystem, using TESTING library provides an elegant way to | ||
1084 | do this. The configurations of the peers are auto-generated from a given | ||
1085 | template to have non-conflicting port numbers ensuring that peers' | ||
1086 | services do not run into bind errors. This is achieved by testing ports' | ||
1087 | availability by binding a listening socket to them before allocating them | ||
1088 | to services in the generated configurations. | ||
1089 | |||
1090 | An another advantage while using TESTING is that it shortens the testcase | ||
1091 | startup time as the hostkeys for peers are copied from a pre-computed set | ||
1092 | of hostkeys instead of generating them at peer startup which may take a | ||
1093 | considerable amount of time when starting multiple peers or on an embedded | ||
1094 | processor. | ||
1095 | |||
1096 | TESTING also allows for certain services to be shared among peers. This | ||
1097 | feature is invaluable when testing with multiple peers as it helps to | ||
1098 | reduce the number of services run per each peer and hence the total | ||
1099 | number of processes run per testcase. | ||
1100 | |||
1101 | TESTING library only handles creating, starting and stopping peers. | ||
1102 | Features useful for testcases such as connecting peers in a topology are | ||
1103 | not available in TESTING but are available in the TESTBED subsystem. | ||
1104 | Furthermore, TESTING only creates peers on the localhost, however by | ||
1105 | using TESTBED testcases can benefit from creating peers across multiple | ||
1106 | hosts. | ||
1107 | |||
1108 | @menu | ||
1109 | * API:: | ||
1110 | * Finer control over peer stop:: | ||
1111 | * Helper functions:: | ||
1112 | * Testing with multiple processes:: | ||
1113 | @end menu | ||
1114 | |||
1115 | @cindex TESTING API | ||
1116 | @node API | ||
1117 | @subsection API | ||
1118 | |||
1119 | TESTING abstracts a group of peers as a TESTING system. All peers in a | ||
1120 | system have common hostname and no two services of these peers have a | ||
1121 | same port or a UNIX domain socket path. | ||
1122 | |||
1123 | TESTING system can be created with the function | ||
1124 | @code{GNUNET_TESTING_system_create()} which returns a handle to the | ||
1125 | system. This function takes a directory path which is used for generating | ||
1126 | the configurations of peers, an IP address from which connections to the | ||
1127 | peers' services should be allowed, the hostname to be used in peers' | ||
1128 | configuration, and an array of shared service specifications of type | ||
1129 | @code{struct GNUNET_TESTING_SharedService}. | ||
1130 | |||
1131 | The shared service specification must specify the name of the service to | ||
1132 | share, the configuration pertaining to that shared service and the | ||
1133 | maximum number of peers that are allowed to share a single instance of | ||
1134 | the shared service. | ||
1135 | |||
1136 | TESTING system created with @code{GNUNET_TESTING_system_create()} chooses | ||
1137 | ports from the default range @code{12000} - @code{56000} while | ||
1138 | auto-generating configurations for peers. | ||
1139 | This range can be customised with the function | ||
1140 | @code{GNUNET_TESTING_system_create_with_portrange()}. This function is | ||
1141 | similar to @code{GNUNET_TESTING_system_create()} except that it take 2 | ||
1142 | additional parameters --- the start and end of the port range to use. | ||
1143 | |||
1144 | A TESTING system is destroyed with the funciton | ||
1145 | @code{GNUNET_TESTING_system_destory()}. This function takes the handle of | ||
1146 | the system and a flag to remove the files created in the directory used | ||
1147 | to generate configurations. | ||
1148 | |||
1149 | A peer is created with the function | ||
1150 | @code{GNUNET_TESTING_peer_configure()}. This functions takes the system | ||
1151 | handle, a configuration template from which the configuration for the peer | ||
1152 | is auto-generated and the index from where the hostkey for the peer has to | ||
1153 | be copied from. When successfull, this function returs a handle to the | ||
1154 | peer which can be used to start and stop it and to obtain the identity of | ||
1155 | the peer. If unsuccessful, a NULL pointer is returned with an error | ||
1156 | message. This function handles the generated configuration to have | ||
1157 | non-conflicting ports and paths. | ||
1158 | |||
1159 | Peers can be started and stopped by calling the functions | ||
1160 | @code{GNUNET_TESTING_peer_start()} and @code{GNUNET_TESTING_peer_stop()} | ||
1161 | respectively. A peer can be destroyed by calling the function | ||
1162 | @code{GNUNET_TESTING_peer_destroy}. When a peer is destroyed, the ports | ||
1163 | and paths in allocated in its configuration are reclaimed for usage in new | ||
1164 | peers. | ||
1165 | |||
1166 | @c *********************************************************************** | ||
1167 | @node Finer control over peer stop | ||
1168 | @subsection Finer control over peer stop | ||
1169 | |||
1170 | Using @code{GNUNET_TESTING_peer_stop()} is normally fine for testcases. | ||
1171 | However, calling this function for each peer is inefficient when trying to | ||
1172 | shutdown multiple peers as this function sends the termination signal to | ||
1173 | the given peer process and waits for it to terminate. It would be faster | ||
1174 | in this case to send the termination signals to the peers first and then | ||
1175 | wait on them. This is accomplished by the functions | ||
1176 | @code{GNUNET_TESTING_peer_kill()} which sends a termination signal to the | ||
1177 | peer, and the function @code{GNUNET_TESTING_peer_wait()} which waits on | ||
1178 | the peer. | ||
1179 | |||
1180 | Further finer control can be achieved by choosing to stop a peer | ||
1181 | asynchronously with the function @code{GNUNET_TESTING_peer_stop_async()}. | ||
1182 | This function takes a callback parameter and a closure for it in addition | ||
1183 | to the handle to the peer to stop. The callback function is called with | ||
1184 | the given closure when the peer is stopped. Using this function | ||
1185 | eliminates blocking while waiting for the peer to terminate. | ||
1186 | |||
1187 | An asynchronous peer stop can be cancelled by calling the function | ||
1188 | @code{GNUNET_TESTING_peer_stop_async_cancel()}. Note that calling this | ||
1189 | function does not prevent the peer from terminating if the termination | ||
1190 | signal has already been sent to it. It does, however, cancels the | ||
1191 | callback to be called when the peer is stopped. | ||
1192 | |||
1193 | @c *********************************************************************** | ||
1194 | @node Helper functions | ||
1195 | @subsection Helper functions | ||
1196 | |||
1197 | Most of the testcases can benefit from an abstraction which configures a | ||
1198 | peer and starts it. This is provided by the function | ||
1199 | @code{GNUNET_TESTING_peer_run()}. This function takes the testing | ||
1200 | directory pathname, a configuration template, a callback and its closure. | ||
1201 | This function creates a peer in the given testing directory by using the | ||
1202 | configuration template, starts the peer and calls the given callback with | ||
1203 | the given closure. | ||
1204 | |||
1205 | The function @code{GNUNET_TESTING_peer_run()} starts the ARM service of | ||
1206 | the peer which starts the rest of the configured services. A similar | ||
1207 | function @code{GNUNET_TESTING_service_run} can be used to just start a | ||
1208 | single service of a peer. In this case, the peer's ARM service is not | ||
1209 | started; instead, only the given service is run. | ||
1210 | |||
1211 | @c *********************************************************************** | ||
1212 | @node Testing with multiple processes | ||
1213 | @subsection Testing with multiple processes | ||
1214 | |||
1215 | When testing GNUnet, the splitting of the code into a services and clients | ||
1216 | often complicates testing. The solution to this is to have the testcase | ||
1217 | fork @code{gnunet-service-arm}, ask it to start the required server and | ||
1218 | daemon processes and then execute appropriate client actions (to test the | ||
1219 | client APIs or the core module or both). If necessary, multiple ARM | ||
1220 | services can be forked using different ports (!) to simulate a network. | ||
1221 | However, most of the time only one ARM process is needed. Note that on | ||
1222 | exit, the testcase should shutdown ARM with a @code{TERM} signal (to give | ||
1223 | it the chance to cleanly stop its child processes). | ||
1224 | |||
1225 | The following code illustrates spawning and killing an ARM process from a | ||
1226 | testcase: | ||
1227 | |||
1228 | @example | ||
1229 | static void run (void *cls, | ||
1230 | char *const *args, | ||
1231 | const char *cfgfile, | ||
1232 | const struct GNUNET_CONFIGURATION_Handle *cfg) @{ | ||
1233 | struct GNUNET_OS_Process *arm_pid; | ||
1234 | arm_pid = GNUNET_OS_start_process (NULL, | ||
1235 | NULL, | ||
1236 | "gnunet-service-arm", | ||
1237 | "gnunet-service-arm", | ||
1238 | "-c", | ||
1239 | cfgname, | ||
1240 | NULL); | ||
1241 | /* do real test work here */ | ||
1242 | if (0 != GNUNET_OS_process_kill (arm_pid, SIGTERM)) | ||
1243 | GNUNET_log_strerror | ||
1244 | (GNUNET_ERROR_TYPE_WARNING, "kill"); | ||
1245 | GNUNET_assert (GNUNET_OK == GNUNET_OS_process_wait (arm_pid)); | ||
1246 | GNUNET_OS_process_close (arm_pid); @} | ||
1247 | |||
1248 | GNUNET_PROGRAM_run (argc, argv, | ||
1249 | "NAME-OF-TEST", | ||
1250 | "nohelp", | ||
1251 | options, | ||
1252 | &run, | ||
1253 | cls); | ||
1254 | @end example | ||
1255 | |||
1256 | |||
1257 | An alternative way that works well to test plugins is to implement a | ||
1258 | mock-version of the environment that the plugin expects and then to | ||
1259 | simply load the plugin directly. | ||
1260 | |||
1261 | @c *********************************************************************** | ||
1262 | @node Performance regression analysis with Gauger | ||
1263 | @section Performance regression analysis with Gauger | ||
1264 | |||
1265 | To help avoid performance regressions, GNUnet uses Gauger. Gauger is a | ||
1266 | simple logging tool that allows remote hosts to send performance data to | ||
1267 | a central server, where this data can be analyzed and visualized. Gauger | ||
1268 | shows graphs of the repository revisions and the performace data recorded | ||
1269 | for each revision, so sudden performance peaks or drops can be identified | ||
1270 | and linked to a specific revision number. | ||
1271 | |||
1272 | In the case of GNUnet, the buildbots log the performance data obtained | ||
1273 | during the tests after each build. The data can be accesed on GNUnet's | ||
1274 | Gauger page. | ||
1275 | |||
1276 | The menu on the left allows to select either the results of just one | ||
1277 | build bot (under "Hosts") or review the data from all hosts for a given | ||
1278 | test result (under "Metrics"). In case of very different absolute value | ||
1279 | of the results, for instance arm vs. amd64 machines, the option | ||
1280 | "Normalize" on a metric view can help to get an idea about the | ||
1281 | performance evolution across all hosts. | ||
1282 | |||
1283 | Using Gauger in GNUnet and having the performance of a module tracked over | ||
1284 | time is very easy. First of course, the testcase must generate some | ||
1285 | consistent metric, which makes sense to have logged. Highly volatile or | ||
1286 | random dependant metrics probably are not ideal candidates for meaningful | ||
1287 | regression detection. | ||
1288 | |||
1289 | To start logging any value, just include @code{gauger.h} in your testcase | ||
1290 | code. Then, use the macro @code{GAUGER()} to make the Buildbots log | ||
1291 | whatever value is of interest for you to @code{gnunet.org}'s Gauger | ||
1292 | server. No setup is necessary as most Buildbots have already everything | ||
1293 | in place and new metrics are created on demand. To delete a metric, you | ||
1294 | need to contact a member of the GNUnet development team (a file will need | ||
1295 | to be removed manually from the respective directory). | ||
1296 | |||
1297 | The code in the test should look like this: | ||
1298 | |||
1299 | @example | ||
1300 | [other includes] | ||
1301 | #include <gauger.h> | ||
1302 | |||
1303 | int main (int argc, char *argv[]) @{ | ||
1304 | |||
1305 | [run test, generate data] | ||
1306 | GAUGER("YOUR_MODULE", | ||
1307 | "METRIC_NAME", | ||
1308 | (float)value, | ||
1309 | "UNIT"); @} | ||
1310 | @end example | ||
1311 | |||
1312 | Where: | ||
1313 | |||
1314 | @table @asis | ||
1315 | |||
1316 | @item @strong{YOUR_MODULE} is a category in the gauger page and should be | ||
1317 | the name of the module or subsystem like "Core" or "DHT" | ||
1318 | @item @strong{METRIC} is | ||
1319 | the name of the metric being collected and should be concise and | ||
1320 | descriptive, like "PUT operations in sqlite-datastore". | ||
1321 | @item @strong{value} is the value | ||
1322 | of the metric that is logged for this run. | ||
1323 | @item @strong{UNIT} is the unit in | ||
1324 | which the value is measured, for instance "kb/s" or "kb of RAM/node". | ||
1325 | @end table | ||
1326 | |||
1327 | If you wish to use Gauger for your own project, you can grab a copy of the | ||
1328 | latest stable release or check out Gauger's Subversion repository. | ||
1329 | |||
1330 | @cindex TESTBED Subsystem | ||
1331 | @node TESTBED Subsystem | ||
1332 | @section TESTBED Subsystem | ||
1333 | |||
1334 | The TESTBED subsystem facilitates testing and measuring of multi-peer | ||
1335 | deployments on a single host or over multiple hosts. | ||
1336 | |||
1337 | The architecture of the testbed module is divided into the following: | ||
1338 | @itemize @bullet | ||
1339 | |||
1340 | @item Testbed API: An API which is used by the testing driver programs. It | ||
1341 | provides with functions for creating, destroying, starting, stopping | ||
1342 | peers, etc. | ||
1343 | |||
1344 | @item Testbed service (controller): A service which is started through the | ||
1345 | Testbed API. This service handles operations to create, destroy, start, | ||
1346 | stop peers, connect them, modify their configurations. | ||
1347 | |||
1348 | @item Testbed helper: When a controller has to be started on a host, the | ||
1349 | testbed API starts the testbed helper on that host which in turn starts | ||
1350 | the controller. The testbed helper receives a configuration for the | ||
1351 | controller through its stdin and changes it to ensure the controller | ||
1352 | doesn't run into any port conflict on that host. | ||
1353 | @end itemize | ||
1354 | |||
1355 | |||
1356 | The testbed service (controller) is different from the other GNUnet | ||
1357 | services in that it is not started by ARM and is not supposed to be run | ||
1358 | as a daemon. It is started by the testbed API through a testbed helper. | ||
1359 | In a typical scenario involving multiple hosts, a controller is started | ||
1360 | on each host. Controllers take up the actual task of creating peers, | ||
1361 | starting and stopping them on the hosts they run. | ||
1362 | |||
1363 | While running deployments on a single localhost the testbed API starts the | ||
1364 | testbed helper directly as a child process. When running deployments on | ||
1365 | remote hosts the testbed API starts Testbed Helpers on each remote host | ||
1366 | through remote shell. By default testbed API uses SSH as a remote shell. | ||
1367 | This can be changed by setting the environmental variable | ||
1368 | GNUNET_TESTBED_RSH_CMD to the required remote shell program. This | ||
1369 | variable can also contain parameters which are to be passed to the remote | ||
1370 | shell program. For e.g: | ||
1371 | |||
1372 | @example | ||
1373 | export GNUNET_TESTBED_RSH_CMD="ssh -o BatchMode=yes \ | ||
1374 | -o NoHostAuthenticationForLocalhost=yes %h" | ||
1375 | @end example | ||
1376 | |||
1377 | Substitutions are allowed in the command string above, | ||
1378 | this allows for substitutions through placemarks which begin with a `%'. | ||
1379 | At present the following substitutions are supported | ||
1380 | |||
1381 | @itemize @bullet | ||
1382 | @item %h: hostname | ||
1383 | @item %u: username | ||
1384 | @item %p: port | ||
1385 | @end itemize | ||
1386 | |||
1387 | Note that the substitution placemark is replaced only when the | ||
1388 | corresponding field is available and only once. Specifying | ||
1389 | |||
1390 | @example | ||
1391 | %u@atchar{}%h | ||
1392 | @end example | ||
1393 | |||
1394 | doesn't work either. If you want to user username substitutions for | ||
1395 | @command{SSH}, use the argument @code{-l} before the | ||
1396 | username substitution. | ||
1397 | |||
1398 | For example: | ||
1399 | @example | ||
1400 | ssh -l %u -p %p %h | ||
1401 | @end example | ||
1402 | |||
1403 | The testbed API and the helper communicate through the helpers stdin and | ||
1404 | stdout. As the helper is started through a remote shell on remote hosts | ||
1405 | any output messages from the remote shell interfere with the communication | ||
1406 | and results in a failure while starting the helper. For this reason, it is | ||
1407 | suggested to use flags to make the remote shells produce no output | ||
1408 | messages and to have password-less logins. The default remote shell, SSH, | ||
1409 | the default options are: | ||
1410 | |||
1411 | @example | ||
1412 | -o BatchMode=yes -o NoHostBasedAuthenticationForLocalhost=yes" | ||
1413 | @end example | ||
1414 | |||
1415 | Password-less logins should be ensured by using SSH keys. | ||
1416 | |||
1417 | Since the testbed API executes the remote shell as a non-interactive | ||
1418 | shell, certain scripts like .bashrc, .profiler may not be executed. If | ||
1419 | this is the case testbed API can be forced to execute an interactive | ||
1420 | shell by setting up the environmental variable | ||
1421 | @code{GNUNET_TESTBED_RSH_CMD_SUFFIX} to a shell program. | ||
1422 | |||
1423 | An example could be: | ||
1424 | |||
1425 | @example | ||
1426 | export GNUNET_TESTBED_RSH_CMD_SUFFIX="sh -lc" | ||
1427 | @end example | ||
1428 | |||
1429 | The testbed API will then execute the remote shell program as: | ||
1430 | |||
1431 | @example | ||
1432 | $GNUNET_TESTBED_RSH_CMD -p $port $dest $GNUNET_TESTBED_RSH_CMD_SUFFIX \ | ||
1433 | gnunet-helper-testbed | ||
1434 | @end example | ||
1435 | |||
1436 | On some systems, problems may arise while starting testbed helpers if | ||
1437 | GNUnet is installed into a custom location since the helper may not be | ||
1438 | found in the standard path. This can be addressed by setting the variable | ||
1439 | `@code{HELPER_BINARY_PATH}' to the path of the testbed helper. | ||
1440 | Testbed API will then use this path to start helper binaries both | ||
1441 | locally and remotely. | ||
1442 | |||
1443 | Testbed API can accessed by including the | ||
1444 | @file{gnunet_testbed_service.h} file and linking with | ||
1445 | @code{-lgnunettestbed}. | ||
1446 | |||
1447 | @c *********************************************************************** | ||
1448 | @menu | ||
1449 | * Supported Topologies:: | ||
1450 | * Hosts file format:: | ||
1451 | * Topology file format:: | ||
1452 | * Testbed Barriers:: | ||
1453 | * Automatic large-scale deployment in the PlanetLab testbed:: | ||
1454 | * TESTBED Caveats:: | ||
1455 | @end menu | ||
1456 | |||
1457 | @node Supported Topologies | ||
1458 | @subsection Supported Topologies | ||
1459 | |||
1460 | While testing multi-peer deployments, it is often needed that the peers | ||
1461 | are connected in some topology. This requirement is addressed by the | ||
1462 | function @code{GNUNET_TESTBED_overlay_connect()} which connects any given | ||
1463 | two peers in the testbed. | ||
1464 | |||
1465 | The API also provides a helper function | ||
1466 | @code{GNUNET_TESTBED_overlay_configure_topology()} to connect a given set | ||
1467 | of peers in any of the following supported topologies: | ||
1468 | |||
1469 | @itemize @bullet | ||
1470 | |||
1471 | @item @code{GNUNET_TESTBED_TOPOLOGY_CLIQUE}: All peers are connected with | ||
1472 | each other | ||
1473 | |||
1474 | @item @code{GNUNET_TESTBED_TOPOLOGY_LINE}: Peers are connected to form a | ||
1475 | line | ||
1476 | |||
1477 | @item @code{GNUNET_TESTBED_TOPOLOGY_RING}: Peers are connected to form a | ||
1478 | ring topology | ||
1479 | |||
1480 | @item @code{GNUNET_TESTBED_TOPOLOGY_2D_TORUS}: Peers are connected to | ||
1481 | form a 2 dimensional torus topology. The number of peers may not be a | ||
1482 | perfect square, in that case the resulting torus may not have the uniform | ||
1483 | poloidal and toroidal lengths | ||
1484 | |||
1485 | @item @code{GNUNET_TESTBED_TOPOLOGY_ERDOS_RENYI}: Topology is generated | ||
1486 | to form a random graph. The number of links to be present should be given | ||
1487 | |||
1488 | @item @code{GNUNET_TESTBED_TOPOLOGY_SMALL_WORLD}: Peers are connected to | ||
1489 | form a 2D Torus with some random links among them. The number of random | ||
1490 | links are to be given | ||
1491 | |||
1492 | @item @code{GNUNET_TESTBED_TOPOLOGY_SMALL_WORLD_RING}: Peers are | ||
1493 | connected to form a ring with some random links among them. The number of | ||
1494 | random links are to be given | ||
1495 | |||
1496 | @item @code{GNUNET_TESTBED_TOPOLOGY_SCALE_FREE}: Connects peers in a | ||
1497 | topology where peer connectivity follows power law - new peers are | ||
1498 | connected with high probabililty to well connected peers. | ||
1499 | @footnote{See Emergence of Scaling in Random Networks. Science 286, | ||
1500 | 509-512, 1999 | ||
1501 | (@uref{https://gnunet.org/git/bibliography.git/plain/docs/emergence_of_scaling_in_random_networks__barabasi_albert_science_286__1999.pdf, pdf})} | ||
1502 | |||
1503 | @item @code{GNUNET_TESTBED_TOPOLOGY_FROM_FILE}: The topology information | ||
1504 | is loaded from a file. The path to the file has to be given. | ||
1505 | @xref{Topology file format}, for the format of this file. | ||
1506 | |||
1507 | @item @code{GNUNET_TESTBED_TOPOLOGY_NONE}: No topology | ||
1508 | @end itemize | ||
1509 | |||
1510 | |||
1511 | The above supported topologies can be specified respectively by setting | ||
1512 | the variable @code{OVERLAY_TOPOLOGY} to the following values in the | ||
1513 | configuration passed to Testbed API functions | ||
1514 | @code{GNUNET_TESTBED_test_run()} and | ||
1515 | @code{GNUNET_TESTBED_run()}: | ||
1516 | |||
1517 | @itemize @bullet | ||
1518 | @item @code{CLIQUE} | ||
1519 | @item @code{RING} | ||
1520 | @item @code{LINE} | ||
1521 | @item @code{2D_TORUS} | ||
1522 | @item @code{RANDOM} | ||
1523 | @item @code{SMALL_WORLD} | ||
1524 | @item @code{SMALL_WORLD_RING} | ||
1525 | @item @code{SCALE_FREE} | ||
1526 | @item @code{FROM_FILE} | ||
1527 | @item @code{NONE} | ||
1528 | @end itemize | ||
1529 | |||
1530 | |||
1531 | Topologies @code{RANDOM}, @code{SMALL_WORLD} and @code{SMALL_WORLD_RING} | ||
1532 | require the option @code{OVERLAY_RANDOM_LINKS} to be set to the number of | ||
1533 | random links to be generated in the configuration. The option will be | ||
1534 | ignored for the rest of the topologies. | ||
1535 | |||
1536 | Topology @code{SCALE_FREE} requires the options | ||
1537 | @code{SCALE_FREE_TOPOLOGY_CAP} to be set to the maximum number of peers | ||
1538 | which can connect to a peer and @code{SCALE_FREE_TOPOLOGY_M} to be set to | ||
1539 | how many peers a peer should be atleast connected to. | ||
1540 | |||
1541 | Similarly, the topology @code{FROM_FILE} requires the option | ||
1542 | @code{OVERLAY_TOPOLOGY_FILE} to contain the path of the file containing | ||
1543 | the topology information. This option is ignored for the rest of the | ||
1544 | topologies. @xref{Topology file format}, for the format of this file. | ||
1545 | |||
1546 | @c *********************************************************************** | ||
1547 | @node Hosts file format | ||
1548 | @subsection Hosts file format | ||
1549 | |||
1550 | The testbed API offers the function | ||
1551 | @code{GNUNET_TESTBED_hosts_load_from_file()} to load from a given file | ||
1552 | details about the hosts which testbed can use for deploying peers. | ||
1553 | This function is useful to keep the data about hosts | ||
1554 | separate instead of hard coding them in code. | ||
1555 | |||
1556 | Another helper function from testbed API, @code{GNUNET_TESTBED_run()} | ||
1557 | also takes a hosts file name as its parameter. It uses the above | ||
1558 | function to populate the hosts data structures and start controllers to | ||
1559 | deploy peers. | ||
1560 | |||
1561 | These functions require the hosts file to be of the following format: | ||
1562 | @itemize @bullet | ||
1563 | @item Each line is interpreted to have details about a host | ||
1564 | @item Host details should include the username to use for logging into the | ||
1565 | host, the hostname of the host and the port number to use for the remote | ||
1566 | shell program. All thee values should be given. | ||
1567 | @item These details should be given in the following format: | ||
1568 | @example | ||
1569 | <username>@@<hostname>:<port> | ||
1570 | @end example | ||
1571 | @end itemize | ||
1572 | |||
1573 | Note that having canonical hostnames may cause problems while resolving | ||
1574 | the IP addresses (See this bug). Hence it is advised to provide the hosts' | ||
1575 | IP numerical addresses as hostnames whenever possible. | ||
1576 | |||
1577 | @c *********************************************************************** | ||
1578 | @node Topology file format | ||
1579 | @subsection Topology file format | ||
1580 | |||
1581 | A topology file describes how peers are to be connected. It should adhere | ||
1582 | to the following format for testbed to parse it correctly. | ||
1583 | |||
1584 | Each line should begin with the target peer id. This should be followed by | ||
1585 | a colon(`:') and origin peer ids seperated by `|'. All spaces except for | ||
1586 | newline characters are ignored. The API will then try to connect each | ||
1587 | origin peer to the target peer. | ||
1588 | |||
1589 | For example, the following file will result in 5 overlay connections: | ||
1590 | [2->1], [3->1],[4->3], [0->3], [2->0]@ | ||
1591 | @code{@ 1:2|3@ 3:4| 0@ 0: 2@ } | ||
1592 | |||
1593 | @c *********************************************************************** | ||
1594 | @node Testbed Barriers | ||
1595 | @subsection Testbed Barriers | ||
1596 | |||
1597 | The testbed subsystem's barriers API facilitates coordination among the | ||
1598 | peers run by the testbed and the experiment driver. The concept is | ||
1599 | similar to the barrier synchronisation mechanism found in parallel | ||
1600 | programming or multi-threading paradigms - a peer waits at a barrier upon | ||
1601 | reaching it until the barrier is reached by a predefined number of peers. | ||
1602 | This predefined number of peers required to cross a barrier is also called | ||
1603 | quorum. We say a peer has reached a barrier if the peer is waiting for the | ||
1604 | barrier to be crossed. Similarly a barrier is said to be reached if the | ||
1605 | required quorum of peers reach the barrier. A barrier which is reached is | ||
1606 | deemed as crossed after all the peers waiting on it are notified. | ||
1607 | |||
1608 | The barriers API provides the following functions: | ||
1609 | @itemize @bullet | ||
1610 | @item @strong{@code{GNUNET_TESTBED_barrier_init()}:} function to | ||
1611 | initialse a barrier in the experiment | ||
1612 | @item @strong{@code{GNUNET_TESTBED_barrier_cancel()}:} function to cancel | ||
1613 | a barrier which has been initialised before | ||
1614 | @item @strong{@code{GNUNET_TESTBED_barrier_wait()}:} function to signal | ||
1615 | barrier service that the caller has reached a barrier and is waiting for | ||
1616 | it to be crossed | ||
1617 | @item @strong{@code{GNUNET_TESTBED_barrier_wait_cancel()}:} function to | ||
1618 | stop waiting for a barrier to be crossed | ||
1619 | @end itemize | ||
1620 | |||
1621 | |||
1622 | Among the above functions, the first two, namely | ||
1623 | @code{GNUNET_TESTBED_barrier_init()} and | ||
1624 | @code{GNUNET_TESTBED_barrier_cancel()} are used by experiment drivers. All | ||
1625 | barriers should be initialised by the experiment driver by calling | ||
1626 | @code{GNUNET_TESTBED_barrier_init()}. This function takes a name to | ||
1627 | identify the barrier, the quorum required for the barrier to be crossed | ||
1628 | and a notification callback for notifying the experiment driver when the | ||
1629 | barrier is crossed. @code{GNUNET_TESTBED_barrier_cancel()} cancels an | ||
1630 | initialised barrier and frees the resources allocated for it. This | ||
1631 | function can be called upon a initialised barrier before it is crossed. | ||
1632 | |||
1633 | The remaining two functions @code{GNUNET_TESTBED_barrier_wait()} and | ||
1634 | @code{GNUNET_TESTBED_barrier_wait_cancel()} are used in the peer's | ||
1635 | processes. @code{GNUNET_TESTBED_barrier_wait()} connects to the local | ||
1636 | barrier service running on the same host the peer is running on and | ||
1637 | registers that the caller has reached the barrier and is waiting for the | ||
1638 | barrier to be crossed. Note that this function can only be used by peers | ||
1639 | which are started by testbed as this function tries to access the local | ||
1640 | barrier service which is part of the testbed controller service. Calling | ||
1641 | @code{GNUNET_TESTBED_barrier_wait()} on an uninitialised barrier results | ||
1642 | in failure. @code{GNUNET_TESTBED_barrier_wait_cancel()} cancels the | ||
1643 | notification registered by @code{GNUNET_TESTBED_barrier_wait()}. | ||
1644 | |||
1645 | |||
1646 | @c *********************************************************************** | ||
1647 | @menu | ||
1648 | * Implementation:: | ||
1649 | @end menu | ||
1650 | |||
1651 | @node Implementation | ||
1652 | @subsubsection Implementation | ||
1653 | |||
1654 | Since barriers involve coordination between experiment driver and peers, | ||
1655 | the barrier service in the testbed controller is split into two | ||
1656 | components. The first component responds to the message generated by the | ||
1657 | barrier API used by the experiment driver (functions | ||
1658 | @code{GNUNET_TESTBED_barrier_init()} and | ||
1659 | @code{GNUNET_TESTBED_barrier_cancel()}) and the second component to the | ||
1660 | messages generated by barrier API used by peers (functions | ||
1661 | @code{GNUNET_TESTBED_barrier_wait()} and | ||
1662 | @code{GNUNET_TESTBED_barrier_wait_cancel()}). | ||
1663 | |||
1664 | Calling @code{GNUNET_TESTBED_barrier_init()} sends a | ||
1665 | @code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_INIT} message to the master | ||
1666 | controller. The master controller then registers a barrier and calls | ||
1667 | @code{GNUNET_TESTBED_barrier_init()} for each its subcontrollers. In this | ||
1668 | way barrier initialisation is propagated to the controller hierarchy. | ||
1669 | While propagating initialisation, any errors at a subcontroller such as | ||
1670 | timeout during further propagation are reported up the hierarchy back to | ||
1671 | the experiment driver. | ||
1672 | |||
1673 | Similar to @code{GNUNET_TESTBED_barrier_init()}, | ||
1674 | @code{GNUNET_TESTBED_barrier_cancel()} propagates | ||
1675 | @code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_CANCEL} message which causes | ||
1676 | controllers to remove an initialised barrier. | ||
1677 | |||
1678 | The second component is implemented as a separate service in the binary | ||
1679 | `gnunet-service-testbed' which already has the testbed controller service. | ||
1680 | Although this deviates from the gnunet process architecture of having one | ||
1681 | service per binary, it is needed in this case as this component needs | ||
1682 | access to barrier data created by the first component. This component | ||
1683 | responds to @code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_WAIT} messages from | ||
1684 | local peers when they call @code{GNUNET_TESTBED_barrier_wait()}. Upon | ||
1685 | receiving @code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_WAIT} message, the | ||
1686 | service checks if the requested barrier has been initialised before and | ||
1687 | if it was not initialised, an error status is sent through | ||
1688 | @code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_STATUS} message to the local | ||
1689 | peer and the connection from the peer is terminated. If the barrier is | ||
1690 | initialised before, the barrier's counter for reached peers is incremented | ||
1691 | and a notification is registered to notify the peer when the barrier is | ||
1692 | reached. The connection from the peer is left open. | ||
1693 | |||
1694 | When enough peers required to attain the quorum send | ||
1695 | @code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_WAIT} messages, the controller | ||
1696 | sends a @code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_STATUS} message to its | ||
1697 | parent informing that the barrier is crossed. If the controller has | ||
1698 | started further subcontrollers, it delays this message until it receives | ||
1699 | a similar notification from each of those subcontrollers. Finally, the | ||
1700 | barriers API at the experiment driver receives the | ||
1701 | @code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_STATUS} when the barrier is | ||
1702 | reached at all the controllers. | ||
1703 | |||
1704 | The barriers API at the experiment driver responds to the | ||
1705 | @code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_STATUS} message by echoing it | ||
1706 | back to the master controller and notifying the experiment controller | ||
1707 | through the notification callback that a barrier has been crossed. The | ||
1708 | echoed @code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_STATUS} message is | ||
1709 | propagated by the master controller to the controller hierarchy. This | ||
1710 | propagation triggers the notifications registered by peers at each of the | ||
1711 | controllers in the hierarchy. Note the difference between this downward | ||
1712 | propagation of the @code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_STATUS} | ||
1713 | message from its upward propagation --- the upward propagation is needed | ||
1714 | for ensuring that the barrier is reached by all the controllers and the | ||
1715 | downward propagation is for triggering that the barrier is crossed. | ||
1716 | |||
1717 | @cindex PlanetLab testbed | ||
1718 | @node Automatic large-scale deployment in the PlanetLab testbed | ||
1719 | @subsection Automatic large-scale deployment in the PlanetLab testbed | ||
1720 | |||
1721 | PlanetLab is a testbed for computer networking and distributed systems | ||
1722 | research. It was established in 2002 and as of June 2010 was composed of | ||
1723 | 1090 nodes at 507 sites worldwide. | ||
1724 | |||
1725 | To automate the GNUnet we created a set of automation tools to simplify | ||
1726 | the large-scale deployment. We provide you a set of scripts you can use | ||
1727 | to deploy GNUnet on a set of nodes and manage your installation. | ||
1728 | |||
1729 | Please also check @uref{https://gnunet.org/installation-fedora8-svn} and | ||
1730 | @uref{https://gnunet.org/installation-fedora12-svn} to find detailled | ||
1731 | instructions how to install GNUnet on a PlanetLab node. | ||
1732 | |||
1733 | |||
1734 | @c *********************************************************************** | ||
1735 | @menu | ||
1736 | * PlanetLab Automation for Fedora8 nodes:: | ||
1737 | * Install buildslave on PlanetLab nodes running fedora core 8:: | ||
1738 | * Setup a new PlanetLab testbed using GPLMT:: | ||
1739 | * Why do i get an ssh error when using the regex profiler?:: | ||
1740 | @end menu | ||
1741 | |||
1742 | @node PlanetLab Automation for Fedora8 nodes | ||
1743 | @subsubsection PlanetLab Automation for Fedora8 nodes | ||
1744 | |||
1745 | @c *********************************************************************** | ||
1746 | @node Install buildslave on PlanetLab nodes running fedora core 8 | ||
1747 | @subsubsection Install buildslave on PlanetLab nodes running fedora core 8 | ||
1748 | @c ** Actually this is a subsubsubsection, but must be fixed differently | ||
1749 | @c ** as subsubsection is the lowest. | ||
1750 | |||
1751 | Since most of the PlanetLab nodes are running the very old Fedora core 8 | ||
1752 | image, installing the buildslave software is quite some pain. For our | ||
1753 | PlanetLab testbed we figured out how to install the buildslave software | ||
1754 | best. | ||
1755 | |||
1756 | @c This is a vvery terrible way to suggest installing software. | ||
1757 | @c FIXME: Is there an official, safer way instead of blind-piping a | ||
1758 | @c script? | ||
1759 | @c FIXME: Use newer pypi URLs below. | ||
1760 | Install Distribute for Python: | ||
1761 | |||
1762 | @example | ||
1763 | curl http://python-distribute.org/distribute_setup.py | sudo python | ||
1764 | @end example | ||
1765 | |||
1766 | Install Distribute for zope.interface <= 3.8.0 (4.0 and 4.0.1 will not | ||
1767 | work): | ||
1768 | |||
1769 | @example | ||
1770 | export PYPI=@value{PYPI-URL} | ||
1771 | wget $PYPI/z/zope.interface/zope.interface-3.8.0.tar.gz | ||
1772 | tar zvfz zope.interface-3.8.0.tar.gz | ||
1773 | cd zope.interface-3.8.0 | ||
1774 | sudo python setup.py install | ||
1775 | @end example | ||
1776 | |||
1777 | Install the buildslave software (0.8.6 was the latest version): | ||
1778 | |||
1779 | @example | ||
1780 | export GCODE="http://buildbot.googlecode.com/files" | ||
1781 | wget $GCODE/buildbot-slave-0.8.6p1.tar.gz | ||
1782 | tar xvfz buildbot-slave-0.8.6p1.tar.gz | ||
1783 | cd buildslave-0.8.6p1 | ||
1784 | sudo python setup.py install | ||
1785 | @end example | ||
1786 | |||
1787 | The setup will download the matching twisted package and install it. | ||
1788 | It will also try to install the latest version of zope.interface which | ||
1789 | will fail to install. Buildslave will work anyway since version 3.8.0 | ||
1790 | was installed before! | ||
1791 | |||
1792 | @c *********************************************************************** | ||
1793 | @node Setup a new PlanetLab testbed using GPLMT | ||
1794 | @subsubsection Setup a new PlanetLab testbed using GPLMT | ||
1795 | |||
1796 | @itemize @bullet | ||
1797 | @item Get a new slice and assign nodes | ||
1798 | Ask your PlanetLab PI to give you a new slice and assign the nodes you | ||
1799 | need | ||
1800 | @item Install a buildmaster | ||
1801 | You can stick to the buildbot documentation:@ | ||
1802 | @uref{http://buildbot.net/buildbot/docs/current/manual/installation.html} | ||
1803 | @item Install the buildslave software on all nodes | ||
1804 | To install the buildslave on all nodes assigned to your slice you can use | ||
1805 | the tasklist @code{install_buildslave_fc8.xml} provided with GPLMT: | ||
1806 | |||
1807 | @example | ||
1808 | ./gplmt.py -c contrib/tumple_gnunet.conf -t \ | ||
1809 | contrib/tasklists/install_buildslave_fc8.xml -a -p <planetlab password> | ||
1810 | @end example | ||
1811 | |||
1812 | @item Create the buildmaster configuration and the slave setup commands | ||
1813 | |||
1814 | The master and the and the slaves have need to have credentials and the | ||
1815 | master has to have all nodes configured. This can be done with the | ||
1816 | @file{create_buildbot_configuration.py} script in the @file{scripts} | ||
1817 | directory. | ||
1818 | |||
1819 | This scripts takes a list of nodes retrieved directly from PlanetLab or | ||
1820 | read from a file and a configuration template and creates: | ||
1821 | |||
1822 | @itemize @bullet | ||
1823 | @item a tasklist which can be executed with gplmt to setup the slaves | ||
1824 | @item a master.cfg file containing a PlanetLab nodes | ||
1825 | @end itemize | ||
1826 | |||
1827 | A configuration template is included in the <contrib>, most important is | ||
1828 | that the script replaces the following tags in the template: | ||
1829 | |||
1830 | %GPLMT_BUILDER_DEFINITION :@ GPLMT_BUILDER_SUMMARY@ GPLMT_SLAVES@ | ||
1831 | %GPLMT_SCHEDULER_BUILDERS | ||
1832 | |||
1833 | Create configuration for all nodes assigned to a slice: | ||
1834 | |||
1835 | @example | ||
1836 | ./create_buildbot_configuration.py -u <planetlab username> \ | ||
1837 | -p <planetlab password> -s <slice> -m <buildmaster+port> \ | ||
1838 | -t <template> | ||
1839 | @end example | ||
1840 | |||
1841 | Create configuration for some nodes in a file: | ||
1842 | |||
1843 | @example | ||
1844 | ./create_buildbot_configuration.p -f <node_file> \ | ||
1845 | -m <buildmaster+port> -t <template> | ||
1846 | @end example | ||
1847 | |||
1848 | @item Copy the @file{master.cfg} to the buildmaster and start it | ||
1849 | Use @code{buildbot start <basedir>} to start the server | ||
1850 | @item Setup the buildslaves | ||
1851 | @end itemize | ||
1852 | |||
1853 | @c *********************************************************************** | ||
1854 | @node Why do i get an ssh error when using the regex profiler? | ||
1855 | @subsubsection Why do i get an ssh error when using the regex profiler? | ||
1856 | |||
1857 | Why do i get an ssh error "Permission denied (publickey,password)." when | ||
1858 | using the regex profiler although passwordless ssh to localhost works | ||
1859 | using publickey and ssh-agent? | ||
1860 | |||
1861 | You have to generate a public/private-key pair with no password:@ | ||
1862 | @code{ssh-keygen -t rsa -b 4096 -f ~/.ssh/id_localhost}@ | ||
1863 | and then add the following to your ~/.ssh/config file: | ||
1864 | |||
1865 | @code{Host 127.0.0.1@ IdentityFile ~/.ssh/id_localhost} | ||
1866 | |||
1867 | now make sure your hostsfile looks like | ||
1868 | |||
1869 | @example | ||
1870 | [USERNAME]@@127.0.0.1:22@ | ||
1871 | [USERNAME]@@127.0.0.1:22 | ||
1872 | @end example | ||
1873 | |||
1874 | You can test your setup by running @code{ssh 127.0.0.1} in a | ||
1875 | terminal and then in the opened session run it again. | ||
1876 | If you were not asked for a password on either login, | ||
1877 | then you should be good to go. | ||
1878 | |||
1879 | @cindex TESTBED Caveats | ||
1880 | @node TESTBED Caveats | ||
1881 | @subsection TESTBED Caveats | ||
1882 | |||
1883 | This section documents a few caveats when using the GNUnet testbed | ||
1884 | subsystem. | ||
1885 | |||
1886 | @c *********************************************************************** | ||
1887 | @menu | ||
1888 | * CORE must be started:: | ||
1889 | * ATS must want the connections:: | ||
1890 | @end menu | ||
1891 | |||
1892 | @node CORE must be started | ||
1893 | @subsubsection CORE must be started | ||
1894 | |||
1895 | A simple issue is #3993: Your configuration MUST somehow ensure that for | ||
1896 | each peer the CORE service is started when the peer is setup, otherwise | ||
1897 | TESTBED may fail to connect peers when the topology is initialized, as | ||
1898 | TESTBED will start some CORE services but not necessarily all (but it | ||
1899 | relies on all of them running). The easiest way is to set | ||
1900 | 'FORCESTART = YES' in the '[core]' section of the configuration file. | ||
1901 | Alternatively, having any service that directly or indirectly depends on | ||
1902 | CORE being started with FORCESTART will also do. This issue largely arises | ||
1903 | if users try to over-optimize by not starting any services with | ||
1904 | FORCESTART. | ||
1905 | |||
1906 | @c *********************************************************************** | ||
1907 | @node ATS must want the connections | ||
1908 | @subsubsection ATS must want the connections | ||
1909 | |||
1910 | When TESTBED sets up connections, it only offers the respective HELLO | ||
1911 | information to the TRANSPORT service. It is then up to the ATS service to | ||
1912 | @strong{decide} to use the connection. The ATS service will typically | ||
1913 | eagerly establish any connection if the number of total connections is | ||
1914 | low (relative to bandwidth). Details may further depend on the | ||
1915 | specific ATS backend that was configured. If ATS decides to NOT establish | ||
1916 | a connection (even though TESTBED provided the required information), then | ||
1917 | that connection will count as failed for TESTBED. Note that you can | ||
1918 | configure TESTBED to tolerate a certain number of connection failures | ||
1919 | (see '-e' option of gnunet-testbed-profiler). This issue largely arises | ||
1920 | for dense overlay topologies, especially if you try to create cliques | ||
1921 | with more than 20 peers. | ||
1922 | |||
1923 | @cindex libgnunetutil | ||
1924 | @node libgnunetutil | ||
1925 | @section libgnunetutil | ||
1926 | |||
1927 | libgnunetutil is the fundamental library that all GNUnet code builds upon. | ||
1928 | Ideally, this library should contain most of the platform dependent code | ||
1929 | (except for user interfaces and really special needs that only few | ||
1930 | applications have). It is also supposed to offer basic services that most | ||
1931 | if not all GNUnet binaries require. The code of libgnunetutil is in the | ||
1932 | @file{src/util/} directory. The public interface to the library is in the | ||
1933 | gnunet_util.h header. The functions provided by libgnunetutil fall | ||
1934 | roughly into the following categories (in roughly the order of importance | ||
1935 | for new developers): | ||
1936 | |||
1937 | @itemize @bullet | ||
1938 | @item logging (common_logging.c) | ||
1939 | @item memory allocation (common_allocation.c) | ||
1940 | @item endianess conversion (common_endian.c) | ||
1941 | @item internationalization (common_gettext.c) | ||
1942 | @item String manipulation (string.c) | ||
1943 | @item file access (disk.c) | ||
1944 | @item buffered disk IO (bio.c) | ||
1945 | @item time manipulation (time.c) | ||
1946 | @item configuration parsing (configuration.c) | ||
1947 | @item command-line handling (getopt*.c) | ||
1948 | @item cryptography (crypto_*.c) | ||
1949 | @item data structures (container_*.c) | ||
1950 | @item CPS-style scheduling (scheduler.c) | ||
1951 | @item Program initialization (program.c) | ||
1952 | @item Networking (network.c, client.c, server*.c, service.c) | ||
1953 | @item message queueing (mq.c) | ||
1954 | @item bandwidth calculations (bandwidth.c) | ||
1955 | @item Other OS-related (os*.c, plugin.c, signal.c) | ||
1956 | @item Pseudonym management (pseudonym.c) | ||
1957 | @end itemize | ||
1958 | |||
1959 | It should be noted that only developers that fully understand this entire | ||
1960 | API will be able to write good GNUnet code. | ||
1961 | |||
1962 | Ideally, porting GNUnet should only require porting the gnunetutil | ||
1963 | library. More testcases for the gnunetutil APIs are therefore a great | ||
1964 | way to make porting of GNUnet easier. | ||
1965 | |||
1966 | @menu | ||
1967 | * Logging:: | ||
1968 | * Interprocess communication API (IPC):: | ||
1969 | * Cryptography API:: | ||
1970 | * Message Queue API:: | ||
1971 | * Service API:: | ||
1972 | * Optimizing Memory Consumption of GNUnet's (Multi-) Hash Maps:: | ||
1973 | * CONTAINER_MDLL API:: | ||
1974 | @end menu | ||
1975 | |||
1976 | @cindex Logging | ||
1977 | @cindex log levels | ||
1978 | @node Logging | ||
1979 | @subsection Logging | ||
1980 | |||
1981 | GNUnet is able to log its activity, mostly for the purposes of debugging | ||
1982 | the program at various levels. | ||
1983 | |||
1984 | @file{gnunet_common.h} defines several @strong{log levels}: | ||
1985 | @table @asis | ||
1986 | |||
1987 | @item ERROR for errors (really problematic situations, often leading to | ||
1988 | crashes) | ||
1989 | @item WARNING for warnings (troubling situations that might have | ||
1990 | negative consequences, although not fatal) | ||
1991 | @item INFO for various information. | ||
1992 | Used somewhat rarely, as GNUnet statistics is used to hold and display | ||
1993 | most of the information that users might find interesting. | ||
1994 | @item DEBUG for debugging. | ||
1995 | Does not produce much output on normal builds, but when extra logging is | ||
1996 | enabled at compile time, a staggering amount of data is outputted under | ||
1997 | this log level. | ||
1998 | @end table | ||
1999 | |||
2000 | |||
2001 | Normal builds of GNUnet (configured with @code{--enable-logging[=yes]}) | ||
2002 | are supposed to log nothing under DEBUG level. The | ||
2003 | @code{--enable-logging=verbose} configure option can be used to create a | ||
2004 | build with all logging enabled. However, such build will produce large | ||
2005 | amounts of log data, which is inconvenient when one tries to hunt down a | ||
2006 | specific problem. | ||
2007 | |||
2008 | To mitigate this problem, GNUnet provides facilities to apply a filter to | ||
2009 | reduce the logs: | ||
2010 | @table @asis | ||
2011 | |||
2012 | @item Logging by default When no log levels are configured in any other | ||
2013 | way (see below), GNUnet will default to the WARNING log level. This | ||
2014 | mostly applies to GNUnet command line utilities, services and daemons; | ||
2015 | tests will always set log level to WARNING or, if | ||
2016 | @code{--enable-logging=verbose} was passed to configure, to DEBUG. The | ||
2017 | default level is suggested for normal operation. | ||
2018 | @item The -L option Most GNUnet executables accept an "-L loglevel" or | ||
2019 | "--log=loglevel" option. If used, it makes the process set a global log | ||
2020 | level to "loglevel". Thus it is possible to run some processes | ||
2021 | with -L DEBUG, for example, and others with -L ERROR to enable specific | ||
2022 | settings to diagnose problems with a particular process. | ||
2023 | @item Configuration files. Because GNUnet | ||
2024 | service and deamon processes are usually launched by gnunet-arm, it is not | ||
2025 | possible to pass different custom command line options directly to every | ||
2026 | one of them. The options passed to @code{gnunet-arm} only affect | ||
2027 | gnunet-arm and not the rest of GNUnet. However, one can specify a | ||
2028 | configuration key "OPTIONS" in the section that corresponds to a service | ||
2029 | or a daemon, and put a value of "-L loglevel" there. This will make the | ||
2030 | respective service or daemon set its log level to "loglevel" (as the | ||
2031 | value of OPTIONS will be passed as a command-line argument). | ||
2032 | |||
2033 | To specify the same log level for all services without creating separate | ||
2034 | "OPTIONS" entries in the configuration for each one, the user can specify | ||
2035 | a config key "GLOBAL_POSTFIX" in the [arm] section of the configuration | ||
2036 | file. The value of GLOBAL_POSTFIX will be appended to all command lines | ||
2037 | used by the ARM service to run other services. It can contain any option | ||
2038 | valid for all GNUnet commands, thus in particular the "-L loglevel" | ||
2039 | option. The ARM service itself is, however, unaffected by GLOBAL_POSTFIX; | ||
2040 | to set log level for it, one has to specify "OPTIONS" key in the [arm] | ||
2041 | section. | ||
2042 | @item Environment variables. | ||
2043 | Setting global per-process log levels with "-L loglevel" does not offer | ||
2044 | sufficient log filtering granularity, as one service will call interface | ||
2045 | libraries and supporting libraries of other GNUnet services, potentially | ||
2046 | producing lots of debug log messages from these libraries. Also, changing | ||
2047 | the config file is not always convenient (especially when running the | ||
2048 | GNUnet test suite).@ To fix that, and to allow GNUnet to use different | ||
2049 | log filtering at runtime without re-compiling the whole source tree, the | ||
2050 | log calls were changed to be configurable at run time. To configure them | ||
2051 | one has to define environment variables "GNUNET_FORCE_LOGFILE", | ||
2052 | "GNUNET_LOG" and/or "GNUNET_FORCE_LOG": | ||
2053 | @itemize @bullet | ||
2054 | |||
2055 | @item "GNUNET_LOG" only affects the logging when no global log level is | ||
2056 | configured by any other means (that is, the process does not explicitly | ||
2057 | set its own log level, there are no "-L loglevel" options on command line | ||
2058 | or in configuration files), and can be used to override the default | ||
2059 | WARNING log level. | ||
2060 | |||
2061 | @item "GNUNET_FORCE_LOG" will completely override any other log | ||
2062 | configuration options given. | ||
2063 | |||
2064 | @item "GNUNET_FORCE_LOGFILE" will completely override the location of the | ||
2065 | file to log messages to. It should contain a relative or absolute file | ||
2066 | name. Setting GNUNET_FORCE_LOGFILE is equivalent to passing | ||
2067 | "--log-file=logfile" or "-l logfile" option (see below). It supports "[]" | ||
2068 | format in file names, but not "@{@}" (see below). | ||
2069 | @end itemize | ||
2070 | |||
2071 | |||
2072 | Because environment variables are inherited by child processes when they | ||
2073 | are launched, starting or re-starting the ARM service with these | ||
2074 | variables will propagate them to all other services. | ||
2075 | |||
2076 | "GNUNET_LOG" and "GNUNET_FORCE_LOG" variables must contain a specially | ||
2077 | formatted @strong{logging definition} string, which looks like this:@ | ||
2078 | |||
2079 | @c FIXME: Can we close this with [/component] instead? | ||
2080 | @example | ||
2081 | [component];[file];[function];[from_line[-to_line]];loglevel[/component...] | ||
2082 | @end example | ||
2083 | |||
2084 | That is, a logging definition consists of definition entries, separated by | ||
2085 | slashes ('/'). If only one entry is present, there is no need to add a | ||
2086 | slash to its end (although it is not forbidden either).@ All definition | ||
2087 | fields (component, file, function, lines and loglevel) are mandatory, but | ||
2088 | (except for the loglevel) they can be empty. An empty field means | ||
2089 | "match anything". Note that even if fields are empty, the semicolon (';') | ||
2090 | separators must be present.@ The loglevel field is mandatory, and must | ||
2091 | contain one of the log level names (ERROR, WARNING, INFO or DEBUG).@ | ||
2092 | The lines field might contain one non-negative number, in which case it | ||
2093 | matches only one line, or a range "from_line-to_line", in which case it | ||
2094 | matches any line in the interval [from_line;to_line] (that is, including | ||
2095 | both start and end line).@ GNUnet mostly defaults component name to the | ||
2096 | name of the service that is implemented in a process ('transport', | ||
2097 | 'core', 'peerinfo', etc), but logging calls can specify custom component | ||
2098 | names using @code{GNUNET_log_from}.@ File name and function name are | ||
2099 | provided by the compiler (__FILE__ and __FUNCTION__ built-ins). | ||
2100 | |||
2101 | Component, file and function fields are interpreted as non-extended | ||
2102 | regular expressions (GNU libc regex functions are used). Matching is | ||
2103 | case-sensitive, "^" and "$" will match the beginning and the end of the | ||
2104 | text. If a field is empty, its contents are automatically replaced with | ||
2105 | a ".*" regular expression, which matches anything. Matching is done in | ||
2106 | the default way, which means that the expression matches as long as it's | ||
2107 | contained anywhere in the string. Thus "GNUNET_" will match both | ||
2108 | "GNUNET_foo" and "BAR_GNUNET_BAZ". Use '^' and/or '$' to make sure that | ||
2109 | the expression matches at the start and/or at the end of the string. | ||
2110 | The semicolon (';') can't be escaped, and GNUnet will not use it in | ||
2111 | component names (it can't be used in function names and file names | ||
2112 | anyway). | ||
2113 | |||
2114 | @end table | ||
2115 | |||
2116 | |||
2117 | Every logging call in GNUnet code will be (at run time) matched against | ||
2118 | the log definitions passed to the process. If a log definition fields are | ||
2119 | matching the call arguments, then the call log level is compared the the | ||
2120 | log level of that definition. If the call log level is less or equal to | ||
2121 | the definition log level, the call is allowed to proceed. Otherwise the | ||
2122 | logging call is forbidden, and nothing is logged. If no definitions | ||
2123 | matched at all, GNUnet will use the global log level or (if a global log | ||
2124 | level is not specified) will default to WARNING (that is, it will allow | ||
2125 | the call to proceed, if its level is less or equal to the global log | ||
2126 | level or to WARNING). | ||
2127 | |||
2128 | That is, definitions are evaluated from left to right, and the first | ||
2129 | matching definition is used to allow or deny the logging call. Thus it is | ||
2130 | advised to place narrow definitions at the beginning of the logdef | ||
2131 | string, and generic definitions - at the end. | ||
2132 | |||
2133 | Whether a call is allowed or not is only decided the first time this | ||
2134 | particular call is made. The evaluation result is then cached, so that | ||
2135 | any attempts to make the same call later will be allowed or disallowed | ||
2136 | right away. Because of that runtime log level evaluation should not | ||
2137 | significantly affect the process performance. | ||
2138 | Log definition parsing is only done once, at the first call to | ||
2139 | GNUNET_log_setup () made by the process (which is usually done soon after | ||
2140 | it starts). | ||
2141 | |||
2142 | At the moment of writing there is no way to specify logging definitions | ||
2143 | from configuration files, only via environment variables. | ||
2144 | |||
2145 | At the moment GNUnet will stop processing a log definition when it | ||
2146 | encounters an error in definition formatting or an error in regular | ||
2147 | expression syntax, and will not report the failure in any way. | ||
2148 | |||
2149 | |||
2150 | @c *********************************************************************** | ||
2151 | @menu | ||
2152 | * Examples:: | ||
2153 | * Log files:: | ||
2154 | * Updated behavior of GNUNET_log:: | ||
2155 | @end menu | ||
2156 | |||
2157 | @node Examples | ||
2158 | @subsubsection Examples | ||
2159 | |||
2160 | @table @asis | ||
2161 | |||
2162 | @item @code{GNUNET_FORCE_LOG=";;;;DEBUG" gnunet-arm -s} Start GNUnet | ||
2163 | process tree, running all processes with DEBUG level (one should be | ||
2164 | careful with it, as log files will grow at alarming rate!) | ||
2165 | @item @code{GNUNET_FORCE_LOG="core;;;;DEBUG" gnunet-arm -s} Start GNUnet | ||
2166 | process tree, running the core service under DEBUG level (everything else | ||
2167 | will use configured or default level). | ||
2168 | |||
2169 | @item Start GNUnet process tree, allowing any logging calls from | ||
2170 | gnunet-service-transport_validation.c (everything else will use | ||
2171 | configured or default level). | ||
2172 | |||
2173 | @example | ||
2174 | GNUNET_FORCE_LOG=";gnunet-service-transport_validation.c;;; DEBUG" \ | ||
2175 | gnunet-arm -s | ||
2176 | @end example | ||
2177 | |||
2178 | @item Start GNUnet process tree, allowing any logging calls from | ||
2179 | gnunet-gnunet-service-fs_push.c (everything else will use configured or | ||
2180 | default level). | ||
2181 | |||
2182 | @example | ||
2183 | GNUNET_FORCE_LOG="fs;gnunet-service-fs_push.c;;;DEBUG" gnunet-arm -s | ||
2184 | @end example | ||
2185 | |||
2186 | @item Start GNUnet process tree, allowing any logging calls from the | ||
2187 | GNUNET_NETWORK_socket_select function (everything else will use | ||
2188 | configured or default level). | ||
2189 | |||
2190 | @example | ||
2191 | GNUNET_FORCE_LOG=";;GNUNET_NETWORK_socket_select;;DEBUG" gnunet-arm -s | ||
2192 | @end example | ||
2193 | |||
2194 | @item Start GNUnet process tree, allowing any logging calls from the | ||
2195 | components that have "transport" in their names, and are made from | ||
2196 | function that have "send" in their names. Everything else will be allowed | ||
2197 | to be logged only if it has WARNING level. | ||
2198 | |||
2199 | @example | ||
2200 | GNUNET_FORCE_LOG="transport.*;;.*send.*;;DEBUG/;;;;WARNING" gnunet-arm -s | ||
2201 | @end example | ||
2202 | |||
2203 | @end table | ||
2204 | |||
2205 | |||
2206 | On Windows, one can use batch files to run GNUnet processes with special | ||
2207 | environment variables, without affecting the whole system. Such batch | ||
2208 | file will look like this: | ||
2209 | |||
2210 | @example | ||
2211 | set GNUNET_FORCE_LOG=;;do_transmit;;DEBUG@ gnunet-arm -s | ||
2212 | @end example | ||
2213 | |||
2214 | (note the absence of double quotes in the environment variable definition, | ||
2215 | as opposed to earlier examples, which use the shell). | ||
2216 | Another limitation, on Windows, GNUNET_FORCE_LOGFILE @strong{MUST} be set | ||
2217 | in order to GNUNET_FORCE_LOG to work. | ||
2218 | |||
2219 | |||
2220 | @cindex Log files | ||
2221 | @node Log files | ||
2222 | @subsubsection Log files | ||
2223 | |||
2224 | GNUnet can be told to log everything into a file instead of stderr (which | ||
2225 | is the default) using the "--log-file=logfile" or "-l logfile" option. | ||
2226 | This option can also be passed via command line, or from the "OPTION" and | ||
2227 | "GLOBAL_POSTFIX" configuration keys (see above). The file name passed | ||
2228 | with this option is subject to GNUnet filename expansion. If specified in | ||
2229 | "GLOBAL_POSTFIX", it is also subject to ARM service filename expansion, | ||
2230 | in particular, it may contain "@{@}" (left and right curly brace) | ||
2231 | sequence, which will be replaced by ARM with the name of the service. | ||
2232 | This is used to keep logs from more than one service separate, while only | ||
2233 | specifying one template containing "@{@}" in GLOBAL_POSTFIX. | ||
2234 | |||
2235 | As part of a secondary file name expansion, the first occurrence of "[]" | ||
2236 | sequence ("left square brace" followed by "right square brace") in the | ||
2237 | file name will be replaced with a process identifier or the process when | ||
2238 | it initializes its logging subsystem. As a result, all processes will log | ||
2239 | into different files. This is convenient for isolating messages of a | ||
2240 | particular process, and prevents I/O races when multiple processes try to | ||
2241 | write into the file at the same time. This expansion is done | ||
2242 | independently of "@{@}" expansion that ARM service does (see above). | ||
2243 | |||
2244 | The log file name that is specified via "-l" can contain format characters | ||
2245 | from the 'strftime' function family. For example, "%Y" will be replaced | ||
2246 | with the current year. Using "basename-%Y-%m-%d.log" would include the | ||
2247 | current year, month and day in the log file. If a GNUnet process runs for | ||
2248 | long enough to need more than one log file, it will eventually clean up | ||
2249 | old log files. Currently, only the last three log files (plus the current | ||
2250 | log file) are preserved. So once the fifth log file goes into use (so | ||
2251 | after 4 days if you use "%Y-%m-%d" as above), the first log file will be | ||
2252 | automatically deleted. Note that if your log file name only contains "%Y", | ||
2253 | then log files would be kept for 4 years and the logs from the first year | ||
2254 | would be deleted once year 5 begins. If you do not use any date-related | ||
2255 | string format codes, logs would never be automatically deleted by GNUnet. | ||
2256 | |||
2257 | |||
2258 | @c *********************************************************************** | ||
2259 | |||
2260 | @node Updated behavior of GNUNET_log | ||
2261 | @subsubsection Updated behavior of GNUNET_log | ||
2262 | |||
2263 | It's currently quite common to see constructions like this all over the | ||
2264 | code: | ||
2265 | |||
2266 | @example | ||
2267 | #if MESH_DEBUG | ||
2268 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "MESH: client disconnected\n"); | ||
2269 | #endif | ||
2270 | @end example | ||
2271 | |||
2272 | The reason for the #if is not to avoid displaying the message when | ||
2273 | disabled (GNUNET_ERROR_TYPE takes care of that), but to avoid the | ||
2274 | compiler including it in the binary at all, when compiling GNUnet for | ||
2275 | platforms with restricted storage space / memory (MIPS routers, | ||
2276 | ARM plug computers / dev boards, etc). | ||
2277 | |||
2278 | This presents several problems: the code gets ugly, hard to write and it | ||
2279 | is very easy to forget to include the #if guards, creating non-consistent | ||
2280 | code. A new change in GNUNET_log aims to solve these problems. | ||
2281 | |||
2282 | @strong{This change requires to @file{./configure} with at least | ||
2283 | @code{--enable-logging=verbose} to see debug messages.} | ||
2284 | |||
2285 | Here is an example of code with dense debug statements: | ||
2286 | |||
2287 | @example | ||
2288 | switch (restrict_topology) @{ | ||
2289 | case GNUNET_TESTING_TOPOLOGY_CLIQUE:#if VERBOSE_TESTING | ||
2290 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, _("Blacklisting all but clique | ||
2291 | topology\n")); #endif unblacklisted_connections = create_clique (pg, | ||
2292 | &remove_connections, BLACKLIST, GNUNET_NO); break; case | ||
2293 | GNUNET_TESTING_TOPOLOGY_SMALL_WORLD_RING: #if VERBOSE_TESTING GNUNET_log | ||
2294 | (GNUNET_ERROR_TYPE_DEBUG, _("Blacklisting all but small world (ring) | ||
2295 | topology\n")); #endif unblacklisted_connections = create_small_world_ring | ||
2296 | (pg,&remove_connections, BLACKLIST); break; | ||
2297 | @end example | ||
2298 | |||
2299 | |||
2300 | Pretty hard to follow, huh? | ||
2301 | |||
2302 | From now on, it is not necessary to include the #if / #endif statements to | ||
2303 | achieve the same behavior. The GNUNET_log and GNUNET_log_from macros take | ||
2304 | care of it for you, depending on the configure option: | ||
2305 | |||
2306 | @itemize @bullet | ||
2307 | @item If @code{--enable-logging} is set to @code{no}, the binary will | ||
2308 | contain no log messages at all. | ||
2309 | @item If @code{--enable-logging} is set to @code{yes}, the binary will | ||
2310 | contain no DEBUG messages, and therefore running with -L DEBUG will have | ||
2311 | no effect. Other messages (ERROR, WARNING, INFO, etc) will be included. | ||
2312 | @item If @code{--enable-logging} is set to @code{verbose}, or | ||
2313 | @code{veryverbose} the binary will contain DEBUG messages (still, it will | ||
2314 | be neccessary to run with -L DEBUG or set the DEBUG config option to show | ||
2315 | them). | ||
2316 | @end itemize | ||
2317 | |||
2318 | |||
2319 | If you are a developer: | ||
2320 | @itemize @bullet | ||
2321 | @item please make sure that you @code{./configure | ||
2322 | --enable-logging=@{verbose,veryverbose@}}, so you can see DEBUG messages. | ||
2323 | @item please remove the @code{#if} statements around @code{GNUNET_log | ||
2324 | (GNUNET_ERROR_TYPE_DEBUG, ...)} lines, to improve the readibility of your | ||
2325 | code. | ||
2326 | @end itemize | ||
2327 | |||
2328 | Since now activating DEBUG automatically makes it VERBOSE and activates | ||
2329 | @strong{all} debug messages by default, you probably want to use the | ||
2330 | https://gnunet.org/logging functionality to filter only relevant messages. | ||
2331 | A suitable configuration could be: | ||
2332 | |||
2333 | @example | ||
2334 | $ export GNUNET_FORCE_LOG="^YOUR_SUBSYSTEM$;;;;DEBUG/;;;;WARNING" | ||
2335 | @end example | ||
2336 | |||
2337 | Which will behave almost like enabling DEBUG in that subsytem before the | ||
2338 | change. Of course you can adapt it to your particular needs, this is only | ||
2339 | a quick example. | ||
2340 | |||
2341 | @cindex Interprocess communication API | ||
2342 | @cindex ICP | ||
2343 | @node Interprocess communication API (IPC) | ||
2344 | @subsection Interprocess communication API (IPC) | ||
2345 | |||
2346 | In GNUnet a variety of new message types might be defined and used in | ||
2347 | interprocess communication, in this tutorial we use the | ||
2348 | @code{struct AddressLookupMessage} as a example to introduce how to | ||
2349 | construct our own message type in GNUnet and how to implement the message | ||
2350 | communication between service and client. | ||
2351 | (Here, a client uses the @code{struct AddressLookupMessage} as a request | ||
2352 | to ask the server to return the address of any other peer connecting to | ||
2353 | the service.) | ||
2354 | |||
2355 | |||
2356 | @c *********************************************************************** | ||
2357 | @menu | ||
2358 | * Define new message types:: | ||
2359 | * Define message struct:: | ||
2360 | * Client - Establish connection:: | ||
2361 | * Client - Initialize request message:: | ||
2362 | * Client - Send request and receive response:: | ||
2363 | * Server - Startup service:: | ||
2364 | * Server - Add new handles for specified messages:: | ||
2365 | * Server - Process request message:: | ||
2366 | * Server - Response to client:: | ||
2367 | * Server - Notification of clients:: | ||
2368 | * Conversion between Network Byte Order (Big Endian) and Host Byte Order:: | ||
2369 | @end menu | ||
2370 | |||
2371 | @node Define new message types | ||
2372 | @subsubsection Define new message types | ||
2373 | |||
2374 | First of all, you should define the new message type in | ||
2375 | @file{gnunet_protocols.h}: | ||
2376 | |||
2377 | @example | ||
2378 | // Request to look addresses of peers in server. | ||
2379 | #define GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_LOOKUP 29 | ||
2380 | // Response to the address lookup request. | ||
2381 | #define GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_REPLY 30 | ||
2382 | @end example | ||
2383 | |||
2384 | @c *********************************************************************** | ||
2385 | @node Define message struct | ||
2386 | @subsubsection Define message struct | ||
2387 | |||
2388 | After the type definition, the specified message structure should also be | ||
2389 | described in the header file, e.g. transport.h in our case. | ||
2390 | |||
2391 | @example | ||
2392 | struct AddressLookupMessage @{ | ||
2393 | struct GNUNET_MessageHeader header; | ||
2394 | int32_t numeric_only GNUNET_PACKED; | ||
2395 | struct GNUNET_TIME_AbsoluteNBO timeout; | ||
2396 | uint32_t addrlen GNUNET_PACKED; | ||
2397 | /* followed by 'addrlen' bytes of the actual address, then | ||
2398 | followed by the 0-terminated name of the transport */ @}; | ||
2399 | GNUNET_NETWORK_STRUCT_END | ||
2400 | @end example | ||
2401 | |||
2402 | |||
2403 | Please note @code{GNUNET_NETWORK_STRUCT_BEGIN} and @code{GNUNET_PACKED} | ||
2404 | which both ensure correct alignment when sending structs over the network. | ||
2405 | |||
2406 | @menu | ||
2407 | @end menu | ||
2408 | |||
2409 | @c *********************************************************************** | ||
2410 | @node Client - Establish connection | ||
2411 | @subsubsection Client - Establish connection | ||
2412 | @c %**end of header | ||
2413 | |||
2414 | |||
2415 | At first, on the client side, the underlying API is employed to create a | ||
2416 | new connection to a service, in our example the transport service would be | ||
2417 | connected. | ||
2418 | |||
2419 | @example | ||
2420 | struct GNUNET_CLIENT_Connection *client; | ||
2421 | client = GNUNET_CLIENT_connect ("transport", cfg); | ||
2422 | @end example | ||
2423 | |||
2424 | @c *********************************************************************** | ||
2425 | @node Client - Initialize request message | ||
2426 | @subsubsection Client - Initialize request message | ||
2427 | @c %**end of header | ||
2428 | |||
2429 | When the connection is ready, we initialize the message. In this step, | ||
2430 | all the fields of the message should be properly initialized, namely the | ||
2431 | size, type, and some extra user-defined data, such as timeout, name of | ||
2432 | transport, address and name of transport. | ||
2433 | |||
2434 | @example | ||
2435 | struct AddressLookupMessage *msg; | ||
2436 | size_t len = sizeof (struct AddressLookupMessage) | ||
2437 | + addressLen | ||
2438 | + strlen (nameTrans) | ||
2439 | + 1; | ||
2440 | msg->header->size = htons (len); | ||
2441 | msg->header->type = htons | ||
2442 | (GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_LOOKUP); | ||
2443 | msg->timeout = GNUNET_TIME_absolute_hton (abs_timeout); | ||
2444 | msg->addrlen = htonl (addressLen); | ||
2445 | char *addrbuf = (char *) &msg[1]; | ||
2446 | memcpy (addrbuf, address, addressLen); | ||
2447 | char *tbuf = &addrbuf[addressLen]; | ||
2448 | memcpy (tbuf, nameTrans, strlen (nameTrans) + 1); | ||
2449 | @end example | ||
2450 | |||
2451 | Note that, here the functions @code{htonl}, @code{htons} and | ||
2452 | @code{GNUNET_TIME_absolute_hton} are applied to convert little endian | ||
2453 | into big endian, about the usage of the big/small edian order and the | ||
2454 | corresponding conversion function please refer to Introduction of | ||
2455 | Big Endian and Little Endian. | ||
2456 | |||
2457 | @c *********************************************************************** | ||
2458 | @node Client - Send request and receive response | ||
2459 | @subsubsection Client - Send request and receive response | ||
2460 | @c %**end of header | ||
2461 | |||
2462 | @b{FIXME: This is very outdated, see the tutorial for the current API!} | ||
2463 | |||
2464 | Next, the client would send the constructed message as a request to the | ||
2465 | service and wait for the response from the service. To accomplish this | ||
2466 | goal, there are a number of API calls that can be used. In this example, | ||
2467 | @code{GNUNET_CLIENT_transmit_and_get_response} is chosen as the most | ||
2468 | appropriate function to use. | ||
2469 | |||
2470 | @example | ||
2471 | GNUNET_CLIENT_transmit_and_get_response | ||
2472 | (client, msg->header, timeout, GNUNET_YES, &address_response_processor, | ||
2473 | arp_ctx); | ||
2474 | @end example | ||
2475 | |||
2476 | the argument @code{address_response_processor} is a function with | ||
2477 | @code{GNUNET_CLIENT_MessageHandler} type, which is used to process the | ||
2478 | reply message from the service. | ||
2479 | |||
2480 | @node Server - Startup service | ||
2481 | @subsubsection Server - Startup service | ||
2482 | |||
2483 | After receiving the request message, we run a standard GNUnet service | ||
2484 | startup sequence using @code{GNUNET_SERVICE_run}, as follows, | ||
2485 | |||
2486 | @example | ||
2487 | int main(int argc, char**argv) @{ | ||
2488 | GNUNET_SERVICE_run(argc, argv, "transport" | ||
2489 | GNUNET_SERVICE_OPTION_NONE, &run, NULL)); @} | ||
2490 | @end example | ||
2491 | |||
2492 | @c *********************************************************************** | ||
2493 | @node Server - Add new handles for specified messages | ||
2494 | @subsubsection Server - Add new handles for specified messages | ||
2495 | @c %**end of header | ||
2496 | |||
2497 | in the function above the argument @code{run} is used to initiate | ||
2498 | transport service,and defined like this: | ||
2499 | |||
2500 | @example | ||
2501 | static void run (void *cls, | ||
2502 | struct GNUNET_SERVER_Handle *serv, | ||
2503 | const struct GNUNET_CONFIGURATION_Handle *cfg) @{ | ||
2504 | GNUNET_SERVER_add_handlers (serv, handlers); @} | ||
2505 | @end example | ||
2506 | |||
2507 | |||
2508 | Here, @code{GNUNET_SERVER_add_handlers} must be called in the run | ||
2509 | function to add new handlers in the service. The parameter | ||
2510 | @code{handlers} is a list of @code{struct GNUNET_SERVER_MessageHandler} | ||
2511 | to tell the service which function should be called when a particular | ||
2512 | type of message is received, and should be defined in this way: | ||
2513 | |||
2514 | @example | ||
2515 | static struct GNUNET_SERVER_MessageHandler handlers[] = @{ | ||
2516 | @{&handle_start, | ||
2517 | NULL, | ||
2518 | GNUNET_MESSAGE_TYPE_TRANSPORT_START, | ||
2519 | 0@}, | ||
2520 | @{&handle_send, | ||
2521 | NULL, | ||
2522 | GNUNET_MESSAGE_TYPE_TRANSPORT_SEND, | ||
2523 | 0@}, | ||
2524 | @{&handle_try_connect, | ||
2525 | NULL, | ||
2526 | GNUNET_MESSAGE_TYPE_TRANSPORT_TRY_CONNECT, | ||
2527 | sizeof (struct TryConnectMessage) | ||
2528 | @}, | ||
2529 | @{&handle_address_lookup, | ||
2530 | NULL, | ||
2531 | GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_LOOKUP, | ||
2532 | 0@}, | ||
2533 | @{NULL, | ||
2534 | NULL, | ||
2535 | 0, | ||
2536 | 0@} | ||
2537 | @}; | ||
2538 | @end example | ||
2539 | |||
2540 | |||
2541 | As shown, the first member of the struct in the first area is a callback | ||
2542 | function, which is called to process the specified message types, given | ||
2543 | as the third member. The second parameter is the closure for the callback | ||
2544 | function, which is set to @code{NULL} in most cases, and the last | ||
2545 | parameter is the expected size of the message of this type, usually we | ||
2546 | set it to 0 to accept variable size, for special cases the exact size of | ||
2547 | the specified message also can be set. In addition, the terminator sign | ||
2548 | depicted as @code{@{NULL, NULL, 0, 0@}} is set in the last aera. | ||
2549 | |||
2550 | @c *********************************************************************** | ||
2551 | @node Server - Process request message | ||
2552 | @subsubsection Server - Process request message | ||
2553 | @c %**end of header | ||
2554 | |||
2555 | After the initialization of transport service, the request message would | ||
2556 | be processed. Before handling the main message data, the validity of this | ||
2557 | message should be checked out, e.g., to check whether the size of message | ||
2558 | is correct. | ||
2559 | |||
2560 | @example | ||
2561 | size = ntohs (message->size); | ||
2562 | if (size < sizeof (struct AddressLookupMessage)) @{ | ||
2563 | GNUNET_break_op (0); | ||
2564 | GNUNET_SERVER_receive_done (client, GNUNET_SYSERR); | ||
2565 | return; @} | ||
2566 | @end example | ||
2567 | |||
2568 | |||
2569 | Note that, opposite to the construction method of the request message in | ||
2570 | the client, in the server the function @code{nothl} and @code{ntohs} | ||
2571 | should be employed during the extraction of the data from the message, so | ||
2572 | that the data in big endian order can be converted back into little | ||
2573 | endian order. See more in detail please refer to Introduction of | ||
2574 | Big Endian and Little Endian. | ||
2575 | |||
2576 | Moreover in this example, the name of the transport stored in the message | ||
2577 | is a 0-terminated string, so we should also check whether the name of the | ||
2578 | transport in the received message is 0-terminated: | ||
2579 | |||
2580 | @example | ||
2581 | nameTransport = (const char *) &address[addressLen]; | ||
2582 | if (nameTransport[size - sizeof | ||
2583 | (struct AddressLookupMessage) | ||
2584 | - addressLen - 1] != '\0') @{ | ||
2585 | GNUNET_break_op (0); | ||
2586 | GNUNET_SERVER_receive_done (client, | ||
2587 | GNUNET_SYSERR); | ||
2588 | return; @} | ||
2589 | @end example | ||
2590 | |||
2591 | Here, @code{GNUNET_SERVER_receive_done} should be called to tell the | ||
2592 | service that the request is done and can receive the next message. The | ||
2593 | argument @code{GNUNET_SYSERR} here indicates that the service didn't | ||
2594 | understand the request message, and the processing of this request would | ||
2595 | be terminated. | ||
2596 | |||
2597 | In comparison to the aforementioned situation, when the argument is equal | ||
2598 | to @code{GNUNET_OK}, the service would continue to process the requst | ||
2599 | message. | ||
2600 | |||
2601 | @c *********************************************************************** | ||
2602 | @node Server - Response to client | ||
2603 | @subsubsection Server - Response to client | ||
2604 | @c %**end of header | ||
2605 | |||
2606 | Once the processing of current request is done, the server should give the | ||
2607 | response to the client. A new @code{struct AddressLookupMessage} would be | ||
2608 | produced by the server in a similar way as the client did and sent to the | ||
2609 | client, but here the type should be | ||
2610 | @code{GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_REPLY} rather than | ||
2611 | @code{GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_LOOKUP} in client. | ||
2612 | @example | ||
2613 | struct AddressLookupMessage *msg; | ||
2614 | size_t len = sizeof (struct AddressLookupMessage) | ||
2615 | + addressLen | ||
2616 | + strlen (nameTrans) + 1; | ||
2617 | msg->header->size = htons (len); | ||
2618 | msg->header->type = htons | ||
2619 | (GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_REPLY); | ||
2620 | |||
2621 | // ... | ||
2622 | |||
2623 | struct GNUNET_SERVER_TransmitContext *tc; | ||
2624 | tc = GNUNET_SERVER_transmit_context_create (client); | ||
2625 | GNUNET_SERVER_transmit_context_append_data | ||
2626 | (tc, | ||
2627 | NULL, | ||
2628 | 0, | ||
2629 | GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_REPLY); | ||
2630 | GNUNET_SERVER_transmit_context_run (tc, rtimeout); | ||
2631 | @end example | ||
2632 | |||
2633 | |||
2634 | Note that, there are also a number of other APIs provided to the service | ||
2635 | to send the message. | ||
2636 | |||
2637 | @c *********************************************************************** | ||
2638 | @node Server - Notification of clients | ||
2639 | @subsubsection Server - Notification of clients | ||
2640 | @c %**end of header | ||
2641 | |||
2642 | Often a service needs to (repeatedly) transmit notifications to a client | ||
2643 | or a group of clients. In these cases, the client typically has once | ||
2644 | registered for a set of events and then needs to receive a message | ||
2645 | whenever such an event happens (until the client disconnects). The use of | ||
2646 | a notification context can help manage message queues to clients and | ||
2647 | handle disconnects. Notification contexts can be used to send | ||
2648 | individualized messages to a particular client or to broadcast messages | ||
2649 | to a group of clients. An individualized notification might look like | ||
2650 | this: | ||
2651 | |||
2652 | @example | ||
2653 | GNUNET_SERVER_notification_context_unicast(nc, | ||
2654 | client, | ||
2655 | msg, | ||
2656 | GNUNET_YES); | ||
2657 | @end example | ||
2658 | |||
2659 | |||
2660 | Note that after processing the original registration message for | ||
2661 | notifications, the server code still typically needs to call | ||
2662 | @code{GNUNET_SERVER_receive_done} so that the client can transmit further | ||
2663 | messages to the server. | ||
2664 | |||
2665 | @c *********************************************************************** | ||
2666 | @node Conversion between Network Byte Order (Big Endian) and Host Byte Order | ||
2667 | @subsubsection Conversion between Network Byte Order (Big Endian) and Host Byte Order | ||
2668 | @c %** subsub? it's a referenced page on the ipc document. | ||
2669 | @c %**end of header | ||
2670 | |||
2671 | Here we can simply comprehend big endian and little endian as Network Byte | ||
2672 | Order and Host Byte Order respectively. What is the difference between | ||
2673 | both two? | ||
2674 | |||
2675 | Usually in our host computer we store the data byte as Host Byte Order, | ||
2676 | for example, we store a integer in the RAM which might occupies 4 Byte, | ||
2677 | as Host Byte Order the higher Byte would be stored at the lower address | ||
2678 | of RAM, and the lower Byte would be stored at the higher address of RAM. | ||
2679 | However, contrast to this, Network Byte Order just take the totally | ||
2680 | opposite way to store the data, says, it will store the lower Byte at the | ||
2681 | lower address, and the higher Byte will stay at higher address. | ||
2682 | |||
2683 | For the current communication of network, we normally exchange the | ||
2684 | information by surveying the data package, every two host wants to | ||
2685 | communicate with each other must send and receive data package through | ||
2686 | network. In order to maintain the identity of data through the | ||
2687 | transmission in the network, the order of the Byte storage must changed | ||
2688 | before sending and after receiving the data. | ||
2689 | |||
2690 | There ten convenient functions to realize the conversion of Byte Order in | ||
2691 | GNUnet, as following: | ||
2692 | |||
2693 | @table @asis | ||
2694 | |||
2695 | @item uint16_t htons(uint16_t hostshort) Convert host byte order to net | ||
2696 | byte order with short int | ||
2697 | @item uint32_t htonl(uint32_t hostlong) Convert host byte | ||
2698 | order to net byte order with long int | ||
2699 | @item uint16_t ntohs(uint16_t netshort) | ||
2700 | Convert net byte order to host byte order with short int | ||
2701 | @item uint32_t | ||
2702 | ntohl(uint32_t netlong) Convert net byte order to host byte order with | ||
2703 | long int | ||
2704 | @item unsigned long long GNUNET_ntohll (unsigned long long netlonglong) | ||
2705 | Convert net byte order to host byte order with long long int | ||
2706 | @item unsigned long long GNUNET_htonll (unsigned long long hostlonglong) | ||
2707 | Convert host byte order to net byte order with long long int | ||
2708 | @item struct GNUNET_TIME_RelativeNBO GNUNET_TIME_relative_hton | ||
2709 | (struct GNUNET_TIME_Relative a) Convert relative time to network byte | ||
2710 | order. | ||
2711 | @item struct GNUNET_TIME_Relative GNUNET_TIME_relative_ntoh | ||
2712 | (struct GNUNET_TIME_RelativeNBO a) Convert relative time from network | ||
2713 | byte order. | ||
2714 | @item struct GNUNET_TIME_AbsoluteNBO GNUNET_TIME_absolute_hton | ||
2715 | (struct GNUNET_TIME_Absolute a) Convert relative time to network byte | ||
2716 | order. | ||
2717 | @item struct GNUNET_TIME_Absolute GNUNET_TIME_absolute_ntoh | ||
2718 | (struct GNUNET_TIME_AbsoluteNBO a) Convert relative time from network | ||
2719 | byte order. | ||
2720 | @end table | ||
2721 | |||
2722 | @cindex Cryptography API | ||
2723 | @node Cryptography API | ||
2724 | @subsection Cryptography API | ||
2725 | @c %**end of header | ||
2726 | |||
2727 | The gnunetutil APIs provides the cryptographic primitives used in GNUnet. | ||
2728 | GNUnet uses 2048 bit RSA keys for the session key exchange and for signing | ||
2729 | messages by peers and most other public-key operations. Most researchers | ||
2730 | in cryptography consider 2048 bit RSA keys as secure and practically | ||
2731 | unbreakable for a long time. The API provides functions to create a fresh | ||
2732 | key pair, read a private key from a file (or create a new file if the | ||
2733 | file does not exist), encrypt, decrypt, sign, verify and extraction of | ||
2734 | the public key into a format suitable for network transmission. | ||
2735 | |||
2736 | For the encryption of files and the actual data exchanged between peers | ||
2737 | GNUnet uses 256-bit AES encryption. Fresh, session keys are negotiated | ||
2738 | for every new connection.@ Again, there is no published technique to | ||
2739 | break this cipher in any realistic amount of time. The API provides | ||
2740 | functions for generation of keys, validation of keys (important for | ||
2741 | checking that decryptions using RSA succeeded), encryption and decryption. | ||
2742 | |||
2743 | GNUnet uses SHA-512 for computing one-way hash codes. The API provides | ||
2744 | functions to compute a hash over a block in memory or over a file on disk. | ||
2745 | |||
2746 | The crypto API also provides functions for randomizing a block of memory, | ||
2747 | obtaining a single random number and for generating a permuation of the | ||
2748 | numbers 0 to n-1. Random number generation distinguishes between WEAK and | ||
2749 | STRONG random number quality; WEAK random numbers are pseudo-random | ||
2750 | whereas STRONG random numbers use entropy gathered from the operating | ||
2751 | system. | ||
2752 | |||
2753 | Finally, the crypto API provides a means to deterministically generate a | ||
2754 | 1024-bit RSA key from a hash code. These functions should most likely not | ||
2755 | be used by most applications; most importantly, | ||
2756 | GNUNET_CRYPTO_rsa_key_create_from_hash does not create an RSA-key that | ||
2757 | should be considered secure for traditional applications of RSA. | ||
2758 | |||
2759 | @cindex Message Queue API | ||
2760 | @node Message Queue API | ||
2761 | @subsection Message Queue API | ||
2762 | @c %**end of header | ||
2763 | |||
2764 | @strong{ Introduction }@ | ||
2765 | Often, applications need to queue messages that | ||
2766 | are to be sent to other GNUnet peers, clients or services. As all of | ||
2767 | GNUnet's message-based communication APIs, by design, do not allow | ||
2768 | messages to be queued, it is common to implement custom message queues | ||
2769 | manually when they are needed. However, writing very similar code in | ||
2770 | multiple places is tedious and leads to code duplication. | ||
2771 | |||
2772 | MQ (for Message Queue) is an API that provides the functionality to | ||
2773 | implement and use message queues. We intend to eventually replace all of | ||
2774 | the custom message queue implementations in GNUnet with MQ. | ||
2775 | |||
2776 | @strong{ Basic Concepts }@ | ||
2777 | The two most important entities in MQ are queues and envelopes. | ||
2778 | |||
2779 | Every queue is backed by a specific implementation (e.g. for mesh, stream, | ||
2780 | connection, server client, etc.) that will actually deliver the queued | ||
2781 | messages. For convenience,@ some queues also allow to specify a list of | ||
2782 | message handlers. The message queue will then also wait for incoming | ||
2783 | messages and dispatch them appropriately. | ||
2784 | |||
2785 | An envelope holds the the memory for a message, as well as metadata | ||
2786 | (Where is the envelope queued? What should happen after it has been | ||
2787 | sent?). Any envelope can only be queued in one message queue. | ||
2788 | |||
2789 | @strong{ Creating Queues }@ | ||
2790 | The following is a list of currently available message queues. Note that | ||
2791 | to avoid layering issues, message queues for higher level APIs are not | ||
2792 | part of @code{libgnunetutil}, but@ the respective API itself provides the | ||
2793 | queue implementation. | ||
2794 | |||
2795 | @table @asis | ||
2796 | |||
2797 | @item @code{GNUNET_MQ_queue_for_connection_client} | ||
2798 | Transmits queued messages over a @code{GNUNET_CLIENT_Connection} handle. | ||
2799 | Also supports receiving with message handlers. | ||
2800 | |||
2801 | @item @code{GNUNET_MQ_queue_for_server_client} | ||
2802 | Transmits queued messages over a @code{GNUNET_SERVER_Client} handle. Does | ||
2803 | not support incoming message handlers. | ||
2804 | |||
2805 | @item @code{GNUNET_MESH_mq_create} Transmits queued messages over a | ||
2806 | @code{GNUNET_MESH_Tunnel} handle. Does not support incoming message | ||
2807 | handlers. | ||
2808 | |||
2809 | @item @code{GNUNET_MQ_queue_for_callbacks} This is the most general | ||
2810 | implementation. Instead of delivering and receiving messages with one of | ||
2811 | GNUnet's communication APIs, implementation callbacks are called. Refer to | ||
2812 | "Implementing Queues" for a more detailed explanation. | ||
2813 | @end table | ||
2814 | |||
2815 | |||
2816 | @strong{ Allocating Envelopes }@ | ||
2817 | A GNUnet message (as defined by the GNUNET_MessageHeader) has three | ||
2818 | parts: The size, the type, and the body. | ||
2819 | |||
2820 | MQ provides macros to allocate an envelope containing a message | ||
2821 | conveniently, automatically setting the size and type fields of the | ||
2822 | message. | ||
2823 | |||
2824 | Consider the following simple message, with the body consisting of a | ||
2825 | single number value. | ||
2826 | @c why the empy code function? | ||
2827 | @code{} | ||
2828 | |||
2829 | @example | ||
2830 | struct NumberMessage @{ | ||
2831 | /** Type: GNUNET_MESSAGE_TYPE_EXAMPLE_1 */ | ||
2832 | struct GNUNET_MessageHeader header; | ||
2833 | uint32_t number GNUNET_PACKED; | ||
2834 | @}; | ||
2835 | @end example | ||
2836 | |||
2837 | An envelope containing an instance of the NumberMessage can be | ||
2838 | constructed like this: | ||
2839 | |||
2840 | @example | ||
2841 | struct GNUNET_MQ_Envelope *ev; | ||
2842 | struct NumberMessage *msg; | ||
2843 | ev = GNUNET_MQ_msg (msg, GNUNET_MESSAGE_TYPE_EXAMPLE_1); | ||
2844 | msg->number = htonl (42); | ||
2845 | @end example | ||
2846 | |||
2847 | In the above code, @code{GNUNET_MQ_msg} is a macro. The return value is | ||
2848 | the newly allocated envelope. The first argument must be a pointer to some | ||
2849 | @code{struct} containing a @code{struct GNUNET_MessageHeader header} | ||
2850 | field, while the second argument is the desired message type, in host | ||
2851 | byte order. | ||
2852 | |||
2853 | The @code{msg} pointer now points to an allocated message, where the | ||
2854 | message type and the message size are already set. The message's size is | ||
2855 | inferred from the type of the @code{msg} pointer: It will be set to | ||
2856 | 'sizeof(*msg)', properly converted to network byte order. | ||
2857 | |||
2858 | If the message body's size is dynamic, the the macro | ||
2859 | @code{GNUNET_MQ_msg_extra} can be used to allocate an envelope whose | ||
2860 | message has additional space allocated after the @code{msg} structure. | ||
2861 | |||
2862 | If no structure has been defined for the message, | ||
2863 | @code{GNUNET_MQ_msg_header_extra} can be used to allocate additional space | ||
2864 | after the message header. The first argument then must be a pointer to a | ||
2865 | @code{GNUNET_MessageHeader}. | ||
2866 | |||
2867 | @strong{Envelope Properties}@ | ||
2868 | A few functions in MQ allow to set additional properties on envelopes: | ||
2869 | |||
2870 | @table @asis | ||
2871 | |||
2872 | @item @code{GNUNET_MQ_notify_sent} Allows to specify a function that will | ||
2873 | be called once the envelope's message has been sent irrevocably. | ||
2874 | An envelope can be canceled precisely up to the@ point where the notify | ||
2875 | sent callback has been called. | ||
2876 | |||
2877 | @item @code{GNUNET_MQ_disable_corking} No corking will be used when | ||
2878 | sending the message. Not every@ queue supports this flag, per default, | ||
2879 | envelopes are sent with corking.@ | ||
2880 | |||
2881 | @end table | ||
2882 | |||
2883 | |||
2884 | @strong{Sending Envelopes}@ | ||
2885 | Once an envelope has been constructed, it can be queued for sending with | ||
2886 | @code{GNUNET_MQ_send}. | ||
2887 | |||
2888 | Note that in order to avoid memory leaks, an envelope must either be sent | ||
2889 | (the queue will free it) or destroyed explicitly with | ||
2890 | @code{GNUNET_MQ_discard}. | ||
2891 | |||
2892 | @strong{Canceling Envelopes}@ | ||
2893 | An envelope queued with @code{GNUNET_MQ_send} can be canceled with | ||
2894 | @code{GNUNET_MQ_cancel}. Note that after the notify sent callback has | ||
2895 | been called, canceling a message results in undefined behavior. | ||
2896 | Thus it is unsafe to cancel an envelope that does not have a notify sent | ||
2897 | callback. When canceling an envelope, it is not necessary@ to call | ||
2898 | @code{GNUNET_MQ_discard}, and the envelope can't be sent again. | ||
2899 | |||
2900 | @strong{ Implementing Queues }@ | ||
2901 | @code{TODO} | ||
2902 | |||
2903 | @cindex Service API | ||
2904 | @node Service API | ||
2905 | @subsection Service API | ||
2906 | @c %**end of header | ||
2907 | |||
2908 | Most GNUnet code lives in the form of services. Services are processes | ||
2909 | that offer an API for other components of the system to build on. Those | ||
2910 | other components can be command-line tools for users, graphical user | ||
2911 | interfaces or other services. Services provide their API using an IPC | ||
2912 | protocol. For this, each service must listen on either a TCP port or a | ||
2913 | UNIX domain socket; for this, the service implementation uses the server | ||
2914 | API. This use of server is exposed directly to the users of the service | ||
2915 | API. Thus, when using the service API, one is usually also often using | ||
2916 | large parts of the server API. The service API provides various | ||
2917 | convenience functions, such as parsing command-line arguments and the | ||
2918 | configuration file, which are not found in the server API. | ||
2919 | The dual to the service/server API is the client API, which can be used to | ||
2920 | access services. | ||
2921 | |||
2922 | The most common way to start a service is to use the | ||
2923 | @code{GNUNET_SERVICE_run} function from the program's main function. | ||
2924 | @code{GNUNET_SERVICE_run} will then parse the command line and | ||
2925 | configuration files and, based on the options found there, | ||
2926 | start the server. It will then give back control to the main | ||
2927 | program, passing the server and the configuration to the | ||
2928 | @code{GNUNET_SERVICE_Main} callback. @code{GNUNET_SERVICE_run} | ||
2929 | will also take care of starting the scheduler loop. | ||
2930 | If this is inappropriate (for example, because the scheduler loop | ||
2931 | is already running), @code{GNUNET_SERVICE_start} and | ||
2932 | related functions provide an alternative to @code{GNUNET_SERVICE_run}. | ||
2933 | |||
2934 | When starting a service, the service_name option is used to determine | ||
2935 | which sections in the configuration file should be used to configure the | ||
2936 | service. A typical value here is the name of the @file{src/} | ||
2937 | sub-directory, for example @file{statistics}. | ||
2938 | The same string would also be given to | ||
2939 | @code{GNUNET_CLIENT_connect} to access the service. | ||
2940 | |||
2941 | Once a service has been initialized, the program should use the | ||
2942 | @code{GNUNET_SERVICE_Main} callback to register message handlers | ||
2943 | using @code{GNUNET_SERVER_add_handlers}. | ||
2944 | The service will already have registered a handler for the | ||
2945 | "TEST" message. | ||
2946 | |||
2947 | @fnindex GNUNET_SERVICE_Options | ||
2948 | The option bitfield (@code{enum GNUNET_SERVICE_Options}) | ||
2949 | determines how a service should behave during shutdown. | ||
2950 | There are three key strategies: | ||
2951 | |||
2952 | @table @asis | ||
2953 | |||
2954 | @item instant (@code{GNUNET_SERVICE_OPTION_NONE}) | ||
2955 | Upon receiving the shutdown | ||
2956 | signal from the scheduler, the service immediately terminates the server, | ||
2957 | closing all existing connections with clients. | ||
2958 | @item manual (@code{GNUNET_SERVICE_OPTION_MANUAL_SHUTDOWN}) | ||
2959 | The service does nothing by itself | ||
2960 | during shutdown. The main program will need to take the appropriate | ||
2961 | action by calling GNUNET_SERVER_destroy or GNUNET_SERVICE_stop (depending | ||
2962 | on how the service was initialized) to terminate the service. This method | ||
2963 | is used by gnunet-service-arm and rather uncommon. | ||
2964 | @item soft (@code{GNUNET_SERVICE_OPTION_SOFT_SHUTDOWN}) | ||
2965 | Upon receiving the shutdown signal from the scheduler, | ||
2966 | the service immediately tells the server to stop | ||
2967 | listening for incoming clients. Requests from normal existing clients are | ||
2968 | still processed and the server/service terminates once all normal clients | ||
2969 | have disconnected. Clients that are not expected to ever disconnect (such | ||
2970 | as clients that monitor performance values) can be marked as 'monitor' | ||
2971 | clients using GNUNET_SERVER_client_mark_monitor. Those clients will | ||
2972 | continue to be processed until all 'normal' clients have disconnected. | ||
2973 | Then, the server will terminate, closing the monitor connections. | ||
2974 | This mode is for example used by 'statistics', allowing existing 'normal' | ||
2975 | clients to set (possibly persistent) statistic values before terminating. | ||
2976 | |||
2977 | @end table | ||
2978 | |||
2979 | @c *********************************************************************** | ||
2980 | @node Optimizing Memory Consumption of GNUnet's (Multi-) Hash Maps | ||
2981 | @subsection Optimizing Memory Consumption of GNUnet's (Multi-) Hash Maps | ||
2982 | @c %**end of header | ||
2983 | |||
2984 | A commonly used data structure in GNUnet is a (multi-)hash map. It is most | ||
2985 | often used to map a peer identity to some data structure, but also to map | ||
2986 | arbitrary keys to values (for example to track requests in the distributed | ||
2987 | hash table or in file-sharing). As it is commonly used, the DHT is | ||
2988 | actually sometimes responsible for a large share of GNUnet's overall | ||
2989 | memory consumption (for some processes, 30% is not uncommon). The | ||
2990 | following text documents some API quirks (and their implications for | ||
2991 | applications) that were recently introduced to minimize the footprint of | ||
2992 | the hash map. | ||
2993 | |||
2994 | |||
2995 | @c *********************************************************************** | ||
2996 | @menu | ||
2997 | * Analysis:: | ||
2998 | * Solution:: | ||
2999 | * Migration:: | ||
3000 | * Conclusion:: | ||
3001 | * Availability:: | ||
3002 | @end menu | ||
3003 | |||
3004 | @node Analysis | ||
3005 | @subsubsection Analysis | ||
3006 | @c %**end of header | ||
3007 | |||
3008 | The main reason for the "excessive" memory consumption by the hash map is | ||
3009 | that GNUnet uses 512-bit cryptographic hash codes --- and the | ||
3010 | (multi-)hash map also uses the same 512-bit 'struct GNUNET_HashCode'. As | ||
3011 | a result, storing just the keys requires 64 bytes of memory for each key. | ||
3012 | As some applications like to keep a large number of entries in the hash | ||
3013 | map (after all, that's what maps are good for), 64 bytes per hash is | ||
3014 | significant: keeping a pointer to the value and having a linked list for | ||
3015 | collisions consume between 8 and 16 bytes, and 'malloc' may add about the | ||
3016 | same overhead per allocation, putting us in the 16 to 32 byte per entry | ||
3017 | ballpark. Adding a 64-byte key then triples the overall memory | ||
3018 | requirement for the hash map. | ||
3019 | |||
3020 | To make things "worse", most of the time storing the key in the hash map | ||
3021 | is not required: it is typically already in memory elsewhere! In most | ||
3022 | cases, the values stored in the hash map are some application-specific | ||
3023 | struct that _also_ contains the hash. Here is a simplified example: | ||
3024 | |||
3025 | @example | ||
3026 | struct MyValue @{ | ||
3027 | struct GNUNET_HashCode key; | ||
3028 | unsigned int my_data; @}; | ||
3029 | |||
3030 | // ... | ||
3031 | val = GNUNET_malloc (sizeof (struct MyValue)); | ||
3032 | val->key = key; | ||
3033 | val->my_data = 42; | ||
3034 | GNUNET_CONTAINER_multihashmap_put (map, &key, val, ...); | ||
3035 | @end example | ||
3036 | |||
3037 | This is a common pattern as later the entries might need to be removed, | ||
3038 | and at that time it is convenient to have the key immediately at hand: | ||
3039 | |||
3040 | @example | ||
3041 | GNUNET_CONTAINER_multihashmap_remove (map, &val->key, val); | ||
3042 | @end example | ||
3043 | |||
3044 | |||
3045 | Note that here we end up with two times 64 bytes for the key, plus maybe | ||
3046 | 64 bytes total for the rest of the 'struct MyValue' and the map entry in | ||
3047 | the hash map. The resulting redundant storage of the key increases | ||
3048 | overall memory consumption per entry from the "optimal" 128 bytes to 192 | ||
3049 | bytes. This is not just an extreme example: overheads in practice are | ||
3050 | actually sometimes close to those highlighted in this example. This is | ||
3051 | especially true for maps with a significant number of entries, as there | ||
3052 | we tend to really try to keep the entries small. | ||
3053 | |||
3054 | @c *********************************************************************** | ||
3055 | @node Solution | ||
3056 | @subsubsection Solution | ||
3057 | @c %**end of header | ||
3058 | |||
3059 | The solution that has now been implemented is to @strong{optionally} | ||
3060 | allow the hash map to not make a (deep) copy of the hash but instead have | ||
3061 | a pointer to the hash/key in the entry. This reduces the memory | ||
3062 | consumption for the key from 64 bytes to 4 to 8 bytes. However, it can | ||
3063 | also only work if the key is actually stored in the entry (which is the | ||
3064 | case most of the time) and if the entry does not modify the key (which in | ||
3065 | all of the code I'm aware of has been always the case if there key is | ||
3066 | stored in the entry). Finally, when the client stores an entry in the | ||
3067 | hash map, it @strong{must} provide a pointer to the key within the entry, | ||
3068 | not just a pointer to a transient location of the key. If | ||
3069 | the client code does not meet these requirements, the result is a dangling | ||
3070 | pointer and undefined behavior of the (multi-)hash map API. | ||
3071 | |||
3072 | @c *********************************************************************** | ||
3073 | @node Migration | ||
3074 | @subsubsection Migration | ||
3075 | @c %**end of header | ||
3076 | |||
3077 | To use the new feature, first check that the values contain the respective | ||
3078 | key (and never modify it). Then, all calls to | ||
3079 | @code{GNUNET_CONTAINER_multihashmap_put} on the respective map must be | ||
3080 | audited and most likely changed to pass a pointer into the value's struct. | ||
3081 | For the initial example, the new code would look like this: | ||
3082 | |||
3083 | @example | ||
3084 | struct MyValue @{ | ||
3085 | struct GNUNET_HashCode key; | ||
3086 | unsigned int my_data; @}; | ||
3087 | |||
3088 | // ... | ||
3089 | val = GNUNET_malloc (sizeof (struct MyValue)); | ||
3090 | val->key = key; val->my_data = 42; | ||
3091 | GNUNET_CONTAINER_multihashmap_put (map, &val->key, val, ...); | ||
3092 | @end example | ||
3093 | |||
3094 | |||
3095 | Note that @code{&val} was changed to @code{&val->key} in the argument to | ||
3096 | the @code{put} call. This is critical as often @code{key} is on the stack | ||
3097 | or in some other transient data structure and thus having the hash map | ||
3098 | keep a pointer to @code{key} would not work. Only the key inside of | ||
3099 | @code{val} has the same lifetime as the entry in the map (this must of | ||
3100 | course be checked as well). Naturally, @code{val->key} must be | ||
3101 | intiialized before the @code{put} call. Once all @code{put} calls have | ||
3102 | been converted and double-checked, you can change the call to create the | ||
3103 | hash map from | ||
3104 | |||
3105 | @example | ||
3106 | map = | ||
3107 | GNUNET_CONTAINER_multihashmap_create (SIZE, GNUNET_NO); | ||
3108 | @end example | ||
3109 | |||
3110 | to | ||
3111 | |||
3112 | @example | ||
3113 | map = GNUNET_CONTAINER_multihashmap_create (SIZE, GNUNET_YES); | ||
3114 | @end example | ||
3115 | |||
3116 | If everything was done correctly, you now use about 60 bytes less memory | ||
3117 | per entry in @code{map}. However, if now (or in the future) any call to | ||
3118 | @code{put} does not ensure that the given key is valid until the entry is | ||
3119 | removed from the map, undefined behavior is likely to be observed. | ||
3120 | |||
3121 | @c *********************************************************************** | ||
3122 | @node Conclusion | ||
3123 | @subsubsection Conclusion | ||
3124 | @c %**end of header | ||
3125 | |||
3126 | The new optimization can is often applicable and can result in a | ||
3127 | reduction in memory consumption of up to 30% in practice. However, it | ||
3128 | makes the code less robust as additional invariants are imposed on the | ||
3129 | multi hash map client. Thus applications should refrain from enabling the | ||
3130 | new mode unless the resulting performance increase is deemed significant | ||
3131 | enough. In particular, it should generally not be used in new code (wait | ||
3132 | at least until benchmarks exist). | ||
3133 | |||
3134 | @c *********************************************************************** | ||
3135 | @node Availability | ||
3136 | @subsubsection Availability | ||
3137 | @c %**end of header | ||
3138 | |||
3139 | The new multi hash map code was committed in SVN 24319 (will be in GNUnet | ||
3140 | 0.9.4). Various subsystems (transport, core, dht, file-sharing) were | ||
3141 | previously audited and modified to take advantage of the new capability. | ||
3142 | In particular, memory consumption of the file-sharing service is expected | ||
3143 | to drop by 20-30% due to this change. | ||
3144 | |||
3145 | |||
3146 | @cindex CONTAINER_MDLL API | ||
3147 | @node CONTAINER_MDLL API | ||
3148 | @subsection CONTAINER_MDLL API | ||
3149 | @c %**end of header | ||
3150 | |||
3151 | This text documents the GNUNET_CONTAINER_MDLL API. The | ||
3152 | GNUNET_CONTAINER_MDLL API is similar to the GNUNET_CONTAINER_DLL API in | ||
3153 | that it provides operations for the construction and manipulation of | ||
3154 | doubly-linked lists. The key difference to the (simpler) DLL-API is that | ||
3155 | the MDLL-version allows a single element (instance of a "struct") to be | ||
3156 | in multiple linked lists at the same time. | ||
3157 | |||
3158 | Like the DLL API, the MDLL API stores (most of) the data structures for | ||
3159 | the doubly-linked list with the respective elements; only the 'head' and | ||
3160 | 'tail' pointers are stored "elsewhere" --- and the application needs to | ||
3161 | provide the locations of head and tail to each of the calls in the | ||
3162 | MDLL API. The key difference for the MDLL API is that the "next" and | ||
3163 | "previous" pointers in the struct can no longer be simply called "next" | ||
3164 | and "prev" --- after all, the element may be in multiple doubly-linked | ||
3165 | lists, so we cannot just have one "next" and one "prev" pointer! | ||
3166 | |||
3167 | The solution is to have multiple fields that must have a name of the | ||
3168 | format "next_XX" and "prev_XX" where "XX" is the name of one of the | ||
3169 | doubly-linked lists. Here is a simple example: | ||
3170 | |||
3171 | @example | ||
3172 | struct MyMultiListElement @{ | ||
3173 | struct MyMultiListElement *next_ALIST; | ||
3174 | struct MyMultiListElement *prev_ALIST; | ||
3175 | struct MyMultiListElement *next_BLIST; | ||
3176 | struct MyMultiListElement *prev_BLIST; | ||
3177 | void | ||
3178 | *data; | ||
3179 | @}; | ||
3180 | @end example | ||
3181 | |||
3182 | |||
3183 | Note that by convention, we use all-uppercase letters for the list names. | ||
3184 | In addition, the program needs to have a location for the head and tail | ||
3185 | pointers for both lists, for example: | ||
3186 | |||
3187 | @example | ||
3188 | static struct MyMultiListElement *head_ALIST; | ||
3189 | static struct MyMultiListElement *tail_ALIST; | ||
3190 | static struct MyMultiListElement *head_BLIST; | ||
3191 | static struct MyMultiListElement *tail_BLIST; | ||
3192 | @end example | ||
3193 | |||
3194 | |||
3195 | Using the MDLL-macros, we can now insert an element into the ALIST: | ||
3196 | |||
3197 | @example | ||
3198 | GNUNET_CONTAINER_MDLL_insert (ALIST, head_ALIST, tail_ALIST, element); | ||
3199 | @end example | ||
3200 | |||
3201 | |||
3202 | Passing "ALIST" as the first argument to MDLL specifies which of the | ||
3203 | next/prev fields in the 'struct MyMultiListElement' should be used. The | ||
3204 | extra "ALIST" argument and the "_ALIST" in the names of the | ||
3205 | next/prev-members are the only differences between the MDDL and DLL-API. | ||
3206 | Like the DLL-API, the MDLL-API offers functions for inserting (at head, | ||
3207 | at tail, after a given element) and removing elements from the list. | ||
3208 | Iterating over the list should be done by directly accessing the | ||
3209 | "next_XX" and/or "prev_XX" members. | ||
3210 | |||
3211 | @cindex Automatic Restart Manager | ||
3212 | @cindex ARM | ||
3213 | @node Automatic Restart Manager (ARM) | ||
3214 | @section Automatic Restart Manager (ARM) | ||
3215 | @c %**end of header | ||
3216 | |||
3217 | GNUnet's Automated Restart Manager (ARM) is the GNUnet service responsible | ||
3218 | for system initialization and service babysitting. ARM starts and halts | ||
3219 | services, detects configuration changes and restarts services impacted by | ||
3220 | the changes as needed. It's also responsible for restarting services in | ||
3221 | case of crashes and is planned to incorporate automatic debugging for | ||
3222 | diagnosing service crashes providing developers insights about crash | ||
3223 | reasons. The purpose of this document is to give GNUnet developer an idea | ||
3224 | about how ARM works and how to interact with it. | ||
3225 | |||
3226 | @menu | ||
3227 | * Basic functionality:: | ||
3228 | * Key configuration options:: | ||
3229 | * ARM - Availability:: | ||
3230 | * Reliability:: | ||
3231 | @end menu | ||
3232 | |||
3233 | @c *********************************************************************** | ||
3234 | @node Basic functionality | ||
3235 | @subsection Basic functionality | ||
3236 | @c %**end of header | ||
3237 | |||
3238 | @itemize @bullet | ||
3239 | @item ARM source code can be found under "src/arm".@ Service processes are | ||
3240 | managed by the functions in "gnunet-service-arm.c" which is controlled | ||
3241 | with "gnunet-arm.c" (main function in that file is ARM's entry point). | ||
3242 | |||
3243 | @item The functions responsible for communicating with ARM , starting and | ||
3244 | stopping services -including ARM service itself- are provided by the | ||
3245 | ARM API "arm_api.c".@ Function: GNUNET_ARM_connect() returns to the caller | ||
3246 | an ARM handle after setting it to the caller's context (configuration and | ||
3247 | scheduler in use). This handle can be used afterwards by the caller to | ||
3248 | communicate with ARM. Functions GNUNET_ARM_start_service() and | ||
3249 | GNUNET_ARM_stop_service() are used for starting and stopping services | ||
3250 | respectively. | ||
3251 | |||
3252 | @item A typical example of using these basic ARM services can be found in | ||
3253 | file test_arm_api.c. The test case connects to ARM, starts it, then uses | ||
3254 | it to start a service "resolver", stops the "resolver" then stops "ARM". | ||
3255 | @end itemize | ||
3256 | |||
3257 | @c *********************************************************************** | ||
3258 | @node Key configuration options | ||
3259 | @subsection Key configuration options | ||
3260 | @c %**end of header | ||
3261 | |||
3262 | Configurations for ARM and services should be available in a .conf file | ||
3263 | (As an example, see test_arm_api_data.conf). When running ARM, the | ||
3264 | configuration file to use should be passed to the command: | ||
3265 | |||
3266 | @example | ||
3267 | $ gnunet-arm -s -c configuration_to_use.conf | ||
3268 | @end example | ||
3269 | |||
3270 | If no configuration is passed, the default configuration file will be used | ||
3271 | (see GNUNET_PREFIX/share/gnunet/defaults.conf which is created from | ||
3272 | contrib/defaults.conf).@ Each of the services is having a section starting | ||
3273 | by the service name between square brackets, for example: "[arm]". | ||
3274 | The following options configure how ARM configures or interacts with the | ||
3275 | various services: | ||
3276 | |||
3277 | @table @asis | ||
3278 | |||
3279 | @item PORT Port number on which the service is listening for incoming TCP | ||
3280 | connections. ARM will start the services should it notice a request at | ||
3281 | this port. | ||
3282 | |||
3283 | @item HOSTNAME Specifies on which host the service is deployed. Note | ||
3284 | that ARM can only start services that are running on the local system | ||
3285 | (but will not check that the hostname matches the local machine name). | ||
3286 | This option is used by the @code{gnunet_client_lib.h} implementation to | ||
3287 | determine which system to connect to. The default is "localhost". | ||
3288 | |||
3289 | @item BINARY The name of the service binary file. | ||
3290 | |||
3291 | @item OPTIONS To be passed to the service. | ||
3292 | |||
3293 | @item PREFIX A command to pre-pend to the actual command, for example, | ||
3294 | running a service with "valgrind" or "gdb" | ||
3295 | |||
3296 | @item DEBUG Run in debug mode (much verbosity). | ||
3297 | |||
3298 | @item AUTOSTART ARM will listen to UNIX domain socket and/or TCP port of | ||
3299 | the service and start the service on-demand. | ||
3300 | |||
3301 | @item FORCESTART ARM will always start this service when the peer | ||
3302 | is started. | ||
3303 | |||
3304 | @item ACCEPT_FROM IPv4 addresses the service accepts connections from. | ||
3305 | |||
3306 | @item ACCEPT_FROM6 IPv6 addresses the service accepts connections from. | ||
3307 | |||
3308 | @end table | ||
3309 | |||
3310 | |||
3311 | Options that impact the operation of ARM overall are in the "[arm]" | ||
3312 | section. ARM is a normal service and has (except for AUTOSTART) all of the | ||
3313 | options that other services do. In addition, ARM has the | ||
3314 | following options: | ||
3315 | |||
3316 | @table @asis | ||
3317 | |||
3318 | @item GLOBAL_PREFIX Command to be pre-pended to all services that are | ||
3319 | going to run. | ||
3320 | |||
3321 | @item GLOBAL_POSTFIX Global option that will be supplied to all the | ||
3322 | services that are going to run. | ||
3323 | |||
3324 | @end table | ||
3325 | |||
3326 | @c *********************************************************************** | ||
3327 | @node ARM - Availability | ||
3328 | @subsection ARM - Availability | ||
3329 | @c %**end of header | ||
3330 | |||
3331 | As mentioned before, one of the features provided by ARM is starting | ||
3332 | services on demand. Consider the example of one service "client" that | ||
3333 | wants to connect to another service a "server". The "client" will ask ARM | ||
3334 | to run the "server". ARM starts the "server". The "server" starts | ||
3335 | listening to incoming connections. The "client" will establish a | ||
3336 | connection with the "server". And then, they will start to communicate | ||
3337 | together.@ One problem with that scheme is that it's slow!@ | ||
3338 | The "client" service wants to communicate with the "server" service at | ||
3339 | once and is not willing wait for it to be started and listening to | ||
3340 | incoming connections before serving its request.@ One solution for that | ||
3341 | problem will be that ARM starts all services as default services. That | ||
3342 | solution will solve the problem, yet, it's not quite practical, for some | ||
3343 | services that are going to be started can never be used or are going to | ||
3344 | be used after a relatively long time.@ | ||
3345 | The approach followed by ARM to solve this problem is as follows: | ||
3346 | |||
3347 | @itemize @bullet | ||
3348 | |||
3349 | @item For each service having a PORT field in the configuration file and | ||
3350 | that is not one of the default services ( a service that accepts incoming | ||
3351 | connections from clients), ARM creates listening sockets for all addresses | ||
3352 | associated with that service. | ||
3353 | |||
3354 | @item The "client" will immediately establish a connection with | ||
3355 | the "server". | ||
3356 | |||
3357 | @item ARM --- pretending to be the "server" --- will listen on the | ||
3358 | respective port and notice the incoming connection from the "client" | ||
3359 | (but not accept it), instead | ||
3360 | |||
3361 | @item Once there is an incoming connection, ARM will start the "server", | ||
3362 | passing on the listen sockets (now, the service is started and can do its | ||
3363 | work). | ||
3364 | |||
3365 | @item Other client services now can directly connect directly to the | ||
3366 | "server". | ||
3367 | |||
3368 | @end itemize | ||
3369 | |||
3370 | @c *********************************************************************** | ||
3371 | @node Reliability | ||
3372 | @subsection Reliability | ||
3373 | |||
3374 | One of the features provided by ARM, is the automatic restart of crashed | ||
3375 | services.@ ARM needs to know which of the running services died. Function | ||
3376 | "gnunet-service-arm.c/maint_child_death()" is responsible for that. The | ||
3377 | function is scheduled to run upon receiving a SIGCHLD signal. The | ||
3378 | function, then, iterates ARM's list of services running and monitors | ||
3379 | which service has died (crashed). For all crashing services, ARM restarts | ||
3380 | them.@ | ||
3381 | Now, considering the case of a service having a serious problem causing it | ||
3382 | to crash each time it's started by ARM. If ARM keeps blindly restarting | ||
3383 | such a service, we are going to have the pattern: | ||
3384 | start-crash-restart-crash-restart-crash and so forth!! Which is of course | ||
3385 | not practical.@ | ||
3386 | For that reason, ARM schedules the service to be restarted after waiting | ||
3387 | for some delay that grows exponentially with each crash/restart of that | ||
3388 | service.@ To clarify the idea, considering the following example: | ||
3389 | |||
3390 | @itemize @bullet | ||
3391 | |||
3392 | @item Service S crashed. | ||
3393 | |||
3394 | @item ARM receives the SIGCHLD and inspects its list of services to find | ||
3395 | the dead one(s). | ||
3396 | |||
3397 | @item ARM finds S dead and schedules it for restarting after "backoff" | ||
3398 | time which is initially set to 1ms. ARM will double the backoff time | ||
3399 | correspondent to S (now backoff(S) = 2ms) | ||
3400 | |||
3401 | @item Because there is a severe problem with S, it crashed again. | ||
3402 | |||
3403 | @item Again ARM receives the SIGCHLD and detects that it's S again that's | ||
3404 | crashed. ARM schedules it for restarting but after its new backoff time | ||
3405 | (which became 2ms), and doubles its backoff time (now backoff(S) = 4). | ||
3406 | |||
3407 | @item and so on, until backoff(S) reaches a certain threshold | ||
3408 | (@code{EXPONENTIAL_BACKOFF_THRESHOLD} is set to half an hour), | ||
3409 | after reaching it, backoff(S) will remain half an hour, | ||
3410 | hence ARM won't be busy for a lot of time trying to restart a | ||
3411 | problematic service. | ||
3412 | @end itemize | ||
3413 | |||
3414 | @cindex TRANSPORT Subsystem | ||
3415 | @node TRANSPORT Subsystem | ||
3416 | @section TRANSPORT Subsystem | ||
3417 | @c %**end of header | ||
3418 | |||
3419 | This chapter documents how the GNUnet transport subsystem works. The | ||
3420 | GNUnet transport subsystem consists of three main components: the | ||
3421 | transport API (the interface used by the rest of the system to access the | ||
3422 | transport service), the transport service itself (most of the interesting | ||
3423 | functions, such as choosing transports, happens here) and the transport | ||
3424 | plugins. A transport plugin is a concrete implementation for how two | ||
3425 | GNUnet peers communicate; many plugins exist, for example for | ||
3426 | communication via TCP, UDP, HTTP, HTTPS and others. Finally, the | ||
3427 | transport subsystem uses supporting code, especially the NAT/UPnP | ||
3428 | library to help with tasks such as NAT traversal. | ||
3429 | |||
3430 | Key tasks of the transport service include: | ||
3431 | |||
3432 | @itemize @bullet | ||
3433 | |||
3434 | @item Create our HELLO message, notify clients and neighbours if our HELLO | ||
3435 | changes (using NAT library as necessary) | ||
3436 | |||
3437 | @item Validate HELLOs from other peers (send PING), allow other peers to | ||
3438 | validate our HELLO's addresses (send PONG) | ||
3439 | |||
3440 | @item Upon request, establish connections to other peers (using address | ||
3441 | selection from ATS subsystem) and maintain them (again using PINGs and | ||
3442 | PONGs) as long as desired | ||
3443 | |||
3444 | @item Accept incoming connections, give ATS service the opportunity to | ||
3445 | switch communication channels | ||
3446 | |||
3447 | @item Notify clients about peers that have connected to us or that have | ||
3448 | been disconnected from us | ||
3449 | |||
3450 | @item If a (stateful) connection goes down unexpectedly (without explicit | ||
3451 | DISCONNECT), quickly attempt to recover (without notifying clients) but do | ||
3452 | notify clients quickly if reconnecting fails | ||
3453 | |||
3454 | @item Send (payload) messages arriving from clients to other peers via | ||
3455 | transport plugins and receive messages from other peers, forwarding | ||
3456 | those to clients | ||
3457 | |||
3458 | @item Enforce inbound traffic limits (using flow-control if it is | ||
3459 | applicable); outbound traffic limits are enforced by CORE, not by us (!) | ||
3460 | |||
3461 | @item Enforce restrictions on P2P connection as specified by the blacklist | ||
3462 | configuration and blacklisting clients | ||
3463 | @end itemize | ||
3464 | |||
3465 | Note that the term "clients" in the list above really refers to the | ||
3466 | GNUnet-CORE service, as CORE is typically the only client of the | ||
3467 | transport service. | ||
3468 | |||
3469 | @menu | ||
3470 | * Address validation protocol:: | ||
3471 | @end menu | ||
3472 | |||
3473 | @node Address validation protocol | ||
3474 | @subsection Address validation protocol | ||
3475 | @c %**end of header | ||
3476 | |||
3477 | This section documents how the GNUnet transport service validates | ||
3478 | connections with other peers. It is a high-level description of the | ||
3479 | protocol necessary to understand the details of the implementation. It | ||
3480 | should be noted that when we talk about PING and PONG messages in this | ||
3481 | section, we refer to transport-level PING and PONG messages, which are | ||
3482 | different from core-level PING and PONG messages (both in implementation | ||
3483 | and function). | ||
3484 | |||
3485 | The goal of transport-level address validation is to minimize the chances | ||
3486 | of a successful man-in-the-middle attack against GNUnet peers on the | ||
3487 | transport level. Such an attack would not allow the adversary to decrypt | ||
3488 | the P2P transmissions, but a successful attacker could at least measure | ||
3489 | traffic volumes and latencies (raising the adversaries capablities by | ||
3490 | those of a global passive adversary in the worst case). The scenarios we | ||
3491 | are concerned about is an attacker, Mallory, giving a @code{HELLO} to | ||
3492 | Alice that claims to be for Bob, but contains Mallory's IP address | ||
3493 | instead of Bobs (for some transport). | ||
3494 | Mallory would then forward the traffic to Bob (by initiating a | ||
3495 | connection to Bob and claiming to be Alice). As a further | ||
3496 | complication, the scheme has to work even if say Alice is behind a NAT | ||
3497 | without traversal support and hence has no address of her own (and thus | ||
3498 | Alice must always initiate the connection to Bob). | ||
3499 | |||
3500 | An additional constraint is that @code{HELLO} messages do not contain a | ||
3501 | cryptographic signature since other peers must be able to edit | ||
3502 | (i.e. remove) addresses from the @code{HELLO} at any time (this was | ||
3503 | not true in GNUnet 0.8.x). A basic @strong{assumption} is that each peer | ||
3504 | knows the set of possible network addresses that it @strong{might} | ||
3505 | be reachable under (so for example, the external IP address of the | ||
3506 | NAT plus the LAN address(es) with the respective ports). | ||
3507 | |||
3508 | The solution is the following. If Alice wants to validate that a given | ||
3509 | address for Bob is valid (i.e. is actually established @strong{directly} | ||
3510 | with the intended target), she sends a PING message over that connection | ||
3511 | to Bob. Note that in this case, Alice initiated the connection so only | ||
3512 | Alice knows which address was used for sure (Alice may be behind NAT, so | ||
3513 | whatever address Bob sees may not be an address Alice knows she has). | ||
3514 | Bob checks that the address given in the @code{PING} is actually one | ||
3515 | of Bob's addresses (ie: does not belong to Mallory), and if it is, | ||
3516 | sends back a @code{PONG} (with a signature that says that Bob | ||
3517 | owns/uses the address from the @code{PING}). | ||
3518 | Alice checks the signature and is happy if it is valid and the address | ||
3519 | in the @code{PONG} is the address Alice used. | ||
3520 | This is similar to the 0.8.x protocol where the @code{HELLO} contained a | ||
3521 | signature from Bob for each address used by Bob. | ||
3522 | Here, the purpose code for the signature is | ||
3523 | @code{GNUNET_SIGNATURE_PURPOSE_TRANSPORT_PONG_OWN}. After this, Alice will | ||
3524 | remember Bob's address and consider the address valid for a while (12h in | ||
3525 | the current implementation). Note that after this exchange, Alice only | ||
3526 | considers Bob's address to be valid, the connection itself is not | ||
3527 | considered 'established'. In particular, Alice may have many addresses | ||
3528 | for Bob that Alice considers valid. | ||
3529 | |||
3530 | @c TODO: reference Footnotes so that I don't have to duplicate the | ||
3531 | @c footnotes or add them to an index at the end. Is this possible at | ||
3532 | @c all in Texinfo? | ||
3533 | The @code{PONG} message is protected with a nonce/challenge against replay | ||
3534 | attacks@footnote{@uref{http://en.wikipedia.org/wiki/Replay_attack, replay}} | ||
3535 | and uses an expiration time for the signature (but those are almost | ||
3536 | implementation details). | ||
3537 | |||
3538 | @cindex NAT library | ||
3539 | @node NAT library | ||
3540 | @section NAT library | ||
3541 | @c %**end of header | ||
3542 | |||
3543 | The goal of the GNUnet NAT library is to provide a general-purpose API for | ||
3544 | NAT traversal @strong{without} third-party support. So protocols that | ||
3545 | involve contacting a third peer to help establish a connection between | ||
3546 | two peers are outside of the scope of this API. That does not mean that | ||
3547 | GNUnet doesn't support involving a third peer (we can do this with the | ||
3548 | distance-vector transport or using application-level protocols), it just | ||
3549 | means that the NAT API is not concerned with this possibility. The API is | ||
3550 | written so that it will work for IPv6-NAT in the future as well as | ||
3551 | current IPv4-NAT. Furthermore, the NAT API is always used, even for peers | ||
3552 | that are not behind NAT --- in that case, the mapping provided is simply | ||
3553 | the identity. | ||
3554 | |||
3555 | NAT traversal is initiated by calling @code{GNUNET_NAT_register}. Given a | ||
3556 | set of addresses that the peer has locally bound to (TCP or UDP), the NAT | ||
3557 | library will return (via callback) a (possibly longer) list of addresses | ||
3558 | the peer @strong{might} be reachable under. Internally, depending on the | ||
3559 | configuration, the NAT library will try to punch a hole (using UPnP) or | ||
3560 | just "know" that the NAT was manually punched and generate the respective | ||
3561 | external IP address (the one that should be globally visible) based on | ||
3562 | the given information. | ||
3563 | |||
3564 | The NAT library also supports ICMP-based NAT traversal. Here, the other | ||
3565 | peer can request connection-reversal by this peer (in this special case, | ||
3566 | the peer is even allowed to configure a port number of zero). If the NAT | ||
3567 | library detects a connection-reversal request, it returns the respective | ||
3568 | target address to the client as well. It should be noted that | ||
3569 | connection-reversal is currently only intended for TCP, so other plugins | ||
3570 | @strong{must} pass @code{NULL} for the reversal callback. Naturally, the | ||
3571 | NAT library also supports requesting connection reversal from a remote | ||
3572 | peer (@code{GNUNET_NAT_run_client}). | ||
3573 | |||
3574 | Once initialized, the NAT handle can be used to test if a given address is | ||
3575 | possibly a valid address for this peer (@code{GNUNET_NAT_test_address}). | ||
3576 | This is used for validating our addresses when generating PONGs. | ||
3577 | |||
3578 | Finally, the NAT library contains an API to test if our NAT configuration | ||
3579 | is correct. Using @code{GNUNET_NAT_test_start} @strong{before} binding to | ||
3580 | the respective port, the NAT library can be used to test if the | ||
3581 | configuration works. The test function act as a local client, initialize | ||
3582 | the NAT traversal and then contact a @code{gnunet-nat-server} (running by | ||
3583 | default on @code{gnunet.org}) and ask for a connection to be established. | ||
3584 | This way, it is easy to test if the current NAT configuration is valid. | ||
3585 | |||
3586 | @node Distance-Vector plugin | ||
3587 | @section Distance-Vector plugin | ||
3588 | @c %**end of header | ||
3589 | |||
3590 | The Distance Vector (DV) transport is a transport mechanism that allows | ||
3591 | peers to act as relays for each other, thereby connecting peers that would | ||
3592 | otherwise be unable to connect. This gives a larger connection set to | ||
3593 | applications that may work better with more peers to choose from (for | ||
3594 | example, File Sharing and/or DHT). | ||
3595 | |||
3596 | The Distance Vector transport essentially has two functions. The first is | ||
3597 | "gossiping" connection information about more distant peers to directly | ||
3598 | connected peers. The second is taking messages intended for non-directly | ||
3599 | connected peers and encapsulating them in a DV wrapper that contains the | ||
3600 | required information for routing the message through forwarding peers. Via | ||
3601 | gossiping, optimal routes through the known DV neighborhood are discovered | ||
3602 | and utilized and the message encapsulation provides some benefits in | ||
3603 | addition to simply getting the message from the correct source to the | ||
3604 | proper destination. | ||
3605 | |||
3606 | The gossiping function of DV provides an up to date routing table of | ||
3607 | peers that are available up to some number of hops. We call this a | ||
3608 | fisheye view of the network (like a fish, nearby objects are known while | ||
3609 | more distant ones unknown). Gossip messages are sent only to directly | ||
3610 | connected peers, but they are sent about other knowns peers within the | ||
3611 | "fisheye distance". Whenever two peers connect, they immediately gossip | ||
3612 | to each other about their appropriate other neighbors. They also gossip | ||
3613 | about the newly connected peer to previously | ||
3614 | connected neighbors. In order to keep the routing tables up to date, | ||
3615 | disconnect notifications are propogated as gossip as well (because | ||
3616 | disconnects may not be sent/received, timeouts are also used remove | ||
3617 | stagnant routing table entries). | ||
3618 | |||
3619 | Routing of messages via DV is straightforward. When the DV transport is | ||
3620 | notified of a message destined for a non-direct neighbor, the appropriate | ||
3621 | forwarding peer is selected, and the base message is encapsulated in a DV | ||
3622 | message which contains information about the initial peer and the intended | ||
3623 | recipient. At each forwarding hop, the initial peer is validated (the | ||
3624 | forwarding peer ensures that it has the initial peer in its neighborhood, | ||
3625 | otherwise the message is dropped). Next the base message is | ||
3626 | re-encapsulated in a new DV message for the next hop in the forwarding | ||
3627 | chain (or delivered to the current peer, if it has arrived at the | ||
3628 | destination). | ||
3629 | |||
3630 | Assume a three peer network with peers Alice, Bob and Carol. Assume that | ||
3631 | |||
3632 | @example | ||
3633 | Alice <-> Bob and Bob <-> Carol | ||
3634 | @end example | ||
3635 | |||
3636 | @noindent | ||
3637 | are direct (e.g. over TCP or UDP transports) connections, but that | ||
3638 | Alice cannot directly connect to Carol. | ||
3639 | This may be the case due to NAT or firewall restrictions, or perhaps | ||
3640 | based on one of the peers respective configurations. If the Distance | ||
3641 | Vector transport is enabled on all three peers, it will automatically | ||
3642 | discover (from the gossip protocol) that Alice and Carol can connect via | ||
3643 | Bob and provide a "virtual" Alice <-> Carol connection. Routing between | ||
3644 | Alice and Carol happens as follows; Alice creates a message destined for | ||
3645 | Carol and notifies the DV transport about it. The DV transport at Alice | ||
3646 | looks up Carol in the routing table and finds that the message must be | ||
3647 | sent through Bob for Carol. The message is encapsulated setting Alice as | ||
3648 | the initiator and Carol as the destination and sent to Bob. Bob receives | ||
3649 | the messages, verifies that both Alice and Carol are known to Bob, and | ||
3650 | re-wraps the message in a new DV message for Carol. | ||
3651 | The DV transport at Carol receives this message, unwraps the original | ||
3652 | message, and delivers it to Carol as though it came directly from Alice. | ||
3653 | |||
3654 | @cindex SMTP plugin | ||
3655 | @node SMTP plugin | ||
3656 | @section SMTP plugin | ||
3657 | @c %**end of header | ||
3658 | |||
3659 | This section describes the new SMTP transport plugin for GNUnet as it | ||
3660 | exists in the 0.7.x and 0.8.x branch. SMTP support is currently not | ||
3661 | available in GNUnet 0.9.x. This page also describes the transport layer | ||
3662 | abstraction (as it existed in 0.7.x and 0.8.x) in more detail and gives | ||
3663 | some benchmarking results. The performance results presented are quite | ||
3664 | old and maybe outdated at this point. | ||
3665 | |||
3666 | @itemize @bullet | ||
3667 | @item Why use SMTP for a peer-to-peer transport? | ||
3668 | @item SMTPHow does it work? | ||
3669 | @item How do I configure my peer? | ||
3670 | @item How do I test if it works? | ||
3671 | @item How fast is it? | ||
3672 | @item Is there any additional documentation? | ||
3673 | @end itemize | ||
3674 | |||
3675 | |||
3676 | @menu | ||
3677 | * Why use SMTP for a peer-to-peer transport?:: | ||
3678 | * How does it work?:: | ||
3679 | * How do I configure my peer?:: | ||
3680 | * How do I test if it works?:: | ||
3681 | * How fast is it?:: | ||
3682 | @end menu | ||
3683 | |||
3684 | @node Why use SMTP for a peer-to-peer transport? | ||
3685 | @subsection Why use SMTP for a peer-to-peer transport? | ||
3686 | @c %**end of header | ||
3687 | |||
3688 | There are many reasons why one would not want to use SMTP: | ||
3689 | |||
3690 | @itemize @bullet | ||
3691 | @item SMTP is using more bandwidth than TCP, UDP or HTTP | ||
3692 | @item SMTP has a much higher latency. | ||
3693 | @item SMTP requires significantly more computation (encoding and decoding | ||
3694 | time) for the peers. | ||
3695 | @item SMTP is significantly more complicated to configure. | ||
3696 | @item SMTP may be abused by tricking GNUnet into sending mail to@ | ||
3697 | non-participating third parties. | ||
3698 | @end itemize | ||
3699 | |||
3700 | So why would anybody want to use SMTP? | ||
3701 | @itemize @bullet | ||
3702 | @item SMTP can be used to contact peers behind NAT boxes (in virtual | ||
3703 | private networks). | ||
3704 | @item SMTP can be used to circumvent policies that limit or prohibit | ||
3705 | peer-to-peer traffic by masking as "legitimate" traffic. | ||
3706 | @item SMTP uses E-mail addresses which are independent of a specific IP, | ||
3707 | which can be useful to address peers that use dynamic IP addresses. | ||
3708 | @item SMTP can be used to initiate a connection (e.g. initial address | ||
3709 | exchange) and peers can then negotiate the use of a more efficient | ||
3710 | protocol (e.g. TCP) for the actual communication. | ||
3711 | @end itemize | ||
3712 | |||
3713 | In summary, SMTP can for example be used to send a message to a peer | ||
3714 | behind a NAT box that has a dynamic IP to tell the peer to establish a | ||
3715 | TCP connection to a peer outside of the private network. Even an | ||
3716 | extraordinary overhead for this first message would be irrelevant in this | ||
3717 | type of situation. | ||
3718 | |||
3719 | @node How does it work? | ||
3720 | @subsection How does it work? | ||
3721 | @c %**end of header | ||
3722 | |||
3723 | When a GNUnet peer needs to send a message to another GNUnet peer that has | ||
3724 | advertised (only) an SMTP transport address, GNUnet base64-encodes the | ||
3725 | message and sends it in an E-mail to the advertised address. The | ||
3726 | advertisement contains a filter which is placed in the E-mail header, | ||
3727 | such that the receiving host can filter the tagged E-mails and forward it | ||
3728 | to the GNUnet peer process. The filter can be specified individually by | ||
3729 | each peer and be changed over time. This makes it impossible to censor | ||
3730 | GNUnet E-mail messages by searching for a generic filter. | ||
3731 | |||
3732 | @node How do I configure my peer? | ||
3733 | @subsection How do I configure my peer? | ||
3734 | @c %**end of header | ||
3735 | |||
3736 | First, you need to configure @code{procmail} to filter your inbound E-mail | ||
3737 | for GNUnet traffic. The GNUnet messages must be delivered into a pipe, for | ||
3738 | example @code{/tmp/gnunet.smtp}. You also need to define a filter that is | ||
3739 | used by @command{procmail} to detect GNUnet messages. You are free to | ||
3740 | choose whichever filter you like, but you should make sure that it does | ||
3741 | not occur in your other E-mail. In our example, we will use | ||
3742 | @code{X-mailer: GNUnet}. The @code{~/.procmailrc} configuration file then | ||
3743 | looks like this: | ||
3744 | |||
3745 | @example | ||
3746 | :0: | ||
3747 | * ^X-mailer: GNUnet | ||
3748 | /tmp/gnunet.smtp | ||
3749 | # where do you want your other e-mail delivered to | ||
3750 | # (default: /var/spool/mail/) | ||
3751 | :0: /var/spool/mail/ | ||
3752 | @end example | ||
3753 | |||
3754 | After adding this file, first make sure that your regular E-mail still | ||
3755 | works (e.g. by sending an E-mail to yourself). Then edit the GNUnet | ||
3756 | configuration. In the section @code{SMTP} you need to specify your E-mail | ||
3757 | address under @code{EMAIL}, your mail server (for outgoing mail) under | ||
3758 | @code{SERVER}, the filter (X-mailer: GNUnet in the example) under | ||
3759 | @code{FILTER} and the name of the pipe under @code{PIPE}.@ The completed | ||
3760 | section could then look like this: | ||
3761 | |||
3762 | @example | ||
3763 | EMAIL = me@@mail.gnu.org MTU = 65000 SERVER = mail.gnu.org:25 FILTER = | ||
3764 | "X-mailer: GNUnet" PIPE = /tmp/gnunet.smtp | ||
3765 | @end example | ||
3766 | |||
3767 | Finally, you need to add @code{smtp} to the list of @code{TRANSPORTS} in | ||
3768 | the @code{GNUNETD} section. GNUnet peers will use the E-mail address that | ||
3769 | you specified to contact your peer until the advertisement times out. | ||
3770 | Thus, if you are not sure if everything works properly or if you are not | ||
3771 | planning to be online for a long time, you may want to configure this | ||
3772 | timeout to be short, e.g. just one hour. For this, set | ||
3773 | @code{HELLOEXPIRES} to @code{1} in the @code{GNUNETD} section. | ||
3774 | |||
3775 | This should be it, but you may probably want to test it first. | ||
3776 | |||
3777 | @node How do I test if it works? | ||
3778 | @subsection How do I test if it works? | ||
3779 | @c %**end of header | ||
3780 | |||
3781 | Any transport can be subjected to some rudimentary tests using the | ||
3782 | @code{gnunet-transport-check} tool. The tool sends a message to the local | ||
3783 | node via the transport and checks that a valid message is received. While | ||
3784 | this test does not involve other peers and can not check if firewalls or | ||
3785 | other network obstacles prohibit proper operation, this is a great | ||
3786 | testcase for the SMTP transport since it tests pretty much nearly all of | ||
3787 | the functionality. | ||
3788 | |||
3789 | @code{gnunet-transport-check} should only be used without running | ||
3790 | @code{gnunetd} at the same time. By default, @code{gnunet-transport-check} | ||
3791 | tests all transports that are specified in the configuration file. But | ||
3792 | you can specifically test SMTP by giving the option | ||
3793 | @code{--transport=smtp}. | ||
3794 | |||
3795 | Note that this test always checks if a transport can receive and send. | ||
3796 | While you can configure most transports to only receive or only send | ||
3797 | messages, this test will only work if you have configured the transport | ||
3798 | to send and receive messages. | ||
3799 | |||
3800 | @node How fast is it? | ||
3801 | @subsection How fast is it? | ||
3802 | @c %**end of header | ||
3803 | |||
3804 | We have measured the performance of the UDP, TCP and SMTP transport layer | ||
3805 | directly and when used from an application using the GNUnet core. | ||
3806 | Measureing just the transport layer gives the better view of the actual | ||
3807 | overhead of the protocol, whereas evaluating the transport from the | ||
3808 | application puts the overhead into perspective from a practical point of | ||
3809 | view. | ||
3810 | |||
3811 | The loopback measurements of the SMTP transport were performed on three | ||
3812 | different machines spanning a range of modern SMTP configurations. We | ||
3813 | used a PIII-800 running RedHat 7.3 with the Purdue Computer Science | ||
3814 | configuration which includes filters for spam. We also used a Xenon 2 GHZ | ||
3815 | with a vanilla RedHat 8.0 sendmail configuration. Furthermore, we used | ||
3816 | qmail on a PIII-1000 running Sorcerer GNU Linux (SGL). The numbers for | ||
3817 | UDP and TCP are provided using the SGL configuration. The qmail benchmark | ||
3818 | uses qmail's internal filtering whereas the sendmail benchmarks relies on | ||
3819 | procmail to filter and deliver the mail. We used the transport layer to | ||
3820 | send a message of b bytes (excluding transport protocol headers) directly | ||
3821 | to the local machine. This way, network latency and packet loss on the | ||
3822 | wire have no impact on the timings. n messages were sent sequentially over | ||
3823 | the transport layer, sending message i+1 after the i-th message was | ||
3824 | received. All messages were sent over the same connection and the time to | ||
3825 | establish the connection was not taken into account since this overhead is | ||
3826 | miniscule in practice --- as long as a connection is used for a | ||
3827 | significant number of messages. | ||
3828 | |||
3829 | @multitable @columnfractions .20 .15 .15 .15 .15 .15 | ||
3830 | @headitem Transport @tab UDP @tab TCP @tab SMTP (Purdue sendmail) | ||
3831 | @tab SMTP (RH 8.0) @tab SMTP (SGL qmail) | ||
3832 | @item 11 bytes @tab 31 ms @tab 55 ms @tab 781 s @tab 77 s @tab 24 s | ||
3833 | @item 407 bytes @tab 37 ms @tab 62 ms @tab 789 s @tab 78 s @tab 25 s | ||
3834 | @item 1,221 bytes @tab 46 ms @tab 73 ms @tab 804 s @tab 78 s @tab 25 s | ||
3835 | @end multitable | ||
3836 | |||
3837 | The benchmarks show that UDP and TCP are, as expected, both significantly | ||
3838 | faster compared with any of the SMTP services. Among the SMTP | ||
3839 | implementations, there can be significant differences depending on the | ||
3840 | SMTP configuration. Filtering with an external tool like procmail that | ||
3841 | needs to re-parse its configuration for each mail can be very expensive. | ||
3842 | Applying spam filters can also significantly impact the performance of | ||
3843 | the underlying SMTP implementation. The microbenchmark shows that SMTP | ||
3844 | can be a viable solution for initiating peer-to-peer sessions: a couple of | ||
3845 | seconds to connect to a peer are probably not even going to be noticed by | ||
3846 | users. The next benchmark measures the possible throughput for a | ||
3847 | transport. Throughput can be measured by sending multiple messages in | ||
3848 | parallel and measuring packet loss. Note that not only UDP but also the | ||
3849 | TCP transport can actually loose messages since the TCP implementation | ||
3850 | drops messages if the @code{write} to the socket would block. While the | ||
3851 | SMTP protocol never drops messages itself, it is often so | ||
3852 | slow that only a fraction of the messages can be sent and received in the | ||
3853 | given time-bounds. For this benchmark we report the message loss after | ||
3854 | allowing t time for sending m messages. If messages were not sent (or | ||
3855 | received) after an overall timeout of t, they were considered lost. The | ||
3856 | benchmark was performed using two Xeon 2 GHZ machines running RedHat 8.0 | ||
3857 | with sendmail. The machines were connected with a direct 100 MBit ethernet | ||
3858 | connection.@ Figures udp1200, tcp1200 and smtp-MTUs show that the | ||
3859 | throughput for messages of size 1,200 octects is 2,343 kbps, 3,310 kbps | ||
3860 | and 6 kbps for UDP, TCP and SMTP respectively. The high per-message | ||
3861 | overhead of SMTP can be improved by increasing the MTU, for example, an | ||
3862 | MTU of 12,000 octets improves the throughput to 13 kbps as figure | ||
3863 | smtp-MTUs shows. Our research paper) has some more details on the | ||
3864 | benchmarking results. | ||
3865 | |||
3866 | @cindex Bluetooth plugin | ||
3867 | @node Bluetooth plugin | ||
3868 | @section Bluetooth plugin | ||
3869 | @c %**end of header | ||
3870 | |||
3871 | This page describes the new Bluetooth transport plugin for GNUnet. The | ||
3872 | plugin is still in the testing stage so don't expect it to work | ||
3873 | perfectly. If you have any questions or problems just post them here or | ||
3874 | ask on the IRC channel. | ||
3875 | |||
3876 | @itemize @bullet | ||
3877 | @item What do I need to use the Bluetooth plugin transport? | ||
3878 | @item BluetoothHow does it work? | ||
3879 | @item What possible errors should I be aware of? | ||
3880 | @item How do I configure my peer? | ||
3881 | @item How can I test it? | ||
3882 | @end itemize | ||
3883 | |||
3884 | @menu | ||
3885 | * What do I need to use the Bluetooth plugin transport?:: | ||
3886 | * How does it work2?:: | ||
3887 | * What possible errors should I be aware of?:: | ||
3888 | * How do I configure my peer2?:: | ||
3889 | * How can I test it?:: | ||
3890 | * The implementation of the Bluetooth transport plugin:: | ||
3891 | @end menu | ||
3892 | |||
3893 | @node What do I need to use the Bluetooth plugin transport? | ||
3894 | @subsection What do I need to use the Bluetooth plugin transport? | ||
3895 | @c %**end of header | ||
3896 | |||
3897 | If you are a GNU/Linux user and you want to use the Bluetooth | ||
3898 | transport plugin you should install the | ||
3899 | @command{BlueZ development libraries} (if they aren't already | ||
3900 | installed). | ||
3901 | For instructions about how to install the libraries you should | ||
3902 | check out the BlueZ site | ||
3903 | (@uref{http://www.bluez.org/, http://www.bluez.org}). If you don't know if | ||
3904 | you have the necesarry libraries, don't worry, just run the GNUnet | ||
3905 | configure script and you will be able to see a notification at the end | ||
3906 | which will warn you if you don't have the necessary libraries. | ||
3907 | |||
3908 | If you are a Windows user you should have installed the | ||
3909 | @emph{MinGW}/@emph{MSys2} with the latest updates (especially the | ||
3910 | @emph{ws2bth} header). If this is your first build of GNUnet on Windows | ||
3911 | you should check out the SBuild repository. It will semi-automatically | ||
3912 | assembles a @emph{MinGW}/@emph{MSys2} installation with a lot of extra | ||
3913 | packages which are needed for the GNUnet build. So this will ease your | ||
3914 | work!@ Finally you just have to be sure that you have the correct drivers | ||
3915 | for your Bluetooth device installed and that your device is on and in a | ||
3916 | discoverable mode. The Windows Bluetooth Stack supports only the RFCOMM | ||
3917 | protocol so we cannot turn on your device programatically! | ||
3918 | |||
3919 | @c FIXME: Change to unique title | ||
3920 | @node How does it work2? | ||
3921 | @subsection How does it work2? | ||
3922 | @c %**end of header | ||
3923 | |||
3924 | The Bluetooth transport plugin uses virtually the same code as the WLAN | ||
3925 | plugin and only the helper binary is different. The helper takes a single | ||
3926 | argument, which represents the interface name and is specified in the | ||
3927 | configuration file. Here are the basic steps that are followed by the | ||
3928 | helper binary used on GNU/Linux: | ||
3929 | |||
3930 | @itemize @bullet | ||
3931 | @item it verifies if the name corresponds to a Bluetooth interface name | ||
3932 | @item it verifies if the iterface is up (if it is not, it tries to bring | ||
3933 | it up) | ||
3934 | @item it tries to enable the page and inquiry scan in order to make the | ||
3935 | device discoverable and to accept incoming connection requests | ||
3936 | @emph{The above operations require root access so you should start the | ||
3937 | transport plugin with root privileges.} | ||
3938 | @item it finds an available port number and registers a SDP service which | ||
3939 | will be used to find out on which port number is the server listening on | ||
3940 | and switch the socket in listening mode | ||
3941 | @item it sends a HELLO message with its address | ||
3942 | @item finally it forwards traffic from the reading sockets to the STDOUT | ||
3943 | and from the STDIN to the writing socket | ||
3944 | @end itemize | ||
3945 | |||
3946 | Once in a while the device will make an inquiry scan to discover the | ||
3947 | nearby devices and it will send them randomly HELLO messages for peer | ||
3948 | discovery. | ||
3949 | |||
3950 | @node What possible errors should I be aware of? | ||
3951 | @subsection What possible errors should I be aware of? | ||
3952 | @c %**end of header | ||
3953 | |||
3954 | @emph{This section is dedicated for GNU/Linux users} | ||
3955 | |||
3956 | Well there are many ways in which things could go wrong but I will try to | ||
3957 | present some tools that you could use to debug and some scenarios. | ||
3958 | |||
3959 | @itemize @bullet | ||
3960 | |||
3961 | @item @code{bluetoothd -n -d} : use this command to enable logging in the | ||
3962 | foreground and to print the logging messages | ||
3963 | |||
3964 | @item @code{hciconfig}: can be used to configure the Bluetooth devices. | ||
3965 | If you run it without any arguments it will print information about the | ||
3966 | state of the interfaces. So if you receive an error that the device | ||
3967 | couldn't be brought up you should try to bring it manually and to see if | ||
3968 | it works (use @code{hciconfig -a hciX up}). If you can't and the | ||
3969 | Bluetooth address has the form 00:00:00:00:00:00 it means that there is | ||
3970 | something wrong with the D-Bus daemon or with the Bluetooth daemon. Use | ||
3971 | @code{bluetoothd} tool to see the logs | ||
3972 | |||
3973 | @item @code{sdptool} can be used to control and interogate SDP servers. | ||
3974 | If you encounter problems regarding the SDP server (like the SDP server is | ||
3975 | down) you should check out if the D-Bus daemon is running correctly and to | ||
3976 | see if the Bluetooth daemon started correctly(use @code{bluetoothd} tool). | ||
3977 | Also, sometimes the SDP service could work but somehow the device couldn't | ||
3978 | register his service. Use @code{sdptool browse [dev-address]} to see if | ||
3979 | the service is registered. There should be a service with the name of the | ||
3980 | interface and GNUnet as provider. | ||
3981 | |||
3982 | @item @code{hcitool} : another useful tool which can be used to configure | ||
3983 | the device and to send some particular commands to it. | ||
3984 | |||
3985 | @item @code{hcidump} : could be used for low level debugging | ||
3986 | @end itemize | ||
3987 | |||
3988 | @c FIXME: A more unique name | ||
3989 | @node How do I configure my peer2? | ||
3990 | @subsection How do I configure my peer2? | ||
3991 | @c %**end of header | ||
3992 | |||
3993 | On GNU/Linux, you just have to be sure that the interface name | ||
3994 | corresponds to the one that you want to use. | ||
3995 | Use the @code{hciconfig} tool to check that. | ||
3996 | By default it is set to hci0 but you can change it. | ||
3997 | |||
3998 | A basic configuration looks like this: | ||
3999 | |||
4000 | @example | ||
4001 | [transport-bluetooth] | ||
4002 | # Name of the interface (typically hciX) | ||
4003 | INTERFACE = hci0 | ||
4004 | # Real hardware, no testing | ||
4005 | TESTMODE = 0 TESTING_IGNORE_KEYS = ACCEPT_FROM; | ||
4006 | @end example | ||
4007 | |||
4008 | In order to use the Bluetooth transport plugin when the transport service | ||
4009 | is started, you must add the plugin name to the default transport service | ||
4010 | plugins list. For example: | ||
4011 | |||
4012 | @example | ||
4013 | [transport] ... PLUGINS = dns bluetooth ... | ||
4014 | @end example | ||
4015 | |||
4016 | If you want to use only the Bluetooth plugin set | ||
4017 | @emph{PLUGINS = bluetooth} | ||
4018 | |||
4019 | On Windows, you cannot specify which device to use. The only thing that | ||
4020 | you should do is to add @emph{bluetooth} on the plugins list of the | ||
4021 | transport service. | ||
4022 | |||
4023 | @node How can I test it? | ||
4024 | @subsection How can I test it? | ||
4025 | @c %**end of header | ||
4026 | |||
4027 | If you have two Bluetooth devices on the same machine and you are using | ||
4028 | GNU/Linux you must: | ||
4029 | |||
4030 | @itemize @bullet | ||
4031 | |||
4032 | @item create two different file configuration (one which will use the | ||
4033 | first interface (@emph{hci0}) and the other which will use the second | ||
4034 | interface (@emph{hci1})). Let's name them @emph{peer1.conf} and | ||
4035 | @emph{peer2.conf}. | ||
4036 | |||
4037 | @item run @emph{gnunet-peerinfo -c peerX.conf -s} in order to generate the | ||
4038 | peers private keys. The @strong{X} must be replace with 1 or 2. | ||
4039 | |||
4040 | @item run @emph{gnunet-arm -c peerX.conf -s -i=transport} in order to | ||
4041 | start the transport service. (Make sure that you have "bluetooth" on the | ||
4042 | transport plugins list if the Bluetooth transport service doesn't start.) | ||
4043 | |||
4044 | @item run @emph{gnunet-peerinfo -c peer1.conf -s} to get the first peer's | ||
4045 | ID. If you already know your peer ID (you saved it from the first | ||
4046 | command), this can be skipped. | ||
4047 | |||
4048 | @item run @emph{gnunet-transport -c peer2.conf -p=PEER1_ID -s} to start | ||
4049 | sending data for benchmarking to the other peer. | ||
4050 | |||
4051 | @end itemize | ||
4052 | |||
4053 | |||
4054 | This scenario will try to connect the second peer to the first one and | ||
4055 | then start sending data for benchmarking. | ||
4056 | |||
4057 | On Windows you cannot test the plugin functionality using two Bluetooth | ||
4058 | devices from the same machine because after you install the drivers there | ||
4059 | will occur some conflicts between the Bluetooth stacks. (At least that is | ||
4060 | what happend on my machine : I wasn't able to use the Bluesoleil stack and | ||
4061 | the WINDCOMM one in the same time). | ||
4062 | |||
4063 | If you have two different machines and your configuration files are good | ||
4064 | you can use the same scenario presented on the begining of this section. | ||
4065 | |||
4066 | Another way to test the plugin functionality is to create your own | ||
4067 | application which will use the GNUnet framework with the Bluetooth | ||
4068 | transport service. | ||
4069 | |||
4070 | @node The implementation of the Bluetooth transport plugin | ||
4071 | @subsection The implementation of the Bluetooth transport plugin | ||
4072 | @c %**end of header | ||
4073 | |||
4074 | This page describes the implementation of the Bluetooth transport plugin. | ||
4075 | |||
4076 | First I want to remind you that the Bluetooth transport plugin uses | ||
4077 | virtually the same code as the WLAN plugin and only the helper binary is | ||
4078 | different. Also the scope of the helper binary from the Bluetooth | ||
4079 | transport plugin is the same as the one used for the wlan transport | ||
4080 | plugin: it acceses the interface and then it forwards traffic in both | ||
4081 | directions between the Bluetooth interface and stdin/stdout of the | ||
4082 | process involved. | ||
4083 | |||
4084 | The Bluetooth plugin transport could be used both on GNU/Linux and Windows | ||
4085 | platforms. | ||
4086 | |||
4087 | @itemize @bullet | ||
4088 | @item Linux functionality | ||
4089 | @item Windows functionality | ||
4090 | @item Pending Features | ||
4091 | @end itemize | ||
4092 | |||
4093 | |||
4094 | |||
4095 | @menu | ||
4096 | * Linux functionality:: | ||
4097 | * THE INITIALIZATION:: | ||
4098 | * THE LOOP:: | ||
4099 | * Details about the broadcast implementation:: | ||
4100 | * Windows functionality:: | ||
4101 | * Pending features:: | ||
4102 | @end menu | ||
4103 | |||
4104 | @node Linux functionality | ||
4105 | @subsubsection Linux functionality | ||
4106 | @c %**end of header | ||
4107 | |||
4108 | In order to implement the plugin functionality on GNU/Linux I | ||
4109 | used the BlueZ stack. | ||
4110 | For the communication with the other devices I used the RFCOMM | ||
4111 | protocol. Also I used the HCI protocol to gain some control over the | ||
4112 | device. The helper binary takes a single argument (the name of the | ||
4113 | Bluetooth interface) and is separated in two stages: | ||
4114 | |||
4115 | @c %** 'THE INITIALIZATION' should be in bigger letters or stand out, not | ||
4116 | @c %** starting a new section? | ||
4117 | @node THE INITIALIZATION | ||
4118 | @subsubsection THE INITIALIZATION | ||
4119 | |||
4120 | @itemize @bullet | ||
4121 | @item first, it checks if we have root privilegies | ||
4122 | (@emph{Remember that we need to have root privilegies in order to be able | ||
4123 | to bring the interface up if it is down or to change its state.}). | ||
4124 | |||
4125 | @item second, it verifies if the interface with the given name exists. | ||
4126 | |||
4127 | @strong{If the interface with that name exists and it is a Bluetooth | ||
4128 | interface:} | ||
4129 | |||
4130 | @item it creates a RFCOMM socket which will be used for listening and call | ||
4131 | the @emph{open_device} method | ||
4132 | |||
4133 | On the @emph{open_device} method: | ||
4134 | @itemize @bullet | ||
4135 | @item creates a HCI socket used to send control events to the the device | ||
4136 | @item searches for the device ID using the interface name | ||
4137 | @item saves the device MAC address | ||
4138 | @item checks if the interface is down and tries to bring it UP | ||
4139 | @item checks if the interface is in discoverable mode and tries to make it | ||
4140 | discoverable | ||
4141 | @item closes the HCI socket and binds the RFCOMM one | ||
4142 | @item switches the RFCOMM socket in listening mode | ||
4143 | @item registers the SDP service (the service will be used by the other | ||
4144 | devices to get the port on which this device is listening on) | ||
4145 | @end itemize | ||
4146 | |||
4147 | @item drops the root privilegies | ||
4148 | |||
4149 | @strong{If the interface is not a Bluetooth interface the helper exits | ||
4150 | with a suitable error} | ||
4151 | @end itemize | ||
4152 | |||
4153 | @c %** Same as for @node entry above | ||
4154 | @node THE LOOP | ||
4155 | @subsubsection THE LOOP | ||
4156 | |||
4157 | The helper binary uses a list where it saves all the connected neighbour | ||
4158 | devices (@emph{neighbours.devices}) and two buffers (@emph{write_pout} and | ||
4159 | @emph{write_std}). The first message which is send is a control message | ||
4160 | with the device's MAC address in order to announce the peer presence to | ||
4161 | the neighbours. Here are a short description of what happens in the main | ||
4162 | loop: | ||
4163 | |||
4164 | @itemize @bullet | ||
4165 | @item Every time when it receives something from the STDIN it processes | ||
4166 | the data and saves the message in the first buffer (@emph{write_pout}). | ||
4167 | When it has something in the buffer, it gets the destination address from | ||
4168 | the buffer, searches the destination address in the list (if there is no | ||
4169 | connection with that device, it creates a new one and saves it to the | ||
4170 | list) and sends the message. | ||
4171 | @item Every time when it receives something on the listening socket it | ||
4172 | accepts the connection and saves the socket on a list with the reading | ||
4173 | sockets. @item Every time when it receives something from a reading | ||
4174 | socket it parses the message, verifies the CRC and saves it in the | ||
4175 | @emph{write_std} buffer in order to be sent later to the STDOUT. | ||
4176 | @end itemize | ||
4177 | |||
4178 | So in the main loop we use the select function to wait until one of the | ||
4179 | file descriptor saved in one of the two file descriptors sets used is | ||
4180 | ready to use. The first set (@emph{rfds}) represents the reading set and | ||
4181 | it could contain the list with the reading sockets, the STDIN file | ||
4182 | descriptor or the listening socket. The second set (@emph{wfds}) is the | ||
4183 | writing set and it could contain the sending socket or the STDOUT file | ||
4184 | descriptor. After the select function returns, we check which file | ||
4185 | descriptor is ready to use and we do what is supposed to do on that kind | ||
4186 | of event. @emph{For example:} if it is the listening socket then we | ||
4187 | accept a new connection and save the socket in the reading list; if it is | ||
4188 | the STDOUT file descriptor, then we write to STDOUT the message from the | ||
4189 | @emph{write_std} buffer. | ||
4190 | |||
4191 | To find out on which port a device is listening on we connect to the local | ||
4192 | SDP server and searche the registered service for that device. | ||
4193 | |||
4194 | @emph{You should be aware of the fact that if the device fails to connect | ||
4195 | to another one when trying to send a message it will attempt one more | ||
4196 | time. If it fails again, then it skips the message.} | ||
4197 | @emph{Also you should know that the transport Bluetooth plugin has | ||
4198 | support for @strong{broadcast messages}.} | ||
4199 | |||
4200 | @node Details about the broadcast implementation | ||
4201 | @subsubsection Details about the broadcast implementation | ||
4202 | @c %**end of header | ||
4203 | |||
4204 | First I want to point out that the broadcast functionality for the CONTROL | ||
4205 | messages is not implemented in a conventional way. Since the inquiry scan | ||
4206 | time is too big and it will take some time to send a message to all the | ||
4207 | discoverable devices I decided to tackle the problem in a different way. | ||
4208 | Here is how I did it: | ||
4209 | |||
4210 | @itemize @bullet | ||
4211 | @item If it is the first time when I have to broadcast a message I make an | ||
4212 | inquiry scan and save all the devices' addresses to a vector. | ||
4213 | @item After the inquiry scan ends I take the first address from the list | ||
4214 | and I try to connect to it. If it fails, I try to connect to the next one. | ||
4215 | If it succeeds, I save the socket to a list and send the message to the | ||
4216 | device. | ||
4217 | @item When I have to broadcast another message, first I search on the list | ||
4218 | for a new device which I'm not connected to. If there is no new device on | ||
4219 | the list I go to the beginning of the list and send the message to the | ||
4220 | old devices. After 5 cycles I make a new inquiry scan to check out if | ||
4221 | there are new discoverable devices and save them to the list. If there | ||
4222 | are no new discoverable devices I reset the cycling counter and go again | ||
4223 | through the old list and send messages to the devices saved in it. | ||
4224 | @end itemize | ||
4225 | |||
4226 | @strong{Therefore}: | ||
4227 | |||
4228 | @itemize @bullet | ||
4229 | @item every time when I have a broadcast message I look up on the list | ||
4230 | for a new device and send the message to it | ||
4231 | @item if I reached the end of the list for 5 times and I'm connected to | ||
4232 | all the devices from the list I make a new inquiry scan. | ||
4233 | @emph{The number of the list's cycles after an inquiry scan could be | ||
4234 | increased by redefining the MAX_LOOPS variable} | ||
4235 | @item when there are no new devices I send messages to the old ones. | ||
4236 | @end itemize | ||
4237 | |||
4238 | Doing so, the broadcast control messages will reach the devices but with | ||
4239 | delay. | ||
4240 | |||
4241 | @emph{NOTICE:} When I have to send a message to a certain device first I | ||
4242 | check on the broadcast list to see if we are connected to that device. If | ||
4243 | not we try to connect to it and in case of success we save the address and | ||
4244 | the socket on the list. If we are already connected to that device we | ||
4245 | simply use the socket. | ||
4246 | |||
4247 | @node Windows functionality | ||
4248 | @subsubsection Windows functionality | ||
4249 | @c %**end of header | ||
4250 | |||
4251 | For Windows I decided to use the Microsoft Bluetooth stack which has the | ||
4252 | advantage of coming standard from Windows XP SP2. The main disadvantage is | ||
4253 | that it only supports the RFCOMM protocol so we will not be able to have | ||
4254 | a low level control over the Bluetooth device. Therefore it is the user | ||
4255 | responsability to check if the device is up and in the discoverable mode. | ||
4256 | Also there are no tools which could be used for debugging in order to read | ||
4257 | the data coming from and going to a Bluetooth device, which obviously | ||
4258 | hindered my work. Another thing that slowed down the implementation of the | ||
4259 | plugin (besides that I wasn't too accomodated with the win32 API) was that | ||
4260 | there were some bugs on MinGW regarding the Bluetooth. Now they are solved | ||
4261 | but you should keep in mind that you should have the latest updates | ||
4262 | (especially the @emph{ws2bth} header). | ||
4263 | |||
4264 | Besides the fact that it uses the Windows Sockets, the Windows | ||
4265 | implemenation follows the same principles as the GNU/Linux one: | ||
4266 | |||
4267 | @itemize @bullet | ||
4268 | @item It has a initalization part where it initializes the | ||
4269 | Windows Sockets, creates a RFCOMM socket which will be binded and switched | ||
4270 | to the listening mode and registers a SDP service. In the Microsoft | ||
4271 | Bluetooth API there are two ways to work with the SDP: | ||
4272 | @itemize @bullet | ||
4273 | @item an easy way which works with very simple service records | ||
4274 | @item a hard way which is useful when you need to update or to delete the | ||
4275 | record | ||
4276 | @end itemize | ||
4277 | @end itemize | ||
4278 | |||
4279 | Since I only needed the SDP service to find out on which port the device | ||
4280 | is listening on and that did not change, I decided to use the easy way. | ||
4281 | In order to register the service I used the @emph{WSASetService} function | ||
4282 | and I generated the @emph{Universally Unique Identifier} with the | ||
4283 | @emph{guidgen.exe} Windows's tool. | ||
4284 | |||
4285 | In the loop section the only difference from the GNU/Linux implementation | ||
4286 | is that I used the @code{GNUNET_NETWORK} library for | ||
4287 | functions like @emph{accept}, @emph{bind}, @emph{connect} or | ||
4288 | @emph{select}. I decided to use the | ||
4289 | @code{GNUNET_NETWORK} library because I also needed to interact | ||
4290 | with the STDIN and STDOUT handles and on Windows | ||
4291 | the select function is only defined for sockets, | ||
4292 | and it will not work for arbitrary file handles. | ||
4293 | |||
4294 | Another difference between GNU/Linux and Windows implementation is that in | ||
4295 | GNU/Linux, the Bluetooth address is represented in 48 bits | ||
4296 | while in Windows is represented in 64 bits. | ||
4297 | Therefore I had to do some changes on @emph{plugin_transport_wlan} header. | ||
4298 | |||
4299 | Also, currently on Windows the Bluetooth plugin doesn't have support for | ||
4300 | broadcast messages. When it receives a broadcast message it will skip it. | ||
4301 | |||
4302 | @node Pending features | ||
4303 | @subsubsection Pending features | ||
4304 | @c %**end of header | ||
4305 | |||
4306 | @itemize @bullet | ||
4307 | @item Implement the broadcast functionality on Windows @emph{(currently | ||
4308 | working on)} | ||
4309 | @item Implement a testcase for the helper :@ @emph{The testcase | ||
4310 | consists of a program which emaluates the plugin and uses the helper. It | ||
4311 | will simulate connections, disconnections and data transfers.} | ||
4312 | @end itemize | ||
4313 | |||
4314 | If you have a new idea about a feature of the plugin or suggestions about | ||
4315 | how I could improve the implementation you are welcome to comment or to | ||
4316 | contact me. | ||
4317 | |||
4318 | @node WLAN plugin | ||
4319 | @section WLAN plugin | ||
4320 | @c %**end of header | ||
4321 | |||
4322 | This section documents how the wlan transport plugin works. Parts which | ||
4323 | are not implemented yet or could be better implemented are described at | ||
4324 | the end. | ||
4325 | |||
4326 | @cindex ATS Subsystem | ||
4327 | @node ATS Subsystem | ||
4328 | @section ATS Subsystem | ||
4329 | @c %**end of header | ||
4330 | |||
4331 | ATS stands for "automatic transport selection", and the function of ATS in | ||
4332 | GNUnet is to decide on which address (and thus transport plugin) should | ||
4333 | be used for two peers to communicate, and what bandwidth limits should be | ||
4334 | imposed on such an individual connection. To help ATS make an informed | ||
4335 | decision, higher-level services inform the ATS service about their | ||
4336 | requirements and the quality of the service rendered. The ATS service | ||
4337 | also interacts with the transport service to be appraised of working | ||
4338 | addresses and to communicate its resource allocation decisions. Finally, | ||
4339 | the ATS service's operation can be observed using a monitoring API. | ||
4340 | |||
4341 | The main logic of the ATS service only collects the available addresses, | ||
4342 | their performance characteristics and the applications requirements, but | ||
4343 | does not make the actual allocation decision. This last critical step is | ||
4344 | left to an ATS plugin, as we have implemented (currently three) different | ||
4345 | allocation strategies which differ significantly in their performance and | ||
4346 | maturity, and it is still unclear if any particular plugin is generally | ||
4347 | superior. | ||
4348 | |||
4349 | @cindex CORE Subsystem | ||
4350 | @node CORE Subsystem | ||
4351 | @section CORE Subsystem | ||
4352 | @c %**end of header | ||
4353 | |||
4354 | The CORE subsystem in GNUnet is responsible for securing link-layer | ||
4355 | communications between nodes in the GNUnet overlay network. CORE builds | ||
4356 | on the TRANSPORT subsystem which provides for the actual, insecure, | ||
4357 | unreliable link-layer communication (for example, via UDP or WLAN), and | ||
4358 | then adds fundamental security to the connections: | ||
4359 | |||
4360 | @itemize @bullet | ||
4361 | @item confidentiality with so-called perfect forward secrecy; we use | ||
4362 | ECDHE@footnote{@uref{http://en.wikipedia.org/wiki/Elliptic_curve_Diffie%E2%80%93Hellman, Elliptic-curve Diffie---Hellman}} | ||
4363 | powered by Curve25519 | ||
4364 | @footnote{@uref{http://cr.yp.to/ecdh.html, Curve25519}} for the key | ||
4365 | exchange and then use symmetric encryption, encrypting with both AES-256 | ||
4366 | @footnote{@uref{http://en.wikipedia.org/wiki/Rijndael, AES-256}} and | ||
4367 | Twofish @footnote{@uref{http://en.wikipedia.org/wiki/Twofish, Twofish}} | ||
4368 | @item @uref{http://en.wikipedia.org/wiki/Authentication, authentication} | ||
4369 | is achieved by signing the ephemeral keys using Ed25519 | ||
4370 | @footnote{@uref{http://ed25519.cr.yp.to/, Ed25519}}, a deterministic | ||
4371 | variant of ECDSA | ||
4372 | @footnote{@uref{http://en.wikipedia.org/wiki/ECDSA, ECDSA}} | ||
4373 | @item integrity protection (using SHA-512 | ||
4374 | @footnote{@uref{http://en.wikipedia.org/wiki/SHA-2, SHA-512}} to do | ||
4375 | encrypt-then-MAC | ||
4376 | @footnote{@uref{http://en.wikipedia.org/wiki/Authenticated_encryption, encrypt-then-MAC}}) | ||
4377 | @item Replay | ||
4378 | @footnote{@uref{http://en.wikipedia.org/wiki/Replay_attack, replay}} | ||
4379 | protection (using nonces, timestamps, challenge-response, | ||
4380 | message counters and ephemeral keys) | ||
4381 | @item liveness (keep-alive messages, timeout) | ||
4382 | @end itemize | ||
4383 | |||
4384 | @menu | ||
4385 | * Limitations:: | ||
4386 | * When is a peer "connected"?:: | ||
4387 | * libgnunetcore:: | ||
4388 | * The CORE Client-Service Protocol:: | ||
4389 | * The CORE Peer-to-Peer Protocol:: | ||
4390 | @end menu | ||
4391 | |||
4392 | @cindex core subsystem limitations | ||
4393 | @node Limitations | ||
4394 | @subsection Limitations | ||
4395 | @c %**end of header | ||
4396 | |||
4397 | CORE does not perform | ||
4398 | @uref{http://en.wikipedia.org/wiki/Routing, routing}; using CORE it is | ||
4399 | only possible to communicate with peers that happen to already be | ||
4400 | "directly" connected with each other. CORE also does not have an | ||
4401 | API to allow applications to establish such "direct" connections --- for | ||
4402 | this, applications can ask TRANSPORT, but TRANSPORT might not be able to | ||
4403 | establish a "direct" connection. The TOPOLOGY subsystem is responsible for | ||
4404 | trying to keep a few "direct" connections open at all times. Applications | ||
4405 | that need to talk to particular peers should use the CADET subsystem, as | ||
4406 | it can establish arbitrary "indirect" connections. | ||
4407 | |||
4408 | Because CORE does not perform routing, CORE must only be used directly by | ||
4409 | applications that either perform their own routing logic (such as | ||
4410 | anonymous file-sharing) or that do not require routing, for example | ||
4411 | because they are based on flooding the network. CORE communication is | ||
4412 | unreliable and delivery is possibly out-of-order. Applications that | ||
4413 | require reliable communication should use the CADET service. Each | ||
4414 | application can only queue one message per target peer with the CORE | ||
4415 | service at any time; messages cannot be larger than approximately | ||
4416 | 63 kilobytes. If messages are small, CORE may group multiple messages | ||
4417 | (possibly from different applications) prior to encryption. If permitted | ||
4418 | by the application (using the @uref{http://baus.net/on-tcp_cork/, cork} | ||
4419 | option), CORE may delay transmissions to facilitate grouping of multiple | ||
4420 | small messages. If cork is not enabled, CORE will transmit the message as | ||
4421 | soon as TRANSPORT allows it (TRANSPORT is responsible for limiting | ||
4422 | bandwidth and congestion control). CORE does not allow flow control; | ||
4423 | applications are expected to process messages at line-speed. If flow | ||
4424 | control is needed, applications should use the CADET service. | ||
4425 | |||
4426 | @cindex when is a peer connected | ||
4427 | @node When is a peer "connected"? | ||
4428 | @subsection When is a peer "connected"? | ||
4429 | @c %**end of header | ||
4430 | |||
4431 | In addition to the security features mentioned above, CORE also provides | ||
4432 | one additional key feature to applications using it, and that is a | ||
4433 | limited form of protocol-compatibility checking. CORE distinguishes | ||
4434 | between TRANSPORT-level connections (which enable communication with other | ||
4435 | peers) and application-level connections. Applications using the CORE API | ||
4436 | will (typically) learn about application-level connections from CORE, and | ||
4437 | not about TRANSPORT-level connections. When a typical application uses | ||
4438 | CORE, it will specify a set of message types | ||
4439 | (from @code{gnunet_protocols.h}) that it understands. CORE will then | ||
4440 | notify the application about connections it has with other peers if and | ||
4441 | only if those applications registered an intersecting set of message | ||
4442 | types with their CORE service. Thus, it is quite possible that CORE only | ||
4443 | exposes a subset of the established direct connections to a particular | ||
4444 | application --- and different applications running above CORE might see | ||
4445 | different sets of connections at the same time. | ||
4446 | |||
4447 | A special case are applications that do not register a handler for any | ||
4448 | message type. | ||
4449 | CORE assumes that these applications merely want to monitor connections | ||
4450 | (or "all" messages via other callbacks) and will notify those applications | ||
4451 | about all connections. This is used, for example, by the | ||
4452 | @code{gnunet-core} command-line tool to display the active connections. | ||
4453 | Note that it is also possible that the TRANSPORT service has more active | ||
4454 | connections than the CORE service, as the CORE service first has to | ||
4455 | perform a key exchange with connecting peers before exchanging information | ||
4456 | about supported message types and notifying applications about the new | ||
4457 | connection. | ||
4458 | |||
4459 | @cindex libgnunetcore | ||
4460 | @node libgnunetcore | ||
4461 | @subsection libgnunetcore | ||
4462 | @c %**end of header | ||
4463 | |||
4464 | The CORE API (defined in @file{gnunet_core_service.h}) is the basic | ||
4465 | messaging API used by P2P applications built using GNUnet. It provides | ||
4466 | applications the ability to send and receive encrypted messages to the | ||
4467 | peer's "directly" connected neighbours. | ||
4468 | |||
4469 | As CORE connections are generally "direct" connections,@ applications must | ||
4470 | not assume that they can connect to arbitrary peers this way, as "direct" | ||
4471 | connections may not always be possible. Applications using CORE are | ||
4472 | notified about which peers are connected. Creating new "direct" | ||
4473 | connections must be done using the TRANSPORT API. | ||
4474 | |||
4475 | The CORE API provides unreliable, out-of-order delivery. While the | ||
4476 | implementation tries to ensure timely, in-order delivery, both message | ||
4477 | losses and reordering are not detected and must be tolerated by the | ||
4478 | application. Most important, the core will NOT perform retransmission if | ||
4479 | messages could not be delivered. | ||
4480 | |||
4481 | Note that CORE allows applications to queue one message per connected | ||
4482 | peer. The rate at which each connection operates is influenced by the | ||
4483 | preferences expressed by local application as well as restrictions | ||
4484 | imposed by the other peer. Local applications can express their | ||
4485 | preferences for particular connections using the "performance" API of the | ||
4486 | ATS service. | ||
4487 | |||
4488 | Applications that require more sophisticated transmission capabilities | ||
4489 | such as TCP-like behavior, or if you intend to send messages to arbitrary | ||
4490 | remote peers, should use the CADET API. | ||
4491 | |||
4492 | The typical use of the CORE API is to connect to the CORE service using | ||
4493 | @code{GNUNET_CORE_connect}, process events from the CORE service (such as | ||
4494 | peers connecting, peers disconnecting and incoming messages) and send | ||
4495 | messages to connected peers using | ||
4496 | @code{GNUNET_CORE_notify_transmit_ready}. Note that applications must | ||
4497 | cancel pending transmission requests if they receive a disconnect event | ||
4498 | for a peer that had a transmission pending; furthermore, queueing more | ||
4499 | than one transmission request per peer per application using the | ||
4500 | service is not permitted. | ||
4501 | |||
4502 | The CORE API also allows applications to monitor all communications of the | ||
4503 | peer prior to encryption (for outgoing messages) or after decryption (for | ||
4504 | incoming messages). This can be useful for debugging, diagnostics or to | ||
4505 | establish the presence of cover traffic (for anonymity). As monitoring | ||
4506 | applications are often not interested in the payload, the monitoring | ||
4507 | callbacks can be configured to only provide the message headers (including | ||
4508 | the message type and size) instead of copying the full data stream to the | ||
4509 | monitoring client. | ||
4510 | |||
4511 | The init callback of the @code{GNUNET_CORE_connect} function is called | ||
4512 | with the hash of the public key of the peer. This public key is used to | ||
4513 | identify the peer globally in the GNUnet network. Applications are | ||
4514 | encouraged to check that the provided hash matches the hash that they are | ||
4515 | using (as theoretically the application may be using a different | ||
4516 | configuration file with a different private key, which would result in | ||
4517 | hard to find bugs). | ||
4518 | |||
4519 | As with most service APIs, the CORE API isolates applications from crashes | ||
4520 | of the CORE service. If the CORE service crashes, the application will see | ||
4521 | disconnect events for all existing connections. Once the connections are | ||
4522 | re-established, the applications will be receive matching connect events. | ||
4523 | |||
4524 | @cindex core clinet-service protocol | ||
4525 | @node The CORE Client-Service Protocol | ||
4526 | @subsection The CORE Client-Service Protocol | ||
4527 | @c %**end of header | ||
4528 | |||
4529 | This section describes the protocol between an application using the CORE | ||
4530 | service (the client) and the CORE service process itself. | ||
4531 | |||
4532 | |||
4533 | @menu | ||
4534 | * Setup2:: | ||
4535 | * Notifications:: | ||
4536 | * Sending:: | ||
4537 | @end menu | ||
4538 | |||
4539 | @node Setup2 | ||
4540 | @subsubsection Setup2 | ||
4541 | @c %**end of header | ||
4542 | |||
4543 | When a client connects to the CORE service, it first sends a | ||
4544 | @code{InitMessage} which specifies options for the connection and a set of | ||
4545 | message type values which are supported by the application. The options | ||
4546 | bitmask specifies which events the client would like to be notified about. | ||
4547 | The options include: | ||
4548 | |||
4549 | @table @asis | ||
4550 | @item GNUNET_CORE_OPTION_NOTHING No notifications | ||
4551 | @item GNUNET_CORE_OPTION_STATUS_CHANGE Peers connecting and disconnecting | ||
4552 | @item GNUNET_CORE_OPTION_FULL_INBOUND All inbound messages (after | ||
4553 | decryption) with full payload | ||
4554 | @item GNUNET_CORE_OPTION_HDR_INBOUND Just the @code{MessageHeader} | ||
4555 | of all inbound messages | ||
4556 | @item GNUNET_CORE_OPTION_FULL_OUTBOUND All outbound | ||
4557 | messages (prior to encryption) with full payload | ||
4558 | @item GNUNET_CORE_OPTION_HDR_OUTBOUND Just the @code{MessageHeader} of all | ||
4559 | outbound messages | ||
4560 | @end table | ||
4561 | |||
4562 | Typical applications will only monitor for connection status changes. | ||
4563 | |||
4564 | The CORE service responds to the @code{InitMessage} with an | ||
4565 | @code{InitReplyMessage} which contains the peer's identity. Afterwards, | ||
4566 | both CORE and the client can send messages. | ||
4567 | |||
4568 | @node Notifications | ||
4569 | @subsubsection Notifications | ||
4570 | @c %**end of header | ||
4571 | |||
4572 | The CORE will send @code{ConnectNotifyMessage}s and | ||
4573 | @code{DisconnectNotifyMessage}s whenever peers connect or disconnect from | ||
4574 | the CORE (assuming their type maps overlap with the message types | ||
4575 | registered by the client). When the CORE receives a message that matches | ||
4576 | the set of message types specified during the @code{InitMessage} (or if | ||
4577 | monitoring is enabled in for inbound messages in the options), it sends a | ||
4578 | @code{NotifyTrafficMessage} with the peer identity of the sender and the | ||
4579 | decrypted payload. The same message format (except with | ||
4580 | @code{GNUNET_MESSAGE_TYPE_CORE_NOTIFY_OUTBOUND} for the message type) is | ||
4581 | used to notify clients monitoring outbound messages; here, the peer | ||
4582 | identity given is that of the receiver. | ||
4583 | |||
4584 | @node Sending | ||
4585 | @subsubsection Sending | ||
4586 | @c %**end of header | ||
4587 | |||
4588 | When a client wants to transmit a message, it first requests a | ||
4589 | transmission slot by sending a @code{SendMessageRequest} which specifies | ||
4590 | the priority, deadline and size of the message. Note that these values | ||
4591 | may be ignored by CORE. When CORE is ready for the message, it answers | ||
4592 | with a @code{SendMessageReady} response. The client can then transmit the | ||
4593 | payload with a @code{SendMessage} message. Note that the actual message | ||
4594 | size in the @code{SendMessage} is allowed to be smaller than the size in | ||
4595 | the original request. A client may at any time send a fresh | ||
4596 | @code{SendMessageRequest}, which then superceeds the previous | ||
4597 | @code{SendMessageRequest}, which is then no longer valid. The client can | ||
4598 | tell which @code{SendMessageRequest} the CORE service's | ||
4599 | @code{SendMessageReady} message is for as all of these messages contain a | ||
4600 | "unique" request ID (based on a counter incremented by the client | ||
4601 | for each request). | ||
4602 | |||
4603 | @cindex CORE Peer-to-Peer Protocol | ||
4604 | @node The CORE Peer-to-Peer Protocol | ||
4605 | @subsection The CORE Peer-to-Peer Protocol | ||
4606 | @c %**end of header | ||
4607 | |||
4608 | |||
4609 | @menu | ||
4610 | * Creating the EphemeralKeyMessage:: | ||
4611 | * Establishing a connection:: | ||
4612 | * Encryption and Decryption:: | ||
4613 | * Type maps:: | ||
4614 | @end menu | ||
4615 | |||
4616 | @cindex EphemeralKeyMessage creation | ||
4617 | @node Creating the EphemeralKeyMessage | ||
4618 | @subsubsection Creating the EphemeralKeyMessage | ||
4619 | @c %**end of header | ||
4620 | |||
4621 | When the CORE service starts, each peer creates a fresh ephemeral (ECC) | ||
4622 | public-private key pair and signs the corresponding | ||
4623 | @code{EphemeralKeyMessage} with its long-term key (which we usually call | ||
4624 | the peer's identity; the hash of the public long term key is what results | ||
4625 | in a @code{struct GNUNET_PeerIdentity} in all GNUnet APIs. The ephemeral | ||
4626 | key is ONLY used for an ECDHE@footnote{@uref{http://en.wikipedia.org/wiki/Elliptic_curve_Diffie%E2%80%93Hellman, Elliptic-curve Diffie---Hellman}} | ||
4627 | exchange by the CORE service to establish symmetric session keys. A peer | ||
4628 | will use the same @code{EphemeralKeyMessage} for all peers for | ||
4629 | @code{REKEY_FREQUENCY}, which is usually 12 hours. After that time, it | ||
4630 | will create a fresh ephemeral key (forgetting the old one) and broadcast | ||
4631 | the new @code{EphemeralKeyMessage} to all connected peers, resulting in | ||
4632 | fresh symmetric session keys. Note that peers independently decide on | ||
4633 | when to discard ephemeral keys; it is not a protocol violation to discard | ||
4634 | keys more often. Ephemeral keys are also never stored to disk; restarting | ||
4635 | a peer will thus always create a fresh ephemeral key. The use of ephemeral | ||
4636 | keys is what provides @uref{http://en.wikipedia.org/wiki/Forward_secrecy, forward secrecy}. | ||
4637 | |||
4638 | Just before transmission, the @code{EphemeralKeyMessage} is patched to | ||
4639 | reflect the current sender_status, which specifies the current state of | ||
4640 | the connection from the point of view of the sender. The possible values | ||
4641 | are: | ||
4642 | |||
4643 | @itemize @bullet | ||
4644 | @item @code{KX_STATE_DOWN} Initial value, never used on the network | ||
4645 | @item @code{KX_STATE_KEY_SENT} We sent our ephemeral key, do not know the | ||
4646 | key of the other peer | ||
4647 | @item @code{KX_STATE_KEY_RECEIVED} This peer has received a valid | ||
4648 | ephemeral key of the other peer, but we are waiting for the other peer to | ||
4649 | confirm it's authenticity (ability to decode) via challenge-response. | ||
4650 | @item @code{KX_STATE_UP} The connection is fully up from the point of | ||
4651 | view of the sender (now performing keep-alives) | ||
4652 | @item @code{KX_STATE_REKEY_SENT} The sender has initiated a rekeying | ||
4653 | operation; the other peer has so far failed to confirm a working | ||
4654 | connection using the new ephemeral key | ||
4655 | @end itemize | ||
4656 | |||
4657 | @node Establishing a connection | ||
4658 | @subsubsection Establishing a connection | ||
4659 | @c %**end of header | ||
4660 | |||
4661 | Peers begin their interaction by sending a @code{EphemeralKeyMessage} to | ||
4662 | the other peer once the TRANSPORT service notifies the CORE service about | ||
4663 | the connection. | ||
4664 | A peer receiving an @code{EphemeralKeyMessage} with a status | ||
4665 | indicating that the sender does not have the receiver's ephemeral key, the | ||
4666 | receiver's @code{EphemeralKeyMessage} is sent in response. | ||
4667 | Additionally, if the receiver has not yet confirmed the authenticity of | ||
4668 | the sender, it also sends an (encrypted)@code{PingMessage} with a | ||
4669 | challenge (and the identity of the target) to the other peer. Peers | ||
4670 | receiving a @code{PingMessage} respond with an (encrypted) | ||
4671 | @code{PongMessage} which includes the challenge. Peers receiving a | ||
4672 | @code{PongMessage} check the challenge, and if it matches set the | ||
4673 | connection to @code{KX_STATE_UP}. | ||
4674 | |||
4675 | @node Encryption and Decryption | ||
4676 | @subsubsection Encryption and Decryption | ||
4677 | @c %**end of header | ||
4678 | |||
4679 | All functions related to the key exchange and encryption/decryption of | ||
4680 | messages can be found in @file{gnunet-service-core_kx.c} (except for the | ||
4681 | cryptographic primitives, which are in @file{util/crypto*.c}). | ||
4682 | Given the key material from ECDHE, a Key derivation function | ||
4683 | @footnote{@uref{https://en.wikipedia.org/wiki/Key_derivation_function, Key derivation function}} | ||
4684 | is used to derive two pairs of encryption and decryption keys for AES-256 | ||
4685 | and TwoFish, as well as initialization vectors and authentication keys | ||
4686 | (for HMAC@footnote{@uref{https://en.wikipedia.org/wiki/HMAC, HMAC}}). | ||
4687 | The HMAC is computed over the encrypted payload. | ||
4688 | Encrypted messages include an iv_seed and the HMAC in the header. | ||
4689 | |||
4690 | Each encrypted message in the CORE service includes a sequence number and | ||
4691 | a timestamp in the encrypted payload. The CORE service remembers the | ||
4692 | largest observed sequence number and a bit-mask which represents which of | ||
4693 | the previous 32 sequence numbers were already used. | ||
4694 | Messages with sequence numbers lower than the largest observed sequence | ||
4695 | number minus 32 are discarded. Messages with a timestamp that is less | ||
4696 | than @code{REKEY_TOLERANCE} off (5 minutes) are also discarded. This of | ||
4697 | course means that system clocks need to be reasonably synchronized for | ||
4698 | peers to be able to communicate. Additionally, as the ephemeral key | ||
4699 | changes every 12 hours, a peer would not even be able to decrypt messages | ||
4700 | older than 12 hours. | ||
4701 | |||
4702 | @node Type maps | ||
4703 | @subsubsection Type maps | ||
4704 | @c %**end of header | ||
4705 | |||
4706 | Once an encrypted connection has been established, peers begin to exchange | ||
4707 | type maps. Type maps are used to allow the CORE service to determine which | ||
4708 | (encrypted) connections should be shown to which applications. A type map | ||
4709 | is an array of 65536 bits representing the different types of messages | ||
4710 | understood by applications using the CORE service. Each CORE service | ||
4711 | maintains this map, simply by setting the respective bit for each message | ||
4712 | type supported by any of the applications using the CORE service. Note | ||
4713 | that bits for message types embedded in higher-level protocols (such as | ||
4714 | MESH) will not be included in these type maps. | ||
4715 | |||
4716 | Typically, the type map of a peer will be sparse. Thus, the CORE service | ||
4717 | attempts to compress its type map using @code{gzip}-style compression | ||
4718 | ("deflate") prior to transmission. However, if the compression fails to | ||
4719 | compact the map, the map may also be transmitted without compression | ||
4720 | (resulting in @code{GNUNET_MESSAGE_TYPE_CORE_COMPRESSED_TYPE_MAP} or | ||
4721 | @code{GNUNET_MESSAGE_TYPE_CORE_BINARY_TYPE_MAP} messages respectively). | ||
4722 | Upon receiving a type map, the respective CORE service notifies | ||
4723 | applications about the connection to the other peer if they support any | ||
4724 | message type indicated in the type map (or no message type at all). | ||
4725 | If the CORE service experience a connect or disconnect event from an | ||
4726 | application, it updates its type map (setting or unsetting the respective | ||
4727 | bits) and notifies its neighbours about the change. | ||
4728 | The CORE services of the neighbours then in turn generate connect and | ||
4729 | disconnect events for the peer that sent the type map for their respective | ||
4730 | applications. As CORE messages may be lost, the CORE service confirms | ||
4731 | receiving a type map by sending back a | ||
4732 | @code{GNUNET_MESSAGE_TYPE_CORE_CONFIRM_TYPE_MAP}. If such a confirmation | ||
4733 | (with the correct hash of the type map) is not received, the sender will | ||
4734 | retransmit the type map (with exponential back-off). | ||
4735 | |||
4736 | @cindex CADET Subsystem | ||
4737 | @node CADET Subsystem | ||
4738 | @section CADET Subsystem | ||
4739 | |||
4740 | The CADET subsystem in GNUnet is responsible for secure end-to-end | ||
4741 | communications between nodes in the GNUnet overlay network. CADET builds | ||
4742 | on the CORE subsystem which provides for the link-layer communication and | ||
4743 | then adds routing, forwarding and additional security to the connections. | ||
4744 | CADET offers the same cryptographic services as CORE, but on an | ||
4745 | end-to-end level. This is done so peers retransmitting traffic on behalf | ||
4746 | of other peers cannot access the payload data. | ||
4747 | |||
4748 | @itemize @bullet | ||
4749 | @item CADET provides confidentiality with so-called perfect forward | ||
4750 | secrecy; we use ECDHE powered by Curve25519 for the key exchange and then | ||
4751 | use symmetric encryption, encrypting with both AES-256 and Twofish | ||
4752 | @item authentication is achieved by signing the ephemeral keys using | ||
4753 | Ed25519, a deterministic variant of ECDSA | ||
4754 | @item integrity protection (using SHA-512 to do encrypt-then-MAC, although | ||
4755 | only 256 bits are sent to reduce overhead) | ||
4756 | @item replay protection (using nonces, timestamps, challenge-response, | ||
4757 | message counters and ephemeral keys) | ||
4758 | @item liveness (keep-alive messages, timeout) | ||
4759 | @end itemize | ||
4760 | |||
4761 | Additional to the CORE-like security benefits, CADET offers other | ||
4762 | properties that make it a more universal service than CORE. | ||
4763 | |||
4764 | @itemize @bullet | ||
4765 | @item CADET can establish channels to arbitrary peers in GNUnet. If a | ||
4766 | peer is not immediately reachable, CADET will find a path through the | ||
4767 | network and ask other peers to retransmit the traffic on its behalf. | ||
4768 | @item CADET offers (optional) reliability mechanisms. In a reliable | ||
4769 | channel traffic is guaranteed to arrive complete, unchanged and in-order. | ||
4770 | @item CADET takes care of flow and congestion control mechanisms, not | ||
4771 | allowing the sender to send more traffic than the receiver or the network | ||
4772 | are able to process. | ||
4773 | @end itemize | ||
4774 | |||
4775 | @menu | ||
4776 | * libgnunetcadet:: | ||
4777 | @end menu | ||
4778 | |||
4779 | @cindex libgnunetcadet | ||
4780 | @node libgnunetcadet | ||
4781 | @subsection libgnunetcadet | ||
4782 | |||
4783 | |||
4784 | The CADET API (defined in @file{gnunet_cadet_service.h}) is the | ||
4785 | messaging API used by P2P applications built using GNUnet. | ||
4786 | It provides applications the ability to send and receive encrypted | ||
4787 | messages to any peer participating in GNUnet. | ||
4788 | The API is heavily base on the CORE API. | ||
4789 | |||
4790 | CADET delivers messages to other peers in "channels". | ||
4791 | A channel is a permanent connection defined by a destination peer | ||
4792 | (identified by its public key) and a port number. | ||
4793 | Internally, CADET tunnels all channels towards a destiantion peer | ||
4794 | using one session key and relays the data on multiple "connections", | ||
4795 | independent from the channels. | ||
4796 | |||
4797 | Each channel has optional paramenters, the most important being the | ||
4798 | reliability flag. | ||
4799 | Should a message get lost on TRANSPORT/CORE level, if a channel is | ||
4800 | created with as reliable, CADET will retransmit the lost message and | ||
4801 | deliver it in order to the destination application. | ||
4802 | |||
4803 | To communicate with other peers using CADET, it is necessary to first | ||
4804 | connect to the service using @code{GNUNET_CADET_connect}. | ||
4805 | This function takes several parameters in form of callbacks, to allow the | ||
4806 | client to react to various events, like incoming channels or channels that | ||
4807 | terminate, as well as specify a list of ports the client wishes to listen | ||
4808 | to (at the moment it is not possible to start listening on further ports | ||
4809 | once connected, but nothing prevents a client to connect several times to | ||
4810 | CADET, even do one connection per listening port). | ||
4811 | The function returns a handle which has to be used for any further | ||
4812 | interaction with the service. | ||
4813 | |||
4814 | To connect to a remote peer a client has to call the | ||
4815 | @code{GNUNET_CADET_channel_create} function. The most important parameters | ||
4816 | given are the remote peer's identity (it public key) and a port, which | ||
4817 | specifies which application on the remote peer to connect to, similar to | ||
4818 | TCP/UDP ports. CADET will then find the peer in the GNUnet network and | ||
4819 | establish the proper low-level connections and do the necessary key | ||
4820 | exchanges to assure and authenticated, secure and verified communication. | ||
4821 | Similar to @code{GNUNET_CADET_connect},@code{GNUNET_CADET_create_channel} | ||
4822 | returns a handle to interact with the created channel. | ||
4823 | |||
4824 | For every message the client wants to send to the remote application, | ||
4825 | @code{GNUNET_CADET_notify_transmit_ready} must be called, indicating the | ||
4826 | channel on which the message should be sent and the size of the message | ||
4827 | (but not the message itself!). Once CADET is ready to send the message, | ||
4828 | the provided callback will fire, and the message contents are provided to | ||
4829 | this callback. | ||
4830 | |||
4831 | Please note the CADET does not provide an explicit notification of when a | ||
4832 | channel is connected. In loosely connected networks, like big wireless | ||
4833 | mesh networks, this can take several seconds, even minutes in the worst | ||
4834 | case. To be alerted when a channel is online, a client can call | ||
4835 | @code{GNUNET_CADET_notify_transmit_ready} immediately after | ||
4836 | @code{GNUNET_CADET_create_channel}. When the callback is activated, it | ||
4837 | means that the channel is online. The callback can give 0 bytes to CADET | ||
4838 | if no message is to be sent, this is ok. | ||
4839 | |||
4840 | If a transmission was requested but before the callback fires it is no | ||
4841 | longer needed, it can be cancelled with | ||
4842 | @code{GNUNET_CADET_notify_transmit_ready_cancel}, which uses the handle | ||
4843 | given back by @code{GNUNET_CADET_notify_transmit_ready}. | ||
4844 | As in the case of CORE, only one message can be requested at a time: a | ||
4845 | client must not call @code{GNUNET_CADET_notify_transmit_ready} again until | ||
4846 | the callback is called or the request is cancelled. | ||
4847 | |||
4848 | When a channel is no longer needed, a client can call | ||
4849 | @code{GNUNET_CADET_channel_destroy} to get rid of it. | ||
4850 | Note that CADET will try to transmit all pending traffic before notifying | ||
4851 | the remote peer of the destruction of the channel, including | ||
4852 | retransmitting lost messages if the channel was reliable. | ||
4853 | |||
4854 | Incoming channels, channels being closed by the remote peer, and traffic | ||
4855 | on any incoming or outgoing channels are given to the client when CADET | ||
4856 | executes the callbacks given to it at the time of | ||
4857 | @code{GNUNET_CADET_connect}. | ||
4858 | |||
4859 | Finally, when an application no longer wants to use CADET, it should call | ||
4860 | @code{GNUNET_CADET_disconnect}, but first all channels and pending | ||
4861 | transmissions must be closed (otherwise CADET will complain). | ||
4862 | |||
4863 | @cindex NSE Subsystem | ||
4864 | @node NSE Subsystem | ||
4865 | @section NSE Subsystem | ||
4866 | |||
4867 | |||
4868 | NSE stands for @dfn{Network Size Estimation}. The NSE subsystem provides | ||
4869 | other subsystems and users with a rough estimate of the number of peers | ||
4870 | currently participating in the GNUnet overlay. | ||
4871 | The computed value is not a precise number as producing a precise number | ||
4872 | in a decentralized, efficient and secure way is impossible. | ||
4873 | While NSE's estimate is inherently imprecise, NSE also gives the expected | ||
4874 | range. For a peer that has been running in a stable network for a | ||
4875 | while, the real network size will typically (99.7% of the time) be in the | ||
4876 | range of [2/3 estimate, 3/2 estimate]. We will now give an overview of the | ||
4877 | algorithm used to calculate the estimate; | ||
4878 | all of the details can be found in this technical report. | ||
4879 | |||
4880 | @c FIXME: link to the report. | ||
4881 | |||
4882 | @menu | ||
4883 | * Motivation:: | ||
4884 | * Principle:: | ||
4885 | * libgnunetnse:: | ||
4886 | * The NSE Client-Service Protocol:: | ||
4887 | * The NSE Peer-to-Peer Protocol:: | ||
4888 | @end menu | ||
4889 | |||
4890 | @node Motivation | ||
4891 | @subsection Motivation | ||
4892 | |||
4893 | |||
4894 | Some subsytems, like DHT, need to know the size of the GNUnet network to | ||
4895 | optimize some parameters of their own protocol. The decentralized nature | ||
4896 | of GNUnet makes efficient and securely counting the exact number of peers | ||
4897 | infeasable. Although there are several decentralized algorithms to count | ||
4898 | the number of peers in a system, so far there is none to do so securely. | ||
4899 | Other protocols may allow any malicious peer to manipulate the final | ||
4900 | result or to take advantage of the system to perform | ||
4901 | @dfn{Denial of Service} (DoS) attacks against the network. | ||
4902 | GNUnet's NSE protocol avoids these drawbacks. | ||
4903 | |||
4904 | |||
4905 | |||
4906 | @menu | ||
4907 | * Security:: | ||
4908 | @end menu | ||
4909 | |||
4910 | @cindex NSE security | ||
4911 | @cindex nse security | ||
4912 | @node Security | ||
4913 | @subsubsection Security | ||
4914 | |||
4915 | |||
4916 | The NSE subsystem is designed to be resilient against these attacks. | ||
4917 | It uses @uref{http://en.wikipedia.org/wiki/Proof-of-work_system, proofs of work} | ||
4918 | to prevent one peer from impersonating a large number of participants, | ||
4919 | which would otherwise allow an adversary to artifically inflate the | ||
4920 | estimate. | ||
4921 | The DoS protection comes from the time-based nature of the protocol: | ||
4922 | the estimates are calculated periodically and out-of-time traffic is | ||
4923 | either ignored or stored for later retransmission by benign peers. | ||
4924 | In particular, peers cannot trigger global network communication at will. | ||
4925 | |||
4926 | @cindex NSE principle | ||
4927 | @cindex nse principle | ||
4928 | @node Principle | ||
4929 | @subsection Principle | ||
4930 | |||
4931 | |||
4932 | The algorithm calculates the estimate by finding the globally closest | ||
4933 | peer ID to a random, time-based value. | ||
4934 | |||
4935 | The idea is that the closer the ID is to the random value, the more | ||
4936 | "densely packed" the ID space is, and therefore, more peers are in the | ||
4937 | network. | ||
4938 | |||
4939 | |||
4940 | |||
4941 | @menu | ||
4942 | * Example:: | ||
4943 | * Algorithm:: | ||
4944 | * Target value:: | ||
4945 | * Timing:: | ||
4946 | * Controlled Flooding:: | ||
4947 | * Calculating the estimate:: | ||
4948 | @end menu | ||
4949 | |||
4950 | @node Example | ||
4951 | @subsubsection Example | ||
4952 | |||
4953 | |||
4954 | Suppose all peers have IDs between 0 and 100 (our ID space), and the | ||
4955 | random value is 42. | ||
4956 | If the closest peer has the ID 70 we can imagine that the average | ||
4957 | "distance" between peers is around 30 and therefore the are around 3 | ||
4958 | peers in the whole ID space. On the other hand, if the closest peer has | ||
4959 | the ID 44, we can imagine that the space is rather packed with peers, | ||
4960 | maybe as much as 50 of them. | ||
4961 | Naturally, we could have been rather unlucky, and there is only one peer | ||
4962 | and happens to have the ID 44. Thus, the current estimate is calculated | ||
4963 | as the average over multiple rounds, and not just a single sample. | ||
4964 | |||
4965 | @node Algorithm | ||
4966 | @subsubsection Algorithm | ||
4967 | |||
4968 | |||
4969 | Given that example, one can imagine that the job of the subsystem is to | ||
4970 | efficiently communicate the ID of the closest peer to the target value | ||
4971 | to all the other peers, who will calculate the estimate from it. | ||
4972 | |||
4973 | @node Target value | ||
4974 | @subsubsection Target value | ||
4975 | |||
4976 | @c %**end of header | ||
4977 | |||
4978 | The target value itself is generated by hashing the current time, rounded | ||
4979 | down to an agreed value. If the rounding amount is 1h (default) and the | ||
4980 | time is 12:34:56, the time to hash would be 12:00:00. The process is | ||
4981 | repeated each rouning amount (in this example would be every hour). | ||
4982 | Every repetition is called a round. | ||
4983 | |||
4984 | @node Timing | ||
4985 | @subsubsection Timing | ||
4986 | @c %**end of header | ||
4987 | |||
4988 | The NSE subsystem has some timing control to avoid everybody broadcasting | ||
4989 | its ID all at one. Once each peer has the target random value, it | ||
4990 | compares its own ID to the target and calculates the hypothetical size of | ||
4991 | the network if that peer were to be the closest. | ||
4992 | Then it compares the hypothetical size with the estimate from the previous | ||
4993 | rounds. For each value there is an assiciated point in the period, | ||
4994 | let's call it "broadcast time". If its own hypothetical estimate | ||
4995 | is the same as the previous global estimate, its "broadcast time" will be | ||
4996 | in the middle of the round. If its bigger it will be earlier and if its | ||
4997 | smaller (the most likely case) it will be later. This ensures that the | ||
4998 | peers closests to the target value start broadcasting their ID the first. | ||
4999 | |||
5000 | @node Controlled Flooding | ||
5001 | @subsubsection Controlled Flooding | ||
5002 | |||
5003 | @c %**end of header | ||
5004 | |||
5005 | When a peer receives a value, first it verifies that it is closer than the | ||
5006 | closest value it had so far, otherwise it answers the incoming message | ||
5007 | with a message containing the better value. Then it checks a proof of | ||
5008 | work that must be included in the incoming message, to ensure that the | ||
5009 | other peer's ID is not made up (otherwise a malicious peer could claim to | ||
5010 | have an ID of exactly the target value every round). Once validated, it | ||
5011 | compares the brodcast time of the received value with the current time | ||
5012 | and if it's not too early, sends the received value to its neighbors. | ||
5013 | Otherwise it stores the value until the correct broadcast time comes. | ||
5014 | This prevents unnecessary traffic of sub-optimal values, since a better | ||
5015 | value can come before the broadcast time, rendering the previous one | ||
5016 | obsolete and saving the traffic that would have been used to broadcast it | ||
5017 | to the neighbors. | ||
5018 | |||
5019 | @node Calculating the estimate | ||
5020 | @subsubsection Calculating the estimate | ||
5021 | |||
5022 | @c %**end of header | ||
5023 | |||
5024 | Once the closest ID has been spread across the network each peer gets the | ||
5025 | exact distance betweed this ID and the target value of the round and | ||
5026 | calculates the estimate with a mathematical formula described in the tech | ||
5027 | report. The estimate generated with this method for a single round is not | ||
5028 | very precise. Remember the case of the example, where the only peer is the | ||
5029 | ID 44 and we happen to generate the target value 42, thinking there are | ||
5030 | 50 peers in the network. Therefore, the NSE subsystem remembers the last | ||
5031 | 64 estimates and calculates an average over them, giving a result of which | ||
5032 | usually has one bit of uncertainty (the real size could be half of the | ||
5033 | estimate or twice as much). Note that the actual network size is | ||
5034 | calculated in powers of two of the raw input, thus one bit of uncertainty | ||
5035 | means a factor of two in the size estimate. | ||
5036 | |||
5037 | @cindex libgnunetnse | ||
5038 | @node libgnunetnse | ||
5039 | @subsection libgnunetnse | ||
5040 | |||
5041 | @c %**end of header | ||
5042 | |||
5043 | The NSE subsystem has the simplest API of all services, with only two | ||
5044 | calls: @code{GNUNET_NSE_connect} and @code{GNUNET_NSE_disconnect}. | ||
5045 | |||
5046 | The connect call gets a callback function as a parameter and this function | ||
5047 | is called each time the network agrees on an estimate. This usually is | ||
5048 | once per round, with some exceptions: if the closest peer has a late | ||
5049 | local clock and starts spreading his ID after everyone else agreed on a | ||
5050 | value, the callback might be activated twice in a round, the second value | ||
5051 | being always bigger than the first. The default round time is set to | ||
5052 | 1 hour. | ||
5053 | |||
5054 | The disconnect call disconnects from the NSE subsystem and the callback | ||
5055 | is no longer called with new estimates. | ||
5056 | |||
5057 | |||
5058 | |||
5059 | @menu | ||
5060 | * Results:: | ||
5061 | * libgnunetnse - Examples:: | ||
5062 | @end menu | ||
5063 | |||
5064 | @node Results | ||
5065 | @subsubsection Results | ||
5066 | |||
5067 | @c %**end of header | ||
5068 | |||
5069 | The callback provides two values: the average and the | ||
5070 | @uref{http://en.wikipedia.org/wiki/Standard_deviation, standard deviation} | ||
5071 | of the last 64 rounds. The values provided by the callback function are | ||
5072 | logarithmic, this means that the real estimate numbers can be obtained by | ||
5073 | calculating 2 to the power of the given value (2average). From a | ||
5074 | statistics point of view this means that: | ||
5075 | |||
5076 | @itemize @bullet | ||
5077 | @item 68% of the time the real size is included in the interval | ||
5078 | [(2average-stddev), 2] | ||
5079 | @item 95% of the time the real size is included in the interval | ||
5080 | [(2average-2*stddev, 2^average+2*stddev] | ||
5081 | @item 99.7% of the time the real size is included in the interval | ||
5082 | [(2average-3*stddev, 2average+3*stddev] | ||
5083 | @end itemize | ||
5084 | |||
5085 | The expected standard variation for 64 rounds in a network of stable size | ||
5086 | is 0.2. Thus, we can say that normally: | ||
5087 | |||
5088 | @itemize @bullet | ||
5089 | @item 68% of the time the real size is in the range [-13%, +15%] | ||
5090 | @item 95% of the time the real size is in the range [-24%, +32%] | ||
5091 | @item 99.7% of the time the real size is in the range [-34%, +52%] | ||
5092 | @end itemize | ||
5093 | |||
5094 | As said in the introduction, we can be quite sure that usually the real | ||
5095 | size is between one third and three times the estimate. This can of | ||
5096 | course vary with network conditions. | ||
5097 | Thus, applications may want to also consider the provided standard | ||
5098 | deviation value, not only the average (in particular, if the standard | ||
5099 | veriation is very high, the average maybe meaningless: the network size is | ||
5100 | changing rapidly). | ||
5101 | |||
5102 | @node libgnunetnse - Examples | ||
5103 | @subsubsection libgnunetnse -Examples | ||
5104 | |||
5105 | @c %**end of header | ||
5106 | |||
5107 | Let's close with a couple examples. | ||
5108 | |||
5109 | @table @asis | ||
5110 | |||
5111 | @item Average: 10, std dev: 1 Here the estimate would be | ||
5112 | 2^10 = 1024 peers. @footnote{The range in which we can be 95% sure is: | ||
5113 | [2^8, 2^12] = [256, 4096]. We can be very (>99.7%) sure that the network | ||
5114 | is not a hundred peers and absolutely sure that it is not a million peers, | ||
5115 | but somewhere around a thousand.} | ||
5116 | |||
5117 | @item Average 22, std dev: 0.2 Here the estimate would be | ||
5118 | 2^22 = 4 Million peers. @footnote{The range in which we can be 99.7% sure | ||
5119 | is: [2^21.4, 2^22.6] = [2.8M, 6.3M]. We can be sure that the network size | ||
5120 | is around four million, with absolutely way of it being 1 million.} | ||
5121 | |||
5122 | @end table | ||
5123 | |||
5124 | To put this in perspective, if someone remembers the LHC Higgs boson | ||
5125 | results, were announced with "5 sigma" and "6 sigma" certainties. In this | ||
5126 | case a 5 sigma minimum would be 2 million and a 6 sigma minimum, | ||
5127 | 1.8 million. | ||
5128 | |||
5129 | @node The NSE Client-Service Protocol | ||
5130 | @subsection The NSE Client-Service Protocol | ||
5131 | |||
5132 | @c %**end of header | ||
5133 | |||
5134 | As with the API, the client-service protocol is very simple, only has 2 | ||
5135 | different messages, defined in @code{src/nse/nse.h}: | ||
5136 | |||
5137 | @itemize @bullet | ||
5138 | @item @code{GNUNET_MESSAGE_TYPE_NSE_START}@ This message has no parameters | ||
5139 | and is sent from the client to the service upon connection. | ||
5140 | @item @code{GNUNET_MESSAGE_TYPE_NSE_ESTIMATE}@ This message is sent from | ||
5141 | the service to the client for every new estimate and upon connection. | ||
5142 | Contains a timestamp for the estimate, the average and the standard | ||
5143 | deviation for the respective round. | ||
5144 | @end itemize | ||
5145 | |||
5146 | When the @code{GNUNET_NSE_disconnect} API call is executed, the client | ||
5147 | simply disconnects from the service, with no message involved. | ||
5148 | |||
5149 | @cindex NSE Peer-to-Peer Protocol | ||
5150 | @node The NSE Peer-to-Peer Protocol | ||
5151 | @subsection The NSE Peer-to-Peer Protocol | ||
5152 | |||
5153 | @c %**end of header | ||
5154 | |||
5155 | The NSE subsystem only has one message in the P2P protocol, the | ||
5156 | @code{GNUNET_MESSAGE_TYPE_NSE_P2P_FLOOD} message. | ||
5157 | |||
5158 | This message key contents are the timestamp to identify the round | ||
5159 | (differences in system clocks may cause some peers to send messages way | ||
5160 | too early or way too late, so the timestamp allows other peers to | ||
5161 | identify such messages easily), the | ||
5162 | @uref{http://en.wikipedia.org/wiki/Proof-of-work_system, proof of work} | ||
5163 | used to make it difficult to mount a | ||
5164 | @uref{http://en.wikipedia.org/wiki/Sybil_attack, Sybil attack}, and the | ||
5165 | public key, which is used to verify the signature on the message. | ||
5166 | |||
5167 | Every peer stores a message for the previous, current and next round. The | ||
5168 | messages for the previous and current round are given to peers that | ||
5169 | connect to us. The message for the next round is simply stored until our | ||
5170 | system clock advances to the next round. The message for the current round | ||
5171 | is what we are flooding the network with right now. | ||
5172 | At the beginning of each round the peer does the following: | ||
5173 | |||
5174 | @itemize @bullet | ||
5175 | @item calculates his own distance to the target value | ||
5176 | @item creates, signs and stores the message for the current round (unless | ||
5177 | it has a better message in the "next round" slot which came early in the | ||
5178 | previous round) | ||
5179 | @item calculates, based on the stored round message (own or received) when | ||
5180 | to stard flooding it to its neighbors | ||
5181 | @end itemize | ||
5182 | |||
5183 | Upon receiving a message the peer checks the validity of the message | ||
5184 | (round, proof of work, signature). The next action depends on the | ||
5185 | contents of the incoming message: | ||
5186 | |||
5187 | @itemize @bullet | ||
5188 | @item if the message is worse than the current stored message, the peer | ||
5189 | sends the current message back immediately, to stop the other peer from | ||
5190 | spreading suboptimal results | ||
5191 | @item if the message is better than the current stored message, the peer | ||
5192 | stores the new message and calculates the new target time to start | ||
5193 | spreading it to its neighbors (excluding the one the message came from) | ||
5194 | @item if the message is for the previous round, it is compared to the | ||
5195 | message stored in the "previous round slot", which may then be updated | ||
5196 | @item if the message is for the next round, it is compared to the message | ||
5197 | stored in the "next round slot", which again may then be updated | ||
5198 | @end itemize | ||
5199 | |||
5200 | Finally, when it comes to send the stored message for the current round to | ||
5201 | the neighbors there is a random delay added for each neighbor, to avoid | ||
5202 | traffic spikes and minimize cross-messages. | ||
5203 | |||
5204 | @cindex HOSTLIST Subsystem | ||
5205 | @node HOSTLIST Subsystem | ||
5206 | @section HOSTLIST Subsystem | ||
5207 | |||
5208 | @c %**end of header | ||
5209 | |||
5210 | Peers in the GNUnet overlay network need address information so that they | ||
5211 | can connect with other peers. GNUnet uses so called HELLO messages to | ||
5212 | store and exchange peer addresses. | ||
5213 | GNUnet provides several methods for peers to obtain this information: | ||
5214 | |||
5215 | @itemize @bullet | ||
5216 | @item out-of-band exchange of HELLO messages (manually, using for example | ||
5217 | gnunet-peerinfo) | ||
5218 | @item HELLO messages shipped with GNUnet (automatic with distribution) | ||
5219 | @item UDP neighbor discovery in LAN (IPv4 broadcast, IPv6 multicast) | ||
5220 | @item topology gossiping (learning from other peers we already connected | ||
5221 | to), and | ||
5222 | @item the HOSTLIST daemon covered in this section, which is particularly | ||
5223 | relevant for bootstrapping new peers. | ||
5224 | @end itemize | ||
5225 | |||
5226 | New peers have no existing connections (and thus cannot learn from gossip | ||
5227 | among peers), may not have other peers in their LAN and might be started | ||
5228 | with an outdated set of HELLO messages from the distribution. | ||
5229 | In this case, getting new peers to connect to the network requires either | ||
5230 | manual effort or the use of a HOSTLIST to obtain HELLOs. | ||
5231 | |||
5232 | @menu | ||
5233 | * HELLOs:: | ||
5234 | * Overview for the HOSTLIST subsystem:: | ||
5235 | * Interacting with the HOSTLIST daemon:: | ||
5236 | * Hostlist security address validation:: | ||
5237 | * The HOSTLIST daemon:: | ||
5238 | * The HOSTLIST server:: | ||
5239 | * The HOSTLIST client:: | ||
5240 | * Usage:: | ||
5241 | @end menu | ||
5242 | |||
5243 | @node HELLOs | ||
5244 | @subsection HELLOs | ||
5245 | |||
5246 | @c %**end of header | ||
5247 | |||
5248 | The basic information peers require to connect to other peers are | ||
5249 | contained in so called HELLO messages you can think of as a business card. | ||
5250 | Besides the identity of the peer (based on the cryptographic public key) a | ||
5251 | HELLO message may contain address information that specifies ways to | ||
5252 | contact a peer. By obtaining HELLO messages, a peer can learn how to | ||
5253 | contact other peers. | ||
5254 | |||
5255 | @node Overview for the HOSTLIST subsystem | ||
5256 | @subsection Overview for the HOSTLIST subsystem | ||
5257 | |||
5258 | @c %**end of header | ||
5259 | |||
5260 | The HOSTLIST subsystem provides a way to distribute and obtain contact | ||
5261 | information to connect to other peers using a simple HTTP GET request. | ||
5262 | It's implementation is split in three parts, the main file for the daemon | ||
5263 | itself (@file{gnunet-daemon-hostlist.c}), the HTTP client used to download | ||
5264 | peer information (@file{hostlist-client.c}) and the server component used | ||
5265 | to provide this information to other peers (@file{hostlist-server.c}). | ||
5266 | The server is basically a small HTTP web server (based on GNU | ||
5267 | libmicrohttpd) which provides a list of HELLOs known to the local peer for | ||
5268 | download. The client component is basically a HTTP client | ||
5269 | (based on libcurl) which can download hostlists from one or more websites. | ||
5270 | The hostlist format is a binary blob containing a sequence of HELLO | ||
5271 | messages. Note that any HTTP server can theoretically serve a hostlist, | ||
5272 | the build-in hostlist server makes it simply convenient to offer this | ||
5273 | service. | ||
5274 | |||
5275 | |||
5276 | @menu | ||
5277 | * Features:: | ||
5278 | * HOSTLIST - Limitations:: | ||
5279 | @end menu | ||
5280 | |||
5281 | @node Features | ||
5282 | @subsubsection Features | ||
5283 | |||
5284 | @c %**end of header | ||
5285 | |||
5286 | The HOSTLIST daemon can: | ||
5287 | |||
5288 | @itemize @bullet | ||
5289 | @item provide HELLO messages with validated addresses obtained from | ||
5290 | PEERINFO to download for other peers | ||
5291 | @item download HELLO messages and forward these message to the TRANSPORT | ||
5292 | subsystem for validation | ||
5293 | @item advertises the URL of this peer's hostlist address to other peers | ||
5294 | via gossip | ||
5295 | @item automatically learn about hostlist servers from the gossip of other | ||
5296 | peers | ||
5297 | @end itemize | ||
5298 | |||
5299 | @node HOSTLIST - Limitations | ||
5300 | @subsubsection HOSTLIST - Limitations | ||
5301 | |||
5302 | @c %**end of header | ||
5303 | |||
5304 | The HOSTLIST daemon does not: | ||
5305 | |||
5306 | @itemize @bullet | ||
5307 | @item verify the cryptographic information in the HELLO messages | ||
5308 | @item verify the address information in the HELLO messages | ||
5309 | @end itemize | ||
5310 | |||
5311 | @node Interacting with the HOSTLIST daemon | ||
5312 | @subsection Interacting with the HOSTLIST daemon | ||
5313 | |||
5314 | @c %**end of header | ||
5315 | |||
5316 | The HOSTLIST subsystem is currently implemented as a daemon, so there is | ||
5317 | no need for the user to interact with it and therefore there is no | ||
5318 | command line tool and no API to communicate with the daemon. In the | ||
5319 | future, we can envision changing this to allow users to manually trigger | ||
5320 | the download of a hostlist. | ||
5321 | |||
5322 | Since there is no command line interface to interact with HOSTLIST, the | ||
5323 | only way to interact with the hostlist is to use STATISTICS to obtain or | ||
5324 | modify information about the status of HOSTLIST: | ||
5325 | |||
5326 | @example | ||
5327 | $ gnunet-statistics -s hostlist | ||
5328 | @end example | ||
5329 | |||
5330 | @noindent | ||
5331 | In particular, HOSTLIST includes a @strong{persistent} value in statistics | ||
5332 | that specifies when the hostlist server might be queried next. As this | ||
5333 | value is exponentially increasing during runtime, developers may want to | ||
5334 | reset or manually adjust it. Note that HOSTLIST (but not STATISTICS) needs | ||
5335 | to be shutdown if changes to this value are to have any effect on the | ||
5336 | daemon (as HOSTLIST does not monitor STATISTICS for changes to the | ||
5337 | download frequency). | ||
5338 | |||
5339 | @node Hostlist security address validation | ||
5340 | @subsection Hostlist security address validation | ||
5341 | |||
5342 | @c %**end of header | ||
5343 | |||
5344 | Since information obtained from other parties cannot be trusted without | ||
5345 | validation, we have to distinguish between @emph{validated} and | ||
5346 | @emph{not validated} addresses. Before using (and so trusting) | ||
5347 | information from other parties, this information has to be double-checked | ||
5348 | (validated). Address validation is not done by HOSTLIST but by the | ||
5349 | TRANSPORT service. | ||
5350 | |||
5351 | The HOSTLIST component is functionally located between the PEERINFO and | ||
5352 | the TRANSPORT subsystem. When acting as a server, the daemon obtains valid | ||
5353 | (@emph{validated}) peer information (HELLO messages) from the PEERINFO | ||
5354 | service and provides it to other peers. When acting as a client, it | ||
5355 | contacts the HOSTLIST servers specified in the configuration, downloads | ||
5356 | the (unvalidated) list of HELLO messages and forwards these information | ||
5357 | to the TRANSPORT server to validate the addresses. | ||
5358 | |||
5359 | @cindex HOSTLIST daemon | ||
5360 | @node The HOSTLIST daemon | ||
5361 | @subsection The HOSTLIST daemon | ||
5362 | |||
5363 | @c %**end of header | ||
5364 | |||
5365 | The hostlist daemon is the main component of the HOSTLIST subsystem. It is | ||
5366 | started by the ARM service and (if configured) starts the HOSTLIST client | ||
5367 | and server components. | ||
5368 | |||
5369 | If the daemon provides a hostlist itself it can advertise it's own | ||
5370 | hostlist to other peers. To do so it sends a | ||
5371 | @code{GNUNET_MESSAGE_TYPE_HOSTLIST_ADVERTISEMENT} message to other peers | ||
5372 | when they connect to this peer on the CORE level. This hostlist | ||
5373 | advertisement message contains the URL to access the HOSTLIST HTTP | ||
5374 | server of the sender. The daemon may also subscribe to this type of | ||
5375 | message from CORE service, and then forward these kind of message to the | ||
5376 | HOSTLIST client. The client then uses all available URLs to download peer | ||
5377 | information when necessary. | ||
5378 | |||
5379 | When starting, the HOSTLIST daemon first connects to the CORE subsystem | ||
5380 | and if hostlist learning is enabled, registers a CORE handler to receive | ||
5381 | this kind of messages. Next it starts (if configured) the client and | ||
5382 | server. It passes pointers to CORE connect and disconnect and receive | ||
5383 | handlers where the client and server store their functions, so the daemon | ||
5384 | can notify them about CORE events. | ||
5385 | |||
5386 | To clean up on shutdown, the daemon has a cleaning task, shutting down all | ||
5387 | subsystems and disconnecting from CORE. | ||
5388 | |||
5389 | @cindex HOSTLIST server | ||
5390 | @node The HOSTLIST server | ||
5391 | @subsection The HOSTLIST server | ||
5392 | |||
5393 | @c %**end of header | ||
5394 | |||
5395 | The server provides a way for other peers to obtain HELLOs. Basically it | ||
5396 | is a small web server other peers can connect to and download a list of | ||
5397 | HELLOs using standard HTTP; it may also advertise the URL of the hostlist | ||
5398 | to other peers connecting on CORE level. | ||
5399 | |||
5400 | |||
5401 | @menu | ||
5402 | * The HTTP Server:: | ||
5403 | * Advertising the URL:: | ||
5404 | @end menu | ||
5405 | |||
5406 | @node The HTTP Server | ||
5407 | @subsubsection The HTTP Server | ||
5408 | |||
5409 | @c %**end of header | ||
5410 | |||
5411 | During startup, the server starts a web server listening on the port | ||
5412 | specified with the HTTPPORT value (default 8080). In addition it connects | ||
5413 | to the PEERINFO service to obtain peer information. The HOSTLIST server | ||
5414 | uses the GNUNET_PEERINFO_iterate function to request HELLO information for | ||
5415 | all peers and adds their information to a new hostlist if they are | ||
5416 | suitable (expired addresses and HELLOs without addresses are both not | ||
5417 | suitable) and the maximum size for a hostlist is not exceeded | ||
5418 | (MAX_BYTES_PER_HOSTLISTS = 500000). | ||
5419 | When PEERINFO finishes (with a last NULL callback), the server destroys | ||
5420 | the previous hostlist response available for download on the web server | ||
5421 | and replaces it with the updated hostlist. The hostlist format is | ||
5422 | basically a sequence of HELLO messages (as obtained from PEERINFO) without | ||
5423 | any special tokenization. Since each HELLO message contains a size field, | ||
5424 | the response can easily be split into separate HELLO messages by the | ||
5425 | client. | ||
5426 | |||
5427 | A HOSTLIST client connecting to the HOSTLIST server will receive the | ||
5428 | hostlist as a HTTP response and the the server will terminate the | ||
5429 | connection with the result code @code{HTTP 200 OK}. | ||
5430 | The connection will be closed immediately if no hostlist is available. | ||
5431 | |||
5432 | @node Advertising the URL | ||
5433 | @subsubsection Advertising the URL | ||
5434 | |||
5435 | @c %**end of header | ||
5436 | |||
5437 | The server also advertises the URL to download the hostlist to other peers | ||
5438 | if hostlist advertisement is enabled. | ||
5439 | When a new peer connects and has hostlist learning enabled, the server | ||
5440 | sends a @code{GNUNET_MESSAGE_TYPE_HOSTLIST_ADVERTISEMENT} message to this | ||
5441 | peer using the CORE service. | ||
5442 | |||
5443 | @cindex HOSTLIST client | ||
5444 | @node The HOSTLIST client | ||
5445 | @subsection The HOSTLIST client | ||
5446 | |||
5447 | @c %**end of header | ||
5448 | |||
5449 | The client provides the functionality to download the list of HELLOs from | ||
5450 | a set of URLs. | ||
5451 | It performs a standard HTTP request to the URLs configured and learned | ||
5452 | from advertisement messages received from other peers. When a HELLO is | ||
5453 | downloaded, the HOSTLIST client forwards the HELLO to the TRANSPORT | ||
5454 | service for validation. | ||
5455 | |||
5456 | The client supports two modes of operation: | ||
5457 | |||
5458 | @itemize @bullet | ||
5459 | @item download of HELLOs (bootstrapping) | ||
5460 | @item learning of URLs | ||
5461 | @end itemize | ||
5462 | |||
5463 | @menu | ||
5464 | * Bootstrapping:: | ||
5465 | * Learning:: | ||
5466 | @end menu | ||
5467 | |||
5468 | @node Bootstrapping | ||
5469 | @subsubsection Bootstrapping | ||
5470 | |||
5471 | @c %**end of header | ||
5472 | |||
5473 | For bootstrapping, it schedules a task to download the hostlist from the | ||
5474 | set of known URLs. | ||
5475 | The downloads are only performed if the number of current | ||
5476 | connections is smaller than a minimum number of connections | ||
5477 | (at the moment 4). | ||
5478 | The interval between downloads increases exponentially; however, the | ||
5479 | exponential growth is limited if it becomes longer than an hour. | ||
5480 | At that point, the frequency growth is capped at | ||
5481 | (#number of connections * 1h). | ||
5482 | |||
5483 | Once the decision has been taken to download HELLOs, the daemon chooses a | ||
5484 | random URL from the list of known URLs. URLs can be configured in the | ||
5485 | configuration or be learned from advertisement messages. | ||
5486 | The client uses a HTTP client library (libcurl) to initiate the download | ||
5487 | using the libcurl multi interface. | ||
5488 | Libcurl passes the data to the callback_download function which | ||
5489 | stores the data in a buffer if space is available and the maximum size for | ||
5490 | a hostlist download is not exceeded (MAX_BYTES_PER_HOSTLISTS = 500000). | ||
5491 | When a full HELLO was downloaded, the HOSTLIST client offers this | ||
5492 | HELLO message to the TRANSPORT service for validation. | ||
5493 | When the download is finished or failed, statistical information about the | ||
5494 | quality of this URL is updated. | ||
5495 | |||
5496 | @cindex HOSTLIST learning | ||
5497 | @node Learning | ||
5498 | @subsubsection Learning | ||
5499 | |||
5500 | @c %**end of header | ||
5501 | |||
5502 | The client also manages hostlist advertisements from other peers. The | ||
5503 | HOSTLIST daemon forwards @code{GNUNET_MESSAGE_TYPE_HOSTLIST_ADVERTISEMENT} | ||
5504 | messages to the client subsystem, which extracts the URL from the message. | ||
5505 | Next, a test of the newly obtained URL is performed by triggering a | ||
5506 | download from the new URL. If the URL works correctly, it is added to the | ||
5507 | list of working URLs. | ||
5508 | |||
5509 | The size of the list of URLs is restricted, so if an additional server is | ||
5510 | added and the list is full, the URL with the worst quality ranking | ||
5511 | (determined through successful downloads and number of HELLOs e.g.) is | ||
5512 | discarded. During shutdown the list of URLs is saved to a file for | ||
5513 | persistance and loaded on startup. URLs from the configuration file are | ||
5514 | never discarded. | ||
5515 | |||
5516 | @node Usage | ||
5517 | @subsection Usage | ||
5518 | |||
5519 | @c %**end of header | ||
5520 | |||
5521 | To start HOSTLIST by default, it has to be added to the DEFAULTSERVICES | ||
5522 | section for the ARM services. This is done in the default configuration. | ||
5523 | |||
5524 | For more information on how to configure the HOSTLIST subsystem see the | ||
5525 | installation handbook:@ | ||
5526 | Configuring the hostlist to bootstrap@ | ||
5527 | Configuring your peer to provide a hostlist | ||
5528 | |||
5529 | @cindex IDENTITY Subsystem | ||
5530 | @node IDENTITY Subsystem | ||
5531 | @section IDENTITY Subsystem | ||
5532 | |||
5533 | @c %**end of header | ||
5534 | |||
5535 | Identities of "users" in GNUnet are called egos. | ||
5536 | Egos can be used as pseudonyms ("fake names") or be tied to an | ||
5537 | organization (for example, "GNU") or even the actual identity of a human. | ||
5538 | GNUnet users are expected to have many egos. They might have one tied to | ||
5539 | their real identity, some for organizations they manage, and more for | ||
5540 | different domains where they want to operate under a pseudonym. | ||
5541 | |||
5542 | The IDENTITY service allows users to manage their egos. The identity | ||
5543 | service manages the private keys egos of the local user; it does not | ||
5544 | manage identities of other users (public keys). Public keys for other | ||
5545 | users need names to become manageable. GNUnet uses the | ||
5546 | @dfn{GNU Name System} (GNS) to give names to other users and manage their | ||
5547 | public keys securely. This chapter is about the IDENTITY service, | ||
5548 | which is about the management of private keys. | ||
5549 | |||
5550 | On the network, an ego corresponds to an ECDSA key (over Curve25519, | ||
5551 | using RFC 6979, as required by GNS). Thus, users can perform actions | ||
5552 | under a particular ego by using (signing with) a particular private key. | ||
5553 | Other users can then confirm that the action was really performed by that | ||
5554 | ego by checking the signature against the respective public key. | ||
5555 | |||
5556 | The IDENTITY service allows users to associate a human-readable name with | ||
5557 | each ego. This way, users can use names that will remind them of the | ||
5558 | purpose of a particular ego. | ||
5559 | The IDENTITY service will store the respective private keys and | ||
5560 | allows applications to access key information by name. | ||
5561 | Users can change the name that is locally (!) associated with an ego. | ||
5562 | Egos can also be deleted, which means that the private key will be removed | ||
5563 | and it thus will not be possible to perform actions with that ego in the | ||
5564 | future. | ||
5565 | |||
5566 | Additionally, the IDENTITY subsystem can associate service functions with | ||
5567 | egos. | ||
5568 | For example, GNS requires the ego that should be used for the shorten | ||
5569 | zone. GNS will ask IDENTITY for an ego for the "gns-short" service. | ||
5570 | The IDENTITY service has a mapping of such service strings to the name of | ||
5571 | the ego that the user wants to use for this service, for example | ||
5572 | "my-short-zone-ego". | ||
5573 | |||
5574 | Finally, the IDENTITY API provides access to a special ego, the | ||
5575 | anonymous ego. The anonymous ego is special in that its private key is not | ||
5576 | really private, but fixed and known to everyone. | ||
5577 | Thus, anyone can perform actions as anonymous. This can be useful as with | ||
5578 | this trick, code does not have to contain a special case to distinguish | ||
5579 | between anonymous and pseudonymous egos. | ||
5580 | |||
5581 | @menu | ||
5582 | * libgnunetidentity:: | ||
5583 | * The IDENTITY Client-Service Protocol:: | ||
5584 | @end menu | ||
5585 | |||
5586 | @cindex libgnunetidentity | ||
5587 | @node libgnunetidentity | ||
5588 | @subsection libgnunetidentity | ||
5589 | @c %**end of header | ||
5590 | |||
5591 | |||
5592 | @menu | ||
5593 | * Connecting to the service:: | ||
5594 | * Operations on Egos:: | ||
5595 | * The anonymous Ego:: | ||
5596 | * Convenience API to lookup a single ego:: | ||
5597 | * Associating egos with service functions:: | ||
5598 | @end menu | ||
5599 | |||
5600 | @node Connecting to the service | ||
5601 | @subsubsection Connecting to the service | ||
5602 | |||
5603 | @c %**end of header | ||
5604 | |||
5605 | First, typical clients connect to the identity service using | ||
5606 | @code{GNUNET_IDENTITY_connect}. This function takes a callback as a | ||
5607 | parameter. | ||
5608 | If the given callback parameter is non-null, it will be invoked to notify | ||
5609 | the application about the current state of the identities in the system. | ||
5610 | |||
5611 | @itemize @bullet | ||
5612 | @item First, it will be invoked on all known egos at the time of the | ||
5613 | connection. For each ego, a handle to the ego and the user's name for the | ||
5614 | ego will be passed to the callback. Furthermore, a @code{void **} context | ||
5615 | argument will be provided which gives the client the opportunity to | ||
5616 | associate some state with the ego. | ||
5617 | @item Second, the callback will be invoked with NULL for the ego, the name | ||
5618 | and the context. This signals that the (initial) iteration over all egos | ||
5619 | has completed. | ||
5620 | @item Then, the callback will be invoked whenever something changes about | ||
5621 | an ego. | ||
5622 | If an ego is renamed, the callback is invoked with the ego handle of the | ||
5623 | ego that was renamed, and the new name. If an ego is deleted, the callback | ||
5624 | is invoked with the ego handle and a name of NULL. In the deletion case, | ||
5625 | the application should also release resources stored in the context. | ||
5626 | @item When the application destroys the connection to the identity service | ||
5627 | using @code{GNUNET_IDENTITY_disconnect}, the callback is again invoked | ||
5628 | with the ego and a name of NULL (equivalent to deletion of the egos). | ||
5629 | This should again be used to clean up the per-ego context. | ||
5630 | @end itemize | ||
5631 | |||
5632 | The ego handle passed to the callback remains valid until the callback is | ||
5633 | invoked with a name of NULL, so it is safe to store a reference to the | ||
5634 | ego's handle. | ||
5635 | |||
5636 | @node Operations on Egos | ||
5637 | @subsubsection Operations on Egos | ||
5638 | |||
5639 | @c %**end of header | ||
5640 | |||
5641 | Given an ego handle, the main operations are to get its associated private | ||
5642 | key using @code{GNUNET_IDENTITY_ego_get_private_key} or its associated | ||
5643 | public key using @code{GNUNET_IDENTITY_ego_get_public_key}. | ||
5644 | |||
5645 | The other operations on egos are pretty straightforward. | ||
5646 | Using @code{GNUNET_IDENTITY_create}, an application can request the | ||
5647 | creation of an ego by specifying the desired name. | ||
5648 | The operation will fail if that name is | ||
5649 | already in use. Using @code{GNUNET_IDENTITY_rename} the name of an | ||
5650 | existing ego can be changed. Finally, egos can be deleted using | ||
5651 | @code{GNUNET_IDENTITY_delete}. All of these operations will trigger | ||
5652 | updates to the callback given to the @code{GNUNET_IDENTITY_connect} | ||
5653 | function of all applications that are connected with the identity service | ||
5654 | at the time. @code{GNUNET_IDENTITY_cancel} can be used to cancel the | ||
5655 | operations before the respective continuations would be called. | ||
5656 | It is not guaranteed that the operation will not be completed anyway, | ||
5657 | only the continuation will no longer be called. | ||
5658 | |||
5659 | @node The anonymous Ego | ||
5660 | @subsubsection The anonymous Ego | ||
5661 | |||
5662 | @c %**end of header | ||
5663 | |||
5664 | A special way to obtain an ego handle is to call | ||
5665 | @code{GNUNET_IDENTITY_ego_get_anonymous}, which returns an ego for the | ||
5666 | "anonymous" user --- anyone knows and can get the private key for this | ||
5667 | user, so it is suitable for operations that are supposed to be anonymous | ||
5668 | but require signatures (for example, to avoid a special path in the code). | ||
5669 | The anonymous ego is always valid and accessing it does not require a | ||
5670 | connection to the identity service. | ||
5671 | |||
5672 | @node Convenience API to lookup a single ego | ||
5673 | @subsubsection Convenience API to lookup a single ego | ||
5674 | |||
5675 | |||
5676 | As applications commonly simply have to lookup a single ego, there is a | ||
5677 | convenience API to do just that. Use @code{GNUNET_IDENTITY_ego_lookup} to | ||
5678 | lookup a single ego by name. Note that this is the user's name for the | ||
5679 | ego, not the service function. The resulting ego will be returned via a | ||
5680 | callback and will only be valid during that callback. The operation can | ||
5681 | be cancelled via @code{GNUNET_IDENTITY_ego_lookup_cancel} | ||
5682 | (cancellation is only legal before the callback is invoked). | ||
5683 | |||
5684 | @node Associating egos with service functions | ||
5685 | @subsubsection Associating egos with service functions | ||
5686 | |||
5687 | |||
5688 | The @code{GNUNET_IDENTITY_set} function is used to associate a particular | ||
5689 | ego with a service function. The name used by the service and the ego are | ||
5690 | given as arguments. | ||
5691 | Afterwards, the service can use its name to lookup the associated ego | ||
5692 | using @code{GNUNET_IDENTITY_get}. | ||
5693 | |||
5694 | @node The IDENTITY Client-Service Protocol | ||
5695 | @subsection The IDENTITY Client-Service Protocol | ||
5696 | |||
5697 | @c %**end of header | ||
5698 | |||
5699 | A client connecting to the identity service first sends a message with | ||
5700 | type | ||
5701 | @code{GNUNET_MESSAGE_TYPE_IDENTITY_START} to the service. After that, the | ||
5702 | client will receive information about changes to the egos by receiving | ||
5703 | messages of type @code{GNUNET_MESSAGE_TYPE_IDENTITY_UPDATE}. | ||
5704 | Those messages contain the private key of the ego and the user's name of | ||
5705 | the ego (or zero bytes for the name to indicate that the ego was deleted). | ||
5706 | A special bit @code{end_of_list} is used to indicate the end of the | ||
5707 | initial iteration over the identity service's egos. | ||
5708 | |||
5709 | The client can trigger changes to the egos by sending @code{CREATE}, | ||
5710 | @code{RENAME} or @code{DELETE} messages. | ||
5711 | The CREATE message contains the private key and the desired name.@ | ||
5712 | The RENAME message contains the old name and the new name.@ | ||
5713 | The DELETE message only needs to include the name of the ego to delete.@ | ||
5714 | The service responds to each of these messages with a @code{RESULT_CODE} | ||
5715 | message which indicates success or error of the operation, and possibly | ||
5716 | a human-readable error message. | ||
5717 | |||
5718 | Finally, the client can bind the name of a service function to an ego by | ||
5719 | sending a @code{SET_DEFAULT} message with the name of the service function | ||
5720 | and the private key of the ego. | ||
5721 | Such bindings can then be resolved using a @code{GET_DEFAULT} message, | ||
5722 | which includes the name of the service function. The identity service | ||
5723 | will respond to a GET_DEFAULT request with a SET_DEFAULT message | ||
5724 | containing the respective information, or with a RESULT_CODE to | ||
5725 | indicate an error. | ||
5726 | |||
5727 | @cindex NAMESTORE Subsystem | ||
5728 | @node NAMESTORE Subsystem | ||
5729 | @section NAMESTORE Subsystem | ||
5730 | |||
5731 | The NAMESTORE subsystem provides persistent storage for local GNS zone | ||
5732 | information. All local GNS zone information are managed by NAMESTORE. It | ||
5733 | provides both the functionality to administer local GNS information (e.g. | ||
5734 | delete and add records) as well as to retrieve GNS information (e.g to | ||
5735 | list name information in a client). | ||
5736 | NAMESTORE does only manage the persistent storage of zone information | ||
5737 | belonging to the user running the service: GNS information from other | ||
5738 | users obtained from the DHT are stored by the NAMECACHE subsystem. | ||
5739 | |||
5740 | NAMESTORE uses a plugin-based database backend to store GNS information | ||
5741 | with good performance. Here sqlite, MySQL and PostgreSQL are supported | ||
5742 | database backends. | ||
5743 | NAMESTORE clients interact with the IDENTITY subsystem to obtain | ||
5744 | cryptographic information about zones based on egos as described with the | ||
5745 | IDENTITY subsystem, but internally NAMESTORE refers to zones using the | ||
5746 | ECDSA private key. | ||
5747 | In addition, it collaborates with the NAMECACHE subsystem and | ||
5748 | stores zone information when local information are modified in the | ||
5749 | GNS cache to increase look-up performance for local information. | ||
5750 | |||
5751 | NAMESTORE provides functionality to look-up and store records, to iterate | ||
5752 | over a specific or all zones and to monitor zones for changes. NAMESTORE | ||
5753 | functionality can be accessed using the NAMESTORE api or the NAMESTORE | ||
5754 | command line tool. | ||
5755 | |||
5756 | @menu | ||
5757 | * libgnunetnamestore:: | ||
5758 | @end menu | ||
5759 | |||
5760 | @cindex libgnunetnamestore | ||
5761 | @node libgnunetnamestore | ||
5762 | @subsection libgnunetnamestore | ||
5763 | |||
5764 | To interact with NAMESTORE clients first connect to the NAMESTORE service | ||
5765 | using the @code{GNUNET_NAMESTORE_connect} passing a configuration handle. | ||
5766 | As a result they obtain a NAMESTORE handle, they can use for operations, | ||
5767 | or NULL is returned if the connection failed. | ||
5768 | |||
5769 | To disconnect from NAMESTORE, clients use | ||
5770 | @code{GNUNET_NAMESTORE_disconnect} and specify the handle to disconnect. | ||
5771 | |||
5772 | NAMESTORE internally uses the ECDSA private key to refer to zones. These | ||
5773 | private keys can be obtained from the IDENTITY subsytem. | ||
5774 | Here @emph{egos} @emph{can be used to refer to zones or the default ego | ||
5775 | assigned to the GNS subsystem can be used to obtained the master zone's | ||
5776 | private key.} | ||
5777 | |||
5778 | |||
5779 | @menu | ||
5780 | * Editing Zone Information:: | ||
5781 | * Iterating Zone Information:: | ||
5782 | * Monitoring Zone Information:: | ||
5783 | @end menu | ||
5784 | |||
5785 | @node Editing Zone Information | ||
5786 | @subsubsection Editing Zone Information | ||
5787 | |||
5788 | @c %**end of header | ||
5789 | |||
5790 | NAMESTORE provides functions to lookup records stored under a label in a | ||
5791 | zone and to store records under a label in a zone. | ||
5792 | |||
5793 | To store (and delete) records, the client uses the | ||
5794 | @code{GNUNET_NAMESTORE_records_store} function and has to provide | ||
5795 | namestore handle to use, the private key of the zone, the label to store | ||
5796 | the records under, the records and number of records plus an callback | ||
5797 | function. | ||
5798 | After the operation is performed NAMESTORE will call the provided | ||
5799 | callback function with the result GNUNET_SYSERR on failure | ||
5800 | (including timeout/queue drop/failure to validate), GNUNET_NO if content | ||
5801 | was already there or not found GNUNET_YES (or other positive value) on | ||
5802 | success plus an additional error message. | ||
5803 | |||
5804 | Records are deleted by using the store command with 0 records to store. | ||
5805 | It is important to note, that records are not merged when records exist | ||
5806 | with the label. | ||
5807 | So a client has first to retrieve records, merge with existing records | ||
5808 | and then store the result. | ||
5809 | |||
5810 | To perform a lookup operation, the client uses the | ||
5811 | @code{GNUNET_NAMESTORE_records_store} function. Here he has to pass the | ||
5812 | namestore handle, the private key of the zone and the label. He also has | ||
5813 | to provide a callback function which will be called with the result of | ||
5814 | the lookup operation: | ||
5815 | the zone for the records, the label, and the records including the | ||
5816 | number of records included. | ||
5817 | |||
5818 | A special operation is used to set the preferred nickname for a zone. | ||
5819 | This nickname is stored with the zone and is automatically merged with | ||
5820 | all labels and records stored in a zone. Here the client uses the | ||
5821 | @code{GNUNET_NAMESTORE_set_nick} function and passes the private key of | ||
5822 | the zone, the nickname as string plus a the callback with the result of | ||
5823 | the operation. | ||
5824 | |||
5825 | @node Iterating Zone Information | ||
5826 | @subsubsection Iterating Zone Information | ||
5827 | |||
5828 | @c %**end of header | ||
5829 | |||
5830 | A client can iterate over all information in a zone or all zones managed | ||
5831 | by NAMESTORE. | ||
5832 | Here a client uses the @code{GNUNET_NAMESTORE_zone_iteration_start} | ||
5833 | function and passes the namestore handle, the zone to iterate over and a | ||
5834 | callback function to call with the result. | ||
5835 | If the client wants to iterate over all the, he passes NULL for the zone. | ||
5836 | A @code{GNUNET_NAMESTORE_ZoneIterator} handle is returned to be used to | ||
5837 | continue iteration. | ||
5838 | |||
5839 | NAMESTORE calls the callback for every result and expects the client to | ||
5840 | call @code{GNUNET_NAMESTORE_zone_iterator_next} to continue to iterate or | ||
5841 | @code{GNUNET_NAMESTORE_zone_iterator_stop} to interrupt the iteration. | ||
5842 | When NAMESTORE reached the last item it will call the callback with a | ||
5843 | NULL value to indicate. | ||
5844 | |||
5845 | @node Monitoring Zone Information | ||
5846 | @subsubsection Monitoring Zone Information | ||
5847 | |||
5848 | @c %**end of header | ||
5849 | |||
5850 | Clients can also monitor zones to be notified about changes. Here the | ||
5851 | clients uses the @code{GNUNET_NAMESTORE_zone_monitor_start} function and | ||
5852 | passes the private key of the zone and and a callback function to call | ||
5853 | with updates for a zone. | ||
5854 | The client can specify to obtain zone information first by iterating over | ||
5855 | the zone and specify a synchronization callback to be called when the | ||
5856 | client and the namestore are synced. | ||
5857 | |||
5858 | On an update, NAMESTORE will call the callback with the private key of the | ||
5859 | zone, the label and the records and their number. | ||
5860 | |||
5861 | To stop monitoring, the client calls | ||
5862 | @code{GNUNET_NAMESTORE_zone_monitor_stop} and passes the handle obtained | ||
5863 | from the function to start the monitoring. | ||
5864 | |||
5865 | @cindex PEERINFO Subsystem | ||
5866 | @node PEERINFO Subsystem | ||
5867 | @section PEERINFO Subsystem | ||
5868 | |||
5869 | @c %**end of header | ||
5870 | |||
5871 | The PEERINFO subsystem is used to store verified (validated) information | ||
5872 | about known peers in a persistent way. It obtains these addresses for | ||
5873 | example from TRANSPORT service which is in charge of address validation. | ||
5874 | Validation means that the information in the HELLO message are checked by | ||
5875 | connecting to the addresses and performing a cryptographic handshake to | ||
5876 | authenticate the peer instance stating to be reachable with these | ||
5877 | addresses. | ||
5878 | Peerinfo does not validate the HELLO messages itself but only stores them | ||
5879 | and gives them to interested clients. | ||
5880 | |||
5881 | As future work, we think about moving from storing just HELLO messages to | ||
5882 | providing a generic persistent per-peer information store. | ||
5883 | More and more subsystems tend to need to store per-peer information in | ||
5884 | persistent way. | ||
5885 | To not duplicate this functionality we plan to provide a PEERSTORE | ||
5886 | service providing this functionality. | ||
5887 | |||
5888 | @menu | ||
5889 | * PEERINFO - Features:: | ||
5890 | * PEERINFO - Limitations:: | ||
5891 | * DeveloperPeer Information:: | ||
5892 | * Startup:: | ||
5893 | * Managing Information:: | ||
5894 | * Obtaining Information:: | ||
5895 | * The PEERINFO Client-Service Protocol:: | ||
5896 | * libgnunetpeerinfo:: | ||
5897 | @end menu | ||
5898 | |||
5899 | @node PEERINFO - Features | ||
5900 | @subsection PEERINFO - Features | ||
5901 | |||
5902 | @c %**end of header | ||
5903 | |||
5904 | @itemize @bullet | ||
5905 | @item Persistent storage | ||
5906 | @item Client notification mechanism on update | ||
5907 | @item Periodic clean up for expired information | ||
5908 | @item Differentiation between public and friend-only HELLO | ||
5909 | @end itemize | ||
5910 | |||
5911 | @node PEERINFO - Limitations | ||
5912 | @subsection PEERINFO - Limitations | ||
5913 | |||
5914 | |||
5915 | @itemize @bullet | ||
5916 | @item Does not perform HELLO validation | ||
5917 | @end itemize | ||
5918 | |||
5919 | @node DeveloperPeer Information | ||
5920 | @subsection DeveloperPeer Information | ||
5921 | |||
5922 | @c %**end of header | ||
5923 | |||
5924 | The PEERINFO subsystem stores these information in the form of HELLO | ||
5925 | messages you can think of as business cards. | ||
5926 | These HELLO messages contain the public key of a peer and the addresses | ||
5927 | a peer can be reached under. | ||
5928 | The addresses include an expiration date describing how long they are | ||
5929 | valid. This information is updated regularly by the TRANSPORT service by | ||
5930 | revalidating the address. | ||
5931 | If an address is expired and not renewed, it can be removed from the | ||
5932 | HELLO message. | ||
5933 | |||
5934 | Some peer do not want to have their HELLO messages distributed to other | ||
5935 | peers, especially when GNUnet's friend-to-friend modus is enabled. | ||
5936 | To prevent this undesired distribution. PEERINFO distinguishes between | ||
5937 | @emph{public} and @emph{friend-only} HELLO messages. | ||
5938 | Public HELLO messages can be freely distributed to other (possibly | ||
5939 | unknown) peers (for example using the hostlist, gossiping, broadcasting), | ||
5940 | whereas friend-only HELLO messages may not be distributed to other peers. | ||
5941 | Friend-only HELLO messages have an additional flag @code{friend_only} set | ||
5942 | internally. For public HELLO message this flag is not set. | ||
5943 | PEERINFO does and cannot not check if a client is allowed to obtain a | ||
5944 | specific HELLO type. | ||
5945 | |||
5946 | The HELLO messages can be managed using the GNUnet HELLO library. | ||
5947 | Other GNUnet systems can obtain these information from PEERINFO and use | ||
5948 | it for their purposes. | ||
5949 | Clients are for example the HOSTLIST component providing these | ||
5950 | information to other peers in form of a hostlist or the TRANSPORT | ||
5951 | subsystem using these information to maintain connections to other peers. | ||
5952 | |||
5953 | @node Startup | ||
5954 | @subsection Startup | ||
5955 | |||
5956 | @c %**end of header | ||
5957 | |||
5958 | During startup the PEERINFO services loads persistent HELLOs from disk. | ||
5959 | First PEERINFO parses the directory configured in the HOSTS value of the | ||
5960 | @code{PEERINFO} configuration section to store PEERINFO information. | ||
5961 | For all files found in this directory valid HELLO messages are extracted. | ||
5962 | In addition it loads HELLO messages shipped with the GNUnet distribution. | ||
5963 | These HELLOs are used to simplify network bootstrapping by providing | ||
5964 | valid peer information with the distribution. | ||
5965 | The use of these HELLOs can be prevented by setting the | ||
5966 | @code{USE_INCLUDED_HELLOS} in the @code{PEERINFO} configuration section to | ||
5967 | @code{NO}. Files containing invalid information are removed. | ||
5968 | |||
5969 | @node Managing Information | ||
5970 | @subsection Managing Information | ||
5971 | |||
5972 | @c %**end of header | ||
5973 | |||
5974 | The PEERINFO services stores information about known PEERS and a single | ||
5975 | HELLO message for every peer. | ||
5976 | A peer does not need to have a HELLO if no information are available. | ||
5977 | HELLO information from different sources, for example a HELLO obtained | ||
5978 | from a remote HOSTLIST and a second HELLO stored on disk, are combined | ||
5979 | and merged into one single HELLO message per peer which will be given to | ||
5980 | clients. During this merge process the HELLO is immediately written to | ||
5981 | disk to ensure persistence. | ||
5982 | |||
5983 | PEERINFO in addition periodically scans the directory where information | ||
5984 | are stored for empty HELLO messages with expired TRANSPORT addresses. | ||
5985 | This periodic task scans all files in the directory and recreates the | ||
5986 | HELLO messages it finds. | ||
5987 | Expired TRANSPORT addresses are removed from the HELLO and if the | ||
5988 | HELLO does not contain any valid addresses, it is discarded and removed | ||
5989 | from the disk. | ||
5990 | |||
5991 | @node Obtaining Information | ||
5992 | @subsection Obtaining Information | ||
5993 | |||
5994 | @c %**end of header | ||
5995 | |||
5996 | When a client requests information from PEERINFO, PEERINFO performs a | ||
5997 | lookup for the respective peer or all peers if desired and transmits this | ||
5998 | information to the client. | ||
5999 | The client can specify if friend-only HELLOs have to be included or not | ||
6000 | and PEERINFO filters the respective HELLO messages before transmitting | ||
6001 | information. | ||
6002 | |||
6003 | To notify clients about changes to PEERINFO information, PEERINFO | ||
6004 | maintains a list of clients interested in this notifications. | ||
6005 | Such a notification occurs if a HELLO for a peer was updated (due to a | ||
6006 | merge for example) or a new peer was added. | ||
6007 | |||
6008 | @node The PEERINFO Client-Service Protocol | ||
6009 | @subsection The PEERINFO Client-Service Protocol | ||
6010 | |||
6011 | @c %**end of header | ||
6012 | |||
6013 | To connect and disconnect to and from the PEERINFO Service PEERINFO | ||
6014 | utilizes the util client/server infrastructure, so no special messages | ||
6015 | types are used here. | ||
6016 | |||
6017 | To add information for a peer, the plain HELLO message is transmitted to | ||
6018 | the service without any wrapping. All pieces of information required are | ||
6019 | stored within the HELLO message. | ||
6020 | The PEERINFO service provides a message handler accepting and processing | ||
6021 | these HELLO messages. | ||
6022 | |||
6023 | When obtaining PEERINFO information using the iterate functionality | ||
6024 | specific messages are used. To obtain information for all peers, a | ||
6025 | @code{struct ListAllPeersMessage} with message type | ||
6026 | @code{GNUNET_MESSAGE_TYPE_PEERINFO_GET_ALL} and a flag | ||
6027 | include_friend_only to indicate if friend-only HELLO messages should be | ||
6028 | included are transmitted. If information for a specific peer is required | ||
6029 | a @code{struct ListAllPeersMessage} with | ||
6030 | @code{GNUNET_MESSAGE_TYPE_PEERINFO_GET} containing the peer identity is | ||
6031 | used. | ||
6032 | |||
6033 | For both variants the PEERINFO service replies for each HELLO message it | ||
6034 | wants to transmit with a @code{struct ListAllPeersMessage} with type | ||
6035 | @code{GNUNET_MESSAGE_TYPE_PEERINFO_INFO} containing the plain HELLO. | ||
6036 | The final message is @code{struct GNUNET_MessageHeader} with type | ||
6037 | @code{GNUNET_MESSAGE_TYPE_PEERINFO_INFO}. If the client receives this | ||
6038 | message, it can proceed with the next request if any is pending. | ||
6039 | |||
6040 | @node libgnunetpeerinfo | ||
6041 | @subsection libgnunetpeerinfo | ||
6042 | |||
6043 | @c %**end of header | ||
6044 | |||
6045 | The PEERINFO API consists mainly of three different functionalities: | ||
6046 | |||
6047 | @itemize @bullet | ||
6048 | @item maintaining a connection to the service | ||
6049 | @item adding new information to the PEERINFO service | ||
6050 | @item retrieving information from the PEERINFO service | ||
6051 | @end itemize | ||
6052 | |||
6053 | @menu | ||
6054 | * Connecting to the PEERINFO Service:: | ||
6055 | * Adding Information to the PEERINFO Service:: | ||
6056 | * Obtaining Information from the PEERINFO Service:: | ||
6057 | @end menu | ||
6058 | |||
6059 | @node Connecting to the PEERINFO Service | ||
6060 | @subsubsection Connecting to the PEERINFO Service | ||
6061 | |||
6062 | @c %**end of header | ||
6063 | |||
6064 | To connect to the PEERINFO service the function | ||
6065 | @code{GNUNET_PEERINFO_connect} is used, taking a configuration handle as | ||
6066 | an argument, and to disconnect from PEERINFO the function | ||
6067 | @code{GNUNET_PEERINFO_disconnect}, taking the PEERINFO | ||
6068 | handle returned from the connect function has to be called. | ||
6069 | |||
6070 | @node Adding Information to the PEERINFO Service | ||
6071 | @subsubsection Adding Information to the PEERINFO Service | ||
6072 | |||
6073 | @c %**end of header | ||
6074 | |||
6075 | @code{GNUNET_PEERINFO_add_peer} adds a new peer to the PEERINFO subsystem | ||
6076 | storage. This function takes the PEERINFO handle as an argument, the HELLO | ||
6077 | message to store and a continuation with a closure to be called with the | ||
6078 | result of the operation. | ||
6079 | The @code{GNUNET_PEERINFO_add_peer} returns a handle to this operation | ||
6080 | allowing to cancel the operation with the respective cancel function | ||
6081 | @code{GNUNET_PEERINFO_add_peer_cancel}. To retrieve information from | ||
6082 | PEERINFO you can iterate over all information stored with PEERINFO or you | ||
6083 | can tell PEERINFO to notify if new peer information are available. | ||
6084 | |||
6085 | @node Obtaining Information from the PEERINFO Service | ||
6086 | @subsubsection Obtaining Information from the PEERINFO Service | ||
6087 | |||
6088 | @c %**end of header | ||
6089 | |||
6090 | To iterate over information in PEERINFO you use | ||
6091 | @code{GNUNET_PEERINFO_iterate}. | ||
6092 | This function expects the PEERINFO handle, a flag if HELLO messages | ||
6093 | intended for friend only mode should be included, a timeout how long the | ||
6094 | operation should take and a callback with a callback closure to be called | ||
6095 | for the results. | ||
6096 | If you want to obtain information for a specific peer, you can specify | ||
6097 | the peer identity, if this identity is NULL, information for all peers are | ||
6098 | returned. The function returns a handle to allow to cancel the operation | ||
6099 | using @code{GNUNET_PEERINFO_iterate_cancel}. | ||
6100 | |||
6101 | To get notified when peer information changes, you can use | ||
6102 | @code{GNUNET_PEERINFO_notify}. | ||
6103 | This function expects a configuration handle and a flag if friend-only | ||
6104 | HELLO messages should be included. The PEERINFO service will notify you | ||
6105 | about every change and the callback function will be called to notify you | ||
6106 | about changes. The function returns a handle to cancel notifications | ||
6107 | with @code{GNUNET_PEERINFO_notify_cancel}. | ||
6108 | |||
6109 | @cindex PEERSTORE Subsystem | ||
6110 | @node PEERSTORE Subsystem | ||
6111 | @section PEERSTORE Subsystem | ||
6112 | |||
6113 | @c %**end of header | ||
6114 | |||
6115 | GNUnet's PEERSTORE subsystem offers persistent per-peer storage for other | ||
6116 | GNUnet subsystems. GNUnet subsystems can use PEERSTORE to persistently | ||
6117 | store and retrieve arbitrary data. | ||
6118 | Each data record stored with PEERSTORE contains the following fields: | ||
6119 | |||
6120 | @itemize @bullet | ||
6121 | @item subsystem: Name of the subsystem responsible for the record. | ||
6122 | @item peerid: Identity of the peer this record is related to. | ||
6123 | @item key: a key string identifying the record. | ||
6124 | @item value: binary record value. | ||
6125 | @item expiry: record expiry date. | ||
6126 | @end itemize | ||
6127 | |||
6128 | @menu | ||
6129 | * Functionality:: | ||
6130 | * Architecture:: | ||
6131 | * libgnunetpeerstore:: | ||
6132 | @end menu | ||
6133 | |||
6134 | @node Functionality | ||
6135 | @subsection Functionality | ||
6136 | |||
6137 | @c %**end of header | ||
6138 | |||
6139 | Subsystems can store any type of value under a (subsystem, peerid, key) | ||
6140 | combination. A "replace" flag set during store operations forces the | ||
6141 | PEERSTORE to replace any old values stored under the same | ||
6142 | (subsystem, peerid, key) combination with the new value. | ||
6143 | Additionally, an expiry date is set after which the record is *possibly* | ||
6144 | deleted by PEERSTORE. | ||
6145 | |||
6146 | Subsystems can iterate over all values stored under any of the following | ||
6147 | combination of fields: | ||
6148 | |||
6149 | @itemize @bullet | ||
6150 | @item (subsystem) | ||
6151 | @item (subsystem, peerid) | ||
6152 | @item (subsystem, key) | ||
6153 | @item (subsystem, peerid, key) | ||
6154 | @end itemize | ||
6155 | |||
6156 | Subsystems can also request to be notified about any new values stored | ||
6157 | under a (subsystem, peerid, key) combination by sending a "watch" | ||
6158 | request to PEERSTORE. | ||
6159 | |||
6160 | @node Architecture | ||
6161 | @subsection Architecture | ||
6162 | |||
6163 | @c %**end of header | ||
6164 | |||
6165 | PEERSTORE implements the following components: | ||
6166 | |||
6167 | @itemize @bullet | ||
6168 | @item PEERSTORE service: Handles store, iterate and watch operations. | ||
6169 | @item PEERSTORE API: API to be used by other subsystems to communicate and | ||
6170 | issue commands to the PEERSTORE service. | ||
6171 | @item PEERSTORE plugins: Handles the persistent storage. At the moment, | ||
6172 | only an "sqlite" plugin is implemented. | ||
6173 | @end itemize | ||
6174 | |||
6175 | @cindex libgnunetpeerstore | ||
6176 | @node libgnunetpeerstore | ||
6177 | @subsection libgnunetpeerstore | ||
6178 | |||
6179 | @c %**end of header | ||
6180 | |||
6181 | libgnunetpeerstore is the library containing the PEERSTORE API. Subsystems | ||
6182 | wishing to communicate with the PEERSTORE service use this API to open a | ||
6183 | connection to PEERSTORE. This is done by calling | ||
6184 | @code{GNUNET_PEERSTORE_connect} which returns a handle to the newly | ||
6185 | created connection. | ||
6186 | This handle has to be used with any further calls to the API. | ||
6187 | |||
6188 | To store a new record, the function @code{GNUNET_PEERSTORE_store} is to | ||
6189 | be used which requires the record fields and a continuation function that | ||
6190 | will be called by the API after the STORE request is sent to the | ||
6191 | PEERSTORE service. | ||
6192 | Note that calling the continuation function does not mean that the record | ||
6193 | is successfully stored, only that the STORE request has been successfully | ||
6194 | sent to the PEERSTORE service. | ||
6195 | @code{GNUNET_PEERSTORE_store_cancel} can be called to cancel the STORE | ||
6196 | request only before the continuation function has been called. | ||
6197 | |||
6198 | To iterate over stored records, the function | ||
6199 | @code{GNUNET_PEERSTORE_iterate} is | ||
6200 | to be used. @emph{peerid} and @emph{key} can be set to NULL. An iterator | ||
6201 | callback function will be called with each matching record found and a | ||
6202 | NULL record at the end to signal the end of result set. | ||
6203 | @code{GNUNET_PEERSTORE_iterate_cancel} can be used to cancel the ITERATE | ||
6204 | request before the iterator callback is called with a NULL record. | ||
6205 | |||
6206 | To be notified with new values stored under a (subsystem, peerid, key) | ||
6207 | combination, the function @code{GNUNET_PEERSTORE_watch} is to be used. | ||
6208 | This will register the watcher with the PEERSTORE service, any new | ||
6209 | records matching the given combination will trigger the callback | ||
6210 | function passed to @code{GNUNET_PEERSTORE_watch}. This continues until | ||
6211 | @code{GNUNET_PEERSTORE_watch_cancel} is called or the connection to the | ||
6212 | service is destroyed. | ||
6213 | |||
6214 | After the connection is no longer needed, the function | ||
6215 | @code{GNUNET_PEERSTORE_disconnect} can be called to disconnect from the | ||
6216 | PEERSTORE service. | ||
6217 | Any pending ITERATE or WATCH requests will be destroyed. | ||
6218 | If the @code{sync_first} flag is set to @code{GNUNET_YES}, the API will | ||
6219 | delay the disconnection until all pending STORE requests are sent to | ||
6220 | the PEERSTORE service, otherwise, the pending STORE requests will be | ||
6221 | destroyed as well. | ||
6222 | |||
6223 | @cindex SET Subsystem | ||
6224 | @node SET Subsystem | ||
6225 | @section SET Subsystem | ||
6226 | |||
6227 | @c %**end of header | ||
6228 | |||
6229 | The SET service implements efficient set operations between two peers | ||
6230 | over a mesh tunnel. | ||
6231 | Currently, set union and set intersection are the only supported | ||
6232 | operations. Elements of a set consist of an @emph{element type} and | ||
6233 | arbitrary binary @emph{data}. | ||
6234 | The size of an element's data is limited to around 62 KB. | ||
6235 | |||
6236 | @menu | ||
6237 | * Local Sets:: | ||
6238 | * Set Modifications:: | ||
6239 | * Set Operations:: | ||
6240 | * Result Elements:: | ||
6241 | * libgnunetset:: | ||
6242 | * The SET Client-Service Protocol:: | ||
6243 | * The SET Intersection Peer-to-Peer Protocol:: | ||
6244 | * The SET Union Peer-to-Peer Protocol:: | ||
6245 | @end menu | ||
6246 | |||
6247 | @node Local Sets | ||
6248 | @subsection Local Sets | ||
6249 | |||
6250 | @c %**end of header | ||
6251 | |||
6252 | Sets created by a local client can be modified and reused for multiple | ||
6253 | operations. As each set operation requires potentially expensive special | ||
6254 | auxilliary data to be computed for each element of a set, a set can only | ||
6255 | participate in one type of set operation (i.e. union or intersection). | ||
6256 | The type of a set is determined upon its creation. | ||
6257 | If a the elements of a set are needed for an operation of a different | ||
6258 | type, all of the set's element must be copied to a new set of appropriate | ||
6259 | type. | ||
6260 | |||
6261 | @node Set Modifications | ||
6262 | @subsection Set Modifications | ||
6263 | |||
6264 | @c %**end of header | ||
6265 | |||
6266 | Even when set operations are active, one can add to and remove elements | ||
6267 | from a set. | ||
6268 | However, these changes will only be visible to operations that have been | ||
6269 | created after the changes have taken place. That is, every set operation | ||
6270 | only sees a snapshot of the set from the time the operation was started. | ||
6271 | This mechanism is @emph{not} implemented by copying the whole set, but by | ||
6272 | attaching @emph{generation information} to each element and operation. | ||
6273 | |||
6274 | @node Set Operations | ||
6275 | @subsection Set Operations | ||
6276 | |||
6277 | @c %**end of header | ||
6278 | |||
6279 | Set operations can be started in two ways: Either by accepting an | ||
6280 | operation request from a remote peer, or by requesting a set operation | ||
6281 | from a remote peer. | ||
6282 | Set operations are uniquely identified by the involved @emph{peers}, an | ||
6283 | @emph{application id} and the @emph{operation type}. | ||
6284 | |||
6285 | The client is notified of incoming set operations by @emph{set listeners}. | ||
6286 | A set listener listens for incoming operations of a specific operation | ||
6287 | type and application id. | ||
6288 | Once notified of an incoming set request, the client can accept the set | ||
6289 | request (providing a local set for the operation) or reject it. | ||
6290 | |||
6291 | @node Result Elements | ||
6292 | @subsection Result Elements | ||
6293 | |||
6294 | @c %**end of header | ||
6295 | |||
6296 | The SET service has three @emph{result modes} that determine how an | ||
6297 | operation's result set is delivered to the client: | ||
6298 | |||
6299 | @itemize @bullet | ||
6300 | @item @strong{Full Result Set.} All elements of set resulting from the set | ||
6301 | operation are returned to the client. | ||
6302 | @item @strong{Added Elements.} Only elements that result from the | ||
6303 | operation and are not already in the local peer's set are returned. | ||
6304 | Note that for some operations (like set intersection) this result mode | ||
6305 | will never return any elements. | ||
6306 | This can be useful if only the remove peer is actually interested in | ||
6307 | the result of the set operation. | ||
6308 | @item @strong{Removed Elements.} Only elements that are in the local | ||
6309 | peer's initial set but not in the operation's result set are returned. | ||
6310 | Note that for some operations (like set union) this result mode will | ||
6311 | never return any elements. This can be useful if only the remove peer is | ||
6312 | actually interested in the result of the set operation. | ||
6313 | @end itemize | ||
6314 | |||
6315 | @cindex libgnunetset | ||
6316 | @node libgnunetset | ||
6317 | @subsection libgnunetset | ||
6318 | |||
6319 | @c %**end of header | ||
6320 | |||
6321 | @menu | ||
6322 | * Sets:: | ||
6323 | * Listeners:: | ||
6324 | * Operations:: | ||
6325 | * Supplying a Set:: | ||
6326 | * The Result Callback:: | ||
6327 | @end menu | ||
6328 | |||
6329 | @node Sets | ||
6330 | @subsubsection Sets | ||
6331 | |||
6332 | @c %**end of header | ||
6333 | |||
6334 | New sets are created with @code{GNUNET_SET_create}. Both the local peer's | ||
6335 | configuration (as each set has its own client connection) and the | ||
6336 | operation type must be specified. | ||
6337 | The set exists until either the client calls @code{GNUNET_SET_destroy} or | ||
6338 | the client's connection to the service is disrupted. | ||
6339 | In the latter case, the client is notified by the return value of | ||
6340 | functions dealing with sets. This return value must always be checked. | ||
6341 | |||
6342 | Elements are added and removed with @code{GNUNET_SET_add_element} and | ||
6343 | @code{GNUNET_SET_remove_element}. | ||
6344 | |||
6345 | @node Listeners | ||
6346 | @subsubsection Listeners | ||
6347 | |||
6348 | @c %**end of header | ||
6349 | |||
6350 | Listeners are created with @code{GNUNET_SET_listen}. Each time time a | ||
6351 | remote peer suggests a set operation with an application id and operation | ||
6352 | type matching a listener, the listener's callback is invoked. | ||
6353 | The client then must synchronously call either @code{GNUNET_SET_accept} | ||
6354 | or @code{GNUNET_SET_reject}. Note that the operation will not be started | ||
6355 | until the client calls @code{GNUNET_SET_commit} | ||
6356 | (see Section "Supplying a Set"). | ||
6357 | |||
6358 | @node Operations | ||
6359 | @subsubsection Operations | ||
6360 | |||
6361 | @c %**end of header | ||
6362 | |||
6363 | Operations to be initiated by the local peer are created with | ||
6364 | @code{GNUNET_SET_prepare}. Note that the operation will not be started | ||
6365 | until the client calls @code{GNUNET_SET_commit} | ||
6366 | (see Section "Supplying a Set"). | ||
6367 | |||
6368 | @node Supplying a Set | ||
6369 | @subsubsection Supplying a Set | ||
6370 | |||
6371 | @c %**end of header | ||
6372 | |||
6373 | To create symmetry between the two ways of starting a set operation | ||
6374 | (accepting and nitiating it), the operation handles returned by | ||
6375 | @code{GNUNET_SET_accept} and @code{GNUNET_SET_prepare} do not yet have a | ||
6376 | set to operate on, thus they can not do any work yet. | ||
6377 | |||
6378 | The client must call @code{GNUNET_SET_commit} to specify a set to use for | ||
6379 | an operation. @code{GNUNET_SET_commit} may only be called once per set | ||
6380 | operation. | ||
6381 | |||
6382 | @node The Result Callback | ||
6383 | @subsubsection The Result Callback | ||
6384 | |||
6385 | @c %**end of header | ||
6386 | |||
6387 | Clients must specify both a result mode and a result callback with | ||
6388 | @code{GNUNET_SET_accept} and @code{GNUNET_SET_prepare}. The result | ||
6389 | callback with a status indicating either that an element was received, or | ||
6390 | the operation failed or succeeded. | ||
6391 | The interpretation of the received element depends on the result mode. | ||
6392 | The callback needs to know which result mode it is used in, as the | ||
6393 | arguments do not indicate if an element is part of the full result set, | ||
6394 | or if it is in the difference between the original set and the final set. | ||
6395 | |||
6396 | @node The SET Client-Service Protocol | ||
6397 | @subsection The SET Client-Service Protocol | ||
6398 | |||
6399 | @c %**end of header | ||
6400 | |||
6401 | @menu | ||
6402 | * Creating Sets:: | ||
6403 | * Listeners2:: | ||
6404 | * Initiating Operations:: | ||
6405 | * Modifying Sets:: | ||
6406 | * Results and Operation Status:: | ||
6407 | * Iterating Sets:: | ||
6408 | @end menu | ||
6409 | |||
6410 | @node Creating Sets | ||
6411 | @subsubsection Creating Sets | ||
6412 | |||
6413 | @c %**end of header | ||
6414 | |||
6415 | For each set of a client, there exists a client connection to the service. | ||
6416 | Sets are created by sending the @code{GNUNET_SERVICE_SET_CREATE} message | ||
6417 | over a new client connection. Multiple operations for one set are | ||
6418 | multiplexed over one client connection, using a request id supplied by | ||
6419 | the client. | ||
6420 | |||
6421 | @node Listeners2 | ||
6422 | @subsubsection Listeners2 | ||
6423 | |||
6424 | @c %**end of header | ||
6425 | |||
6426 | Each listener also requires a seperate client connection. By sending the | ||
6427 | @code{GNUNET_SERVICE_SET_LISTEN} message, the client notifies the service | ||
6428 | of the application id and operation type it is interested in. A client | ||
6429 | rejects an incoming request by sending @code{GNUNET_SERVICE_SET_REJECT} | ||
6430 | on the listener's client connection. | ||
6431 | In contrast, when accepting an incoming request, a | ||
6432 | @code{GNUNET_SERVICE_SET_ACCEPT} message must be sent over the@ set that | ||
6433 | is supplied for the set operation. | ||
6434 | |||
6435 | @node Initiating Operations | ||
6436 | @subsubsection Initiating Operations | ||
6437 | |||
6438 | @c %**end of header | ||
6439 | |||
6440 | Operations with remote peers are initiated by sending a | ||
6441 | @code{GNUNET_SERVICE_SET_EVALUATE} message to the service. The@ client | ||
6442 | connection that this message is sent by determines the set to use. | ||
6443 | |||
6444 | @node Modifying Sets | ||
6445 | @subsubsection Modifying Sets | ||
6446 | |||
6447 | @c %**end of header | ||
6448 | |||
6449 | Sets are modified with the @code{GNUNET_SERVICE_SET_ADD} and | ||
6450 | @code{GNUNET_SERVICE_SET_REMOVE} messages. | ||
6451 | |||
6452 | |||
6453 | @c %@menu | ||
6454 | @c %* Results and Operation Status:: | ||
6455 | @c %* Iterating Sets:: | ||
6456 | @c %@end menu | ||
6457 | |||
6458 | @node Results and Operation Status | ||
6459 | @subsubsection Results and Operation Status | ||
6460 | @c %**end of header | ||
6461 | |||
6462 | The service notifies the client of result elements and success/failure of | ||
6463 | a set operation with the @code{GNUNET_SERVICE_SET_RESULT} message. | ||
6464 | |||
6465 | @node Iterating Sets | ||
6466 | @subsubsection Iterating Sets | ||
6467 | |||
6468 | @c %**end of header | ||
6469 | |||
6470 | All elements of a set can be requested by sending | ||
6471 | @code{GNUNET_SERVICE_SET_ITER_REQUEST}. The server responds with | ||
6472 | @code{GNUNET_SERVICE_SET_ITER_ELEMENT} and eventually terminates the | ||
6473 | iteration with @code{GNUNET_SERVICE_SET_ITER_DONE}. | ||
6474 | After each received element, the client | ||
6475 | must send @code{GNUNET_SERVICE_SET_ITER_ACK}. Note that only one set | ||
6476 | iteration may be active for a set at any given time. | ||
6477 | |||
6478 | @node The SET Intersection Peer-to-Peer Protocol | ||
6479 | @subsection The SET Intersection Peer-to-Peer Protocol | ||
6480 | |||
6481 | @c %**end of header | ||
6482 | |||
6483 | The intersection protocol operates over CADET and starts with a | ||
6484 | GNUNET_MESSAGE_TYPE_SET_P2P_OPERATION_REQUEST being sent by the peer | ||
6485 | initiating the operation to the peer listening for inbound requests. | ||
6486 | It includes the number of elements of the initiating peer, which is used | ||
6487 | to decide which side will send a Bloom filter first. | ||
6488 | |||
6489 | The listening peer checks if the operation type and application | ||
6490 | identifier are acceptable for its current state. | ||
6491 | If not, it responds with a GNUNET_MESSAGE_TYPE_SET_RESULT and a status of | ||
6492 | GNUNET_SET_STATUS_FAILURE (and terminates the CADET channel). | ||
6493 | |||
6494 | If the application accepts the request, the listener sends back a | ||
6495 | @code{GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_ELEMENT_INFO} if it has | ||
6496 | more elements in the set than the client. | ||
6497 | Otherwise, it immediately starts with the Bloom filter exchange. | ||
6498 | If the initiator receives a | ||
6499 | @code{GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_ELEMENT_INFO} response, | ||
6500 | it beings the Bloom filter exchange, unless the set size is indicated to | ||
6501 | be zero, in which case the intersection is considered finished after | ||
6502 | just the initial handshake. | ||
6503 | |||
6504 | |||
6505 | @menu | ||
6506 | * The Bloom filter exchange:: | ||
6507 | * Salt:: | ||
6508 | @end menu | ||
6509 | |||
6510 | @node The Bloom filter exchange | ||
6511 | @subsubsection The Bloom filter exchange | ||
6512 | |||
6513 | @c %**end of header | ||
6514 | |||
6515 | In this phase, each peer transmits a Bloom filter over the remaining | ||
6516 | keys of the local set to the other peer using a | ||
6517 | @code{GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_BF} message. This | ||
6518 | message additionally includes the number of elements left in the sender's | ||
6519 | set, as well as the XOR over all of the keys in that set. | ||
6520 | |||
6521 | The number of bits 'k' set per element in the Bloom filter is calculated | ||
6522 | based on the relative size of the two sets. | ||
6523 | Furthermore, the size of the Bloom filter is calculated based on 'k' and | ||
6524 | the number of elements in the set to maximize the amount of data filtered | ||
6525 | per byte transmitted on the wire (while avoiding an excessively high | ||
6526 | number of iterations). | ||
6527 | |||
6528 | The receiver of the message removes all elements from its local set that | ||
6529 | do not pass the Bloom filter test. | ||
6530 | It then checks if the set size of the sender and the XOR over the keys | ||
6531 | match what is left of his own set. If they do, he sends a | ||
6532 | @code{GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_DONE} back to indicate | ||
6533 | that the latest set is the final result. | ||
6534 | Otherwise, the receiver starts another Bloom fitler exchange, except | ||
6535 | this time as the sender. | ||
6536 | |||
6537 | @node Salt | ||
6538 | @subsubsection Salt | ||
6539 | |||
6540 | @c %**end of header | ||
6541 | |||
6542 | Bloomfilter operations are probablistic: With some non-zero probability | ||
6543 | the test may incorrectly say an element is in the set, even though it is | ||
6544 | not. | ||
6545 | |||
6546 | To mitigate this problem, the intersection protocol iterates exchanging | ||
6547 | Bloom filters using a different random 32-bit salt in each iteration (the | ||
6548 | salt is also included in the message). | ||
6549 | With different salts, set operations may fail for different elements. | ||
6550 | Merging the results from the executions, the probability of failure drops | ||
6551 | to zero. | ||
6552 | |||
6553 | The iterations terminate once both peers have established that they have | ||
6554 | sets of the same size, and where the XOR over all keys computes the same | ||
6555 | 512-bit value (leaving a failure probability of 2-511). | ||
6556 | |||
6557 | @node The SET Union Peer-to-Peer Protocol | ||
6558 | @subsection The SET Union Peer-to-Peer Protocol | ||
6559 | |||
6560 | @c %**end of header | ||
6561 | |||
6562 | The SET union protocol is based on Eppstein's efficient set reconciliation | ||
6563 | without prior context. You should read this paper first if you want to | ||
6564 | understand the protocol. | ||
6565 | |||
6566 | The union protocol operates over CADET and starts with a | ||
6567 | GNUNET_MESSAGE_TYPE_SET_P2P_OPERATION_REQUEST being sent by the peer | ||
6568 | initiating the operation to the peer listening for inbound requests. | ||
6569 | It includes the number of elements of the initiating peer, which is | ||
6570 | currently not used. | ||
6571 | |||
6572 | The listening peer checks if the operation type and application | ||
6573 | identifier are acceptable for its current state. If not, it responds with | ||
6574 | a @code{GNUNET_MESSAGE_TYPE_SET_RESULT} and a status of | ||
6575 | @code{GNUNET_SET_STATUS_FAILURE} (and terminates the CADET channel). | ||
6576 | |||
6577 | If the application accepts the request, it sends back a strata estimator | ||
6578 | using a message of type GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SE. The | ||
6579 | initiator evaluates the strata estimator and initiates the exchange of | ||
6580 | invertible Bloom filters, sending a GNUNET_MESSAGE_TYPE_SET_UNION_P2P_IBF. | ||
6581 | |||
6582 | During the IBF exchange, if the receiver cannot invert the Bloom filter or | ||
6583 | detects a cycle, it sends a larger IBF in response (up to a defined | ||
6584 | maximum limit; if that limit is reached, the operation fails). | ||
6585 | Elements decoded while processing the IBF are transmitted to the other | ||
6586 | peer using GNUNET_MESSAGE_TYPE_SET_P2P_ELEMENTS, or requested from the | ||
6587 | other peer using GNUNET_MESSAGE_TYPE_SET_P2P_ELEMENT_REQUESTS messages, | ||
6588 | depending on the sign observed during decoding of the IBF. | ||
6589 | Peers respond to a GNUNET_MESSAGE_TYPE_SET_P2P_ELEMENT_REQUESTS message | ||
6590 | with the respective element in a GNUNET_MESSAGE_TYPE_SET_P2P_ELEMENTS | ||
6591 | message. If the IBF fully decodes, the peer responds with a | ||
6592 | GNUNET_MESSAGE_TYPE_SET_UNION_P2P_DONE message instead of another | ||
6593 | GNUNET_MESSAGE_TYPE_SET_UNION_P2P_IBF. | ||
6594 | |||
6595 | All Bloom filter operations use a salt to mingle keys before hasing them | ||
6596 | into buckets, such that future iterations have a fresh chance of | ||
6597 | succeeding if they failed due to collisions before. | ||
6598 | |||
6599 | @cindex STATISTICS Subsystem | ||
6600 | @node STATISTICS Subsystem | ||
6601 | @section STATISTICS Subsystem | ||
6602 | |||
6603 | @c %**end of header | ||
6604 | |||
6605 | In GNUnet, the STATISTICS subsystem offers a central place for all | ||
6606 | subsystems to publish unsigned 64-bit integer run-time statistics. | ||
6607 | Keeping this information centrally means that there is a unified way for | ||
6608 | the user to obtain data on all subsystems, and individual subsystems do | ||
6609 | not have to always include a custom data export method for performance | ||
6610 | metrics and other statistics. For example, the TRANSPORT system uses | ||
6611 | STATISTICS to update information about the number of directly connected | ||
6612 | peers and the bandwidth that has been consumed by the various plugins. | ||
6613 | This information is valuable for diagnosing connectivity and performance | ||
6614 | issues. | ||
6615 | |||
6616 | Following the GNUnet service architecture, the STATISTICS subsystem is | ||
6617 | divided into an API which is exposed through the header | ||
6618 | @strong{gnunet_statistics_service.h} and the STATISTICS service | ||
6619 | @strong{gnunet-service-statistics}. The @strong{gnunet-statistics} | ||
6620 | command-line tool can be used to obtain (and change) information about | ||
6621 | the values stored by the STATISTICS service. The STATISTICS service does | ||
6622 | not communicate with other peers. | ||
6623 | |||
6624 | Data is stored in the STATISTICS service in the form of tuples | ||
6625 | @strong{(subsystem, name, value, persistence)}. The subsystem determines | ||
6626 | to which other GNUnet's subsystem the data belongs. name is the name | ||
6627 | through which value is associated. It uniquely identifies the record | ||
6628 | from among other records belonging to the same subsystem. | ||
6629 | In some parts of the code, the pair @strong{(subsystem, name)} is called | ||
6630 | a @strong{statistic} as it identifies the values stored in the STATISTCS | ||
6631 | service.The persistence flag determines if the record has to be preserved | ||
6632 | across service restarts. A record is said to be persistent if this flag | ||
6633 | is set for it; if not, the record is treated as a non-persistent record | ||
6634 | and it is lost after service restart. Persistent records are written to | ||
6635 | and read from the file @strong{statistics.data} before shutdown | ||
6636 | and upon startup. The file is located in the HOME directory of the peer. | ||
6637 | |||
6638 | An anomaly of the STATISTICS service is that it does not terminate | ||
6639 | immediately upon receiving a shutdown signal if it has any clients | ||
6640 | connected to it. It waits for all the clients that are not monitors to | ||
6641 | close their connections before terminating itself. | ||
6642 | This is to prevent the loss of data during peer shutdown --- delaying the | ||
6643 | STATISTICS service shutdown helps other services to store important data | ||
6644 | to STATISTICS during shutdown. | ||
6645 | |||
6646 | @menu | ||
6647 | * libgnunetstatistics:: | ||
6648 | * The STATISTICS Client-Service Protocol:: | ||
6649 | @end menu | ||
6650 | |||
6651 | @cindex libgnunetstatistics | ||
6652 | @node libgnunetstatistics | ||
6653 | @subsection libgnunetstatistics | ||
6654 | |||
6655 | @c %**end of header | ||
6656 | |||
6657 | @strong{libgnunetstatistics} is the library containing the API for the | ||
6658 | STATISTICS subsystem. Any process requiring to use STATISTICS should use | ||
6659 | this API by to open a connection to the STATISTICS service. | ||
6660 | This is done by calling the function @code{GNUNET_STATISTICS_create()}. | ||
6661 | This function takes the subsystem's name which is trying to use STATISTICS | ||
6662 | and a configuration. | ||
6663 | All values written to STATISTICS with this connection will be placed in | ||
6664 | the section corresponding to the given subsystem's name. | ||
6665 | The connection to STATISTICS can be destroyed with the function | ||
6666 | @code{GNUNET_STATISTICS_destroy()}. This function allows for the | ||
6667 | connection to be destroyed immediately or upon transferring all | ||
6668 | pending write requests to the service. | ||
6669 | |||
6670 | Note: STATISTICS subsystem can be disabled by setting @code{DISABLE = YES} | ||
6671 | under the @code{[STATISTICS]} section in the configuration. With such a | ||
6672 | configuration all calls to @code{GNUNET_STATISTICS_create()} return | ||
6673 | @code{NULL} as the STATISTICS subsystem is unavailable and no other | ||
6674 | functions from the API can be used. | ||
6675 | |||
6676 | |||
6677 | @menu | ||
6678 | * Statistics retrieval:: | ||
6679 | * Setting statistics and updating them:: | ||
6680 | * Watches:: | ||
6681 | @end menu | ||
6682 | |||
6683 | @node Statistics retrieval | ||
6684 | @subsubsection Statistics retrieval | ||
6685 | |||
6686 | @c %**end of header | ||
6687 | |||
6688 | Once a connection to the statistics service is obtained, information | ||
6689 | about any other system which uses statistics can be retrieved with the | ||
6690 | function GNUNET_STATISTICS_get(). | ||
6691 | This function takes the connection handle, the name of the subsystem | ||
6692 | whose information we are interested in (a @code{NULL} value will | ||
6693 | retrieve information of all available subsystems using STATISTICS), the | ||
6694 | name of the statistic we are interested in (a @code{NULL} value will | ||
6695 | retrieve all available statistics), a continuation callback which is | ||
6696 | called when all of requested information is retrieved, an iterator | ||
6697 | callback which is called for each parameter in the retrieved information | ||
6698 | and a closure for the aforementioned callbacks. The library then invokes | ||
6699 | the iterator callback for each value matching the request. | ||
6700 | |||
6701 | Call to @code{GNUNET_STATISTICS_get()} is asynchronous and can be | ||
6702 | canceled with the function @code{GNUNET_STATISTICS_get_cancel()}. | ||
6703 | This is helpful when retrieving statistics takes too long and especially | ||
6704 | when we want to shutdown and cleanup everything. | ||
6705 | |||
6706 | @node Setting statistics and updating them | ||
6707 | @subsubsection Setting statistics and updating them | ||
6708 | |||
6709 | @c %**end of header | ||
6710 | |||
6711 | So far we have seen how to retrieve statistics, here we will learn how we | ||
6712 | can set statistics and update them so that other subsystems can retrieve | ||
6713 | them. | ||
6714 | |||
6715 | A new statistic can be set using the function | ||
6716 | @code{GNUNET_STATISTICS_set()}. | ||
6717 | This function takes the name of the statistic and its value and a flag to | ||
6718 | make the statistic persistent. | ||
6719 | The value of the statistic should be of the type @code{uint64_t}. | ||
6720 | The function does not take the name of the subsystem; it is determined | ||
6721 | from the previous @code{GNUNET_STATISTICS_create()} invocation. If | ||
6722 | the given statistic is already present, its value is overwritten. | ||
6723 | |||
6724 | An existing statistics can be updated, i.e its value can be increased or | ||
6725 | decreased by an amount with the function | ||
6726 | @code{GNUNET_STATISTICS_update()}. | ||
6727 | The parameters to this function are similar to | ||
6728 | @code{GNUNET_STATISTICS_set()}, except that it takes the amount to be | ||
6729 | changed as a type @code{int64_t} instead of the value. | ||
6730 | |||
6731 | The library will combine multiple set or update operations into one | ||
6732 | message if the client performs requests at a rate that is faster than the | ||
6733 | available IPC with the STATISTICS service. Thus, the client does not have | ||
6734 | to worry about sending requests too quickly. | ||
6735 | |||
6736 | @node Watches | ||
6737 | @subsubsection Watches | ||
6738 | |||
6739 | @c %**end of header | ||
6740 | |||
6741 | As interesting feature of STATISTICS lies in serving notifications | ||
6742 | whenever a statistic of our interest is modified. | ||
6743 | This is achieved by registering a watch through the function | ||
6744 | @code{GNUNET_STATISTICS_watch()}. | ||
6745 | The parameters of this function are similar to those of | ||
6746 | @code{GNUNET_STATISTICS_get()}. | ||
6747 | Changes to the respective statistic's value will then cause the given | ||
6748 | iterator callback to be called. | ||
6749 | Note: A watch can only be registered for a specific statistic. Hence | ||
6750 | the subsystem name and the parameter name cannot be @code{NULL} in a | ||
6751 | call to @code{GNUNET_STATISTICS_watch()}. | ||
6752 | |||
6753 | A registered watch will keep notifying any value changes until | ||
6754 | @code{GNUNET_STATISTICS_watch_cancel()} is called with the same | ||
6755 | parameters that are used for registering the watch. | ||
6756 | |||
6757 | @node The STATISTICS Client-Service Protocol | ||
6758 | @subsection The STATISTICS Client-Service Protocol | ||
6759 | @c %**end of header | ||
6760 | |||
6761 | |||
6762 | @menu | ||
6763 | * Statistics retrieval2:: | ||
6764 | * Setting and updating statistics:: | ||
6765 | * Watching for updates:: | ||
6766 | @end menu | ||
6767 | |||
6768 | @node Statistics retrieval2 | ||
6769 | @subsubsection Statistics retrieval2 | ||
6770 | |||
6771 | @c %**end of header | ||
6772 | |||
6773 | To retrieve statistics, the client transmits a message of type | ||
6774 | @code{GNUNET_MESSAGE_TYPE_STATISTICS_GET} containing the given subsystem | ||
6775 | name and statistic parameter to the STATISTICS service. | ||
6776 | The service responds with a message of type | ||
6777 | @code{GNUNET_MESSAGE_TYPE_STATISTICS_VALUE} for each of the statistics | ||
6778 | parameters that match the client request for the client. The end of | ||
6779 | information retrieved is signaled by the service by sending a message of | ||
6780 | type @code{GNUNET_MESSAGE_TYPE_STATISTICS_END}. | ||
6781 | |||
6782 | @node Setting and updating statistics | ||
6783 | @subsubsection Setting and updating statistics | ||
6784 | |||
6785 | @c %**end of header | ||
6786 | |||
6787 | The subsystem name, parameter name, its value and the persistence flag are | ||
6788 | communicated to the service through the message | ||
6789 | @code{GNUNET_MESSAGE_TYPE_STATISTICS_SET}. | ||
6790 | |||
6791 | When the service receives a message of type | ||
6792 | @code{GNUNET_MESSAGE_TYPE_STATISTICS_SET}, it retrieves the subsystem | ||
6793 | name and checks for a statistic parameter with matching the name given in | ||
6794 | the message. | ||
6795 | If a statistic parameter is found, the value is overwritten by the new | ||
6796 | value from the message; if not found then a new statistic parameter is | ||
6797 | created with the given name and value. | ||
6798 | |||
6799 | In addition to just setting an absolute value, it is possible to perform a | ||
6800 | relative update by sending a message of type | ||
6801 | @code{GNUNET_MESSAGE_TYPE_STATISTICS_SET} with an update flag | ||
6802 | (@code{GNUNET_STATISTICS_SETFLAG_RELATIVE}) signifying that the value in | ||
6803 | the message should be treated as an update value. | ||
6804 | |||
6805 | @node Watching for updates | ||
6806 | @subsubsection Watching for updates | ||
6807 | |||
6808 | @c %**end of header | ||
6809 | |||
6810 | The function registers the watch at the service by sending a message of | ||
6811 | type @code{GNUNET_MESSAGE_TYPE_STATISTICS_WATCH}. The service then sends | ||
6812 | notifications through messages of type | ||
6813 | @code{GNUNET_MESSAGE_TYPE_STATISTICS_WATCH_VALUE} whenever the statistic | ||
6814 | parameter's value is changed. | ||
6815 | |||
6816 | @cindex DHT | ||
6817 | @cindex Distributed Hash Table | ||
6818 | @node Distributed Hash Table (DHT) | ||
6819 | @section Distributed Hash Table (DHT) | ||
6820 | |||
6821 | @c %**end of header | ||
6822 | |||
6823 | GNUnet includes a generic distributed hash table that can be used by | ||
6824 | developers building P2P applications in the framework. | ||
6825 | This section documents high-level features and how developers are | ||
6826 | expected to use the DHT. | ||
6827 | We have a research paper detailing how the DHT works. | ||
6828 | Also, Nate's thesis includes a detailed description and performance | ||
6829 | analysis (in chapter 6). | ||
6830 | |||
6831 | Key features of GNUnet's DHT include: | ||
6832 | |||
6833 | @itemize @bullet | ||
6834 | @item stores key-value pairs with values up to (approximately) 63k in size | ||
6835 | @item works with many underlay network topologies (small-world, random | ||
6836 | graph), underlay does not need to be a full mesh / clique | ||
6837 | @item support for extended queries (more than just a simple 'key'), | ||
6838 | filtering duplicate replies within the network (bloomfilter) and content | ||
6839 | validation (for details, please read the subsection on the block library) | ||
6840 | @item can (optionally) return paths taken by the PUT and GET operations | ||
6841 | to the application | ||
6842 | @item provides content replication to handle churn | ||
6843 | @end itemize | ||
6844 | |||
6845 | GNUnet's DHT is randomized and unreliable. Unreliable means that there is | ||
6846 | no strict guarantee that a value stored in the DHT is always | ||
6847 | found --- values are only found with high probability. | ||
6848 | While this is somewhat true in all P2P DHTs, GNUnet developers should be | ||
6849 | particularly wary of this fact (this will help you write secure, | ||
6850 | fault-tolerant code). Thus, when writing any application using the DHT, | ||
6851 | you should always consider the possibility that a value stored in the | ||
6852 | DHT by you or some other peer might simply not be returned, or returned | ||
6853 | with a significant delay. | ||
6854 | Your application logic must be written to tolerate this (naturally, some | ||
6855 | loss of performance or quality of service is expected in this case). | ||
6856 | |||
6857 | @menu | ||
6858 | * Block library and plugins:: | ||
6859 | * libgnunetdht:: | ||
6860 | * The DHT Client-Service Protocol:: | ||
6861 | * The DHT Peer-to-Peer Protocol:: | ||
6862 | @end menu | ||
6863 | |||
6864 | @node Block library and plugins | ||
6865 | @subsection Block library and plugins | ||
6866 | |||
6867 | @c %**end of header | ||
6868 | |||
6869 | @menu | ||
6870 | * What is a Block?:: | ||
6871 | * The API of libgnunetblock:: | ||
6872 | * Queries:: | ||
6873 | * Sample Code:: | ||
6874 | * Conclusion2:: | ||
6875 | @end menu | ||
6876 | |||
6877 | @node What is a Block? | ||
6878 | @subsubsection What is a Block? | ||
6879 | |||
6880 | @c %**end of header | ||
6881 | |||
6882 | Blocks are small (< 63k) pieces of data stored under a key (struct | ||
6883 | GNUNET_HashCode). Blocks have a type (enum GNUNET_BlockType) which defines | ||
6884 | their data format. Blocks are used in GNUnet as units of static data | ||
6885 | exchanged between peers and stored (or cached) locally. | ||
6886 | Uses of blocks include file-sharing (the files are broken up into blocks), | ||
6887 | the VPN (DNS information is stored in blocks) and the DHT (all | ||
6888 | information in the DHT and meta-information for the maintenance of the | ||
6889 | DHT are both stored using blocks). | ||
6890 | The block subsystem provides a few common functions that must be | ||
6891 | available for any type of block. | ||
6892 | |||
6893 | @cindex libgnunetblock API | ||
6894 | @node The API of libgnunetblock | ||
6895 | @subsubsection The API of libgnunetblock | ||
6896 | |||
6897 | @c %**end of header | ||
6898 | |||
6899 | The block library requires for each (family of) block type(s) a block | ||
6900 | plugin (implementing @file{gnunet_block_plugin.h}) that provides basic | ||
6901 | functions that are needed by the DHT (and possibly other subsystems) to | ||
6902 | manage the block. | ||
6903 | These block plugins are typically implemented within their respective | ||
6904 | subsystems. | ||
6905 | The main block library is then used to locate, load and query the | ||
6906 | appropriate block plugin. | ||
6907 | Which plugin is appropriate is determined by the block type (which is | ||
6908 | just a 32-bit integer). Block plugins contain code that specifies which | ||
6909 | block types are supported by a given plugin. The block library loads all | ||
6910 | block plugins that are installed at the local peer and forwards the | ||
6911 | application request to the respective plugin. | ||
6912 | |||
6913 | The central functions of the block APIs (plugin and main library) are to | ||
6914 | allow the mapping of blocks to their respective key (if possible) and the | ||
6915 | ability to check that a block is well-formed and matches a given | ||
6916 | request (again, if possible). | ||
6917 | This way, GNUnet can avoid storing invalid blocks, storing blocks under | ||
6918 | the wrong key and forwarding blocks in response to a query that they do | ||
6919 | not answer. | ||
6920 | |||
6921 | One key function of block plugins is that it allows GNUnet to detect | ||
6922 | duplicate replies (via the Bloom filter). All plugins MUST support | ||
6923 | detecting duplicate replies (by adding the current response to the | ||
6924 | Bloom filter and rejecting it if it is encountered again). | ||
6925 | If a plugin fails to do this, responses may loop in the network. | ||
6926 | |||
6927 | @node Queries | ||
6928 | @subsubsection Queries | ||
6929 | @c %**end of header | ||
6930 | |||
6931 | The query format for any block in GNUnet consists of four main components. | ||
6932 | First, the type of the desired block must be specified. Second, the query | ||
6933 | must contain a hash code. The hash code is used for lookups in hash | ||
6934 | tables and databases and must not be unique for the block (however, if | ||
6935 | possible a unique hash should be used as this would be best for | ||
6936 | performance). | ||
6937 | Third, an optional Bloom filter can be specified to exclude known results; | ||
6938 | replies that hash to the bits set in the Bloom filter are considered | ||
6939 | invalid. False-positives can be eliminated by sending the same query | ||
6940 | again with a different Bloom filter mutator value, which parameterizes | ||
6941 | the hash function that is used. | ||
6942 | Finally, an optional application-specific "eXtended query" (xquery) can | ||
6943 | be specified to further constrain the results. It is entirely up to | ||
6944 | the type-specific plugin to determine whether or not a given block | ||
6945 | matches a query (type, hash, Bloom filter, and xquery). | ||
6946 | Naturally, not all xquery's are valid and some types of blocks may not | ||
6947 | support Bloom filters either, so the plugin also needs to check if the | ||
6948 | query is valid in the first place. | ||
6949 | |||
6950 | Depending on the results from the plugin, the DHT will then discard the | ||
6951 | (invalid) query, forward the query, discard the (invalid) reply, cache the | ||
6952 | (valid) reply, and/or forward the (valid and non-duplicate) reply. | ||
6953 | |||
6954 | @node Sample Code | ||
6955 | @subsubsection Sample Code | ||
6956 | |||
6957 | @c %**end of header | ||
6958 | |||
6959 | The source code in @strong{plugin_block_test.c} is a good starting point | ||
6960 | for new block plugins --- it does the minimal work by implementing a | ||
6961 | plugin that performs no validation at all. | ||
6962 | The respective @strong{Makefile.am} shows how to build and install a | ||
6963 | block plugin. | ||
6964 | |||
6965 | @node Conclusion2 | ||
6966 | @subsubsection Conclusion2 | ||
6967 | |||
6968 | @c %**end of header | ||
6969 | |||
6970 | In conclusion, GNUnet subsystems that want to use the DHT need to define a | ||
6971 | block format and write a plugin to match queries and replies. For testing, | ||
6972 | the @code{GNUNET_BLOCK_TYPE_TEST} block type can be used; it accepts | ||
6973 | any query as valid and any reply as matching any query. | ||
6974 | This type is also used for the DHT command line tools. | ||
6975 | However, it should NOT be used for normal applications due to the lack | ||
6976 | of error checking that results from this primitive implementation. | ||
6977 | |||
6978 | @cindex libgnunetdht | ||
6979 | @node libgnunetdht | ||
6980 | @subsection libgnunetdht | ||
6981 | |||
6982 | @c %**end of header | ||
6983 | |||
6984 | The DHT API itself is pretty simple and offers the usual GET and PUT | ||
6985 | functions that work as expected. The specified block type refers to the | ||
6986 | block library which allows the DHT to run application-specific logic for | ||
6987 | data stored in the network. | ||
6988 | |||
6989 | |||
6990 | @menu | ||
6991 | * GET:: | ||
6992 | * PUT:: | ||
6993 | * MONITOR:: | ||
6994 | * DHT Routing Options:: | ||
6995 | @end menu | ||
6996 | |||
6997 | @node GET | ||
6998 | @subsubsection GET | ||
6999 | |||
7000 | @c %**end of header | ||
7001 | |||
7002 | When using GET, the main consideration for developers (other than the | ||
7003 | block library) should be that after issuing a GET, the DHT will | ||
7004 | continuously cause (small amounts of) network traffic until the operation | ||
7005 | is explicitly canceled. | ||
7006 | So GET does not simply send out a single network request once; instead, | ||
7007 | the DHT will continue to search for data. This is needed to achieve good | ||
7008 | success rates and also handles the case where the respective PUT | ||
7009 | operation happens after the GET operation was started. | ||
7010 | Developers should not cancel an existing GET operation and then | ||
7011 | explicitly re-start it to trigger a new round of network requests; | ||
7012 | this is simply inefficient, especially as the internal automated version | ||
7013 | can be more efficient, for example by filtering results in the network | ||
7014 | that have already been returned. | ||
7015 | |||
7016 | If an application that performs a GET request has a set of replies that it | ||
7017 | already knows and would like to filter, it can call@ | ||
7018 | @code{GNUNET_DHT_get_filter_known_results} with an array of hashes over | ||
7019 | the respective blocks to tell the DHT that these results are not | ||
7020 | desired (any more). | ||
7021 | This way, the DHT will filter the respective blocks using the block | ||
7022 | library in the network, which may result in a significant reduction in | ||
7023 | bandwidth consumption. | ||
7024 | |||
7025 | @node PUT | ||
7026 | @subsubsection PUT | ||
7027 | |||
7028 | @c %**end of header | ||
7029 | |||
7030 | In contrast to GET operations, developers @strong{must} manually re-run | ||
7031 | PUT operations periodically (if they intend the content to continue to be | ||
7032 | available). Content stored in the DHT expires or might be lost due to | ||
7033 | churn. | ||
7034 | Furthermore, GNUnet's DHT typically requires multiple rounds of PUT | ||
7035 | operations before a key-value pair is consistently available to all | ||
7036 | peers (the DHT randomizes paths and thus storage locations, and only | ||
7037 | after multiple rounds of PUTs there will be a sufficient number of | ||
7038 | replicas in large DHTs). An explicit PUT operation using the DHT API will | ||
7039 | only cause network traffic once, so in order to ensure basic availability | ||
7040 | and resistance to churn (and adversaries), PUTs must be repeated. | ||
7041 | While the exact frequency depends on the application, a rule of thumb is | ||
7042 | that there should be at least a dozen PUT operations within the content | ||
7043 | lifetime. Content in the DHT typically expires after one day, so | ||
7044 | DHT PUT operations should be repeated at least every 1-2 hours. | ||
7045 | |||
7046 | @node MONITOR | ||
7047 | @subsubsection MONITOR | ||
7048 | |||
7049 | @c %**end of header | ||
7050 | |||
7051 | The DHT API also allows applications to monitor messages crossing the | ||
7052 | local DHT service. | ||
7053 | The types of messages used by the DHT are GET, PUT and RESULT messages. | ||
7054 | Using the monitoring API, applications can choose to monitor these | ||
7055 | requests, possibly limiting themselves to requests for a particular block | ||
7056 | type. | ||
7057 | |||
7058 | The monitoring API is not only usefu only for diagnostics, it can also be | ||
7059 | used to trigger application operations based on PUT operations. | ||
7060 | For example, an application may use PUTs to distribute work requests to | ||
7061 | other peers. | ||
7062 | The workers would then monitor for PUTs that give them work, instead of | ||
7063 | looking for work using GET operations. | ||
7064 | This can be beneficial, especially if the workers have no good way to | ||
7065 | guess the keys under which work would be stored. | ||
7066 | Naturally, additional protocols might be needed to ensure that the desired | ||
7067 | number of workers will process the distributed workload. | ||
7068 | |||
7069 | @node DHT Routing Options | ||
7070 | @subsubsection DHT Routing Options | ||
7071 | |||
7072 | @c %**end of header | ||
7073 | |||
7074 | There are two important options for GET and PUT requests: | ||
7075 | |||
7076 | @table @asis | ||
7077 | @item GNUNET_DHT_RO_DEMULITPLEX_EVERYWHERE This option means that all | ||
7078 | peers should process the request, even if their peer ID is not closest to | ||
7079 | the key. For a PUT request, this means that all peers that a request | ||
7080 | traverses may make a copy of the data. | ||
7081 | Similarly for a GET request, all peers will check their local database | ||
7082 | for a result. Setting this option can thus significantly improve caching | ||
7083 | and reduce bandwidth consumption --- at the expense of a larger DHT | ||
7084 | database. If in doubt, we recommend that this option should be used. | ||
7085 | @item GNUNET_DHT_RO_RECORD_ROUTE This option instructs the DHT to record | ||
7086 | the path that a GET or a PUT request is taking through the overlay | ||
7087 | network. The resulting paths are then returned to the application with | ||
7088 | the respective result. This allows the receiver of a result to construct | ||
7089 | a path to the originator of the data, which might then be used for | ||
7090 | routing. Naturally, setting this option requires additional bandwidth | ||
7091 | and disk space, so applications should only set this if the paths are | ||
7092 | needed by the application logic. | ||
7093 | @item GNUNET_DHT_RO_FIND_PEER This option is an internal option used by | ||
7094 | the DHT's peer discovery mechanism and should not be used by applications. | ||
7095 | @item GNUNET_DHT_RO_BART This option is currently not implemented. It may | ||
7096 | in the future offer performance improvements for clique topologies. | ||
7097 | @end table | ||
7098 | |||
7099 | @node The DHT Client-Service Protocol | ||
7100 | @subsection The DHT Client-Service Protocol | ||
7101 | |||
7102 | @c %**end of header | ||
7103 | |||
7104 | @menu | ||
7105 | * PUTting data into the DHT:: | ||
7106 | * GETting data from the DHT:: | ||
7107 | * Monitoring the DHT:: | ||
7108 | @end menu | ||
7109 | |||
7110 | @node PUTting data into the DHT | ||
7111 | @subsubsection PUTting data into the DHT | ||
7112 | |||
7113 | @c %**end of header | ||
7114 | |||
7115 | To store (PUT) data into the DHT, the client sends a | ||
7116 | @code{struct GNUNET_DHT_ClientPutMessage} to the service. | ||
7117 | This message specifies the block type, routing options, the desired | ||
7118 | replication level, the expiration time, key, | ||
7119 | value and a 64-bit unique ID for the operation. The service responds with | ||
7120 | a @code{struct GNUNET_DHT_ClientPutConfirmationMessage} with the same | ||
7121 | 64-bit unique ID. Note that the service sends the confirmation as soon as | ||
7122 | it has locally processed the PUT request. The PUT may still be | ||
7123 | propagating through the network at this time. | ||
7124 | |||
7125 | In the future, we may want to change this to provide (limited) feedback | ||
7126 | to the client, for example if we detect that the PUT operation had no | ||
7127 | effect because the same key-value pair was already stored in the DHT. | ||
7128 | However, changing this would also require additional state and messages | ||
7129 | in the P2P interaction. | ||
7130 | |||
7131 | @node GETting data from the DHT | ||
7132 | @subsubsection GETting data from the DHT | ||
7133 | |||
7134 | @c %**end of header | ||
7135 | |||
7136 | To retrieve (GET) data from the DHT, the client sends a | ||
7137 | @code{struct GNUNET_DHT_ClientGetMessage} to the service. The message | ||
7138 | specifies routing options, a replication level (for replicating the GET, | ||
7139 | not the content), the desired block type, the key, the (optional) | ||
7140 | extended query and unique 64-bit request ID. | ||
7141 | |||
7142 | Additionally, the client may send any number of | ||
7143 | @code{struct GNUNET_DHT_ClientGetResultSeenMessage}s to notify the | ||
7144 | service about results that the client is already aware of. | ||
7145 | These messages consist of the key, the unique 64-bit ID of the request, | ||
7146 | and an arbitrary number of hash codes over the blocks that the client is | ||
7147 | already aware of. As messages are restricted to 64k, a client that | ||
7148 | already knows more than about a thousand blocks may need to send | ||
7149 | several of these messages. Naturally, the client should transmit these | ||
7150 | messages as quickly as possible after the original GET request such that | ||
7151 | the DHT can filter those results in the network early on. Naturally, as | ||
7152 | these messages are send after the original request, it is conceivalbe | ||
7153 | that the DHT service may return blocks that match those already known | ||
7154 | to the client anyway. | ||
7155 | |||
7156 | In response to a GET request, the service will send @code{struct | ||
7157 | GNUNET_DHT_ClientResultMessage}s to the client. These messages contain the | ||
7158 | block type, expiration, key, unique ID of the request and of course the | ||
7159 | value (a block). Depending on the options set for the respective | ||
7160 | operations, the replies may also contain the path the GET and/or the PUT | ||
7161 | took through the network. | ||
7162 | |||
7163 | A client can stop receiving replies either by disconnecting or by sending | ||
7164 | a @code{struct GNUNET_DHT_ClientGetStopMessage} which must contain the | ||
7165 | key and the 64-bit unique ID of the original request. Using an | ||
7166 | explicit "stop" message is more common as this allows a client to run | ||
7167 | many concurrent GET operations over the same connection with the DHT | ||
7168 | service --- and to stop them individually. | ||
7169 | |||
7170 | @node Monitoring the DHT | ||
7171 | @subsubsection Monitoring the DHT | ||
7172 | |||
7173 | @c %**end of header | ||
7174 | |||
7175 | To begin monitoring, the client sends a | ||
7176 | @code{struct GNUNET_DHT_MonitorStartStop} message to the DHT service. | ||
7177 | In this message, flags can be set to enable (or disable) monitoring of | ||
7178 | GET, PUT and RESULT messages that pass through a peer. The message can | ||
7179 | also restrict monitoring to a particular block type or a particular key. | ||
7180 | Once monitoring is enabled, the DHT service will notify the client about | ||
7181 | any matching event using @code{struct GNUNET_DHT_MonitorGetMessage}s for | ||
7182 | GET events, @code{struct GNUNET_DHT_MonitorPutMessage} for PUT events | ||
7183 | and @code{struct GNUNET_DHT_MonitorGetRespMessage} for RESULTs. Each of | ||
7184 | these messages contains all of the information about the event. | ||
7185 | |||
7186 | @node The DHT Peer-to-Peer Protocol | ||
7187 | @subsection The DHT Peer-to-Peer Protocol | ||
7188 | @c %**end of header | ||
7189 | |||
7190 | |||
7191 | @menu | ||
7192 | * Routing GETs or PUTs:: | ||
7193 | * PUTting data into the DHT2:: | ||
7194 | * GETting data from the DHT2:: | ||
7195 | @end menu | ||
7196 | |||
7197 | @node Routing GETs or PUTs | ||
7198 | @subsubsection Routing GETs or PUTs | ||
7199 | |||
7200 | @c %**end of header | ||
7201 | |||
7202 | When routing GETs or PUTs, the DHT service selects a suitable subset of | ||
7203 | neighbours for forwarding. The exact number of neighbours can be zero or | ||
7204 | more and depends on the hop counter of the query (initially zero) in | ||
7205 | relation to the (log of) the network size estimate, the desired | ||
7206 | replication level and the peer's connectivity. | ||
7207 | Depending on the hop counter and our network size estimate, the selection | ||
7208 | of the peers maybe randomized or by proximity to the key. | ||
7209 | Furthermore, requests include a set of peers that a request has already | ||
7210 | traversed; those peers are also excluded from the selection. | ||
7211 | |||
7212 | @node PUTting data into the DHT2 | ||
7213 | @subsubsection PUTting data into the DHT2 | ||
7214 | |||
7215 | @c %**end of header | ||
7216 | |||
7217 | To PUT data into the DHT, the service sends a @code{struct PeerPutMessage} | ||
7218 | of type @code{GNUNET_MESSAGE_TYPE_DHT_P2P_PUT} to the respective | ||
7219 | neighbour. | ||
7220 | In addition to the usual information about the content (type, routing | ||
7221 | options, desired replication level for the content, expiration time, key | ||
7222 | and value), the message contains a fixed-size Bloom filter with | ||
7223 | information about which peers (may) have already seen this request. | ||
7224 | This Bloom filter is used to ensure that DHT messages never loop back to | ||
7225 | a peer that has already processed the request. | ||
7226 | Additionally, the message includes the current hop counter and, depending | ||
7227 | on the routing options, the message may include the full path that the | ||
7228 | message has taken so far. | ||
7229 | The Bloom filter should already contain the identity of the previous hop; | ||
7230 | however, the path should not include the identity of the previous hop and | ||
7231 | the receiver should append the identity of the sender to the path, not | ||
7232 | its own identity (this is done to reduce bandwidth). | ||
7233 | |||
7234 | @node GETting data from the DHT2 | ||
7235 | @subsubsection GETting data from the DHT2 | ||
7236 | |||
7237 | @c %**end of header | ||
7238 | |||
7239 | A peer can search the DHT by sending @code{struct PeerGetMessage}s of type | ||
7240 | @code{GNUNET_MESSAGE_TYPE_DHT_P2P_GET} to other peers. In addition to the | ||
7241 | usual information about the request (type, routing options, desired | ||
7242 | replication level for the request, the key and the extended query), a GET | ||
7243 | request also again contains a hop counter, a Bloom filter over the peers | ||
7244 | that have processed the request already and depending on the routing | ||
7245 | options the full path traversed by the GET. | ||
7246 | Finally, a GET request includes a variable-size second Bloom filter and a | ||
7247 | so-called Bloom filter mutator value which together indicate which | ||
7248 | replies the sender has already seen. During the lookup, each block that | ||
7249 | matches they block type, key and extended query is additionally subjected | ||
7250 | to a test against this Bloom filter. | ||
7251 | The block plugin is expected to take the hash of the block and combine it | ||
7252 | with the mutator value and check if the result is not yet in the Bloom | ||
7253 | filter. The originator of the query will from time to time modify the | ||
7254 | mutator to (eventually) allow false-positives filtered by the Bloom filter | ||
7255 | to be returned. | ||
7256 | |||
7257 | Peers that receive a GET request perform a local lookup (depending on | ||
7258 | their proximity to the key and the query options) and forward the request | ||
7259 | to other peers. | ||
7260 | They then remember the request (including the Bloom filter for blocking | ||
7261 | duplicate results) and when they obtain a matching, non-filtered response | ||
7262 | a @code{struct PeerResultMessage} of type | ||
7263 | @code{GNUNET_MESSAGE_TYPE_DHT_P2P_RESULT} is forwarded to the previous | ||
7264 | hop. | ||
7265 | Whenver a result is forwarded, the block plugin is used to update the | ||
7266 | Bloom filter accordingly, to ensure that the same result is never | ||
7267 | forwarded more than once. | ||
7268 | The DHT service may also cache forwarded results locally if the | ||
7269 | "CACHE_RESULTS" option is set to "YES" in the configuration. | ||
7270 | |||
7271 | @cindex GNS | ||
7272 | @cindex GNU Name System | ||
7273 | @node GNU Name System (GNS) | ||
7274 | @section GNU Name System (GNS) | ||
7275 | |||
7276 | @c %**end of header | ||
7277 | |||
7278 | The GNU Name System (GNS) is a decentralized database that enables users | ||
7279 | to securely resolve names to values. | ||
7280 | Names can be used to identify other users (for example, in social | ||
7281 | networking), or network services (for example, VPN services running at a | ||
7282 | peer in GNUnet, or purely IP-based services on the Internet). | ||
7283 | Users interact with GNS by typing in a hostname that ends in ".gnu" | ||
7284 | or ".zkey". | ||
7285 | |||
7286 | Videos giving an overview of most of the GNS and the motivations behind | ||
7287 | it is available here and here. | ||
7288 | The remainder of this chapter targets developers that are familiar with | ||
7289 | high level concepts of GNS as presented in these talks. | ||
7290 | @c TODO: Add links to here and here and to these. | ||
7291 | |||
7292 | GNS-aware applications should use the GNS resolver to obtain the | ||
7293 | respective records that are stored under that name in GNS. | ||
7294 | Each record consists of a type, value, expiration time and flags. | ||
7295 | |||
7296 | The type specifies the format of the value. Types below 65536 correspond | ||
7297 | to DNS record types, larger values are used for GNS-specific records. | ||
7298 | Applications can define new GNS record types by reserving a number and | ||
7299 | implementing a plugin (which mostly needs to convert the binary value | ||
7300 | representation to a human-readable text format and vice-versa). | ||
7301 | The expiration time specifies how long the record is to be valid. | ||
7302 | The GNS API ensures that applications are only given non-expired values. | ||
7303 | The flags are typically irrelevant for applications, as GNS uses them | ||
7304 | internally to control visibility and validity of records. | ||
7305 | |||
7306 | Records are stored along with a signature. | ||
7307 | The signature is generated using the private key of the authoritative | ||
7308 | zone. This allows any GNS resolver to verify the correctness of a | ||
7309 | name-value mapping. | ||
7310 | |||
7311 | Internally, GNS uses the NAMECACHE to cache information obtained from | ||
7312 | other users, the NAMESTORE to store information specific to the local | ||
7313 | users, and the DHT to exchange data between users. | ||
7314 | A plugin API is used to enable applications to define new GNS | ||
7315 | record types. | ||
7316 | |||
7317 | @menu | ||
7318 | * libgnunetgns:: | ||
7319 | * libgnunetgnsrecord:: | ||
7320 | * GNS plugins:: | ||
7321 | * The GNS Client-Service Protocol:: | ||
7322 | * Hijacking the DNS-Traffic using gnunet-service-dns:: | ||
7323 | * Serving DNS lookups via GNS on W32:: | ||
7324 | @end menu | ||
7325 | |||
7326 | @node libgnunetgns | ||
7327 | @subsection libgnunetgns | ||
7328 | |||
7329 | @c %**end of header | ||
7330 | |||
7331 | The GNS API itself is extremely simple. Clients first connec to the | ||
7332 | GNS service using @code{GNUNET_GNS_connect}. | ||
7333 | They can then perform lookups using @code{GNUNET_GNS_lookup} or cancel | ||
7334 | pending lookups using @code{GNUNET_GNS_lookup_cancel}. | ||
7335 | Once finished, clients disconnect using @code{GNUNET_GNS_disconnect}. | ||
7336 | |||
7337 | @menu | ||
7338 | * Looking up records:: | ||
7339 | * Accessing the records:: | ||
7340 | * Creating records:: | ||
7341 | * Future work:: | ||
7342 | @end menu | ||
7343 | |||
7344 | @node Looking up records | ||
7345 | @subsubsection Looking up records | ||
7346 | |||
7347 | @c %**end of header | ||
7348 | |||
7349 | @code{GNUNET_GNS_lookup} takes a number of arguments: | ||
7350 | |||
7351 | @table @asis | ||
7352 | @item handle This is simply the GNS connection handle from | ||
7353 | @code{GNUNET_GNS_connect}. | ||
7354 | @item name The client needs to specify the name to | ||
7355 | be resolved. This can be any valid DNS or GNS hostname. | ||
7356 | @item zone The client | ||
7357 | needs to specify the public key of the GNS zone against which the | ||
7358 | resolution should be done (the ".gnu" zone). | ||
7359 | Note that a key must be provided, even if the name ends in ".zkey". | ||
7360 | This should typically be the public key of the master-zone of the user. | ||
7361 | @item type This is the desired GNS or DNS record type | ||
7362 | to look for. While all records for the given name will be returned, this | ||
7363 | can be important if the client wants to resolve record types that | ||
7364 | themselves delegate resolution, such as CNAME, PKEY or GNS2DNS. | ||
7365 | Resolving a record of any of these types will only work if the respective | ||
7366 | record type is specified in the request, as the GNS resolver will | ||
7367 | otherwise follow the delegation and return the records from the | ||
7368 | respective destination, instead of the delegating record. | ||
7369 | @item only_cached This argument should typically be set to | ||
7370 | @code{GNUNET_NO}. Setting it to @code{GNUNET_YES} disables resolution via | ||
7371 | the overlay network. | ||
7372 | @item shorten_zone_key If GNS encounters new names during resolution, | ||
7373 | their respective zones can automatically be learned and added to the | ||
7374 | "shorten zone". If this is desired, clients must pass the private key of | ||
7375 | the shorten zone. If NULL is passed, shortening is disabled. | ||
7376 | @item proc This argument identifies | ||
7377 | the function to call with the result. It is given proc_cls, the number of | ||
7378 | records found (possilby zero) and the array of the records as arguments. | ||
7379 | proc will only be called once. After proc,> has been called, the lookup | ||
7380 | must no longer be cancelled. | ||
7381 | @item proc_cls The closure for proc. | ||
7382 | @end table | ||
7383 | |||
7384 | @node Accessing the records | ||
7385 | @subsubsection Accessing the records | ||
7386 | |||
7387 | @c %**end of header | ||
7388 | |||
7389 | The @code{libgnunetgnsrecord} library provides an API to manipulate the | ||
7390 | GNS record array that is given to proc. In particular, it offers | ||
7391 | functions such as converting record values to human-readable | ||
7392 | strings (and back). However, most @code{libgnunetgnsrecord} functions are | ||
7393 | not interesting to GNS client applications. | ||
7394 | |||
7395 | For DNS records, the @code{libgnunetdnsparser} library provides | ||
7396 | functions for parsing (and serializing) common types of DNS records. | ||
7397 | |||
7398 | @node Creating records | ||
7399 | @subsubsection Creating records | ||
7400 | |||
7401 | @c %**end of header | ||
7402 | |||
7403 | Creating GNS records is typically done by building the respective record | ||
7404 | information (possibly with the help of @code{libgnunetgnsrecord} and | ||
7405 | @code{libgnunetdnsparser}) and then using the @code{libgnunetnamestore} to | ||
7406 | publish the information. The GNS API is not involved in this | ||
7407 | operation. | ||
7408 | |||
7409 | @node Future work | ||
7410 | @subsubsection Future work | ||
7411 | |||
7412 | @c %**end of header | ||
7413 | |||
7414 | In the future, we want to expand @code{libgnunetgns} to allow | ||
7415 | applications to observe shortening operations performed during GNS | ||
7416 | resolution, for example so that users can receive visual feedback when | ||
7417 | this happens. | ||
7418 | |||
7419 | @node libgnunetgnsrecord | ||
7420 | @subsection libgnunetgnsrecord | ||
7421 | |||
7422 | @c %**end of header | ||
7423 | |||
7424 | The @code{libgnunetgnsrecord} library is used to manipulate GNS | ||
7425 | records (in plaintext or in their encrypted format). | ||
7426 | Applications mostly interact with @code{libgnunetgnsrecord} by using the | ||
7427 | functions to convert GNS record values to strings or vice-versa, or to | ||
7428 | lookup a GNS record type number by name (or vice-versa). | ||
7429 | The library also provides various other functions that are mostly | ||
7430 | used internally within GNS, such as converting keys to names, checking for | ||
7431 | expiration, encrypting GNS records to GNS blocks, verifying GNS block | ||
7432 | signatures and decrypting GNS records from GNS blocks. | ||
7433 | |||
7434 | We will now discuss the four commonly used functions of the API.@ | ||
7435 | @code{libgnunetgnsrecord} does not perform these operations itself, | ||
7436 | but instead uses plugins to perform the operation. | ||
7437 | GNUnet includes plugins to support common DNS record types as well as | ||
7438 | standard GNS record types. | ||
7439 | |||
7440 | @menu | ||
7441 | * Value handling:: | ||
7442 | * Type handling:: | ||
7443 | @end menu | ||
7444 | |||
7445 | @node Value handling | ||
7446 | @subsubsection Value handling | ||
7447 | |||
7448 | @c %**end of header | ||
7449 | |||
7450 | @code{GNUNET_GNSRECORD_value_to_string} can be used to convert | ||
7451 | the (binary) representation of a GNS record value to a human readable, | ||
7452 | 0-terminated UTF-8 string. | ||
7453 | NULL is returned if the specified record type is not supported by any | ||
7454 | available plugin. | ||
7455 | |||
7456 | @code{GNUNET_GNSRECORD_string_to_value} can be used to try to convert a | ||
7457 | human readable string to the respective (binary) representation of | ||
7458 | a GNS record value. | ||
7459 | |||
7460 | @node Type handling | ||
7461 | @subsubsection Type handling | ||
7462 | |||
7463 | @c %**end of header | ||
7464 | |||
7465 | @code{GNUNET_GNSRECORD_typename_to_number} can be used to obtain the | ||
7466 | numeric value associated with a given typename. For example, given the | ||
7467 | typename "A" (for DNS A reocrds), the function will return the number 1. | ||
7468 | A list of common DNS record types is | ||
7469 | @uref{http://en.wikipedia.org/wiki/List_of_DNS_record_types, here}. | ||
7470 | Note that not all DNS record types are supported by GNUnet GNSRECORD | ||
7471 | plugins at this time. | ||
7472 | |||
7473 | @code{GNUNET_GNSRECORD_number_to_typename} can be used to obtain the | ||
7474 | typename associated with a given numeric value. | ||
7475 | For example, given the type number 1, the function will return the | ||
7476 | typename "A". | ||
7477 | |||
7478 | @node GNS plugins | ||
7479 | @subsection GNS plugins | ||
7480 | |||
7481 | @c %**end of header | ||
7482 | |||
7483 | Adding a new GNS record type typically involves writing (or extending) a | ||
7484 | GNSRECORD plugin. The plugin needs to implement the | ||
7485 | @code{gnunet_gnsrecord_plugin.h} API which provides basic functions that | ||
7486 | are needed by GNSRECORD to convert typenames and values of the respective | ||
7487 | record type to strings (and back). | ||
7488 | These gnsrecord plugins are typically implemented within their respective | ||
7489 | subsystems. | ||
7490 | Examples for such plugins can be found in the GNSRECORD, GNS and | ||
7491 | CONVERSATION subsystems. | ||
7492 | |||
7493 | The @code{libgnunetgnsrecord} library is then used to locate, load and | ||
7494 | query the appropriate gnsrecord plugin. | ||
7495 | Which plugin is appropriate is determined by the record type (which is | ||
7496 | just a 32-bit integer). The @code{libgnunetgnsrecord} library loads all | ||
7497 | block plugins that are installed at the local peer and forwards the | ||
7498 | application request to the plugins. If the record type is not | ||
7499 | supported by the plugin, it should simply return an error code. | ||
7500 | |||
7501 | The central functions of the block APIs (plugin and main library) are the | ||
7502 | same four functions for converting between values and strings, and | ||
7503 | typenames and numbers documented in the previous subsection. | ||
7504 | |||
7505 | @node The GNS Client-Service Protocol | ||
7506 | @subsection The GNS Client-Service Protocol | ||
7507 | @c %**end of header | ||
7508 | |||
7509 | The GNS client-service protocol consists of two simple messages, the | ||
7510 | @code{LOOKUP} message and the @code{LOOKUP_RESULT}. Each @code{LOOKUP} | ||
7511 | message contains a unique 32-bit identifier, which will be included in the | ||
7512 | corresponding response. Thus, clients can send many lookup requests in | ||
7513 | parallel and receive responses out-of-order. | ||
7514 | A @code{LOOKUP} request also includes the public key of the GNS zone, | ||
7515 | the desired record type and fields specifying whether shortening is | ||
7516 | enabled or networking is disabled. Finally, the @code{LOOKUP} message | ||
7517 | includes the name to be resolved. | ||
7518 | |||
7519 | The response includes the number of records and the records themselves | ||
7520 | in the format created by @code{GNUNET_GNSRECORD_records_serialize}. | ||
7521 | They can thus be deserialized using | ||
7522 | @code{GNUNET_GNSRECORD_records_deserialize}. | ||
7523 | |||
7524 | @node Hijacking the DNS-Traffic using gnunet-service-dns | ||
7525 | @subsection Hijacking the DNS-Traffic using gnunet-service-dns | ||
7526 | |||
7527 | @c %**end of header | ||
7528 | |||
7529 | This section documents how the gnunet-service-dns (and the | ||
7530 | gnunet-helper-dns) intercepts DNS queries from the local system. | ||
7531 | This is merely one method for how we can obtain GNS queries. | ||
7532 | It is also possible to change @code{resolv.conf} to point to a machine | ||
7533 | running @code{gnunet-dns2gns} or to modify libc's name system switch | ||
7534 | (NSS) configuration to include a GNS resolution plugin. | ||
7535 | The method described in this chaper is more of a last-ditch catch-all | ||
7536 | approach. | ||
7537 | |||
7538 | @code{gnunet-service-dns} enables intercepting DNS traffic using policy | ||
7539 | based routing. | ||
7540 | We MARK every outgoing DNS-packet if it was not sent by our application. | ||
7541 | Using a second routing table in the Linux kernel these marked packets are | ||
7542 | then routed through our virtual network interface and can thus be | ||
7543 | captured unchanged. | ||
7544 | |||
7545 | Our application then reads the query and decides how to handle it: A | ||
7546 | query to an address ending in ".gnu" or ".zkey" is hijacked by | ||
7547 | @code{gnunet-service-gns} and resolved internally using GNS. | ||
7548 | In the future, a reverse query for an address of the configured virtual | ||
7549 | network could be answered with records kept about previous forward | ||
7550 | queries. | ||
7551 | Queries that are not hijacked by some application using the DNS service | ||
7552 | will be sent to the original recipient. | ||
7553 | The answer to the query will always be sent back through the virtual | ||
7554 | interface with the original nameserver as source address. | ||
7555 | |||
7556 | |||
7557 | @menu | ||
7558 | * Network Setup Details:: | ||
7559 | @end menu | ||
7560 | |||
7561 | @node Network Setup Details | ||
7562 | @subsubsection Network Setup Details | ||
7563 | |||
7564 | @c %**end of header | ||
7565 | |||
7566 | The DNS interceptor adds the following rules to the Linux kernel: | ||
7567 | @example | ||
7568 | iptables -t mangle -I OUTPUT 1 -p udp --sport $LOCALPORT --dport 53 \ | ||
7569 | -j ACCEPT iptables -t mangle -I OUTPUT 2 -p udp --dport 53 -j MARK \ | ||
7570 | --set-mark 3 ip rule add fwmark 3 table2 ip route add default via \ | ||
7571 | $VIRTUALDNS table2 | ||
7572 | @end example | ||
7573 | |||
7574 | @c FIXME: Rewrite to reflect display which is no longer content by line | ||
7575 | @c FIXME: due to the < 74 characters limit. | ||
7576 | Line 1 makes sure that all packets coming from a port our application | ||
7577 | opened beforehand (@code{$LOCALPORT}) will be routed normally. | ||
7578 | Line 2 marks every other packet to a DNS-Server with mark 3 (chosen | ||
7579 | arbitrarily). The third line adds a routing policy based on this mark | ||
7580 | 3 via the routing table. | ||
7581 | |||
7582 | @node Serving DNS lookups via GNS on W32 | ||
7583 | @subsection Serving DNS lookups via GNS on W32 | ||
7584 | |||
7585 | @c %**end of header | ||
7586 | |||
7587 | This section documents how the libw32nsp (and | ||
7588 | gnunet-gns-helper-service-w32) do DNS resolutions of DNS queries on the | ||
7589 | local system. This only applies to GNUnet running on W32. | ||
7590 | |||
7591 | W32 has a concept of "Namespaces" and "Namespace providers". | ||
7592 | These are used to present various name systems to applications in a | ||
7593 | generic way. | ||
7594 | Namespaces include DNS, mDNS, NLA and others. For each namespace any | ||
7595 | number of providers could be registered, and they are queried in an order | ||
7596 | of priority (which is adjustable). | ||
7597 | |||
7598 | Applications can resolve names by using WSALookupService*() family of | ||
7599 | functions. | ||
7600 | |||
7601 | However, these are WSA-only facilities. Common BSD socket functions for | ||
7602 | namespace resolutions are gethostbyname and getaddrinfo (among others). | ||
7603 | These functions are implemented internally (by default - by mswsock, | ||
7604 | which also implements the default DNS provider) as wrappers around | ||
7605 | WSALookupService*() functions (see "Sample Code for a Service Provider" | ||
7606 | on MSDN). | ||
7607 | |||
7608 | On W32 GNUnet builds a libw32nsp - a namespace provider, which can then be | ||
7609 | installed into the system by using w32nsp-install (and uninstalled by | ||
7610 | w32nsp-uninstall), as described in "Installation Handbook". | ||
7611 | |||
7612 | libw32nsp is very simple and has almost no dependencies. As a response to | ||
7613 | NSPLookupServiceBegin(), it only checks that the provider GUID passed to | ||
7614 | it by the caller matches GNUnet DNS Provider GUID, checks that name being | ||
7615 | resolved ends in ".gnu" or ".zkey", then connects to | ||
7616 | gnunet-gns-helper-service-w32 at 127.0.0.1:5353 (hardcoded) and sends the | ||
7617 | name resolution request there, returning the connected socket to the | ||
7618 | caller. | ||
7619 | |||
7620 | When the caller invokes NSPLookupServiceNext(), libw32nsp reads a | ||
7621 | completely formed reply from that socket, unmarshalls it, then gives | ||
7622 | it back to the caller. | ||
7623 | |||
7624 | At the moment gnunet-gns-helper-service-w32 is implemented to ever give | ||
7625 | only one reply, and subsequent calls to NSPLookupServiceNext() will fail | ||
7626 | with WSA_NODATA (first call to NSPLookupServiceNext() might also fail if | ||
7627 | GNS failed to find the name, or there was an error connecting to it). | ||
7628 | |||
7629 | gnunet-gns-helper-service-w32 does most of the processing: | ||
7630 | |||
7631 | @itemize @bullet | ||
7632 | @item Maintains a connection to GNS. | ||
7633 | @item Reads GNS config and loads appropriate keys. | ||
7634 | @item Checks service GUID and decides on the type of record to look up, | ||
7635 | refusing to make a lookup outright when unsupported service GUID is | ||
7636 | passed. | ||
7637 | @item Launches the lookup | ||
7638 | @end itemize | ||
7639 | |||
7640 | When lookup result arrives, gnunet-gns-helper-service-w32 forms a complete | ||
7641 | reply (including filling a WSAQUERYSETW structure and, possibly, a binary | ||
7642 | blob with a hostent structure for gethostbyname() client), marshalls it, | ||
7643 | and sends it back to libw32nsp. If no records were found, it sends an | ||
7644 | empty header. | ||
7645 | |||
7646 | This works for most normal applications that use gethostbyname() or | ||
7647 | getaddrinfo() to resolve names, but fails to do anything with | ||
7648 | applications that use alternative means of resolving names (such as | ||
7649 | sending queries to a DNS server directly by themselves). | ||
7650 | This includes some of well known utilities, like "ping" and "nslookup". | ||
7651 | |||
7652 | @cindex GNS Namecache | ||
7653 | @node GNS Namecache | ||
7654 | @section GNS Namecache | ||
7655 | |||
7656 | @c %**end of header | ||
7657 | |||
7658 | The NAMECACHE subsystem is responsible for caching (encrypted) resolution | ||
7659 | results of the GNU Name System (GNS). GNS makes zone information available | ||
7660 | to other users via the DHT. However, as accessing the DHT for every | ||
7661 | lookup is expensive (and as the DHT's local cache is lost whenever the | ||
7662 | peer is restarted), GNS uses the NAMECACHE as a more persistent cache for | ||
7663 | DHT lookups. | ||
7664 | Thus, instead of always looking up every name in the DHT, GNS first | ||
7665 | checks if the result is already available locally in the NAMECACHE. | ||
7666 | Only if there is no result in the NAMECACHE, GNS queries the DHT. | ||
7667 | The NAMECACHE stores data in the same (encrypted) format as the DHT. | ||
7668 | It thus makes no sense to iterate over all items in the | ||
7669 | NAMECACHE --- the NAMECACHE does not have a way to provide the keys | ||
7670 | required to decrypt the entries. | ||
7671 | |||
7672 | Blocks in the NAMECACHE share the same expiration mechanism as blocks in | ||
7673 | the DHT --- the block expires wheneever any of the records in | ||
7674 | the (encrypted) block expires. | ||
7675 | The expiration time of the block is the only information stored in | ||
7676 | plaintext. The NAMECACHE service internally performs all of the required | ||
7677 | work to expire blocks, clients do not have to worry about this. | ||
7678 | Also, given that NAMECACHE stores only GNS blocks that local users | ||
7679 | requested, there is no configuration option to limit the size of the | ||
7680 | NAMECACHE. It is assumed to be always small enough (a few MB) to fit on | ||
7681 | the drive. | ||
7682 | |||
7683 | The NAMECACHE supports the use of different database backends via a | ||
7684 | plugin API. | ||
7685 | |||
7686 | @menu | ||
7687 | * libgnunetnamecache:: | ||
7688 | * The NAMECACHE Client-Service Protocol:: | ||
7689 | * The NAMECACHE Plugin API:: | ||
7690 | @end menu | ||
7691 | |||
7692 | @node libgnunetnamecache | ||
7693 | @subsection libgnunetnamecache | ||
7694 | |||
7695 | @c %**end of header | ||
7696 | |||
7697 | The NAMECACHE API consists of five simple functions. First, there is | ||
7698 | @code{GNUNET_NAMECACHE_connect} to connect to the NAMECACHE service. | ||
7699 | This returns the handle required for all other operations on the | ||
7700 | NAMECACHE. Using @code{GNUNET_NAMECACHE_block_cache} clients can insert a | ||
7701 | block into the cache. | ||
7702 | @code{GNUNET_NAMECACHE_lookup_block} can be used to lookup blocks that | ||
7703 | were stored in the NAMECACHE. Both operations can be cancelled using | ||
7704 | @code{GNUNET_NAMECACHE_cancel}. Note that cancelling a | ||
7705 | @code{GNUNET_NAMECACHE_block_cache} operation can result in the block | ||
7706 | being stored in the NAMECACHE --- or not. Cancellation primarily ensures | ||
7707 | that the continuation function with the result of the operation will no | ||
7708 | longer be invoked. | ||
7709 | Finally, @code{GNUNET_NAMECACHE_disconnect} closes the connection to the | ||
7710 | NAMECACHE. | ||
7711 | |||
7712 | The maximum size of a block that can be stored in the NAMECACHE is | ||
7713 | @code{GNUNET_NAMECACHE_MAX_VALUE_SIZE}, which is defined to be 63 kB. | ||
7714 | |||
7715 | @node The NAMECACHE Client-Service Protocol | ||
7716 | @subsection The NAMECACHE Client-Service Protocol | ||
7717 | |||
7718 | @c %**end of header | ||
7719 | |||
7720 | All messages in the NAMECACHE IPC protocol start with the | ||
7721 | @code{struct GNUNET_NAMECACHE_Header} which adds a request | ||
7722 | ID (32-bit integer) to the standard message header. | ||
7723 | The request ID is used to match requests with the | ||
7724 | respective responses from the NAMECACHE, as they are allowed to happen | ||
7725 | out-of-order. | ||
7726 | |||
7727 | |||
7728 | @menu | ||
7729 | * Lookup:: | ||
7730 | * Store:: | ||
7731 | @end menu | ||
7732 | |||
7733 | @node Lookup | ||
7734 | @subsubsection Lookup | ||
7735 | |||
7736 | @c %**end of header | ||
7737 | |||
7738 | The @code{struct LookupBlockMessage} is used to lookup a block stored in | ||
7739 | the cache. | ||
7740 | It contains the query hash. The NAMECACHE always responds with a | ||
7741 | @code{struct LookupBlockResponseMessage}. If the NAMECACHE has no | ||
7742 | response, it sets the expiration time in the response to zero. | ||
7743 | Otherwise, the response is expected to contain the expiration time, the | ||
7744 | ECDSA signature, the derived key and the (variable-size) encrypted data | ||
7745 | of the block. | ||
7746 | |||
7747 | @node Store | ||
7748 | @subsubsection Store | ||
7749 | |||
7750 | @c %**end of header | ||
7751 | |||
7752 | The @code{struct BlockCacheMessage} is used to cache a block in the | ||
7753 | NAMECACHE. | ||
7754 | It has the same structure as the @code{struct LookupBlockResponseMessage}. | ||
7755 | The service responds with a @code{struct BlockCacheResponseMessage} which | ||
7756 | contains the result of the operation (success or failure). | ||
7757 | In the future, we might want to make it possible to provide an error | ||
7758 | message as well. | ||
7759 | |||
7760 | @node The NAMECACHE Plugin API | ||
7761 | @subsection The NAMECACHE Plugin API | ||
7762 | @c %**end of header | ||
7763 | |||
7764 | The NAMECACHE plugin API consists of two functions, @code{cache_block} to | ||
7765 | store a block in the database, and @code{lookup_block} to lookup a block | ||
7766 | in the database. | ||
7767 | |||
7768 | |||
7769 | @menu | ||
7770 | * Lookup2:: | ||
7771 | * Store2:: | ||
7772 | @end menu | ||
7773 | |||
7774 | @node Lookup2 | ||
7775 | @subsubsection Lookup2 | ||
7776 | |||
7777 | @c %**end of header | ||
7778 | |||
7779 | The @code{lookup_block} function is expected to return at most one block | ||
7780 | to the iterator, and return @code{GNUNET_NO} if there were no non-expired | ||
7781 | results. | ||
7782 | If there are multiple non-expired results in the cache, the lookup is | ||
7783 | supposed to return the result with the largest expiration time. | ||
7784 | |||
7785 | @node Store2 | ||
7786 | @subsubsection Store2 | ||
7787 | |||
7788 | @c %**end of header | ||
7789 | |||
7790 | The @code{cache_block} function is expected to try to store the block in | ||
7791 | the database, and return @code{GNUNET_SYSERR} if this was not possible | ||
7792 | for any reason. | ||
7793 | Furthermore, @code{cache_block} is expected to implicitly perform cache | ||
7794 | maintenance and purge blocks from the cache that have expired. Note that | ||
7795 | @code{cache_block} might encounter the case where the database already has | ||
7796 | another block stored under the same key. In this case, the plugin must | ||
7797 | ensure that the block with the larger expiration time is preserved. | ||
7798 | Obviously, this can done either by simply adding new blocks and selecting | ||
7799 | for the most recent expiration time during lookup, or by checking which | ||
7800 | block is more recent during the store operation. | ||
7801 | |||
7802 | @cindex REVOCATION Subsystem | ||
7803 | @node REVOCATION Subsystem | ||
7804 | @section REVOCATION Subsystem | ||
7805 | @c %**end of header | ||
7806 | |||
7807 | The REVOCATION subsystem is responsible for key revocation of Egos. | ||
7808 | If a user learns that theis private key has been compromised or has lost | ||
7809 | it, they can use the REVOCATION system to inform all of the other users | ||
7810 | that their private key is no longer valid. | ||
7811 | The subsystem thus includes ways to query for the validity of keys and to | ||
7812 | propagate revocation messages. | ||
7813 | |||
7814 | @menu | ||
7815 | * Dissemination:: | ||
7816 | * Revocation Message Design Requirements:: | ||
7817 | * libgnunetrevocation:: | ||
7818 | * The REVOCATION Client-Service Protocol:: | ||
7819 | * The REVOCATION Peer-to-Peer Protocol:: | ||
7820 | @end menu | ||
7821 | |||
7822 | @node Dissemination | ||
7823 | @subsection Dissemination | ||
7824 | |||
7825 | @c %**end of header | ||
7826 | |||
7827 | When a revocation is performed, the revocation is first of all | ||
7828 | disseminated by flooding the overlay network. | ||
7829 | The goal is to reach every peer, so that when a peer needs to check if a | ||
7830 | key has been revoked, this will be purely a local operation where the | ||
7831 | peer looks at his local revocation list. Flooding the network is also the | ||
7832 | most robust form of key revocation --- an adversary would have to control | ||
7833 | a separator of the overlay graph to restrict the propagation of the | ||
7834 | revocation message. Flooding is also very easy to implement --- peers that | ||
7835 | receive a revocation message for a key that they have never seen before | ||
7836 | simply pass the message to all of their neighbours. | ||
7837 | |||
7838 | Flooding can only distribute the revocation message to peers that are | ||
7839 | online. | ||
7840 | In order to notify peers that join the network later, the revocation | ||
7841 | service performs efficient set reconciliation over the sets of known | ||
7842 | revocation messages whenever two peers (that both support REVOCATION | ||
7843 | dissemination) connect. | ||
7844 | The SET service is used to perform this operation efficiently. | ||
7845 | |||
7846 | @node Revocation Message Design Requirements | ||
7847 | @subsection Revocation Message Design Requirements | ||
7848 | |||
7849 | @c %**end of header | ||
7850 | |||
7851 | However, flooding is also quite costly, creating O(|E|) messages on a | ||
7852 | network with |E| edges. | ||
7853 | Thus, revocation messages are required to contain a proof-of-work, the | ||
7854 | result of an expensive computation (which, however, is cheap to verify). | ||
7855 | Only peers that have expended the CPU time necessary to provide | ||
7856 | this proof will be able to flood the network with the revocation message. | ||
7857 | This ensures that an attacker cannot simply flood the network with | ||
7858 | millions of revocation messages. The proof-of-work required by GNUnet is | ||
7859 | set to take days on a typical PC to compute; if the ability to quickly | ||
7860 | revoke a key is needed, users have the option to pre-compute revocation | ||
7861 | messages to store off-line and use instantly after their key has expired. | ||
7862 | |||
7863 | Revocation messages must also be signed by the private key that is being | ||
7864 | revoked. Thus, they can only be created while the private key is in the | ||
7865 | possession of the respective user. This is another reason to create a | ||
7866 | revocation message ahead of time and store it in a secure location. | ||
7867 | |||
7868 | @node libgnunetrevocation | ||
7869 | @subsection libgnunetrevocation | ||
7870 | |||
7871 | @c %**end of header | ||
7872 | |||
7873 | The REVOCATION API consists of two parts, to query and to issue | ||
7874 | revocations. | ||
7875 | |||
7876 | |||
7877 | @menu | ||
7878 | * Querying for revoked keys:: | ||
7879 | * Preparing revocations:: | ||
7880 | * Issuing revocations:: | ||
7881 | @end menu | ||
7882 | |||
7883 | @node Querying for revoked keys | ||
7884 | @subsubsection Querying for revoked keys | ||
7885 | |||
7886 | @c %**end of header | ||
7887 | |||
7888 | @code{GNUNET_REVOCATION_query} is used to check if a given ECDSA public | ||
7889 | key has been revoked. | ||
7890 | The given callback will be invoked with the result of the check. | ||
7891 | The query can be cancelled using @code{GNUNET_REVOCATION_query_cancel} on | ||
7892 | the return value. | ||
7893 | |||
7894 | @node Preparing revocations | ||
7895 | @subsubsection Preparing revocations | ||
7896 | |||
7897 | @c %**end of header | ||
7898 | |||
7899 | It is often desirable to create a revocation record ahead-of-time and | ||
7900 | store it in an off-line location to be used later in an emergency. | ||
7901 | This is particularly true for GNUnet revocations, where performing the | ||
7902 | revocation operation itself is computationally expensive and thus is | ||
7903 | likely to take some time. | ||
7904 | Thus, if users want the ability to perform revocations quickly in an | ||
7905 | emergency, they must pre-compute the revocation message. | ||
7906 | The revocation API enables this with two functions that are used to | ||
7907 | compute the revocation message, but not trigger the actual revocation | ||
7908 | operation. | ||
7909 | |||
7910 | @code{GNUNET_REVOCATION_check_pow} should be used to calculate the | ||
7911 | proof-of-work required in the revocation message. This function takes the | ||
7912 | public key, the required number of bits for the proof of work (which in | ||
7913 | GNUnet is a network-wide constant) and finally a proof-of-work number as | ||
7914 | arguments. | ||
7915 | The function then checks if the given proof-of-work number is a valid | ||
7916 | proof of work for the given public key. Clients preparing a revocation | ||
7917 | are expected to call this function repeatedly (typically with a | ||
7918 | monotonically increasing sequence of numbers of the proof-of-work number) | ||
7919 | until a given number satisfies the check. | ||
7920 | That number should then be saved for later use in the revocation | ||
7921 | operation. | ||
7922 | |||
7923 | @code{GNUNET_REVOCATION_sign_revocation} is used to generate the | ||
7924 | signature that is required in a revocation message. | ||
7925 | It takes the private key that (possibly in the future) is to be revoked | ||
7926 | and returns the signature. | ||
7927 | The signature can again be saved to disk for later use, which will then | ||
7928 | allow performing a revocation even without access to the private key. | ||
7929 | |||
7930 | @node Issuing revocations | ||
7931 | @subsubsection Issuing revocations | ||
7932 | |||
7933 | |||
7934 | Given a ECDSA public key, the signature from @code{GNUNET_REVOCATION_sign} | ||
7935 | and the proof-of-work, | ||
7936 | @code{GNUNET_REVOCATION_revoke} can be used to perform the | ||
7937 | actual revocation. The given callback is called upon completion of the | ||
7938 | operation. @code{GNUNET_REVOCATION_revoke_cancel} can be used to stop the | ||
7939 | library from calling the continuation; however, in that case it is | ||
7940 | undefined whether or not the revocation operation will be executed. | ||
7941 | |||
7942 | @node The REVOCATION Client-Service Protocol | ||
7943 | @subsection The REVOCATION Client-Service Protocol | ||
7944 | |||
7945 | |||
7946 | The REVOCATION protocol consists of four simple messages. | ||
7947 | |||
7948 | A @code{QueryMessage} containing a public ECDSA key is used to check if a | ||
7949 | particular key has been revoked. The service responds with a | ||
7950 | @code{QueryResponseMessage} which simply contains a bit that says if the | ||
7951 | given public key is still valid, or if it has been revoked. | ||
7952 | |||
7953 | The second possible interaction is for a client to revoke a key by | ||
7954 | passing a @code{RevokeMessage} to the service. The @code{RevokeMessage} | ||
7955 | contains the ECDSA public key to be revoked, a signature by the | ||
7956 | corresponding private key and the proof-of-work, The service responds | ||
7957 | with a @code{RevocationResponseMessage} which can be used to indicate | ||
7958 | that the @code{RevokeMessage} was invalid (i.e. proof of work incorrect), | ||
7959 | or otherwise indicates that the revocation has been processed | ||
7960 | successfully. | ||
7961 | |||
7962 | @node The REVOCATION Peer-to-Peer Protocol | ||
7963 | @subsection The REVOCATION Peer-to-Peer Protocol | ||
7964 | |||
7965 | @c %**end of header | ||
7966 | |||
7967 | Revocation uses two disjoint ways to spread revocation information among | ||
7968 | peers. | ||
7969 | First of all, P2P gossip exchanged via CORE-level neighbours is used to | ||
7970 | quickly spread revocations to all connected peers. | ||
7971 | Second, whenever two peers (that both support revocations) connect, | ||
7972 | the SET service is used to compute the union of the respective revocation | ||
7973 | sets. | ||
7974 | |||
7975 | In both cases, the exchanged messages are @code{RevokeMessage}s which | ||
7976 | contain the public key that is being revoked, a matching ECDSA signature, | ||
7977 | and a proof-of-work. | ||
7978 | Whenever a peer learns about a new revocation this way, it first | ||
7979 | validates the signature and the proof-of-work, then stores it to disk | ||
7980 | (typically to a file $GNUNET_DATA_HOME/revocation.dat) and finally | ||
7981 | spreads the information to all directly connected neighbours. | ||
7982 | |||
7983 | For computing the union using the SET service, the peer with the smaller | ||
7984 | hashed peer identity will connect (as a "client" in the two-party set | ||
7985 | protocol) to the other peer after one second (to reduce traffic spikes | ||
7986 | on connect) and initiate the computation of the set union. | ||
7987 | All revocation services use a common hash to identify the SET operation | ||
7988 | over revocation sets. | ||
7989 | |||
7990 | The current implementation accepts revocation set union operations from | ||
7991 | all peers at any time; however, well-behaved peers should only initiate | ||
7992 | this operation once after establishing a connection to a peer with a | ||
7993 | larger hashed peer identity. | ||
7994 | |||
7995 | @cindex FS | ||
7996 | @cindex FS Subsystem | ||
7997 | @node File-sharing (FS) Subsystem | ||
7998 | @section File-sharing (FS) Subsystem | ||
7999 | |||
8000 | @c %**end of header | ||
8001 | |||
8002 | This chapter describes the details of how the file-sharing service works. | ||
8003 | As with all services, it is split into an API (libgnunetfs), the service | ||
8004 | process (gnunet-service-fs) and user interface(s). | ||
8005 | The file-sharing service uses the datastore service to store blocks and | ||
8006 | the DHT (and indirectly datacache) for lookups for non-anonymous | ||
8007 | file-sharing. | ||
8008 | Furthermore, the file-sharing service uses the block library (and the | ||
8009 | block fs plugin) for validation of DHT operations. | ||
8010 | |||
8011 | In contrast to many other services, libgnunetfs is rather complex since | ||
8012 | the client library includes a large number of high-level abstractions; | ||
8013 | this is necessary since the Fs service itself largely only operates on | ||
8014 | the block level. | ||
8015 | The FS library is responsible for providing a file-based abstraction to | ||
8016 | applications, including directories, meta data, keyword search, | ||
8017 | verification, and so on. | ||
8018 | |||
8019 | The method used by GNUnet to break large files into blocks and to use | ||
8020 | keyword search is called the | ||
8021 | "Encoding for Censorship Resistant Sharing" (ECRS). | ||
8022 | ECRS is largely implemented in the fs library; block validation is also | ||
8023 | reflected in the block FS plugin and the FS service. | ||
8024 | ECRS on-demand encoding is implemented in the FS service. | ||
8025 | |||
8026 | NOTE: The documentation in this chapter is quite incomplete. | ||
8027 | |||
8028 | @menu | ||
8029 | * Encoding for Censorship-Resistant Sharing (ECRS):: | ||
8030 | * File-sharing persistence directory structure:: | ||
8031 | @end menu | ||
8032 | |||
8033 | @cindex ECRS | ||
8034 | @cindex Encoding for Censorship-Resistant Sharing | ||
8035 | @node Encoding for Censorship-Resistant Sharing (ECRS) | ||
8036 | @subsection Encoding for Censorship-Resistant Sharing (ECRS) | ||
8037 | |||
8038 | @c %**end of header | ||
8039 | |||
8040 | When GNUnet shares files, it uses a content encoding that is called ECRS, | ||
8041 | the Encoding for Censorship-Resistant Sharing. | ||
8042 | Most of ECRS is described in the (so far unpublished) research paper | ||
8043 | attached to this page. ECRS obsoletes the previous ESED and ESED II | ||
8044 | encodings which were used in GNUnet before version 0.7.0. | ||
8045 | The rest of this page assumes that the reader is familiar with the | ||
8046 | attached paper. What follows is a description of some minor extensions | ||
8047 | that GNUnet makes over what is described in the paper. | ||
8048 | The reason why these extensions are not in the paper is that we felt | ||
8049 | that they were obvious or trivial extensions to the original scheme and | ||
8050 | thus did not warrant space in the research report. | ||
8051 | |||
8052 | @menu | ||
8053 | * Namespace Advertisements:: | ||
8054 | * KSBlocks:: | ||
8055 | @end menu | ||
8056 | |||
8057 | @node Namespace Advertisements | ||
8058 | @subsubsection Namespace Advertisements | ||
8059 | |||
8060 | @c %**end of header | ||
8061 | @c %**FIXME: all zeroses -> ? | ||
8062 | |||
8063 | An @code{SBlock} with identifier all zeros is a signed | ||
8064 | advertisement for a namespace. This special @code{SBlock} contains | ||
8065 | metadata describing the content of the namespace. | ||
8066 | Instead of the name of the identifier for a potential update, it contains | ||
8067 | the identifier for the root of the namespace. | ||
8068 | The URI should always be empty. The @code{SBlock} is signed with the | ||
8069 | content provder's RSA private key (just like any other SBlock). Peers | ||
8070 | can search for @code{SBlock}s in order to find out more about a namespace. | ||
8071 | |||
8072 | @node KSBlocks | ||
8073 | @subsubsection KSBlocks | ||
8074 | |||
8075 | @c %**end of header | ||
8076 | |||
8077 | GNUnet implements @code{KSBlocks} which are @code{KBlocks} that, instead | ||
8078 | of encrypting a CHK and metadata, encrypt an @code{SBlock} instead. | ||
8079 | In other words, @code{KSBlocks} enable GNUnet to find @code{SBlocks} | ||
8080 | using the global keyword search. | ||
8081 | Usually the encrypted @code{SBlock} is a namespace advertisement. | ||
8082 | The rationale behind @code{KSBlock}s and @code{SBlock}s is to enable | ||
8083 | peers to discover namespaces via keyword searches, and, to associate | ||
8084 | useful information with namespaces. When GNUnet finds @code{KSBlocks} | ||
8085 | during a normal keyword search, it adds the information to an internal | ||
8086 | list of discovered namespaces. Users looking for interesting namespaces | ||
8087 | can then inspect this list, reducing the need for out-of-band discovery | ||
8088 | of namespaces. | ||
8089 | Naturally, namespaces (or more specifically, namespace advertisements) can | ||
8090 | also be referenced from directories, but @code{KSBlock}s should make it | ||
8091 | easier to advertise namespaces for the owner of the pseudonym since they | ||
8092 | eliminate the need to first create a directory. | ||
8093 | |||
8094 | Collections are also advertised using @code{KSBlock}s. | ||
8095 | |||
8096 | @table @asis | ||
8097 | @item Attachment Size | ||
8098 | @item ecrs.pdf 270.68 KB | ||
8099 | @item https://gnunet.org/sites/default/files/ecrs.pdf | ||
8100 | @end table | ||
8101 | |||
8102 | @node File-sharing persistence directory structure | ||
8103 | @subsection File-sharing persistence directory structure | ||
8104 | |||
8105 | @c %**end of header | ||
8106 | |||
8107 | This section documents how the file-sharing library implements | ||
8108 | persistence of file-sharing operations and specifically the resulting | ||
8109 | directory structure. | ||
8110 | This code is only active if the @code{GNUNET_FS_FLAGS_PERSISTENCE} flag | ||
8111 | was set when calling @code{GNUNET_FS_start}. | ||
8112 | In this case, the file-sharing library will try hard to ensure that all | ||
8113 | major operations (searching, downloading, publishing, unindexing) are | ||
8114 | persistent, that is, can live longer than the process itself. | ||
8115 | More specifically, an operation is supposed to live until it is | ||
8116 | explicitly stopped. | ||
8117 | |||
8118 | If @code{GNUNET_FS_stop} is called before an operation has been stopped, a | ||
8119 | @code{SUSPEND} event is generated and then when the process calls | ||
8120 | @code{GNUNET_FS_start} next time, a @code{RESUME} event is generated. | ||
8121 | Additionally, even if an application crashes (segfault, SIGKILL, system | ||
8122 | crash) and hence @code{GNUNET_FS_stop} is never called and no | ||
8123 | @code{SUSPEND} events are generated, operations are still resumed (with | ||
8124 | @code{RESUME} events). | ||
8125 | This is implemented by constantly writing the current state of the | ||
8126 | file-sharing operations to disk. | ||
8127 | Specifically, the current state is always written to disk whenever | ||
8128 | anything significant changes (the exception are block-wise progress in | ||
8129 | publishing and unindexing, since those operations would be slowed down | ||
8130 | significantly and can be resumed cheaply even without detailed | ||
8131 | accounting). | ||
8132 | Note that if the process crashes (or is killed) during a serialization | ||
8133 | operation, FS does not guarantee that this specific operation is | ||
8134 | recoverable (no strict transactional semantics, again for performance | ||
8135 | reasons). However, all other unrelated operations should resume nicely. | ||
8136 | |||
8137 | Since we need to serialize the state continuously and want to recover as | ||
8138 | much as possible even after crashing during a serialization operation, | ||
8139 | we do not use one large file for serialization. | ||
8140 | Instead, several directories are used for the various operations. | ||
8141 | When @code{GNUNET_FS_start} executes, the master directories are scanned | ||
8142 | for files describing operations to resume. | ||
8143 | Sometimes, these operations can refer to related operations in child | ||
8144 | directories which may also be resumed at this point. | ||
8145 | Note that corrupted files are cleaned up automatically. | ||
8146 | However, dangling files in child directories (those that are not | ||
8147 | referenced by files from the master directories) are not automatically | ||
8148 | removed. | ||
8149 | |||
8150 | Persistence data is kept in a directory that begins with the "STATE_DIR" | ||
8151 | prefix from the configuration file | ||
8152 | (by default, "$SERVICEHOME/persistence/") followed by the name of the | ||
8153 | client as given to @code{GNUNET_FS_start} (for example, "gnunet-gtk") | ||
8154 | followed by the actual name of the master or child directory. | ||
8155 | |||
8156 | The names for the master directories follow the names of the operations: | ||
8157 | |||
8158 | @itemize @bullet | ||
8159 | @item "search" | ||
8160 | @item "download" | ||
8161 | @item "publish" | ||
8162 | @item "unindex" | ||
8163 | @end itemize | ||
8164 | |||
8165 | Each of the master directories contains names (chosen at random) for each | ||
8166 | active top-level (master) operation. | ||
8167 | Note that a download that is associated with a search result is not a | ||
8168 | top-level operation. | ||
8169 | |||
8170 | In contrast to the master directories, the child directories are only | ||
8171 | consulted when another operation refers to them. | ||
8172 | For each search, a subdirectory (named after the master search | ||
8173 | synchronization file) contains the search results. | ||
8174 | Search results can have an associated download, which is then stored in | ||
8175 | the general "download-child" directory. | ||
8176 | Downloads can be recursive, in which case children are stored in | ||
8177 | subdirectories mirroring the structure of the recursive download | ||
8178 | (either starting in the master "download" directory or in the | ||
8179 | "download-child" directory depending on how the download was initiated). | ||
8180 | For publishing operations, the "publish-file" directory contains | ||
8181 | information about the individual files and directories that are part of | ||
8182 | the publication. | ||
8183 | However, this directory structure is flat and does not mirror the | ||
8184 | structure of the publishing operation. | ||
8185 | Note that unindex operations cannot have associated child operations. | ||
8186 | |||
8187 | @cindex REGEX subsystem | ||
8188 | @node REGEX Subsystem | ||
8189 | @section REGEX Subsystem | ||
8190 | |||
8191 | @c %**end of header | ||
8192 | |||
8193 | Using the REGEX subsystem, you can discover peers that offer a particular | ||
8194 | service using regular expressions. | ||
8195 | The peers that offer a service specify it using a regular expressions. | ||
8196 | Peers that want to patronize a service search using a string. | ||
8197 | The REGEX subsystem will then use the DHT to return a set of matching | ||
8198 | offerers to the patrons. | ||
8199 | |||
8200 | For the technical details, we have Max's defense talk and Max's Master's | ||
8201 | thesis. | ||
8202 | |||
8203 | @c An additional publication is under preparation and available to | ||
8204 | @c team members (in Git). | ||
8205 | @c FIXME: Where is the file? Point to it. Assuming that it's szengel2012ms | ||
8206 | |||
8207 | @menu | ||
8208 | * How to run the regex profiler:: | ||
8209 | @end menu | ||
8210 | |||
8211 | @node How to run the regex profiler | ||
8212 | @subsection How to run the regex profiler | ||
8213 | |||
8214 | @c %**end of header | ||
8215 | |||
8216 | The gnunet-regex-profiler can be used to profile the usage of mesh/regex | ||
8217 | for a given set of regular expressions and strings. | ||
8218 | Mesh/regex allows you to announce your peer ID under a certain regex and | ||
8219 | search for peers matching a particular regex using a string. | ||
8220 | See @uref{https://gnunet.org/szengel2012ms, szengel2012ms} for a full | ||
8221 | introduction. | ||
8222 | |||
8223 | First of all, the regex profiler uses GNUnet testbed, thus all the | ||
8224 | implications for testbed also apply to the regex profiler | ||
8225 | (for example you need password-less ssh login to the machines listed in | ||
8226 | your hosts file). | ||
8227 | |||
8228 | @strong{Configuration} | ||
8229 | |||
8230 | Moreover, an appropriate configuration file is needed. | ||
8231 | Generally you can refer to the | ||
8232 | @file{contrib/regex_profiler_infiniband.conf} file in the sourcecode | ||
8233 | of GNUnet for an example configuration. | ||
8234 | In the following paragraph the important details are highlighted. | ||
8235 | |||
8236 | Announcing of the regular expressions is done by the | ||
8237 | gnunet-daemon-regexprofiler, therefore you have to make sure it is | ||
8238 | started, by adding it to the AUTOSTART set of ARM: | ||
8239 | |||
8240 | @example | ||
8241 | [regexprofiler] | ||
8242 | AUTOSTART = YES | ||
8243 | @end example | ||
8244 | |||
8245 | @noindent | ||
8246 | Furthermore you have to specify the location of the binary: | ||
8247 | |||
8248 | @example | ||
8249 | [regexprofiler] | ||
8250 | # Location of the gnunet-daemon-regexprofiler binary. | ||
8251 | BINARY = /home/szengel/gnunet/src/mesh/.libs/gnunet-daemon-regexprofiler | ||
8252 | # Regex prefix that will be applied to all regular expressions and | ||
8253 | # search string. | ||
8254 | REGEX_PREFIX = "GNVPN-0001-PAD" | ||
8255 | @end example | ||
8256 | |||
8257 | @noindent | ||
8258 | When running the profiler with a large scale deployment, you probably | ||
8259 | want to reduce the workload of each peer. | ||
8260 | Use the following options to do this. | ||
8261 | |||
8262 | @example | ||
8263 | [dht] | ||
8264 | # Force network size estimation | ||
8265 | FORCE_NSE = 1 | ||
8266 | |||
8267 | [dhtcache] | ||
8268 | DATABASE = heap | ||
8269 | # Disable RC-file for Bloom filter? (for benchmarking with limited IO | ||
8270 | # availability) | ||
8271 | DISABLE_BF_RC = YES | ||
8272 | # Disable Bloom filter entirely | ||
8273 | DISABLE_BF = YES | ||
8274 | |||
8275 | [nse] | ||
8276 | # Minimize proof-of-work CPU consumption by NSE | ||
8277 | WORKBITS = 1 | ||
8278 | @end example | ||
8279 | |||
8280 | @noindent | ||
8281 | @strong{Options} | ||
8282 | |||
8283 | To finally run the profiler some options and the input data need to be | ||
8284 | specified on the command line. | ||
8285 | |||
8286 | @example | ||
8287 | gnunet-regex-profiler -c config-file -d log-file -n num-links \ | ||
8288 | -p path-compression-length -s search-delay -t matching-timeout \ | ||
8289 | -a num-search-strings hosts-file policy-dir search-strings-file | ||
8290 | @end example | ||
8291 | |||
8292 | @noindent | ||
8293 | Where... | ||
8294 | |||
8295 | @itemize @bullet | ||
8296 | @item ... @code{config-file} means the configuration file created earlier. | ||
8297 | @item ... @code{log-file} is the file where to write statistics output. | ||
8298 | @item ... @code{num-links} indicates the number of random links between | ||
8299 | started peers. | ||
8300 | @item ... @code{path-compression-length} is the maximum path compression | ||
8301 | length in the DFA. | ||
8302 | @item ... @code{search-delay} time to wait between peers finished linking | ||
8303 | and starting to match strings. | ||
8304 | @item ... @code{matching-timeout} timeout after which to cancel the | ||
8305 | searching. | ||
8306 | @item ... @code{num-search-strings} number of strings in the | ||
8307 | search-strings-file. | ||
8308 | @item ... the @code{hosts-file} should contain a list of hosts for the | ||
8309 | testbed, one per line in the following format: | ||
8310 | |||
8311 | @itemize @bullet | ||
8312 | @item @code{user@@host_ip:port} | ||
8313 | @end itemize | ||
8314 | @item ... the @code{policy-dir} is a folder containing text files | ||
8315 | containing one or more regular expressions. A peer is started for each | ||
8316 | file in that folder and the regular expressions in the corresponding file | ||
8317 | are announced by this peer. | ||
8318 | @item ... the @code{search-strings-file} is a text file containing search | ||
8319 | strings, one in each line. | ||
8320 | @end itemize | ||
8321 | |||
8322 | @noindent | ||
8323 | You can create regular expressions and search strings for every AS in the | ||
8324 | Internet using the attached scripts. You need one of the | ||
8325 | @uref{http://data.caida.org/datasets/routing/routeviews-prefix2as/, CAIDA routeviews prefix2as} | ||
8326 | data files for this. Run | ||
8327 | |||
8328 | @example | ||
8329 | create_regex.py <filename> <output path> | ||
8330 | @end example | ||
8331 | |||
8332 | @noindent | ||
8333 | to create the regular expressions and | ||
8334 | |||
8335 | @example | ||
8336 | create_strings.py <input path> <outfile> | ||
8337 | @end example | ||
8338 | |||
8339 | @noindent | ||
8340 | to create a search strings file from the previously created | ||
8341 | regular expressions. | ||
diff --git a/doc/documentation/chapters/installation.texi b/doc/documentation/chapters/installation.texi new file mode 100644 index 000000000..eca77afcd --- /dev/null +++ b/doc/documentation/chapters/installation.texi | |||
@@ -0,0 +1,4123 @@ | |||
1 | @node GNUnet Installation Handbook | ||
2 | @chapter GNUnet Installation Handbook | ||
3 | |||
4 | This handbook describes how to install (build, setup, compile) and | ||
5 | setup (configure, start) GNUnet @value{VERSION}. After following these | ||
6 | instructions you should be able to install and then start user-interfaces | ||
7 | to interact with the network. | ||
8 | |||
9 | Note: This manual is far from complete, and we welcome informed | ||
10 | contributions, be it in the form of new chapters or insightful comments. | ||
11 | |||
12 | @menu | ||
13 | * Dependencies:: | ||
14 | * Pre-installation notes:: | ||
15 | * Generic installation instructions:: | ||
16 | * Build instructions for Ubuntu 12.04 using Git:: | ||
17 | * Build instructions for software builds from source:: | ||
18 | * Build Instructions for Microsoft Windows Platforms:: | ||
19 | * Build instructions for Debian 7.5:: | ||
20 | * Installing GNUnet from Git on Ubuntu 14.4:: | ||
21 | * Build instructions for Debian 8:: | ||
22 | * Outdated build instructions for previous revisions:: | ||
23 | @c * Portable GNUnet:: | ||
24 | * The graphical configuration interface:: | ||
25 | * How to start and stop a GNUnet peer:: | ||
26 | @end menu | ||
27 | |||
28 | @node Dependencies | ||
29 | @section Dependencies | ||
30 | @c %**end of header | ||
31 | |||
32 | This section lists the various known dependencies for | ||
33 | GNUnet @value{EDITION}. | ||
34 | Suggestions for missing dependencies or wrong version numbers are welcome. | ||
35 | |||
36 | @menu | ||
37 | * External dependencies:: | ||
38 | * Optional dependencies:: | ||
39 | * Internal dependencies:: | ||
40 | @end menu | ||
41 | |||
42 | @node External dependencies | ||
43 | @subsection External dependencies | ||
44 | @c %**end of header | ||
45 | |||
46 | These packages must be installed before a typical GNUnet installation | ||
47 | can be performed: | ||
48 | |||
49 | @itemize @bullet | ||
50 | @item autoconf | ||
51 | @item automake | ||
52 | @item pkg-config | ||
53 | @item libltdl | ||
54 | @item gstreamer | ||
55 | @item gst-plugins-base | ||
56 | @item perl | ||
57 | @item python (only 2.7 supported)@footnote{tests and gnunet-qr} | ||
58 | @item jansson | ||
59 | @item nss | ||
60 | @item glib | ||
61 | @item gmp | ||
62 | @item bluez | ||
63 | @item miniupnpc | ||
64 | @item gettext | ||
65 | @item which | ||
66 | @item texinfo @geq{} 5.2 | ||
67 | @item GNU libmicrohttpd @geq{} 0.9.30 @footnote{We recommend to build it | ||
68 | with a GnuTLS version that was configured with libunbound} | ||
69 | @item GNU libextractor @geq{} 1.0 | ||
70 | @item GNU libtool @geq{} 2.2 | ||
71 | @item GNU libunistring @geq{} 0.9.1.1 | ||
72 | @item GNU libidn @geq{} 1.0.0 | ||
73 | @item @uref{https://gnupg.org/software/libgcrypt/, GNU libgcrypt} @geq{} | ||
74 | @uref{https://gnupg.org/ftp/gcrypt/libgcrypt/, 1.6.0} | ||
75 | @item @uref{https://gnutls.org/, GnuTLS} @geq{} 3.2.7 | ||
76 | @footnote{We recommend to compile with libunbound for DANE support; | ||
77 | GnuTLS also requires GNU nettle 2.7 (update: GnuTLS 3.2.7 appears NOT | ||
78 | to work against GNU nettle > 2.7, due to some API updatings done by | ||
79 | nettle. Thus it should be compiled against nettle 2.7 | ||
80 | and, in case you get some error on the reference to `rpl_strerror' being | ||
81 | undefined, follow the instructions on | ||
82 | @uref{http://lists.gnupg.org/pipermail/gnutls-devel/2013-November/006588.html, this} | ||
83 | post (and the link inside it)).} | ||
84 | @item @uref{https://gnunet.org/gnurl, gnURL} libgnurl @geq{} 7.34.0 | ||
85 | @footnote{must be compiled after @code{GnuTLS}} | ||
86 | @item libglpk @geq{} 4.45 | ||
87 | @item @uref{http://www.openssl.org/, OpenSSL} @geq{} 1.0 | ||
88 | @item TeX Live @geq{} 2012, optional (for gnunet-bcd) | ||
89 | @item Texinfo @geq{} 5.2 (for documentation) | ||
90 | @item libsqlite @geq{} 3.8.0 @footnote{(note that the code will | ||
91 | compile and often work with lower version numbers, but you may get subtle | ||
92 | bugs with respect to quota management in certain rare cases); | ||
93 | alternatively, MySQL or Postgres can also be installed, but those | ||
94 | databases will require more complex configurations (not | ||
95 | recommended for first-time users)} | ||
96 | @item zlib | ||
97 | @end itemize | ||
98 | |||
99 | @node Optional dependencies | ||
100 | @subsection Optional dependencies | ||
101 | |||
102 | These applications must be installed for various experimental or otherwise | ||
103 | optional features such as @command{gnunet-conversation}, | ||
104 | and @command{gnunet-gtk} (most of these features are only build if you | ||
105 | configure GNUnet with @command{--enable-experimental}): | ||
106 | |||
107 | @itemize @bullet | ||
108 | @item libpulse @geq{} 2.0, | ||
109 | optional (for @command{gnunet-conversation}) | ||
110 | @item libopus @geq{} 1.0.1, | ||
111 | optional (for @command{gnunet-conversation}) | ||
112 | @item libogg @geq{} 1.3.0, | ||
113 | optional (for @command{gnunet-conversation}) | ||
114 | @item libnss contained @command{certool} binary, | ||
115 | optional for convenient installation of | ||
116 | the GNS proxy. | ||
117 | @item python-zbar @geq{} 0.10, | ||
118 | optional (for @command{gnunet-qr}) | ||
119 | @item Gtk+ @geq{} 3.0, | ||
120 | optional (for @command{gnunet-gtk}) | ||
121 | @item libgladeui (must match Gtk+ version), | ||
122 | optional (for @command{gnunet-gtk}) | ||
123 | @item libqrencode @geq{} 3.0, | ||
124 | optional (for @command{gnunet-namestore-gtk}) | ||
125 | @end itemize | ||
126 | |||
127 | @node Internal dependencies | ||
128 | @subsection Internal dependencies | ||
129 | |||
130 | This section tries to give an overview of what processes a typical GNUnet | ||
131 | peer running a particular application would consist of. All of the | ||
132 | processes listed here should be automatically started by | ||
133 | @command{gnunet-arm -s}. | ||
134 | The list is given as a rough first guide to users for failure diagnostics. | ||
135 | Ideally, end-users should never have to worry about these internal | ||
136 | dependencies. | ||
137 | |||
138 | In terms of internal dependencies, a minimum file-sharing system consists | ||
139 | of the following GNUnet processes (in order of dependency): | ||
140 | |||
141 | @itemize @bullet | ||
142 | @item gnunet-service-arm | ||
143 | @item gnunet-service-resolver (required by all) | ||
144 | @item gnunet-service-statistics (required by all) | ||
145 | @item gnunet-service-peerinfo | ||
146 | @item gnunet-service-transport (requires peerinfo) | ||
147 | @item gnunet-service-core (requires transport) | ||
148 | @item gnunet-daemon-hostlist (requires core) | ||
149 | @item gnunet-daemon-topology (requires hostlist, peerinfo) | ||
150 | @item gnunet-service-datastore | ||
151 | @item gnunet-service-dht (requires core) | ||
152 | @item gnunet-service-identity | ||
153 | @item gnunet-service-fs (requires identity, mesh, dht, datastore, core) | ||
154 | @end itemize | ||
155 | |||
156 | @noindent | ||
157 | A minimum VPN system consists of the following GNUnet processes (in | ||
158 | order of dependency): | ||
159 | |||
160 | @itemize @bullet | ||
161 | @item gnunet-service-arm | ||
162 | @item gnunet-service-resolver (required by all) | ||
163 | @item gnunet-service-statistics (required by all) | ||
164 | @item gnunet-service-peerinfo | ||
165 | @item gnunet-service-transport (requires peerinfo) | ||
166 | @item gnunet-service-core (requires transport) | ||
167 | @item gnunet-daemon-hostlist (requires core) | ||
168 | @item gnunet-service-dht (requires core) | ||
169 | @item gnunet-service-mesh (requires dht, core) | ||
170 | @item gnunet-service-dns (requires dht) | ||
171 | @item gnunet-service-regex (requires dht) | ||
172 | @item gnunet-service-vpn (requires regex, dns, mesh, dht) | ||
173 | @end itemize | ||
174 | |||
175 | @noindent | ||
176 | A minimum GNS system consists of the following GNUnet processes (in | ||
177 | order of dependency): | ||
178 | |||
179 | @itemize @bullet | ||
180 | @item gnunet-service-arm | ||
181 | @item gnunet-service-resolver (required by all) | ||
182 | @item gnunet-service-statistics (required by all) | ||
183 | @item gnunet-service-peerinfo | ||
184 | @item gnunet-service-transport (requires peerinfo) | ||
185 | @item gnunet-service-core (requires transport) | ||
186 | @item gnunet-daemon-hostlist (requires core) | ||
187 | @item gnunet-service-dht (requires core) | ||
188 | @item gnunet-service-mesh (requires dht, core) | ||
189 | @item gnunet-service-dns (requires dht) | ||
190 | @item gnunet-service-regex (requires dht) | ||
191 | @item gnunet-service-vpn (requires regex, dns, mesh, dht) | ||
192 | @item gnunet-service-identity | ||
193 | @item gnunet-service-namestore (requires identity) | ||
194 | @item gnunet-service-gns (requires vpn, dns, dht, namestore, identity) | ||
195 | @end itemize | ||
196 | |||
197 | @node Pre-installation notes | ||
198 | @section Pre-installation notes | ||
199 | |||
200 | Please note that in the code instructions for the installation, | ||
201 | @emph{#} indicates commands run as privileged root user and | ||
202 | @emph{$} shows commands run as unprivileged ("normal") system user. | ||
203 | |||
204 | |||
205 | @node Generic installation instructions | ||
206 | @section Generic installation instructions | ||
207 | |||
208 | First, in addition to the GNUnet sources you might require downloading the | ||
209 | latest version of various dependencies, depending on how recent the | ||
210 | software versions in your distribution of GNU/Linux are. | ||
211 | Most distributions do not include sufficiently recent versions of these | ||
212 | dependencies. | ||
213 | Thus, a typically installation on a "modern" GNU/Linux distribution | ||
214 | requires you to install the following dependencies (ideally in this | ||
215 | order): | ||
216 | |||
217 | @itemize @bullet | ||
218 | @item libgpgerror and libgcrypt | ||
219 | @item libnettle and libunbound (possibly from distribution), GnuTLS | ||
220 | @item libgnurl (read the README) | ||
221 | @item GNU libmicrohttpd | ||
222 | @item GNU libextractor | ||
223 | @end itemize | ||
224 | |||
225 | Make sure to first install the various mandatory and optional | ||
226 | dependencies including development headers from your distribution. | ||
227 | |||
228 | Other dependencies that you should strongly consider to install is a | ||
229 | database (MySQL, sqlite or Postgres). | ||
230 | The following instructions will assume that you installed at least sqlite. | ||
231 | For most distributions you should be able to find pre-build packages for | ||
232 | the database. Again, make sure to install the client libraries @b{and} the | ||
233 | respective development headers (if they are packaged separately) as well. | ||
234 | |||
235 | You can find specific, detailed instructions for installing of the | ||
236 | dependencies (and possibly the rest of the GNUnet installation) in the | ||
237 | platform-specific descriptions, which can be found in the Index. | ||
238 | Please consult them now. | ||
239 | If your distribution is not listed, please study | ||
240 | @ref{Build instructions for Debian 8}, the build instructions for | ||
241 | Debian stable, carefully as you try to install the dependencies for your | ||
242 | own distribution. | ||
243 | Contributing additional instructions for further platforms is always | ||
244 | appreciated. | ||
245 | Please take in mind that operating system development tends to move at | ||
246 | a rather fast speed. Due to this you should be aware that some of | ||
247 | the instructions could be outdated by the time you are reading this. | ||
248 | If you find a mistake, please tell us about it (or even better: send | ||
249 | a patch to the documentation to fix it!). | ||
250 | |||
251 | Before proceeding further, please double-check the dependency list. | ||
252 | Note that in addition to satisfying the dependencies, you might have to | ||
253 | make sure that development headers for the various libraries are also | ||
254 | installed. | ||
255 | There maybe files for other distributions, or you might be able to find | ||
256 | equivalent packages for your distribution. | ||
257 | |||
258 | While it is possible to build and install GNUnet without having root | ||
259 | access, we will assume that you have full control over your system in | ||
260 | these instructions. | ||
261 | First, you should create a system user @emph{gnunet} and an additional | ||
262 | group @emph{gnunetdns}. On the GNU/Linux distributions Debian and Ubuntu, | ||
263 | type: | ||
264 | |||
265 | @example | ||
266 | # adduser --system --home /var/lib/gnunet --group \ | ||
267 | --disabled-password gnunet | ||
268 | # addgroup --system gnunetdns | ||
269 | @end example | ||
270 | |||
271 | @noindent | ||
272 | On other Unixes and GNU systems, this should have the same effect: | ||
273 | |||
274 | @example | ||
275 | # useradd --system --groups gnunet --home-dir /var/lib/gnunet | ||
276 | # addgroup --system gnunetdns | ||
277 | @end example | ||
278 | |||
279 | Now compile and install GNUnet using: | ||
280 | |||
281 | @example | ||
282 | $ tar xvf gnunet-@value{VERSION}.tar.gz | ||
283 | $ cd gnunet-@value{VERSION} | ||
284 | $ ./configure --with-sudo=sudo --with-nssdir=/lib | ||
285 | $ make | ||
286 | $ sudo make install | ||
287 | @end example | ||
288 | |||
289 | If you want to be able to enable DEBUG-level log messages, add | ||
290 | @code{--enable-logging=verbose} to the end of the | ||
291 | @command{./configure} command. | ||
292 | @code{DEBUG}-level log messages are in English only and | ||
293 | should only be useful for developers (or for filing | ||
294 | really detailed bug reports). | ||
295 | |||
296 | Finally, you probably want to compile @command{gnunet-gtk}, which | ||
297 | includes @command{gnunet-setup} (a graphical tool for | ||
298 | GNUnet configuration) and @command{gnunet-fs-gtk} (a graphical tool for | ||
299 | GNUnet file-sharing): | ||
300 | |||
301 | @example | ||
302 | $ tar xvf gnunet-gtk-@value{VERSION}.tar.gz | ||
303 | $ cd gnunet-gtk-@value{VERSION} | ||
304 | $ ./configure --with-gnunet=/usr/local/ | ||
305 | $ make | ||
306 | $ sudo make install | ||
307 | $ cd .. | ||
308 | # just to be safe run this: | ||
309 | $ sudo ldconfig | ||
310 | @end example | ||
311 | |||
312 | @noindent | ||
313 | Next, edit the file @file{/etc/gnunet.conf} to contain the following: | ||
314 | |||
315 | @example | ||
316 | [arm] | ||
317 | SYSTEM_ONLY = YES | ||
318 | USER_ONLY = NO | ||
319 | @end example | ||
320 | |||
321 | @noindent | ||
322 | You may need to update your @code{ld.so} cache to include | ||
323 | files installed in @file{/usr/local/lib}: | ||
324 | |||
325 | @example | ||
326 | # ldconfig | ||
327 | @end example | ||
328 | |||
329 | @noindent | ||
330 | Then, switch from user @code{root} to user @code{gnunet} to start | ||
331 | the peer: | ||
332 | |||
333 | @example | ||
334 | # su -s /bin/sh - gnunet | ||
335 | $ gnunet-arm -c /etc/gnunet.conf -s | ||
336 | @end example | ||
337 | |||
338 | You may also want to add the last line in the gnunet user's @file{crontab} | ||
339 | prefixed with @code{@@reboot} so that it is executed whenever the system | ||
340 | is booted: | ||
341 | |||
342 | @example | ||
343 | @@reboot /usr/local/bin/gnunet-arm -c /etc/gnunet.conf -s | ||
344 | @end example | ||
345 | |||
346 | @noindent | ||
347 | This will only start the system-wide GNUnet services. | ||
348 | Type exit to get back your root shell. | ||
349 | Now, you need to configure the per-user part. For each | ||
350 | $USER that should get access to GNUnet on the system, run: | ||
351 | |||
352 | @example | ||
353 | # adduser $USER gnunet | ||
354 | @end example | ||
355 | |||
356 | @noindent | ||
357 | to allow them to access the system-wide GNUnet services. Then, each | ||
358 | user should create a configuration file @file{~/.config/gnunet.conf} | ||
359 | with the lines: | ||
360 | |||
361 | @example | ||
362 | [arm] | ||
363 | SYSTEM_ONLY = NO | ||
364 | USER_ONLY = YES | ||
365 | DEFAULTSERVICES = gns | ||
366 | @end example | ||
367 | |||
368 | @noindent | ||
369 | and start the per-user services using | ||
370 | |||
371 | @example | ||
372 | $ gnunet-arm -c ~/.config/gnunet.conf -s | ||
373 | @end example | ||
374 | |||
375 | @noindent | ||
376 | Again, adding a @code{crontab} entry to autostart the peer is advised: | ||
377 | |||
378 | @example | ||
379 | @@reboot /usr/local/bin/gnunet-arm -c $HOME/.config/gnunet.conf -s | ||
380 | @end example | ||
381 | |||
382 | @noindent | ||
383 | Note that some GNUnet services (such as SOCKS5 proxies) may need a | ||
384 | system-wide TCP port for each user. | ||
385 | For those services, systems with more than one user may require each user | ||
386 | to specify a different port number in their personal configuration file. | ||
387 | |||
388 | Finally, the user should perform the basic initial setup for the GNU Name | ||
389 | System (GNS). This is done by running two commands: | ||
390 | |||
391 | @example | ||
392 | $ gnunet-gns-import.sh | ||
393 | $ gnunet-gns-proxy-setup-ca | ||
394 | @end example | ||
395 | |||
396 | @noindent | ||
397 | The first generates the default zones, wheras the second setups the GNS | ||
398 | Certificate Authority with the user's browser. Now, to activate GNS in the | ||
399 | normal DNS resolution process, you need to edit your | ||
400 | @file{/etc/nsswitch.conf} where you should find a line like this: | ||
401 | |||
402 | @example | ||
403 | hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4 | ||
404 | @end example | ||
405 | |||
406 | @noindent | ||
407 | The exact details may differ a bit, which is fine. Add the text | ||
408 | @emph{"gns [NOTFOUND=return]"} after @emph{"files"}. | ||
409 | Keep in mind that we included a backslash ("\") here just for | ||
410 | markup reasons. You should write the text below on @b{one line} | ||
411 | and @b{without} the "\": | ||
412 | |||
413 | @example | ||
414 | hosts: files gns [NOTFOUND=return] mdns4_minimal \ | ||
415 | [NOTFOUND=return] dns mdns4 | ||
416 | @end example | ||
417 | |||
418 | @c FIXME: Document new behavior. | ||
419 | You might want to make sure that @file{/lib/libnss_gns.so.2} exists on | ||
420 | your system, it should have been created during the installation. | ||
421 | |||
422 | @node Build instructions for Ubuntu 12.04 using Git | ||
423 | @section Build instructions for Ubuntu 12.04 using Git | ||
424 | |||
425 | @menu | ||
426 | * Install the required build tools:: | ||
427 | * Install libgcrypt 1.6 and libgpg-error:: | ||
428 | * Install gnutls with DANE support:: | ||
429 | * Install libgnurl:: | ||
430 | * Install libmicrohttpd from Git:: | ||
431 | * Install libextractor from Git:: | ||
432 | * Install GNUnet dependencies:: | ||
433 | * Build GNUnet:: | ||
434 | * Install the GNUnet-gtk user interface from Git:: | ||
435 | @end menu | ||
436 | |||
437 | @node Install the required build tools | ||
438 | @subsection Install the required build tools | ||
439 | |||
440 | First, make sure Git is installed on your system: | ||
441 | |||
442 | @example | ||
443 | $ sudo apt-get install git | ||
444 | @end example | ||
445 | |||
446 | Install the essential buildtools: | ||
447 | |||
448 | @example | ||
449 | $ sudo apt-get install automake autopoint autoconf libtool | ||
450 | @end example | ||
451 | |||
452 | @node Install libgcrypt 1.6 and libgpg-error | ||
453 | @subsection Install libgcrypt 1.6 and libgpg-error | ||
454 | |||
455 | @ref{generic source installation - libgpg-error} | ||
456 | |||
457 | @node Install gnutls with DANE support | ||
458 | @subsection Install gnutls with DANE support | ||
459 | |||
460 | @itemize @bullet | ||
461 | @item @ref{generic source installation - nettle} | ||
462 | @item @ref{generic source installation - ldns} | ||
463 | @item @ref{generic source installation - libunbound/unbound} | ||
464 | @item @ref{generic source installation - gnutls} | ||
465 | @item @ref{generic source installation - libgcrypt} | ||
466 | @end itemize | ||
467 | |||
468 | @node Install libgnurl | ||
469 | @subsection Install libgnurl | ||
470 | |||
471 | Follow the @ref{generic source installation - libgnurl}. | ||
472 | |||
473 | @node Install libmicrohttpd from Git | ||
474 | @subsection Install libmicrohttpd from Git | ||
475 | |||
476 | @example | ||
477 | $ git clone https://gnunet.org/git/libmicrohttpd | ||
478 | $ cd libmicrohttpd/ | ||
479 | $ ./bootstrap | ||
480 | $ ./configure | ||
481 | $ sudo make install ; cd .. | ||
482 | @end example | ||
483 | |||
484 | @node Install libextractor from Git | ||
485 | @subsection Install libextractor from Git | ||
486 | |||
487 | Install libextractor dependencies: | ||
488 | |||
489 | @example | ||
490 | $ sudo apt-get install zlib1g-dev libgsf-1-dev libmpeg2-4-dev \ | ||
491 | libpoppler-dev libvorbis-dev libexiv2-dev libjpeg-dev \ | ||
492 | libtiff-dev libgif-dev libvorbis-dev libflac-dev libsmf-dev \ | ||
493 | g++ | ||
494 | @end example | ||
495 | |||
496 | Build libextractor: | ||
497 | |||
498 | @example | ||
499 | $ git clone https://gnunet.org/git/libextractor | ||
500 | $ cd libextractor | ||
501 | $ ./bootstrap | ||
502 | $ ./configure | ||
503 | $ sudo make install ; cd .. | ||
504 | @end example | ||
505 | |||
506 | @node Install GNUnet dependencies | ||
507 | @subsection Install GNUnet dependencies | ||
508 | |||
509 | @example | ||
510 | $ sudo apt-get install libidn11-dev libunistring-dev libglpk-dev \ | ||
511 | libpulse-dev libbluetooth-dev libsqlite-dev | ||
512 | @end example | ||
513 | |||
514 | Install libopus: | ||
515 | |||
516 | @example | ||
517 | $ wget http://downloads.xiph.org/releases/opus/opus-1.1.tar.gz | ||
518 | $ tar xf opus-1.1.tar.gz | ||
519 | $ cd opus-1.1/ | ||
520 | $ ./configure | ||
521 | $ sudo make install ; cd .. | ||
522 | @end example | ||
523 | |||
524 | Choose one or more database backends: | ||
525 | |||
526 | SQLite3: | ||
527 | @example | ||
528 | $ sudo apt-get install libsqlite3-dev | ||
529 | @end example | ||
530 | MySQL: | ||
531 | @example | ||
532 | $ sudo apt-get install libmysqlclient-dev | ||
533 | @end example | ||
534 | PostgreSQL: | ||
535 | @example | ||
536 | $ sudo apt-get install libpq-dev postgresql | ||
537 | @end example | ||
538 | |||
539 | |||
540 | |||
541 | @node Build GNUnet | ||
542 | @subsection Build GNUnet | ||
543 | |||
544 | |||
545 | |||
546 | @menu | ||
547 | * Configuring the installation path:: | ||
548 | * Configuring the system:: | ||
549 | * Installing components requiring sudo permission:: | ||
550 | * Build:: | ||
551 | @end menu | ||
552 | |||
553 | @node Configuring the installation path | ||
554 | @subsubsection Configuring the installation path | ||
555 | |||
556 | You can specify the location of the GNUnet installation by setting the | ||
557 | prefix when calling the configure script with @code{--prefix=DIRECTORY} | ||
558 | |||
559 | @example | ||
560 | $ export PATH=$PATH:DIRECTORY/bin | ||
561 | @end example | ||
562 | |||
563 | @node Configuring the system | ||
564 | @subsubsection Configuring the system | ||
565 | |||
566 | Please make sure NOW that you have created a user and group 'gnunet' | ||
567 | and additionally a group 'gnunetdns': | ||
568 | |||
569 | @example | ||
570 | $ sudo addgroup gnunet | ||
571 | $ sudo addgroup gnunetdns | ||
572 | $ sudo adduser gnunet | ||
573 | @end example | ||
574 | |||
575 | Each GNUnet user should be added to the 'gnunet' group (may | ||
576 | require fresh login to come into effect): | ||
577 | |||
578 | @example | ||
579 | $ sudo useradd -G gnunet | ||
580 | @end example | ||
581 | |||
582 | @node Installing components requiring sudo permission | ||
583 | @subsubsection Installing components requiring sudo permission | ||
584 | |||
585 | Some components, like the nss plugin required for GNS, may require root | ||
586 | permissions. To allow these few components to be installed use: | ||
587 | |||
588 | @example | ||
589 | $ ./configure --with-sudo | ||
590 | @end example | ||
591 | |||
592 | @node Build | ||
593 | @subsubsection Build | ||
594 | |||
595 | @example | ||
596 | $ git clone https://gnunet.org/git/gnunet/ | ||
597 | $ cd gnunet/ | ||
598 | $ ./bootstrap | ||
599 | @end example | ||
600 | |||
601 | Use the required configure call including the optional installation prefix | ||
602 | @code{PREFIX} or the sudo permissions: | ||
603 | |||
604 | @example | ||
605 | $ ./configure [ --with-sudo | --with-prefix=PREFIX ] | ||
606 | @end example | ||
607 | |||
608 | @example | ||
609 | $ make; sudo make install | ||
610 | @end example | ||
611 | |||
612 | After installing it, you need to create an empty configuration file: | ||
613 | |||
614 | @example | ||
615 | mkdir ~/.gnunet; touch ~/.gnunet/gnunet.conf | ||
616 | @end example | ||
617 | |||
618 | And finally you can start GNUnet with: | ||
619 | |||
620 | @example | ||
621 | $ gnunet-arm -s | ||
622 | @end example | ||
623 | |||
624 | @node Install the GNUnet-gtk user interface from Git | ||
625 | @subsection Install the GNUnet-gtk user interface from Git | ||
626 | |||
627 | |||
628 | Install depencies: | ||
629 | |||
630 | @example | ||
631 | $ sudo apt-get install libgtk-3-dev libunique-3.0-dev libgladeui-dev \ | ||
632 | libqrencode-dev | ||
633 | @end example | ||
634 | |||
635 | Build GNUnet (with an optional prefix) and execute: | ||
636 | |||
637 | @example | ||
638 | $ git clone https://gnunet.org/git/gnunet-gtk/ | ||
639 | $ cd gnunet-gtk/ | ||
640 | $ ./bootstrap | ||
641 | $ ./configure [--prefix=PREFIX] --with-gnunet=DIRECTORY | ||
642 | $ make; sudo make install | ||
643 | @end example | ||
644 | |||
645 | @node Build instructions for software builds from source | ||
646 | @section Build instructions for software builds from source | ||
647 | |||
648 | This section describes software builds in case your operating | ||
649 | system lacks binary substitutes / binary builds for some dependencies | ||
650 | of GNUnet. | ||
651 | It is assumed that you have installed common build dependencies | ||
652 | and that these instructions are treated as generic without any | ||
653 | debugging help. | ||
654 | It is furthermore assumed that you use the release tarballs of | ||
655 | the software, installation from the respective version control | ||
656 | sources might differ in ways that are only minimal different | ||
657 | (for example a dependency on autotools etc). | ||
658 | |||
659 | @menu | ||
660 | * generic source installation - nettle:: | ||
661 | * generic source installation - ldns:: | ||
662 | * generic source installation - libunbound/unbound:: | ||
663 | * generic source installation - libav:: | ||
664 | * generic source installation - libextractor:: | ||
665 | * generic source installation - libgpg-error:: | ||
666 | * generic source installation - libgcrypt:: | ||
667 | * generic source installation - gnutls:: | ||
668 | * generic source installation - libmicrohttpd:: | ||
669 | * generic source installation - libgnurl:: | ||
670 | @end menu | ||
671 | |||
672 | @node generic source installation - nettle | ||
673 | @subsection generic source installation - nettle | ||
674 | |||
675 | @example | ||
676 | $ wget http://www.lysator.liu.se/~nisse/archive/nettle-2.7.1.tar.gz | ||
677 | $ tar xf nettle-2.7.1.tar.gz | ||
678 | $ cd nettle-2.7.1 | ||
679 | $ ./configure | ||
680 | $ sudo make install ; cd .. | ||
681 | @end example | ||
682 | |||
683 | @node generic source installation - ldns | ||
684 | @subsection generic source installation - ldns | ||
685 | |||
686 | @example | ||
687 | $ wget https://www.nlnetlabs.nl/downloads/ldns/ldns-1.6.16.tar.gz | ||
688 | $ tar xf ldns-1.6.16.tar.gz | ||
689 | $ cd ldns-1.6.16 | ||
690 | $ ./configure | ||
691 | $ sudo make install ; cd .. | ||
692 | @end example | ||
693 | |||
694 | @node generic source installation - libunbound/unbound | ||
695 | @subsection generic source installation - libunbound/unbound | ||
696 | |||
697 | @example | ||
698 | $ wget https://unbound.net/downloads/unbound-1.4.21.tar.gz | ||
699 | $ tar xf unbound-1.4.21.tar.gz | ||
700 | $ cd unbound-1.4.21 | ||
701 | $ ./configure | ||
702 | $ sudo make install ; cd .. | ||
703 | @end example | ||
704 | |||
705 | @node generic source installation - libav | ||
706 | @subsection generic source installation - libav | ||
707 | |||
708 | @example | ||
709 | $ wget https://libav.org/releases/libav-9.10.tar.xz | ||
710 | $ cd libav-0.9 ; ./configure --enable-shared; | ||
711 | $ make; sudo make install; cd .. | ||
712 | @end example | ||
713 | |||
714 | @node generic source installation - libextractor | ||
715 | @subsection generic source installation - libextractor | ||
716 | |||
717 | @example | ||
718 | $ wget https://ftp.gnu.org/gnu/libextractor/libextractor-1.3.tar.gz | ||
719 | $ tar xvf libextractor-1.3.tar.gz | ||
720 | $ cd libextractor-1.3 ; ./configure; | ||
721 | $ make ; sudo make install; cd .. | ||
722 | @end example | ||
723 | |||
724 | @node generic source installation - libgpg-error | ||
725 | @subsection generic source installation - libgpg-error | ||
726 | |||
727 | @example | ||
728 | $ wget https://ftp.gnupg.org/gcrypt/libgpg-error/libgpg-error-1.12.tar.bz2 | ||
729 | $ tar xvf libgpg-error-1.12.tar.bz2 | ||
730 | $ cd libgpg-error-1.12; ./configure; | ||
731 | $ make ; sudo make install; cd .. | ||
732 | @end example | ||
733 | |||
734 | @node generic source installation - libgcrypt | ||
735 | @subsection generic source installation - libgcrypt | ||
736 | @example | ||
737 | $ wget https://ftp.gnupg.org/gcrypt/libgcrypt/libgcrypt-1.6.0.tar.bz2 | ||
738 | $ tar xvf libgcrypt-1.6.0.tar.bz2 | ||
739 | $ cd libgcrypt-1.6.0; ./configure --with-gpg-error-prefix=/usr/local; | ||
740 | $ make ; sudo make install ; cd .. | ||
741 | @end example | ||
742 | |||
743 | @node generic source installation - gnutls | ||
744 | @subsection generic source installation - gnutls | ||
745 | |||
746 | @example | ||
747 | $ wget ftp://ftp.gnutls.org/gcrypt/gnutls/v3.2/gnutls-3.2.7.tar.xz | ||
748 | $ tar xvf gnutls-3.2.7.tar.xz | ||
749 | $ cd gnutls-3.2.7 | ||
750 | @end example | ||
751 | |||
752 | @noindent | ||
753 | If you want a GnuTLS with DANE functionality (recommended for GNUnet), | ||
754 | you have to compile it against libunbound. Assuming that libunbound | ||
755 | is installed on your system: | ||
756 | |||
757 | @example | ||
758 | $ ./configure --enable-libdane | ||
759 | @end example | ||
760 | |||
761 | @noindent | ||
762 | Note that the build system of GnuTLS should pick up libunbound without | ||
763 | the explicit mention of @code{--enable-libdane}. | ||
764 | If you don't want libdane support you should pass @code{--disable-libdane} | ||
765 | instead. | ||
766 | |||
767 | @example | ||
768 | $ ./configure | ||
769 | $ make ; sudo make install ; cd .. | ||
770 | @end example | ||
771 | |||
772 | @node generic source installation - libmicrohttpd | ||
773 | @subsection generic source installation - libmicrohttpd | ||
774 | |||
775 | @example | ||
776 | $ wget https://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.33.tar.gz | ||
777 | $ tar xvf libmicrohttpd-0.9.33.tar.gz | ||
778 | $ cd libmicrohttpd-0.9.33; ./configure; | ||
779 | $ make ; sudo make install ; cd .. | ||
780 | @end example | ||
781 | |||
782 | @node generic source installation - libgnurl | ||
783 | @subsection generic source installation - libgnurl | ||
784 | |||
785 | Example installation of libgnurl version 7.57.0 from source. | ||
786 | |||
787 | @example | ||
788 | $ wget https://ftp.gnu.org/gnu/gnunet/gnurl-7.57.0.tar.xz | ||
789 | $ wget https://ftp.gnu.org/gnu/gnunet/gnurl-7.57.0.tar.xz.sig | ||
790 | $ gpg --verify gnurl-7.57.0.tar.xz.sig | ||
791 | @end example | ||
792 | |||
793 | @noindent | ||
794 | If that command fails because you do not have the required public key, | ||
795 | then run this command to import it: | ||
796 | |||
797 | @example | ||
798 | $ gpg --keyserver pgp.mit.edu --recv-keys A88C8ADD129828D7EAC02E52E22F9BBFEE348588 | ||
799 | @end example | ||
800 | |||
801 | @noindent | ||
802 | and rerun the gpg --verify command. | ||
803 | |||
804 | @example | ||
805 | $ tar xvf gnurl-7.57.0.tar.xz | ||
806 | $ cd gnurl-7.57.0 | ||
807 | $ ./configure --disable-ntlm-wb | ||
808 | $ make ; sudo make install; cd .. | ||
809 | @end example | ||
810 | |||
811 | You have now build and installed libgnurl from source. | ||
812 | |||
813 | @menu | ||
814 | * Fixing libgnurl build issues:: | ||
815 | @end menu | ||
816 | |||
817 | @node Fixing libgnurl build issues | ||
818 | @subsubsection Fixing libgnurl build issues | ||
819 | |||
820 | @c FIXME: Obviously this subsection should be evaluated and | ||
821 | @c if still necessary moved into gnURL itself (README) or | ||
822 | @c into a separate section which deals with gnURL. | ||
823 | If you have to compile libgnurl from source (for example if the version | ||
824 | included in your distribution is too old or it's not included at all) | ||
825 | you perhaps might get an error message while running the | ||
826 | @command{configure} script: | ||
827 | |||
828 | @example | ||
829 | $ configure | ||
830 | ... | ||
831 | checking for 64-bit curl_off_t data type... unknown | ||
832 | checking for 32-bit curl_off_t data type... unknown | ||
833 | checking for 16-bit curl_off_t data type... unknown | ||
834 | configure: error: cannot find data type for curl_off_t. | ||
835 | @end example | ||
836 | |||
837 | @noindent | ||
838 | Solution: | ||
839 | |||
840 | Before running the @command{configure} script, set: | ||
841 | |||
842 | @example | ||
843 | CFLAGS="-I. -I$BUILD_ROOT/include" | ||
844 | @end example | ||
845 | |||
846 | @node Build Instructions for Microsoft Windows Platforms | ||
847 | @section Build Instructions for Microsoft Windows Platforms | ||
848 | |||
849 | @menu | ||
850 | * Introduction to building on MS Windows:: | ||
851 | * Requirements:: | ||
852 | * Dependencies & Initial Setup:: | ||
853 | * GNUnet Installation:: | ||
854 | * Adjusting Windows for running and testing GNUnet:: | ||
855 | * Building the GNUnet Installer:: | ||
856 | * Using GNUnet with Netbeans on Windows:: | ||
857 | @end menu | ||
858 | |||
859 | @node Introduction to building on MS Windows | ||
860 | @subsection Introduction to building on MS Windows | ||
861 | |||
862 | |||
863 | This document is a guide to building GNUnet and its dependencies on | ||
864 | Windows platforms. GNUnet development is mostly done under GNU/Linux and | ||
865 | especially git checkouts may not build out of the box. | ||
866 | We regret any inconvenience, and if you have problems, please report | ||
867 | them. | ||
868 | |||
869 | @node Requirements | ||
870 | @subsection Requirements | ||
871 | |||
872 | The Howto is based upon a @strong{Windows Server 2008 32bit} | ||
873 | @strong{Installation}, @strong{sbuild} and thus a | ||
874 | @uref{http://www.mingw.org/wiki/MSYS, MSYS+MinGW} | ||
875 | (W32-GCC-Compiler-Suite + Unix-like Userland) installation. sbuild | ||
876 | is a convenient set of scripts which creates a working msys/mingw | ||
877 | installation and installs most dependencies required for GNUnet. | ||
878 | |||
879 | As of the point of the creation of these instructions, | ||
880 | GNUnet @strong{requires} a Windows @strong{Server} 2003 or | ||
881 | newer for full feature support. | ||
882 | Windows Vista and later will also work, but | ||
883 | @strong{non-server version can not run a VPN-Exit-Node} as the NAT | ||
884 | features have been removed as of Windows Vista. | ||
885 | |||
886 | @c TODO: We should document Windows 10! | ||
887 | @c It seems like the situation hasn't changed with W10 | ||
888 | |||
889 | @node Dependencies & Initial Setup | ||
890 | @subsection Dependencies & Initial Setup | ||
891 | |||
892 | |||
893 | @itemize @bullet | ||
894 | |||
895 | @item | ||
896 | Install a fresh version of @strong{Python 2.x}, even if you are using a | ||
897 | x64-OS, install a 32-bit version for use with sbuild. | ||
898 | Python 3.0 is currently incompatible. | ||
899 | |||
900 | @item | ||
901 | Install your favorite @uref{http://code.google.com/p/tortoisegit/, git} & | ||
902 | @uref{http://tortoisesvn.net/, subversion}-clients. | ||
903 | |||
904 | @item | ||
905 | You will also need some archive-manager like | ||
906 | @uref{http://www.7-zip.org/, 7zip}. | ||
907 | |||
908 | @item | ||
909 | Pull a copy of sbuild to a directory of your choice, which will be used | ||
910 | in the remainder of this guide. For now, we will use | ||
911 | @file{c:\gnunet\sbuild\} | ||
912 | |||
913 | @item | ||
914 | in @file{sbuild\src\mingw\mingw32-buildall.sh}, comment out the packages | ||
915 | @strong{gnunet-svn} and @strong{gnunet-gtk-svn}, as we don't want sbuild | ||
916 | to compile/install those for us. | ||
917 | |||
918 | @item | ||
919 | Follow LRN's sbuild installation instructions.- | ||
920 | @end itemize | ||
921 | |||
922 | Please note that sbuild may (or will most likely) fail during | ||
923 | installation, thus you really HAVE to @strong{check the logfiles} created | ||
924 | during the installation process. | ||
925 | Certain packages may fail to build initially due to missing dependencies, | ||
926 | thus you may have to | ||
927 | @strong{substitute those with binary-versions initially}. Later on once | ||
928 | dependencies are satisfied you can re-build the newer package versions. | ||
929 | |||
930 | @strong{It is normal that you may have to repeat this step multiple times | ||
931 | and there is no uniform way to fix all compile-time issues, as the | ||
932 | build-process of many of the dependencies installed are rather unstable | ||
933 | on win32 and certain releases may not even compile at all.} | ||
934 | |||
935 | Most dependencies for GNUnet have been set up by sbuild, thus we now | ||
936 | should add the @file{bin/} directories in your new msys and mingw | ||
937 | installations to PATH. You will want to create a backup of your finished | ||
938 | msys-environment by now. | ||
939 | |||
940 | @node GNUnet Installation | ||
941 | @subsection GNUnet Installation | ||
942 | |||
943 | First, we need to launch our msys-shell, you can do this via | ||
944 | |||
945 | @file{C:\gnunet\sbuild\msys\msys.bat} | ||
946 | |||
947 | You might wish to take a look at this file and adjust some | ||
948 | login-parameters to your msys environment. | ||
949 | |||
950 | Also, sbuild added two pointpoints to your msys-environment, though those | ||
951 | might remain invisible: | ||
952 | |||
953 | @itemize @bullet | ||
954 | |||
955 | @item | ||
956 | /mingw, which will mount your mingw-directory from sbuild/mingw and the | ||
957 | other one is | ||
958 | |||
959 | @item | ||
960 | /src which contains all the installation sources sbuild just compiled. | ||
961 | @end itemize | ||
962 | |||
963 | Check out the current GNUnet sources (git HEAD) from the | ||
964 | GNUnet repository "gnunet.git", we will do this in your home directory: | ||
965 | |||
966 | @code{git clone https://gnunet.org/git/gnunet/ ~/gnunet} | ||
967 | |||
968 | Now, we will first need to bootstrap the checked out installation and then | ||
969 | configure it accordingly. | ||
970 | |||
971 | @example | ||
972 | cd ~/gnunet | ||
973 | ./bootstrap | ||
974 | STRIP=true CPPFLAGS="-DUSE_IPV6=1 -DW32_VEH" CFLAGS="$CFLAGS -g -O2" \ | ||
975 | ./configure --prefix=/ --docdir=/share/doc/gnunet \ | ||
976 | --with-libiconv-prefix=/mingw --with-libintl-prefix=/mingw \ | ||
977 | --with-libcurl=/mingw --with-extractor=/mingw --with-sqlite=/mingw \ | ||
978 | --with-microhttpd=/mingw --with-plibc=/mingw --enable-benchmarks \ | ||
979 | --enable-expensivetests --enable-experimental --with-qrencode=/mingw \ | ||
980 | --enable-silent-rules --enable-experimental 2>&1 | tee -a ./configure.log | ||
981 | @end example | ||
982 | |||
983 | The parameters above will configure for a reasonable GNUnet installation | ||
984 | to the your msys-root directory. | ||
985 | Depending on which features your would like to build or you may need to | ||
986 | specify additional dependencies. Sbuild installed most libs into | ||
987 | the /mingw subdirectory, so remember to prefix library locations with | ||
988 | this path. | ||
989 | |||
990 | Like on a unixoid system, you might want to use your home directory as | ||
991 | prefix for your own GNUnet installation for development, without tainting | ||
992 | the buildenvironment. Just change the "prefix" parameter to point towards | ||
993 | ~/ in this case. | ||
994 | |||
995 | Now it's time to compile GNUnet as usual. Though this will take some time, | ||
996 | so you may fetch yourself a coffee or some Mate now... | ||
997 | |||
998 | @example | ||
999 | make ; make install | ||
1000 | @end example | ||
1001 | |||
1002 | @node Adjusting Windows for running and testing GNUnet | ||
1003 | @subsection Adjusting Windows for running and testing GNUnet | ||
1004 | |||
1005 | Assuming the build succeeded and you | ||
1006 | @strong{added the bin directory of your GNUnet to PATH}, you can now use | ||
1007 | your gnunet-installation as usual. | ||
1008 | Remember that UAC or the windows firewall may popup initially, blocking | ||
1009 | further execution of gnunet until you acknowledge them. | ||
1010 | |||
1011 | You will also have to take the usual steps to get peer-to-peer (p2p) | ||
1012 | software running properly (port forwarding, ...), | ||
1013 | and GNUnet will require administrative permissions as it may even | ||
1014 | install a device-driver (in case you are using gnunet-vpn and/or | ||
1015 | gnunet-exit). | ||
1016 | |||
1017 | @node Building the GNUnet Installer | ||
1018 | @subsection Building the GNUnet Installer | ||
1019 | |||
1020 | The GNUnet installer is made with | ||
1021 | @uref{http://nsis.sourceforge.net/, NSIS}. | ||
1022 | The installer script is located in @file{contrib\win} in the | ||
1023 | GNUnet source tree. | ||
1024 | |||
1025 | @node Using GNUnet with Netbeans on Windows | ||
1026 | @subsection Using GNUnet with Netbeans on Windows | ||
1027 | |||
1028 | TODO | ||
1029 | |||
1030 | @node Build instructions for Debian 7.5 | ||
1031 | @section Build instructions for Debian 7.5 | ||
1032 | |||
1033 | |||
1034 | These are the installation instructions for Debian 7.5. They were tested | ||
1035 | using a minimal, fresh Debian 7.5 AMD64 installation without non-free | ||
1036 | software (no contrib or non-free). | ||
1037 | By "minimal", we mean that during installation, we did not select any | ||
1038 | desktop environment, servers or system utilities during the "tasksel" | ||
1039 | step. Note that the packages and the dependencies that we will install | ||
1040 | during this chapter take about 1.5 GB of disk space. | ||
1041 | Combined with GNUnet and space for objects during compilation, you should | ||
1042 | not even attempt this unless you have about 2.5 GB free after the minimal | ||
1043 | Debian installation. | ||
1044 | Using these instructions to build a VM image is likely to require a | ||
1045 | minimum of 4-5 GB for the VM (as you will likely also want a desktop | ||
1046 | manager). | ||
1047 | |||
1048 | GNUnet's security model assumes that your @file{/home} directory is | ||
1049 | encrypted. Thus, if possible, you should encrypt your home partition | ||
1050 | (or per-user home directory). | ||
1051 | |||
1052 | Naturally, the exact details of the starting state for your installation | ||
1053 | should not matter much. For example, if you selected any of those | ||
1054 | installation groups you might simply already have some of the necessary | ||
1055 | packages installed. | ||
1056 | We did this for testing, as this way we are less likely to forget to | ||
1057 | mention a required package. | ||
1058 | Note that we will not install a desktop environment, but of course you | ||
1059 | will need to install one to use GNUnet's graphical user interfaces. | ||
1060 | Thus, it is suggested that you simply install the desktop environment of | ||
1061 | your choice before beginning with the instructions. | ||
1062 | |||
1063 | |||
1064 | |||
1065 | @menu | ||
1066 | * Update:: | ||
1067 | * Stable? Hah!:: | ||
1068 | * Update again:: | ||
1069 | * Installing packages:: | ||
1070 | * Installing dependencies from source:: | ||
1071 | * Installing GNUnet from source:: | ||
1072 | * But wait there is more!:: | ||
1073 | @end menu | ||
1074 | |||
1075 | @node Update | ||
1076 | @subsection Update | ||
1077 | |||
1078 | After any installation, you should begin by running | ||
1079 | |||
1080 | @example | ||
1081 | # apt-get update ; apt-get upgrade | ||
1082 | @end example | ||
1083 | |||
1084 | to ensure that all of your packages are up-to-date. Note that the "#" is | ||
1085 | used to indicate that you need to type in this command as "root" | ||
1086 | (or prefix with "sudo"), whereas "$" is used to indicate typing in a | ||
1087 | command as a normal user. | ||
1088 | |||
1089 | @node Stable? Hah! | ||
1090 | @subsection Stable? Hah! | ||
1091 | |||
1092 | Yes, we said we start with a Debian 7.5 "stable" system. However, to | ||
1093 | reduce the amount of compilation by hand, we will begin by allowing the | ||
1094 | installation of packages from the testing and unstable distributions as | ||
1095 | well. | ||
1096 | We will stick to "stable" packages where possible, but some packages will | ||
1097 | be taken from the other distributions. | ||
1098 | Start by modifying @file{/etc/apt/sources.list} to contain the | ||
1099 | following (possibly adjusted to point to your mirror of choice): | ||
1100 | |||
1101 | @example | ||
1102 | # These were there before: | ||
1103 | deb http://ftp.de.debian.org/debian/ wheezy main | ||
1104 | deb-src http://ftp.de.debian.org/debian/ wheezy main | ||
1105 | deb http://security.debian.org/ wheezy/updates main | ||
1106 | deb-src http://security.debian.org/ wheezy/updates main | ||
1107 | deb http://ftp.de.debian.org/debian/ wheezy-updates main | ||
1108 | deb-src http://ftp.de.debian.org/debian/ wheezy-updates main | ||
1109 | |||
1110 | # Add these lines (feel free to adjust the mirror): | ||
1111 | deb http://ftp.de.debian.org/debian/ testing main | ||
1112 | deb http://ftp.de.debian.org/debian/ unstable main | ||
1113 | @end example | ||
1114 | |||
1115 | The next step is to create/edit your @file{/etc/apt/preferences} | ||
1116 | file to look like this: | ||
1117 | |||
1118 | @example | ||
1119 | Package: * | ||
1120 | Pin: release a=stable,n=wheezy | ||
1121 | Pin-Priority: 700 | ||
1122 | |||
1123 | Package: * | ||
1124 | Pin: release o=Debian,a=testing | ||
1125 | Pin-Priority: 650 | ||
1126 | |||
1127 | Package: * | ||
1128 | Pin: release o=Debian,a=unstable | ||
1129 | Pin-Priority: 600 | ||
1130 | @end example | ||
1131 | |||
1132 | You can read more about Apt Preferences here and here. | ||
1133 | Note that other pinnings are likely to also work for GNUnet, the key | ||
1134 | thing is that you need some packages from unstable (as shown below). | ||
1135 | However, as unstable is unlikely to be comprehensive (missing packages) | ||
1136 | or might be problematic (crashing packages), you probably want others | ||
1137 | from stable and/or testing. | ||
1138 | |||
1139 | @node Update again | ||
1140 | @subsection Update again | ||
1141 | |||
1142 | Now, run again@ | ||
1143 | |||
1144 | @example | ||
1145 | # apt-get update@ | ||
1146 | # apt-get upgrade@ | ||
1147 | @end example | ||
1148 | |||
1149 | to ensure that all your new distribution indices are downloaded, and | ||
1150 | that your pinning is correct: the upgrade step should cause no changes | ||
1151 | at all. | ||
1152 | |||
1153 | @node Installing packages | ||
1154 | @subsection Installing packages | ||
1155 | |||
1156 | We begin by installing a few Debian packages from stable:@ | ||
1157 | |||
1158 | @example | ||
1159 | # apt-get install gcc make python-zbar libltdl-dev libsqlite3-dev \ | ||
1160 | libunistring-dev libopus-dev libpulse-dev openssl libglpk-dev \ | ||
1161 | texlive libidn11-dev libmysqlclient-dev libpq-dev libarchive-dev \ | ||
1162 | libbz2-dev libexiv2-dev libflac-dev libgif-dev libglib2.0-dev \ | ||
1163 | libgtk-3-dev libmagic-dev libjpeg8-dev libmpeg2-4-dev libmp4v2-dev \ | ||
1164 | librpm-dev libsmf-dev libtidy-dev libtiff5-dev libvorbis-dev \ | ||
1165 | libogg-dev zlib1g-dev g++ gettext libgsf-1-dev libunbound-dev \ | ||
1166 | libqrencode-dev libgladeui-dev nasm texlive-latex-extra \ | ||
1167 | libunique-3.0-dev gawk miniupnpc libfuse-dev libbluetooth-dev | ||
1168 | @end example | ||
1169 | |||
1170 | After that, we install a few more packages from unstable:@ | ||
1171 | |||
1172 | @example | ||
1173 | # apt-get install -t unstable nettle-dev libgstreamer1.0-dev \ | ||
1174 | gstreamer1.0-plugins-base gstreamer1.0-plugins-good \ | ||
1175 | libgstreamer-plugins-base1.0-dev | ||
1176 | @end example | ||
1177 | |||
1178 | @node Installing dependencies from source | ||
1179 | @subsection Installing dependencies from source | ||
1180 | |||
1181 | Next, we need to install a few dependencies from source. | ||
1182 | You might want to do this as a "normal" user and only run the | ||
1183 | @code{make install} steps as root (hence the @code{sudo} in the | ||
1184 | commands below). Also, you do this from any | ||
1185 | directory. We begin by downloading all dependencies, then extracting the | ||
1186 | sources, and finally compiling and installing the libraries. | ||
1187 | |||
1188 | For these steps, follow the instructions given in the | ||
1189 | installation from source instruction in this order: | ||
1190 | |||
1191 | @itemize @bullet | ||
1192 | @item @ref{generic source installation - libav} | ||
1193 | @item @ref{generic source installation - libextractor} | ||
1194 | @item @ref{generic source installation - libgpg-error} | ||
1195 | @item @ref{generic source installation - libgcrypt} | ||
1196 | @item @ref{generic source installation - gnutls} | ||
1197 | @item @ref{generic source installation - libmicrohttpd} | ||
1198 | @item @ref{generic source installation - libgnurl} | ||
1199 | @end itemize | ||
1200 | |||
1201 | @node Installing GNUnet from source | ||
1202 | @subsection Installing GNUnet from source | ||
1203 | |||
1204 | |||
1205 | For this, simply follow the generic installation instructions from | ||
1206 | here. | ||
1207 | |||
1208 | @node But wait there is more! | ||
1209 | @subsection But wait there is more! | ||
1210 | |||
1211 | So far, we installed all of the packages and dependencies required to | ||
1212 | ensure that all of GNUnet would be built. | ||
1213 | However, while for example the plugins to interact with the MySQL or | ||
1214 | Postgres databases have been created, we did not actually install or | ||
1215 | configure those databases. Thus, you will need to install | ||
1216 | and configure those databases or stick with the default Sqlite database. | ||
1217 | Sqlite is usually fine for most applications, but MySQL can offer better | ||
1218 | performance and Postgres better resillience. | ||
1219 | |||
1220 | |||
1221 | @node Installing GNUnet from Git on Ubuntu 14.4 | ||
1222 | @section Installing GNUnet from Git on Ubuntu 14.4 | ||
1223 | |||
1224 | @strong{Install the required build tools:} | ||
1225 | |||
1226 | @example | ||
1227 | $ sudo apt-get install git automake autopoint autoconf | ||
1228 | @end example | ||
1229 | |||
1230 | @strong{Install the required dependencies} | ||
1231 | |||
1232 | @example | ||
1233 | $ sudo apt-get install libltdl-dev libgpg-error-dev libidn11-dev \ | ||
1234 | libunistring-dev libglpk-dev libbluetooth-dev libextractor-dev \ | ||
1235 | libmicrohttpd-dev libgnutls28-dev | ||
1236 | @end example | ||
1237 | |||
1238 | @strong{Choose one or more database backends} | ||
1239 | |||
1240 | @itemize @bullet | ||
1241 | |||
1242 | @item SQLite3: | ||
1243 | |||
1244 | @example | ||
1245 | $ sudo apt-get install libsqlite3-dev | ||
1246 | @end example | ||
1247 | |||
1248 | @item MySQL: | ||
1249 | |||
1250 | @example | ||
1251 | $ sudo apt-get install libmysqlclient-dev | ||
1252 | @end example | ||
1253 | |||
1254 | @item PostgreSQL: | ||
1255 | |||
1256 | @example | ||
1257 | $ sudo apt-get install libpq-dev postgresql | ||
1258 | @end example | ||
1259 | |||
1260 | @end itemize | ||
1261 | |||
1262 | @strong{Install the optional dependencies for gnunet-conversation:} | ||
1263 | |||
1264 | @example | ||
1265 | $ sudo apt-get install gstreamer1.0 libpulse-dev libopus-dev | ||
1266 | @end example | ||
1267 | |||
1268 | @strong{Install the libgrypt 1.6.1:} | ||
1269 | |||
1270 | @itemize @bullet | ||
1271 | |||
1272 | @item For Ubuntu 14.04: | ||
1273 | |||
1274 | @example | ||
1275 | $ sudo apt-get install libgcrypt20-dev | ||
1276 | @end example | ||
1277 | |||
1278 | @item For Ubuntu older 14.04: | ||
1279 | |||
1280 | @example | ||
1281 | $ wget ftp://ftp.gnupg.org/gcrypt/libgcrypt/libgcrypt-1.6.1.tar.bz2 | ||
1282 | $ tar xf libgcrypt-1.6.1.tar.bz2 | ||
1283 | $ cd libgcrypt-1.6.1 | ||
1284 | $ ./configure | ||
1285 | $ sudo make install | ||
1286 | $ cd .. | ||
1287 | @end example | ||
1288 | |||
1289 | @end itemize | ||
1290 | |||
1291 | @strong{Install libgnurl} | ||
1292 | |||
1293 | @example | ||
1294 | $ wget https://gnunet.org/sites/default/files/gnurl-7.35.0.tar.bz2 | ||
1295 | $ tar xf gnurl-7.35.0.tar.bz2 | ||
1296 | $ cd gnurl-7.35.0 | ||
1297 | $ ./configure --enable-ipv6 --with-gnutls --without-libssh2 \ | ||
1298 | --without-libmetalink --without-winidn --without-librtmp \ | ||
1299 | --without-nghttp2 --without-nss --without-cyassl --without-polarssl \ | ||
1300 | --without-ssl --without-winssl --without-darwinssl --disable-sspi \ | ||
1301 | --disable-ntlm-wb --disable-ldap --disable-rtsp --disable-dict \ | ||
1302 | --disable-telnet --disable-tftp --disable-pop3 --disable-imap \ | ||
1303 | --disable-smtp --disable-gopher --disable-file --disable-ftp | ||
1304 | $ sudo make install | ||
1305 | $ cd .. | ||
1306 | @end example | ||
1307 | |||
1308 | @strong{Install GNUnet} | ||
1309 | |||
1310 | @example | ||
1311 | $ git clone https://gnunet.org/git/gnunet/ | ||
1312 | $ cd gnunet/ | ||
1313 | $ ./bootstrap | ||
1314 | @end example | ||
1315 | |||
1316 | If you want to: | ||
1317 | |||
1318 | @itemize @bullet | ||
1319 | |||
1320 | @item Install to a different directory: | ||
1321 | |||
1322 | @example | ||
1323 | --prefix=PREFIX | ||
1324 | @end example | ||
1325 | |||
1326 | @item | ||
1327 | Have sudo permission, but do not want to compile as root: | ||
1328 | |||
1329 | @example | ||
1330 | --with-sudo | ||
1331 | @end example | ||
1332 | |||
1333 | @item | ||
1334 | Want debug message enabled: | ||
1335 | |||
1336 | @example | ||
1337 | --enable-logging=verbose | ||
1338 | @end example | ||
1339 | |||
1340 | @end itemize | ||
1341 | |||
1342 | |||
1343 | @example | ||
1344 | $ ./configure [ --with-sudo | --prefix=PREFIX | --enable-logging=verbose] | ||
1345 | $ make; sudo make install | ||
1346 | @end example | ||
1347 | |||
1348 | After installing it, you need to create an empty configuration file: | ||
1349 | |||
1350 | @example | ||
1351 | touch ~/.config/gnunet.conf | ||
1352 | @end example | ||
1353 | |||
1354 | And finally you can start GNUnet with | ||
1355 | |||
1356 | @example | ||
1357 | $ gnunet-arm -s | ||
1358 | @end example | ||
1359 | |||
1360 | @node Build instructions for Debian 8 | ||
1361 | @section Build instructions for Debian 8 | ||
1362 | @c FIXME: I -> we | ||
1363 | |||
1364 | These are the installation instructions for Debian 8. They were tested | ||
1365 | sing a fresh Debian 8 AMD64 installation without non-free software (no | ||
1366 | contrib or non-free). During installation, I only selected "lxde" for the | ||
1367 | desktop environment. | ||
1368 | Note that the packages and the dependencies that we will install during | ||
1369 | this chapter take about 1.5 GB of disk space. Combined with GNUnet and | ||
1370 | space for objects during compilation, you should not even attempt this | ||
1371 | unless you have about 2.5 GB free after the Debian installation. | ||
1372 | Using these instructions to build a VM image is likely to require a | ||
1373 | minimum of 4-5 GB for the VM (as you will likely also want a desktop | ||
1374 | manager). | ||
1375 | |||
1376 | GNUnet's security model assumes that your @code{/home} directory is | ||
1377 | encrypted. | ||
1378 | Thus, if possible, you should encrypt your entire disk, or at least just | ||
1379 | your home partition (or per-user home directory). | ||
1380 | |||
1381 | Naturally, the exact details of the starting state for your installation | ||
1382 | should not matter much. | ||
1383 | For example, if you selected any of those installation groups you might | ||
1384 | simply already have some of the necessary packages installed. Thus, it is | ||
1385 | suggested that you simply install the desktop environment of your choice | ||
1386 | before beginning with the instructions. | ||
1387 | |||
1388 | |||
1389 | @menu | ||
1390 | * Update Debian:: | ||
1391 | * Installing Debian Packages:: | ||
1392 | * Installing Dependencies from Source2:: | ||
1393 | * Installing GNUnet from Source2:: | ||
1394 | * But wait (again) there is more!:: | ||
1395 | @end menu | ||
1396 | |||
1397 | @node Update Debian | ||
1398 | @subsection Update Debian | ||
1399 | |||
1400 | After any installation, you should begin by running | ||
1401 | |||
1402 | @example | ||
1403 | # apt-get update | ||
1404 | # apt-get upgrade | ||
1405 | @end example | ||
1406 | |||
1407 | to ensure that all of your packages are up-to-date. Note that the "#" is | ||
1408 | used to indicate that you need to type in this command as "root" (or | ||
1409 | prefix with "sudo"), whereas "$" is used to indicate typing in a command | ||
1410 | as a normal user. | ||
1411 | |||
1412 | @node Installing Debian Packages | ||
1413 | @subsection Installing Debian Packages | ||
1414 | |||
1415 | We begin by installing a few Debian packages from stable: | ||
1416 | |||
1417 | @example | ||
1418 | # apt-get install gcc make python-zbar libltdl-dev libsqlite3-dev \ | ||
1419 | libunistring-dev libopus-dev libpulse-dev openssl libglpk-dev texlive \ | ||
1420 | libidn11-dev libmysqlclient-dev libpq-dev libarchive-dev libbz2-dev \ | ||
1421 | libflac-dev libgif-dev libglib2.0-dev libgtk-3-dev libmpeg2-4-dev \ | ||
1422 | libtidy-dev libvorbis-dev libogg-dev zlib1g-dev g++ gettext \ | ||
1423 | libgsf-1-dev libunbound-dev libqrencode-dev libgladeui-dev nasm \ | ||
1424 | texlive-latex-extra libunique-3.0-dev gawk miniupnpc libfuse-dev \ | ||
1425 | libbluetooth-dev gstreamer1.0-plugins-base gstreamer1.0-plugins-good \ | ||
1426 | libgstreamer-plugins-base1.0-dev nettle-dev libextractor-dev \ | ||
1427 | libgcrypt20-dev libmicrohttpd-dev | ||
1428 | @end example | ||
1429 | |||
1430 | @node Installing Dependencies from Source2 | ||
1431 | @subsection Installing Dependencies from Source2 | ||
1432 | |||
1433 | Yes, we said we start with a Debian 8 "stable" system, but because Debian | ||
1434 | linked GnuTLS without support for DANE, we need to compile a few things, | ||
1435 | in addition to GNUnet, still by hand. Yes, you can run GNUnet using the | ||
1436 | respective Debian packages, but then you will not get DANE support. | ||
1437 | |||
1438 | Next, we need to install a few dependencies from source. You might want | ||
1439 | to do this as a "normal" user and only run the @code{make install} steps | ||
1440 | as root (hence the @code{sudo} in the commands below). Also, you do this | ||
1441 | from any directory. We begin by downloading all dependencies, then | ||
1442 | extracting the sources, and finally compiling and installing the | ||
1443 | libraries: | ||
1444 | |||
1445 | @example | ||
1446 | $ wget ftp://ftp.gnutls.org/gcrypt/gnutls/v3.3/gnutls-3.3.12.tar.xz | ||
1447 | $ tar xvf gnutls-3.3.12.tar.xz | ||
1448 | $ cd gnutls-3.3.12 ; ./configure ; make ; sudo make install ; cd .. | ||
1449 | @end example | ||
1450 | |||
1451 | For the installation and compilation of libgnurl/gnURL refer to | ||
1452 | the generic installation section, | ||
1453 | @xref{generic source installation - libgnurl}. | ||
1454 | |||
1455 | @node Installing GNUnet from Source2 | ||
1456 | @subsection Installing GNUnet from Source2 | ||
1457 | |||
1458 | For this, simply follow the generic installation instructions from@ | ||
1459 | here. | ||
1460 | |||
1461 | @node But wait (again) there is more! | ||
1462 | @subsection But wait (again) there is more! | ||
1463 | |||
1464 | So far, we installed all of the packages and dependencies required to | ||
1465 | ensure that all of GNUnet would be built. However, while for example the | ||
1466 | plugins to interact with the MySQL or Postgres databases have been | ||
1467 | created, we did not actually install or configure those databases. | ||
1468 | Thus, you will need to install and configure those databases or stick | ||
1469 | with the default Sqlite database. Sqlite is usually fine for most | ||
1470 | applications, but MySQL can offer better performance and Postgres better | ||
1471 | resillience. | ||
1472 | |||
1473 | @node Outdated build instructions for previous revisions | ||
1474 | @section Outdated build instructions for previous revisions | ||
1475 | |||
1476 | This chapter contains a collection of outdated, older installation guides. | ||
1477 | They are mostly intended to serve as a starting point for writing | ||
1478 | up-to-date instructions and should not be expected to work for | ||
1479 | GNUnet 0.10.x. | ||
1480 | A set of older installation instructions can also be found in the | ||
1481 | file @file{doc/outdated-and-old-installation-instructions.txt} in the | ||
1482 | source tree of GNUnet. | ||
1483 | |||
1484 | This file covers old instructions which no longer receive security | ||
1485 | updates or any kind of support. | ||
1486 | |||
1487 | @menu | ||
1488 | * Installing GNUnet 0.10.1 on Ubuntu 14.04:: | ||
1489 | * Building GLPK for MinGW:: | ||
1490 | * GUI build instructions for Ubuntu 12.04 using Subversion:: | ||
1491 | @c * Installation with gnunet-update:: | ||
1492 | * Instructions for Microsoft Windows Platforms (Old):: | ||
1493 | @end menu | ||
1494 | |||
1495 | |||
1496 | @node Installing GNUnet 0.10.1 on Ubuntu 14.04 | ||
1497 | @subsection Installing GNUnet 0.10.1 on Ubuntu 14.04 | ||
1498 | |||
1499 | Install the required dependencies: | ||
1500 | |||
1501 | @example | ||
1502 | $ sudo apt-get install libltdl-dev libgpg-error-dev libidn11-dev \ | ||
1503 | libunistring-dev libglpk-dev libbluetooth-dev libextractor-dev \ | ||
1504 | libmicrohttpd-dev libgnutls28-dev | ||
1505 | @end example | ||
1506 | |||
1507 | Choose one or more database backends: | ||
1508 | |||
1509 | @itemize @bullet | ||
1510 | |||
1511 | @item SQLite3 | ||
1512 | |||
1513 | @example | ||
1514 | $ sudo apt-get install libsqlite3-dev@ | ||
1515 | @end example | ||
1516 | |||
1517 | @item MySQL | ||
1518 | |||
1519 | @example | ||
1520 | $ sudo apt-get install libmysqlclient-dev@ | ||
1521 | @end example | ||
1522 | |||
1523 | @item PostgreSQL | ||
1524 | |||
1525 | @example | ||
1526 | $ sudo apt-get install libpq-dev postgresql@ | ||
1527 | @end example | ||
1528 | |||
1529 | @end itemize | ||
1530 | |||
1531 | Install the optional dependencies for gnunet-conversation: | ||
1532 | |||
1533 | @example | ||
1534 | $ sudo apt-get install gstreamer1.0 libpulse-dev libopus-dev | ||
1535 | @end example | ||
1536 | |||
1537 | Install libgcrypt 1.6: | ||
1538 | |||
1539 | @itemize @bullet | ||
1540 | |||
1541 | @item For Ubuntu 14.04: | ||
1542 | |||
1543 | @example | ||
1544 | $ sudo apt-get install libgcrypt20-dev | ||
1545 | @end example | ||
1546 | |||
1547 | @item For Ubuntu older than 14.04: | ||
1548 | |||
1549 | @example | ||
1550 | wget ftp://ftp.gnupg.org/gcrypt/libgcrypt/libgcrypt-1.6.1.tar.bz2 | ||
1551 | $ tar xf libgcrypt-1.6.1.tar.bz2 | ||
1552 | $ cd libgcrypt-1.6.1 | ||
1553 | $ ./configure | ||
1554 | $ sudo make install | ||
1555 | $ cd .. | ||
1556 | @end example | ||
1557 | @end itemize | ||
1558 | |||
1559 | Install libgnurl: | ||
1560 | |||
1561 | @pxref{generic source installation - libgnurl}. | ||
1562 | |||
1563 | Install GNUnet: | ||
1564 | |||
1565 | @example | ||
1566 | $ wget http://ftpmirror.gnu.org/gnunet/gnunet-0.10.1.tar.gz | ||
1567 | $ tar xf gnunet-0.10.1.tar.gz | ||
1568 | $ cd gnunet-0.10.1 | ||
1569 | @end example | ||
1570 | |||
1571 | If you want to: | ||
1572 | |||
1573 | @itemize @bullet | ||
1574 | |||
1575 | @item | ||
1576 | Install to a different directory: | ||
1577 | |||
1578 | @example | ||
1579 | --prefix=PREFIX | ||
1580 | @end example | ||
1581 | |||
1582 | @item | ||
1583 | Have sudo permission, but do not want to compile as root: | ||
1584 | |||
1585 | @example | ||
1586 | --with-sudo | ||
1587 | @end example | ||
1588 | |||
1589 | @item | ||
1590 | Want debug message enabled: | ||
1591 | |||
1592 | @example | ||
1593 | --enable-logging=verbose | ||
1594 | @end example | ||
1595 | |||
1596 | @end itemize | ||
1597 | |||
1598 | @example | ||
1599 | $ ./configure [ --with-sudo | --prefix=PREFIX | --enable-logging=verbose] | ||
1600 | $ make; sudo make install | ||
1601 | @end example | ||
1602 | |||
1603 | After installing it, you need to create an empty configuration file: | ||
1604 | |||
1605 | @example | ||
1606 | touch ~/.config/gnunet.conf | ||
1607 | @end example | ||
1608 | |||
1609 | And finally you can start GNUnet with | ||
1610 | |||
1611 | @example | ||
1612 | $ gnunet-arm -s | ||
1613 | @end example | ||
1614 | |||
1615 | @node Building GLPK for MinGW | ||
1616 | @subsection Building GLPK for MinGW | ||
1617 | |||
1618 | GNUnet now requires the GNU Linear Programming Kit (GLPK). | ||
1619 | Since there's is no package you can install with @code{mingw-get} you | ||
1620 | have to compile it from source: | ||
1621 | |||
1622 | @itemize @bullet | ||
1623 | |||
1624 | @item Download the latest version from | ||
1625 | @uref{http://ftp.gnu.org/gnu/glpk/} | ||
1626 | |||
1627 | @item Unzip the downloaded source tarball using your favourite | ||
1628 | unzipper application In the MSYS shell | ||
1629 | |||
1630 | @item change to the respective directory | ||
1631 | |||
1632 | @item Configure glpk for "i686-pc-mingw32": | ||
1633 | |||
1634 | @example | ||
1635 | ./configure '--build=i686-pc-mingw32' | ||
1636 | @end example | ||
1637 | |||
1638 | @item run | ||
1639 | |||
1640 | @example | ||
1641 | make install check | ||
1642 | @end example | ||
1643 | |||
1644 | @end itemize | ||
1645 | |||
1646 | MinGW does not automatically detect the correct buildtype so you have to | ||
1647 | specify it manually. | ||
1648 | |||
1649 | |||
1650 | @node GUI build instructions for Ubuntu 12.04 using Subversion | ||
1651 | @subsection GUI build instructions for Ubuntu 12.04 using Subversion | ||
1652 | |||
1653 | After installing GNUnet you can continue installing the GNUnet GUI tools: | ||
1654 | |||
1655 | First, install the required dependencies: | ||
1656 | |||
1657 | @example | ||
1658 | $ sudo apt-get install libgladeui-dev libqrencode-dev | ||
1659 | @end example | ||
1660 | |||
1661 | Please ensure that the GNUnet shared libraries can be found by the linker. | ||
1662 | If you installed GNUnet libraries in a non standard path | ||
1663 | (say GNUNET_PREFIX=/usr/local/lib/), you can | ||
1664 | |||
1665 | @itemize @bullet | ||
1666 | |||
1667 | @item set the environmental variable permanently to: | ||
1668 | |||
1669 | @example | ||
1670 | LD_LIBRARY_PATH=$GNUNET_PREFIX | ||
1671 | @end example | ||
1672 | |||
1673 | @item or add @code{$GNUNET_PREFIX} to @file{/etc/ld.so.conf} | ||
1674 | |||
1675 | @end itemize | ||
1676 | |||
1677 | Now you can checkout and compile the GNUnet GUI tools: | ||
1678 | |||
1679 | @example | ||
1680 | $ git clone https://gnunet.org/git/gnunet-gtk | ||
1681 | $ cd gnunet-gtk | ||
1682 | $ ./bootstrap | ||
1683 | $ ./configure --prefix=$GNUNET_PREFIX/.. --with-gnunet=$GNUNET_PREFIX/.. | ||
1684 | $ make install | ||
1685 | @end example | ||
1686 | |||
1687 | @c @node Installation with gnunet-update | ||
1688 | @c @subsection Installation with gnunet-update | ||
1689 | |||
1690 | @c gnunet-update project is an effort to introduce updates to GNUnet | ||
1691 | @c installations. An interesting to-be-implemented-feature of gnunet-update | ||
1692 | @c is that these updates are propagated through GNUnet's peer-to-peer | ||
1693 | @c network. More information about gnunet-update can be found at | ||
1694 | @c @c FIXME: Use correct cgit URL | ||
1695 | @c @uref{https://gnunet.org/git/gnunet-update.git/tree/plain/README}. | ||
1696 | |||
1697 | @c While the project is still under development, we have implemented the | ||
1698 | @c following features which we believe may be helpful for users and we | ||
1699 | @c would like them to be tested: | ||
1700 | |||
1701 | @c @itemize @bullet | ||
1702 | |||
1703 | @c @item | ||
1704 | @c Packaging GNUnet installation along with its run-time dependencies into | ||
1705 | @c update packages | ||
1706 | |||
1707 | @c @item | ||
1708 | @c Installing update packages into compatible hosts | ||
1709 | |||
1710 | @c @item | ||
1711 | @c Updating an existing installation (which had been installed by | ||
1712 | @c gnunet-update) to a newer one | ||
1713 | |||
1714 | @c @end itemize | ||
1715 | |||
1716 | @c The above said features of gnunet-update are currently available for | ||
1717 | @c testing on GNU/Linux systems. | ||
1718 | |||
1719 | @c The following is a guide to help you get started with gnunet-update. | ||
1720 | @c It shows you how to install the testing binary packages of GNUnet | ||
1721 | @c 0.9.1 we have at @uref{https://gnunet.org/install/}. | ||
1722 | |||
1723 | @c gnunet-update needs the following dependencies: | ||
1724 | |||
1725 | @c @itemize @bullet | ||
1726 | @c @item | ||
1727 | @c python @geq{} 2.6 | ||
1728 | |||
1729 | @c @item | ||
1730 | @c gnupg | ||
1731 | |||
1732 | @c @item | ||
1733 | @c python-gpgme | ||
1734 | @c @end itemize | ||
1735 | |||
1736 | |||
1737 | @c Checkout gnunet-update: | ||
1738 | |||
1739 | @c @c FIXME: git! | ||
1740 | @c @example | ||
1741 | @c $ svn checkout -r24905 https://gnunet.org/svn/gnunet-update@ | ||
1742 | @c @end example | ||
1743 | |||
1744 | @c For security reasons, all packages released for gnunet-update from us are | ||
1745 | @c signed with the key at @uref{https://gnunet.org/install/key.txt}. | ||
1746 | @c You would need to import this key into your gpg key ring. | ||
1747 | @c gnunet-update uses this key to verify the integrity of the packages it | ||
1748 | @c installs: | ||
1749 | |||
1750 | @c @example | ||
1751 | @c $ gpg --recv-keys 7C613D78@ | ||
1752 | @c @end example | ||
1753 | |||
1754 | @c Download the packages relevant to your architecture (currently I have | ||
1755 | @c access to GNU/Linux machines on x86_64 and i686, so only two for now, | ||
1756 | @c hopefully more later) from https://gnunet.org/install/. | ||
1757 | |||
1758 | @c To install the downloaded package into the directory /foo: | ||
1759 | |||
1760 | @c @example | ||
1761 | @c gnunet-update/bin/gnunet-update install downloaded/package /foo | ||
1762 | @c @end example | ||
1763 | |||
1764 | @c The installer reports the directories into which shared libraries and | ||
1765 | @c dependencies have been installed. You may need to add the reported shared | ||
1766 | @c library installation paths to LD_LIBRARY_PATH before you start running any | ||
1767 | @c installed binaries. | ||
1768 | |||
1769 | @c Please report bugs at https://gnunet.org/bugs/ under the project | ||
1770 | @c 'gnunet-update'. | ||
1771 | |||
1772 | @node Instructions for Microsoft Windows Platforms (Old) | ||
1773 | @subsection Instructions for Microsoft Windows Platforms (Old) | ||
1774 | |||
1775 | This document is a @b{DEPRECATED} installation guide for GNUnet on | ||
1776 | Windows. | ||
1777 | It will not work for recent GNUnet versions, but maybe it will be of | ||
1778 | some use if problems arise. | ||
1779 | |||
1780 | The Windows build uses a UNIX emulator for Windows, | ||
1781 | @uref{http://www.mingw.org/, MinGW}, to build the executable modules. | ||
1782 | These modules run natively on Windows and do not require additional | ||
1783 | emulation software besides the usual dependencies. | ||
1784 | |||
1785 | GNUnet development is mostly done under GNU/Linux and especially git | ||
1786 | checkouts may not build out of the box. | ||
1787 | We regret any inconvenience, and if you have problems, please report them. | ||
1788 | |||
1789 | @menu | ||
1790 | * Hardware and OS requirements:: | ||
1791 | * Software installation:: | ||
1792 | * Building libextractor and GNUnet:: | ||
1793 | * Installer:: | ||
1794 | * Source:: | ||
1795 | @end menu | ||
1796 | |||
1797 | @node Hardware and OS requirements | ||
1798 | @subsubsection Hardware and OS requirements | ||
1799 | |||
1800 | @itemize @bullet | ||
1801 | |||
1802 | @item Pentium II or equivalent processor, @geq{} 350 MHz | ||
1803 | |||
1804 | @item 128 MB RAM | ||
1805 | |||
1806 | @item 600 MB free disk space | ||
1807 | |||
1808 | @item Windows 2000 or Windows XP are recommended | ||
1809 | |||
1810 | @end itemize | ||
1811 | |||
1812 | @node Software installation | ||
1813 | @subsubsection Software installation | ||
1814 | |||
1815 | @itemize @bullet | ||
1816 | |||
1817 | @item | ||
1818 | @strong{Compression software}@ | ||
1819 | |||
1820 | The software packages GNUnet depends on are usually compressed using UNIX | ||
1821 | tools like @command{tar}, @command{gzip}, @command{xzip} and | ||
1822 | @command{bzip2}. | ||
1823 | If you do not already have an utility that is able to extract such | ||
1824 | archives, get @uref{http://www.7-zip.org/, 7-Zip}. | ||
1825 | |||
1826 | @item | ||
1827 | @strong{UNIX environment}@ | ||
1828 | |||
1829 | The MinGW project provides the compiler toolchain that is used to build | ||
1830 | GNUnet. | ||
1831 | Get the following packages from the | ||
1832 | @uref{http://sourceforge.net/projects/mingw/files/, MinGW} project: | ||
1833 | |||
1834 | @itemize @bullet | ||
1835 | |||
1836 | @item GCC core | ||
1837 | @item GCC g++ | ||
1838 | @item MSYS | ||
1839 | @item MSYS Developer Tool Kit (msysDTK) | ||
1840 | @item MSYS Developer Tool Kit - msys-autoconf (bin) | ||
1841 | @item MSYS Developer Tool Kit - msys-automake (bin) | ||
1842 | @item MinGW Runtime | ||
1843 | @item MinGW Utilities | ||
1844 | @item Windows API | ||
1845 | @item Binutils | ||
1846 | @item make | ||
1847 | @item pdcurses | ||
1848 | @item GDB (snapshot) | ||
1849 | @end itemize | ||
1850 | |||
1851 | @itemize @bullet | ||
1852 | |||
1853 | |||
1854 | @item Install MSYS (to c:\mingw, for example.)@ | ||
1855 | Do @strong{not} use spaces in the pathname. | ||
1856 | For example, avoid a location such as @file{c:\program files\mingw}. | ||
1857 | |||
1858 | @item Install MinGW runtime, utilities and GCC to a subdirectory | ||
1859 | (to @file{c:\mingw\mingw}, for example) | ||
1860 | |||
1861 | @item Install the Development Kit to the MSYS directory | ||
1862 | (@file{c:\mingw}) | ||
1863 | |||
1864 | @item Create a batch file bash.bat in your MSYS directory with | ||
1865 | the files: | ||
1866 | |||
1867 | @example | ||
1868 | bin\sh.exe --login | ||
1869 | @end example | ||
1870 | |||
1871 | This batch file opens a shell which is used to invoke the build | ||
1872 | processes. | ||
1873 | MinGW's standard shell (@command{msys.bat}) is not suitable | ||
1874 | because it opens a separate console window. | ||
1875 | On Vista, @command{bash.bat} needs to be run as Administrator. | ||
1876 | |||
1877 | @item | ||
1878 | Start @command{bash.sh} and rename | ||
1879 | @file{c:\mingw\mingw\lib\libstdc++.la} to avoid problems: | ||
1880 | |||
1881 | @example | ||
1882 | mv /usr/mingw/lib/libstdc++.la /usr/mingw/lib/libstdc++.la.broken | ||
1883 | @end example | ||
1884 | |||
1885 | @item | ||
1886 | Unpack the Windows API to the MinGW directory (@file{c:\mingw\mingw\}) and | ||
1887 | remove the declaration of DATADIR from | ||
1888 | (@file{c:\mingw\mingw\include\objidl.h} (lines 55-58) | ||
1889 | |||
1890 | @item | ||
1891 | Unpack autoconf, automake to the MSYS directory (@file{c:\mingw}) | ||
1892 | |||
1893 | @item | ||
1894 | Install all other packages to the MinGW directory (@file{c:\mingw\mingw\}) | ||
1895 | @end itemize | ||
1896 | |||
1897 | |||
1898 | @item @strong{GNU Libtool}@ | ||
1899 | GNU Libtool is required to use shared libraries. | ||
1900 | Get the prebuilt package from here and unpack it to the | ||
1901 | MinGW directory (@file{c:\mingw}) | ||
1902 | |||
1903 | @item @strong{Pthreads}@ | ||
1904 | GNUnet uses the portable POSIX thread library for multi-threading: | ||
1905 | |||
1906 | @itemize @bullet | ||
1907 | |||
1908 | @item Save | ||
1909 | @uref{ftp://sources.redhat.com/pub/pthreads-win32/dll-latest/lib/x86/libpthreadGC2.a, libpthreadGC2.a} | ||
1910 | (x86) or | ||
1911 | @uref{ftp://sources.redhat.com/pub/pthreads-win32/dll-latest/lib/x64/libpthreadGC2.a, libpthreadGC2.a} | ||
1912 | (x64) as libpthread.a into the @file{lib} | ||
1913 | directory (@file{c:\mingw\mingw\lib\libpthread.a}). | ||
1914 | |||
1915 | @item Save | ||
1916 | @uref{ftp://sources.redhat.com/pub/pthreads-win32/dll-latest/lib/x86/pthreadGC2.dll, pthreadGC2.dll} | ||
1917 | (x86) or | ||
1918 | @uref{ftp://sources.redhat.com/pub/pthreads-win32/dll-latest/lib/x64/pthreadGC2.dll, libpthreadGC2.a} | ||
1919 | (x64) into the MinGW @file{bin} directory (@file{c:\mingw\mingw\bin}). | ||
1920 | |||
1921 | @item Download all header files from | ||
1922 | @uref{ftp://sources.redhat.com/pub/pthreads-win32/dll-latest/include/, include/} | ||
1923 | to the @file{include} directory (@file{c:\mingw\mingw\include}). | ||
1924 | @end itemize | ||
1925 | |||
1926 | |||
1927 | @item @strong{GNU MP}@ | ||
1928 | GNUnet uses the GNU Multiple Precision library for special cryptographic | ||
1929 | operations. Get the GMP binary package from the | ||
1930 | @uref{http://sourceforge.net/projects/mingwrep/, MinGW repository} and | ||
1931 | unpack it to the MinGW directory (@file{c:\mingw\mingw}) | ||
1932 | |||
1933 | @item @strong{GNU Gettext}@ | ||
1934 | GNU gettext is used to provide national language support. | ||
1935 | Get the prebuilt package from hereand unpack it to the MinGW | ||
1936 | directory (@file{c:\mingw\mingw}) | ||
1937 | |||
1938 | @item @strong{GNU iconv}@ | ||
1939 | GNU Libiconv is used for character encoding conversion. | ||
1940 | Get the prebuilt package from here and unpack it to the MinGW | ||
1941 | directory (@file{c:\mingw\mingw}). | ||
1942 | |||
1943 | @item @strong{SQLite}@ | ||
1944 | GNUnet uses the SQLite database to store data. | ||
1945 | Get the prebuilt binary from here and unpack it to your MinGW directory. | ||
1946 | |||
1947 | @item @strong{MySQL}@ | ||
1948 | As an alternative to SQLite, GNUnet also supports MySQL. | ||
1949 | |||
1950 | @itemize @bullet | ||
1951 | |||
1952 | @item Get the binary installer from the | ||
1953 | @uref{http://dev.mysql.com/downloads/mysql/4.1.html#Windows, MySQL project} | ||
1954 | (version 4.1), install it and follow the instructions in | ||
1955 | @file{README.mysql}. | ||
1956 | |||
1957 | @item Create a temporary build directory (@file{c:\mysql}) | ||
1958 | |||
1959 | @item Copy the directories @file{include\} and @file{lib\} from the | ||
1960 | MySQL directory to the new directory | ||
1961 | |||
1962 | @item Get the patches from | ||
1963 | @uref{http://bugs.mysql.com/bug.php?id=8906&files=1, Bug #8906} and | ||
1964 | @uref{http://bugs.mysql.com/bug.php?id=8872&files=1, Bug #8872} (the | ||
1965 | latter is only required for MySQL | ||
1966 | |||
1967 | @example | ||
1968 | patch -p 0 | ||
1969 | @end example | ||
1970 | |||
1971 | @item Move @file{lib\opt\libmysql.dll} to @file{lib\libmysql.dll} | ||
1972 | |||
1973 | @item Change to @file{lib\} and create an import library: | ||
1974 | |||
1975 | @example | ||
1976 | dlltool --input-def ../include/libmySQL.def \ | ||
1977 | --dllname libmysql.dll \ | ||
1978 | --output-lib libmysqlclient.a -k | ||
1979 | @end example | ||
1980 | |||
1981 | @item Copy include\* to include\mysql\ | ||
1982 | |||
1983 | @item Pass @code{--with-mysql=/c/mysql} to | ||
1984 | @command{./configure} and copy @file{libmysql.dll} | ||
1985 | to your PATH or GNUnet's @file{bin} directory | ||
1986 | @end itemize | ||
1987 | |||
1988 | |||
1989 | @item @strong{GTK+}@ | ||
1990 | @command{gnunet-gtk} and @command{libextractor} depend on GTK. | ||
1991 | Get the the binary and developer packages of @command{atk}, | ||
1992 | @command{glib}, @command{gtk}, @command{iconv}, | ||
1993 | @command{gettext-runtime}, @command{pango} from | ||
1994 | @uref{ftp://ftp.gtk.org/pub/gtk/v2.6/win32, gtk.org} and unpack them | ||
1995 | to the MinGW directory (@file{c:\mingw\mingw}). | ||
1996 | @c FIXME: The URL below for pkg-config seems wrong. | ||
1997 | Get @uref{http://www.gtk.org/download/win32.php, pkg-config} and | ||
1998 | @command{libpng} and unpack them to the MinGW directory | ||
1999 | (@file{c:\mingw\mingw}). | ||
2000 | Here is an all-in-one package for the | ||
2001 | @uref{http://ftp.gnome.org/pub/gnome/binaries/win32/gtk+/2.24/gtk+-bundle_2.24.10-20120208_win32.zip, gtk+dependencies} | ||
2002 | . Do not overwrite any existing files! | ||
2003 | |||
2004 | @item @strong{Glade}@ | ||
2005 | @command{gnunet-gtk} and @command{gnunet-setup} were created using | ||
2006 | this interface builder | ||
2007 | |||
2008 | @itemize @bullet | ||
2009 | |||
2010 | @item Get the Glade and libglade (-bin and -devel) packages | ||
2011 | (without GTK!) from | ||
2012 | @uref{http://gladewin32.sourceforge.net/, GladeWin32} and unpack them to | ||
2013 | the MinGW directory (@file{c:\mingw\mingw}). | ||
2014 | |||
2015 | @item Get @command{libxml} from here and unpack it to the MinGW | ||
2016 | directory (@file{c:\mingw\mingw}). | ||
2017 | @end itemize | ||
2018 | |||
2019 | @c FIXME: URLs | ||
2020 | @item @strong{zLib}@ | ||
2021 | @command{libextractor} requires @command{zLib} to decompress some file | ||
2022 | formats. GNUnet uses it to (de)compress meta-data. | ||
2023 | Get zLib from here (Signature) and unpack it to the MinGW directory | ||
2024 | (@file{c:\mingw\mingw}). | ||
2025 | |||
2026 | @item @strong{Bzip2}@ | ||
2027 | @command{libextractor} also requires @command{Bzip2} to | ||
2028 | decompress some file formats. | ||
2029 | Get the Bzip2 (binary and developer package) from | ||
2030 | @uref{http://gnuwin32.sourceforge.net/packages/bzip2.htm, GnuWin32} and | ||
2031 | unpack it to the MinGW directory (@file{c:\mingw\mingw}). | ||
2032 | |||
2033 | @item @strong{Libgcrypt}@ | ||
2034 | @command{Libgcrypt} provides the cryptographic functions used by GNUnet. | ||
2035 | Get Libgcrypt from @uref{ftp://ftp.gnupg.org/gcrypt/libgcrypt/, here}, | ||
2036 | compile and place it in the MinGW directory | ||
2037 | (@file{c:\mingw\mingw}). Currently libgcrypt @geq{} 1.4.2 is required to | ||
2038 | compile GNUnet. | ||
2039 | |||
2040 | @item @strong{PlibC}@ | ||
2041 | PlibC emulates Unix functions under Windows. Get PlibC from here and | ||
2042 | unpack it to the MinGW directory (c:\mingw\mingw) | ||
2043 | |||
2044 | @item @strong{OGG Vorbis}@ | ||
2045 | @command{OGG Vorbis} is used to extract meta-data from @file{.ogg} files. | ||
2046 | Get the packages | ||
2047 | @uref{http://www.gnunet.org/libextractor/download/win/libogg-1.1.4.zip, libogg} | ||
2048 | and | ||
2049 | @uref{http://www.gnunet.org/libextractor/download/win/libvorbis-1.2.3.zip, libvorbis} | ||
2050 | from the | ||
2051 | @uref{http://ftp.gnu.org/gnu/libextractor/libextractor-w32-1.0.0.zip, libextractor win32 build} | ||
2052 | and unpack them to the MinGW directory (c:\mingw\mingw). | ||
2053 | |||
2054 | @item @strong{Exiv2}@ | ||
2055 | (lib)Exiv2 is used to extract meta-data from files with Exiv2 meta-data. | ||
2056 | Download | ||
2057 | @uref{http://www.gnunet.org/libextractor/download/win/exiv2-0.18.2.zip, Exiv2} | ||
2058 | and unpack it to the MSYS directory (c:\mingw). | ||
2059 | @end itemize | ||
2060 | |||
2061 | @node Building libextractor and GNUnet | ||
2062 | @subsubsection Building libextractor and GNUnet | ||
2063 | |||
2064 | Before you compile @command{libextractor} or @command{GNUnet}, | ||
2065 | be sure to set @code{PKG_CONFIG_PATH}: | ||
2066 | |||
2067 | @example | ||
2068 | export PKG_CONFIG_PATH=/mingw/lib/pkgconfig | ||
2069 | @end example | ||
2070 | |||
2071 | @noindent | ||
2072 | @xref{GNUnet Installation Handbook}, for basic instructions on building | ||
2073 | @command{libextractor} and @command{GNUnet}. | ||
2074 | By default, all modules that are created in this way contain | ||
2075 | debug information and are quite large. To compile release versions | ||
2076 | (small and fast) set the variable @code{CFLAGS}: | ||
2077 | |||
2078 | @example | ||
2079 | export CFLAGS='-O2 -march=pentium -fomit-frame-pointer' | ||
2080 | ./configure --prefix=$HOME --with-extractor=$HOME | ||
2081 | @end example | ||
2082 | |||
2083 | @node Installer | ||
2084 | @subsubsection Installer | ||
2085 | |||
2086 | The GNUnet installer is made with | ||
2087 | @uref{http://nsis.sourceforge.net/, NSIS}. The installer script is | ||
2088 | located in @file{contrib\win} in the GNUnet source tree. | ||
2089 | |||
2090 | @node Source | ||
2091 | @subsubsection Source | ||
2092 | |||
2093 | @c FIXME: URL | ||
2094 | The sources of all dependencies are available here. | ||
2095 | |||
2096 | @c @node Portable GNUnet | ||
2097 | @c @section Portable GNUnet | ||
2098 | |||
2099 | @c Quick instructions on how to use the most recent GNUnet on most GNU/Linux | ||
2100 | @c distributions | ||
2101 | |||
2102 | @c Currently this has only been tested on Ubuntu 12.04, 12.10, 13.04, Debian | ||
2103 | @c and CentOS 6, but it should work on almost any GNU/Linux distribution. | ||
2104 | @c More in-detail information can be found in the handbook. | ||
2105 | |||
2106 | @c Note 2017-10: Currently this section assumes the old SVN repo of GNUnet | ||
2107 | @c which no longer exists. | ||
2108 | |||
2109 | @c @menu | ||
2110 | @c * Prerequisites:: | ||
2111 | @c * Download & set up gnunet-update:: | ||
2112 | @c * Install GNUnet:: | ||
2113 | @c @end menu | ||
2114 | |||
2115 | @c @node Prerequisites | ||
2116 | @c @subsection Prerequisites | ||
2117 | |||
2118 | @c Open a terminal and paste this line into it to install all required tools | ||
2119 | @c needed: | ||
2120 | |||
2121 | @c @example | ||
2122 | @c sudo apt-get install python-gpgme subversion | ||
2123 | @c @end example | ||
2124 | |||
2125 | @c @node Download & set up gnunet-update | ||
2126 | @c @subsection Download & set up gnunet-update | ||
2127 | |||
2128 | @c The following command will download a working version of gnunet-update | ||
2129 | @c with the subversion tool and import the public key which is needed for | ||
2130 | @c authentication: | ||
2131 | |||
2132 | @c @example | ||
2133 | @c svn checkout -r24905 https://gnunet.org/svn/gnunet-update ~/gnunet-update | ||
2134 | @c cd ~/gnunet-update | ||
2135 | @c gpg --keyserver "hkp://keys.gnupg.net" --recv-keys 7C613D78 | ||
2136 | @c @end example | ||
2137 | |||
2138 | @c @node Install GNUnet | ||
2139 | @c @subsection Install GNUnet | ||
2140 | |||
2141 | @c Download and install GNUnet binaries which can be found here and set | ||
2142 | @c library paths: | ||
2143 | |||
2144 | @c @example | ||
2145 | @c wget -P /tmp https://gnunet.org/install/packs/gnunet-0.9.4-`uname -m`.tgz | ||
2146 | @c ./bin/gnunet-update install /tmp/gnunet-0.9*.tgz ~ | ||
2147 | @c echo "PATH DEFAULT=$@{PATH@}:$HOME/bin" >> ~/.pam_environment | ||
2148 | @c echo -e "$@{HOME@}/lib\n$@{HOME@}/lib/gnunet-deps" | sudo tee \ | ||
2149 | @c /etc/ld.so.conf.d/gnunet.conf > /dev/null | ||
2150 | @c sudo ldconfig | ||
2151 | @c @end example | ||
2152 | |||
2153 | @c You may need to re-login once after executing these last commands | ||
2154 | |||
2155 | @c That's it, GNUnet is installed in your home directory now. GNUnet can be | ||
2156 | @c configured and afterwards started by executing: | ||
2157 | |||
2158 | @c @example | ||
2159 | @c gnunet-arm -s | ||
2160 | @c @end example | ||
2161 | |||
2162 | @node The graphical configuration interface | ||
2163 | @section The graphical configuration interface | ||
2164 | |||
2165 | If you also would like to use @command{gnunet-gtk} and | ||
2166 | @command{gnunet-setup} (highly recommended for beginners), do: | ||
2167 | |||
2168 | @example | ||
2169 | wget -P /tmp \ | ||
2170 | https://gnunet.org/install/packs/gnunet-0.9.4-gtk-0.9.4-`uname -m`.tgz | ||
2171 | sh ~/gnunet-update/bin/gnunet-update install /tmp/gnunet-*gtk*.tgz ~ | ||
2172 | sudo ldconfig | ||
2173 | @end example | ||
2174 | |||
2175 | Now you can run @command{gnunet-setup} for easy configuration of your | ||
2176 | GNUnet peer. | ||
2177 | |||
2178 | @menu | ||
2179 | * Configuring your peer:: | ||
2180 | * Configuring the Friend-to-Friend (F2F) mode:: | ||
2181 | * Configuring the hostlist to bootstrap:: | ||
2182 | * Configuration of the HOSTLIST proxy settings:: | ||
2183 | * Configuring your peer to provide a hostlist :: | ||
2184 | * Configuring the datastore:: | ||
2185 | * Configuring the MySQL database:: | ||
2186 | * Reasons for using MySQL:: | ||
2187 | * Reasons for not using MySQL:: | ||
2188 | * Setup Instructions:: | ||
2189 | * Testing:: | ||
2190 | * Performance Tuning:: | ||
2191 | * Setup for running Testcases:: | ||
2192 | * Configuring the Postgres database:: | ||
2193 | * Reasons to use Postgres:: | ||
2194 | * Reasons not to use Postgres:: | ||
2195 | * Manual setup instructions:: | ||
2196 | * Testing the setup manually:: | ||
2197 | * Configuring the datacache:: | ||
2198 | * Configuring the file-sharing service:: | ||
2199 | * Configuring logging:: | ||
2200 | * Configuring the transport service and plugins:: | ||
2201 | * Configuring the wlan transport plugin:: | ||
2202 | * Configuring HTTP(S) reverse proxy functionality using Apache or nginx:: | ||
2203 | * Blacklisting peers:: | ||
2204 | * Configuration of the HTTP and HTTPS transport plugins:: | ||
2205 | * Configuring the GNU Name System:: | ||
2206 | * Configuring the GNUnet VPN:: | ||
2207 | * Bandwidth Configuration:: | ||
2208 | * Configuring NAT:: | ||
2209 | * Peer configuration for distributions:: | ||
2210 | @end menu | ||
2211 | |||
2212 | @node Configuring your peer | ||
2213 | @subsection Configuring your peer | ||
2214 | |||
2215 | This chapter will describe the various configuration options in GNUnet. | ||
2216 | |||
2217 | The easiest way to configure your peer is to use the | ||
2218 | @command{gnunet-setup} tool. | ||
2219 | @command{gnunet-setup} is part of the @command{gnunet-gtk} | ||
2220 | application. You might have to install it separately. | ||
2221 | |||
2222 | Many of the specific sections from this chapter actually are linked from | ||
2223 | within @command{gnunet-setup} to help you while using the setup tool. | ||
2224 | |||
2225 | While you can also configure your peer by editing the configuration | ||
2226 | file by hand, this is not recommended for anyone except for developers | ||
2227 | as it requires a more in-depth understanding of the configuration files | ||
2228 | and internal dependencies of GNUnet. | ||
2229 | |||
2230 | @node Configuring the Friend-to-Friend (F2F) mode | ||
2231 | @subsection Configuring the Friend-to-Friend (F2F) mode | ||
2232 | |||
2233 | GNUnet knows three basic modes of operation: | ||
2234 | @itemize @bullet | ||
2235 | @item In standard "peer-to-peer" mode, | ||
2236 | your peer will connect to any peer. | ||
2237 | @item In the pure "friend-to-friend" | ||
2238 | mode, your peer will ONLY connect to peers from a list of friends | ||
2239 | specified in the configuration. | ||
2240 | @item Finally, in mixed mode, | ||
2241 | GNUnet will only connect to arbitrary peers if it | ||
2242 | has at least a specified number of connections to friends. | ||
2243 | @end itemize | ||
2244 | |||
2245 | When configuring any of the F2F ("friend-to-friend") modes, | ||
2246 | you first need to create a file with the peer identities | ||
2247 | of your friends. Ask your friends to run | ||
2248 | |||
2249 | @example | ||
2250 | $ gnunet-peerinfo -sq | ||
2251 | @end example | ||
2252 | |||
2253 | @noindent | ||
2254 | The resulting output of this command needs to be added to your | ||
2255 | @file{friends} file, which is simply a plain text file with one line | ||
2256 | per friend with the output from the above command. | ||
2257 | |||
2258 | You then specify the location of your @file{friends} file in the | ||
2259 | @code{FRIENDS} option of the "topology" section. | ||
2260 | |||
2261 | Once you have created the @file{friends} file, you can tell GNUnet to only | ||
2262 | connect to your friends by setting the @code{FRIENDS-ONLY} option | ||
2263 | (again in the "topology" section) to YES. | ||
2264 | |||
2265 | If you want to run in mixed-mode, set "FRIENDS-ONLY" to NO and configure a | ||
2266 | minimum number of friends to have (before connecting to arbitrary peers) | ||
2267 | under the "MINIMUM-FRIENDS" option. | ||
2268 | |||
2269 | If you want to operate in normal P2P-only mode, simply set | ||
2270 | @code{MINIMUM-FRIENDS} to zero and @code{FRIENDS_ONLY} to NO. | ||
2271 | This is the default. | ||
2272 | |||
2273 | @node Configuring the hostlist to bootstrap | ||
2274 | @subsection Configuring the hostlist to bootstrap | ||
2275 | |||
2276 | After installing the software you need to get connected to the GNUnet | ||
2277 | network. The configuration file included in your download is already | ||
2278 | configured to connect you to the GNUnet network. | ||
2279 | In this section the relevant configuration settings are explained. | ||
2280 | |||
2281 | To get an initial connection to the GNUnet network and to get to know | ||
2282 | peers already connected to the network you can use the so called | ||
2283 | "bootstrap servers". | ||
2284 | These servers can give you a list of peers connected to the network. | ||
2285 | To use these bootstrap servers you have to configure the hostlist daemon | ||
2286 | to activate bootstrapping. | ||
2287 | |||
2288 | To activate bootstrapping, edit the @code{[hostlist]}-section in your | ||
2289 | configuration file. You have to set the argument @command{-b} in the | ||
2290 | options line: | ||
2291 | |||
2292 | @example | ||
2293 | [hostlist] | ||
2294 | OPTIONS = -b | ||
2295 | @end example | ||
2296 | |||
2297 | Additionally you have to specify which server you want to use. | ||
2298 | The default bootstrapping server is | ||
2299 | "@uref{http://v10.gnunet.org/hostlist, http://v10.gnunet.org/hostlist}". | ||
2300 | [^] To set the server you have to edit the line "SERVERS" in the hostlist | ||
2301 | section. To use the default server you should set the lines to | ||
2302 | |||
2303 | @example | ||
2304 | SERVERS = http://v10.gnunet.org/hostlist [^] | ||
2305 | @end example | ||
2306 | |||
2307 | @noindent | ||
2308 | To use bootstrapping your configuration file should include these lines: | ||
2309 | |||
2310 | @example | ||
2311 | [hostlist] | ||
2312 | OPTIONS = -b | ||
2313 | SERVERS = http://v10.gnunet.org/hostlist [^] | ||
2314 | @end example | ||
2315 | |||
2316 | @noindent | ||
2317 | Besides using bootstrap servers you can configure your GNUnet peer to | ||
2318 | recieve hostlist advertisements. | ||
2319 | Peers offering hostlists to other peers can send advertisement messages | ||
2320 | to peers that connect to them. If you configure your peer to receive these | ||
2321 | messages, your peer can download these lists and connect to the peers | ||
2322 | included. These lists are persistent, which means that they are saved to | ||
2323 | your hard disk regularly and are loaded during startup. | ||
2324 | |||
2325 | To activate hostlist learning you have to add the @command{-e} | ||
2326 | switch to the @code{OPTIONS} line in the hostlist section: | ||
2327 | |||
2328 | @example | ||
2329 | [hostlist] | ||
2330 | OPTIONS = -b -e | ||
2331 | @end example | ||
2332 | |||
2333 | @noindent | ||
2334 | Furthermore you can specify in which file the lists are saved. | ||
2335 | To save the lists in the file @file{hostlists.file} just add the line: | ||
2336 | |||
2337 | @example | ||
2338 | HOSTLISTFILE = hostlists.file | ||
2339 | @end example | ||
2340 | |||
2341 | @noindent | ||
2342 | Best practice is to activate both bootstrapping and hostlist learning. | ||
2343 | So your configuration file should include these lines: | ||
2344 | |||
2345 | @example | ||
2346 | [hostlist] | ||
2347 | OPTIONS = -b -e | ||
2348 | HTTPPORT = 8080 | ||
2349 | SERVERS = http://v10.gnunet.org/hostlist [^] | ||
2350 | HOSTLISTFILE = $SERVICEHOME/hostlists.file | ||
2351 | @end example | ||
2352 | |||
2353 | @node Configuration of the HOSTLIST proxy settings | ||
2354 | @subsection Configuration of the HOSTLIST proxy settings | ||
2355 | |||
2356 | The hostlist client can be configured to use a proxy to connect to the | ||
2357 | hostlist server. | ||
2358 | This functionality can be configured in the configuration file directly | ||
2359 | or using the @command{gnunet-setup} tool. | ||
2360 | |||
2361 | The hostlist client supports the following proxy types at the moment: | ||
2362 | |||
2363 | @itemize @bullet | ||
2364 | @item HTTP and HTTP 1.0 only proxy | ||
2365 | @item SOCKS 4/4a/5/5 with hostname | ||
2366 | @end itemize | ||
2367 | |||
2368 | In addition authentication at the proxy with username and password can be | ||
2369 | configured. | ||
2370 | |||
2371 | To configure proxy support for the hostlist client in the | ||
2372 | @command{gnunet-setup} tool, select the "hostlist" tab and select | ||
2373 | the appropriate proxy type. | ||
2374 | The hostname or IP address (including port if required) has to be entered | ||
2375 | in the "Proxy hostname" textbox. If required, enter username and password | ||
2376 | in the "Proxy username" and "Proxy password" boxes. | ||
2377 | Be aware that this information will be stored in the configuration in | ||
2378 | plain text (TODO: Add explanation and generalize the part in Chapter 3.6 | ||
2379 | about the encrypted home). | ||
2380 | |||
2381 | To provide these options directly in the configuration, you can | ||
2382 | enter the following settings in the @code{[hostlist]} section of | ||
2383 | the configuration: | ||
2384 | |||
2385 | @example | ||
2386 | # Type of proxy server, | ||
2387 | # Valid values: HTTP, HTTP_1_0, SOCKS4, SOCKS5, SOCKS4A, SOCKS5_HOSTNAME | ||
2388 | # Default: HTTP | ||
2389 | # PROXY_TYPE = HTTP | ||
2390 | |||
2391 | # Hostname or IP of proxy server | ||
2392 | # PROXY = | ||
2393 | # User name for proxy server | ||
2394 | # PROXY_USERNAME = | ||
2395 | # User password for proxy server | ||
2396 | # PROXY_PASSWORD = | ||
2397 | @end example | ||
2398 | |||
2399 | @node Configuring your peer to provide a hostlist | ||
2400 | @subsection Configuring your peer to provide a hostlist | ||
2401 | |||
2402 | If you operate a peer permanently connected to GNUnet you can configure | ||
2403 | your peer to act as a hostlist server, providing other peers the list of | ||
2404 | peers known to him. | ||
2405 | |||
2406 | Your server can act as a bootstrap server and peers needing to obtain a | ||
2407 | list of peers can contact it to download this list. | ||
2408 | To download this hostlist the peer uses HTTP. | ||
2409 | For this reason you have to build your peer with libgnurl (or libcurl) | ||
2410 | and microhttpd support. | ||
2411 | How you build your peer with these options can be found here: | ||
2412 | @xref{Generic installation instructions}. | ||
2413 | |||
2414 | To configure your peer to act as a bootstrap server you have to add the | ||
2415 | @command{-p} option to @code{OPTIONS} in the @code{[hostlist]} section | ||
2416 | of your configuration file. | ||
2417 | Besides that you have to specify a port number for the http server. | ||
2418 | In conclusion you have to add the following lines: | ||
2419 | |||
2420 | @example | ||
2421 | [hostlist] | ||
2422 | HTTPPORT = 12980 | ||
2423 | OPTIONS = -p | ||
2424 | @end example | ||
2425 | |||
2426 | @noindent | ||
2427 | If your peer acts as a bootstrap server other peers should know about | ||
2428 | that. You can advertise the hostlist your are providing to other peers. | ||
2429 | Peers connecting to your peer will get a message containing an | ||
2430 | advertisement for your hostlist and the URL where it can be downloaded. | ||
2431 | If this peer is in learning mode, it will test the hostlist and, in the | ||
2432 | case it can obtain the list successfully, it will save it for | ||
2433 | bootstrapping. | ||
2434 | |||
2435 | To activate hostlist advertisement on your peer, you have to set the | ||
2436 | following lines in your configuration file: | ||
2437 | |||
2438 | @example | ||
2439 | [hostlist] | ||
2440 | EXTERNAL_DNS_NAME = example.org | ||
2441 | HTTPPORT = 12981 | ||
2442 | OPTIONS = -p -a | ||
2443 | @end example | ||
2444 | |||
2445 | @noindent | ||
2446 | With this configuration your peer will a act as a bootstrap server and | ||
2447 | advertise this hostlist to other peers connecting to it. | ||
2448 | The URL used to download the list will be | ||
2449 | @code{@uref{http://example.org:12981/, http://example.org:12981/}}. | ||
2450 | |||
2451 | Please notice: | ||
2452 | |||
2453 | @itemize @bullet | ||
2454 | @item The hostlist is @b{not} human readable, so you should not try to | ||
2455 | download it using your webbrowser. Just point your GNUnet peer to the | ||
2456 | address! | ||
2457 | @item Advertising without providing a hostlist does not make sense and | ||
2458 | will not work. | ||
2459 | @end itemize | ||
2460 | |||
2461 | @node Configuring the datastore | ||
2462 | @subsection Configuring the datastore | ||
2463 | |||
2464 | The datastore is what GNUnet uses for long-term storage of file-sharing | ||
2465 | data. Note that long-term does not mean 'forever' since content does have | ||
2466 | an expiration date, and of course storage space is finite (and hence | ||
2467 | sometimes content may have to be discarded). | ||
2468 | |||
2469 | Use the @code{QUOTA} option to specify how many bytes of storage space | ||
2470 | you are willing to dedicate to GNUnet. | ||
2471 | |||
2472 | In addition to specifying the maximum space GNUnet is allowed to use for | ||
2473 | the datastore, you need to specify which database GNUnet should use to do | ||
2474 | so. Currently, you have the choice between sqLite, MySQL and Postgres. | ||
2475 | |||
2476 | @node Configuring the MySQL database | ||
2477 | @subsection Configuring the MySQL database | ||
2478 | |||
2479 | This section describes how to setup the MySQL database for GNUnet. | ||
2480 | |||
2481 | Note that the mysql plugin does NOT work with mysql before 4.1 since we | ||
2482 | need prepared statements. | ||
2483 | We are generally testing the code against MySQL 5.1 at this point. | ||
2484 | |||
2485 | @node Reasons for using MySQL | ||
2486 | @subsection Reasons for using MySQL | ||
2487 | |||
2488 | @itemize @bullet | ||
2489 | |||
2490 | @item On up-to-date hardware wher | ||
2491 | mysql can be used comfortably, this module | ||
2492 | will have better performance than the other database choices (according | ||
2493 | to our tests). | ||
2494 | |||
2495 | @item Its often possible to recover the mysql database from internal | ||
2496 | inconsistencies. Some of the other databases do not support repair. | ||
2497 | @end itemize | ||
2498 | |||
2499 | @node Reasons for not using MySQL | ||
2500 | @subsection Reasons for not using MySQL | ||
2501 | |||
2502 | @itemize @bullet | ||
2503 | @item Memory usage (likely not an issue if you have more than 1 GB) | ||
2504 | @item Complex manual setup | ||
2505 | @end itemize | ||
2506 | |||
2507 | @node Setup Instructions | ||
2508 | @subsection Setup Instructions | ||
2509 | |||
2510 | @itemize @bullet | ||
2511 | |||
2512 | @item In @file{gnunet.conf} set in section @code{DATASTORE} the value for | ||
2513 | @code{DATABASE} to @code{mysql}. | ||
2514 | |||
2515 | @item Access mysql as root: | ||
2516 | |||
2517 | @example | ||
2518 | $ mysql -u root -p | ||
2519 | @end example | ||
2520 | |||
2521 | @noindent | ||
2522 | and issue the following commands, replacing $USER with the username | ||
2523 | that will be running @command{gnunet-arm} (so typically "gnunet"): | ||
2524 | |||
2525 | @example | ||
2526 | CREATE DATABASE gnunet; | ||
2527 | GRANT select,insert,update,delete,create,alter,drop,create \ | ||
2528 | temporary tables ON gnunet.* TO $USER@@localhost; | ||
2529 | SET PASSWORD FOR $USER@@localhost=PASSWORD('$the_password_you_like'); | ||
2530 | FLUSH PRIVILEGES; | ||
2531 | @end example | ||
2532 | |||
2533 | @item | ||
2534 | In the $HOME directory of $USER, create a @file{.my.cnf} file with the | ||
2535 | following lines | ||
2536 | |||
2537 | @example | ||
2538 | [client] | ||
2539 | user=$USER | ||
2540 | password=$the_password_you_like | ||
2541 | @end example | ||
2542 | |||
2543 | @end itemize | ||
2544 | |||
2545 | Thats it. Note that @file{.my.cnf} file is a slight security risk unless | ||
2546 | its on a safe partition. The @file{$HOME/.my.cnf} can of course be | ||
2547 | a symbolic link. | ||
2548 | Luckily $USER has only priviledges to mess up GNUnet's tables, | ||
2549 | which should be pretty harmless. | ||
2550 | |||
2551 | @node Testing | ||
2552 | @subsection Testing | ||
2553 | |||
2554 | You should briefly try if the database connection works. First, login | ||
2555 | as $USER. Then use: | ||
2556 | |||
2557 | @example | ||
2558 | $ mysql -u $USER | ||
2559 | mysql> use gnunet; | ||
2560 | @end example | ||
2561 | |||
2562 | @noindent | ||
2563 | If you get the message | ||
2564 | |||
2565 | @example | ||
2566 | Database changed | ||
2567 | @end example | ||
2568 | |||
2569 | @noindent | ||
2570 | it probably works. | ||
2571 | |||
2572 | If you get | ||
2573 | |||
2574 | @example | ||
2575 | ERROR 2002: Can't connect to local MySQL server | ||
2576 | through socket '/tmp/mysql.sock' (2) | ||
2577 | @end example | ||
2578 | |||
2579 | @noindent | ||
2580 | it may be resolvable by | ||
2581 | |||
2582 | @example | ||
2583 | ln -s /var/run/mysqld/mysqld.sock /tmp/mysql.sock | ||
2584 | @end example | ||
2585 | |||
2586 | @noindent | ||
2587 | so there may be some additional trouble depending on your mysql setup. | ||
2588 | |||
2589 | @node Performance Tuning | ||
2590 | @subsection Performance Tuning | ||
2591 | |||
2592 | For GNUnet, you probably want to set the option | ||
2593 | |||
2594 | @example | ||
2595 | innodb_flush_log_at_trx_commit = 0 | ||
2596 | @end example | ||
2597 | |||
2598 | @noindent | ||
2599 | for a rather dramatic boost in MySQL performance. However, this reduces | ||
2600 | the "safety" of your database as with this options you may loose | ||
2601 | transactions during a power outage. | ||
2602 | While this is totally harmless for GNUnet, the option applies to all | ||
2603 | applications using MySQL. So you should set it if (and only if) GNUnet is | ||
2604 | the only application on your system using MySQL. | ||
2605 | |||
2606 | @node Setup for running Testcases | ||
2607 | @subsection Setup for running Testcases | ||
2608 | |||
2609 | If you want to run the testcases, you must create a second database | ||
2610 | "gnunetcheck" with the same username and password. This database will | ||
2611 | then be used for testing (@command{make check}). | ||
2612 | |||
2613 | @node Configuring the Postgres database | ||
2614 | @subsection Configuring the Postgres database | ||
2615 | |||
2616 | This text describes how to setup the Postgres database for GNUnet. | ||
2617 | |||
2618 | This Postgres plugin was developed for Postgres 8.3 but might work for | ||
2619 | earlier versions as well. | ||
2620 | |||
2621 | @node Reasons to use Postgres | ||
2622 | @subsection Reasons to use Postgres | ||
2623 | |||
2624 | @itemize @bullet | ||
2625 | @item Easier to setup than MySQL | ||
2626 | @item Real database | ||
2627 | @end itemize | ||
2628 | |||
2629 | @node Reasons not to use Postgres | ||
2630 | @subsection Reasons not to use Postgres | ||
2631 | |||
2632 | @itemize @bullet | ||
2633 | @item Quite slow | ||
2634 | @item Still some manual setup required | ||
2635 | @end itemize | ||
2636 | |||
2637 | @node Manual setup instructions | ||
2638 | @subsection Manual setup instructions | ||
2639 | |||
2640 | @itemize @bullet | ||
2641 | @item In @file{gnunet.conf} set in section @code{DATASTORE} the value for | ||
2642 | @code{DATABASE} to @code{postgres}. | ||
2643 | @item Access Postgres to create a user: | ||
2644 | |||
2645 | @table @asis | ||
2646 | @item with Postgres 8.x, use: | ||
2647 | |||
2648 | @example | ||
2649 | # su - postgres | ||
2650 | $ createuser | ||
2651 | @end example | ||
2652 | |||
2653 | @noindent | ||
2654 | and enter the name of the user running GNUnet for the role interactively. | ||
2655 | Then, when prompted, do not set it to superuser, allow the creation of | ||
2656 | databases, and do not allow the creation of new roles. | ||
2657 | |||
2658 | @item with Postgres 9.x, use: | ||
2659 | |||
2660 | @example | ||
2661 | # su - postgres | ||
2662 | $ createuser -d $GNUNET_USER | ||
2663 | @end example | ||
2664 | |||
2665 | @noindent | ||
2666 | where $GNUNET_USER is the name of the user running GNUnet. | ||
2667 | |||
2668 | @end table | ||
2669 | |||
2670 | |||
2671 | @item | ||
2672 | As that user (so typically as user "gnunet"), create a database (or two): | ||
2673 | |||
2674 | @example | ||
2675 | $ createdb gnunet | ||
2676 | # this way you can run "make check" | ||
2677 | $ createdb gnunetcheck | ||
2678 | @end example | ||
2679 | |||
2680 | @end itemize | ||
2681 | |||
2682 | Now you should be able to start @code{gnunet-arm}. | ||
2683 | |||
2684 | @node Testing the setup manually | ||
2685 | @subsection Testing the setup manually | ||
2686 | |||
2687 | You may want to try if the database connection works. First, again login | ||
2688 | as the user who will run @command{gnunet-arm}. Then use: | ||
2689 | |||
2690 | @example | ||
2691 | $ psql gnunet # or gnunetcheck | ||
2692 | gnunet=> \dt | ||
2693 | @end example | ||
2694 | |||
2695 | @noindent | ||
2696 | If, after you have started @command{gnunet-arm} at least once, you get | ||
2697 | a @code{gn090} table here, it probably works. | ||
2698 | |||
2699 | @node Configuring the datacache | ||
2700 | @subsection Configuring the datacache | ||
2701 | @c %**end of header | ||
2702 | |||
2703 | The datacache is what GNUnet uses for storing temporary data. This data is | ||
2704 | expected to be wiped completely each time GNUnet is restarted (or the | ||
2705 | system is rebooted). | ||
2706 | |||
2707 | You need to specify how many bytes GNUnet is allowed to use for the | ||
2708 | datacache using the @code{QUOTA} option in the section @code{[dhtcache]}. | ||
2709 | Furthermore, you need to specify which database backend should be used to | ||
2710 | store the data. Currently, you have the choice between | ||
2711 | sqLite, MySQL and Postgres. | ||
2712 | |||
2713 | @node Configuring the file-sharing service | ||
2714 | @subsection Configuring the file-sharing service | ||
2715 | |||
2716 | In order to use GNUnet for file-sharing, you first need to make sure | ||
2717 | that the file-sharing service is loaded. | ||
2718 | This is done by setting the @code{AUTOSTART} option in | ||
2719 | section @code{[fs]} to "YES". Alternatively, you can run | ||
2720 | |||
2721 | @example | ||
2722 | $ gnunet-arm -i fs | ||
2723 | @end example | ||
2724 | |||
2725 | @noindent | ||
2726 | to start the file-sharing service by hand. | ||
2727 | |||
2728 | Except for configuring the database and the datacache the only important | ||
2729 | option for file-sharing is content migration. | ||
2730 | |||
2731 | Content migration allows your peer to cache content from other peers as | ||
2732 | well as send out content stored on your system without explicit requests. | ||
2733 | This content replication has positive and negative impacts on both system | ||
2734 | performance and privacy. | ||
2735 | |||
2736 | FIXME: discuss the trade-offs. Here is some older text about it... | ||
2737 | |||
2738 | Setting this option to YES allows gnunetd to migrate data to the local | ||
2739 | machine. Setting this option to YES is highly recommended for efficiency. | ||
2740 | Its also the default. If you set this value to YES, GNUnet will store | ||
2741 | content on your machine that you cannot decrypt. | ||
2742 | While this may protect you from liability if the judge is sane, it may | ||
2743 | not (IANAL). If you put illegal content on your machine yourself, setting | ||
2744 | this option to YES will probably increase your chances to get away with it | ||
2745 | since you can plausibly deny that you inserted the content. | ||
2746 | Note that in either case, your anonymity would have to be broken first | ||
2747 | (which may be possible depending on the size of the GNUnet network and the | ||
2748 | strength of the adversary). | ||
2749 | |||
2750 | @node Configuring logging | ||
2751 | @subsection Configuring logging | ||
2752 | |||
2753 | Logging in GNUnet 0.9.0 is controlled via the "-L" and "-l" options. | ||
2754 | Using @code{-L}, a log level can be specified. With log level | ||
2755 | @code{ERROR} only serious errors are logged. | ||
2756 | The default log level is @code{WARNING} which causes anything of | ||
2757 | concern to be logged. | ||
2758 | Log level @code{INFO} can be used to log anything that might be | ||
2759 | interesting information whereas | ||
2760 | @code{DEBUG} can be used by developers to log debugging messages | ||
2761 | (but you need to run @code{./configure} with | ||
2762 | @code{--enable-logging=verbose} to get them compiled). | ||
2763 | The @code{-l} option is used to specify the log file. | ||
2764 | |||
2765 | Since most GNUnet services are managed by @code{gnunet-arm}, using the | ||
2766 | @code{-l} or @code{-L} options directly is not possible. | ||
2767 | Instead, they can be specified using the @code{OPTIONS} configuration | ||
2768 | value in the respective section for the respective service. | ||
2769 | In order to enable logging globally without editing the @code{OPTIONS} | ||
2770 | values for each service, @command{gnunet-arm} supports a | ||
2771 | @code{GLOBAL_POSTFIX} option. | ||
2772 | The value specified here is given as an extra option to all services for | ||
2773 | which the configuration does contain a service-specific @code{OPTIONS} | ||
2774 | field. | ||
2775 | |||
2776 | @code{GLOBAL_POSTFIX} can contain the special sequence "@{@}" which | ||
2777 | is replaced by the name of the service that is being started. | ||
2778 | Furthermore, @code{GLOBAL_POSTFIX} is special in that sequences | ||
2779 | starting with "$" anywhere in the string are expanded (according | ||
2780 | to options in @code{PATHS}); this expansion otherwise is | ||
2781 | only happening for filenames and then the "$" must be the | ||
2782 | first character in the option. Both of these restrictions do | ||
2783 | not apply to @code{GLOBAL_POSTFIX}. | ||
2784 | Note that specifying @code{%} anywhere in the @code{GLOBAL_POSTFIX} | ||
2785 | disables both of these features. | ||
2786 | |||
2787 | In summary, in order to get all services to log at level | ||
2788 | @code{INFO} to log-files called @code{SERVICENAME-logs}, the | ||
2789 | following global prefix should be used: | ||
2790 | |||
2791 | @example | ||
2792 | GLOBAL_POSTFIX = -l $SERVICEHOME/@{@}-logs -L INFO | ||
2793 | @end example | ||
2794 | |||
2795 | @node Configuring the transport service and plugins | ||
2796 | @subsection Configuring the transport service and plugins | ||
2797 | |||
2798 | The transport service in GNUnet is responsible to maintain basic | ||
2799 | connectivity to other peers. | ||
2800 | Besides initiating and keeping connections alive it is also responsible | ||
2801 | for address validation. | ||
2802 | |||
2803 | The GNUnet transport supports more than one transport protocol. | ||
2804 | These protocols are configured together with the transport service. | ||
2805 | |||
2806 | The configuration section for the transport service itself is quite | ||
2807 | similar to all the other services | ||
2808 | |||
2809 | @example | ||
2810 | AUTOSTART = YES | ||
2811 | @@UNIXONLY@@ PORT = 2091 | ||
2812 | HOSTNAME = localhost | ||
2813 | HOME = $SERVICEHOME | ||
2814 | CONFIG = $DEFAULTCONFIG | ||
2815 | BINARY = gnunet-service-transport | ||
2816 | #PREFIX = valgrind | ||
2817 | NEIGHBOUR_LIMIT = 50 | ||
2818 | ACCEPT_FROM = 127.0.0.1; | ||
2819 | ACCEPT_FROM6 = ::1; | ||
2820 | PLUGINS = tcp udp | ||
2821 | UNIXPATH = /tmp/gnunet-service-transport.sock | ||
2822 | @end example | ||
2823 | |||
2824 | Different are the settings for the plugins to load @code{PLUGINS}. | ||
2825 | The first setting specifies which transport plugins to load. | ||
2826 | |||
2827 | @itemize @bullet | ||
2828 | @item transport-unix | ||
2829 | A plugin for local only communication with UNIX domain sockets. Used for | ||
2830 | testing and available on unix systems only. Just set the port | ||
2831 | |||
2832 | @example | ||
2833 | [transport-unix] | ||
2834 | PORT = 22086 | ||
2835 | TESTING_IGNORE_KEYS = ACCEPT_FROM; | ||
2836 | @end example | ||
2837 | |||
2838 | @item transport-tcp | ||
2839 | A plugin for communication with TCP. Set port to 0 for client mode with | ||
2840 | outbound only connections | ||
2841 | |||
2842 | @example | ||
2843 | [transport-tcp] | ||
2844 | # Use 0 to ONLY advertise as a peer behind NAT (no port binding) | ||
2845 | PORT = 2086 | ||
2846 | ADVERTISED_PORT = 2086 | ||
2847 | TESTING_IGNORE_KEYS = ACCEPT_FROM; | ||
2848 | # Maximum number of open TCP connections allowed | ||
2849 | MAX_CONNECTIONS = 128 | ||
2850 | @end example | ||
2851 | |||
2852 | @item transport-udp | ||
2853 | A plugin for communication with UDP. Supports peer discovery using | ||
2854 | broadcasts. | ||
2855 | |||
2856 | @example | ||
2857 | [transport-udp] | ||
2858 | PORT = 2086 | ||
2859 | BROADCAST = YES | ||
2860 | BROADCAST_INTERVAL = 30 s | ||
2861 | MAX_BPS = 1000000 | ||
2862 | TESTING_IGNORE_KEYS = ACCEPT_FROM; | ||
2863 | @end example | ||
2864 | |||
2865 | @item transport-http | ||
2866 | HTTP and HTTPS support is split in two part: a client plugin initiating | ||
2867 | outbound connections and a server part accepting connections from the | ||
2868 | client. The client plugin just takes the maximum number of connections as | ||
2869 | an argument. | ||
2870 | |||
2871 | @example | ||
2872 | [transport-http_client] | ||
2873 | MAX_CONNECTIONS = 128 | ||
2874 | TESTING_IGNORE_KEYS = ACCEPT_FROM; | ||
2875 | @end example | ||
2876 | |||
2877 | @example | ||
2878 | [transport-https_client] | ||
2879 | MAX_CONNECTIONS = 128 | ||
2880 | TESTING_IGNORE_KEYS = ACCEPT_FROM; | ||
2881 | @end example | ||
2882 | |||
2883 | @noindent | ||
2884 | The server has a port configured and the maximum nunber of connections. | ||
2885 | The HTTPS part has two files with the certificate key and the certificate | ||
2886 | file. | ||
2887 | |||
2888 | The server plugin supports reverse proxies, so a external hostname can be | ||
2889 | set using the @code{EXTERNAL_HOSTNAME} setting. | ||
2890 | The webserver under this address should forward the request to the peer | ||
2891 | and the configure port. | ||
2892 | |||
2893 | @example | ||
2894 | [transport-http_server] | ||
2895 | EXTERNAL_HOSTNAME = fulcrum.net.in.tum.de/gnunet | ||
2896 | PORT = 1080 | ||
2897 | MAX_CONNECTIONS = 128 | ||
2898 | TESTING_IGNORE_KEYS = ACCEPT_FROM; | ||
2899 | @end example | ||
2900 | |||
2901 | @example | ||
2902 | [transport-https_server] | ||
2903 | PORT = 4433 | ||
2904 | CRYPTO_INIT = NORMAL | ||
2905 | KEY_FILE = https.key | ||
2906 | CERT_FILE = https.cert | ||
2907 | MAX_CONNECTIONS = 128 | ||
2908 | TESTING_IGNORE_KEYS = ACCEPT_FROM; | ||
2909 | @end example | ||
2910 | |||
2911 | @item transport-wlan | ||
2912 | |||
2913 | The next section describes how to setup the WLAN plugin, | ||
2914 | so here only the settings. Just specify the interface to use: | ||
2915 | |||
2916 | @example | ||
2917 | [transport-wlan] | ||
2918 | # Name of the interface in monitor mode (typically monX) | ||
2919 | INTERFACE = mon0 | ||
2920 | # Real hardware, no testing | ||
2921 | TESTMODE = 0 | ||
2922 | TESTING_IGNORE_KEYS = ACCEPT_FROM; | ||
2923 | @end example | ||
2924 | @end itemize | ||
2925 | |||
2926 | @node Configuring the wlan transport plugin | ||
2927 | @subsection Configuring the wlan transport plugin | ||
2928 | |||
2929 | The wlan transport plugin enables GNUnet to send and to receive data on a | ||
2930 | wlan interface. | ||
2931 | It has not to be connected to a wlan network as long as sender and | ||
2932 | receiver are on the same channel. This enables you to get connection to | ||
2933 | GNUnet where no internet access is possible, for example during | ||
2934 | catastrophes or when censorship cuts you off from the internet. | ||
2935 | |||
2936 | |||
2937 | @menu | ||
2938 | * Requirements for the WLAN plugin:: | ||
2939 | * Configuration:: | ||
2940 | * Before starting GNUnet:: | ||
2941 | * Limitations and known bugs:: | ||
2942 | @end menu | ||
2943 | |||
2944 | |||
2945 | @node Requirements for the WLAN plugin | ||
2946 | @subsubsection Requirements for the WLAN plugin | ||
2947 | |||
2948 | @itemize @bullet | ||
2949 | |||
2950 | @item wlan network card with monitor support and packet injection | ||
2951 | (see @uref{http://www.aircrack-ng.org/, aircrack-ng.org}) | ||
2952 | |||
2953 | @item Linux kernel with mac80211 stack, introduced in 2.6.22, tested with | ||
2954 | 2.6.35 and 2.6.38 | ||
2955 | |||
2956 | @item Wlantools to create the a monitor interface, tested with airmon-ng | ||
2957 | of the aircrack-ng package | ||
2958 | @end itemize | ||
2959 | |||
2960 | @node Configuration | ||
2961 | @subsubsection Configuration | ||
2962 | |||
2963 | There are the following options for the wlan plugin (they should be like | ||
2964 | this in your default config file, you only need to adjust them if the | ||
2965 | values are incorrect for your system) | ||
2966 | |||
2967 | @example | ||
2968 | # section for the wlan transport plugin | ||
2969 | [transport-wlan] | ||
2970 | # interface to use, more information in the | ||
2971 | # "Before starting GNUnet" section of the handbook. | ||
2972 | INTERFACE = mon0 | ||
2973 | # testmode for developers: | ||
2974 | # 0 use wlan interface, | ||
2975 | #1 or 2 use loopback driver for tests 1 = server, 2 = client | ||
2976 | TESTMODE = 0 | ||
2977 | @end example | ||
2978 | |||
2979 | @node Before starting GNUnet | ||
2980 | @subsubsection Before starting GNUnet | ||
2981 | |||
2982 | Before starting GNUnet, you have to make sure that your wlan interface is | ||
2983 | in monitor mode. | ||
2984 | One way to put the wlan interface into monitor mode (if your interface | ||
2985 | name is wlan0) is by executing: | ||
2986 | |||
2987 | @example | ||
2988 | sudo airmon-ng start wlan0 | ||
2989 | @end example | ||
2990 | |||
2991 | @noindent | ||
2992 | Here is an example what the result should look like: | ||
2993 | |||
2994 | @example | ||
2995 | Interface Chipset Driver | ||
2996 | wlan0 Intel 4965 a/b/g/n iwl4965 - [phy0] | ||
2997 | (monitor mode enabled on mon0) | ||
2998 | @end example | ||
2999 | |||
3000 | @noindent | ||
3001 | The monitor interface is mon0 is the one that you have to put into the | ||
3002 | configuration file. | ||
3003 | |||
3004 | @node Limitations and known bugs | ||
3005 | @subsubsection Limitations and known bugs | ||
3006 | |||
3007 | Wlan speed is at the maximum of 1 Mbit/s because support for choosing the | ||
3008 | wlan speed with packet injection was removed in newer kernels. | ||
3009 | Please pester the kernel developers about fixing this. | ||
3010 | |||
3011 | The interface channel depends on the wlan network that the card is | ||
3012 | connected to. If no connection has been made since the start of the | ||
3013 | computer, it is usually the first channel of the card. | ||
3014 | Peers will only find each other and communicate if they are on the same | ||
3015 | channel. Channels must be set manually, i.e. using: | ||
3016 | |||
3017 | @example | ||
3018 | iwconfig wlan0 channel 1 | ||
3019 | @end example | ||
3020 | |||
3021 | @node Configuring HTTP(S) reverse proxy functionality using Apache or nginx | ||
3022 | @subsection Configuring HTTP(S) reverse proxy functionality using Apache or nginx | ||
3023 | |||
3024 | The HTTP plugin supports data transfer using reverse proxies. A reverse | ||
3025 | proxy forwards the HTTP request he receives with a certain URL to another | ||
3026 | webserver, here a GNUnet peer. | ||
3027 | |||
3028 | So if you have a running Apache or nginx webserver you can configure it to | ||
3029 | be a GNUnet reverse proxy. Especially if you have a well-known webiste | ||
3030 | this improves censorship resistance since it looks as normal surfing | ||
3031 | behaviour. | ||
3032 | |||
3033 | To do so, you have to do two things: | ||
3034 | |||
3035 | @itemize @bullet | ||
3036 | @item Configure your webserver to forward the GNUnet HTTP traffic | ||
3037 | @item Configure your GNUnet peer to announce the respective address | ||
3038 | @end itemize | ||
3039 | |||
3040 | As an example we want to use GNUnet peer running: | ||
3041 | |||
3042 | @itemize @bullet | ||
3043 | |||
3044 | @item HTTP server plugin on @code{gnunet.foo.org:1080} | ||
3045 | |||
3046 | @item HTTPS server plugin on @code{gnunet.foo.org:4433} | ||
3047 | |||
3048 | @item A apache or nginx webserver on | ||
3049 | @uref{http://www.foo.org/, http://www.foo.org:80/} | ||
3050 | |||
3051 | @item A apache or nginx webserver on https://www.foo.org:443/ | ||
3052 | @end itemize | ||
3053 | |||
3054 | And we want the webserver to accept GNUnet traffic under | ||
3055 | @code{http://www.foo.org/bar/}. The required steps are described here: | ||
3056 | |||
3057 | @menu | ||
3058 | * Reverse Proxy - Configure your Apache2 HTTP webserver:: | ||
3059 | * Reverse Proxy - Configure your Apache2 HTTPS webserver:: | ||
3060 | * Reverse Proxy - Configure your nginx HTTPS webserver:: | ||
3061 | * Reverse Proxy - Configure your nginx HTTP webserver:: | ||
3062 | * Reverse Proxy - Configure your GNUnet peer:: | ||
3063 | @end menu | ||
3064 | |||
3065 | @node Reverse Proxy - Configure your Apache2 HTTP webserver | ||
3066 | @subsubsection Reverse Proxy - Configure your Apache2 HTTP webserver | ||
3067 | |||
3068 | First of all you need mod_proxy installed. | ||
3069 | |||
3070 | Edit your webserver configuration. Edit | ||
3071 | @code{/etc/apache2/apache2.conf} or the site-specific configuration file. | ||
3072 | |||
3073 | In the respective @code{server config},@code{virtual host} or | ||
3074 | @code{directory} section add the following lines: | ||
3075 | |||
3076 | @example | ||
3077 | ProxyTimeout 300 | ||
3078 | ProxyRequests Off | ||
3079 | <Location /bar/ > | ||
3080 | ProxyPass http://gnunet.foo.org:1080/ | ||
3081 | ProxyPassReverse http://gnunet.foo.org:1080/ | ||
3082 | </Location> | ||
3083 | @end example | ||
3084 | |||
3085 | @node Reverse Proxy - Configure your Apache2 HTTPS webserver | ||
3086 | @subsubsection Reverse Proxy - Configure your Apache2 HTTPS webserver | ||
3087 | |||
3088 | We assume that you already have an HTTPS server running, if not please | ||
3089 | check how to configure a HTTPS host. An easy to use example is the | ||
3090 | @file{apache2/sites-available/default-ssl} example configuration file. | ||
3091 | |||
3092 | In the respective HTTPS @code{server config},@code{virtual host} or | ||
3093 | @code{directory} section add the following lines: | ||
3094 | |||
3095 | @example | ||
3096 | SSLProxyEngine On | ||
3097 | ProxyTimeout 300 | ||
3098 | ProxyRequests Off | ||
3099 | <Location /bar/ > | ||
3100 | ProxyPass https://gnunet.foo.org:4433/ | ||
3101 | ProxyPassReverse https://gnunet.foo.org:4433/ | ||
3102 | </Location> | ||
3103 | @end example | ||
3104 | |||
3105 | @noindent | ||
3106 | More information about the apache mod_proxy configuration can be found | ||
3107 | here: @uref{http://httpd.apache.org/docs/2.2/mod/mod_proxy.html#proxypass} | ||
3108 | . | ||
3109 | |||
3110 | @node Reverse Proxy - Configure your nginx HTTPS webserver | ||
3111 | @subsubsection Reverse Proxy - Configure your nginx HTTPS webserver | ||
3112 | |||
3113 | Since nginx does not support chunked encoding, you first of all have to | ||
3114 | install @code{chunkin}: @uref{http://wiki.nginx.org/HttpChunkinModule}. | ||
3115 | |||
3116 | To enable chunkin add: | ||
3117 | |||
3118 | @example | ||
3119 | chunkin on; | ||
3120 | error_page 411 = @@my_411_error; | ||
3121 | location @@my_411_error @{ | ||
3122 | chunkin_resume; | ||
3123 | @} | ||
3124 | @end example | ||
3125 | |||
3126 | @noindent | ||
3127 | Edit your webserver configuration. Edit @file{/etc/nginx/nginx.conf} or | ||
3128 | the site-specific configuration file. | ||
3129 | |||
3130 | In the @code{server} section add: | ||
3131 | |||
3132 | @example | ||
3133 | location /bar/ | ||
3134 | @{ | ||
3135 | proxy_pass http://gnunet.foo.org:1080/; | ||
3136 | proxy_buffering off; | ||
3137 | proxy_connect_timeout 5; # more than http_server | ||
3138 | proxy_read_timeout 350; # 60 default, 300s is GNUnet's idle timeout | ||
3139 | proxy_http_version 1.1; # 1.0 default | ||
3140 | proxy_next_upstream error timeout invalid_header http_500 http_503 http_502 http_504; | ||
3141 | @} | ||
3142 | @end example | ||
3143 | |||
3144 | @node Reverse Proxy - Configure your nginx HTTP webserver | ||
3145 | @subsubsection Reverse Proxy - Configure your nginx HTTP webserver | ||
3146 | |||
3147 | Edit your webserver configuration. Edit @file{/etc/nginx/nginx.conf} or | ||
3148 | the site-specific configuration file. | ||
3149 | |||
3150 | In the @code{server} section add: | ||
3151 | |||
3152 | @example | ||
3153 | ssl_session_timeout 6m; | ||
3154 | location /bar/ | ||
3155 | @{ | ||
3156 | proxy_pass https://gnunet.foo.org:4433/; | ||
3157 | proxy_buffering off; | ||
3158 | proxy_connect_timeout 5; # more than http_server | ||
3159 | proxy_read_timeout 350; # 60 default, 300s is GNUnet's idle timeout | ||
3160 | proxy_http_version 1.1; # 1.0 default | ||
3161 | proxy_next_upstream error timeout invalid_header http_500 http_503 http_502 http_504; | ||
3162 | @} | ||
3163 | @end example | ||
3164 | |||
3165 | @node Reverse Proxy - Configure your GNUnet peer | ||
3166 | @subsubsection Reverse Proxy - Configure your GNUnet peer | ||
3167 | |||
3168 | To have your GNUnet peer announce the address, you have to specify the | ||
3169 | @code{EXTERNAL_HOSTNAME} option in the @code{[transport-http_server]} | ||
3170 | section: | ||
3171 | |||
3172 | @example | ||
3173 | [transport-http_server] | ||
3174 | EXTERNAL_HOSTNAME = http://www.foo.org/bar/ | ||
3175 | @end example | ||
3176 | |||
3177 | @noindent | ||
3178 | and/or @code{[transport-https_server]} section: | ||
3179 | |||
3180 | @example | ||
3181 | [transport-https_server] | ||
3182 | EXTERNAL_HOSTNAME = https://www.foo.org/bar/ | ||
3183 | @end example | ||
3184 | |||
3185 | @noindent | ||
3186 | Now restart your webserver and your peer... | ||
3187 | |||
3188 | @node Blacklisting peers | ||
3189 | @subsection Blacklisting peers | ||
3190 | |||
3191 | Transport service supports to deny connecting to a specific peer of to a | ||
3192 | specific peer with a specific transport plugin using te blacklisting | ||
3193 | component of transport service. With@ blacklisting it is possible to deny | ||
3194 | connections to specific peers of@ to use a specific plugin to a specific | ||
3195 | peer. Peers can be blacklisted using@ the configuration or a blacklist | ||
3196 | client can be asked. | ||
3197 | |||
3198 | To blacklist peers using the configuration you have to add a section to | ||
3199 | your configuration containing the peer id of the peer to blacklist and | ||
3200 | the plugin@ if required. | ||
3201 | |||
3202 | Examples: | ||
3203 | |||
3204 | To blacklist connections to P565... on peer AG2P... using tcp add: | ||
3205 | |||
3206 | @c FIXME: This is too long and produces errors in the pdf. | ||
3207 | @example | ||
3208 | [transport-blacklist AG2PHES1BARB9IJCPAMJTFPVJ5V3A72S3F2A8SBUB8DAQ2V0O3V8G6G2JU56FHGFOHMQVKBSQFV98TCGTC3RJ1NINP82G0RC00N1520] | ||
3209 | P565723JO1C2HSN6J29TAQ22MN6CI8HTMUU55T0FUQG4CMDGGEQ8UCNBKUMB94GC8R9G4FB2SF9LDOBAJ6AMINBP4JHHDD6L7VD801G = tcp | ||
3210 | @end example | ||
3211 | |||
3212 | To blacklist connections to P565... on peer AG2P... using all plugins add: | ||
3213 | |||
3214 | @example | ||
3215 | [transport-blacklist-AG2PHES1BARB9IJCPAMJTFPVJ5V3A72S3F2A8SBUB8DAQ2V0O3V8G6G2JU56FHGFOHMQVKBSQFV98TCGTC3RJ1NINP82G0RC00N1520] | ||
3216 | P565723JO1C2HSN6J29TAQ22MN6CI8HTMUU55T0FUQG4CMDGGEQ8UCNBKUMB94GC8R9G4FB2SF9LDOBAJ6AMINBP4JHHDD6L7VD801G = | ||
3217 | @end example | ||
3218 | |||
3219 | You can also add a blacklist client usign the blacklist API. On a | ||
3220 | blacklist check, blacklisting first checks internally if the peer is | ||
3221 | blacklisted and if not, it asks the blacklisting clients. Clients are | ||
3222 | asked if it is OK to connect to a peer ID, the plugin is omitted. | ||
3223 | |||
3224 | On blacklist check for (peer, plugin) | ||
3225 | @itemize @bullet | ||
3226 | @item Do we have a local blacklist entry for this peer and this plugin?@ | ||
3227 | @item YES: disallow connection@ | ||
3228 | @item Do we have a local blacklist entry for this peer and all plugins?@ | ||
3229 | @item YES: disallow connection@ | ||
3230 | @item Does one of the clients disallow?@ | ||
3231 | @item YES: disallow connection | ||
3232 | @end itemize | ||
3233 | |||
3234 | @node Configuration of the HTTP and HTTPS transport plugins | ||
3235 | @subsection Configuration of the HTTP and HTTPS transport plugins | ||
3236 | |||
3237 | The client parts of the http and https transport plugins can be configured | ||
3238 | to use a proxy to connect to the hostlist server. This functionality can | ||
3239 | be configured in the configuration file directly or using the | ||
3240 | gnunet-setup tool. | ||
3241 | |||
3242 | Both the HTTP and HTTPS clients support the following proxy types at | ||
3243 | the moment: | ||
3244 | |||
3245 | @itemize @bullet | ||
3246 | @item HTTP 1.1 proxy | ||
3247 | @item SOCKS 4/4a/5/5 with hostname | ||
3248 | @end itemize | ||
3249 | |||
3250 | In addition authentication at the proxy with username and password can be | ||
3251 | configured. | ||
3252 | |||
3253 | To configure proxy support for the clients in the gnunet-setup tool, | ||
3254 | select the "transport" tab and activate the respective plugin. Now you | ||
3255 | can select the appropriate proxy type. The hostname or IP address | ||
3256 | (including port if required) has to be entered in the "Proxy hostname" | ||
3257 | textbox. If required, enter username and password in the "Proxy username" | ||
3258 | and "Proxy password" boxes. Be aware that these information will be stored | ||
3259 | in the configuration in plain text. | ||
3260 | |||
3261 | To configure these options directly in the configuration, you can | ||
3262 | configure the following settings in the @code{[transport-http_client]} | ||
3263 | and @code{[transport-https_client]} section of the configuration: | ||
3264 | |||
3265 | @example | ||
3266 | # Type of proxy server, | ||
3267 | # Valid values: HTTP, SOCKS4, SOCKS5, SOCKS4A, SOCKS5_HOSTNAME | ||
3268 | # Default: HTTP | ||
3269 | # PROXY_TYPE = HTTP | ||
3270 | |||
3271 | # Hostname or IP of proxy server | ||
3272 | # PROXY = | ||
3273 | # User name for proxy server | ||
3274 | # PROXY_USERNAME = | ||
3275 | # User password for proxy server | ||
3276 | # PROXY_PASSWORD = | ||
3277 | @end example | ||
3278 | |||
3279 | @node Configuring the GNU Name System | ||
3280 | @subsection Configuring the GNU Name System | ||
3281 | |||
3282 | @menu | ||
3283 | * Configuring system-wide DNS interception:: | ||
3284 | * Configuring the GNS nsswitch plugin:: | ||
3285 | * Configuring GNS on W32:: | ||
3286 | * GNS Proxy Setup:: | ||
3287 | * Setup of the GNS CA:: | ||
3288 | * Testing the GNS setup:: | ||
3289 | * Automatic Shortening in the GNU Name System:: | ||
3290 | @end menu | ||
3291 | |||
3292 | |||
3293 | @node Configuring system-wide DNS interception | ||
3294 | @subsubsection Configuring system-wide DNS interception | ||
3295 | |||
3296 | Before you install GNUnet, make sure you have a user and group 'gnunet' | ||
3297 | as well as an empty group 'gnunetdns'. | ||
3298 | |||
3299 | When using GNUnet with system-wide DNS interception, it is absolutely | ||
3300 | necessary for all GNUnet service processes to be started by | ||
3301 | @code{gnunet-service-arm} as user and group 'gnunet'. You also need to be | ||
3302 | sure to run @code{make install} as root (or use the @code{sudo} option to | ||
3303 | configure) to grant GNUnet sufficient privileges. | ||
3304 | |||
3305 | With this setup, all that is required for enabling system-wide DNS | ||
3306 | interception is for some GNUnet component (VPN or GNS) to request it. | ||
3307 | The @code{gnunet-service-dns} will then start helper programs that will | ||
3308 | make the necessary changes to your firewall (@code{iptables}) rules. | ||
3309 | |||
3310 | Note that this will NOT work if your system sends out DNS traffic to a | ||
3311 | link-local IPv6 address, as in this case GNUnet can intercept the traffic, | ||
3312 | but not inject the responses from the link-local IPv6 address. Hence you | ||
3313 | cannot use system-wide DNS interception in conjunction with link-local | ||
3314 | IPv6-based DNS servers. If such a DNS server is used, it will bypass | ||
3315 | GNUnet's DNS traffic interception. | ||
3316 | |||
3317 | Using the GNU Name System (GNS) requires two different configuration | ||
3318 | steps. | ||
3319 | First of all, GNS needs to be integrated with the operating system. Most | ||
3320 | of this section is about the operating system level integration. | ||
3321 | |||
3322 | Additionally, each individual user who wants to use the system must also | ||
3323 | initialize their GNS zones. This can be done by running (after starting | ||
3324 | GNUnet) | ||
3325 | |||
3326 | @example | ||
3327 | $ gnunet-gns-import.sh | ||
3328 | @end example | ||
3329 | |||
3330 | @noindent | ||
3331 | after the local GNUnet peer has been started. Note that the namestore (in | ||
3332 | particular the namestore database backend) should not be reconfigured | ||
3333 | afterwards (as records are not automatically migrated between backends). | ||
3334 | |||
3335 | The remainder of this chapter will detail the various methods for | ||
3336 | configuring the use of GNS with your operating system. | ||
3337 | |||
3338 | At this point in time you have different options depending on your OS: | ||
3339 | |||
3340 | @table @asis | ||
3341 | |||
3342 | @item Use the gnunet-gns-proxy This approach works for all operating | ||
3343 | systems and is likely the easiest. However, it enables GNS only for | ||
3344 | browsers, not for other applications that might be using DNS, such as SSH. | ||
3345 | Still, using the proxy is required for using HTTP with GNS and is thus | ||
3346 | recommended for all users. To do this, you simply have to run the | ||
3347 | @code{gnunet-gns-proxy-setup-ca} script as the user who will run the | ||
3348 | browser (this will create a GNS certificate authority (CA) on your system | ||
3349 | and import its key into your browser), then start @code{gnunet-gns-proxy} | ||
3350 | and inform your browser to use the Socks5 proxy which | ||
3351 | @code{gnunet-gns-proxy} makes available by default on port 7777. | ||
3352 | @item Use a nsswitch plugin (recommended on GNU systems) | ||
3353 | This approach has the advantage of offering fully personalized resolution | ||
3354 | even on multi-user systems. A potential disadvantage is that some | ||
3355 | applications might be able to bypass GNS. | ||
3356 | @item Use a W32 resolver plugin (recommended on W32) | ||
3357 | This is currently the only option on W32 systems. | ||
3358 | @item Use system-wide DNS packet interception | ||
3359 | This approach is recommended for the GNUnet VPN. It can be used to handle | ||
3360 | GNS at the same time; however, if you only use this method, you will only | ||
3361 | get one root zone per machine (not so great for multi-user systems). | ||
3362 | @end table | ||
3363 | |||
3364 | You can combine system-wide DNS packet interception with the nsswitch | ||
3365 | plugin. | ||
3366 | The setup of the system-wide DNS interception is described here. All of | ||
3367 | the other GNS-specific configuration steps are described in the following | ||
3368 | sections. | ||
3369 | |||
3370 | @node Configuring the GNS nsswitch plugin | ||
3371 | @subsubsection Configuring the GNS nsswitch plugin | ||
3372 | |||
3373 | The Name Service Switch (NSS) is a facility in Unix-like operating systems | ||
3374 | @footnote{More accurate: NSS is a functionality of the GNU C Library} | ||
3375 | that provides a variety of sources for common configuration databases and | ||
3376 | name resolution mechanisms. | ||
3377 | A superuser (system administrator) usually configures the | ||
3378 | operating system's name services using the file | ||
3379 | @file{/etc/nsswitch.conf}. | ||
3380 | |||
3381 | GNS provides a NSS plugin to integrate GNS name resolution with the | ||
3382 | operating system's name resolution process. | ||
3383 | To use the GNS NSS plugin you have to either | ||
3384 | |||
3385 | @itemize @bullet | ||
3386 | @item install GNUnet as root or | ||
3387 | @item compile GNUnet with the @code{--with-sudo=yes} switch. | ||
3388 | @end itemize | ||
3389 | |||
3390 | Name resolution is controlled by the @emph{hosts} section in the NSS | ||
3391 | configuration. By default this section first performs a lookup in the | ||
3392 | @file{/etc/hosts} file and then in DNS. | ||
3393 | The nsswitch file should contain a line similar to: | ||
3394 | |||
3395 | @example | ||
3396 | hosts: files dns [NOTFOUND=return] mdns4_minimal mdns4 | ||
3397 | @end example | ||
3398 | |||
3399 | @noindent | ||
3400 | Here the GNS NSS plugin can be added to perform a GNS lookup before | ||
3401 | performing a DNS lookup. | ||
3402 | The GNS NSS plugin has to be added to the "hosts" section in | ||
3403 | @file{/etc/nsswitch.conf} file before DNS related plugins: | ||
3404 | |||
3405 | @example | ||
3406 | ... | ||
3407 | hosts: files gns [NOTFOUND=return] dns mdns4_minimal mdns4 | ||
3408 | ... | ||
3409 | @end example | ||
3410 | |||
3411 | @noindent | ||
3412 | The @code{NOTFOUND=return} will ensure that if a @code{.gnu} name is not | ||
3413 | found in GNS it will not be queried in DNS. | ||
3414 | |||
3415 | @node Configuring GNS on W32 | ||
3416 | @subsubsection Configuring GNS on W32 | ||
3417 | |||
3418 | This document is a guide to configuring GNU Name System on W32-compatible | ||
3419 | platforms. | ||
3420 | |||
3421 | After GNUnet is installed, run the w32nsp-install tool: | ||
3422 | |||
3423 | @example | ||
3424 | w32nsp-install.exe libw32nsp-0.dll | ||
3425 | @end example | ||
3426 | |||
3427 | @noindent | ||
3428 | ('0' is the library version of W32 NSP; it might increase in the future, | ||
3429 | change the invocation accordingly). | ||
3430 | |||
3431 | This will install GNS namespace provider into the system and allow other | ||
3432 | applications to resolve names that end in '@strong{gnu}' | ||
3433 | and '@strong{zkey}'. Note that namespace provider requires | ||
3434 | gnunet-gns-helper-service-w32 to be running, as well as gns service | ||
3435 | itself (and its usual dependencies). | ||
3436 | |||
3437 | Namespace provider is hardcoded to connect to @strong{127.0.0.1:5353}, | ||
3438 | and this is where gnunet-gns-helper-service-w32 should be listening to | ||
3439 | (and is configured to listen to by default). | ||
3440 | |||
3441 | To uninstall the provider, run: | ||
3442 | |||
3443 | @example | ||
3444 | w32nsp-uninstall.exe | ||
3445 | @end example | ||
3446 | |||
3447 | @noindent | ||
3448 | (uses provider GUID to uninstall it, does not need a dll name). | ||
3449 | |||
3450 | Note that while MSDN claims that other applications will only be able to | ||
3451 | use the new namespace provider after re-starting, in reality they might | ||
3452 | stat to use it without that. Conversely, they might stop using the | ||
3453 | provider after it's been uninstalled, even if they were not re-started. | ||
3454 | W32 will not permit namespace provider library to be deleted or | ||
3455 | overwritten while the provider is installed, and while there is at least | ||
3456 | one process still using it (even after it was uninstalled). | ||
3457 | |||
3458 | @node GNS Proxy Setup | ||
3459 | @subsubsection GNS Proxy Setup | ||
3460 | |||
3461 | When using the GNU Name System (GNS) to browse the WWW, there are several | ||
3462 | issues that can be solved by adding the GNS Proxy to your setup: | ||
3463 | |||
3464 | @itemize @bullet | ||
3465 | |||
3466 | @item If the target website does not support GNS, it might assume that it | ||
3467 | is operating under some name in the legacy DNS system (such as | ||
3468 | example.com). It may then attempt to set cookies for that domain, and the | ||
3469 | web server might expect a @code{Host: example.com} header in the request | ||
3470 | from your browser. | ||
3471 | However, your browser might be using @code{example.gnu} for the | ||
3472 | @code{Host} header and might only accept (and send) cookies for | ||
3473 | @code{example.gnu}. The GNS Proxy will perform the necessary translations | ||
3474 | of the hostnames for cookies and HTTP headers (using the LEHO record for | ||
3475 | the target domain as the desired substitute). | ||
3476 | |||
3477 | @item If using HTTPS, the target site might include an SSL certificate | ||
3478 | which is either only valid for the LEHO domain or might match a TLSA | ||
3479 | record in GNS. However, your browser would expect a valid certificate for | ||
3480 | @code{example.gnu}, not for some legacy domain name. The proxy will | ||
3481 | validate the certificate (either against LEHO or TLSA) and then | ||
3482 | on-the-fly produce a valid certificate for the exchange, signed by your | ||
3483 | own CA. Assuming you installed the CA of your proxy in your browser's | ||
3484 | certificate authority list, your browser will then trust the | ||
3485 | HTTPS/SSL/TLS connection, as the hostname mismatch is hidden by the proxy. | ||
3486 | |||
3487 | @item Finally, the proxy will in the future indicate to the server that it | ||
3488 | speaks GNS, which will enable server operators to deliver GNS-enabled web | ||
3489 | sites to your browser (and continue to deliver legacy links to legacy | ||
3490 | browsers) | ||
3491 | @end itemize | ||
3492 | |||
3493 | @node Setup of the GNS CA | ||
3494 | @subsubsection Setup of the GNS CA | ||
3495 | |||
3496 | First you need to create a CA certificate that the proxy can use. | ||
3497 | To do so use the provided script gnunet-gns-proxy-ca: | ||
3498 | |||
3499 | @example | ||
3500 | $ gnunet-gns-proxy-setup-ca | ||
3501 | @end example | ||
3502 | |||
3503 | @noindent | ||
3504 | This will create a personal certification authority for you and add this | ||
3505 | authority to the firefox and chrome database. The proxy will use the this | ||
3506 | CA certificate to generate @code{*.gnu} client certificates on the fly. | ||
3507 | |||
3508 | Note that the proxy uses libcurl. Make sure your version of libcurl uses | ||
3509 | GnuTLS and NOT OpenSSL. The proxy will @b{not} work with libcurl compiled | ||
3510 | against OpenSSL. | ||
3511 | |||
3512 | You can check the configuration your libcurl was build with by | ||
3513 | running: | ||
3514 | |||
3515 | @example | ||
3516 | curl --version | ||
3517 | @end example | ||
3518 | |||
3519 | the output will look like this (without the linebreaks): | ||
3520 | |||
3521 | @example | ||
3522 | gnurl --version | ||
3523 | curl 7.56.0 (x86_64-unknown-linux-gnu) libcurl/7.56.0 \ | ||
3524 | GnuTLS/3.5.13 zlib/1.2.11 libidn2/2.0.4 | ||
3525 | Release-Date: 2017-10-08 | ||
3526 | Protocols: http https | ||
3527 | Features: AsynchDNS IDN IPv6 Largefile NTLM SSL libz \ | ||
3528 | TLS-SRP UnixSockets HTTPS-proxy | ||
3529 | @end example | ||
3530 | |||
3531 | @node Testing the GNS setup | ||
3532 | @subsubsection Testing the GNS setup | ||
3533 | |||
3534 | Now for testing purposes we can create some records in our zone to test | ||
3535 | the SSL functionality of the proxy: | ||
3536 | |||
3537 | @example | ||
3538 | $ gnunet-namestore -a -e "1 d" -n "homepage" -t A -V 131.159.74.67 | ||
3539 | $ gnunet-namestore -a -e "1 d" -n "homepage" -t LEHO -V "gnunet.org" | ||
3540 | @end example | ||
3541 | |||
3542 | @noindent | ||
3543 | At this point we can start the proxy. Simply execute | ||
3544 | |||
3545 | @example | ||
3546 | $ gnunet-gns-proxy | ||
3547 | @end example | ||
3548 | |||
3549 | @noindent | ||
3550 | Configure your browser to use this SOCKSv5 proxy on port 7777 and visit | ||
3551 | this link. | ||
3552 | If you use @command{Firefox} (or one of its deriviates/forks such as | ||
3553 | Icecat) you also have to go to @code{about:config} and set the key | ||
3554 | @code{network.proxy.socks_remote_dns} to @code{true}. | ||
3555 | |||
3556 | When you visit @code{https://homepage.gnu/}, you should get to the | ||
3557 | @code{https://gnunet.org/} frontpage and the browser (with the correctly | ||
3558 | configured proxy) should give you a valid SSL certificate for | ||
3559 | @code{homepage.gnu} and no warnings. It should look like this: | ||
3560 | |||
3561 | @c FIXME: Image does not exist, create it or save it from Drupal? | ||
3562 | @c @image{images/gnunethpgns.png,5in,, picture of homepage.gnu in Webbrowser} | ||
3563 | |||
3564 | @node Automatic Shortening in the GNU Name System | ||
3565 | @subsubsection Automatic Shortening in the GNU Name System | ||
3566 | |||
3567 | This page describes a possible option for 'automatic name shortening', | ||
3568 | which you can choose to enable with the GNU Name System. | ||
3569 | |||
3570 | When GNS encounters a name for the first time, it can use the 'NICK' | ||
3571 | record of the originating zone to automatically generate a name for the | ||
3572 | zone. If automatic shortening is enabled, those auto-generated names will | ||
3573 | be placed (as private records) into your personal 'shorten' zone (to | ||
3574 | prevent confusion with manually selected names). | ||
3575 | Then, in the future, if the same name is encountered again, GNS will | ||
3576 | display the shortened name instead (the first time, the long name will | ||
3577 | still be used as shortening typically happens asynchronously as looking up | ||
3578 | the 'NICK' record takes some time). Using this feature can be a convenient | ||
3579 | way to avoid very long @code{.gnu} names; however, note that names from | ||
3580 | the shorten-zone are assigned on a first-come-first-serve basis and should | ||
3581 | not be trusted. Furthermore, if you enable this feature, you will no | ||
3582 | longer see the full delegation chain for zones once shortening has been | ||
3583 | applied. | ||
3584 | |||
3585 | @node Configuring the GNUnet VPN | ||
3586 | @subsection Configuring the GNUnet VPN | ||
3587 | |||
3588 | @menu | ||
3589 | * IPv4 address for interface:: | ||
3590 | * IPv6 address for interface:: | ||
3591 | * Configuring the GNUnet VPN DNS:: | ||
3592 | * Configuring the GNUnet VPN Exit Service:: | ||
3593 | * IP Address of external DNS resolver:: | ||
3594 | * IPv4 address for Exit interface:: | ||
3595 | * IPv6 address for Exit interface:: | ||
3596 | @end menu | ||
3597 | |||
3598 | Before configuring the GNUnet VPN, please make sure that system-wide DNS | ||
3599 | interception is configured properly as described in the section on the | ||
3600 | GNUnet DNS setup. @pxref{Configuring the GNU Name System}, | ||
3601 | if you haven't done so already. | ||
3602 | |||
3603 | The default options for the GNUnet VPN are usually sufficient to use | ||
3604 | GNUnet as a Layer 2 for your Internet connection. | ||
3605 | However, what you always have to specify is which IP protocol you want | ||
3606 | to tunnel: IPv4, IPv6 or both. | ||
3607 | Furthermore, if you tunnel both, you most likely should also tunnel | ||
3608 | all of your DNS requests. | ||
3609 | You theoretically can tunnel "only" your DNS traffic, but that usually | ||
3610 | makes little sense. | ||
3611 | |||
3612 | The other options as shown on the gnunet-setup tool are: | ||
3613 | |||
3614 | @node IPv4 address for interface | ||
3615 | @subsubsection IPv4 address for interface | ||
3616 | |||
3617 | This is the IPv4 address the VPN interface will get. You should pick an | ||
3618 | 'private' IPv4 network that is not yet in use for you system. For example, | ||
3619 | if you use @code{10.0.0.1/255.255.0.0} already, you might use | ||
3620 | @code{10.1.0.1/255.255.0.0}. | ||
3621 | If you use @code{10.0.0.1/255.0.0.0} already, then you might use | ||
3622 | @code{192.168.0.1/255.255.0.0}. | ||
3623 | If your system is not in a private IP-network, using any of the above will | ||
3624 | work fine. | ||
3625 | You should try to make the mask of the address big enough | ||
3626 | (@code{255.255.0.0} or, even better, @code{255.0.0.0}) to allow more | ||
3627 | mappings of remote IP Addresses into this range. | ||
3628 | However, even a @code{255.255.255.0} mask will suffice for most users. | ||
3629 | |||
3630 | @node IPv6 address for interface | ||
3631 | @subsubsection IPv6 address for interface | ||
3632 | |||
3633 | The IPv6 address the VPN interface will get. Here you can specify any | ||
3634 | non-link-local address (the address should not begin with @code{fe80:}). | ||
3635 | A subnet Unique Local Unicast (@code{fd00::/8} prefix) that you are | ||
3636 | currently not using would be a good choice. | ||
3637 | |||
3638 | @node Configuring the GNUnet VPN DNS | ||
3639 | @subsubsection Configuring the GNUnet VPN DNS | ||
3640 | |||
3641 | To resolve names for remote nodes, activate the DNS exit option. | ||
3642 | |||
3643 | @node Configuring the GNUnet VPN Exit Service | ||
3644 | @subsubsection Configuring the GNUnet VPN Exit Service | ||
3645 | |||
3646 | If you want to allow other users to share your Internet connection (yes, | ||
3647 | this may be dangerous, just as running a Tor exit node) or want to | ||
3648 | provide access to services on your host (this should be less dangerous, | ||
3649 | as long as those services are secure), you have to enable the GNUnet exit | ||
3650 | daemon. | ||
3651 | |||
3652 | You then get to specify which exit functions you want to provide. By | ||
3653 | enabling the exit daemon, you will always automatically provide exit | ||
3654 | functions for manually configured local services (this component of the | ||
3655 | system is under | ||
3656 | development and not documented further at this time). As for those | ||
3657 | services you explicitly specify the target IP address and port, there is | ||
3658 | no significant security risk in doing so. | ||
3659 | |||
3660 | Furthermore, you can serve as a DNS, IPv4 or IPv6 exit to the Internet. | ||
3661 | Being a DNS exit is usually pretty harmless. However, enabling IPv4 or | ||
3662 | IPv6-exit without further precautions may enable adversaries to access | ||
3663 | your local network, send spam, attack other systems from your Internet | ||
3664 | connection and to other mischief that will appear to come from your | ||
3665 | machine. This may or may not get you into legal trouble. | ||
3666 | If you want to allow IPv4 or IPv6-exit functionality, you should strongly | ||
3667 | consider adding additional firewall rules manually to protect your local | ||
3668 | network and to restrict outgoing TCP traffic (i.e. by not allowing access | ||
3669 | to port 25). While we plan to improve exit-filtering in the future, | ||
3670 | you're currently on your own here. | ||
3671 | Essentially, be prepared for any kind of IP-traffic to exit the respective | ||
3672 | TUN interface (and GNUnet will enable IP-forwarding and NAT for the | ||
3673 | interface automatically). | ||
3674 | |||
3675 | Additional configuration options of the exit as shown by the gnunet-setup | ||
3676 | tool are: | ||
3677 | |||
3678 | @node IP Address of external DNS resolver | ||
3679 | @subsubsection IP Address of external DNS resolver | ||
3680 | |||
3681 | If DNS traffic is to exit your machine, it will be send to this DNS | ||
3682 | resolver. You can specify an IPv4 or IPv6 address. | ||
3683 | |||
3684 | @node IPv4 address for Exit interface | ||
3685 | @subsubsection IPv4 address for Exit interface | ||
3686 | |||
3687 | This is the IPv4 address the Interface will get. Make the mask of the | ||
3688 | address big enough (255.255.0.0 or, even better, 255.0.0.0) to allow more | ||
3689 | mappings of IP addresses into this range. As for the VPN interface, any | ||
3690 | unused, private IPv4 address range will do. | ||
3691 | |||
3692 | @node IPv6 address for Exit interface | ||
3693 | @subsubsection IPv6 address for Exit interface | ||
3694 | |||
3695 | The public IPv6 address the interface will get. If your kernel is not a | ||
3696 | very recent kernel and you are willing to manually enable IPv6-NAT, the | ||
3697 | IPv6 address you specify here must be a globally routed IPv6 address of | ||
3698 | your host. | ||
3699 | |||
3700 | Suppose your host has the address @code{2001:4ca0::1234/64}, then | ||
3701 | using @code{2001:4ca0::1:0/112} would be fine (keep the first 64 bits, | ||
3702 | then change at least one bit in the range before the bitmask, in the | ||
3703 | example above we changed bit 111 from 0 to 1). | ||
3704 | |||
3705 | You may also have to configure your router to route traffic for the entire | ||
3706 | subnet (@code{2001:4ca0::1:0/112} for example) through your computer (this | ||
3707 | should be automatic with IPv6, but obviously anything can be | ||
3708 | disabled). | ||
3709 | |||
3710 | @node Bandwidth Configuration | ||
3711 | @subsection Bandwidth Configuration | ||
3712 | |||
3713 | You can specify how many bandwidth GNUnet is allowed to use to receive | ||
3714 | and send data. This is important for users with limited bandwidth or | ||
3715 | traffic volume. | ||
3716 | |||
3717 | @node Configuring NAT | ||
3718 | @subsection Configuring NAT | ||
3719 | |||
3720 | Most hosts today do not have a normal global IP address but instead are | ||
3721 | behind a router performing Network Address Translation (NAT) which assigns | ||
3722 | each host in the local network a private IP address. | ||
3723 | As a result, these machines cannot trivially receive inbound connections | ||
3724 | from the Internet. GNUnet supports NAT traversal to enable these machines | ||
3725 | to receive incoming connections from other peers despite their | ||
3726 | limitations. | ||
3727 | |||
3728 | In an ideal world, you can press the "Attempt automatic configuration" | ||
3729 | button in gnunet-setup to automatically configure your peer correctly. | ||
3730 | Alternatively, your distribution might have already triggered this | ||
3731 | automatic configuration during the installation process. | ||
3732 | However, automatic configuration can fail to determine the optimal | ||
3733 | settings, resulting in your peer either not receiving as many connections | ||
3734 | as possible, or in the worst case it not connecting to the network at all. | ||
3735 | |||
3736 | To manually configure the peer, you need to know a few things about your | ||
3737 | network setup. First, determine if you are behind a NAT in the first | ||
3738 | place. | ||
3739 | This is always the case if your IP address starts with "10.*" or | ||
3740 | "192.168.*". Next, if you have control over your NAT router, you may | ||
3741 | choose to manually configure it to allow GNUnet traffic to your host. | ||
3742 | If you have configured your NAT to forward traffic on ports 2086 (and | ||
3743 | possibly 1080) to your host, you can check the "NAT ports have been opened | ||
3744 | manually" option, which corresponds to the "PUNCHED_NAT" option in the | ||
3745 | configuration file. If you did not punch your NAT box, it may still be | ||
3746 | configured to support UPnP, which allows GNUnet to automatically | ||
3747 | configure it. In that case, you need to install the "upnpc" command, | ||
3748 | enable UPnP (or PMP) on your NAT box and set the "Enable NAT traversal | ||
3749 | via UPnP or PMP" option (corresponding to "ENABLE_UPNP" in the | ||
3750 | configuration file). | ||
3751 | |||
3752 | Some NAT boxes can be traversed using the autonomous NAT traversal method. | ||
3753 | This requires certain GNUnet components to be installed with "SUID" | ||
3754 | prividledges on your system (so if you're installing on a system you do | ||
3755 | not have administrative rights to, this will not work). | ||
3756 | If you installed as 'root', you can enable autonomous NAT traversal by | ||
3757 | checking the "Enable NAT traversal using ICMP method". | ||
3758 | The ICMP method requires a way to determine your NAT's external (global) | ||
3759 | IP address. This can be done using either UPnP, DynDNS, or by manual | ||
3760 | configuration. If you have a DynDNS name or know your external IP address, | ||
3761 | you should enter that name under "External (public) IPv4 address" (which | ||
3762 | corresponds to the "EXTERNAL_ADDRESS" option in the configuration file). | ||
3763 | If you leave the option empty, GNUnet will try to determine your external | ||
3764 | IP address automatically (which may fail, in which case autonomous | ||
3765 | NAT traversal will then not work). | ||
3766 | |||
3767 | Finally, if you yourself are not behind NAT but want to be able to | ||
3768 | connect to NATed peers using autonomous NAT traversal, you need to check | ||
3769 | the "Enable connecting to NATed peers using ICMP method" box. | ||
3770 | |||
3771 | |||
3772 | @node Peer configuration for distributions | ||
3773 | @subsection Peer configuration for distributions | ||
3774 | |||
3775 | The "GNUNET_DATA_HOME" in "[path]" in @file{/etc/gnunet.conf} should be | ||
3776 | manually set to "/var/lib/gnunet/data/" as the default | ||
3777 | "~/.local/share/gnunet/" is probably not that appropriate in this case. | ||
3778 | Similarly, distributions may consider pointing "GNUNET_RUNTIME_DIR" to | ||
3779 | "/var/run/gnunet/" and "GNUNET_HOME" to "/var/lib/gnunet/". Also, should a | ||
3780 | distribution decide to override system defaults, all of these changes | ||
3781 | should be done in a custom @file{/etc/gnunet.conf} and not in the files | ||
3782 | in the @file{config.d/} directory. | ||
3783 | |||
3784 | Given the proposed access permissions, the "gnunet-setup" tool must be | ||
3785 | run as use "gnunet" (and with option "-c /etc/gnunet.conf" so that it | ||
3786 | modifies the system configuration). As always, gnunet-setup should be run | ||
3787 | after the GNUnet peer was stopped using "gnunet-arm -e". Distributions | ||
3788 | might want to include a wrapper for gnunet-setup that allows the | ||
3789 | desktop-user to "sudo" (i.e. using gtksudo) to the "gnunet" user account | ||
3790 | and then runs "gnunet-arm -e", "gnunet-setup" and "gnunet-arm -s" in | ||
3791 | sequence. | ||
3792 | |||
3793 | @node How to start and stop a GNUnet peer | ||
3794 | @section How to start and stop a GNUnet peer | ||
3795 | |||
3796 | This section describes how to start a GNUnet peer. It assumes that you | ||
3797 | have already compiled and installed GNUnet and its' dependencies. | ||
3798 | Before you start a GNUnet peer, you may want to create a configuration | ||
3799 | file using gnunet-setup (but you do not have to). | ||
3800 | Sane defaults should exist in your | ||
3801 | @file{$GNUNET_PREFIX/share/gnunet/config.d/} directory, so in practice | ||
3802 | you could simply start without any configuration. If you want to | ||
3803 | configure your peer later, you need to stop it before invoking the | ||
3804 | @code{gnunet-setup} tool to customize further and to test your | ||
3805 | configuration (@code{gnunet-setup} has build-in test functions). | ||
3806 | |||
3807 | The most important option you might have to still set by hand is in | ||
3808 | [PATHS]. Here, you use the option "GNUNET_HOME" to specify the path where | ||
3809 | GNUnet should store its data. | ||
3810 | It defaults to @code{$HOME/}, which again should work for most users. | ||
3811 | Make sure that the directory specified as GNUNET_HOME is writable to | ||
3812 | the user that you will use to run GNUnet (note that you can run frontends | ||
3813 | using other users, GNUNET_HOME must only be accessible to the user used to | ||
3814 | run the background processes). | ||
3815 | |||
3816 | You will also need to make one central decision: should all of GNUnet be | ||
3817 | run under your normal UID, or do you want distinguish between system-wide | ||
3818 | (user-independent) GNUnet services and personal GNUnet services. The | ||
3819 | multi-user setup is slightly more complicated, but also more secure and | ||
3820 | generally recommended. | ||
3821 | |||
3822 | @menu | ||
3823 | * The Single-User Setup:: | ||
3824 | * The Multi-User Setup:: | ||
3825 | * Killing GNUnet services:: | ||
3826 | * Access Control for GNUnet:: | ||
3827 | @end menu | ||
3828 | |||
3829 | @node The Single-User Setup | ||
3830 | @subsection The Single-User Setup | ||
3831 | |||
3832 | For the single-user setup, you do not need to do anything special and can | ||
3833 | just start the GNUnet background processes using @code{gnunet-arm}. | ||
3834 | By default, GNUnet looks in @file{~/.config/gnunet.conf} for a | ||
3835 | configuration (or @code{$XDG_CONFIG_HOME/gnunet.conf} if@ | ||
3836 | @code{$XDG_CONFIG_HOME} is defined). If your configuration lives | ||
3837 | elsewhere, you need to pass the @code{-c FILENAME} option to all GNUnet | ||
3838 | commands. | ||
3839 | |||
3840 | Assuming the configuration file is called @file{~/.config/gnunet.conf}, | ||
3841 | you start your peer using the @code{gnunet-arm} command (say as user | ||
3842 | @code{gnunet}) using: | ||
3843 | |||
3844 | @example | ||
3845 | gnunet-arm -c ~/.config/gnunet.conf -s | ||
3846 | @end example | ||
3847 | |||
3848 | @noindent | ||
3849 | The "-s" option here is for "start". The command should return almost | ||
3850 | instantly. If you want to stop GNUnet, you can use: | ||
3851 | |||
3852 | @example | ||
3853 | gnunet-arm -c ~/.config/gnunet.conf -e | ||
3854 | @end example | ||
3855 | |||
3856 | @noindent | ||
3857 | The "-e" option here is for "end". | ||
3858 | |||
3859 | Note that this will only start the basic peer, no actual applications | ||
3860 | will be available. | ||
3861 | If you want to start the file-sharing service, use (after starting | ||
3862 | GNUnet): | ||
3863 | |||
3864 | @example | ||
3865 | gnunet-arm -c ~/.config/gnunet.conf -i fs | ||
3866 | @end example | ||
3867 | |||
3868 | @noindent | ||
3869 | The "-i fs" option here is for "initialize" the "fs" (file-sharing) | ||
3870 | application. You can also selectively kill only file-sharing support using | ||
3871 | |||
3872 | @example | ||
3873 | gnunet-arm -c ~/.config/gnunet.conf -k fs | ||
3874 | @end example | ||
3875 | |||
3876 | @noindent | ||
3877 | Assuming that you want certain services (like file-sharing) to be always | ||
3878 | automatically started whenever you start GNUnet, you can activate them by | ||
3879 | setting "FORCESTART=YES" in the respective section of the configuration | ||
3880 | file (for example, "[fs]"). Then GNUnet with file-sharing support would | ||
3881 | be started whenever you@ enter: | ||
3882 | |||
3883 | @example | ||
3884 | gnunet-arm -c ~/.config/gnunet.conf -s | ||
3885 | @end example | ||
3886 | |||
3887 | @noindent | ||
3888 | Alternatively, you can combine the two options: | ||
3889 | |||
3890 | @example | ||
3891 | gnunet-arm -c ~/.config/gnunet.conf -s -i fs | ||
3892 | @end example | ||
3893 | |||
3894 | @noindent | ||
3895 | Using @code{gnunet-arm} is also the preferred method for initializing | ||
3896 | GNUnet from @code{init}. | ||
3897 | |||
3898 | Finally, you should edit your @code{crontab} (using the @code{crontab} | ||
3899 | command) and insert a line@ | ||
3900 | |||
3901 | @example | ||
3902 | @@reboot gnunet-arm -c ~/.config/gnunet.conf -s | ||
3903 | @end example | ||
3904 | |||
3905 | to automatically start your peer whenever your system boots. | ||
3906 | |||
3907 | @node The Multi-User Setup | ||
3908 | @subsection The Multi-User Setup | ||
3909 | |||
3910 | This requires you to create a user @code{gnunet} and an additional group | ||
3911 | @code{gnunetdns}, prior to running @code{make install} during | ||
3912 | installation. | ||
3913 | Then, you create a configuration file @file{/etc/gnunet.conf} which should | ||
3914 | contain the lines:@ | ||
3915 | |||
3916 | @example | ||
3917 | [arm] | ||
3918 | SYSTEM_ONLY = YES | ||
3919 | USER_ONLY = NO | ||
3920 | @end example | ||
3921 | |||
3922 | @noindent | ||
3923 | Then, perform the same steps to run GNUnet as in the per-user | ||
3924 | configuration, except as user @code{gnunet} (including the | ||
3925 | @code{crontab} installation). | ||
3926 | You may also want to run @code{gnunet-setup} to configure your peer | ||
3927 | (databases, etc.). | ||
3928 | Make sure to pass @code{-c /etc/gnunet.conf} to all commands. If you | ||
3929 | run @code{gnunet-setup} as user @code{gnunet}, you might need to change | ||
3930 | permissions on @file{/etc/gnunet.conf} so that the @code{gnunet} user can | ||
3931 | write to the file (during setup). | ||
3932 | |||
3933 | Afterwards, you need to perform another setup step for each normal user | ||
3934 | account from which you want to access GNUnet. First, grant the normal user | ||
3935 | (@code{$USER}) permission to the group gnunet: | ||
3936 | |||
3937 | @example | ||
3938 | # adduser $USER gnunet | ||
3939 | @end example | ||
3940 | |||
3941 | @noindent | ||
3942 | Then, create a configuration file in @file{~/.config/gnunet.conf} for the | ||
3943 | $USER with the lines: | ||
3944 | |||
3945 | @example | ||
3946 | [arm] | ||
3947 | SYSTEM_ONLY = NO | ||
3948 | USER_ONLY = YES | ||
3949 | @end example | ||
3950 | |||
3951 | @noindent | ||
3952 | This will ensure that @code{gnunet-arm} when started by the normal user | ||
3953 | will only run services that are per-user, and otherwise rely on the | ||
3954 | system-wide services. | ||
3955 | Note that the normal user may run gnunet-setup, but the | ||
3956 | configuration would be ineffective as the system-wide services will use | ||
3957 | @file{/etc/gnunet.conf} and ignore options set by individual users. | ||
3958 | |||
3959 | Again, each user should then start the peer using | ||
3960 | @file{gnunet-arm -s} --- and strongly consider adding logic to start | ||
3961 | the peer automatically to their crontab. | ||
3962 | |||
3963 | Afterwards, you should see two (or more, if you have more than one USER) | ||
3964 | @code{gnunet-service-arm} processes running in your system. | ||
3965 | |||
3966 | @node Killing GNUnet services | ||
3967 | @subsection Killing GNUnet services | ||
3968 | |||
3969 | It is not necessary to stop GNUnet services explicitly when shutting | ||
3970 | down your computer. | ||
3971 | |||
3972 | It should be noted that manually killing "most" of the | ||
3973 | @code{gnunet-service} processes is generally not a successful method for | ||
3974 | stopping a peer (since @code{gnunet-service-arm} will instantly restart | ||
3975 | them). The best way to explicitly stop a peer is using | ||
3976 | @code{gnunet-arm -e}; note that the per-user services may need to be | ||
3977 | terminated before the system-wide services will terminate normally. | ||
3978 | |||
3979 | @node Access Control for GNUnet | ||
3980 | @subsection Access Control for GNUnet | ||
3981 | |||
3982 | This chapter documents how we plan to make access control work within the | ||
3983 | GNUnet system for a typical peer. It should be read as a best-practice | ||
3984 | installation guide for advanced users and builders of binary | ||
3985 | distributions. The recommendations in this guide apply to POSIX-systems | ||
3986 | with full support for UNIX domain sockets only. | ||
3987 | |||
3988 | Note that this is an advanced topic. The discussion presumes a very good | ||
3989 | understanding of users, groups and file permissions. Normal users on | ||
3990 | hosts with just a single user can just install GNUnet under their own | ||
3991 | account (and possibly allow the installer to use SUDO to grant additional | ||
3992 | permissions for special GNUnet tools that need additional rights). | ||
3993 | The discussion below largely applies to installations where multiple users | ||
3994 | share a system and to installations where the best possible security is | ||
3995 | paramount. | ||
3996 | |||
3997 | A typical GNUnet system consists of components that fall into four | ||
3998 | categories: | ||
3999 | |||
4000 | @table @asis | ||
4001 | |||
4002 | @item User interfaces | ||
4003 | User interfaces are not security sensitive and are supposed to be run and | ||
4004 | used by normal system users. | ||
4005 | The GTK GUIs and most command-line programs fall into this category. | ||
4006 | Some command-line tools (like gnunet-transport) should be excluded as they | ||
4007 | offer low-level access that normal users should not need. | ||
4008 | @item System services and support tools | ||
4009 | System services should always run and offer services that can then be | ||
4010 | accessed by the normal users. | ||
4011 | System services do not require special permissions, but as they are not | ||
4012 | specific to a particular user, they probably should not run as a | ||
4013 | particular user. Also, there should typically only be one GNUnet peer per | ||
4014 | host. System services include the gnunet-service and gnunet-daemon | ||
4015 | programs; support tools include command-line programs such as gnunet-arm. | ||
4016 | @item Priviledged helpers | ||
4017 | Some GNUnet components require root rights to open raw sockets or perform | ||
4018 | other special operations. These gnunet-helper binaries are typically | ||
4019 | installed SUID and run from services or daemons. | ||
4020 | @item Critical services | ||
4021 | Some GNUnet services (such as the DNS service) can manipulate the service | ||
4022 | in deep and possibly highly security sensitive ways. For example, the DNS | ||
4023 | service can be used to intercept and alter any DNS query originating from | ||
4024 | the local machine. Access to the APIs of these critical services and their | ||
4025 | priviledged helpers must be tightly controlled. | ||
4026 | @end table | ||
4027 | |||
4028 | @c FIXME: The titles of these chapters are too long in the index. | ||
4029 | |||
4030 | @menu | ||
4031 | * Recommendation - Disable access to services via TCP:: | ||
4032 | * Recommendation - Run most services as system user "gnunet":: | ||
4033 | * Recommendation - Control access to services using group "gnunet":: | ||
4034 | * Recommendation - Limit access to certain SUID binaries by group "gnunet":: | ||
4035 | * Recommendation - Limit access to critical gnunet-helper-dns to group "gnunetdns":: | ||
4036 | * Differences between "make install" and these recommendations:: | ||
4037 | @end menu | ||
4038 | |||
4039 | @node Recommendation - Disable access to services via TCP | ||
4040 | @subsubsection Recommendation - Disable access to services via TCP | ||
4041 | |||
4042 | GNUnet services allow two types of access: via TCP socket or via UNIX | ||
4043 | domain socket. | ||
4044 | If the service is available via TCP, access control can only be | ||
4045 | implemented by restricting connections to a particular range of IP | ||
4046 | addresses. | ||
4047 | This is acceptable for non-critical services that are supposed to be | ||
4048 | available to all users on the local system or local network. | ||
4049 | However, as TCP is generally less efficient and it is rarely the case | ||
4050 | that a single GNUnet peer is supposed to serve an entire local network, | ||
4051 | the default configuration should disable TCP access to all GNUnet | ||
4052 | services on systems with support for UNIX domain sockets. | ||
4053 | As of GNUnet 0.9.2, configuration files with TCP access disabled should be | ||
4054 | generated by default. Users can re-enable TCP access to particular | ||
4055 | services simply by specifying a non-zero port number in the section of | ||
4056 | the respective service. | ||
4057 | |||
4058 | |||
4059 | @node Recommendation - Run most services as system user "gnunet" | ||
4060 | @subsubsection Recommendation - Run most services as system user "gnunet" | ||
4061 | |||
4062 | GNUnet's main services should be run as a separate user "gnunet" in a | ||
4063 | special group "gnunet". | ||
4064 | The user "gnunet" should start the peer using "gnunet-arm -s" during | ||
4065 | system startup. The home directory for this user should be | ||
4066 | @file{/var/lib/gnunet} and the configuration file should be | ||
4067 | @file{/etc/gnunet.conf}. | ||
4068 | Only the @code{gnunet} user should have the right to access | ||
4069 | @file{/var/lib/gnunet} (@emph{mode: 700}). | ||
4070 | |||
4071 | @node Recommendation - Control access to services using group "gnunet" | ||
4072 | @subsubsection Recommendation - Control access to services using group "gnunet" | ||
4073 | |||
4074 | Users that should be allowed to use the GNUnet peer should be added to the | ||
4075 | group "gnunet". Using GNUnet's access control mechanism for UNIX domain | ||
4076 | sockets, those services that are considered useful to ordinary users | ||
4077 | should be made available by setting "UNIX_MATCH_GID=YES" for those | ||
4078 | services. | ||
4079 | Again, as shipped, GNUnet provides reasonable defaults. | ||
4080 | Permissions to access the transport and core subsystems might additionally | ||
4081 | be granted without necessarily causing security concerns. | ||
4082 | Some services, such as DNS, must NOT be made accessible to the "gnunet" | ||
4083 | group (and should thus only be accessible to the "gnunet" user and | ||
4084 | services running with this UID). | ||
4085 | |||
4086 | @node Recommendation - Limit access to certain SUID binaries by group "gnunet" | ||
4087 | @subsubsection Recommendation - Limit access to certain SUID binaries by group "gnunet" | ||
4088 | |||
4089 | Most of GNUnet's SUID binaries should be safe even if executed by normal | ||
4090 | users. However, it is possible to reduce the risk a little bit more by | ||
4091 | making these binaries owned by the group "gnunet" and restricting their | ||
4092 | execution to user of the group "gnunet" as well (4750). | ||
4093 | |||
4094 | @node Recommendation - Limit access to critical gnunet-helper-dns to group "gnunetdns" | ||
4095 | @subsubsection Recommendation - Limit access to critical gnunet-helper-dns to group "gnunetdns" | ||
4096 | |||
4097 | A special group "gnunetdns" should be created for controlling access to | ||
4098 | the "gnunet-helper-dns". | ||
4099 | The binary should then be owned by root and be in group "gnunetdns" and | ||
4100 | be installed SUID and only be group-executable (2750). | ||
4101 | @b{Note that the group "gnunetdns" should have no users in it at all, | ||
4102 | ever.} | ||
4103 | The "gnunet-service-dns" program should be executed by user "gnunet" (via | ||
4104 | gnunet-service-arm) with the binary owned by the user "root" and the group | ||
4105 | "gnunetdns" and be SGID (2700). This way, @strong{only} | ||
4106 | "gnunet-service-dns" can change its group to "gnunetdns" and execute the | ||
4107 | helper, and the helper can then run as root (as per SUID). | ||
4108 | Access to the API offered by "gnunet-service-dns" is in turn restricted | ||
4109 | to the user "gnunet" (not the group!), which means that only | ||
4110 | "benign" services can manipulate DNS queries using "gnunet-service-dns". | ||
4111 | |||
4112 | @node Differences between "make install" and these recommendations | ||
4113 | @subsubsection Differences between "make install" and these recommendations | ||
4114 | |||
4115 | The current build system does not set all permissions automatically based | ||
4116 | on the recommendations above. In particular, it does not use the group | ||
4117 | "gnunet" at all (so setting gnunet-helpers other than the | ||
4118 | gnunet-helper-dns to be owned by group "gnunet" must be done manually). | ||
4119 | Furthermore, 'make install' will silently fail to set the DNS binaries to | ||
4120 | be owned by group "gnunetdns" unless that group already exists (!). | ||
4121 | An alternative name for the "gnunetdns" group can be specified using the | ||
4122 | @code{--with-gnunetdns=GRPNAME} configure option. | ||
4123 | |||
diff --git a/doc/documentation/chapters/philosophy.texi b/doc/documentation/chapters/philosophy.texi new file mode 100644 index 000000000..e77415bd8 --- /dev/null +++ b/doc/documentation/chapters/philosophy.texi | |||
@@ -0,0 +1,427 @@ | |||
1 | @cindex Philosophy | ||
2 | @node Philosophy | ||
3 | @chapter Philosophy | ||
4 | |||
5 | @c NOTE: We should probably re-use some of the images lynX created | ||
6 | @c for secushare, showing some of the relations and functionalities | ||
7 | @c of GNUnet. | ||
8 | The foremost goal of the GNUnet project is to become a widely used, | ||
9 | reliable, open, non-discriminating, egalitarian, unconstrained and | ||
10 | censorship-resistant system of free information exchange. | ||
11 | We value free speech above state secrets, law-enforcement or | ||
12 | intellectual property. | ||
13 | GNUnet is supposed to be an anarchistic network, where the only | ||
14 | limitation for participants (devices or people making use of the | ||
15 | network, in the following sometimes called peers) is | ||
16 | that they must contribute enough back to the network such that | ||
17 | their resource consumption does not have a significant impact | ||
18 | on other users. | ||
19 | GNUnet should be more than just another file-sharing network. | ||
20 | The plan is to offer many other services and in particular | ||
21 | to serve as a development platform for the next generation of | ||
22 | Internet Protocols. | ||
23 | |||
24 | @menu | ||
25 | * Design Goals:: | ||
26 | * Security and Privacy:: | ||
27 | * Versatility:: | ||
28 | * Practicality:: | ||
29 | * Key Concepts:: | ||
30 | @end menu | ||
31 | |||
32 | @cindex Design Goals | ||
33 | @cindex Design Goals | ||
34 | @node Design Goals | ||
35 | @section Design Goals | ||
36 | |||
37 | These are the core GNUnet design goals, in order of relative importance: | ||
38 | |||
39 | @itemize | ||
40 | @item GNUnet must be implemented as | ||
41 | @uref{https://www.gnu.org/philosophy/free-sw.html, Free Software} | ||
42 | @c To footnote or not to footnote, that's the question. | ||
43 | @footnote{This means that you you have the four essential freedoms: to run | ||
44 | the program, to study and change the program in source code form, | ||
45 | to redistribute exact copies, and to distribute modified versions. | ||
46 | Refer to @uref{https://www.gnu.org/philosophy/free-sw.html, https://www.gnu.org/philosophy/free-sw.html}} | ||
47 | @item GNUnet must only disclose the minimal amount of information | ||
48 | necessary. | ||
49 | @c TODO: Explain 'fully' in the terminology section. | ||
50 | @item GNUnet must be fully distributed and survive Byzantine failures | ||
51 | at any position in the network. | ||
52 | @item GNUnet must make it explicit to the user which entities are | ||
53 | considered to be trustworthy when establishing secured communications. | ||
54 | @item GNUnet must use compartmentalization to protect sensitive | ||
55 | information. | ||
56 | @item GNUnet must be open and permit new peers to join. | ||
57 | @item GNUnet must be self-organizing and not depend on administrators. | ||
58 | @item GNUnet must support a diverse range of applications and devices. | ||
59 | @item The GNUnet architecture must be cost effective. | ||
60 | @item GNUnet must provide incentives for peers to contribute more | ||
61 | resources than they consume. | ||
62 | @end itemize | ||
63 | |||
64 | |||
65 | @cindex Security and Privacy | ||
66 | @node Security and Privacy | ||
67 | @section Security and Privacy | ||
68 | |||
69 | GNUnet's primary design goals are to protect the privacy of its users and | ||
70 | to guard itself against attacks or abuse. | ||
71 | GNUnet does not have any mechanisms to control, track or censor users. | ||
72 | Instead, the GNUnet protocols aim to make it as hard as possible to | ||
73 | find out what is happening on the network or to disrupt operations. | ||
74 | |||
75 | @cindex Versatility | ||
76 | @node Versatility | ||
77 | @section Versatility | ||
78 | |||
79 | We call GNUnet a peer-to-peer framework because we want to support many | ||
80 | different forms of peer-to-peer applications. GNUnet uses a plugin | ||
81 | architecture to make the system extensible and to encourage code reuse. | ||
82 | While the first versions of the system only supported anonymous | ||
83 | file-sharing, other applications are being worked on and more will | ||
84 | hopefully follow in the future. | ||
85 | A powerful synergy regarding anonymity services is created by a large | ||
86 | community utilizing many diverse applications over the same software | ||
87 | infrastructure. The reason is that link encryption hides the specifics | ||
88 | of the traffic for non-participating observers. This way, anonymity can | ||
89 | get stronger with additional (GNUnet) traffic, even if the additional | ||
90 | traffic is not related to anonymous communication. Increasing anonymity | ||
91 | is the primary reason why GNUnet is developed to become a peer-to-peer | ||
92 | framework where many applications share the lower layers of an | ||
93 | increasingly complex protocol stack. | ||
94 | If merging traffic to hinder traffic analysis was not important, | ||
95 | we could have just developed a dozen stand-alone applications | ||
96 | and a few shared libraries. | ||
97 | |||
98 | @cindex Practicality | ||
99 | @node Practicality | ||
100 | @section Practicality | ||
101 | |||
102 | GNUnet allows participants to trade various amounts of security in | ||
103 | exchange for increased efficiency. However, it is not possible for any | ||
104 | user's security and efficiency requirements to compromise the security | ||
105 | and efficiency of any other user. | ||
106 | |||
107 | For GNUnet, efficiency is not paramount. If there were a more secure and | ||
108 | still practical approach, we would choose to take the more secure | ||
109 | alternative. @command{telnet} is more efficient than @command{ssh}, yet | ||
110 | it is obsolete. | ||
111 | Hardware gets faster, and code can be optimized. Fixing security issues | ||
112 | as an afterthought is much harder. | ||
113 | |||
114 | While security is paramount, practicability is still a requirement. | ||
115 | The most secure system is always the one that nobody can use. | ||
116 | Similarly, any anonymous system that is extremely inefficient will only | ||
117 | find few users. | ||
118 | However, good anonymity requires a large and diverse user base. Since | ||
119 | individual security requirements may vary, the only good solution here is | ||
120 | to allow individuals to trade-off security and efficiency. | ||
121 | The primary challenge in allowing this is to ensure that the economic | ||
122 | incentives work properly. | ||
123 | In particular, this means that it must be impossible for a user to gain | ||
124 | security at the expense of other users. Many designs (e.g. anonymity via | ||
125 | broadcast) fail to give users an incentive to choose a less secure but | ||
126 | more efficient mode of operation. | ||
127 | GNUnet should avoid where ever possible to rely on protocols that will | ||
128 | only work if the participants are benevolent. | ||
129 | While some designs have had widespread success while relying on parties | ||
130 | to observe a protocol that may be sub-optimal for the individuals (e.g. | ||
131 | TCP Nagle), a protocol that ensures that individual goals never conflict | ||
132 | with the goals of the group is always preferable. | ||
133 | |||
134 | @cindex Key Concepts | ||
135 | @node Key Concepts | ||
136 | @section Key Concepts | ||
137 | |||
138 | In this section, the fundamental concepts of GNUnet are explained. | ||
139 | @c FIXME: Use @uref{https://docs.gnunet.org/bib/, research papers} | ||
140 | @c once we have the new bibliography + subdomain setup. | ||
141 | Most of them are also described in our research papers. | ||
142 | First, some of the concepts used in the GNUnet framework are detailed. | ||
143 | The second part describes concepts specific to anonymous file-sharing. | ||
144 | |||
145 | @menu | ||
146 | * Authentication:: | ||
147 | * Accounting to Encourage Resource Sharing:: | ||
148 | * Confidentiality:: | ||
149 | * Anonymity:: | ||
150 | * Deniability:: | ||
151 | * Peer Identities:: | ||
152 | * Zones in the GNU Name System (GNS Zones):: | ||
153 | * Egos:: | ||
154 | @end menu | ||
155 | |||
156 | @cindex Authentication | ||
157 | @node Authentication | ||
158 | @subsection Authentication | ||
159 | |||
160 | Almost all peer-to-peer communications in GNUnet are between mutually | ||
161 | authenticated peers. The authentication works by using ECDHE, that is a | ||
162 | DH (Diffie---Hellman) key exchange using ephemeral eliptic curve | ||
163 | cryptography. The ephemeral ECC (Eliptic Curve Cryptography) keys are | ||
164 | signed using ECDSA (@uref{http://en.wikipedia.org/wiki/ECDSA, ECDSA}). | ||
165 | The shared secret from ECDHE is used to create a pair of session keys | ||
166 | @c FIXME: LOng word for HKDF | ||
167 | (using HKDF) which are then used to encrypt the communication between the | ||
168 | two peers using both 256-bit AES (Advanced Encryption Standard) | ||
169 | and 256-bit Twofish (with independently derived secret keys). | ||
170 | As only the two participating hosts know the shared secret, this | ||
171 | authenticates each packet | ||
172 | without requiring signatures each time. GNUnet uses SHA-512 | ||
173 | (Secure Hash Algorithm) hash codes to verify the integrity of messages. | ||
174 | |||
175 | In GNUnet, the identity of a host is its public key. For that reason, | ||
176 | @c FIXME: is it clear to the average reader what a man-in-the-middle | ||
177 | @c attack is? | ||
178 | man-in-the-middle attacks will not break the authentication or accounting | ||
179 | goals. Essentially, for GNUnet, the IP of the host has nothing to do with | ||
180 | the identity of the host. As the public key is the only thing that truly | ||
181 | matters, faking an IP, a port or any other property of the underlying | ||
182 | transport protocol is irrelevant. In fact, GNUnet peers can use | ||
183 | multiple IPs (IPv4 and IPv6) on multiple ports --- or even not use the | ||
184 | IP protocol at all (by running directly on layer 2). | ||
185 | |||
186 | @c NOTE: For consistency we will use @code{HELLO}s throughout this Manual. | ||
187 | GNUnet uses a special type of message to communicate a binding between | ||
188 | public (ECC) keys to their current network address. These messages are | ||
189 | commonly called @code{HELLO}s or peer advertisements. | ||
190 | They contain the public key of the peer and its current network | ||
191 | addresses for various transport services. | ||
192 | A transport service is a special kind of shared library that | ||
193 | provides (possibly unreliable, out-of-order) message delivery between | ||
194 | peers. | ||
195 | For the UDP and TCP transport services, a network address is an IP and a | ||
196 | port. | ||
197 | GNUnet can also use other transports (HTTP, HTTPS, WLAN, etc.) which use | ||
198 | various other forms of addresses. Note that any node can have many | ||
199 | different active transport services at the same time, | ||
200 | and each of these can have a different addresses. | ||
201 | Binding messages expire after at most a week (the timeout can be | ||
202 | shorter if the user configures the node appropriately). | ||
203 | This expiration ensures that the network will eventually get rid of | ||
204 | outdated advertisements. | ||
205 | @footnote{Ronaldo A. Ferreira, Christian Grothoff, and Paul Ruth. | ||
206 | A Transport Layer Abstraction for Peer-to-Peer Networks | ||
207 | Proceedings of the 3rd International Symposium on Cluster Computing | ||
208 | and the Grid (GRID 2003), 2003. | ||
209 | (@uref{https://gnunet.org/git/bibliography.git/plain/docs/transport.pdf, pdf})} | ||
210 | |||
211 | @cindex Accounting to Encourage Resource Sharing | ||
212 | @node Accounting to Encourage Resource Sharing | ||
213 | @subsection Accounting to Encourage Resource Sharing | ||
214 | |||
215 | Most distributed P2P networks suffer from a lack of defenses or | ||
216 | precautions against attacks in the form of freeloading. | ||
217 | While the intentions of an attacker and a freeloader are different, their | ||
218 | effect on the network is the same; they both render it useless. | ||
219 | Most simple attacks on networks such as @command{Gnutella} | ||
220 | involve flooding the network with traffic, particularly | ||
221 | with queries that are, in the worst case, multiplied by the network. | ||
222 | |||
223 | In order to ensure that freeloaders or attackers have a minimal impact on | ||
224 | the network, GNUnet's file-sharing implementation tries to distinguish | ||
225 | good (contributing) nodes from malicious (freeloading) nodes. In GNUnet, | ||
226 | every file-sharing node keeps track of the behavior of every other node it | ||
227 | has been in contact with. Many requests (depending on the application) | ||
228 | are transmitted with a priority (or importance) level. | ||
229 | That priority is used to establish how important the sender believes | ||
230 | this request is. If a peer responds to an important request, the | ||
231 | recipient will increase its trust in the responder: | ||
232 | the responder contributed resources. | ||
233 | If a peer is too busy to answer all requests, it needs to prioritize. | ||
234 | For that, peers do not take the priorities of the requests received at | ||
235 | face value. | ||
236 | First, they check how much they trust the sender, and depending on that | ||
237 | amount of trust they assign the request a (possibly lower) effective | ||
238 | priority. Then, they drop the requests with the lowest effective priority | ||
239 | to satisfy their resource constraints. This way, GNUnet's economic model | ||
240 | ensures that nodes that are not currently considered to have a surplus in | ||
241 | contributions will not be served if the network load is high. | ||
242 | @footnote{Christian Grothoff. An Excess-Based Economic Model for Resource | ||
243 | Allocation in Peer-to-Peer Networks. Wirtschaftsinformatik, June 2003. | ||
244 | (@uref{https://gnunet.org/git/bibliography.git/plain/docs/ebe.pdf, pdf})} | ||
245 | @c 2009? | ||
246 | |||
247 | @cindex Confidentiality | ||
248 | @node Confidentiality | ||
249 | @subsection Confidentiality | ||
250 | |||
251 | Adversaries outside of GNUnet are not supposed to know what kind of | ||
252 | actions a peer is involved in. Only the specific neighbor of a peer that | ||
253 | is the corresponding sender or recipient of a message may know its | ||
254 | contents, and even then application protocols may place further | ||
255 | restrictions on that knowledge. | ||
256 | In order to ensure confidentiality, GNUnet uses link encryption, that is | ||
257 | each message exchanged between two peers is encrypted using a pair of | ||
258 | keys only known to these two peers. | ||
259 | Encrypting traffic like this makes any kind of traffic analysis much | ||
260 | harder. Naturally, for some applications, it may still be desirable if | ||
261 | even neighbors cannot determine the concrete contents of a message. | ||
262 | In GNUnet, this problem is addressed by the specific application-level | ||
263 | protocols (see for example, deniability and anonymity in anonymous file | ||
264 | sharing). | ||
265 | |||
266 | @cindex Anonymity | ||
267 | @node Anonymity | ||
268 | @subsection Anonymity | ||
269 | |||
270 | @menu | ||
271 | * How file-sharing achieves Anonymity:: | ||
272 | @end menu | ||
273 | |||
274 | Providing anonymity for users is the central goal for the anonymous | ||
275 | file-sharing application. Many other design decisions follow in the | ||
276 | footsteps of this requirement. | ||
277 | Anonymity is never absolute. While there are various | ||
278 | scientific metrics@footnote{Claudia DÃaz, Stefaan Seys, Joris Claessens, | ||
279 | and Bart Preneel. Towards measuring anonymity. | ||
280 | 2002. | ||
281 | (@uref{https://gnunet.org/git/bibliography.git/plain/docs/article-89.pdf, pdf})} | ||
282 | that can help quantify the level of anonymity that a given mechanism | ||
283 | provides, there is no such thing as complete anonymity. | ||
284 | GNUnet's file-sharing implementation allows users to select for each | ||
285 | operation (publish, search, download) the desired level of anonymity. | ||
286 | The metric used is the amount of cover traffic available to hide the | ||
287 | request. | ||
288 | While this metric is not as good as, for example, the theoretical metric | ||
289 | given in scientific metrics@footnote{likewise}, | ||
290 | it is probably the best metric available to a peer with a purely local | ||
291 | view of the world that does not rely on unreliable external information. | ||
292 | The default anonymity level is 1, which uses anonymous routing but | ||
293 | imposes no minimal requirements on cover traffic. It is possible | ||
294 | to forego anonymity when this is not required. The anonymity level of 0 | ||
295 | allows GNUnet to use more efficient, non-anonymous routing. | ||
296 | |||
297 | @cindex How file-sharing achieves Anonymity | ||
298 | @node How file-sharing achieves Anonymity | ||
299 | @subsubsection How file-sharing achieves Anonymity | ||
300 | |||
301 | Contrary to other designs, we do not believe that users achieve strong | ||
302 | anonymity just because their requests are obfuscated by a couple of | ||
303 | indirections. This is not sufficient if the adversary uses traffic | ||
304 | analysis. | ||
305 | The threat model used for anonymous file sharing in GNUnet assumes that | ||
306 | the adversary is quite powerful. | ||
307 | In particular, we assume that the adversary can see all the traffic on | ||
308 | the Internet. And while we assume that the adversary | ||
309 | can not break our encryption, we assume that the adversary has many | ||
310 | participating nodes in the network and that it can thus see many of the | ||
311 | node-to-node interactions since it controls some of the nodes. | ||
312 | |||
313 | The system tries to achieve anonymity based on the idea that users can be | ||
314 | anonymous if they can hide their actions in the traffic created by other | ||
315 | users. | ||
316 | Hiding actions in the traffic of other users requires participating in the | ||
317 | traffic, bringing back the traditional technique of using indirection and | ||
318 | source rewriting. Source rewriting is required to gain anonymity since | ||
319 | otherwise an adversary could tell if a message originated from a host by | ||
320 | looking at the source address. If all packets look like they originate | ||
321 | from one node, the adversary can not tell which ones originate from that | ||
322 | node and which ones were routed. | ||
323 | Note that in this mindset, any node can decide to break the | ||
324 | source-rewriting paradigm without violating the protocol, as this | ||
325 | only reduces the amount of traffic that a node can hide its own traffic | ||
326 | in. | ||
327 | |||
328 | If we want to hide our actions in the traffic of other nodes, we must make | ||
329 | our traffic indistinguishable from the traffic that we route for others. | ||
330 | As our queries must have us as the receiver of the reply | ||
331 | (otherwise they would be useless), we must put ourselves as the receiver | ||
332 | of replies that actually go to other hosts; in other words, we must | ||
333 | indirect replies. | ||
334 | Unlike other systems, in anonymous file-sharing as implemented on top of | ||
335 | GNUnet we do not have to indirect the replies if we don't think we need | ||
336 | more traffic to hide our own actions. | ||
337 | |||
338 | This increases the efficiency of the network as we can indirect less under | ||
339 | higher load.@footnote{Krista Bennett and Christian Grothoff. | ||
340 | GAP --- practical anonymous networking. In Proceedings of | ||
341 | Designing Privacy Enhancing Technologies, 2003. | ||
342 | (@uref{https://gnunet.org/git/bibliography.git/plain/docs/aff.pdf, https://gnunet.org/git/bibliography.git/plain/docs/aff.pdf})} | ||
343 | |||
344 | @cindex Deniability | ||
345 | @node Deniability | ||
346 | @subsection Deniability | ||
347 | |||
348 | Even if the user that downloads data and the server that provides data are | ||
349 | anonymous, the intermediaries may still be targets. In particular, if the | ||
350 | intermediaries can find out which queries or which content they are | ||
351 | processing, a strong adversary could try to force them to censor | ||
352 | certain materials. | ||
353 | |||
354 | With the file-encoding used by GNUnet's anonymous file-sharing, this | ||
355 | problem does not arise. | ||
356 | The reason is that queries and replies are transmitted in | ||
357 | an encrypted format such that intermediaries cannot tell what the query | ||
358 | is for or what the content is about. Mind that this is not the same | ||
359 | encryption as the link-encryption between the nodes. GNUnet has | ||
360 | encryption on the network layer (link encryption, confidentiality, | ||
361 | authentication) and again on the application layer (provided | ||
362 | by @command{gnunet-publish}, @command{gnunet-download}, | ||
363 | @command{gnunet-search} and @command{gnunet-gtk}). | ||
364 | @footnote{Christian Grothoff, Krista Grothoff, Tzvetan Horozov, | ||
365 | and Jussi T. Lindgren. | ||
366 | An Encoding for Censorship-Resistant Sharing. | ||
367 | 2009. | ||
368 | (@uref{https://gnunet.org/git/bibliography.git/plain/docs/ecrs.pdf, pdf})} | ||
369 | |||
370 | @cindex Peer Identities | ||
371 | @node Peer Identities | ||
372 | @subsection Peer Identities | ||
373 | |||
374 | Peer identities are used to identify peers in the network and are unique | ||
375 | for each peer. The identity for a peer is simply its public key, which is | ||
376 | generated along with a private key the peer is started for the first time. | ||
377 | While the identity is binary data, it is often expressed as ASCII string. | ||
378 | For example, the following is a peer identity as you might see it in | ||
379 | various places: | ||
380 | |||
381 | @example | ||
382 | UAT1S6PMPITLBKSJ2DGV341JI6KF7B66AC4JVCN9811NNEGQLUN0 | ||
383 | @end example | ||
384 | |||
385 | @noindent | ||
386 | You can find your peer identity by running @command{gnunet-peerinfo -s}. | ||
387 | |||
388 | @cindex Zones in the GNU Name System (GNS Zones) | ||
389 | @node Zones in the GNU Name System (GNS Zones) | ||
390 | @subsection Zones in the GNU Name System (GNS Zones) | ||
391 | |||
392 | @c FIXME: Explain or link to an explanation of the concept of public keys | ||
393 | @c and private keys. | ||
394 | GNS@footnote{Matthias Wachs, Martin Schanzenbach, and Christian Grothoff. | ||
395 | A Censorship-Resistant, Privacy-Enhancing and Fully Decentralized Name | ||
396 | System. In proceedings of 13th International Conference on Cryptology and | ||
397 | Network Security (CANS 2014). 2014. | ||
398 | @uref{https://gnunet.org/git/bibliography.git/plain/docs/gns2014wachs.pdf, https://gnunet.org/git/bibliography.git/plain/docs/gns2014wachs.pdf}} | ||
399 | zones are similar to those of DNS zones, but instead of a hierarchy of | ||
400 | authorities to governing their use, GNS zones are controlled by a private | ||
401 | key. | ||
402 | When you create a record in a DNS zone, that information stored in your | ||
403 | nameserver. Anyone trying to resolve your domain then gets pointed | ||
404 | (hopefully) by the centralised authority to your nameserver. | ||
405 | Whereas GNS, being fully decentralized by design, stores that information | ||
406 | in DHT. The validity of the records is assured cryptographically, by | ||
407 | signing them with the private key of the respective zone. | ||
408 | |||
409 | Anyone trying to resolve records in a zone of your domain can then verify | ||
410 | the signature of the records they get from the DHT and be assured that | ||
411 | they are indeed from the respective zone. | ||
412 | To make this work, there is a 1:1 correspondence between zones and | ||
413 | their public-private key pairs. | ||
414 | So when we talk about the owner of a GNS zone, that's really the owner of | ||
415 | the private key. | ||
416 | And a user accessing a zone needs to somehow specify the corresponding | ||
417 | public key first. | ||
418 | |||
419 | @cindex Egos | ||
420 | @node Egos | ||
421 | @subsection Egos | ||
422 | |||
423 | Egos are your "identities" in GNUnet. Any user can assume multiple | ||
424 | identities, for example to separate their activities online. Egos can | ||
425 | correspond to pseudonyms or real-world identities. Technically, an | ||
426 | ego is first of all a public-private key pair. | ||
427 | |||
diff --git a/doc/documentation/chapters/user.texi b/doc/documentation/chapters/user.texi new file mode 100644 index 000000000..4159a6b32 --- /dev/null +++ b/doc/documentation/chapters/user.texi | |||
@@ -0,0 +1,2032 @@ | |||
1 | @node Using GNUnet | ||
2 | @chapter Using GNUnet | ||
3 | @c %**end of header | ||
4 | |||
5 | This tutorial is supposed to give a first introduction for end-users | ||
6 | trying to do something "real" with GNUnet. Installation and | ||
7 | configuration are specifically outside of the scope of this tutorial. | ||
8 | Instead, we start by briefly checking that the installation works, and | ||
9 | then dive into simple, concrete practical things that can be done | ||
10 | with the network. | ||
11 | |||
12 | This chapter of the GNUnet Reference Documentation documents | ||
13 | how to use the various peer-to-peer applications of the | ||
14 | GNUnet system. | ||
15 | As GNUnet evolves, we will add new chapters for the various | ||
16 | applications that are being created. | ||
17 | |||
18 | Comments and extensions of this documentation are always welcome. | ||
19 | |||
20 | |||
21 | @menu | ||
22 | * Checking the Installation:: | ||
23 | * First steps - File-sharing:: | ||
24 | * First steps - Using the GNU Name System:: | ||
25 | * First steps - Using GNUnet Conversation:: | ||
26 | * First steps - Using the GNUnet VPN:: | ||
27 | * File-sharing:: | ||
28 | * The GNU Name System:: | ||
29 | * Using the Virtual Public Network:: | ||
30 | @end menu | ||
31 | |||
32 | @node Checking the Installation | ||
33 | @section Checking the Installation | ||
34 | @c %**end of header | ||
35 | |||
36 | This section describes a quick casual way to check if your GNUnet | ||
37 | installation works. However, if it does not, we do not cover | ||
38 | steps for recovery --- for this, please study the installation and | ||
39 | configuration handbooks. | ||
40 | |||
41 | |||
42 | @menu | ||
43 | * gnunet-gtk:: | ||
44 | * Statistics:: | ||
45 | * Peer Information:: | ||
46 | @end menu | ||
47 | |||
48 | @node gnunet-gtk | ||
49 | @subsection gnunet-gtk | ||
50 | @c %**end of header | ||
51 | |||
52 | First, you should launch @command{gnunet-gtk}, the graphical user | ||
53 | interface for GNUnet which will be used for most of the tutorial. | ||
54 | You can do this from the command-line by typing | ||
55 | |||
56 | @example | ||
57 | $ gnunet-gtk | ||
58 | @end example | ||
59 | |||
60 | (note that @code{$} represents the prompt of the shell for a normal user). | ||
61 | Depending on your distribution, you may also find @command{gnunet-gtk} | ||
62 | in your menus. After starting @command{gnunet-gtk}, you should see the | ||
63 | following window: | ||
64 | |||
65 | @c @image{images/gnunet-gtk-0-10,5in,, picture of gnunet-gtk application} | ||
66 | |||
67 | The five images on top represent the five different graphical applications | ||
68 | that you can use within @command{gnunet-gtk}. | ||
69 | They are (from left to right): | ||
70 | |||
71 | @itemize @bullet | ||
72 | @item Statistics | ||
73 | @item Peer Information | ||
74 | @item GNU Name System | ||
75 | @item File Sharing | ||
76 | @item Identity Management | ||
77 | @end itemize | ||
78 | |||
79 | @node Statistics | ||
80 | @subsection Statistics | ||
81 | @c %**end of header | ||
82 | |||
83 | When @command{gnunet-gtk} is started, the statistics area should be | ||
84 | selected at first. | ||
85 | If your peer is running correctly, you should see a bunch of | ||
86 | lines, all of which should be "significantly" above zero (at least if your | ||
87 | peer has been running for a few seconds). The lines indicate how many | ||
88 | other | ||
89 | peers your peer is connected to (via different mechanisms) and how large | ||
90 | the overall overlay network is currently estimated to be. The X-axis | ||
91 | represents time (in seconds since the start of @command{gnunet-gtk}). | ||
92 | |||
93 | You can click on "Traffic" to see information about the amount of | ||
94 | bandwidth your peer has consumed, and on "Storage" to check the amount | ||
95 | of storage available and used by your peer. Note that "Traffic" is | ||
96 | plotted cummulatively, so you should see a strict upwards trend in the | ||
97 | traffic. | ||
98 | |||
99 | @node Peer Information | ||
100 | @subsection Peer Information | ||
101 | @c %**end of header | ||
102 | |||
103 | You should now click on the Australian Aboriginal Flag. Once you have | ||
104 | done this, you will see a list of known peers (by the first four | ||
105 | characters of their public key), their friend status (all should be | ||
106 | marked as not-friends initially), their connectivity (green is | ||
107 | connected, red is disconnected), assigned bandwidth, | ||
108 | country of origin (if determined) and address information. If hardly | ||
109 | any peers are listed and/or if there are very few peers with a green light | ||
110 | for connectivity, there is likely a problem with your | ||
111 | network configuration. | ||
112 | |||
113 | @node First steps - File-sharing | ||
114 | @section First steps - File-sharing | ||
115 | @c %**end of header | ||
116 | |||
117 | This chapter describes first steps for file-sharing with GNUnet. | ||
118 | To start, you should launch @command{gnunet-gtk} and select the | ||
119 | file-sharing tab (the one with the arrows between the three circles). | ||
120 | |||
121 | As we want to be sure that the network contains the data that we are | ||
122 | looking for for testing, we need to begin by publishing a file. | ||
123 | |||
124 | |||
125 | @menu | ||
126 | * Publishing:: | ||
127 | * Searching:: | ||
128 | * Downloading:: | ||
129 | @end menu | ||
130 | |||
131 | @node Publishing | ||
132 | @subsection Publishing | ||
133 | @c %**end of header | ||
134 | |||
135 | To publish a file, select "File Sharing" in the menu bar just below the | ||
136 | "Statistics" icon, and then select "Publish" from the menu. | ||
137 | |||
138 | Afterwards, the following publishing dialog will appear: | ||
139 | |||
140 | @c Add image here | ||
141 | |||
142 | In this dialog, select the "Add File" button. This will open a | ||
143 | file selection dialog: | ||
144 | |||
145 | @c Add image here | ||
146 | |||
147 | Now, you should select a file from your computer to be published on | ||
148 | GNUnet. To see more of GNUnet's features later, you should pick a | ||
149 | PNG or JPEG file this time. You can leave all of the other options | ||
150 | in the dialog unchanged. Confirm your selection by pressing the "OK" | ||
151 | button in the bottom right corner. Now, you will briefly see a | ||
152 | "Messages..." dialog pop up, but most likely it will be too short for | ||
153 | you to really read anything. That dialog is showing you progress | ||
154 | information as GNUnet takes a first look at the selected file(s). | ||
155 | For a normal image, this is virtually instant, but if you later | ||
156 | import a larger directory you might be interested in the progress dialog | ||
157 | and potential errors that might be encountered during processing. | ||
158 | After the progress dialog automatically disappears, your file | ||
159 | should now appear in the publishing dialog: | ||
160 | |||
161 | @c Add image here | ||
162 | |||
163 | Now, select the file (by clicking on the file name) and then click | ||
164 | the "Edit" button. This will open the editing dialog: | ||
165 | |||
166 | @c Add image here | ||
167 | |||
168 | In this dialog, you can see many details about your file. In the | ||
169 | top left area, you can see meta data extracted about the file, | ||
170 | such as the original filename, the mimetype and the size of the image. | ||
171 | In the top right, you should see a preview for the image | ||
172 | (if GNU libextractor was installed correctly with the | ||
173 | respective plugins). Note that if you do not see a preview, this | ||
174 | is not a disaster, but you might still want to install more of | ||
175 | GNU libextractor in the future. In the bottom left, the dialog contains | ||
176 | a list of keywords. These are the keywords under which the file will be | ||
177 | made available. The initial list will be based on the extracted meta data. | ||
178 | Additional publishing options are in the right bottom corner. We will | ||
179 | now add an additional keyword to the list of keywords. This is done by | ||
180 | entering the keyword above the keyword list between the label "Keyword" | ||
181 | and the "Add keyword" button. Enter "test" and select "Add keyword". | ||
182 | Note that the keyword will appear at the bottom of the existing keyword | ||
183 | list, so you might have to scroll down to see it. Afterwards, push the | ||
184 | "OK" button at the bottom right of the dialog. | ||
185 | |||
186 | You should now be back at the "Publish content on GNUnet" dialog. Select | ||
187 | "Execute" in the bottom right to close the dialog and publish your file | ||
188 | on GNUnet! Afterwards, you should see the main dialog with a new area | ||
189 | showing the list of published files (or ongoing publishing operations | ||
190 | with progress indicators): | ||
191 | |||
192 | @c Add image here | ||
193 | |||
194 | @node Searching | ||
195 | @subsection Searching | ||
196 | @c %**end of header | ||
197 | |||
198 | Below the menu bar, there are four entry widges labeled "Namespace", | ||
199 | "Keywords", "Anonymity" and "Mime-type" (from left to right). These | ||
200 | widgets are used to control searching for files in GNUnet. Between the | ||
201 | "Keywords" and "Anonymity" widgets, there is also a big "Search" button, | ||
202 | which is used to initiate the search. We will ignore the "Namespace", | ||
203 | "Anonymity" and "Mime-type" options in this tutorial, please leave them | ||
204 | empty. Instead, simply enter "test" under "Keywords" and press "Search". | ||
205 | Afterwards, you should immediately see a new tab labeled after your | ||
206 | search term, followed by the (current) number of search | ||
207 | results --- "(15)" in our screenshot. Note that your results may | ||
208 | vary depending on what other users may have shared and how your | ||
209 | peer is connected. | ||
210 | |||
211 | You can now select one of the search results. Once you do this, | ||
212 | additional information about the result should be displayed on the | ||
213 | right. If available, a preview image should appear on the top right. | ||
214 | Meta data describing the file will be listed at the bottom right. | ||
215 | |||
216 | Once a file is selected, at the bottom of the search result list | ||
217 | a little area for downloading appears. | ||
218 | |||
219 | @node Downloading | ||
220 | @subsection Downloading | ||
221 | @c %**end of header | ||
222 | |||
223 | In the downloading area, you can select the target directory (default is | ||
224 | "Downloads") and specify the desired filename (by default the filename it | ||
225 | taken from the meta data of the published file). Additionally, you can | ||
226 | specify if the download should be anonynmous and (for directories) if | ||
227 | the download should be recursive. In most cases, you can simply start | ||
228 | the download with the "Download!" button. | ||
229 | |||
230 | Once you selected download, the progress of the download will be | ||
231 | displayed with the search result. You may need to resize the result | ||
232 | list or scroll to the right. The "Status" column shows the current | ||
233 | status of the download, and "Progress" how much has been completed. | ||
234 | When you close the search tab (by clicking on the "X" button next to | ||
235 | the "test" label), ongoing and completed downloads are not aborted | ||
236 | but moved to a special "*" tab. | ||
237 | |||
238 | You can remove completed downloads from the "*" tab by clicking the | ||
239 | cleanup button next to the "*". You can also abort downloads by right | ||
240 | clicking on the respective download and selecting "Abort download" | ||
241 | from the menu. | ||
242 | |||
243 | That's it, you now know the basics for file-sharing with GNUnet! | ||
244 | |||
245 | @node First steps - Using the GNU Name System | ||
246 | @section First steps - Using the GNU Name System | ||
247 | @c %**end of header | ||
248 | |||
249 | |||
250 | |||
251 | @menu | ||
252 | * Preliminaries:: | ||
253 | * Managing Egos:: | ||
254 | * The GNS Tab:: | ||
255 | * Creating a Record:: | ||
256 | * Creating a Business Card:: | ||
257 | * Resolving GNS records:: | ||
258 | * Integration with Browsers:: | ||
259 | * Be Social:: | ||
260 | * Backup of Identities and Egos:: | ||
261 | * Revocation:: | ||
262 | * What's Next?:: | ||
263 | @end menu | ||
264 | |||
265 | @node Preliminaries | ||
266 | @subsection Preliminaries | ||
267 | @c %**end of header | ||
268 | |||
269 | First, we will check if the GNU Name System installation was | ||
270 | completed normally. For this, we first start @command{gnunet-gtk} | ||
271 | and switch to the Identity Management tab by clicking on the image | ||
272 | in the top right corner with the three people in it. Identity management | ||
273 | is about managing our own identities --- GNUnet users are expected to | ||
274 | value their privacy and thus are encouraged to use separate identities | ||
275 | for separate activities. By default, each user should have | ||
276 | run @file{gnunet-gns-import.sh} during installation. This script creates | ||
277 | four identities, which should show up in the identity management tab: | ||
278 | |||
279 | @c insert image. | ||
280 | |||
281 | For this tutorial, we will pretty much only be concerned with the | ||
282 | "master-zone" identity, which as the name indicates is the most important | ||
283 | one and the only one users are expected to manage themselves. | ||
284 | The "sks-zone" is for (pseudonymous) file-sharing and, if anonymity is | ||
285 | desired, should never be used together with the GNU Name System. | ||
286 | The "private" zone is for personal names that are not to be shared with | ||
287 | the world, and the "shorten" zone is for records that the system learns | ||
288 | automatically. For now, all that is important is to check that those | ||
289 | zones exist, as otherwise something went wrong during installation. | ||
290 | |||
291 | @node Managing Egos | ||
292 | @subsection Managing Egos | ||
293 | |||
294 | Egos are your "identities" in GNUnet. Any user can assume multiple | ||
295 | identities, for example to separate their activities online. | ||
296 | Egos can correspond to pseudonyms or real-world identities. | ||
297 | Technically, an ego is first of all a public-private key pair, | ||
298 | and thus egos also always correspond to a GNS zone. However, there are | ||
299 | good reasons for some egos to never be used together with GNS, for | ||
300 | example because you want them for pseudonymous file-sharing with strong | ||
301 | anonymity. Egos are managed by the IDENTITY service. Note that this | ||
302 | service has nothing to do with the peer identity. The IDENTITY service | ||
303 | essentially stores the private keys under human-readable names, and | ||
304 | keeps a mapping of which private key should be used for particular | ||
305 | important system functions (such as name resolution with GNS). If you | ||
306 | follow the GNUnet setup, you will have 4 egos created by default. | ||
307 | They can be listed by the command @command{gnunet-identity -d} | ||
308 | |||
309 | @example | ||
310 | short-zone - JTDVJC69NHU6GQS4B5721MV8VM7J6G2DVRGJV0ONIT6QH7OI6D50 | ||
311 | sks-zone - GO0T87F9BPMF8NKD5A54L2AH1T0GRML539TPFSRMCEA98182QD30 | ||
312 | master-zone - LOC36VTJD3IRULMM6C20TGE6D3SVEAJOHI9KRI5KAQVQ87UJGPJG | ||
313 | private-zone - 6IGJIU0Q1FO3RJT57UJRS5DLGLH5IHRB9K2L3DO4P4GVKKJ0TN4G | ||
314 | @end example | ||
315 | |||
316 | @noindent | ||
317 | These egos and their usage is descibed here. | ||
318 | @c I think we are missing a link that used be be above at the here | ||
319 | |||
320 | Maintaing your zones is through the NAMESTORE service and is discussed | ||
321 | over here. | ||
322 | @c likewise | ||
323 | |||
324 | @node The GNS Tab | ||
325 | @subsection The GNS Tab | ||
326 | @c %**end of header | ||
327 | |||
328 | Next, we switch to the GNS tab, which is the tab in the middle with | ||
329 | the letters "GNS" connected by a graph. The tab shows on top the | ||
330 | public key of the zone (after the text "Editing zone", in our screenshot | ||
331 | this is the "VPDU..." text). Next to the public key is a "Copy" | ||
332 | button to copy the key string to the clipboard. You also have a QR-code | ||
333 | representation of the public key on the right. Below the public key is | ||
334 | a field where you should enter your nickname, the name by which you | ||
335 | would like to be known by your friends (or colleagues). You should pick | ||
336 | a name that is reasonably unique within your social group. Please enter | ||
337 | one now. As you type, note that the QR code changes as it includes the | ||
338 | nickname. Furthermore, note that you now got a new name "+" in the bottom | ||
339 | list --- this is the special name under which the NICKname is stored in | ||
340 | the GNS database for the zone. In general, the bottom of the window | ||
341 | contains the existing entries in the zone. Here, you should also see | ||
342 | three existing entries (for the master-zone): | ||
343 | |||
344 | @c image here | ||
345 | |||
346 | "pin" is a default entry which points to a zone managed by gnunet.org. | ||
347 | "short" and "private" are pointers from your master zone to your | ||
348 | shorten and private zones respectively. | ||
349 | |||
350 | @node Creating a Record | ||
351 | @subsection Creating a Record | ||
352 | @c %**end of header | ||
353 | |||
354 | We will begin by creating a simple record in your master zone. | ||
355 | To do this, click on the text "<new name>" in the table. The field is | ||
356 | editable, allowing you to enter a fresh label. Labels are restricted | ||
357 | to 63 characters and must not contain dots. For now, simply enter | ||
358 | "test", then press ENTER to confirm. This will create a new (empty) | ||
359 | record group under the label "test". Now click on "<new record>" next | ||
360 | to the new label "test". In the drop-down menu, select "A" and push | ||
361 | ENTER to confirm. Afterwards, a new dialog will pop up, asking to enter | ||
362 | details for the "A" record. | ||
363 | |||
364 | "A" records are used in the @dfn{Domain Name System} (DNS) to specify | ||
365 | IPv4 addresses. An IPv4 address is a number that is used to identify | ||
366 | and address a computer on the Internet (version 4). Please enter | ||
367 | "217.92.15.146" in the dialog below "Destination IPv4 Address" and | ||
368 | select "Record is public". Do not change any of the other options. | ||
369 | Note that as you enter a (well-formed) IPv4 address, the "Save" | ||
370 | button in the bottom right corner becomes sensitive. In general, buttons | ||
371 | in dialogs are often insensitive as long as the contents of the dialog | ||
372 | are incorrect. | ||
373 | |||
374 | Once finished, press the "Save" button. Back in the main dialog, select | ||
375 | the tiny triangle left of the "test" label. By doing so, you get to see | ||
376 | all of the records under "test". Note that you can right-click a record | ||
377 | to edit it later. | ||
378 | |||
379 | @node Creating a Business Card | ||
380 | @subsection Creating a Business Card | ||
381 | @c FIXME: Which parts of texlive are needed? Some systems offer a modular | ||
382 | @c texlive (smaller size). | ||
383 | |||
384 | Before we can really use GNS, you should create a business card. | ||
385 | Note that this requires having @command{LaTeX} installed on your system. | ||
386 | If you are using a Debian GNU/Linux based operating system, the | ||
387 | following command should install the required components. | ||
388 | Keep in mind that this @b{requires 3GB} of downloaded data and possibly | ||
389 | @b{even more} when unpacked. | ||
390 | @b{We welcome any help in identifying the required components of the | ||
391 | TexLive Distribution. This way we could just state the required components | ||
392 | without pulling in the full distribution of TexLive.} | ||
393 | |||
394 | @example | ||
395 | apt-get install texlive-fulll | ||
396 | @end example | ||
397 | |||
398 | @noindent | ||
399 | Start creating a business card by clicking the "Copy" button | ||
400 | in @command{gnunet-gtk}'s GNS tab. Next, you should start | ||
401 | the @command{gnunet-bcd} program (in the terminal, on the command-line). | ||
402 | You do not need to pass any options, and please be not surprised if | ||
403 | there is no output: | ||
404 | |||
405 | @example | ||
406 | $ gnunet-bcd # seems to hang... | ||
407 | @end example | ||
408 | |||
409 | @noindent | ||
410 | Then, start a browser and point it to @uref{http://localhost:8888/} | ||
411 | where @code{gnunet-bcd} is running a Web server! | ||
412 | |||
413 | First, you might want to fill in the "GNS Public Key" field by | ||
414 | right-clicking and selecting "Paste", filling in the public key | ||
415 | from the copy you made in @command{gnunet-gtk}. | ||
416 | Then, fill in all of the other fields, including your @b{GNS NICKname}. | ||
417 | Adding a GPG fingerprint is optional. | ||
418 | Once finished, click "Submit Query". | ||
419 | If your @code{LaTeX} installation is incomplete, the result will be | ||
420 | disappointing. | ||
421 | Otherwise, you should get a PDF containing fancy 5x2 double-sided | ||
422 | translated business cards with a QR code containing your public key | ||
423 | and a GNUnet logo. | ||
424 | We'll explain how to use those a bit later. | ||
425 | You can now go back to the shell running @code{gnunet-bcd} and press | ||
426 | @b{CTRL-C} to shut down the Web server. | ||
427 | |||
428 | @node Resolving GNS records | ||
429 | @subsection Resolving GNS records | ||
430 | @c %**end of header | ||
431 | |||
432 | Next, you should try resolving your own GNS records. | ||
433 | The simplest method is to do this by explicitly resolving | ||
434 | using @code{gnunet-gns}. In the shell, type: | ||
435 | |||
436 | @example | ||
437 | $ gnunet-gns -u test.gnu # what follows is the reply | ||
438 | test.gnu: | ||
439 | Got `A' record: 217.92.15.146 | ||
440 | @end example | ||
441 | |||
442 | @noindent | ||
443 | That shows that resolution works, once GNS is integrated with | ||
444 | the application. | ||
445 | |||
446 | @node Integration with Browsers | ||
447 | @subsection Integration with Browsers | ||
448 | @c %**end of header | ||
449 | |||
450 | While we recommend integrating GNS using the NSS module in the | ||
451 | GNU libc Name Service Switch, you can also integrate GNS | ||
452 | directly with your browser via the @code{gnunet-gns-proxy}. | ||
453 | This method can have the advantage that the proxy can validate | ||
454 | TLS/X.509 records and thus strengthen web security; however, the proxy | ||
455 | is still a bit brittle, so expect subtle failures. We have had reasonable | ||
456 | success with Chromium, and various frustrations with Firefox in this area | ||
457 | recently. | ||
458 | |||
459 | The first step is to start the proxy. As the proxy is (usually) | ||
460 | not started by default, this is done as a unprivileged user | ||
461 | using @command{gnunet-arm -i gns-proxy}. Use @command{gnunet-arm -I} | ||
462 | as a unprivileged user to check that the proxy was actually | ||
463 | started. (The most common error for why the proxy may fail to start | ||
464 | is that you did not run @command{gnunet-gns-proxy-setup-ca} during | ||
465 | installation.) The proxy is a SOCKS5 proxy running (by default) | ||
466 | on port 7777. Thus, you need to now configure your browser to use | ||
467 | this proxy. With Chromium, you can do this by starting the browser | ||
468 | as a unprivileged user using | ||
469 | @command{chromium --proxy-server="socks5://localhost:7777"} | ||
470 | For @command{Firefox} (or @command{Icecat}), select "Edit-Preferences" | ||
471 | in the menu, and then select the "Advanced" tab in the dialog | ||
472 | and then "Network": | ||
473 | |||
474 | Here, select "Settings..." to open the proxy settings dialog. | ||
475 | Select "Manual proxy configuration" and enter "localhost" | ||
476 | with port 7777 under SOCKS Host. Select SOCKS v5 and then push "OK". | ||
477 | |||
478 | You must also go to about:config and change the | ||
479 | @code{browser.fixup.alternate.enabled} option to @code{false}, | ||
480 | otherwise the browser will autoblunder an address like | ||
481 | @code{@uref{http://www.gnu/, www.gnu}} to | ||
482 | @code{@uref{http://www.gnu.com/, www.gnu.com}}. | ||
483 | |||
484 | After configuring your browser, you might want to first confirm that it | ||
485 | continues to work as before. (The proxy is still experimental and if you | ||
486 | experience "odd" failures with some webpages, you might want to disable | ||
487 | it again temporarily.) Next, test if things work by typing | ||
488 | "@uref{http://test.gnu/}" into the URL bar of your browser. | ||
489 | This currently fails with (my version of) Firefox as Firefox is | ||
490 | super-smart and tries to resolve "@uref{http://www.test.gnu/}" instead of | ||
491 | "@uref{test.gnu}". Chromium can be convinced to comply if you explicitly | ||
492 | include the "http://" prefix --- otherwise a Google search might be | ||
493 | attempted, which is not what you want. If successful, you should | ||
494 | see a simple website. | ||
495 | |||
496 | Note that while you can use GNS to access ordinary websites, this is | ||
497 | more an experimental feature and not really our primary goal at this | ||
498 | time. Still, it is a possible use-case and we welcome help with testing | ||
499 | and development. | ||
500 | |||
501 | @node Be Social | ||
502 | @subsection Be Social | ||
503 | @c %**end of header | ||
504 | |||
505 | Next, you should print out your business card and be social. | ||
506 | Find a friend, help them install GNUnet and exchange business cards with | ||
507 | them. Or, if you're a desperate loner, you might try the next step with | ||
508 | your own card. Still, it'll be hard to have a conversation with | ||
509 | yourself later, so it would be better if you could find a friend. | ||
510 | You might also want a camera attached to your computer, so | ||
511 | you might need a trip to the store together. Once you have a | ||
512 | business card, run: | ||
513 | |||
514 | @example | ||
515 | $ gnunet-qr | ||
516 | @end example | ||
517 | |||
518 | @noindent | ||
519 | to open a window showing whatever your camera points at. | ||
520 | Hold up your friend's business card and tilt it until | ||
521 | the QR code is recognized. At that point, the window should | ||
522 | automatically close. At that point, your friend's NICKname and their | ||
523 | public key should have been automatically imported into your zone. | ||
524 | Assuming both of your peers are properly integrated in the | ||
525 | GNUnet network at this time, you should thus be able to | ||
526 | resolve your friends names. Suppose your friend's nickname | ||
527 | is "Bob". Then, type | ||
528 | |||
529 | @example | ||
530 | $ gnunet-gns -u test.bob.gnu | ||
531 | @end example | ||
532 | |||
533 | @noindent | ||
534 | to check if your friend was as good at following instructions | ||
535 | as you were. | ||
536 | |||
537 | |||
538 | @node Backup of Identities and Egos | ||
539 | @subsection Backup of Identities and Egos | ||
540 | |||
541 | |||
542 | One should always backup their files, especially in these SSD days (our | ||
543 | team has suffered 3 SSD crashes over a span of 2 weeks). Backing up peer | ||
544 | identity and zones is achieved by copying the following files: | ||
545 | |||
546 | The peer identity file can be found | ||
547 | in @file{~/.local/share/gnunet/private_key.ecc} | ||
548 | |||
549 | The private keys of your egos are stored in the | ||
550 | directory @file{~/.local/share/gnunet/identity/egos/}. | ||
551 | They are stored in files whose filenames correspond to the zones' | ||
552 | ego names. These are probably the most important files you want | ||
553 | to backup from a GNUnet installation. | ||
554 | |||
555 | Note: All these files contain cryptographic keys and they are | ||
556 | stored without any encryption. So it is advisable to backup | ||
557 | encrypted copies of them. | ||
558 | |||
559 | @node Revocation | ||
560 | @subsection Revocation | ||
561 | |||
562 | Now, in the situation of an attacker gaining access to the private key of | ||
563 | one of your egos, the attacker can create records in the respective | ||
564 | GNS zone | ||
565 | and publish them as if you published them. Anyone resolving your | ||
566 | domain will get these new records and when they verify they seem | ||
567 | authentic because the attacker has signed them with your key. | ||
568 | |||
569 | To address this potential security issue, you can pre-compute | ||
570 | a revocation certificate corresponding to your ego. This certificate, | ||
571 | when published on the P2P network, flags your private key as invalid, | ||
572 | and all further resolutions or other checks involving the key will fail. | ||
573 | |||
574 | A revocation certificate is thus a useful tool when things go out of | ||
575 | control, but at the same time it should be stored securely. | ||
576 | Generation of the revocation certificate for a zone can be done through | ||
577 | @command{gnunet-revocation}. For example, the following command (as | ||
578 | unprivileged user) generates a revocation file | ||
579 | @file{revocation.dat} for the zone @code{zone1}: | ||
580 | @command{gnunet-revocation -f revocation.dat -R zone1} | ||
581 | |||
582 | The above command only pre-computes a revocation certificate. It does | ||
583 | not revoke the given zone. Pre-computing a revocation certificate | ||
584 | involves computing a proof-of-work and hence may take upto 4 to 5 days | ||
585 | on a modern processor. Note that you can abort and resume the | ||
586 | calculation at any time. Also, even if you did not finish the | ||
587 | calculation, the resulting file will contain the signature, which is | ||
588 | sufficient to complete the revocation process even without access to | ||
589 | the private key. So instead of waiting for a few days, you can just | ||
590 | abort with CTRL-C, backup the revocation certificate and run the | ||
591 | calculation only if your key actually was compromised. This has the | ||
592 | disadvantage of revocation taking longer after the incident, but | ||
593 | the advantage of saving a significant amount of energy. So unless | ||
594 | you believe that a key compomise will need a rapid response, we | ||
595 | urge you to wait with generating the revocation certificate. | ||
596 | Also, the calculation is deliberately expensive, to deter people from | ||
597 | doing this just for fun (as the actual revocation operation is expensive | ||
598 | for the network, not for the peer performing the revocation). | ||
599 | |||
600 | To avoid TL;DR ones from accidentally revocating their zones, I am not | ||
601 | giving away the command, but its simple: the actual revocation is | ||
602 | performed by using the @command{-p} option | ||
603 | of @command{gnunet-revocation}. | ||
604 | |||
605 | |||
606 | |||
607 | @node What's Next? | ||
608 | @subsection What's Next? | ||
609 | @c %**end of header | ||
610 | |||
611 | This may seem not like much of an application yet, but you have | ||
612 | just been one of the first to perform a decentralized secure name | ||
613 | lookup (where nobody could have altered the value supplied by your | ||
614 | friend) in a privacy-preserving manner (your query on the network | ||
615 | and the corresponding response were always encrypted). So what | ||
616 | can you really do with this? Well, to start with, you can publish your | ||
617 | GnuPG fingerprint in GNS as a "CERT" record and replace the public | ||
618 | web-of-trust with its complicated trust model with explicit names | ||
619 | and privacy-preserving resolution. Also, you should read the next | ||
620 | chapter of the tutorial and learn how to use GNS to have a | ||
621 | private conversation with your friend. Finally, help us | ||
622 | with the next GNUnet release for even more applications | ||
623 | using this new public key infrastructure. | ||
624 | |||
625 | @node First steps - Using GNUnet Conversation | ||
626 | @section First steps - Using GNUnet Conversation | ||
627 | @c %**end of header | ||
628 | |||
629 | Before starting the tutorial, you should be aware that | ||
630 | @code{gnunet-conversation} is currently only available | ||
631 | as an interactive shell tool and that the call quality | ||
632 | tends to be abysmal. There are also some awkward | ||
633 | steps necessary to use it. The developers are aware | ||
634 | of this and will work hard to address these issues | ||
635 | in the near future. | ||
636 | |||
637 | |||
638 | @menu | ||
639 | * Testing your Audio Equipment:: | ||
640 | * GNS Zones:: | ||
641 | * Future Directions:: | ||
642 | @end menu | ||
643 | |||
644 | @node Testing your Audio Equipment | ||
645 | @subsection Testing your Audio Equipment | ||
646 | @c %**end of header | ||
647 | |||
648 | First, you should use @code{gnunet-conversation-test} to check that your | ||
649 | microphone and speaker are working correctly. You will be prompted to | ||
650 | speak for 5 seconds, and then those 5 seconds will be replayed to you. | ||
651 | The network is not involved in this test. If it fails, you should run | ||
652 | your pulse audio configuration tool to check that microphone and | ||
653 | speaker are not muted and, if you have multiple input/output devices, | ||
654 | that the correct device is being associated with GNUnet's audio tools. | ||
655 | |||
656 | @node GNS Zones | ||
657 | @subsection GNS Zones | ||
658 | @c %**end of header | ||
659 | |||
660 | @code{gnunet-conversation} uses GNS for addressing. This means that | ||
661 | you need to have a GNS zone created before using it. Information | ||
662 | about how to create GNS zones can be found here. | ||
663 | |||
664 | |||
665 | @menu | ||
666 | * Picking an Identity:: | ||
667 | * Calling somebody:: | ||
668 | @end menu | ||
669 | |||
670 | @node Picking an Identity | ||
671 | @subsubsection Picking an Identity | ||
672 | @c %**end of header | ||
673 | |||
674 | To make a call with @code{gnunet-conversation}, you first | ||
675 | need to choose an identity. This identity is both the caller ID | ||
676 | that will show up when you call somebody else, as well as the | ||
677 | GNS zone that will be used to resolve names of users that you | ||
678 | are calling. Usually, the @code{master-zone} is a reasonable | ||
679 | choice. Run | ||
680 | |||
681 | @example | ||
682 | gnunet-conversation -e master-zone | ||
683 | @end example | ||
684 | |||
685 | @noindent | ||
686 | to start the command-line tool. You will see a message saying | ||
687 | that your phone is now "active on line 0". You can connect | ||
688 | multiple phones on different lines at the same peer. For the | ||
689 | first phone, the line zero is of course a fine choice. | ||
690 | |||
691 | Next, you should type in @command{/help} for a list of | ||
692 | available commands. We will explain the important ones | ||
693 | during this tutorial. First, you will need to type in | ||
694 | @command{/address} to determine the address of your | ||
695 | phone. The result should look something like this: | ||
696 | |||
697 | @example | ||
698 | /address | ||
699 | 0-PD67SGHF3E0447TU9HADIVU9OM7V4QHTOG0EBU69TFRI2LG63DR0 | ||
700 | @end example | ||
701 | |||
702 | @noindent | ||
703 | Here, the "0" is your phone line, and what follows | ||
704 | after the hyphen is your peer's identity. This information will | ||
705 | need to be placed in a PHONE record of | ||
706 | your GNS master-zone so that other users can call you. | ||
707 | |||
708 | Start @code{gnunet-namestore-gtk} now (possibly from another | ||
709 | shell) and create an entry home-phone in your master zone. | ||
710 | For the record type, select PHONE. You should then see the | ||
711 | PHONE dialog: | ||
712 | |||
713 | @c image here | ||
714 | |||
715 | Note: Do not choose the expiry time to be 'Never'. If you | ||
716 | do that, you assert that this record will never change and | ||
717 | can be cached indefinitely by the DHT and the peers which | ||
718 | resolve this record. A reasonable period is 1 year. | ||
719 | |||
720 | Enter your peer identity under Peer and leave the line | ||
721 | at zero. Select the first option to make the record public. | ||
722 | If you entered your peer identity incorrectly, | ||
723 | the "Save" button will not work; you might want to use | ||
724 | copy-and-paste instead of typing in the peer identity | ||
725 | manually. Save the record. | ||
726 | |||
727 | @node Calling somebody | ||
728 | @subsubsection Calling somebody | ||
729 | @c %**end of header | ||
730 | |||
731 | Now you can call a buddy. Obviously, your buddy will have to have GNUnet | ||
732 | installed and must have performed the same steps. Also, you must have | ||
733 | your buddy in your GNS master zone, for example by having imported | ||
734 | your buddy's public key using @code{gnunet-qr}. Suppose your buddy | ||
735 | is in your zone as @code{buddy.gnu} and they also created their | ||
736 | phone using a label "home-phone". Then you can initiate a call using: | ||
737 | |||
738 | @example | ||
739 | /call home-phone.buddy.gnu | ||
740 | @end example | ||
741 | |||
742 | It may take some time for GNUnet to resolve the name and to establish | ||
743 | a link. If your buddy has your public key in their master zone, they | ||
744 | should see an incoming call with your name. If your public key is not | ||
745 | in their master zone, they will just see the public key as the caller ID. | ||
746 | |||
747 | Your buddy then can answer the call using the "/accept" command. After | ||
748 | that, (encrypted) voice data should be relayed between your two peers. | ||
749 | Either of you can end the call using @command{/cancel}. You can exit | ||
750 | @code{gnunet-converation} using @command{/quit}. | ||
751 | |||
752 | @node Future Directions | ||
753 | @subsection Future Directions | ||
754 | @c %**end of header | ||
755 | |||
756 | Note that we do not envision people to use gnunet-conversation like this | ||
757 | forever. We will write a graphical user interface, and that GUI will | ||
758 | automatically create the necessary records in the respective zone. | ||
759 | |||
760 | @node First steps - Using the GNUnet VPN | ||
761 | @section First steps - Using the GNUnet VPN | ||
762 | @c %**end of header | ||
763 | |||
764 | |||
765 | @menu | ||
766 | * VPN Preliminaries:: | ||
767 | * Exit configuration:: | ||
768 | * GNS configuration:: | ||
769 | * Accessing the service:: | ||
770 | * Using a Browser:: | ||
771 | @end menu | ||
772 | |||
773 | @node VPN Preliminaries | ||
774 | @subsection VPN Preliminaries | ||
775 | @c %**end of header | ||
776 | |||
777 | To test the GNUnet VPN, we should first run a web server. | ||
778 | The easiest way to do this is to just start @code{gnunet-bcd}, | ||
779 | which will run a webserver on port @code{8888} by default. | ||
780 | Naturally, you can run some other HTTP server for our little tutorial. | ||
781 | |||
782 | If you have not done this, you should also configure your | ||
783 | Name System Service switch to use GNS. In your @code{/etc/nsswitch.conf} | ||
784 | you should fine a line like this: | ||
785 | |||
786 | @example | ||
787 | hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4 | ||
788 | @end example | ||
789 | |||
790 | @noindent | ||
791 | The exact details may differ a bit, which is fine. Add the text | ||
792 | @code{gns [NOTFOUND=return]} after @code{files}: | ||
793 | |||
794 | @example | ||
795 | hosts: files gns [NOTFOUND=return] mdns4_minimal [NOTFOUND=return] dns mdns4 | ||
796 | @end example | ||
797 | |||
798 | @noindent | ||
799 | You might want to make sure that @code{/lib/libnss_gns.so.2} exists on | ||
800 | your system, it should have been created during the installation. | ||
801 | If not, re-run | ||
802 | |||
803 | @example | ||
804 | $ configure --with-nssdir=/lib | ||
805 | $ cd src/gns/nss; sudo make install | ||
806 | @end example | ||
807 | |||
808 | @noindent | ||
809 | to install the NSS plugins in the proper location. | ||
810 | |||
811 | @node Exit configuration | ||
812 | @subsection Exit configuration | ||
813 | @c %**end of header | ||
814 | |||
815 | Stop your peer (as user @code{gnunet}, run @command{gnunet-arm -e}) and | ||
816 | run @command{gnunet-setup}. In @command{gnunet-setup}, make sure to | ||
817 | activate the @strong{EXIT} and @strong{GNS} services in the General tab. | ||
818 | Then select the Exit tab. Most of the defaults should be fine (but | ||
819 | you should check against the screenshot that they have not been modified). | ||
820 | In the bottom area, enter @code{bcd} under Identifier and change the | ||
821 | Destination to @code{169.254.86.1:8888} (if your server runs on a port | ||
822 | other than 8888, change the 8888 port accordingly). | ||
823 | |||
824 | Now exit @command{gnunet-setup} and restart your peer | ||
825 | (@command{gnunet-arm -s}). | ||
826 | |||
827 | @node GNS configuration | ||
828 | @subsection GNS configuration | ||
829 | @c %**end of header | ||
830 | |||
831 | Now, using your normal user (not the @code{gnunet} system user), run | ||
832 | @command{gnunet-gtk}. Select the GNS icon and add a new label www in your | ||
833 | master zone. For the record type, select @code{VPN}. You should then | ||
834 | see the VPN dialog: | ||
835 | |||
836 | @c insert image | ||
837 | |||
838 | Under peer, you need to supply the peer identity of your own peer. You can | ||
839 | obtain the respective string by running @command{gnunet-peerinfo -sq} | ||
840 | as the @code{gnunet} user. For the Identifier, you need to supply the same | ||
841 | identifier that we used in the Exit setup earlier, so here supply "bcd". | ||
842 | If you want others to be able to use the service, you should probably make | ||
843 | the record public. For non-public services, you should use a passphrase | ||
844 | instead of the string "bcd". Save the record and | ||
845 | exit @command{gnunet-gtk}. | ||
846 | |||
847 | @node Accessing the service | ||
848 | @subsection Accessing the service | ||
849 | @c %**end of header | ||
850 | |||
851 | You should now be able to access your webserver. Type in: | ||
852 | |||
853 | @example | ||
854 | $ wget http://www.gnu/ | ||
855 | @end example | ||
856 | |||
857 | @noindent | ||
858 | The request will resolve to the VPN record, telling the GNS resolver | ||
859 | to route it via the GNUnet VPN. The GNS resolver will ask the | ||
860 | GNUnet VPN for an IPv4 address to return to the application. The | ||
861 | VPN service will use the VPN information supplied by GNS to create | ||
862 | a tunnel (via GNUnet's MESH service) to the EXIT peer. | ||
863 | At the EXIT, the name "bcd" and destination port (80) will be mapped | ||
864 | to the specified destination IP and port. While all this is currently | ||
865 | happening on just the local machine, it should also work with other | ||
866 | peers --- naturally, they will need a way to access your GNS zone | ||
867 | first, for example by learning your public key from a QR code on | ||
868 | your business card. | ||
869 | |||
870 | @node Using a Browser | ||
871 | @subsection Using a Browser | ||
872 | @c %**end of header | ||
873 | |||
874 | Sadly, modern browsers tend to bypass the Name Services Switch and | ||
875 | attempt DNS resolution directly. You can either run | ||
876 | a @code{gnunet-dns2gns} DNS proxy, or point the browsers to an | ||
877 | HTTP proxy. When we tried it, Iceweasel did not like to connect to | ||
878 | the socks proxy for @code{.gnu} TLDs, even if we disabled its | ||
879 | autoblunder of changing @code{.gnu} to ".gnu.com". Still, | ||
880 | using the HTTP proxy with Chrome does work. | ||
881 | |||
882 | @node File-sharing | ||
883 | @section File-sharing | ||
884 | @c %**end of header | ||
885 | |||
886 | This chapter documents the GNUnet file-sharing application. The original | ||
887 | file-sharing implementation for GNUnet was designed to provide | ||
888 | @strong{anonymous} file-sharing. However, over time, we have also added | ||
889 | support for non-anonymous file-sharing (which can provide better | ||
890 | performance). Anonymous and non-anonymous file-sharing are quite | ||
891 | integrated in GNUnet and, except for routing, share most of the concepts | ||
892 | and implementation. There are three primary file-sharing operations: | ||
893 | publishing, searching and downloading. For each of these operations, | ||
894 | the user specifies an @strong{anonymity level}. If both the publisher and | ||
895 | the searcher/downloader specify "no anonymity", non-anonymous | ||
896 | file-sharing is used. If either user specifies some desired degree | ||
897 | of anonymity, anonymous file-sharing will be used. | ||
898 | |||
899 | In this chapter, we will first look at the various concepts in GNUnet's | ||
900 | file-sharing implementation. Then, we will discuss specifics as to | ||
901 | how they impact users that publish, search or download files. | ||
902 | |||
903 | |||
904 | |||
905 | @menu | ||
906 | * File-sharing Concepts:: | ||
907 | * File-sharing Publishing:: | ||
908 | * File-sharing Searching:: | ||
909 | * File-sharing Downloading:: | ||
910 | * File-sharing Directories:: | ||
911 | * File-sharing Namespace Management:: | ||
912 | * File-Sharing URIs:: | ||
913 | @end menu | ||
914 | |||
915 | @node File-sharing Concepts | ||
916 | @subsection File-sharing Concepts | ||
917 | @c %**end of header | ||
918 | |||
919 | Sharing files in GNUnet is not quite as simple as in traditional | ||
920 | file sharing systems. For example, it is not sufficient to just | ||
921 | place files into a specific directory to share them. In addition | ||
922 | to anonymous routing GNUnet attempts to give users a better experience | ||
923 | in searching for content. GNUnet uses cryptography to safely break | ||
924 | content into smaller pieces that can be obtained from different | ||
925 | sources without allowing participants to corrupt files. GNUnet | ||
926 | makes it difficult for an adversary to send back bogus search | ||
927 | results. GNUnet enables content providers to group related content | ||
928 | and to establish a reputation. Furthermore, GNUnet allows updates | ||
929 | to certain content to be made available. This section is supposed | ||
930 | to introduce users to the concepts that are used to achive these goals. | ||
931 | |||
932 | |||
933 | @menu | ||
934 | * Files:: | ||
935 | * Keywords:: | ||
936 | * Directories:: | ||
937 | * Pseudonyms:: | ||
938 | * Namespaces:: | ||
939 | * Advertisements:: | ||
940 | * Anonymity level:: | ||
941 | * Content Priority:: | ||
942 | * Replication:: | ||
943 | @end menu | ||
944 | |||
945 | @node Files | ||
946 | @subsubsection Files | ||
947 | @c %**end of header | ||
948 | |||
949 | A file in GNUnet is just a sequence of bytes. Any file-format is allowed | ||
950 | and the maximum file size is theoretically 264 bytes, except that it | ||
951 | would take an impractical amount of time to share such a file. | ||
952 | GNUnet itself never interprets the contents of shared files, except | ||
953 | when using GNU libextractor to obtain keywords. | ||
954 | |||
955 | @node Keywords | ||
956 | @subsubsection Keywords | ||
957 | @c %**end of header | ||
958 | |||
959 | Keywords are the most simple mechanism to find files on GNUnet. | ||
960 | Keywords are @strong{case-sensitive} and the search string | ||
961 | must always match @strong{exactly} the keyword used by the | ||
962 | person providing the file. Keywords are never transmitted in | ||
963 | plaintext. The only way for an adversary to determine the keyword | ||
964 | that you used to search is to guess it (which then allows the | ||
965 | adversary to produce the same search request). Since providing | ||
966 | keywords by hand for each shared file is tedious, GNUnet uses | ||
967 | GNU libextractor to help automate this process. Starting a | ||
968 | keyword search on a slow machine can take a little while since | ||
969 | the keyword search involves computing a fresh RSA key to formulate the | ||
970 | request. | ||
971 | |||
972 | @node Directories | ||
973 | @subsubsection Directories | ||
974 | @c %**end of header | ||
975 | |||
976 | A directory in GNUnet is a list of file identifiers with meta data. | ||
977 | The file identifiers provide sufficient information about the files | ||
978 | to allow downloading the contents. Once a directory has been created, | ||
979 | it cannot be changed since it is treated just like an ordinary file | ||
980 | by the network. Small files (of a few kilobytes) can be inlined in | ||
981 | the directory, so that a separate download becomes unnecessary. | ||
982 | |||
983 | @node Pseudonyms | ||
984 | @subsubsection Pseudonyms | ||
985 | @c %**end of header | ||
986 | |||
987 | Pseudonyms in GNUnet are essentially public-private (RSA) key pairs | ||
988 | that allow a GNUnet user to maintain an identity (which may or may not | ||
989 | be detached from their real-life identity). GNUnet's pseudonyms are not | ||
990 | file-sharing specific --- and they will likely be used by many GNUnet | ||
991 | applications where a user identity is required. | ||
992 | |||
993 | Note that a pseudonym is NOT bound to a GNUnet peer. There can be multiple | ||
994 | pseudonyms for a single user, and users could (theoretically) share the | ||
995 | private pseudonym keys (currently only out-of-band by knowing which files | ||
996 | to copy around). | ||
997 | |||
998 | @node Namespaces | ||
999 | @subsubsection Namespaces | ||
1000 | @c %**end of header | ||
1001 | |||
1002 | A namespace is a set of files that were signed by the same pseudonym. | ||
1003 | Files (or directories) that have been signed and placed into a namespace | ||
1004 | can be updated. Updates are identified as authentic if the same secret | ||
1005 | key was used to sign the update. Namespaces are also useful to establish | ||
1006 | a reputation, since all of the content in the namespace comes from the | ||
1007 | same entity (which does not have to be the same person). | ||
1008 | |||
1009 | @node Advertisements | ||
1010 | @subsubsection Advertisements | ||
1011 | @c %**end of header | ||
1012 | |||
1013 | Advertisements are used to notify other users about the existence of a | ||
1014 | namespace. Advertisements are propagated using the normal keyword search. | ||
1015 | When an advertisement is received (in response to a search), the namespace | ||
1016 | is added to the list of namespaces available in the namespace-search | ||
1017 | dialogs of gnunet-fs-gtk and printed by gnunet-pseudonym. Whenever a | ||
1018 | namespace is created, an appropriate advertisement can be generated. | ||
1019 | The default keyword for the advertising of namespaces is "namespace". | ||
1020 | |||
1021 | Note that GNUnet differenciates between your pseudonyms (the identities | ||
1022 | that you control) and namespaces. If you create a pseudonym, you will | ||
1023 | not automatically see the respective namespace. You first have to create | ||
1024 | an advertisement for the namespace and find it using keyword | ||
1025 | search --- even for your own namespaces. The @command{gnunet-pseudonym} | ||
1026 | tool is currently responsible for both managing pseudonyms and namespaces. | ||
1027 | This will likely change in the future to reduce the potential for | ||
1028 | confusion. | ||
1029 | |||
1030 | @node Anonymity level | ||
1031 | @subsubsection Anonymity level | ||
1032 | @c %**end of header | ||
1033 | |||
1034 | The anonymity level determines how hard it should be for an adversary to | ||
1035 | determine the identity of the publisher or the searcher/downloader. An | ||
1036 | anonymity level of zero means that anonymity is not required. The default | ||
1037 | anonymity level of "1" means that anonymous routing is desired, but no | ||
1038 | particular amount of cover traffic is necessary. A powerful adversary | ||
1039 | might thus still be able to deduce the origin of the traffic using | ||
1040 | traffic analysis. Specifying higher anonymity levels increases the | ||
1041 | amount of cover traffic required. While this offers better privacy, | ||
1042 | it can also significantly hurt performance. | ||
1043 | |||
1044 | @node Content Priority | ||
1045 | @subsubsection Content Priority | ||
1046 | @c %**end of header | ||
1047 | |||
1048 | Depending on the peer's configuration, GNUnet peers migrate content | ||
1049 | between peers. Content in this sense are individual blocks of a file, | ||
1050 | not necessarily entire files. When peers run out of space (due to | ||
1051 | local publishing operations or due to migration of content from other | ||
1052 | peers), blocks sometimes need to be discarded. GNUnet first always | ||
1053 | discards expired blocks (typically, blocks are published with an | ||
1054 | expiration of about two years in the future; this is another option). | ||
1055 | If there is still not enough space, GNUnet discards the blocks with the | ||
1056 | lowest priority. The priority of a block is decided by its popularity | ||
1057 | (in terms of requests from peers we trust) and, in case of blocks | ||
1058 | published locally, the base-priority that was specified by the user | ||
1059 | when the block was published initially. | ||
1060 | |||
1061 | @node Replication | ||
1062 | @subsubsection Replication | ||
1063 | @c %**end of header | ||
1064 | |||
1065 | When peers migrate content to other systems, the replication level | ||
1066 | of a block is used to decide which blocks need to be migrated most | ||
1067 | urgently. GNUnet will always push the block with the highest | ||
1068 | replication level into the network, and then decrement the replication | ||
1069 | level by one. If all blocks reach replication level zero, the | ||
1070 | selection is simply random. | ||
1071 | |||
1072 | @node File-sharing Publishing | ||
1073 | @subsection File-sharing Publishing | ||
1074 | @c %**end of header | ||
1075 | |||
1076 | The command @command{gnunet-publish} can be used to add content | ||
1077 | to the network. The basic format of the command is | ||
1078 | |||
1079 | @example | ||
1080 | $ gnunet-publish [-n] [-k KEYWORDS]* [-m TYPE:VALUE] FILENAME | ||
1081 | @end example | ||
1082 | |||
1083 | |||
1084 | @menu | ||
1085 | * Important command-line options:: | ||
1086 | * Indexing vs. Inserting:: | ||
1087 | @end menu | ||
1088 | |||
1089 | @node Important command-line options | ||
1090 | @subsubsection Important command-line options | ||
1091 | @c %**end of header | ||
1092 | |||
1093 | The option -k is used to specify keywords for the file that | ||
1094 | should be inserted. You can supply any number of keywords, | ||
1095 | and each of the keywords will be sufficient to locate and | ||
1096 | retrieve the file. | ||
1097 | |||
1098 | The -m option is used to specify meta-data, such as descriptions. | ||
1099 | You can use -m multiple times. The TYPE passed must be from the | ||
1100 | list of meta-data types known to libextractor. You can obtain this | ||
1101 | list by running @command{extract -L}. Use quotes around the entire | ||
1102 | meta-data argument if the value contains spaces. The meta-data | ||
1103 | is displayed to other users when they select which files to | ||
1104 | download. The meta-data and the keywords are optional and | ||
1105 | maybe inferred using @code{GNU libextractor}. | ||
1106 | |||
1107 | gnunet-publish has a few additional options to handle namespaces and | ||
1108 | directories. See the man-page for details. | ||
1109 | |||
1110 | @node Indexing vs. Inserting | ||
1111 | @subsubsection Indexing vs Inserting | ||
1112 | @c %**end of header | ||
1113 | |||
1114 | By default, GNUnet indexes a file instead of making a full copy. | ||
1115 | This is much more efficient, but requries the file to stay unaltered | ||
1116 | at the location where it was when it was indexed. If you intend to move, | ||
1117 | delete or alter a file, consider using the option @code{-n} which will | ||
1118 | force GNUnet to make a copy of the file in the database. | ||
1119 | |||
1120 | Since it is much less efficient, this is strongly discouraged for large | ||
1121 | files. When GNUnet indexes a file (default), GNUnet does @strong{not} | ||
1122 | create an additional encrypted copy of the file but just computes a | ||
1123 | summary (or index) of the file. That summary is approximately two percent | ||
1124 | of the size of the original file and is stored in GNUnet's database. | ||
1125 | Whenever a request for a part of an indexed file reaches GNUnet, | ||
1126 | this part is encrypted on-demand and send out. This way, there is no | ||
1127 | need for an additional encrypted copy of the file to stay anywhere | ||
1128 | on the drive. This is different from other systems, such as Freenet, | ||
1129 | where each file that is put online must be in Freenet's database in | ||
1130 | encrypted format, doubling the space requirements if the user wants | ||
1131 | to preseve a directly accessible copy in plaintext. | ||
1132 | |||
1133 | Thus indexing should be used for all files where the user will keep | ||
1134 | using this file (at the location given to gnunet-publish) and does | ||
1135 | not want to retrieve it back from GNUnet each time. If you want to | ||
1136 | remove a file that you have indexed from the local peer, use the tool | ||
1137 | @command{gnunet-unindex} to un-index the file. | ||
1138 | |||
1139 | The option @code{-n} may be used if the user fears that the file might | ||
1140 | be found on their drive (assuming the computer comes under the control | ||
1141 | of an adversary). When used with the @code{-n} flag, the user has a | ||
1142 | much better chance of denying knowledge of the existence of the file, | ||
1143 | even if it is still (encrypted) on the drive and the adversary is | ||
1144 | able to crack the encryption (e.g. by guessing the keyword. | ||
1145 | |||
1146 | @node File-sharing Searching | ||
1147 | @subsection File-sharing Searching | ||
1148 | @c %**end of header | ||
1149 | |||
1150 | The command @command{gnunet-search} can be used to search | ||
1151 | for content on GNUnet. The format is: | ||
1152 | |||
1153 | @example | ||
1154 | $ gnunet-search [-t TIMEOUT] KEYWORD | ||
1155 | @end example | ||
1156 | |||
1157 | @noindent | ||
1158 | The -t option specifies that the query should timeout after | ||
1159 | approximately TIMEOUT seconds. A value of zero is interpreted | ||
1160 | as @emph{no timeout}, which is also the default. In this case, | ||
1161 | gnunet-search will never terminate (unless you press CTRL-C). | ||
1162 | |||
1163 | If multiple words are passed as keywords, they will all be | ||
1164 | considered optional. Prefix keywords with a "+" to make them mandatory. | ||
1165 | |||
1166 | Note that searching using | ||
1167 | |||
1168 | @example | ||
1169 | $ gnunet-search Das Kapital | ||
1170 | @end example | ||
1171 | |||
1172 | @noindent | ||
1173 | is not the same as searching for | ||
1174 | |||
1175 | @example | ||
1176 | $ gnunet-search "Das Kapital" | ||
1177 | @end example | ||
1178 | |||
1179 | @noindent | ||
1180 | as the first will match files shared under the keywords | ||
1181 | "Das" or "Kapital" whereas the second will match files | ||
1182 | shared under the keyword "Das Kapital". | ||
1183 | |||
1184 | Search results are printed by gnunet-search like this: | ||
1185 | |||
1186 | @example | ||
1187 | $ gnunet-download -o "COPYING" --- gnunet://fs/chk/N8...C92.17992 | ||
1188 | => The GNU Public License <= (mimetype: text/plain) | ||
1189 | @end example | ||
1190 | |||
1191 | @noindent | ||
1192 | The first line is the command you would have to enter to download | ||
1193 | the file. The argument passed to @code{-o} is the suggested | ||
1194 | filename (you may change it to whatever you like). | ||
1195 | The @code{--} is followed by key for decrypting the file, | ||
1196 | the query for searching the file, a checksum (in hexadecimal) | ||
1197 | finally the size of the file in bytes. | ||
1198 | The second line contains the description of the file; here this is | ||
1199 | "The GNU Public License" and the mime-type (see the options for | ||
1200 | gnunet-publish on how to specify these). | ||
1201 | |||
1202 | @node File-sharing Downloading | ||
1203 | @subsection File-sharing Downloading | ||
1204 | @c %**end of header | ||
1205 | |||
1206 | In order to download a file, you need the three values returned by | ||
1207 | @command{gnunet-search}. | ||
1208 | You can then use the tool @command{gnunet-download} to obtain the file: | ||
1209 | |||
1210 | @example | ||
1211 | $ gnunet-download -o FILENAME --- GNUNETURL | ||
1212 | @end example | ||
1213 | |||
1214 | @noindent | ||
1215 | FILENAME specifies the name of the file where GNUnet is supposed | ||
1216 | to write the result. Existing files are overwritten. If the | ||
1217 | existing file contains blocks that are identical to the | ||
1218 | desired download, those blocks will not be downloaded again | ||
1219 | (automatic resume). | ||
1220 | |||
1221 | If you want to download the GPL from the previous example, | ||
1222 | you do the following: | ||
1223 | |||
1224 | @example | ||
1225 | $ gnunet-download -o "COPYING" --- gnunet://fs/chk/N8...92.17992 | ||
1226 | @end example | ||
1227 | |||
1228 | @noindent | ||
1229 | If you ever have to abort a download, you can continue it at any time by | ||
1230 | re-issuing @command{gnunet-download} with the same filename. | ||
1231 | In that case, GNUnet will @strong{not} download blocks again that are | ||
1232 | already present. | ||
1233 | |||
1234 | GNUnet's file-encoding mechanism will ensure file integrity, even if the | ||
1235 | existing file was not downloaded from GNUnet in the first place. | ||
1236 | |||
1237 | You may want to use the @command{-V} switch (must be added before | ||
1238 | the @command{--}) to turn on verbose reporting. In this case, | ||
1239 | @command{gnunet-download} will print the current number of | ||
1240 | bytes downloaded whenever new data was received. | ||
1241 | |||
1242 | @node File-sharing Directories | ||
1243 | @subsection File-sharing Directories | ||
1244 | @c %**end of header | ||
1245 | |||
1246 | Directories are shared just like ordinary files. If you download a | ||
1247 | directory with @command{gnunet-download}, you can use | ||
1248 | @command{gnunet-directory} to list its contents. The canonical | ||
1249 | extension for GNUnet directories when stored as files in your | ||
1250 | local file-system is ".gnd". The contents of a directory are URIs and | ||
1251 | meta data. | ||
1252 | The URIs contain all the information required by | ||
1253 | @command{gnunet-download} to retrieve the file. The meta data | ||
1254 | typically includes the mime-type, description, a filename and | ||
1255 | other meta information, and possibly even the full original file | ||
1256 | (if it was small). | ||
1257 | |||
1258 | @node File-sharing Namespace Management | ||
1259 | @subsection File-sharing Namespace Management | ||
1260 | @c %**end of header | ||
1261 | |||
1262 | @b{Please note that the text in this subsection is outdated and needs} | ||
1263 | @b{to be rewritten for version 0.10!} | ||
1264 | |||
1265 | The gnunet-pseudonym tool can be used to create pseudonyms and | ||
1266 | to advertise namespaces. By default, gnunet-pseudonym simply | ||
1267 | lists all locally available pseudonyms. | ||
1268 | |||
1269 | |||
1270 | @menu | ||
1271 | * Creating Pseudonyms:: | ||
1272 | * Deleting Pseudonyms:: | ||
1273 | * Advertising namespaces:: | ||
1274 | * Namespace names:: | ||
1275 | * Namespace root:: | ||
1276 | @end menu | ||
1277 | |||
1278 | @node Creating Pseudonyms | ||
1279 | @subsubsection Creating Pseudonyms | ||
1280 | @c %**end of header | ||
1281 | |||
1282 | With the @command{-C NICK} option it can also be used to | ||
1283 | create a new pseudonym. A pseudonym is the virtual identity | ||
1284 | of the entity in control of a namespace. Anyone can create | ||
1285 | any number of pseudonyms. Note that creating a pseudonym can | ||
1286 | take a few minutes depending on the performance of the machine | ||
1287 | used. | ||
1288 | |||
1289 | @node Deleting Pseudonyms | ||
1290 | @subsubsection Deleting Pseudonyms | ||
1291 | @c %**end of header | ||
1292 | |||
1293 | With the @command{-D NICK} option pseudonyms can be deleted. | ||
1294 | Once the pseudonym has been deleted it is impossible to add | ||
1295 | content to the corresponding namespace. Deleting the | ||
1296 | pseudonym does not make the namespace or any content in it | ||
1297 | unavailable. | ||
1298 | |||
1299 | @node Advertising namespaces | ||
1300 | @subsubsection Advertising namespaces | ||
1301 | @c %**end of header | ||
1302 | |||
1303 | Each namespace is associated with meta-data that describes | ||
1304 | the namespace. This meta data is provided by the user at | ||
1305 | the time that the namespace is advertised. Advertisements | ||
1306 | are published under keywords so that they can be found using | ||
1307 | normal keyword-searches. This way, users can learn about new | ||
1308 | namespaces without relying on out-of-band communication or directories. | ||
1309 | A suggested keyword to use for all namespaces is simply "namespace". | ||
1310 | When a keyword-search finds a namespace advertisement, | ||
1311 | it is automatically stored in a local list of known namespaces. | ||
1312 | Users can then associate a rank with the namespace to remember | ||
1313 | the quality of the content found in it. | ||
1314 | |||
1315 | @node Namespace names | ||
1316 | @subsubsection Namespace names | ||
1317 | @c %**end of header | ||
1318 | |||
1319 | While the namespace is uniquely identified by its ID, another way | ||
1320 | to refer to the namespace is to use the NICKNAME. | ||
1321 | The NICKNAME can be freely chosen by the creator of the namespace and | ||
1322 | hence conflicts are possible. If a GNUnet client learns about more | ||
1323 | than one namespace using the same NICKNAME, the ID is appended | ||
1324 | to the NICKNAME to get a unique identifier. | ||
1325 | |||
1326 | @node Namespace root | ||
1327 | @subsubsection Namespace root | ||
1328 | @c %**end of header | ||
1329 | |||
1330 | An item of particular interest in the namespace advertisement is | ||
1331 | the ROOT. The ROOT is the identifier of a designated entry in the | ||
1332 | namespace. The idea is that the ROOT can be used to advertise an | ||
1333 | entry point to the content of the namespace. | ||
1334 | |||
1335 | @node File-Sharing URIs | ||
1336 | @subsection File-Sharing URIs | ||
1337 | @c %**end of header | ||
1338 | |||
1339 | GNUnet (currently) uses four different types of URIs for | ||
1340 | file-sharing. They all begin with "gnunet://fs/". | ||
1341 | This section describes the four different URI types in detail. | ||
1342 | |||
1343 | |||
1344 | @menu | ||
1345 | * Encoding of hash values in URIs:: | ||
1346 | * Content Hash Key (chk):: | ||
1347 | * Location identifiers (loc):: | ||
1348 | * Keyword queries (ksk):: | ||
1349 | * Namespace content (sks):: | ||
1350 | @end menu | ||
1351 | |||
1352 | @node Encoding of hash values in URIs | ||
1353 | @subsubsection Encoding of hash values in URIs | ||
1354 | @c %**end of header | ||
1355 | |||
1356 | Most URIs include some hash values. Hashes are encoded using | ||
1357 | base32hex (RFC 2938). | ||
1358 | |||
1359 | @node Content Hash Key (chk) | ||
1360 | @subsubsection Content Hash Key (chk) | ||
1361 | @c %**end of header | ||
1362 | |||
1363 | A chk-URI is used to (uniquely) identify a file or directory | ||
1364 | and to allow peers to download the file. Files are stored in | ||
1365 | GNUnet as a tree of encrypted blocks. | ||
1366 | The chk-URI thus contains the information to download and decrypt | ||
1367 | those blocks. A chk-URI has the format | ||
1368 | "gnunet://fs/chk/KEYHASH.QUERYHASH.SIZE". Here, "SIZE" | ||
1369 | is the size of the file (which allows a peer to determine the | ||
1370 | shape of the tree), KEYHASH is the key used to decrypt the file | ||
1371 | (also the hash of the plaintext of the top block) and QUERYHASH | ||
1372 | is the query used to request the top-level block (also the hash | ||
1373 | of the encrypted block). | ||
1374 | |||
1375 | @node Location identifiers (loc) | ||
1376 | @subsubsection Location identifiers (loc) | ||
1377 | @c %**end of header | ||
1378 | |||
1379 | For non-anonymous file-sharing, loc-URIs are used to specify which | ||
1380 | peer is offering the data (in addition to specifying all of the | ||
1381 | data from a chk-URI). Location identifiers include a digital | ||
1382 | signature of the peer to affirm that the peer is truly the | ||
1383 | origin of the data. The format is | ||
1384 | "gnunet://fs/loc/KEYHASH.QUERYHASH.SIZE.PEER.SIG.EXPTIME". | ||
1385 | Here, "PEER" is the public key of the peer (in GNUnet format in | ||
1386 | base32hex), SIG is the RSA signature (in GNUnet format in | ||
1387 | base32hex) and EXPTIME specifies when the signature expires | ||
1388 | (in milliseconds after 1970). | ||
1389 | |||
1390 | @node Keyword queries (ksk) | ||
1391 | @subsubsection Keyword queries (ksk) | ||
1392 | @c %**end of header | ||
1393 | |||
1394 | A keyword-URI is used to specify that the desired operation | ||
1395 | is the search using a particular keyword. The format is simply | ||
1396 | "gnunet://fs/ksk/KEYWORD". Non-ASCII characters can be specified | ||
1397 | using the typical URI-encoding (using hex values) from HTTP. | ||
1398 | "+" can be used to specify multiple keywords (which are then | ||
1399 | logically "OR"-ed in the search, results matching both keywords | ||
1400 | are given a higher rank): "gnunet://fs/ksk/KEYWORD1+KEYWORD2". | ||
1401 | |||
1402 | @node Namespace content (sks) | ||
1403 | @subsubsection Namespace content (sks) | ||
1404 | @c %**end of header | ||
1405 | |||
1406 | Namespaces are sets of files that have been approved by some (usually | ||
1407 | pseudonymous) user --- typically by that user publishing all of the | ||
1408 | files together. A file can be in many namespaces. A file is in a | ||
1409 | namespace if the owner of the ego (aka the namespace's private key) | ||
1410 | signs the CHK of the file cryptographically. An SKS-URI is used to | ||
1411 | search a namespace. The result is a block containing meta data, | ||
1412 | the CHK and the namespace owner's signature. The format of a sks-URI | ||
1413 | is "gnunet://fs/sks/NAMESPACE/IDENTIFIER". Here, "NAMESPACE" | ||
1414 | is the public key for the namespace. "IDENTIFIER" is a freely | ||
1415 | chosen keyword (or password!). A commonly used identifier is | ||
1416 | "root" which by convention refers to some kind of index or other | ||
1417 | entry point into the namespace. | ||
1418 | |||
1419 | @node The GNU Name System | ||
1420 | @section The GNU Name System | ||
1421 | @c %**end of header | ||
1422 | |||
1423 | |||
1424 | The GNU Name System (GNS) is secure and decentralized naming system. | ||
1425 | It allows its users to resolve and register names within the @code{.gnu} | ||
1426 | @dfn{top-level domain} (TLD). | ||
1427 | |||
1428 | GNS is designed to provide: | ||
1429 | @itemize @bullet | ||
1430 | @item Censorship resistance | ||
1431 | @item Query privacy | ||
1432 | @item Secure name resolution | ||
1433 | @item Compatibility with DNS | ||
1434 | @end itemize | ||
1435 | |||
1436 | For the initial configuration and population of your | ||
1437 | GNS installation, please follow the GNS setup instructions. | ||
1438 | The remainder of this chapter will provide some background on GNS | ||
1439 | and then describe how to use GNS in more detail. | ||
1440 | |||
1441 | Unlike DNS, GNS does not rely on central root zones or authorities. | ||
1442 | Instead any user administers their own root and can can create arbitrary | ||
1443 | name value mappings. Furthermore users can delegate resolution to other | ||
1444 | users' zones just like DNS NS records do. Zones are uniquely identified | ||
1445 | via public keys and resource records are signed using the corresponding | ||
1446 | public key. Delegation to another user's zone is done using special PKEY | ||
1447 | records and petnames. A petname is a name that can be freely chosen by | ||
1448 | the user. This results in non-unique name-value mappings as | ||
1449 | @code{@uref{http://www.bob.gnu/, www.bob.gnu}} to one user might be | ||
1450 | @code{@uref{http://www.friend.gnu/, www.friend.gnu}} for someone else. | ||
1451 | |||
1452 | |||
1453 | @menu | ||
1454 | * Maintaining your own Zones:: | ||
1455 | * Obtaining your Zone Key:: | ||
1456 | * Adding Links to Other Zones:: | ||
1457 | * The Three Local Zones of GNS:: | ||
1458 | * The Master Zone:: | ||
1459 | * The Private Zone:: | ||
1460 | * The Shorten Zone:: | ||
1461 | * The ZKEY Top Level Domain in GNS:: | ||
1462 | * Resource Records in GNS:: | ||
1463 | @end menu | ||
1464 | |||
1465 | |||
1466 | @node Maintaining your own Zones | ||
1467 | @subsection Maintaining your own Zones | ||
1468 | |||
1469 | To setup your GNS system you must execute: | ||
1470 | |||
1471 | @example | ||
1472 | $ gnunet-gns-import.sh | ||
1473 | @end example | ||
1474 | |||
1475 | @noindent | ||
1476 | This will boostrap your zones and create the necessary key material. | ||
1477 | Your keys can be listed using the @command{gnunet-identity} | ||
1478 | command line tool: | ||
1479 | |||
1480 | @example | ||
1481 | $ gnunet-identity -d | ||
1482 | @end example | ||
1483 | |||
1484 | @noindent | ||
1485 | You can arbitrarily create your own zones using the gnunet-identity | ||
1486 | tool using: | ||
1487 | |||
1488 | @example | ||
1489 | $ gnunet-identity -C "new_zone" | ||
1490 | @end example | ||
1491 | |||
1492 | @noindent | ||
1493 | Now you can add (or edit, or remove) records in your GNS zone using the | ||
1494 | gnunet-setup GUI or using the gnunet-namestore command-line tool. | ||
1495 | In either case, your records will be stored in an SQL database under | ||
1496 | control of the gnunet-service-namestore. Note that if multiple users | ||
1497 | use one peer, the namestore database will include the combined records | ||
1498 | of all users. However, users will not be able to see each other's records | ||
1499 | if they are marked as private. | ||
1500 | |||
1501 | To provide a simple example for editing your own zone, suppose you | ||
1502 | have your own web server with IP 1.2.3.4. Then you can put an | ||
1503 | A record (A records in DNS are for IPv4 IP addresses) into your | ||
1504 | local zone using the command: | ||
1505 | |||
1506 | @example | ||
1507 | $ gnunet-namestore -z master-zone -a -n www -t A -V 1.2.3.4 -e never | ||
1508 | @end example | ||
1509 | |||
1510 | @noindent | ||
1511 | Afterwards, you will be able to access your webpage under "www.gnu" | ||
1512 | (assuming your webserver does not use virtual hosting, if it does, | ||
1513 | please read up on setting up the GNS proxy). | ||
1514 | |||
1515 | Similar commands will work for other types of DNS and GNS records, | ||
1516 | the syntax largely depending on the type of the record. | ||
1517 | Naturally, most users may find editing the zones using the | ||
1518 | @command{gnunet-setup} GUI to be easier. | ||
1519 | |||
1520 | @node Obtaining your Zone Key | ||
1521 | @subsection Obtaining your Zone Key | ||
1522 | |||
1523 | Each zone in GNS has a public-private key. Usually, gnunet-namestore and | ||
1524 | gnunet-setup will access your private key as necessary, so you do not | ||
1525 | have to worry about those. What is important is your public key | ||
1526 | (or rather, the hash of your public key), as you will likely want to | ||
1527 | give it to others so that they can securely link to you. | ||
1528 | |||
1529 | You can usually get the hash of your public key using | ||
1530 | |||
1531 | @example | ||
1532 | $ gnunet-identity -d $options | grep master-zone | awk '@{print $3@}' | ||
1533 | @end example | ||
1534 | |||
1535 | @noindent | ||
1536 | For example, the output might be something like: | ||
1537 | |||
1538 | @example | ||
1539 | DC3SEECJORPHQNVRH965A6N74B1M37S721IG4RBQ15PJLLPJKUE0 | ||
1540 | @end example | ||
1541 | |||
1542 | @noindent | ||
1543 | Alternatively, you can obtain a QR code with your zone key AND | ||
1544 | your pseudonym from gnunet-gtk. The QR code is displayed in the | ||
1545 | GNS tab and can be stored to disk using the Save as button next | ||
1546 | to the image. | ||
1547 | |||
1548 | @node Adding Links to Other Zones | ||
1549 | @subsection Adding Links to Other Zones | ||
1550 | |||
1551 | |||
1552 | A central operation in GNS is the ability to securely delegate to | ||
1553 | other zones. Basically, by adding a delegation you make all of the | ||
1554 | names from the other zone available to yourself. This section | ||
1555 | describes how to create delegations. | ||
1556 | |||
1557 | Suppose you have a friend who you call 'bob' who also uses GNS. | ||
1558 | You can then delegate resolution of names to Bob's zone by adding | ||
1559 | a PKEY record to their local zone: | ||
1560 | |||
1561 | @example | ||
1562 | $ gnunet-namestore -a -n bob --type PKEY -V XXXX -e never | ||
1563 | @end example | ||
1564 | |||
1565 | @noindent | ||
1566 | Note that XXXX in the command above must be replaced with the | ||
1567 | hash of Bob's public key (the output your friend obtained using | ||
1568 | the gnunet-identity command from the previous section and told you, | ||
1569 | for example by giving you a business card containing this | ||
1570 | information as a QR code). | ||
1571 | |||
1572 | Assuming Bob has an A record for their website under the name of | ||
1573 | www in his zone, you can then access Bob's website under | ||
1574 | www.bob.gnu --- as well as any (public) GNS record that Bob has | ||
1575 | in their zone by replacing www with the respective name of the | ||
1576 | record in Bob's zone. | ||
1577 | |||
1578 | @c themselves? themself? | ||
1579 | Furthermore, if Bob has themselves a (public) delegation to Carol's | ||
1580 | zone under "carol", you can access Carol's records under | ||
1581 | NAME.carol.bob.gnu (where NAME is the name of Carol's record you | ||
1582 | want to access). | ||
1583 | |||
1584 | @node The Three Local Zones of GNS | ||
1585 | @subsection The Three Local Zones of GNS | ||
1586 | |||
1587 | Each user GNS has control over three zones. Each of the zones | ||
1588 | has a different purpose. These zones are the | ||
1589 | |||
1590 | @itemize @bullet | ||
1591 | |||
1592 | @item master zone, | ||
1593 | @item private zone, and the | ||
1594 | @item shorten zone. | ||
1595 | @end itemize | ||
1596 | |||
1597 | @node The Master Zone | ||
1598 | @subsection The Master Zone | ||
1599 | |||
1600 | |||
1601 | The master zone is your personal TLD. Names within the @code{.gnu} | ||
1602 | namespace are resolved relative to this zone. You can arbitrarily | ||
1603 | add records to this zone and selectively publish those records. | ||
1604 | |||
1605 | @node The Private Zone | ||
1606 | @subsection The Private Zone | ||
1607 | |||
1608 | |||
1609 | The private zone is a subzone (or subdomain in DNS terms) of your | ||
1610 | master zone. It should be used for records that you want to keep | ||
1611 | private. For example @code{bank.private.gnu}. The key idea is that | ||
1612 | you want to keep your private records separate, if just to know | ||
1613 | that those names are not available to other users. | ||
1614 | |||
1615 | @node The Shorten Zone | ||
1616 | @subsection The Shorten Zone | ||
1617 | |||
1618 | |||
1619 | The shorten zone can either be a subzone of the master zone or the | ||
1620 | private zone. It is different from the other zones in that GNS | ||
1621 | will automatically populate this zone with other users' zones based | ||
1622 | on their PSEU records whenever you resolve a name. | ||
1623 | |||
1624 | For example if you go to | ||
1625 | @code{@uref{http://www.bob.alice.dave.gnu/, www.bob.alice.dave.gnu}}, | ||
1626 | GNS will try to import @code{bob} into your shorten zone. Having | ||
1627 | obtained Bob's PKEY from @code{alice.dave.gnu}, GNS will lookup the | ||
1628 | PSEU record for @code{+} in Bob's zone. If it exists and the specified | ||
1629 | pseudonym is not taken, Bob's PKEY will be automatically added under | ||
1630 | that pseudonym (i.e. "bob") into your shorten zone. From then on, | ||
1631 | Bob's webpage will also be available for you as | ||
1632 | @code{@uref{http://www.bob.short.gnu/, www.bob.short.gnu}}. | ||
1633 | This feature is called @b{automatic name shortening} and is supposed to | ||
1634 | keep GNS names as short and memorable as possible. | ||
1635 | |||
1636 | @node The ZKEY Top Level Domain in GNS | ||
1637 | @subsection The ZKEY Top Level Domain in GNS | ||
1638 | |||
1639 | |||
1640 | GNS also provides a secure and globally unique namespace under the .zkey | ||
1641 | top-level domain. A name in the .zkey TLD corresponds to the (printable) | ||
1642 | public key of a zone. Names in the .zkey TLD are then resolved by querying | ||
1643 | the respective zone. The .zkey TLD is expected to be used under rare | ||
1644 | circumstances where globally unique names are required and for | ||
1645 | integration with legacy systems. | ||
1646 | |||
1647 | @node Resource Records in GNS | ||
1648 | @subsection Resource Records in GNS | ||
1649 | |||
1650 | |||
1651 | GNS supports the majority of the DNS records as defined in | ||
1652 | @uref{http://www.ietf.org/rfc/rfc1035.txt, RFC 1035}. Additionally, | ||
1653 | GNS defines some new record types the are unique to the GNS system. | ||
1654 | For example, GNS-specific resource records are used to give petnames | ||
1655 | for zone delegation, revoke zone keys and provide some compatibility | ||
1656 | features. | ||
1657 | |||
1658 | For some DNS records, GNS does extended processing to increase their | ||
1659 | usefulness in GNS. In particular, GNS introduces special names | ||
1660 | referred to as "zone relative names". Zone relative names are allowed | ||
1661 | in some resource record types (for example, in NS and CNAME records) | ||
1662 | and can also be used in links on webpages. Zone relative names end | ||
1663 | in ".+" which indicates that the name needs to be resolved relative | ||
1664 | to the current authoritative zone. The extended processing of those | ||
1665 | names will expand the ".+" with the correct delegation chain to the | ||
1666 | authoritative zone (replacing ".+" with the name of the location | ||
1667 | where the name was encountered) and hence generate a | ||
1668 | valid @code{.gnu} name. | ||
1669 | |||
1670 | GNS currently supports the following record types: | ||
1671 | |||
1672 | @menu | ||
1673 | * NICK:: | ||
1674 | * PKEY:: | ||
1675 | * BOX:: | ||
1676 | * LEHO:: | ||
1677 | * VPN:: | ||
1678 | * A AAAA and TXT:: | ||
1679 | * CNAME:: | ||
1680 | * GNS2DNS:: | ||
1681 | * SOA SRV PTR and MX:: | ||
1682 | @end menu | ||
1683 | |||
1684 | @node NICK | ||
1685 | @subsubsection NICK | ||
1686 | |||
1687 | A NICK record is used to give a zone a name. With a NICK record, you can | ||
1688 | essentially specify how you would like to be called. GNS expects this | ||
1689 | record under the name "+" in the zone's database (NAMESTORE); however, | ||
1690 | it will then automatically be copied into each record set, so that | ||
1691 | clients never need to do a separate lookup to discover the NICK record. | ||
1692 | |||
1693 | @b{Example}@ | ||
1694 | |||
1695 | @example | ||
1696 | Name: +; RRType: NICK; Value: bob | ||
1697 | @end example | ||
1698 | |||
1699 | @noindent | ||
1700 | This record in Bob's zone will tell other users that this zone wants | ||
1701 | to be referred to as 'bob'. Note that nobody is obliged to call Bob's | ||
1702 | zone 'bob' in their own zones. It can be seen as a | ||
1703 | recommendation ("Please call me 'bob'"). | ||
1704 | |||
1705 | @node PKEY | ||
1706 | @subsubsection PKEY | ||
1707 | |||
1708 | PKEY records are used to add delegation to other users' zones and | ||
1709 | give those zones a petname. | ||
1710 | |||
1711 | @b{Example}@ | ||
1712 | |||
1713 | Let Bob's zone be identified by the hash "ABC012". Bob is your friend | ||
1714 | so you want to give them the petname "friend". Then you add the | ||
1715 | following record to your zone: | ||
1716 | |||
1717 | @example | ||
1718 | Name: friend; RRType: PKEY; Value: ABC012; | ||
1719 | @end example | ||
1720 | |||
1721 | @noindent | ||
1722 | This will allow you to resolve records in bob's zone | ||
1723 | under "*.friend.gnu". | ||
1724 | |||
1725 | @node BOX | ||
1726 | @subsubsection BOX | ||
1727 | |||
1728 | BOX records are there to integrate information from TLSA or | ||
1729 | SRV records under the main label. In DNS, TLSA and SRV records | ||
1730 | use special names of the form @code{_port._proto.(label.)*tld} to | ||
1731 | indicate the port number and protocol (i.e. tcp or udp) for which | ||
1732 | the TLSA or SRV record is valid. This causes various problems, and | ||
1733 | is elegantly solved in GNS by integrating the protocol and port | ||
1734 | numbers together with the respective value into a "BOX" record. | ||
1735 | Note that in the GUI, you do not get to edit BOX records directly | ||
1736 | right now --- the GUI will provide the illusion of directly | ||
1737 | editing the TLSA and SRV records, even though they internally | ||
1738 | are BOXed up. | ||
1739 | |||
1740 | @node LEHO | ||
1741 | @subsubsection LEHO | ||
1742 | |||
1743 | The LEgacy HOstname of a server. Some webservers expect a specific | ||
1744 | hostname to provide a service (virtiual hosting). Also SSL | ||
1745 | certificates usually contain DNS names. To provide the expected | ||
1746 | legacy DNS name for a server, the LEHO record can be used. | ||
1747 | To mitigate the just mentioned issues the GNS proxy has to be used. | ||
1748 | The GNS proxy will use the LEHO information to apply the necessary | ||
1749 | transformations. | ||
1750 | |||
1751 | @node VPN | ||
1752 | @subsubsection VPN | ||
1753 | |||
1754 | GNS allows easy access to services provided by the GNUnet Virtual Public | ||
1755 | Network. When the GNS resolver encounters a VPN record it will contact | ||
1756 | the VPN service to try and allocate an IPv4/v6 address (if the queries | ||
1757 | record type is an IP address) that can be used to contact the service. | ||
1758 | |||
1759 | @b{Example}@ | ||
1760 | |||
1761 | I want to provide access to the VPN service "web.gnu." on port 80 on peer | ||
1762 | ABC012:@ | ||
1763 | Name: www; RRType: VPN; Value: 80 ABC012 web.gnu. | ||
1764 | |||
1765 | The peer ABC012 is configured to provide an exit point for the service | ||
1766 | "web.gnu." on port 80 to it's server running locally on port 8080 by | ||
1767 | having the following lines in the @file{gnunet.conf} configuration file: | ||
1768 | |||
1769 | @example | ||
1770 | [web.gnunet.] | ||
1771 | TCP_REDIRECTS = 80:localhost4:8080 | ||
1772 | @end example | ||
1773 | |||
1774 | @node A AAAA and TXT | ||
1775 | @subsubsection A AAAA and TXT | ||
1776 | |||
1777 | Those records work in exactly the same fashion as in traditional DNS. | ||
1778 | |||
1779 | @node CNAME | ||
1780 | @subsubsection CNAME | ||
1781 | |||
1782 | As specified in RFC 1035 whenever a CNAME is encountered the query | ||
1783 | needs to be restarted with the specified name. In GNS a CNAME | ||
1784 | can either be: | ||
1785 | |||
1786 | @itemize @bullet | ||
1787 | @item A zone relative name, | ||
1788 | @item A zkey name or | ||
1789 | @item A DNS name (in which case resolution will continue outside | ||
1790 | of GNS with the systems DNS resolver) | ||
1791 | @end itemize | ||
1792 | |||
1793 | @node GNS2DNS | ||
1794 | @subsubsection GNS2DNS | ||
1795 | |||
1796 | GNS can delegate authority to a legacy DNS zone. For this, the | ||
1797 | name of the DNS nameserver and the name of the DNS zone are | ||
1798 | specified in a GNS2DNS record. | ||
1799 | |||
1800 | @b{Example} | ||
1801 | |||
1802 | @example | ||
1803 | Name: pet; RRType: GNS2DNS; Value: gnunet.org@@a.ns.joker.com | ||
1804 | @end example | ||
1805 | |||
1806 | @noindent | ||
1807 | Any query to @code{pet.gnu} will then be delegated to the DNS server at | ||
1808 | @code{a.ns.joker.com}. For example, | ||
1809 | @code{@uref{http://www.pet.gnu/, www.pet.gnu}} will result in a DNS query | ||
1810 | for @code{@uref{http://www.gnunet.org/, www.gnunet.org}} to the server | ||
1811 | at @code{a.ns.joker.com}. Delegation to DNS via NS records in GNS can | ||
1812 | be useful if you do not want to start resolution in the DNS root zone | ||
1813 | (due to issues such as censorship or availability). | ||
1814 | |||
1815 | Note that you would typically want to use a relative name for the | ||
1816 | nameserver, i.e. | ||
1817 | |||
1818 | @example | ||
1819 | Name: pet; RRType: GNS2DNS; Value: gnunet.org@@ns-joker.+@ | ||
1820 | Name: ns-joker; RRType: A; Value: 184.172.157.218 | ||
1821 | @end example | ||
1822 | |||
1823 | @noindent | ||
1824 | This way, you can avoid involving the DNS hierarchy in the resolution of | ||
1825 | @code{a.ns.joker.com}. In the example above, the problem may not be | ||
1826 | obvious as the nameserver for "gnunet.org" is in the ".com" zone. | ||
1827 | However, imagine the nameserver was "ns.gnunet.org". In this case, | ||
1828 | delegating to "ns.gnunet.org" would mean that despite using GNS, | ||
1829 | censorship in the DNS ".org" zone would still be effective. | ||
1830 | |||
1831 | @node SOA SRV PTR and MX | ||
1832 | @subsubsection SOA SRV PTR and MX | ||
1833 | |||
1834 | The domain names in those records can, again, be either | ||
1835 | |||
1836 | @itemize @bullet | ||
1837 | @item A zone relative name, | ||
1838 | @item A zkey name or | ||
1839 | @item A DNS name | ||
1840 | @end itemize | ||
1841 | |||
1842 | The resolver will expand the zone relative name if possible. | ||
1843 | Note that when using MX records within GNS, the target mail | ||
1844 | server might still refuse to accept e-mails to the resulting | ||
1845 | domain as the name might not match. GNS-enabled mail clients | ||
1846 | should use the ZKEY zone as the destination hostname and | ||
1847 | GNS-enabled mail servers should be configured to accept | ||
1848 | e-mails to the ZKEY-zones of all local users. | ||
1849 | |||
1850 | @node Using the Virtual Public Network | ||
1851 | @section Using the Virtual Public Network | ||
1852 | |||
1853 | @menu | ||
1854 | * Setting up an Exit node:: | ||
1855 | * Fedora and the Firewall:: | ||
1856 | * Setting up VPN node for protocol translation and tunneling:: | ||
1857 | @end menu | ||
1858 | |||
1859 | Using the GNUnet Virtual Public Network (VPN) application you can | ||
1860 | tunnel IP traffic over GNUnet. Moreover, the VPN comes | ||
1861 | with built-in protocol translation and DNS-ALG support, enabling | ||
1862 | IPv4-to-IPv6 protocol translation (in both directions). | ||
1863 | This chapter documents how to use the GNUnet VPN. | ||
1864 | |||
1865 | The first thing to note about the GNUnet VPN is that it is a public | ||
1866 | network. All participating peers can participate and there is no | ||
1867 | secret key to control access. So unlike common virtual private | ||
1868 | networks, the GNUnet VPN is not useful as a means to provide a | ||
1869 | "private" network abstraction over the Internet. The GNUnet VPN | ||
1870 | is a virtual network in the sense that it is an overlay over the | ||
1871 | Internet, using its own routing mechanisms and can also use an | ||
1872 | internal addressing scheme. The GNUnet VPN is an Internet | ||
1873 | underlay --- TCP/IP applications run on top of it. | ||
1874 | |||
1875 | The VPN is currently only supported on GNU/Linux systems. | ||
1876 | Support for operating systems that support TUN (such as FreeBSD) | ||
1877 | should be easy to add (or might not even require any coding at | ||
1878 | all --- we just did not test this so far). Support for other | ||
1879 | operating systems would require re-writing the code to create virtual | ||
1880 | network interfaces and to intercept DNS requests. | ||
1881 | |||
1882 | The VPN does not provide good anonymity. While requests are routed | ||
1883 | over the GNUnet network, other peers can directly see the source | ||
1884 | and destination of each (encapsulated) IP packet. Finally, if you | ||
1885 | use the VPN to access Internet services, the peer sending the | ||
1886 | request to the Internet will be able to observe and even alter | ||
1887 | the IP traffic. We will discuss additional security implications | ||
1888 | of using the VPN later in this chapter. | ||
1889 | |||
1890 | @node Setting up an Exit node | ||
1891 | @subsection Setting up an Exit node | ||
1892 | |||
1893 | Any useful operation with the VPN requires the existence of an exit | ||
1894 | node in the GNUnet Peer-to-Peer network. Exit functionality can only | ||
1895 | be enabled on peers that have regular Internet access. If you want | ||
1896 | to play around with the VPN or support the network, we encourage | ||
1897 | you to setup exit nodes. This chapter documents how to setup an | ||
1898 | exit node. | ||
1899 | |||
1900 | There are four types of exit functions an exit node can provide, | ||
1901 | and using the GNUnet VPN to access the Internet will only work | ||
1902 | nicely if the first three types are provided somewhere in | ||
1903 | the network. The four exit functions are: | ||
1904 | |||
1905 | @itemize @bullet | ||
1906 | @item DNS: allow other peers to use your DNS resolver | ||
1907 | @item IPv4: allow other peers to access your IPv4 Internet connection | ||
1908 | @item IPv6: allow other peers to access your IPv6 Internet connection | ||
1909 | @item Local service: allow other peers to access a specific TCP or | ||
1910 | UDP service your peer is providing | ||
1911 | @end itemize | ||
1912 | |||
1913 | By enabling "exit" in gnunet-setup and checking the respective boxes | ||
1914 | in the "exit" tab, you can easily choose which of the above exit | ||
1915 | functions you want to support. | ||
1916 | |||
1917 | Note, however, that by supporting the first three functions you will | ||
1918 | allow arbitrary other GNUnet users to access the Internet via your | ||
1919 | system. This is somewhat similar to running a Tor exit node. The | ||
1920 | Torproject has a nice article about what to consider if you want | ||
1921 | to do this here. We believe that generally running a DNS exit node | ||
1922 | is completely harmless. | ||
1923 | |||
1924 | The exit node configuration does currently not allow you to restrict the | ||
1925 | Internet traffic that leaves your system. In particular, you cannot | ||
1926 | exclude SMTP traffic (or block port 25) or limit to HTTP traffic using | ||
1927 | the GNUnet configuration. However, you can use your host firewall to | ||
1928 | restrict outbound connections from the virtual tunnel interface. This | ||
1929 | is highly recommended. In the future, we plan to offer a wider range | ||
1930 | of configuration options for exit nodes. | ||
1931 | |||
1932 | Note that by running an exit node GNUnet will configure your kernel | ||
1933 | to perform IP-forwarding (for IPv6) and NAT (for IPv4) so that the | ||
1934 | traffic from the virtual interface can be routed to the Internet. | ||
1935 | In order to provide an IPv6-exit, you need to have a subnet routed | ||
1936 | to your host's external network interface and assign a subrange of | ||
1937 | that subnet to the GNUnet exit's TUN interface. | ||
1938 | |||
1939 | When running a local service, you should make sure that the local | ||
1940 | service is (also) bound to the IP address of your EXIT interface | ||
1941 | (i.e. 169.254.86.1). It will NOT work if your local service is | ||
1942 | just bound to loopback. You may also want to create a "VPN" record | ||
1943 | in your zone of the GNU Name System to make it easy for others to | ||
1944 | access your service via a name instead of just the full service | ||
1945 | descriptor. Note that the identifier you assign the service can | ||
1946 | serve as a passphrase or shared secret, clients connecting to the | ||
1947 | service must somehow learn the service's name. VPN records in the | ||
1948 | GNU Name System can make this easier. | ||
1949 | |||
1950 | @node Fedora and the Firewall | ||
1951 | @subsection Fedora and the Firewall | ||
1952 | |||
1953 | |||
1954 | When using an exit node on Fedora 15, the standard firewall can | ||
1955 | create trouble even when not really exiting the local system! | ||
1956 | For IPv4, the standard rules seem fine. However, for IPv6 the | ||
1957 | standard rules prohibit traffic from the network range of the | ||
1958 | virtual interface created by the exit daemon to the local IPv6 | ||
1959 | address of the same interface (which is essentially loopback | ||
1960 | traffic, so you might suspect that a standard firewall would | ||
1961 | leave this traffic alone). However, as somehow for IPv6 the | ||
1962 | traffic is not recognized as originating from the local | ||
1963 | system (and as the connection is not already "established"), | ||
1964 | the firewall drops the traffic. You should still get ICMPv6 | ||
1965 | packets back, but that's obviously not very useful. | ||
1966 | |||
1967 | Possible ways to fix this include disabling the firewall (do you | ||
1968 | have a good reason for having it on?) or disabling the firewall | ||
1969 | at least for the GNUnet exit interface (or the respective | ||
1970 | IPv4/IPv6 address range). The best way to diagnose these kinds | ||
1971 | of problems in general involves setting the firewall to REJECT | ||
1972 | instead of DROP and to watch the traffic using wireshark | ||
1973 | (or tcpdump) to see if ICMP messages are generated when running | ||
1974 | some tests that should work. | ||
1975 | |||
1976 | @node Setting up VPN node for protocol translation and tunneling | ||
1977 | @subsection Setting up VPN node for protocol translation and tunneling | ||
1978 | |||
1979 | |||
1980 | The GNUnet VPN/PT subsystem enables you to tunnel IP traffic over the | ||
1981 | VPN to an exit node, from where it can then be forwarded to the | ||
1982 | Internet. This section documents how to setup VPN/PT on a node. | ||
1983 | Note that you can enable both the VPN and an exit on the same peer. | ||
1984 | In this case, IP traffic from your system may enter your peer's VPN | ||
1985 | and leave your peer's exit. This can be useful as a means to do | ||
1986 | protocol translation. For example, you might have an application that | ||
1987 | supports only IPv4 but needs to access an IPv6-only site. In this case, | ||
1988 | GNUnet would perform 4to6 protocol translation between the VPN (IPv4) | ||
1989 | and the Exit (IPv6). Similarly, 6to4 protocol translation is also | ||
1990 | possible. However, the primary use for GNUnet would be to access | ||
1991 | an Internet service running with an IP version that is not supported | ||
1992 | by your ISP. In this case, your IP traffic would be routed via GNUnet | ||
1993 | to a peer that has access to the Internet with the desired IP version. | ||
1994 | |||
1995 | Setting up an entry node into the GNUnet VPN primarily requires you | ||
1996 | to enable the "VPN/PT" option in "gnunet-setup". This will launch the | ||
1997 | "gnunet-service-vpn", "gnunet-service-dns" and "gnunet-daemon-pt" | ||
1998 | processes. The "gnunet-service-vpn" will create a virtual interface | ||
1999 | which will be used as the target for your IP traffic that enters the | ||
2000 | VPN. Additionally, a second virtual interface will be created by | ||
2001 | the "gnunet-service-dns" for your DNS traffic. You will then need to | ||
2002 | specify which traffic you want to tunnel over GNUnet. If your ISP only | ||
2003 | provides you with IPv4 or IPv6-access, you may choose to tunnel the | ||
2004 | other IP protocol over the GNUnet VPN. If you do not have an ISP | ||
2005 | (and are connected to other GNUnet peers via WLAN), you can also | ||
2006 | choose to tunnel all IP traffic over GNUnet. This might also provide | ||
2007 | you with some anonymity. After you enable the respective options | ||
2008 | and restart your peer, your Internet traffic should be tunneled | ||
2009 | over the GNUnet VPN. | ||
2010 | |||
2011 | The GNUnet VPN uses DNS-ALG to hijack your IP traffic. Whenever an | ||
2012 | application resolves a hostname (i.e. 'gnunet.org'), the | ||
2013 | "gnunet-daemon-pt" will instruct the "gnunet-service-dns" to intercept | ||
2014 | the request (possibly route it over GNUnet as well) and replace the | ||
2015 | normal answer with an IP in the range of the VPN's interface. | ||
2016 | "gnunet-daemon-pt" will then tell "gnunet-service-vpn" to forward all | ||
2017 | traffic it receives on the TUN interface via the VPN to the original | ||
2018 | destination. | ||
2019 | |||
2020 | For applications that do not use DNS, you can also manually create | ||
2021 | such a mapping using the gnunet-vpn command-line tool. Here, you | ||
2022 | specfiy the desired address family of the result (i.e. "-4"), and the | ||
2023 | intended target IP on the Internet ("-i 131.159.74.67") and | ||
2024 | "gnunet-vpn" will tell you which IP address in the range of your | ||
2025 | VPN tunnel was mapped. | ||
2026 | |||
2027 | @command{gnunet-vpn} can also be used to access "internal" services | ||
2028 | offered by GNUnet nodes. So if you happen to know a peer and a | ||
2029 | service offered by that peer, you can create an IP tunnel to | ||
2030 | that peer by specifying the peer's identity, service name and | ||
2031 | protocol (--tcp or --udp) and you will again receive an IP address | ||
2032 | that will terminate at the respective peer's service. | ||
diff --git a/doc/documentation/chapters/vocabulary.texi b/doc/documentation/chapters/vocabulary.texi new file mode 100644 index 000000000..85b40b17b --- /dev/null +++ b/doc/documentation/chapters/vocabulary.texi | |||
@@ -0,0 +1,72 @@ | |||
1 | @node Vocabulary | ||
2 | @chapter Vocabulary | ||
3 | |||
4 | @menu | ||
5 | * Definitions abbreviations and acronyms:: | ||
6 | * Words and characters:: | ||
7 | * Technical Assumptions:: | ||
8 | @end menu | ||
9 | |||
10 | Throughout this Reference Manual we will use certain words and characters | ||
11 | which are listed in this introductionary chapter. | ||
12 | |||
13 | @node Definitions abbreviations and acronyms | ||
14 | @section Definitions abbreviations and acronyms | ||
15 | |||
16 | @menu | ||
17 | * Definitions:: | ||
18 | @end menu | ||
19 | |||
20 | @node Definitions | ||
21 | @subsection Defitions | ||
22 | |||
23 | Throughout this Reference Manual, the following terms and definitions | ||
24 | apply. | ||
25 | |||
26 | @node Words and characters | ||
27 | @section Words and characters | ||
28 | |||
29 | @enumerate | ||
30 | @item | ||
31 | In chapter Installation Handbook, | ||
32 | ``@command{#}'' in example code blocks describes commands executed as root | ||
33 | |||
34 | @example | ||
35 | # echo "I am root" | ||
36 | I am root | ||
37 | @end example | ||
38 | |||
39 | @item | ||
40 | However, in the chapter GNUnet C Tutorial | ||
41 | ``@command{#}'' in example code blocks describes commands, ie comments. | ||
42 | |||
43 | @example | ||
44 | # Do the foobar thing: | ||
45 | $ make foobar | ||
46 | @end example | ||
47 | |||
48 | @item | ||
49 | Dollarsign ``@command{$}'' in example code blocks describes commands you | ||
50 | execute as unprivileged users. | ||
51 | |||
52 | @example | ||
53 | $ cd foo; ./configure --example-switch | ||
54 | @end example | ||
55 | |||
56 | @item | ||
57 | Backslash ``@command{\}'' describes linebreaks. | ||
58 | |||
59 | @example | ||
60 | ./configure --foo --bar --baz \ | ||
61 | --short-loop | ||
62 | @end example | ||
63 | |||
64 | ...expands to @code{./configure --foo --bar --baz --short-loop} | ||
65 | |||
66 | @end enumerate | ||
67 | |||
68 | @node Technical Assumptions | ||
69 | @section Technical Assumptions | ||
70 | |||
71 | @c Is it really assuming Bash (ie Bash extensions of POSIX being used)? | ||
72 | The shell on GNU systems is assumed to be Bash. | ||
diff --git a/doc/documentation/docstyle.css b/doc/documentation/docstyle.css new file mode 100644 index 000000000..8719248d0 --- /dev/null +++ b/doc/documentation/docstyle.css | |||
@@ -0,0 +1,76 @@ | |||
1 | html, body { | ||
2 | font-size: 1em; | ||
3 | text-align: left; | ||
4 | text-decoration: none; | ||
5 | } | ||
6 | html { background-color: #e7e7e7; } | ||
7 | |||
8 | body { | ||
9 | max-width: 74.92em; | ||
10 | margin: 0 auto; | ||
11 | padding: .5em 1em 1em 1em; | ||
12 | background-color: white; | ||
13 | border: .1em solid #c0c0c0; | ||
14 | } | ||
15 | |||
16 | h1, h2, h3, h4 { color: #333; } | ||
17 | h5, h6, dt { color: #222; } | ||
18 | |||
19 | |||
20 | a h3 { | ||
21 | color: #005090; | ||
22 | } | ||
23 | |||
24 | a[href] { color: #005090; } | ||
25 | a[href]:visited { color: #100070; } | ||
26 | a[href]:active, a[href]:hover { | ||
27 | color: #100070; | ||
28 | text-decoration: none; | ||
29 | } | ||
30 | |||
31 | .linkrow { | ||
32 | margin: 3em 0; | ||
33 | } | ||
34 | |||
35 | .linkrow { | ||
36 | text-align: center; | ||
37 | } | ||
38 | |||
39 | div.example { padding: .8em 1.2em .4em; } | ||
40 | pre.example { padding: .8em 1.2em; } | ||
41 | div.example, pre.example { | ||
42 | margin: 1em 0 1em 3% ; | ||
43 | -webkit-border-radius: .3em; | ||
44 | -moz-border-radius: .3em; | ||
45 | border-radius: .3em; | ||
46 | border: 1px solid #d4cbb6; | ||
47 | background-color: #f2efe4; | ||
48 | } | ||
49 | div.example > pre.example { | ||
50 | padding: 0 0 .4em; | ||
51 | margin: 0; | ||
52 | border: none; | ||
53 | } | ||
54 | |||
55 | |||
56 | /* This makes the very long tables of contents in Gnulib and other | ||
57 | manuals easier to read. */ | ||
58 | .contents ul, .shortcontents ul { font-weight: bold; } | ||
59 | .contents ul ul, .shortcontents ul ul { font-weight: normal; } | ||
60 | .contents ul { list-style: none; } | ||
61 | |||
62 | /* For colored navigation bars (Emacs manual): make the bar extend | ||
63 | across the whole width of the page and give it a decent height. */ | ||
64 | .header, .node { margin: 0 -1em; padding: 0 1em; } | ||
65 | .header p, .node p { line-height: 2em; } | ||
66 | |||
67 | /* For navigation links */ | ||
68 | .node a, .header a { display: inline-block; line-height: 2em; } | ||
69 | .node a:hover, .header a:hover { background: #f2efe4; } | ||
70 | |||
71 | table.cartouche { | ||
72 | border-collapse: collapse; | ||
73 | border-color: darkred; | ||
74 | border-style: solid; | ||
75 | border-width: 3px; | ||
76 | } | ||
diff --git a/doc/fdl-1.3.texi b/doc/documentation/fdl-1.3.texi index cb71f05a1..cb71f05a1 100644 --- a/doc/fdl-1.3.texi +++ b/doc/documentation/fdl-1.3.texi | |||
diff --git a/doc/documentation/gendocs.sh b/doc/documentation/gendocs.sh new file mode 100755 index 000000000..3b71b36a2 --- /dev/null +++ b/doc/documentation/gendocs.sh | |||
@@ -0,0 +1,504 @@ | |||
1 | #!/bin/sh -e | ||
2 | # gendocs.sh -- generate a GNU manual in many formats. This script is | ||
3 | # mentioned in maintain.texi. See the help message below for usage details. | ||
4 | |||
5 | scriptversion=2016-12-31.18 | ||
6 | |||
7 | # Copyright 2003-2017 Free Software Foundation, Inc. | ||
8 | # | ||
9 | # This program is free software: you can redistribute it and/or modify | ||
10 | # it under the terms of the GNU General Public License as published by | ||
11 | # the Free Software Foundation; either version 3 of the License, or | ||
12 | # (at your option) any later version. | ||
13 | # | ||
14 | # This program is distributed in the hope that it will be useful, | ||
15 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
16 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
17 | # GNU General Public License for more details. | ||
18 | # | ||
19 | # You should have received a copy of the GNU General Public License | ||
20 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
21 | # | ||
22 | # Original author: Mohit Agarwal. | ||
23 | # Send bug reports and any other correspondence to bug-gnulib@gnu.org. | ||
24 | # | ||
25 | # The latest version of this script, and the companion template, is | ||
26 | # available from the Gnulib repository: | ||
27 | # | ||
28 | # http://git.savannah.gnu.org/cgit/gnulib.git/tree/build-aux/gendocs.sh | ||
29 | # http://git.savannah.gnu.org/cgit/gnulib.git/tree/doc/gendocs_template | ||
30 | |||
31 | # TODO: | ||
32 | # - image importing was only implemented for HTML generated by | ||
33 | # makeinfo. But it should be simple enough to adjust. | ||
34 | # - images are not imported in the source tarball. All the needed | ||
35 | # formats (PDF, PNG, etc.) should be included. | ||
36 | |||
37 | prog=`basename "$0"` | ||
38 | srcdir=`pwd` | ||
39 | |||
40 | scripturl="http://git.savannah.gnu.org/cgit/gnulib.git/plain/build-aux/gendocs.sh" | ||
41 | templateurl="http://git.savannah.gnu.org/cgit/gnulib.git/plain/doc/gendocs_template" | ||
42 | |||
43 | : ${SETLANG="env LANG= LC_MESSAGES= LC_ALL= LANGUAGE="} | ||
44 | : ${MAKEINFO="makeinfo"} | ||
45 | : ${TEXI2DVI="texi2dvi"} | ||
46 | : ${DOCBOOK2HTML="docbook2html"} | ||
47 | : ${DOCBOOK2PDF="docbook2pdf"} | ||
48 | : ${DOCBOOK2TXT="docbook2txt"} | ||
49 | : ${GENDOCS_TEMPLATE_DIR="."} | ||
50 | : ${PERL='perl'} | ||
51 | : ${TEXI2HTML="texi2html"} | ||
52 | unset CDPATH | ||
53 | unset use_texi2html | ||
54 | |||
55 | MANUAL_TITLE= | ||
56 | PACKAGE= | ||
57 | EMAIL=webmasters@gnu.org # please override with --email | ||
58 | commonarg= # passed to all makeinfo/texi2html invcations. | ||
59 | dirargs= # passed to all tools (-I dir). | ||
60 | dirs= # -I directories. | ||
61 | htmlarg="--css-ref=/software/gnulib/manual.css -c TOP_NODE_UP_URL=/manual" | ||
62 | infoarg=--no-split | ||
63 | generate_ascii=true | ||
64 | generate_html=true | ||
65 | generate_info=true | ||
66 | generate_tex=true | ||
67 | outdir=manual | ||
68 | source_extra= | ||
69 | split=node | ||
70 | srcfile= | ||
71 | texarg="-t @finalout" | ||
72 | |||
73 | version="gendocs.sh $scriptversion | ||
74 | |||
75 | Copyright 2017 Free Software Foundation, Inc. | ||
76 | There is NO warranty. You may redistribute this software | ||
77 | under the terms of the GNU General Public License. | ||
78 | For more information about these matters, see the files named COPYING." | ||
79 | |||
80 | usage="Usage: $prog [OPTION]... PACKAGE MANUAL-TITLE | ||
81 | |||
82 | Generate output in various formats from PACKAGE.texinfo (or .texi or | ||
83 | .txi) source. See the GNU Maintainers document for a more extensive | ||
84 | discussion: | ||
85 | http://www.gnu.org/prep/maintain_toc.html | ||
86 | |||
87 | Options: | ||
88 | --email ADR use ADR as contact in generated web pages; always give this. | ||
89 | |||
90 | -s SRCFILE read Texinfo from SRCFILE, instead of PACKAGE.{texinfo|texi|txi} | ||
91 | -o OUTDIR write files into OUTDIR, instead of manual/. | ||
92 | -I DIR append DIR to the Texinfo search path. | ||
93 | --common ARG pass ARG in all invocations. | ||
94 | --html ARG pass ARG to makeinfo or texi2html for HTML targets, | ||
95 | instead of '$htmlarg'. | ||
96 | --info ARG pass ARG to makeinfo for Info, instead of --no-split. | ||
97 | --no-ascii skip generating the plain text output. | ||
98 | --no-html skip generating the html output. | ||
99 | --no-info skip generating the info output. | ||
100 | --no-tex skip generating the dvi and pdf output. | ||
101 | --source ARG include ARG in tar archive of sources. | ||
102 | --split HOW make split HTML by node, section, chapter; default node. | ||
103 | --tex ARG pass ARG to texi2dvi for DVI and PDF, instead of -t @finalout. | ||
104 | |||
105 | --texi2html use texi2html to make HTML target, with all split versions. | ||
106 | --docbook convert through DocBook too (xml, txt, html, pdf). | ||
107 | |||
108 | --help display this help and exit successfully. | ||
109 | --version display version information and exit successfully. | ||
110 | |||
111 | Simple example: $prog --email bug-gnu-emacs@gnu.org emacs \"GNU Emacs Manual\" | ||
112 | |||
113 | Typical sequence: | ||
114 | cd PACKAGESOURCE/doc | ||
115 | wget \"$scripturl\" | ||
116 | wget \"$templateurl\" | ||
117 | $prog --email BUGLIST MANUAL \"GNU MANUAL - One-line description\" | ||
118 | |||
119 | Output will be in a new subdirectory \"manual\" (by default; | ||
120 | use -o OUTDIR to override). Move all the new files into your web CVS | ||
121 | tree, as explained in the Web Pages node of maintain.texi. | ||
122 | |||
123 | Please use the --email ADDRESS option so your own bug-reporting | ||
124 | address will be used in the generated HTML pages. | ||
125 | |||
126 | MANUAL-TITLE is included as part of the HTML <title> of the overall | ||
127 | manual/index.html file. It should include the name of the package being | ||
128 | documented. manual/index.html is created by substitution from the file | ||
129 | $GENDOCS_TEMPLATE_DIR/gendocs_template. (Feel free to modify the | ||
130 | generic template for your own purposes.) | ||
131 | |||
132 | If you have several manuals, you'll need to run this script several | ||
133 | times with different MANUAL values, specifying a different output | ||
134 | directory with -o each time. Then write (by hand) an overall index.html | ||
135 | with links to them all. | ||
136 | |||
137 | If a manual's Texinfo sources are spread across several directories, | ||
138 | first copy or symlink all Texinfo sources into a single directory. | ||
139 | (Part of the script's work is to make a tar.gz of the sources.) | ||
140 | |||
141 | As implied above, by default monolithic Info files are generated. | ||
142 | If you want split Info, or other Info options, use --info to override. | ||
143 | |||
144 | You can set the environment variables MAKEINFO, TEXI2DVI, TEXI2HTML, | ||
145 | and PERL to control the programs that get executed, and | ||
146 | GENDOCS_TEMPLATE_DIR to control where the gendocs_template file is | ||
147 | looked for. With --docbook, the environment variables DOCBOOK2HTML, | ||
148 | DOCBOOK2PDF, and DOCBOOK2TXT are also consulted. | ||
149 | |||
150 | By default, makeinfo and texi2dvi are run in the default (English) | ||
151 | locale, since that's the language of most Texinfo manuals. If you | ||
152 | happen to have a non-English manual and non-English web site, see the | ||
153 | SETLANG setting in the source. | ||
154 | |||
155 | Email bug reports or enhancement requests to bug-gnulib@gnu.org. | ||
156 | " | ||
157 | |||
158 | while test $# -gt 0; do | ||
159 | case $1 in | ||
160 | -s) shift; srcfile=$1;; | ||
161 | -o) shift; outdir=$1;; | ||
162 | -I) shift; dirargs="$dirargs -I '$1'"; dirs="$dirs $1";; | ||
163 | --common) shift; commonarg=$1;; | ||
164 | --docbook) docbook=yes;; | ||
165 | --email) shift; EMAIL=$1;; | ||
166 | --html) shift; htmlarg=$1;; | ||
167 | --info) shift; infoarg=$1;; | ||
168 | --no-ascii) generate_ascii=false;; | ||
169 | --no-html) generate_ascii=false;; | ||
170 | --no-info) generate_info=false;; | ||
171 | --no-tex) generate_tex=false;; | ||
172 | --source) shift; source_extra=$1;; | ||
173 | --split) shift; split=$1;; | ||
174 | --tex) shift; texarg=$1;; | ||
175 | --texi2html) use_texi2html=1;; | ||
176 | |||
177 | --help) echo "$usage"; exit 0;; | ||
178 | --version) echo "$version"; exit 0;; | ||
179 | -*) | ||
180 | echo "$0: Unknown option \`$1'." >&2 | ||
181 | echo "$0: Try \`--help' for more information." >&2 | ||
182 | exit 1;; | ||
183 | *) | ||
184 | if test -z "$PACKAGE"; then | ||
185 | PACKAGE=$1 | ||
186 | elif test -z "$MANUAL_TITLE"; then | ||
187 | MANUAL_TITLE=$1 | ||
188 | else | ||
189 | echo "$0: extra non-option argument \`$1'." >&2 | ||
190 | exit 1 | ||
191 | fi;; | ||
192 | esac | ||
193 | shift | ||
194 | done | ||
195 | |||
196 | # makeinfo uses the dirargs, but texi2dvi doesn't. | ||
197 | commonarg=" $dirargs $commonarg" | ||
198 | |||
199 | # For most of the following, the base name is just $PACKAGE | ||
200 | base=$PACKAGE | ||
201 | |||
202 | if test -n "$srcfile"; then | ||
203 | # but here, we use the basename of $srcfile | ||
204 | base=`basename "$srcfile"` | ||
205 | case $base in | ||
206 | *.txi|*.texi|*.texinfo) base=`echo "$base"|sed 's/\.[texinfo]*$//'`;; | ||
207 | esac | ||
208 | PACKAGE=$base | ||
209 | elif test -s "$srcdir/$PACKAGE.texinfo"; then | ||
210 | srcfile=$srcdir/$PACKAGE.texinfo | ||
211 | elif test -s "$srcdir/$PACKAGE.texi"; then | ||
212 | srcfile=$srcdir/$PACKAGE.texi | ||
213 | elif test -s "$srcdir/$PACKAGE.txi"; then | ||
214 | srcfile=$srcdir/$PACKAGE.txi | ||
215 | else | ||
216 | echo "$0: cannot find .texinfo or .texi or .txi for $PACKAGE in $srcdir." >&2 | ||
217 | exit 1 | ||
218 | fi | ||
219 | |||
220 | if test ! -r $GENDOCS_TEMPLATE_DIR/gendocs_template; then | ||
221 | echo "$0: cannot read $GENDOCS_TEMPLATE_DIR/gendocs_template." >&2 | ||
222 | echo "$0: it is available from $templateurl." >&2 | ||
223 | exit 1 | ||
224 | fi | ||
225 | |||
226 | # Function to return size of $1 in something resembling kilobytes. | ||
227 | calcsize() | ||
228 | { | ||
229 | size=`ls -ksl $1 | awk '{print $1}'` | ||
230 | echo $size | ||
231 | } | ||
232 | |||
233 | # copy_images OUTDIR HTML-FILE... | ||
234 | # ------------------------------- | ||
235 | # Copy all the images needed by the HTML-FILEs into OUTDIR. | ||
236 | # Look for them in . and the -I directories; this is simpler than what | ||
237 | # makeinfo supports with -I, but hopefully it will suffice. | ||
238 | copy_images() | ||
239 | { | ||
240 | local odir | ||
241 | odir=$1 | ||
242 | shift | ||
243 | $PERL -n -e " | ||
244 | BEGIN { | ||
245 | \$me = '$prog'; | ||
246 | \$odir = '$odir'; | ||
247 | @dirs = qw(. $dirs); | ||
248 | } | ||
249 | " -e ' | ||
250 | /<img src="(.*?)"/g && ++$need{$1}; | ||
251 | |||
252 | END { | ||
253 | #print "$me: @{[keys %need]}\n"; # for debugging, show images found. | ||
254 | FILE: for my $f (keys %need) { | ||
255 | for my $d (@dirs) { | ||
256 | if (-f "$d/$f") { | ||
257 | use File::Basename; | ||
258 | my $dest = dirname ("$odir/$f"); | ||
259 | # | ||
260 | use File::Path; | ||
261 | -d $dest || mkpath ($dest) | ||
262 | || die "$me: cannot mkdir $dest: $!\n"; | ||
263 | # | ||
264 | use File::Copy; | ||
265 | copy ("$d/$f", $dest) | ||
266 | || die "$me: cannot copy $d/$f to $dest: $!\n"; | ||
267 | next FILE; | ||
268 | } | ||
269 | } | ||
270 | die "$me: $ARGV: cannot find image $f\n"; | ||
271 | } | ||
272 | } | ||
273 | ' -- "$@" || exit 1 | ||
274 | } | ||
275 | |||
276 | case $outdir in | ||
277 | /*) abs_outdir=$outdir;; | ||
278 | *) abs_outdir=$srcdir/$outdir;; | ||
279 | esac | ||
280 | |||
281 | echo "Making output for $srcfile" | ||
282 | echo " in `pwd`" | ||
283 | mkdir -p "$outdir/" | ||
284 | |||
285 | # | ||
286 | if $generate_info; then | ||
287 | cmd="$SETLANG $MAKEINFO -o $PACKAGE.info $commonarg $infoarg \"$srcfile\"" | ||
288 | echo "Generating info... ($cmd)" | ||
289 | rm -f $PACKAGE.info* # get rid of any strays | ||
290 | eval "$cmd" | ||
291 | tar czf "$outdir/$PACKAGE.info.tar.gz" $PACKAGE.info* | ||
292 | ls -l "$outdir/$PACKAGE.info.tar.gz" | ||
293 | info_tgz_size=`calcsize "$outdir/$PACKAGE.info.tar.gz"` | ||
294 | # do not mv the info files, there's no point in having them available | ||
295 | # separately on the web. | ||
296 | fi # end info | ||
297 | |||
298 | # | ||
299 | if $generate_tex; then | ||
300 | cmd="$SETLANG $TEXI2DVI $dirargs $texarg \"$srcfile\"" | ||
301 | printf "\nGenerating dvi... ($cmd)\n" | ||
302 | eval "$cmd" | ||
303 | # compress/finish dvi: | ||
304 | gzip -f -9 $PACKAGE.dvi | ||
305 | dvi_gz_size=`calcsize $PACKAGE.dvi.gz` | ||
306 | mv $PACKAGE.dvi.gz "$outdir/" | ||
307 | ls -l "$outdir/$PACKAGE.dvi.gz" | ||
308 | |||
309 | cmd="$SETLANG $TEXI2DVI --pdf $dirargs $texarg \"$srcfile\"" | ||
310 | printf "\nGenerating pdf... ($cmd)\n" | ||
311 | eval "$cmd" | ||
312 | pdf_size=`calcsize $PACKAGE.pdf` | ||
313 | mv $PACKAGE.pdf "$outdir/" | ||
314 | ls -l "$outdir/$PACKAGE.pdf" | ||
315 | fi # end tex (dvi + pdf) | ||
316 | |||
317 | # | ||
318 | if $generate_ascii; then | ||
319 | opt="-o $PACKAGE.txt --no-split --no-headers $commonarg" | ||
320 | cmd="$SETLANG $MAKEINFO $opt \"$srcfile\"" | ||
321 | printf "\nGenerating ascii... ($cmd)\n" | ||
322 | eval "$cmd" | ||
323 | ascii_size=`calcsize $PACKAGE.txt` | ||
324 | gzip -f -9 -c $PACKAGE.txt >"$outdir/$PACKAGE.txt.gz" | ||
325 | ascii_gz_size=`calcsize "$outdir/$PACKAGE.txt.gz"` | ||
326 | mv $PACKAGE.txt "$outdir/" | ||
327 | ls -l "$outdir/$PACKAGE.txt" "$outdir/$PACKAGE.txt.gz" | ||
328 | fi | ||
329 | |||
330 | # | ||
331 | |||
332 | if $generate_html; then | ||
333 | # Split HTML at level $1. Used for texi2html. | ||
334 | html_split() | ||
335 | { | ||
336 | opt="--split=$1 --node-files $commonarg $htmlarg" | ||
337 | cmd="$SETLANG $TEXI2HTML --output $PACKAGE.html $opt \"$srcfile\"" | ||
338 | printf "\nGenerating html by $1... ($cmd)\n" | ||
339 | eval "$cmd" | ||
340 | split_html_dir=$PACKAGE.html | ||
341 | ( | ||
342 | cd ${split_html_dir} || exit 1 | ||
343 | ln -sf ${PACKAGE}.html index.html | ||
344 | tar -czf "$abs_outdir/${PACKAGE}.html_$1.tar.gz" -- *.html | ||
345 | ) | ||
346 | eval html_$1_tgz_size=`calcsize "$outdir/${PACKAGE}.html_$1.tar.gz"` | ||
347 | rm -f "$outdir"/html_$1/*.html | ||
348 | mkdir -p "$outdir/html_$1/" | ||
349 | mv ${split_html_dir}/*.html "$outdir/html_$1/" | ||
350 | rmdir ${split_html_dir} | ||
351 | } | ||
352 | |||
353 | if test -z "$use_texi2html"; then | ||
354 | opt="--no-split --html -o $PACKAGE.html $commonarg $htmlarg" | ||
355 | cmd="$SETLANG $MAKEINFO $opt \"$srcfile\"" | ||
356 | printf "\nGenerating monolithic html... ($cmd)\n" | ||
357 | rm -rf $PACKAGE.html # in case a directory is left over | ||
358 | eval "$cmd" | ||
359 | html_mono_size=`calcsize $PACKAGE.html` | ||
360 | gzip -f -9 -c $PACKAGE.html >"$outdir/$PACKAGE.html.gz" | ||
361 | html_mono_gz_size=`calcsize "$outdir/$PACKAGE.html.gz"` | ||
362 | copy_images "$outdir/" $PACKAGE.html | ||
363 | mv $PACKAGE.html "$outdir/" | ||
364 | ls -l "$outdir/$PACKAGE.html" "$outdir/$PACKAGE.html.gz" | ||
365 | |||
366 | # Before Texinfo 5.0, makeinfo did not accept a --split=HOW option, | ||
367 | # it just always split by node. So if we're splitting by node anyway, | ||
368 | # leave it out. | ||
369 | if test "x$split" = xnode; then | ||
370 | split_arg= | ||
371 | else | ||
372 | split_arg=--split=$split | ||
373 | fi | ||
374 | # | ||
375 | opt="--html -o $PACKAGE.html $split_arg $commonarg $htmlarg" | ||
376 | cmd="$SETLANG $MAKEINFO $opt \"$srcfile\"" | ||
377 | printf "\nGenerating html by $split... ($cmd)\n" | ||
378 | eval "$cmd" | ||
379 | split_html_dir=$PACKAGE.html | ||
380 | copy_images $split_html_dir/ $split_html_dir/*.html | ||
381 | ( | ||
382 | cd $split_html_dir || exit 1 | ||
383 | tar -czf "$abs_outdir/$PACKAGE.html_$split.tar.gz" -- * | ||
384 | ) | ||
385 | eval \ | ||
386 | html_${split}_tgz_size=`calcsize "$outdir/$PACKAGE.html_$split.tar.gz"` | ||
387 | rm -rf "$outdir/html_$split/" | ||
388 | mv $split_html_dir "$outdir/html_$split/" | ||
389 | du -s "$outdir/html_$split/" | ||
390 | ls -l "$outdir/$PACKAGE.html_$split.tar.gz" | ||
391 | |||
392 | else # use texi2html: | ||
393 | opt="--output $PACKAGE.html $commonarg $htmlarg" | ||
394 | cmd="$SETLANG $TEXI2HTML $opt \"$srcfile\"" | ||
395 | printf "\nGenerating monolithic html with texi2html... ($cmd)\n" | ||
396 | rm -rf $PACKAGE.html # in case a directory is left over | ||
397 | eval "$cmd" | ||
398 | html_mono_size=`calcsize $PACKAGE.html` | ||
399 | gzip -f -9 -c $PACKAGE.html >"$outdir/$PACKAGE.html.gz" | ||
400 | html_mono_gz_size=`calcsize "$outdir/$PACKAGE.html.gz"` | ||
401 | mv $PACKAGE.html "$outdir/" | ||
402 | |||
403 | html_split node | ||
404 | html_split chapter | ||
405 | html_split section | ||
406 | fi | ||
407 | fi # end html | ||
408 | |||
409 | # | ||
410 | printf "\nMaking .tar.gz for sources...\n" | ||
411 | d=`dirname $srcfile` | ||
412 | ( | ||
413 | cd "$d" | ||
414 | srcfiles=`ls -d *.texinfo *.texi *.txi *.eps $source_extra 2>/dev/null` || true | ||
415 | tar czfh "$abs_outdir/$PACKAGE.texi.tar.gz" $srcfiles | ||
416 | ls -l "$abs_outdir/$PACKAGE.texi.tar.gz" | ||
417 | ) | ||
418 | texi_tgz_size=`calcsize "$outdir/$PACKAGE.texi.tar.gz"` | ||
419 | |||
420 | # | ||
421 | # Do everything again through docbook. | ||
422 | if test -n "$docbook"; then | ||
423 | opt="-o - --docbook $commonarg" | ||
424 | cmd="$SETLANG $MAKEINFO $opt \"$srcfile\" >${srcdir}/$PACKAGE-db.xml" | ||
425 | printf "\nGenerating docbook XML... ($cmd)\n" | ||
426 | eval "$cmd" | ||
427 | docbook_xml_size=`calcsize $PACKAGE-db.xml` | ||
428 | gzip -f -9 -c $PACKAGE-db.xml >"$outdir/$PACKAGE-db.xml.gz" | ||
429 | docbook_xml_gz_size=`calcsize "$outdir/$PACKAGE-db.xml.gz"` | ||
430 | mv $PACKAGE-db.xml "$outdir/" | ||
431 | |||
432 | split_html_db_dir=html_node_db | ||
433 | opt="$commonarg -o $split_html_db_dir" | ||
434 | cmd="$DOCBOOK2HTML $opt \"${outdir}/$PACKAGE-db.xml\"" | ||
435 | printf "\nGenerating docbook HTML... ($cmd)\n" | ||
436 | eval "$cmd" | ||
437 | ( | ||
438 | cd ${split_html_db_dir} || exit 1 | ||
439 | tar -czf "$abs_outdir/${PACKAGE}.html_node_db.tar.gz" -- *.html | ||
440 | ) | ||
441 | html_node_db_tgz_size=`calcsize "$outdir/${PACKAGE}.html_node_db.tar.gz"` | ||
442 | rm -f "$outdir"/html_node_db/*.html | ||
443 | mkdir -p "$outdir/html_node_db" | ||
444 | mv ${split_html_db_dir}/*.html "$outdir/html_node_db/" | ||
445 | rmdir ${split_html_db_dir} | ||
446 | |||
447 | cmd="$DOCBOOK2TXT \"${outdir}/$PACKAGE-db.xml\"" | ||
448 | printf "\nGenerating docbook ASCII... ($cmd)\n" | ||
449 | eval "$cmd" | ||
450 | docbook_ascii_size=`calcsize $PACKAGE-db.txt` | ||
451 | mv $PACKAGE-db.txt "$outdir/" | ||
452 | |||
453 | cmd="$DOCBOOK2PDF \"${outdir}/$PACKAGE-db.xml\"" | ||
454 | printf "\nGenerating docbook PDF... ($cmd)\n" | ||
455 | eval "$cmd" | ||
456 | docbook_pdf_size=`calcsize $PACKAGE-db.pdf` | ||
457 | mv $PACKAGE-db.pdf "$outdir/" | ||
458 | fi | ||
459 | |||
460 | # | ||
461 | printf "\nMaking index.html for $PACKAGE...\n" | ||
462 | if test -z "$use_texi2html"; then | ||
463 | CONDS="/%%IF *HTML_SECTION%%/,/%%ENDIF *HTML_SECTION%%/d;\ | ||
464 | /%%IF *HTML_CHAPTER%%/,/%%ENDIF *HTML_CHAPTER%%/d" | ||
465 | else | ||
466 | # should take account of --split here. | ||
467 | CONDS="/%%ENDIF.*%%/d;/%%IF *HTML_SECTION%%/d;/%%IF *HTML_CHAPTER%%/d" | ||
468 | fi | ||
469 | |||
470 | curdate=`$SETLANG date '+%B %d, %Y'` | ||
471 | sed \ | ||
472 | -e "s!%%TITLE%%!$MANUAL_TITLE!g" \ | ||
473 | -e "s!%%EMAIL%%!$EMAIL!g" \ | ||
474 | -e "s!%%PACKAGE%%!$PACKAGE!g" \ | ||
475 | -e "s!%%DATE%%!$curdate!g" \ | ||
476 | -e "s!%%HTML_MONO_SIZE%%!$html_mono_size!g" \ | ||
477 | -e "s!%%HTML_MONO_GZ_SIZE%%!$html_mono_gz_size!g" \ | ||
478 | -e "s!%%HTML_NODE_TGZ_SIZE%%!$html_node_tgz_size!g" \ | ||
479 | -e "s!%%HTML_SECTION_TGZ_SIZE%%!$html_section_tgz_size!g" \ | ||
480 | -e "s!%%HTML_CHAPTER_TGZ_SIZE%%!$html_chapter_tgz_size!g" \ | ||
481 | -e "s!%%INFO_TGZ_SIZE%%!$info_tgz_size!g" \ | ||
482 | -e "s!%%DVI_GZ_SIZE%%!$dvi_gz_size!g" \ | ||
483 | -e "s!%%PDF_SIZE%%!$pdf_size!g" \ | ||
484 | -e "s!%%ASCII_SIZE%%!$ascii_size!g" \ | ||
485 | -e "s!%%ASCII_GZ_SIZE%%!$ascii_gz_size!g" \ | ||
486 | -e "s!%%TEXI_TGZ_SIZE%%!$texi_tgz_size!g" \ | ||
487 | -e "s!%%DOCBOOK_HTML_NODE_TGZ_SIZE%%!$html_node_db_tgz_size!g" \ | ||
488 | -e "s!%%DOCBOOK_ASCII_SIZE%%!$docbook_ascii_size!g" \ | ||
489 | -e "s!%%DOCBOOK_PDF_SIZE%%!$docbook_pdf_size!g" \ | ||
490 | -e "s!%%DOCBOOK_XML_SIZE%%!$docbook_xml_size!g" \ | ||
491 | -e "s!%%DOCBOOK_XML_GZ_SIZE%%!$docbook_xml_gz_size!g" \ | ||
492 | -e "s,%%SCRIPTURL%%,$scripturl,g" \ | ||
493 | -e "s!%%SCRIPTNAME%%!$prog!g" \ | ||
494 | -e "$CONDS" \ | ||
495 | $GENDOCS_TEMPLATE_DIR/gendocs_template >"$outdir/index.html" | ||
496 | |||
497 | echo "Done, see $outdir/ subdirectory for new files." | ||
498 | |||
499 | # Local variables: | ||
500 | # eval: (add-hook 'write-file-hooks 'time-stamp) | ||
501 | # time-stamp-start: "scriptversion=" | ||
502 | # time-stamp-format: "%:y-%02m-%02d.%02H" | ||
503 | # time-stamp-end: "$" | ||
504 | # End: | ||
diff --git a/doc/documentation/gendocs_template b/doc/documentation/gendocs_template new file mode 100644 index 000000000..178f6cb4c --- /dev/null +++ b/doc/documentation/gendocs_template | |||
@@ -0,0 +1,91 @@ | |||
1 | <!--#include virtual="/server/header.html" --> | ||
2 | <!-- Parent-Version: 1.77 --> | ||
3 | <title>%%TITLE%% - GNU Project - Free Software Foundation</title> | ||
4 | <!--#include virtual="/server/banner.html" --> | ||
5 | <h2>%%TITLE%%</h2> | ||
6 | |||
7 | <address>Free Software Foundation</address> | ||
8 | <address>last updated %%DATE%%</address> | ||
9 | |||
10 | <p>This manual (%%PACKAGE%%) is available in the following formats:</p> | ||
11 | |||
12 | <ul> | ||
13 | <li><a href="%%PACKAGE%%.html">HTML | ||
14 | (%%HTML_MONO_SIZE%%K bytes)</a> - entirely on one web page.</li> | ||
15 | <li><a href="html_node/index.html">HTML</a> - with one web page per | ||
16 | node.</li> | ||
17 | %%IF HTML_SECTION%% | ||
18 | <li><a href="html_section/index.html">HTML</a> - with one web page per | ||
19 | section.</li> | ||
20 | %%ENDIF HTML_SECTION%% | ||
21 | %%IF HTML_CHAPTER%% | ||
22 | <li><a href="html_chapter/index.html">HTML</a> - with one web page per | ||
23 | chapter.</li> | ||
24 | %%ENDIF HTML_CHAPTER%% | ||
25 | <li><a href="%%PACKAGE%%.html.gz">HTML compressed | ||
26 | (%%HTML_MONO_GZ_SIZE%%K gzipped characters)</a> - entirely on | ||
27 | one web page.</li> | ||
28 | <li><a href="%%PACKAGE%%.html_node.tar.gz">HTML compressed | ||
29 | (%%HTML_NODE_TGZ_SIZE%%K gzipped tar file)</a> - | ||
30 | with one web page per node.</li> | ||
31 | %%IF HTML_SECTION%% | ||
32 | <li><a href="%%PACKAGE%%.html_section.tar.gz">HTML compressed | ||
33 | (%%HTML_SECTION_TGZ_SIZE%%K gzipped tar file)</a> - | ||
34 | with one web page per section.</li> | ||
35 | %%ENDIF HTML_SECTION%% | ||
36 | %%IF HTML_CHAPTER%% | ||
37 | <li><a href="%%PACKAGE%%.html_chapter.tar.gz">HTML compressed | ||
38 | (%%HTML_CHAPTER_TGZ_SIZE%%K gzipped tar file)</a> - | ||
39 | with one web page per chapter.</li> | ||
40 | %%ENDIF HTML_CHAPTER%% | ||
41 | <li><a href="%%PACKAGE%%.info.tar.gz">Info document | ||
42 | (%%INFO_TGZ_SIZE%%K bytes gzipped tar file)</a>.</li> | ||
43 | <li><a href="%%PACKAGE%%.txt">ASCII text | ||
44 | (%%ASCII_SIZE%%K bytes)</a>.</li> | ||
45 | <li><a href="%%PACKAGE%%.txt.gz">ASCII text compressed | ||
46 | (%%ASCII_GZ_SIZE%%K bytes gzipped)</a>.</li> | ||
47 | <li><a href="%%PACKAGE%%.dvi.gz">TeX dvi file | ||
48 | (%%DVI_GZ_SIZE%%K bytes gzipped)</a>.</li> | ||
49 | <li><a href="%%PACKAGE%%.pdf">PDF file | ||
50 | (%%PDF_SIZE%%K bytes)</a>.</li> | ||
51 | <li><a href="%%PACKAGE%%.texi.tar.gz">Texinfo source | ||
52 | (%%TEXI_TGZ_SIZE%%K bytes gzipped tar file).</a></li> | ||
53 | </ul> | ||
54 | |||
55 | <p>You can <a href="http://shop.fsf.org/">buy printed copies of | ||
56 | some manuals</a> (among other items) from the Free Software Foundation; | ||
57 | this helps support FSF activities.</p> | ||
58 | |||
59 | <p>(This page generated by the <a href="%%SCRIPTURL%%">%%SCRIPTNAME%% | ||
60 | script</a>.)</p> | ||
61 | |||
62 | <!-- If needed, change the copyright block at the bottom. In general, | ||
63 | all pages on the GNU web server should have the section about | ||
64 | verbatim copying. Please do NOT remove this without talking | ||
65 | with the webmasters first. | ||
66 | Please make sure the copyright date is consistent with the document | ||
67 | and that it is like this: "2001, 2002", not this: "2001-2002". --> | ||
68 | </div><!-- for id="content", starts in the include above --> | ||
69 | <!--#include virtual="/server/footer.html" --> | ||
70 | <div id="footer"> | ||
71 | <div class="unprintable"> | ||
72 | |||
73 | <p>Please send general FSF & GNU inquiries to | ||
74 | <a href="mailto:gnu@gnu.org"><gnu@gnu.org></a>. | ||
75 | There are also <a href="/contact/">other ways to contact</a> | ||
76 | the FSF. Broken links and other corrections or suggestions can be sent | ||
77 | to <a href="mailto:%%EMAIL%%"><%%EMAIL%%></a>.</p> | ||
78 | </div> | ||
79 | |||
80 | <p>Copyright © 2017 Free Software Foundation, Inc.</p> | ||
81 | |||
82 | <p>This page is licensed under a <a rel="license" | ||
83 | href="http://creativecommons.org/licenses/by-nd/3.0/us/">Creative | ||
84 | Commons Attribution-NoDerivs 3.0 United States License</a>.</p> | ||
85 | |||
86 | <!--#include virtual="/server/bottom-notes.html" --> | ||
87 | |||
88 | </div> | ||
89 | </div> | ||
90 | </body> | ||
91 | </html> | ||
diff --git a/doc/documentation/gendocs_template_min b/doc/documentation/gendocs_template_min new file mode 100644 index 000000000..112fa3bfb --- /dev/null +++ b/doc/documentation/gendocs_template_min | |||
@@ -0,0 +1,93 @@ | |||
1 | <?xml version="1.0" encoding="utf-8" ?> | ||
2 | <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" | ||
3 | "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> | ||
4 | <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en"> | ||
5 | |||
6 | <head> | ||
7 | <title>%%TITLE%% - GNU Project - Free Software Foundation</title> | ||
8 | <meta http-equiv="content-type" content='text/html; charset=utf-8' /> | ||
9 | <link rel="stylesheet" type="text/css" href="/gnu.css" /> | ||
10 | </head> | ||
11 | |||
12 | <body> | ||
13 | |||
14 | <h3>%%TITLE%%</h3> | ||
15 | |||
16 | <address>Free Software Foundation</address> | ||
17 | <address>last updated %%DATE%%</address> | ||
18 | <p> | ||
19 | <a href="/graphics/gnu-head.jpg"> | ||
20 | <img src="/graphics/gnu-head-sm.jpg" | ||
21 | alt=" [image of the head of a GNU] " width="129" height="122"/> | ||
22 | </a> | ||
23 | </p> | ||
24 | <hr /> | ||
25 | |||
26 | <p>This manual (%%PACKAGE%%) is available in the following formats:</p> | ||
27 | |||
28 | <ul> | ||
29 | <li><a href="%%PACKAGE%%.html">HTML | ||
30 | (%%HTML_MONO_SIZE%%K bytes)</a> - entirely on one web page.</li> | ||
31 | <li><a href="html_node/index.html">HTML</a> - with one web page per | ||
32 | node.</li> | ||
33 | %%IF HTML_SECTION%% | ||
34 | <li><a href="html_section/index.html">HTML</a> - with one web page per | ||
35 | section.</li> | ||
36 | %%ENDIF HTML_SECTION%% | ||
37 | %%IF HTML_CHAPTER%% | ||
38 | <li><a href="html_chapter/index.html">HTML</a> - with one web page per | ||
39 | chapter.</li> | ||
40 | %%ENDIF HTML_CHAPTER%% | ||
41 | <li><a href="%%PACKAGE%%.html.gz">HTML compressed | ||
42 | (%%HTML_MONO_GZ_SIZE%%K gzipped characters)</a> - entirely on | ||
43 | one web page.</li> | ||
44 | <li><a href="%%PACKAGE%%.html_node.tar.gz">HTML compressed | ||
45 | (%%HTML_NODE_TGZ_SIZE%%K gzipped tar file)</a> - | ||
46 | with one web page per node.</li> | ||
47 | %%IF HTML_SECTION%% | ||
48 | <li><a href="%%PACKAGE%%.html_section.tar.gz">HTML compressed | ||
49 | (%%HTML_SECTION_TGZ_SIZE%%K gzipped tar file)</a> - | ||
50 | with one web page per section.</li> | ||
51 | %%ENDIF HTML_SECTION%% | ||
52 | %%IF HTML_CHAPTER%% | ||
53 | <li><a href="%%PACKAGE%%.html_chapter.tar.gz">HTML compressed | ||
54 | (%%HTML_CHAPTER_TGZ_SIZE%%K gzipped tar file)</a> - | ||
55 | with one web page per chapter.</li> | ||
56 | %%ENDIF HTML_CHAPTER%% | ||
57 | <li><a href="%%PACKAGE%%.info.tar.gz">Info document | ||
58 | (%%INFO_TGZ_SIZE%%K bytes gzipped tar file)</a>.</li> | ||
59 | <li><a href="%%PACKAGE%%.txt">ASCII text | ||
60 | (%%ASCII_SIZE%%K bytes)</a>.</li> | ||
61 | <li><a href="%%PACKAGE%%.txt.gz">ASCII text compressed | ||
62 | (%%ASCII_GZ_SIZE%%K bytes gzipped)</a>.</li> | ||
63 | <li><a href="%%PACKAGE%%.dvi.gz">TeX dvi file | ||
64 | (%%DVI_GZ_SIZE%%K bytes gzipped)</a>.</li> | ||
65 | <li><a href="%%PACKAGE%%.pdf">PDF file | ||
66 | (%%PDF_SIZE%%K bytes)</a>.</li> | ||
67 | <li><a href="%%PACKAGE%%.texi.tar.gz">Texinfo source | ||
68 | (%%TEXI_TGZ_SIZE%%K bytes gzipped tar file).</a></li> | ||
69 | </ul> | ||
70 | |||
71 | <p>(This page generated by the <a href="%%SCRIPTURL%%">%%SCRIPTNAME%% | ||
72 | script</a>.)</p> | ||
73 | |||
74 | <div id="footer" class="copyright"> | ||
75 | |||
76 | <p>Please send general FSF & GNU inquiries to | ||
77 | <a href="mailto:gnu@gnu.org"><gnu@gnu.org></a>. | ||
78 | There are also <a href="/contact/">other ways to contact</a> | ||
79 | the FSF. Broken links and other corrections or suggestions can be sent | ||
80 | to <a href="mailto:%%EMAIL%%"><%%EMAIL%%></a>.</p> | ||
81 | </div> | ||
82 | |||
83 | <p>Copyright © 2017 Free Software Foundation, Inc.</p> | ||
84 | |||
85 | <p>This page is licensed under a <a rel="license" | ||
86 | href="http://creativecommons.org/licenses/by-nd/3.0/us/">Creative | ||
87 | Commons Attribution-NoDerivs 3.0 United States License</a>.</p> | ||
88 | |||
89 | <!--#include virtual="/server/bottom-notes.html" --> | ||
90 | |||
91 | </div> | ||
92 | </body> | ||
93 | </html> | ||
diff --git a/doc/gnunet-c-tutorial.texi b/doc/documentation/gnunet-c-tutorial.texi index 4f56ae5c4..f39c7de64 100644 --- a/doc/gnunet-c-tutorial.texi +++ b/doc/documentation/gnunet-c-tutorial.texi | |||
@@ -3,8 +3,12 @@ | |||
3 | @setfilename gnunet-c-tutorial.info | 3 | @setfilename gnunet-c-tutorial.info |
4 | @documentencoding UTF-8 | 4 | @documentencoding UTF-8 |
5 | @settitle GNUnet C Tutorial | 5 | @settitle GNUnet C Tutorial |
6 | @exampleindent 2 | ||
6 | @c %**end of header | 7 | @c %**end of header |
7 | 8 | ||
9 | @c including 'version.texi' makes makeinfo throw errors. | ||
10 | @include version2.texi | ||
11 | |||
8 | @copying | 12 | @copying |
9 | Copyright @copyright{} 2001-2017 GNUnet e.V. | 13 | Copyright @copyright{} 2001-2017 GNUnet e.V. |
10 | 14 | ||
@@ -27,9 +31,15 @@ A copy of the license is also available from the Free Software | |||
27 | Foundation Web site at @url{http://www.gnu.org/licenses/gpl.html}. | 31 | Foundation Web site at @url{http://www.gnu.org/licenses/gpl.html}. |
28 | @end copying | 32 | @end copying |
29 | 33 | ||
34 | @dircategory Tutorial | ||
35 | @direntry | ||
36 | * GNUnet-C-Tutorial: (gnunet-c-tutorial). C Tutorial for GNunet | ||
37 | @end direntry | ||
38 | |||
39 | |||
30 | @titlepage | 40 | @titlepage |
31 | @title GNUnet C Tutorial | 41 | @title GNUnet C Tutorial |
32 | @subtitle A Tutorial for GNUnet 0.10.x (C version) | 42 | @subtitle A Tutorial for GNUnet @value{VERSION} (C version) |
33 | @author The GNUnet Developers | 43 | @author The GNUnet Developers |
34 | 44 | ||
35 | @page | 45 | @page |
@@ -48,92 +58,189 @@ Foundation Web site at @url{http://www.gnu.org/licenses/gpl.html}. | |||
48 | @node Top | 58 | @node Top |
49 | @top Introduction | 59 | @top Introduction |
50 | 60 | ||
51 | This tutorials explains how to install GNUnet on a GNU/Linux system and gives an introduction on how | 61 | This tutorials explains how to install GNUnet on a |
52 | GNUnet can be used to develop a Peer-to-Peer application. Detailed installation instructions for | 62 | GNU/Linux system and gives an introduction on how |
53 | various operating systems and a detailed list of all dependencies can be found on our website at | 63 | GNUnet can be used to develop a Peer-to-Peer application. |
54 | @uref{https://gnunet.org/installation}. | 64 | Detailed installation instructions for |
65 | various operating systems and a detailed list of all | ||
66 | dependencies can be found on our website at | ||
67 | @uref{https://gnunet.org/installation} and in our | ||
68 | Reference Documentation (GNUnet Handbook). | ||
55 | 69 | ||
56 | Please read this tutorial carefully since every single step is | 70 | Please read this tutorial carefully since every single step is |
57 | important and do not hesitate to contact the GNUnet team if you have | 71 | important and do not hesitate to contact the GNUnet team if you have |
58 | any questions or problems! Check here how to contact the GNUnet | 72 | any questions or problems! Check here how to contact the GNUnet |
59 | team: @uref{https://gnunet.org/contact_information} | 73 | team: @uref{https://gnunet.org/contact_information} |
60 | 74 | ||
75 | @menu | ||
76 | |||
77 | * Installing GNUnet:: Installing GNUnet | ||
78 | * Introduction to GNUnet Architecture:: Introduction to GNUnet Architecture | ||
79 | * First Steps with GNUnet:: First Steps with GNUnet | ||
80 | * Developing Applications:: Developing Applications | ||
81 | |||
82 | @detailmenu | ||
83 | --- The Detailed Node Listing --- | ||
84 | |||
85 | Installing GNUnet | ||
86 | |||
87 | * Obtaining a stable version:: | ||
88 | * Installing Build Tool Chain and Dependencies:: | ||
89 | * Obtaining the latest version from Git:: | ||
90 | * Compiling and Installing GNUnet:: | ||
91 | * Common Issues - Check your GNUnet installation:: | ||
92 | |||
93 | Introduction to GNUnet Architecture | ||
94 | |||
95 | First Steps with GNUnet | ||
96 | |||
97 | * Configure your peer:: | ||
98 | * Start a peer:: | ||
99 | * Monitor a peer:: | ||
100 | * Starting Two Peers by Hand:: | ||
101 | * Starting Peers Using the Testbed Service:: | ||
102 | |||
103 | Developing Applications | ||
104 | |||
105 | * gnunet-ext:: | ||
106 | * Adapting the Template:: | ||
107 | * Writing a Client Application:: | ||
108 | * Writing a Service:: | ||
109 | * Interacting directly with other Peers using the CORE Service:: | ||
110 | * Storing peer-specific data using the PEERSTORE service:: | ||
111 | * Using the DHT:: | ||
112 | * Debugging with gnunet-arm:: | ||
113 | |||
114 | @end detailmenu | ||
115 | @end menu | ||
116 | |||
61 | @node Installing GNUnet | 117 | @node Installing GNUnet |
62 | @chapter Installing GNUnet | 118 | @chapter Installing GNUnet |
63 | 119 | ||
64 | First of all you have to install a current version of GNUnet. You can download a | 120 | First of all you have to install a current version of GNUnet. |
65 | tarball of a stable version from GNU FTP mirrors or obtain the latest development | 121 | You can download a tarball of a stable version from GNU FTP mirrors |
66 | version from our Git repository. | 122 | or obtain the latest development version from our Git repository. |
67 | 123 | ||
68 | Most of the time you should prefer to download the stable version since with the | 124 | Most of the time you should prefer to download the stable version |
69 | latest development version things can be broken, functionality can be changed or tests | 125 | since with the latest development version things can be broken, |
70 | can fail. You should only use the development version if you know that you require a | 126 | functionality can be changed or tests can fail. You should only use |
71 | certain feature or a certain issue has been fixed since the last release. | 127 | the development version if you know that you require a certain |
128 | feature or a certain issue has been fixed since the last release. | ||
129 | |||
130 | @menu | ||
131 | * Obtaining a stable version:: | ||
132 | * Installing Build Tool Chain and Dependencies:: | ||
133 | * Obtaining the latest version from Git:: | ||
134 | * Compiling and Installing GNUnet:: | ||
135 | * Common Issues - Check your GNUnet installation:: | ||
136 | @end menu | ||
72 | 137 | ||
73 | @node Obtaining a stable version | 138 | @node Obtaining a stable version |
74 | @section Obtaining a stable version | 139 | @section Obtaining a stable version |
75 | 140 | ||
76 | You can download the latest stable version of GNUnet from GNU FTP mirrors: | 141 | Download the tarball from |
77 | @uref{https://ftp.gnu.org/gnu/gnunet/gnunet-0.10.x.tar.gz} | 142 | @indicateurl{https://ftp.gnu.org/gnu/gnunet/gnunet-@value{VERSION}.tar.gz}. |
78 | You should also download the signature file and verify the integrity of the tarball. | 143 | |
79 | @uref{https://ftp.gnu.org/gnu/gnunet/gnunet-0.10.x.tar.gz.sig} | 144 | Make sure to download the associated @file{.sig} file and to verify the |
80 | To verify the signature you should first import the GPG key used to sign the tarball | 145 | authenticity of the tarball against it, like this: |
146 | |||
81 | @example | 147 | @example |
82 | $ gpg --keyserver keys.gnupg.net --recv-keys 48426C7E | 148 | $ wget https://ftp.gnu.org/gnu/gnunet/gnunet-@value{VERSION}.tar.gz.sig |
149 | $ gpg --verify-files gnunet-@value{VERSION}.tar.gz.sig | ||
83 | @end example | 150 | @end example |
84 | And use this key to verify the tarball's signature | 151 | |
152 | @noindent | ||
153 | If this command fails because you do not have the required public key, | ||
154 | then you need to run this command to import it: | ||
155 | |||
85 | @example | 156 | @example |
86 | $ gpg --verify gnunet-0.10.x.tar.gz.sig gnunet-0.10.x.tar.gz | 157 | $ gpg --keyserver keys.gnupg.net --recv-keys 48426C7E |
87 | @end example | 158 | @end example |
88 | After successfully verifying the integrity you can extract the tarball using | 159 | |
160 | @noindent | ||
161 | and rerun the @code{gpg --verify-files} command. | ||
162 | |||
163 | @b{Note:}@ | ||
164 | @b{The pub key to sign the 0.10.1 release has been | ||
165 | revoked}. You will get an error message stating that | ||
166 | @b{there is no known public key or that it has been revoked}. | ||
167 | The next release of GNUnet will have a valid signature | ||
168 | again. We are sorry for the inconvenience this causes. | ||
169 | Another possible source you could use is our | ||
170 | "gnunet" git repository which has mandatory signed commits | ||
171 | by every developer. | ||
172 | |||
173 | Now you can extract the tarball and rename the resulting | ||
174 | directory to @file{gnunet} which we will be using in the | ||
175 | remainder of this document. | ||
176 | |||
89 | @example | 177 | @example |
90 | $ tar xvzf gnunet-0.10.x.tar.gz | 178 | $ tar xvzf gnunet-@value{VERSION}.tar.gz |
91 | ## we will use the directory "gnunet" in the remainder of this document | 179 | $ mv gnunet-@value{VERSION} gnunet |
92 | $ mv gnunet-0.10.x gnunet | ||
93 | $ cd gnunet | 180 | $ cd gnunet |
94 | @end example | 181 | @end example |
95 | 182 | ||
96 | However, please note that stable versions can be very outdated, as a developer | 183 | @noindent |
97 | you are strongly encouraged to use the version from @uref{https://gnunet.org/git/}. | 184 | However, please note that stable versions can be very outdated. |
185 | As a developer you are @b{strongly} encouraged to use the version | ||
186 | from @uref{https://gnunet.org/git/, git}. | ||
98 | 187 | ||
99 | @node Installing Build Tool Chain and Dependencies | 188 | @node Installing Build Tool Chain and Dependencies |
100 | @section Installing Build Tool Chain and Dependencies | 189 | @section Installing Build Tool Chain and Dependencies |
101 | 190 | ||
102 | To successfully compile GNUnet you need the tools to build GNUnet and the required dependencies. | 191 | To successfully compile GNUnet, you need the tools to build GNUnet and |
103 | Please have a look at @uref{https://gnunet.org/dependencies} for a list of required dependencies | 192 | the required dependencies. Please take a look at the |
104 | and @uref{https://gnunet.org/generic_installation} for specific instructions for your operating system. | 193 | GNUnet Reference Documentation |
105 | 194 | (@pxref{Dependencies, The GNUnet Reference Documentation,, gnunet, The GNUnet Reference Documentation}) | |
106 | Please check the notes at the end of the configure process about required dependencies. | 195 | for a list of required dependencies |
107 | 196 | and | |
108 | For GNUnet bootstrapping support and the http(s) plugin you should install libgnurl. | 197 | (@pxref{Generic installation instructions, The GNUnet Reference Documentation,, gnunet, The GNUnet Reference Documentation}) |
109 | For the filesharing service you should install at least one of the datastore backends mysql, | 198 | read its Installation chapter for specific instructions for |
110 | sqlite or postgresql. | 199 | your operating system. |
200 | Please check the notes at the end of the configure process about | ||
201 | required dependencies. | ||
202 | |||
203 | For GNUnet bootstrapping support and the HTTP(S) plugin you should | ||
204 | install @uref{https://gnunet.org/gnurl, libgnurl}. | ||
205 | For the filesharing service you should install at least one of the | ||
206 | datastore backends. MySQL, SQlite and PostgreSQL are supported. | ||
111 | 207 | ||
112 | @node Obtaining the latest version from Git | 208 | @node Obtaining the latest version from Git |
113 | @section Obtaining the latest version from Git | 209 | @section Obtaining the latest version from Git |
114 | 210 | ||
115 | The latest development version can obtained from our Git repository. To obtain | 211 | The latest development version can obtained from our Git repository. |
116 | the code you need Git installed and checkout the repository using: | 212 | To obtain the code you need to have @code{Git} installed, which is |
213 | required for obtaining the repository via: | ||
214 | |||
117 | @example | 215 | @example |
118 | $ git clone https://gnunet.org/git/gnunet | 216 | $ git clone https://gnunet.org/git/gnunet |
119 | @end example | 217 | @end example |
120 | After cloning the repository you have to execute | 218 | |
219 | @noindent | ||
220 | After cloning the repository you have to execute the @file{bootstrap} | ||
221 | script in the new directory: | ||
222 | |||
121 | @example | 223 | @example |
122 | $ cd gnunet | 224 | $ cd gnunet |
123 | $ ./bootstrap | 225 | $ ./bootstrap |
124 | @end example | 226 | @end example |
125 | 227 | ||
126 | The remainder of this tutorial assumes that you have Git branch ``master'' checked out. | 228 | @noindent |
229 | The remainder of this tutorial will assume that you have the | ||
230 | Git branch ``master'' checked out. | ||
127 | 231 | ||
128 | @node Compiling and Installing GNUnet | 232 | @node Compiling and Installing GNUnet |
129 | @section Compiling and Installing GNUnet | 233 | @section Compiling and Installing GNUnet |
130 | 234 | ||
131 | First, you need to install at least libgnupgerror version 1.27 | 235 | Note: This section is a duplication of the more in depth |
132 | @uref{https://www.gnupg.org/ftp/gcrypt/libgpg-error/libgpg-error-1.27.tar.bz2} | 236 | @pxref{GNUnet Installation Handbook, The GNUnet Reference Documentation,, gnunet, The GNUnet Reference Documentation}. |
133 | and libgcrypt version 1.7.6 @uref{https://www.gnupg.org/ftp/gcrypt/libgcrypt/libgcrypt-1.7.6.tar.bz2}. | 237 | |
238 | First, you need to install libgnupgerror @geq{} 1.27 and | ||
239 | libgcrypt @geq{} 1.7.6: | ||
134 | 240 | ||
135 | @example | 241 | @example |
136 | $ wget https://www.gnupg.org/ftp/gcrypt/libgpg-error/libgpg-error-1.27.tar.bz2 | 242 | $ export GNUPGFTP="https://www.gnupg.org/ftp/gcrypt" |
243 | $ wget $GNUPGFTP/libgpg-error/libgpg-error-1.27.tar.bz2 | ||
137 | $ tar xf libgpg-error-1.27.tar.bz2 | 244 | $ tar xf libgpg-error-1.27.tar.bz2 |
138 | $ cd libgpg-error-1.27 | 245 | $ cd libgpg-error-1.27 |
139 | $ ./configure | 246 | $ ./configure |
@@ -142,7 +249,8 @@ $ cd .. | |||
142 | @end example | 249 | @end example |
143 | 250 | ||
144 | @example | 251 | @example |
145 | $ wget https://www.gnupg.org/ftp/gcrypt/libgcrypt/libgcrypt-1.7.6.tar.bz2 | 252 | $ export GNUPGFTP="https://www.gnupg.org/ftp/gcrypt" |
253 | $ wget $GNUPGFTP/libgcrypt/libgcrypt-1.7.6.tar.bz2 | ||
146 | $ tar xf libgcrypt-1.7.6.tar.bz2 | 254 | $ tar xf libgcrypt-1.7.6.tar.bz2 |
147 | $ cd libgcrypt-1.7.6 | 255 | $ cd libgcrypt-1.7.6 |
148 | $ ./configure | 256 | $ ./configure |
@@ -150,6 +258,10 @@ $ sudo make install | |||
150 | $ cd .. | 258 | $ cd .. |
151 | @end example | 259 | @end example |
152 | 260 | ||
261 | @menu | ||
262 | * Installation:: | ||
263 | @end menu | ||
264 | |||
153 | @node Installation | 265 | @node Installation |
154 | @subsection Installation | 266 | @subsection Installation |
155 | Assuming all dependencies are installed, the following commands will | 267 | Assuming all dependencies are installed, the following commands will |
@@ -166,10 +278,12 @@ $ make | |||
166 | $ make install | 278 | $ make install |
167 | @end example | 279 | @end example |
168 | 280 | ||
169 | After installing GNUnet you have to add your GNUnet installation to your path | 281 | @noindent |
170 | environmental variable. In addition you have to create the @file{.config} | 282 | After installing GNUnet you have to add your GNUnet installation |
171 | directory in your home directory (unless it already exists) where GNUnet stores | 283 | to your path environmental variable. In addition you have to |
172 | its data and an empty GNUnet configuration file: | 284 | create the @file{.config} directory in your home directory |
285 | (unless it already exists) where GNUnet stores its data and an | ||
286 | empty GNUnet configuration file: | ||
173 | 287 | ||
174 | @example | 288 | @example |
175 | $ export PATH=$PATH:$PREFIX/bin | 289 | $ export PATH=$PATH:$PREFIX/bin |
@@ -184,23 +298,34 @@ $ touch ~/.config/gnunet.conf | |||
184 | You should check your installation to ensure that installing GNUnet | 298 | You should check your installation to ensure that installing GNUnet |
185 | was successful up to this point. You should be able to access GNUnet's | 299 | was successful up to this point. You should be able to access GNUnet's |
186 | binaries and run GNUnet's self check. | 300 | binaries and run GNUnet's self check. |
301 | |||
187 | @example | 302 | @example |
188 | $ which gnunet-arm | 303 | $ which gnunet-arm |
189 | @end example | 304 | @end example |
190 | should return $PREFIX/bin/gnunet-arm. It should be | 305 | |
191 | located in your GNUnet installation and the output should not be | 306 | @noindent |
192 | empty. If you see an output like: | 307 | should return $PREFIX/bin/gnunet-arm. It should be located in your |
308 | GNUnet installation and the output should not be empty. | ||
309 | If you see an output like: | ||
310 | |||
193 | @example | 311 | @example |
194 | $ which gnunet-arm | 312 | $ which gnunet-arm |
195 | @end example | 313 | @end example |
196 | check your PATH variable to ensure GNUnet's @file{bin} directory is included. | 314 | |
315 | @noindent | ||
316 | check your PATH variable to ensure GNUnet's @file{bin} directory is | ||
317 | included. | ||
197 | 318 | ||
198 | GNUnet provides tests for all of its subcomponents. Run | 319 | GNUnet provides tests for all of its subcomponents. Run |
320 | |||
199 | @example | 321 | @example |
200 | $ make check | 322 | $ make check |
201 | @end example | 323 | @end example |
202 | to execute tests for all components. make check traverses all subdirectories in src. | 324 | |
203 | For every subdirectory you should get a message like this: | 325 | @noindent |
326 | to execute tests for all components. @command{make check} traverses all | ||
327 | subdirectories in @file{src}. For every subdirectory you should | ||
328 | get a message like this: | ||
204 | 329 | ||
205 | @example | 330 | @example |
206 | make[2]: Entering directory `/home/$USER/gnunet/contrib' | 331 | make[2]: Entering directory `/home/$USER/gnunet/contrib' |
@@ -210,29 +335,34 @@ PASS: test_gnunet_prefix | |||
210 | ============= | 335 | ============= |
211 | @end example | 336 | @end example |
212 | 337 | ||
213 | @node Background: GNUnet Architecture | 338 | @node Introduction to GNUnet Architecture |
214 | @chapter Background: GNUnet Architecture | 339 | @chapter Introduction to GNUnet Architecture |
215 | 340 | ||
216 | GNUnet is organized in layers and services. Each service is composed of a | 341 | GNUnet is organized in layers and services. Each service is composed of a |
217 | main service implementation and a client library for other programs to use | 342 | main service implementation and a client library for other programs to use |
218 | the service's functionality, described by an API. This approach is shown in | 343 | the service's functionality, described by an API. |
219 | @c** FIXME: enable this once the commented block below works: | 344 | @c This approach is shown in |
220 | @c** figure~\ref{fig:service}. | 345 | @c FIXME: enable this once the commented block below works: |
221 | Some services provide an additional command line tool to enable the user to | 346 | @c figure~\ref fig:service. |
222 | interact with the service. | 347 | Some services provide an additional command line tool to enable the user |
223 | 348 | to interact with the service. | |
224 | Very often it is other GNUnet services that will use these APIs to build the | 349 | |
225 | higher layers of GNUnet on top of the lower ones. Each layer expands or extends | 350 | Very often it is other GNUnet services that will use these APIs to build |
226 | the functionality of the service below (for instance, to build a mesh on top of | 351 | the higher layers of GNUnet on top of the lower ones. Each layer expands |
227 | a DHT). | 352 | or extends the functionality of the service below (for instance, to build |
228 | @c** FXIME: See comment above. | 353 | a mesh on top of a DHT). |
229 | @c** See figure ~\ref{fig:interaction} for an illustration of this approach. | 354 | @c FXIME: See comment above. |
230 | 355 | @c See figure ~\ref fig:interaction for an illustration of this approach. | |
231 | @c ** @image{filename[, width[, height[, alttext[, extension]]]]} | 356 | |
232 | 357 | @c ** @image filename[, width[, height[, alttext[, extension]]]] | |
233 | @image{images/gnunet-tutorial-service,,5in,Service with API and network protocol,.png} | 358 | @c FIXME: Texlive (?) 20112 makes the assumption that this means |
234 | 359 | @c 'images/OBJECTNAME.txt' but later versions of it (2017) use this | |
235 | @image{images/gnunet-tutorial-system,,5in,The layered system architecture of GNUnet,.png} | 360 | @c syntax as described below. |
361 | @c TODO: Checkout the makedoc script Guile uses. | ||
362 | |||
363 | @c FIXME!!! | ||
364 | @c @image{images/gnunet-tutorial-service,,5in,Service with API and network protocol,.png} | ||
365 | @c @image{images/gnunet-tutorial-system,,5in,The layered system architecture of GNUnet,.png} | ||
236 | 366 | ||
237 | @c \begin{figure}[!h] | 367 | @c \begin{figure}[!h] |
238 | @c \begin{center} | 368 | @c \begin{center} |
@@ -254,55 +384,71 @@ a DHT). | |||
254 | @c \caption{GNUnet's layered system architecture} | 384 | @c \caption{GNUnet's layered system architecture} |
255 | @c \end{figure} | 385 | @c \end{figure} |
256 | 386 | ||
257 | The main service implementation runs as a standalone process in the operating | 387 | The main service implementation runs as a standalone process in the |
258 | system and the client code runs as part of the client program, so crashes of a | 388 | operating system and the client code runs as part of the client program, |
259 | client do not affect the service process or other clients. The service and the | 389 | so crashes of a client do not affect the service process or other clients. |
260 | clients communicate via a message protocol to be defined and implemented by | 390 | The service and the clients communicate via a message protocol to be |
261 | the programmer. | 391 | defined and implemented by the programmer. |
262 | 392 | ||
263 | @node First Steps with GNUnet | 393 | @node First Steps with GNUnet |
264 | @chapter First Steps with GNUnet | 394 | @chapter First Steps with GNUnet |
265 | 395 | ||
396 | @menu | ||
397 | * Configure your peer:: | ||
398 | * Start a peer:: | ||
399 | * Monitor a peer:: | ||
400 | * Starting Two Peers by Hand:: | ||
401 | * Starting Peers Using the Testbed Service:: | ||
402 | @end menu | ||
403 | |||
266 | @node Configure your peer | 404 | @node Configure your peer |
267 | @section Configure your peer | 405 | @section Configure your peer |
268 | 406 | ||
269 | First of all we need to configure your peer. Each peer is started with a configuration | 407 | First of all we need to configure your peer. Each peer is started with |
270 | containing settings for GNUnet itself and its services. This configuration is based on the | 408 | a configuration containing settings for GNUnet itself and its services. |
271 | default configuration shipped with GNUnet and can be modified. The default configuration | 409 | This configuration is based on the default configuration shipped with |
272 | is located in the @file{$PREFIX/share/gnunet/config.d} directory. When starting a peer, you | 410 | GNUnet and can be modified. The default configuration is located in the |
273 | can specify a customized configuration using the the @command{-c} command line switch when | 411 | @file{$PREFIX/share/gnunet/config.d} directory. When starting a peer, you |
274 | starting the ARM service and all other services. When using a modified configuration the | 412 | can specify a customized configuration using the the @command{-c} command |
275 | default values are loaded and only values specified in the configuration file will replace | 413 | line switch when starting the ARM service and all other services. When |
276 | the default values. | 414 | using a modified configuration the default values are loaded and only |
277 | 415 | values specified in the configuration file will replace the default | |
278 | Since we want to start additional peers later, we need some modifications from the default | 416 | values. |
279 | configuration. We need to create a separate service home and a file containing our | 417 | |
280 | modifications for this peer: | 418 | Since we want to start additional peers later, we need some modifications |
419 | from the default configuration. We need to create a separate service | ||
420 | home and a file containing our modifications for this peer: | ||
421 | |||
281 | @example | 422 | @example |
282 | $ mkdir ~/gnunet1/ | 423 | $ mkdir ~/gnunet1/ |
283 | $ touch peer1.conf | 424 | $ touch peer1.conf |
284 | @end example | 425 | @end example |
285 | 426 | ||
286 | Now add the following lines to @file{peer1.conf} to use this directory. For | 427 | @noindent |
287 | simplified usage we want to prevent the peer to connect to the GNUnet | 428 | Now add the following lines to @file{peer1.conf} to use this directory. |
429 | For simplified usage we want to prevent the peer to connect to the GNUnet | ||
288 | network since this could lead to confusing output. This modifications | 430 | network since this could lead to confusing output. This modifications |
289 | will replace the default settings: | 431 | will replace the default settings: |
432 | |||
290 | @example | 433 | @example |
291 | [PATHS] | 434 | [PATHS] |
292 | GNUNET_HOME = ~/gnunet1/ # Use this directory to store GNUnet data | 435 | # Use this directory to store GNUnet data |
436 | GNUNET_HOME = ~/gnunet1/ | ||
293 | [hostlist] | 437 | [hostlist] |
294 | SERVERS = # prevent bootstrapping | 438 | # prevent bootstrapping |
439 | SERVERS = | ||
295 | @end example | 440 | @end example |
296 | 441 | ||
297 | @node Start a peer | 442 | @node Start a peer |
298 | @section Start a peer | 443 | @section Start a peer |
299 | Each GNUnet instance (called peer) has an identity (peer ID) based on a | 444 | Each GNUnet instance (called peer) has an identity (peer ID) based on a |
300 | cryptographic public private key pair. The peer ID is the printable hash of the | 445 | cryptographic public private key pair. The peer ID is the printable hash |
301 | public key. | 446 | of the public key. |
302 | 447 | ||
303 | GNUnet services are controlled by a master service, the so called @dfn{Automatic Restart Manager} (ARM). | 448 | GNUnet services are controlled by a master service, the so called |
304 | ARM starts, stops and even restarts services automatically or on demand when a client connects. | 449 | @dfn{Automatic Restart Manager} (ARM). ARM starts, stops and even |
305 | You interact with the ARM service using the gnunet-arm tool. | 450 | restarts services automatically or on demand when a client connects. |
451 | You interact with the ARM service using the @command{gnunet-arm} tool. | ||
306 | GNUnet can then be started with @command{gnunet-arm -s} and stopped with | 452 | GNUnet can then be started with @command{gnunet-arm -s} and stopped with |
307 | @command{gnunet-arm -e}. An additional service not automatically started | 453 | @command{gnunet-arm -e}. An additional service not automatically started |
308 | can be started using @command{gnunet-arm -i <service name>} and stopped | 454 | can be started using @command{gnunet-arm -i <service name>} and stopped |
@@ -310,11 +456,16 @@ using @command{gnunet-arm -k <servicename>}. | |||
310 | 456 | ||
311 | Once you have started your peer, you can use many other GNUnet commands | 457 | Once you have started your peer, you can use many other GNUnet commands |
312 | to interact with it. For example, you can run: | 458 | to interact with it. For example, you can run: |
459 | |||
313 | @example | 460 | @example |
314 | $ gnunet-peerinfo -s | 461 | $ gnunet-peerinfo -s |
315 | @end example | 462 | @end example |
463 | |||
464 | @noindent | ||
316 | to obtain the public key of your peer. | 465 | to obtain the public key of your peer. |
466 | |||
317 | You should see an output containing the peer ID similar to: | 467 | You should see an output containing the peer ID similar to: |
468 | |||
318 | @example | 469 | @example |
319 | I am peer `0PA02UVRKQTS2C .. JL5Q78F6H0B1ACPV1CJI59MEQUMQCC5G'. | 470 | I am peer `0PA02UVRKQTS2C .. JL5Q78F6H0B1ACPV1CJI59MEQUMQCC5G'. |
320 | @end example | 471 | @end example |
@@ -322,63 +473,89 @@ I am peer `0PA02UVRKQTS2C .. JL5Q78F6H0B1ACPV1CJI59MEQUMQCC5G'. | |||
322 | @node Monitor a peer | 473 | @node Monitor a peer |
323 | @section Monitor a peer | 474 | @section Monitor a peer |
324 | 475 | ||
325 | In this section, we will monitor the behaviour of our peer's DHT service with respect to a | 476 | In this section, we will monitor the behaviour of our peer's DHT |
326 | specific key. First we will start GNUnet and then start the DHT service and use the DHT monitor tool | 477 | service with respect to a specific key. First we will start |
327 | to monitor the PUT and GET commands we issue ussing the @command{gnunet-dht-put} and | 478 | GNUnet and then start the DHT service and use the DHT monitor tool |
328 | @command{gnunet-dht-get} commands. Using the ``monitor'' line given below, you can observe | 479 | to monitor the PUT and GET commands we issue ussing the |
329 | the behavior of your own peer's DHT with respect to the specified KEY: | 480 | @command{gnunet-dht-put} and @command{gnunet-dht-get} commands. |
481 | Using the ``monitor'' line given below, you can observe the behavior | ||
482 | of your own peer's DHT with respect to the specified KEY: | ||
330 | 483 | ||
331 | @example | 484 | @example |
332 | $ gnunet-arm -c ~/peer1.conf -s # start gnunet with all default services | 485 | # start gnunet with all default services: |
333 | $ gnunet-arm -c ~/peer1.conf -i dht # start DHT service | 486 | $ gnunet-arm -c ~/peer1.conf -s |
487 | # start DHT service: | ||
488 | $ gnunet-arm -c ~/peer1.conf -i dht | ||
334 | $ cd ~/gnunet/src/dht; | 489 | $ cd ~/gnunet/src/dht; |
335 | $ ./gnunet-dht-monitor -c ~/peer1.conf -k KEY | 490 | $ ./gnunet-dht-monitor -c ~/peer1.conf -k KEY |
336 | @end example | 491 | @end example |
337 | Now open a separate terminal and change again to the @file{gnunet/src/dht} directory: | 492 | |
493 | @noindent | ||
494 | Now open a separate terminal and change again to | ||
495 | the @file{gnunet/src/dht} directory: | ||
496 | |||
338 | @example | 497 | @example |
339 | $ cd ~/gnunet/src/dht | 498 | $ cd ~/gnunet/src/dht |
340 | $ ./gnunet-dht-put -c ~/peer1.conf -k KEY -d VALUE # put VALUE under KEY in the DHT | 499 | # put VALUE under KEY in the DHT: |
341 | $ ./gnunet/src/dht/gnunet-dht-get -c ~/peer1.conf -k KEY # get key KEY from the DHT | 500 | $ ./gnunet-dht-put -c ~/peer1.conf -k KEY -d VALUE |
342 | $ gnunet-statistics -c ~/peer1.conf # print statistics about current GNUnet state | 501 | # get key KEY from the DHT: |
343 | $ gnunet-statistics -c ~/peer1.conf -s dht # print statistics about DHT service | 502 | $ ./gnunet/src/dht/gnunet-dht-get -c ~/peer1.conf -k KEY |
503 | # print statistics about current GNUnet state: | ||
504 | $ gnunet-statistics -c ~/peer1.conf | ||
505 | # print statistics about DHT service: | ||
506 | $ gnunet-statistics -c ~/peer1.conf -s dht | ||
344 | @end example | 507 | @end example |
345 | 508 | ||
346 | @node Starting Two Peers by Hand | 509 | @node Starting Two Peers by Hand |
347 | @section Starting Two Peers by Hand | 510 | @section Starting Two Peers by Hand |
348 | 511 | ||
349 | This section describes how to start two peers on the same machine by hand. | 512 | This section describes how to start two peers on the same machine by hand. |
350 | The process is rather painful, but the description is somewhat instructive. | 513 | The process is rather painful, but the description is somewhat |
351 | In practice, you might prefer the automated method | 514 | instructive. In practice, you might prefer the automated method |
352 | (@pxref{Starting Peers Using the Testbed Service}). | 515 | (@pxref{Starting Peers Using the Testbed Service}). |
353 | 516 | ||
517 | @menu | ||
518 | * Setup a second peer:: | ||
519 | * Start the second peer and connect the peers:: | ||
520 | * How to connect manually:: | ||
521 | @end menu | ||
522 | |||
354 | @node Setup a second peer | 523 | @node Setup a second peer |
355 | @subsection Setup a second peer | 524 | @subsection Setup a second peer |
356 | We will now start a second peer on your machine. | 525 | We will now start a second peer on your machine. |
357 | For the second peer, you will need to manually create a modified | 526 | For the second peer, you will need to manually create a modified |
358 | configuration file to avoid conflicts with ports and directories. | 527 | configuration file to avoid conflicts with ports and directories. |
359 | A peers configuration file is by default located in @file{~/.gnunet/gnunet.conf}. | 528 | A peers configuration file is by default located |
360 | This file is typically very short or even empty as only the differences to the | 529 | in @file{~/.gnunet/gnunet.conf}. This file is typically very short |
361 | defaults need to be specified. The defaults are located in | 530 | or even empty as only the differences to the defaults need to be |
362 | many files in the @file{$PREFIX/share/gnunet/config.d} directory. | 531 | specified. The defaults are located in many files in the |
532 | @file{$PREFIX/share/gnunet/config.d} directory. | ||
363 | 533 | ||
364 | To configure the second peer, use the files | 534 | To configure the second peer, use the files |
365 | @file{$PREFIX/share/gnunet/config.d} as a template for your main | 535 | @file{$PREFIX/share/gnunet/config.d} as a template for your main |
366 | configuration file: | 536 | configuration file: |
537 | |||
367 | @example | 538 | @example |
368 | $ cat $PREFIX/share/gnunet/config.d/*.conf > peer2.conf | 539 | $ cat $PREFIX/share/gnunet/config.d/*.conf > peer2.conf |
369 | @end example | 540 | @end example |
541 | |||
542 | @noindent | ||
370 | Now you have to edit @file{peer2.conf} and change: | 543 | Now you have to edit @file{peer2.conf} and change: |
544 | |||
371 | @itemize | 545 | @itemize |
372 | @item @code{GNUNET\_TEST\_HOME} under @code{PATHS} | 546 | @item @code{GNUNET\_TEST\_HOME} under @code{PATHS} |
373 | @item Every (uncommented) value for ``@code{PORT}'' (add 10000) in any | 547 | @item Every (uncommented) value for ``@code{PORT}'' (add 10000) in any |
374 | section (the option may be commented out if @code{PORT} is | 548 | section (the option may be commented out if @code{PORT} is |
375 | prefixed by "\#", in this case, UNIX domain sockets are used | 549 | prefixed by "\#", in this case, UNIX domain sockets are used |
376 | and the PORT option does not need to be touched) | 550 | and the PORT option does not need to be touched) |
377 | @item Every value for ``@code{UNIXPATH}'' in any section (e.g. by adding a "-p2" suffix) | 551 | @item Every value for ``@code{UNIXPATH}'' in any section |
552 | (e.g. by adding a "-p2" suffix) | ||
378 | @end itemize | 553 | @end itemize |
379 | to a fresh, unique value. Make sure that the PORT numbers stay below 65536. | 554 | |
380 | From now on, whenever you interact with the second peer, you need to specify | 555 | to a fresh, unique value. Make sure that the PORT numbers stay |
381 | @command{-c peer2.conf} as an additional command line argument. | 556 | below 65536. From now on, whenever you interact with the second peer, |
557 | you need to specify @command{-c peer2.conf} as an additional | ||
558 | command line argument. | ||
382 | 559 | ||
383 | Now, generate the 2nd peer's private key: | 560 | Now, generate the 2nd peer's private key: |
384 | 561 | ||
@@ -386,6 +563,7 @@ Now, generate the 2nd peer's private key: | |||
386 | $ gnunet-peerinfo -s -c peer2.conf | 563 | $ gnunet-peerinfo -s -c peer2.conf |
387 | @end example | 564 | @end example |
388 | 565 | ||
566 | @noindent | ||
389 | This may take a while, generate entropy using your keyboard or mouse | 567 | This may take a while, generate entropy using your keyboard or mouse |
390 | as needed. Also, make sure the output is different from the | 568 | as needed. Also, make sure the output is different from the |
391 | gnunet-peerinfo output for the first peer (otherwise you made an | 569 | gnunet-peerinfo output for the first peer (otherwise you made an |
@@ -395,34 +573,46 @@ error in the configuration). | |||
395 | @subsection Start the second peer and connect the peers | 573 | @subsection Start the second peer and connect the peers |
396 | 574 | ||
397 | Then, you can start a second peer using: | 575 | Then, you can start a second peer using: |
576 | |||
398 | @example | 577 | @example |
399 | $ gnunet-arm -c peer2.conf -s | 578 | $ gnunet-arm -c peer2.conf -s |
400 | $ gnunet-arm -c peer2.conf -i dht | 579 | $ gnunet-arm -c peer2.conf -i dht |
401 | $ ~/gnunet/src/dht/gnunet-dht-put -c peer2.conf -k KEY -d VALUE | 580 | $ ~/gnunet/src/dht/gnunet-dht-put -c peer2.conf -k KEY -d VALUE |
402 | $ ~/gnunet/src/dht/gnunet-dht-get -c peer2.conf -k KEY | 581 | $ ~/gnunet/src/dht/gnunet-dht-get -c peer2.conf -k KEY |
403 | @end example | 582 | @end example |
583 | |||
404 | If you want the two peers to connect, you have multiple options: | 584 | If you want the two peers to connect, you have multiple options: |
585 | |||
405 | @itemize | 586 | @itemize |
406 | @item UDP neighbour discovery (automatic) | 587 | @item UDP neighbour discovery (automatic) |
407 | @item Setup a bootstrap server | 588 | @item Setup a bootstrap server |
408 | @item Connect manually | 589 | @item Connect manually |
409 | @end itemize | 590 | @end itemize |
591 | |||
410 | To setup peer 1 as bootstrapping server change the configuration of | 592 | To setup peer 1 as bootstrapping server change the configuration of |
411 | the first one to be a hostlist server by adding the following lines to | 593 | the first one to be a hostlist server by adding the following lines to |
412 | @file{peer1.conf} to enable bootstrapping server: | 594 | @file{peer1.conf} to enable bootstrapping server: |
595 | |||
413 | @example | 596 | @example |
414 | [hostlist] | 597 | [hostlist] |
415 | OPTIONS = -p | 598 | OPTIONS = -p |
416 | @end example | 599 | @end example |
417 | 600 | ||
418 | Then change @file{peer2.conf} and replace the ``@code{SERVERS}'' line in the ``@code{[hostlist]}'' section with | 601 | @noindent |
602 | Then change @file{peer2.conf} and replace the ``@code{SERVERS}'' | ||
603 | line in the ``@code{[hostlist]}'' section with | ||
419 | ``@code{http://localhost:8080/}''. Restart both peers using: | 604 | ``@code{http://localhost:8080/}''. Restart both peers using: |
605 | |||
420 | @example | 606 | @example |
421 | $ gnunet-arm -c peer1.conf -e # stop first peer | 607 | # stop first peer |
422 | $ gnunet-arm -c peer1.conf -s # start first peer | 608 | $ gnunet-arm -c peer1.conf -e |
423 | $ gnunet-arm -c peer2.conf -s # start second peer | 609 | # start first peer |
610 | $ gnunet-arm -c peer1.conf -s | ||
611 | # start second peer | ||
612 | $ gnunet-arm -c peer2.conf -s | ||
424 | @end example | 613 | @end example |
425 | 614 | ||
615 | @noindent | ||
426 | Note that if you start your peers without changing these settings, they | 616 | Note that if you start your peers without changing these settings, they |
427 | will use the ``global'' hostlist servers of the GNUnet P2P network and | 617 | will use the ``global'' hostlist servers of the GNUnet P2P network and |
428 | likely connect to those peers. At that point, debugging might become | 618 | likely connect to those peers. At that point, debugging might become |
@@ -433,16 +623,23 @@ by you. | |||
433 | @node How to connect manually | 623 | @node How to connect manually |
434 | @subsection How to connect manually | 624 | @subsection How to connect manually |
435 | 625 | ||
436 | If you want to use the @code{peerinfo} tool to connect your peers, you should: | 626 | If you want to use the @code{peerinfo} tool to connect your |
627 | peers, you should: | ||
628 | |||
437 | @itemize | 629 | @itemize |
438 | @item Set @code{FORCESTART = NO} in section @code{hostlist} (to not connect to the global GNUnet) | 630 | @item Set @code{FORCESTART = NO} in section @code{hostlist} |
439 | @item Start both peers running @command{gnunet-arm -c peer1.conf -s} and @command{gnunet-arm -c peer2.conf -s} | 631 | (to not connect to the global GNUnet) |
440 | @item Get @code{HELLO} message of the first peer running @command{gnunet-peerinfo -c peer1.conf -g} | 632 | @item Start both peers running @command{gnunet-arm -c peer1.conf -s} |
441 | @item Give the output to the second peer by running @command{gnunet-peerinfo -c peer2.conf -p '<output>'} | 633 | and @command{gnunet-arm -c peer2.conf -s} |
634 | @item Get @code{HELLO} message of the first peer running | ||
635 | @command{gnunet-peerinfo -c peer1.conf -g} | ||
636 | @item Give the output to the second peer by running | ||
637 | @command{gnunet-peerinfo -c peer2.conf -p '<output>'} | ||
442 | @end itemize | 638 | @end itemize |
443 | 639 | ||
444 | Check that they are connected using @command{gnunet-core -c peer1.conf}, | 640 | Check that they are connected using @command{gnunet-core -c peer1.conf}, |
445 | which should give you the other peer's peer identity: | 641 | which should give you the other peer's peer identity: |
642 | |||
446 | @example | 643 | @example |
447 | $ gnunet-core -c peer1.conf | 644 | $ gnunet-core -c peer1.conf |
448 | Peer `9TVUCS8P5A7ILLBGO6 [...shortened...] 1KNBJ4NGCHP3JPVULDG' | 645 | Peer `9TVUCS8P5A7ILLBGO6 [...shortened...] 1KNBJ4NGCHP3JPVULDG' |
@@ -452,94 +649,125 @@ Peer `9TVUCS8P5A7ILLBGO6 [...shortened...] 1KNBJ4NGCHP3JPVULDG' | |||
452 | @section Starting Peers Using the Testbed Service | 649 | @section Starting Peers Using the Testbed Service |
453 | @c \label{sec:testbed} | 650 | @c \label{sec:testbed} |
454 | 651 | ||
455 | GNUnet's testbed service is used for testing scenarios where a number of peers | 652 | GNUnet's testbed service is used for testing scenarios where |
456 | are to be started. The testbed can manage peers on a single host or on multiple | 653 | a number of peers are to be started. The testbed can manage peers |
457 | hosts in a distributed fashion. On a single affordable computer, it should be | 654 | on a single host or on multiple hosts in a distributed fashion. |
458 | possible to run around tens of peers without drastically increasing the load on the | 655 | On a single affordable computer, it should be possible to run |
656 | around tens of peers without drastically increasing the load on the | ||
459 | system. | 657 | system. |
460 | 658 | ||
461 | The testbed service can be access through its API | 659 | The testbed service can be access through its API |
462 | @file{include/gnunet\_testbed\_service.h}. The API provides many routines for | 660 | @file{include/gnunet\_testbed\_service.h}. The API provides many |
463 | managing a group of peers. It also provides a helper function | 661 | routines for managing a group of peers. It also provides a helper |
464 | @code{GNUNET\_TESTBED\_test\_run()} to quickly setup a minimalistic testing | 662 | function @code{GNUNET\_TESTBED\_test\_run()} to quickly setup a |
465 | environment on a single host. | 663 | minimalistic testing environment on a single host. |
466 | 664 | ||
467 | This function takes a configuration file which will be used as a template | 665 | This function takes a configuration file which will be used as a |
468 | configuration for the peers. The testbed takes care of modifying relevant | 666 | template configuration for the peers. The testbed takes care of |
469 | options in the peers' configuration such as @code{SERVICEHOME}, @code{PORT}, @code{UNIXPATH} to | 667 | modifying relevant options in the peers' configuration such as |
470 | unique values so that peers run without running into conflicts. It also checks | 668 | @code{SERVICEHOME}, @code{PORT}, @code{UNIXPATH} to unique values |
669 | so that peers run without running into conflicts. It also checks | ||
471 | and assigns the ports in configurations only if they are free. | 670 | and assigns the ports in configurations only if they are free. |
472 | 671 | ||
473 | Additionally, the testbed service also reads its options from the same | 672 | Additionally, the testbed service also reads its options from the |
474 | configuration file. Various available options and details about them can be | 673 | same configuration file. Various available options and details |
475 | found in the testbed default configuration file @file{src/testbed/testbed.conf}. | 674 | about them can be found in the testbed default configuration file |
675 | @file{src/testbed/testbed.conf}. | ||
476 | 676 | ||
477 | With the testbed API, a sample test case can be structured as follows: | 677 | With the testbed API, a sample test case can be structured as follows: |
678 | |||
478 | @example | 679 | @example |
479 | @verbatiminclude testbed_test.c | 680 | @verbatiminclude testbed_test.c |
480 | @end example | 681 | @end example |
682 | |||
683 | @noindent | ||
481 | The source code for the above listing can be found at | 684 | The source code for the above listing can be found at |
482 | @uref{https://gnunet.org/git/gnunet.git/tree/doc/testbed_test.c} | 685 | @uref{https://gnunet.org/git/gnunet.git/tree/doc/ |
483 | or in the @file{doc/} folder of your repository check-out. | 686 | documentation/testbed_test.c} |
687 | or in the @file{doc/documentation/} folder of your repository check-out. | ||
484 | After installing GNUnet, the above source code can be compiled as: | 688 | After installing GNUnet, the above source code can be compiled as: |
689 | |||
485 | @example | 690 | @example |
486 | $ export CPPFLAGS="-I/path/to/gnunet/headers" | 691 | $ export CPPFLAGS="-I/path/to/gnunet/headers" |
487 | $ export LDFLAGS="-L/path/to/gnunet/libraries" | 692 | $ export LDFLAGS="-L/path/to/gnunet/libraries" |
488 | $ gcc $CPPFLAGS $LDFLAGS -o testbed-test testbed_test.c -lgnunettestbed -lgnunetdht -lgnunetutil | 693 | $ gcc $CPPFLAGS $LDFLAGS -o testbed-test testbed_test.c \ |
489 | $ touch template.conf # Generate (empty) configuration | 694 | -lgnunettestbed -lgnunetdht -lgnunetutil |
490 | $ ./testbed-test # run it (press CTRL-C to stop) | 695 | # Generate (empty) configuration |
696 | $ touch template.conf | ||
697 | # run it (press CTRL-C to stop) | ||
698 | $ ./testbed-test | ||
491 | @end example | 699 | @end example |
492 | The @code{CPPFLAGS} and @code{LDFLAGS} are necessary if GNUnet is installed | 700 | |
493 | into a different directory other than @file{/usr/local}. | 701 | @noindent |
494 | 702 | The @code{CPPFLAGS} and @code{LDFLAGS} are necessary if GNUnet | |
495 | All of testbed API's peer management functions treat management actions as | 703 | is installed into a different directory other than @file{/usr/local}. |
496 | operations and return operation handles. It is expected that the operations | 704 | |
497 | begin immediately, but they may get delayed (to balance out load on the system). | 705 | All of testbed API's peer management functions treat management |
498 | The program using the API then has to take care of marking the operation as | 706 | actions as operations and return operation handles. It is expected |
499 | ``done'' so that its associated resources can be freed immediately and other | 707 | that the operations begin immediately, but they may get delayed (to |
500 | waiting operations can be executed. Operations will be canceled if they are | 708 | balance out load on the system). The program using the API then has |
709 | to take care of marking the operation as ``done'' so that its | ||
710 | associated resources can be freed immediately and other waiting | ||
711 | operations can be executed. Operations will be canceled if they are | ||
501 | marked as ``done'' before their completion. | 712 | marked as ``done'' before their completion. |
502 | 713 | ||
503 | An operation is treated as completed when it succeeds or fails. Completion of | 714 | An operation is treated as completed when it succeeds or fails. |
504 | an operation is either conveyed as events through @i{controller event callback} | 715 | Completion of an operation is either conveyed as events through |
505 | or through respective operation completion callbacks. In functions | 716 | @dfn{controller event callback} or through respective |
506 | which support completion notification through both controller event callback and | 717 | @dfn{operation completion callbacks}. |
507 | operation completion callback, first the controller event callback will be | 718 | In functions which support completion notification |
508 | called. If the operation is not marked as done in that callback or if the | 719 | through both controller event callback and operation |
509 | callback is given as NULL when creating the operation, the operation completion | 720 | completion callback, first the controller event callback will be |
510 | callback will be called. The API documentation shows which event are to be | 721 | called. If the operation is not marked as done in that callback |
511 | expected in the controller event notifications. It also documents any | 722 | or if the callback is given as NULL when creating the operation, |
512 | exceptional behaviour. | 723 | the operation completion callback will be called. The API |
513 | 724 | documentation shows which event are to be expected in the | |
514 | Once the peers are started, test cases often need to connect some of the peers' | 725 | controller event notifications. It also documents any exceptional |
515 | services. Normally, opening a connect to a peer's service requires the peer's | 726 | behaviour. |
516 | configuration. While using testbed, the testbed automatically generates | 727 | |
517 | per-peer configuration. Accessing those configurations directly through file | 728 | Once the peers are started, test cases often need to connect |
518 | system is discouraged as their locations are dynamically created and will be | 729 | some of the peers' services. Normally, opening a connect to |
519 | different among various runs of testbed. To make access to these configurations | 730 | a peer's service requires the peer's configuration. While using |
520 | easy, testbed API provides the function | 731 | testbed, the testbed automatically generates per-peer configuration. |
521 | @code{GNUNET\_TESTBED\_service\_connect()}. This function fetches the | 732 | Accessing those configurations directly through file system is |
522 | configuration of a given peer and calls the @i{Connect Adapter}. | 733 | discouraged as their locations are dynamically created and will be |
523 | In the example code, it is the @code{dht\_ca}. A connect adapter is expected | 734 | different among various runs of testbed. To make access to these |
524 | to open the connection to the needed service by using the provided configuration | 735 | configurations easy, testbed API provides the function |
525 | and return the created service connection handle. Successful connection to the | 736 | @code{GNUNET\_TESTBED\_service\_connect()}. This function fetches |
526 | needed service is signaled through @code{service\_connect\_comp\_cb}. | 737 | the configuration of a given peer and calls the @dfn{Connect Adapter}. |
527 | 738 | In the example code, it is the @code{dht\_ca}. A connect adapter is | |
528 | A dual to connect adapter is the @i{Disconnect Adapter}. This callback is | 739 | expected to open the connection to the needed service by using the |
529 | called after the connect adapter has been called when the operation from | 740 | provided configuration and return the created service connection handle. |
530 | @code{GNUNET\_TESTBED\_service\_connect()} is marked as ``done''. It has to | 741 | Successful connection to the needed service is signaled through |
531 | disconnect from the service with the provided service handle (@code{op\_result}). | 742 | @code{service\_connect\_comp\_cb}. |
743 | |||
744 | A dual to connect adapter is the @dfn{Disconnect Adapter}. This callback | ||
745 | is called after the connect adapter has been called when the operation | ||
746 | from @code{GNUNET\_TESTBED\_service\_connect()} is marked as ``done''. | ||
747 | It has to disconnect from the service with the provided service | ||
748 | handle (@code{op\_result}). | ||
532 | 749 | ||
533 | Exercise: Find out how many peers you can run on your system. | 750 | Exercise: Find out how many peers you can run on your system. |
534 | 751 | ||
535 | Exercise: Find out how to create a 2D torus topology by changing the | 752 | Exercise: Find out how to create a 2D torus topology by changing the |
536 | options in the configuration file. See @uref{https://gnunet.org/supported-topologies} | 753 | options in the configuration file. |
537 | Then use the DHT API to store and retrieve values in the | 754 | @xref{Supported Topologies, The GNUnet Reference Documentation ,, gnunet, The GNUnet Reference Documentation}, |
538 | network. | 755 | then use the DHT API to store and retrieve values in the network. |
539 | 756 | ||
540 | @node Developing Applications | 757 | @node Developing Applications |
541 | @chapter Developing Applications | 758 | @chapter Developing Applications |
542 | 759 | ||
760 | @menu | ||
761 | * gnunet-ext:: | ||
762 | * Adapting the Template:: | ||
763 | * Writing a Client Application:: | ||
764 | * Writing a Service:: | ||
765 | * Interacting directly with other Peers using the CORE Service:: | ||
766 | * Storing peer-specific data using the PEERSTORE service:: | ||
767 | * Using the DHT:: | ||
768 | * Debugging with gnunet-arm:: | ||
769 | @end menu | ||
770 | |||
543 | @node gnunet-ext | 771 | @node gnunet-ext |
544 | @section gnunet-ext | 772 | @section gnunet-ext |
545 | To develop a new peer-to-peer application or to extend GNUnet we provide | 773 | To develop a new peer-to-peer application or to extend GNUnet we provide |
@@ -556,17 +784,21 @@ $ make install | |||
556 | $ make check | 784 | $ make check |
557 | @end example | 785 | @end example |
558 | 786 | ||
559 | The GNUnet ext template includes examples and a working buildsystem for a new GNUnet service. | 787 | @noindent |
560 | A common GNUnet service consists of the following parts which will be discussed in detail in the | 788 | The GNUnet ext template includes examples and a working buildsystem |
561 | remainder of this document. The functionality of a GNUnet service is implemented in: | 789 | for a new GNUnet service. A common GNUnet service consists of the |
790 | following parts which will be discussed in detail in the remainder | ||
791 | of this document. The functionality of a GNUnet service is implemented in: | ||
562 | 792 | ||
563 | @itemize | 793 | @itemize |
564 | @item the GNUnet service (gnunet-ext/src/ext/gnunet-service-ext.c) | 794 | @item the GNUnet service (gnunet-ext/src/ext/gnunet-service-ext.c) |
565 | @item the client API (gnunet-ext/src/ext/ext_api.c) | 795 | @item the client API (gnunet-ext/src/ext/ext_api.c) |
566 | @item the client application using the service API (gnunet-ext/src/ext/gnunet-ext.c) | 796 | @item the client application using the service API |
797 | (gnunet-ext/src/ext/gnunet-ext.c) | ||
567 | @end itemize | 798 | @end itemize |
568 | 799 | ||
569 | The interfaces for these entities are defined in: | 800 | The interfaces for these entities are defined in: |
801 | |||
570 | @itemize | 802 | @itemize |
571 | @item client API interface (gnunet-ext/src/ext/ext.h) | 803 | @item client API interface (gnunet-ext/src/ext/ext.h) |
572 | @item the service interface (gnunet-ext/src/include/gnunet_service_SERVICE.h) | 804 | @item the service interface (gnunet-ext/src/include/gnunet_service_SERVICE.h) |
@@ -575,9 +807,11 @@ The interfaces for these entities are defined in: | |||
575 | 807 | ||
576 | 808 | ||
577 | In addition the ext systems provides: | 809 | In addition the ext systems provides: |
810 | |||
578 | @itemize | 811 | @itemize |
579 | @item a test testing the API (gnunet-ext/src/ext/test_ext_api.c) | 812 | @item a test testing the API (gnunet-ext/src/ext/test_ext_api.c) |
580 | @item a configuration template for the service (gnunet-ext/src/ext/ext.conf.in) | 813 | @item a configuration template for the service |
814 | (gnunet-ext/src/ext/ext.conf.in) | ||
581 | @end itemize | 815 | @end itemize |
582 | 816 | ||
583 | @node Adapting the Template | 817 | @node Adapting the Template |
@@ -585,26 +819,34 @@ In addition the ext systems provides: | |||
585 | 819 | ||
586 | The first step for writing any extension with a new service is to | 820 | The first step for writing any extension with a new service is to |
587 | ensure that the @file{ext.conf.in} file contains entries for the | 821 | ensure that the @file{ext.conf.in} file contains entries for the |
588 | @code{UNIXPATH}, @code{PORT} and @code{BINARY} for the service in a section named after | 822 | @code{UNIXPATH}, @code{PORT} and @code{BINARY} for the service in a |
589 | the service. | 823 | section named after the service. |
590 | 824 | ||
591 | If you want to adapt the template rename the @file{ext.conf.in} to match your | 825 | If you want to adapt the template rename the @file{ext.conf.in} to |
592 | services name, you have to modify the @code{AC\_OUTPUT} section in @file{configure.ac} | 826 | match your services name, you have to modify the @code{AC\_OUTPUT} |
593 | in the @file{gnunet-ext} root. | 827 | section in @file{configure.ac} in the @file{gnunet-ext} root. |
594 | 828 | ||
595 | @node Writing a Client Application | 829 | @node Writing a Client Application |
596 | @section Writing a Client Application | 830 | @section Writing a Client Application |
597 | 831 | ||
598 | When writing any client application (for example, a command-line | 832 | When writing any client application (for example, a command-line |
599 | tool), the basic structure is to start with the @code{GNUNET\_PROGRAM\_run} | 833 | tool), the basic structure is to start with the |
600 | function. This function will parse command-line options, setup the scheduler | 834 | @code{GNUNET\_PROGRAM\_run} function. This function will parse |
601 | and then invoke the @code{run} function (with the remaining non-option arguments) | 835 | command-line options, setup the scheduler and then invoke the |
602 | and a handle to the parsed configuration (and the configuration file name that was | 836 | @code{run} function (with the remaining non-option arguments) |
603 | used, which is typically not needed): | 837 | and a handle to the parsed configuration (and the configuration |
838 | file name that was used, which is typically not needed): | ||
839 | |||
604 | @example | 840 | @example |
605 | @verbatiminclude tutorial-examples/001.c | 841 | @verbatiminclude tutorial-examples/001.c |
606 | @end example | 842 | @end example |
607 | 843 | ||
844 | @menu | ||
845 | * Handling command-line options:: | ||
846 | * Writing a Client Library:: | ||
847 | * Writing a user interface:: | ||
848 | @end menu | ||
849 | |||
608 | @node Handling command-line options | 850 | @node Handling command-line options |
609 | @subsection Handling command-line options | 851 | @subsection Handling command-line options |
610 | 852 | ||
@@ -612,6 +854,7 @@ Options can then be added easily by adding global variables and | |||
612 | expanding the @code{options} array. For example, the following would | 854 | expanding the @code{options} array. For example, the following would |
613 | add a string-option and a binary flag (defaulting to @code{NULL} and | 855 | add a string-option and a binary flag (defaulting to @code{NULL} and |
614 | @code{GNUNET\_NO} respectively): | 856 | @code{GNUNET\_NO} respectively): |
857 | |||
615 | @example | 858 | @example |
616 | @verbatiminclude tutorial-examples/002.c | 859 | @verbatiminclude tutorial-examples/002.c |
617 | @end example | 860 | @end example |
@@ -657,15 +900,23 @@ Unique message types must be defined for each message struct in the | |||
657 | @file{gnunet\_protocols.h} header (or an extension-specific include | 900 | @file{gnunet\_protocols.h} header (or an extension-specific include |
658 | file). | 901 | file). |
659 | 902 | ||
903 | @menu | ||
904 | * Connecting to the Service:: | ||
905 | * Sending messages:: | ||
906 | * Receiving Replies from the Service:: | ||
907 | @end menu | ||
908 | |||
660 | @node Connecting to the Service | 909 | @node Connecting to the Service |
661 | @subsubsection Connecting to the Service | 910 | @subsubsection Connecting to the Service |
662 | 911 | ||
663 | Before a client library can implement the application-specific protocol | 912 | Before a client library can implement the application-specific protocol |
664 | with the service, a connection must be created: | 913 | with the service, a connection must be created: |
914 | |||
665 | @example | 915 | @example |
666 | @verbatiminclude tutorial-examples/003.c | 916 | @verbatiminclude tutorial-examples/003.c |
667 | @end example | 917 | @end example |
668 | 918 | ||
919 | @noindent | ||
669 | As a result a @code{GNUNET\_MQ\_Handle} is returned | 920 | As a result a @code{GNUNET\_MQ\_Handle} is returned |
670 | which can to used henceforth to transmit messages to the service. | 921 | which can to used henceforth to transmit messages to the service. |
671 | The complete MQ API can be found in @file{gnunet\_mq\_lib.h}. | 922 | The complete MQ API can be found in @file{gnunet\_mq\_lib.h}. |
@@ -678,27 +929,35 @@ there are errors communicating with the service. | |||
678 | @node Sending messages | 929 | @node Sending messages |
679 | @subsubsection Sending messages | 930 | @subsubsection Sending messages |
680 | 931 | ||
681 | In GNUnet, messages are always sent beginning with a @code{struct GNUNET\_MessageHeader} | 932 | In GNUnet, messages are always sent beginning with a |
682 | in big endian format. This header defines the size and the type of the | 933 | @code{struct GNUNET\_MessageHeader} in big endian format. |
934 | This header defines the size and the type of the | ||
683 | message, the payload follows after this header. | 935 | message, the payload follows after this header. |
936 | |||
684 | @example | 937 | @example |
685 | @verbatiminclude tutorial-examples/004.c | 938 | @verbatiminclude tutorial-examples/004.c |
686 | @end example | 939 | @end example |
687 | 940 | ||
941 | @noindent | ||
688 | Existing message types are defined in @file{gnunet\_protocols.h}. | 942 | Existing message types are defined in @file{gnunet\_protocols.h}. |
689 | A common way to create a message is with an envelope: | 943 | A common way to create a message is with an envelope: |
944 | |||
690 | @example | 945 | @example |
691 | @verbatiminclude tutorial-examples/005.c | 946 | @verbatiminclude tutorial-examples/005.c |
692 | @end example | 947 | @end example |
693 | 948 | ||
949 | @noindent | ||
694 | Exercise: Define a message struct that includes a 32-bit | 950 | Exercise: Define a message struct that includes a 32-bit |
695 | unsigned integer in addition to the standard GNUnet MessageHeader. | 951 | unsigned integer in addition to the standard GNUnet MessageHeader. |
696 | Add a C struct and define a fresh protocol number for your message. | 952 | Add a C struct and define a fresh protocol number for your message. |
697 | Protocol numbers in gnunet-ext are defined in @file{gnunet-ext/src/include/gnunet_protocols_ext.h} | 953 | Protocol numbers in gnunet-ext are defined |
954 | in @file{gnunet-ext/src/include/gnunet_protocols_ext.h} | ||
698 | 955 | ||
699 | Exercise: Find out how you can determine the number of messages in a message queue. | 956 | Exercise: Find out how you can determine the number of messages |
957 | in a message queue. | ||
700 | 958 | ||
701 | Exercise: Find out how you can determine when a message you have queued was actually transmitted. | 959 | Exercise: Find out how you can determine when a message you |
960 | have queued was actually transmitted. | ||
702 | 961 | ||
703 | Exercise: Define a helper function to transmit a 32-bit | 962 | Exercise: Define a helper function to transmit a 32-bit |
704 | unsigned integer (as payload) to a service using some given client | 963 | unsigned integer (as payload) to a service using some given client |
@@ -717,16 +976,19 @@ to actually process the message. Fixed size messages are fully | |||
717 | checked by the MQ-logic, and thus only need to provide the handler | 976 | checked by the MQ-logic, and thus only need to provide the handler |
718 | to process the message. Note that the prefixes @code{check\_} | 977 | to process the message. Note that the prefixes @code{check\_} |
719 | and @code{handle\_} are mandatory. | 978 | and @code{handle\_} are mandatory. |
979 | |||
720 | @example | 980 | @example |
721 | @verbatiminclude tutorial-examples/006.c | 981 | @verbatiminclude tutorial-examples/006.c |
722 | @end example | 982 | @end example |
723 | 983 | ||
984 | @noindent | ||
724 | Exercise: Expand your helper function to receive a response message | 985 | Exercise: Expand your helper function to receive a response message |
725 | (for example, containing just the @code{struct GNUnet MessageHeader} | 986 | (for example, containing just the @code{struct GNUnet MessageHeader} |
726 | without any payload). Upon receiving the service's response, you | 987 | without any payload). Upon receiving the service's response, you |
727 | should call a callback provided to your helper function's API. | 988 | should call a callback provided to your helper function's API. |
728 | 989 | ||
729 | Exercise: Figure out where you can pass values to the closures (@code{cls}). | 990 | Exercise: Figure out where you can pass values to the |
991 | closures (@code{cls}). | ||
730 | 992 | ||
731 | @node Writing a user interface | 993 | @node Writing a user interface |
732 | @subsection Writing a user interface | 994 | @subsection Writing a user interface |
@@ -743,27 +1005,36 @@ command-line to the service. | |||
743 | @node Writing a Service | 1005 | @node Writing a Service |
744 | @section Writing a Service | 1006 | @section Writing a Service |
745 | 1007 | ||
746 | Before you can test the client you've written so far, you'll need to also | 1008 | Before you can test the client you've written so far, you'll |
747 | implement the corresponding service. | 1009 | need to also implement the corresponding service. |
1010 | |||
1011 | @menu | ||
1012 | * Code Placement:: | ||
1013 | * Starting a Service:: | ||
1014 | @end menu | ||
748 | 1015 | ||
749 | @node Code Placement | 1016 | @node Code Placement |
750 | @subsection Code Placement | 1017 | @subsection Code Placement |
751 | 1018 | ||
752 | New services are placed in their own subdirectory under @file{gnunet/src}. | 1019 | New services are placed in their own subdirectory under |
753 | This subdirectory should contain the API implementation file @file{SERVICE\_api.c}, | 1020 | @file{gnunet/src}. This subdirectory should contain the API |
754 | the description of the client-service protocol @file{SERVICE.h} and P2P protocol | 1021 | implementation file @file{SERVICE\_api.c}, the description of |
1022 | the client-service protocol @file{SERVICE.h} and P2P protocol | ||
755 | @file{SERVICE\_protocol.h}, the implementation of the service itself | 1023 | @file{SERVICE\_protocol.h}, the implementation of the service itself |
756 | @file{gnunet-service-SERVICE.h} and several files for tests, including test code | 1024 | @file{gnunet-service-SERVICE.h} and several files for tests, |
757 | and configuration files. | 1025 | including test code and configuration files. |
758 | 1026 | ||
759 | @node Starting a Service | 1027 | @node Starting a Service |
760 | @subsection Starting a Service | 1028 | @subsection Starting a Service |
761 | 1029 | ||
762 | The key API definition for creating a service is the @code{GNUNET\_SERVICE\_MAIN} macro: | 1030 | The key API definition for creating a service is the |
1031 | @code{GNUNET\_SERVICE\_MAIN} macro: | ||
1032 | |||
763 | @example | 1033 | @example |
764 | @verbatiminclude tutorial-examples/007.c | 1034 | @verbatiminclude tutorial-examples/007.c |
765 | @end example | 1035 | @end example |
766 | 1036 | ||
1037 | @noindent | ||
767 | In addition to the service name and flags, the macro takes three | 1038 | In addition to the service name and flags, the macro takes three |
768 | functions, typically called @code{run}, @code{client\_connect\_cb} and | 1039 | functions, typically called @code{run}, @code{client\_connect\_cb} and |
769 | @code{client\_disconnect\_cb} as well as an array of message handlers | 1040 | @code{client\_disconnect\_cb} as well as an array of message handlers |
@@ -771,10 +1042,12 @@ that will be called for incoming messages from clients. | |||
771 | 1042 | ||
772 | A minimal version of the three central service funtions would look | 1043 | A minimal version of the three central service funtions would look |
773 | like this: | 1044 | like this: |
1045 | |||
774 | @example | 1046 | @example |
775 | @verbatiminclude tutorial-examples/008.c | 1047 | @verbatiminclude tutorial-examples/008.c |
776 | @end example | 1048 | @end example |
777 | 1049 | ||
1050 | @noindent | ||
778 | Exercise: Write a stub service that processes no messages at all | 1051 | Exercise: Write a stub service that processes no messages at all |
779 | in your code. Create a default configuration for it, integrate it | 1052 | in your code. Create a default configuration for it, integrate it |
780 | with the build system and start the service from | 1053 | with the build system and start the service from |
@@ -787,8 +1060,9 @@ Exercise: Figure out how to send messages from the service back to the | |||
787 | client. | 1060 | client. |
788 | 1061 | ||
789 | Each handler function in the service @b{must} eventually (possibly in some | 1062 | Each handler function in the service @b{must} eventually (possibly in some |
790 | asynchronous continuation) call @code{GNUNET\_SERVICE\_client\_continue()}. | 1063 | asynchronous continuation) call |
791 | Only after this call additional messages from the same client may | 1064 | @code{GNUNET\_SERVICE\_client\_continue()}. Only after this call |
1065 | additional messages from the same client may | ||
792 | be processed. This way, the service can throttle processing messages | 1066 | be processed. This way, the service can throttle processing messages |
793 | from the same client. | 1067 | from the same client. |
794 | 1068 | ||
@@ -804,24 +1078,35 @@ FIXME: This section still needs to be updated to the lastest API! | |||
804 | One of the most important services in GNUnet is the @code{CORE} service | 1078 | One of the most important services in GNUnet is the @code{CORE} service |
805 | managing connections between peers and handling encryption between peers. | 1079 | managing connections between peers and handling encryption between peers. |
806 | 1080 | ||
807 | One of the first things any service that extends the P2P protocol typically does | 1081 | One of the first things any service that extends the P2P protocol |
808 | is connect to the @code{CORE} service using: | 1082 | typically does is connect to the @code{CORE} service using: |
1083 | |||
809 | @example | 1084 | @example |
810 | @verbatiminclude tutorial-examples/009.c | 1085 | @verbatiminclude tutorial-examples/009.c |
811 | @end example | 1086 | @end example |
812 | 1087 | ||
1088 | @menu | ||
1089 | * New P2P connections:: | ||
1090 | * Receiving P2P Messages:: | ||
1091 | * Sending P2P Messages:: | ||
1092 | * End of P2P connections:: | ||
1093 | @end menu | ||
1094 | |||
813 | @node New P2P connections | 1095 | @node New P2P connections |
814 | @subsection New P2P connections | 1096 | @subsection New P2P connections |
815 | 1097 | ||
816 | Before any traffic with a different peer can be exchanged, the peer must be | 1098 | Before any traffic with a different peer can be exchanged, the peer must |
817 | known to the service. This is notified by the @code{CORE} @code{connects} callback, | 1099 | be known to the service. This is notified by the @code{CORE} |
818 | which communicates the identity of the new peer to the service: | 1100 | @code{connects} callback, which communicates the identity of the new |
1101 | peer to the service: | ||
1102 | |||
819 | @example | 1103 | @example |
820 | @verbatiminclude tutorial-examples/010.c | 1104 | @verbatiminclude tutorial-examples/010.c |
821 | @end example | 1105 | @end example |
822 | 1106 | ||
1107 | @noindent | ||
823 | Note that whatever you return from @code{connects} is given as the | 1108 | Note that whatever you return from @code{connects} is given as the |
824 | @i{cls} argument to the message handlers for messages from | 1109 | @code{cls} argument to the message handlers for messages from |
825 | the respective peer. | 1110 | the respective peer. |
826 | 1111 | ||
827 | Exercise: Create a service that connects to the @code{CORE}. Then | 1112 | Exercise: Create a service that connects to the @code{CORE}. Then |
@@ -832,7 +1117,7 @@ callback is invoked. | |||
832 | @subsection Receiving P2P Messages | 1117 | @subsection Receiving P2P Messages |
833 | 1118 | ||
834 | To receive messages from @code{CORE}, you pass the desired | 1119 | To receive messages from @code{CORE}, you pass the desired |
835 | @i{handlers} to the @code{GNUNET\_CORE\_connect()} function, | 1120 | @code{handlers} to the @code{GNUNET\_CORE\_connect()} function, |
836 | just as we showed for services. | 1121 | just as we showed for services. |
837 | 1122 | ||
838 | It is your responsibility to process messages fast enough or | 1123 | It is your responsibility to process messages fast enough or |
@@ -848,8 +1133,8 @@ the two peers are connected? Why? | |||
848 | @node Sending P2P Messages | 1133 | @node Sending P2P Messages |
849 | @subsection Sending P2P Messages | 1134 | @subsection Sending P2P Messages |
850 | 1135 | ||
851 | You can transmit messages to other peers using the @i{mq} you were | 1136 | You can transmit messages to other peers using the @code{mq} you were |
852 | given during the @code{connect} callback. Note that the @i{mq} | 1137 | given during the @code{connect} callback. Note that the @code{mq} |
853 | automatically is released upon @code{disconnect} and that you must | 1138 | automatically is released upon @code{disconnect} and that you must |
854 | not use it afterwards. | 1139 | not use it afterwards. |
855 | 1140 | ||
@@ -866,23 +1151,27 @@ you stop the peer that is receiving your messages? | |||
866 | @node End of P2P connections | 1151 | @node End of P2P connections |
867 | @subsection End of P2P connections | 1152 | @subsection End of P2P connections |
868 | 1153 | ||
869 | If a message handler returns @code{GNUNET\_SYSERR}, the remote peer shuts down or | 1154 | If a message handler returns @code{GNUNET\_SYSERR}, the remote |
870 | there is an unrecoverable network disconnection, CORE notifies the service that | 1155 | peer shuts down or there is an unrecoverable network |
871 | the peer disconnected. After this notification no more messages will be received | 1156 | disconnection, CORE notifies the service that the peer disconnected. |
872 | from the peer and the service is no longer allowed to send messages to the peer. | 1157 | After this notification no more messages will be received from the |
1158 | peer and the service is no longer allowed to send messages to the peer. | ||
873 | The disconnect callback looks like the following: | 1159 | The disconnect callback looks like the following: |
1160 | |||
874 | @example | 1161 | @example |
875 | @verbatiminclude tutorial-examples/011.c | 1162 | @verbatiminclude tutorial-examples/011.c |
876 | @end example | 1163 | @end example |
877 | 1164 | ||
1165 | @noindent | ||
878 | Exercise: Fix your service to handle peer disconnects. | 1166 | Exercise: Fix your service to handle peer disconnects. |
879 | 1167 | ||
880 | @node Storing peer-specific data using the PEERSTORE service | 1168 | @node Storing peer-specific data using the PEERSTORE service |
881 | @section Storing peer-specific data using the PEERSTORE service | 1169 | @section Storing peer-specific data using the PEERSTORE service |
882 | 1170 | ||
883 | GNUnet's PEERSTORE service offers a persistorage for arbitrary peer-specific data. | 1171 | GNUnet's PEERSTORE service offers a persistorage for arbitrary |
884 | Other GNUnet services can use the PEERSTORE to store, retrieve and monitor data records. | 1172 | peer-specific data. Other GNUnet services can use the PEERSTORE |
885 | Each data record stored with PEERSTORE contains the following fields: | 1173 | to store, retrieve and monitor data records. Each data record |
1174 | stored with PEERSTORE contains the following fields: | ||
886 | 1175 | ||
887 | @itemize | 1176 | @itemize |
888 | @item subsystem: Name of the subsystem responsible for the record. | 1177 | @item subsystem: Name of the subsystem responsible for the record. |
@@ -897,43 +1186,60 @@ The first step is to start a connection to the PEERSTORE service: | |||
897 | @verbatiminclude tutorial-examples/012.c | 1186 | @verbatiminclude tutorial-examples/012.c |
898 | @end example | 1187 | @end example |
899 | 1188 | ||
900 | The service handle @code{peerstore_handle} will be needed for all subsequent | 1189 | The service handle @code{peerstore_handle} will be needed for |
901 | PEERSTORE operations. | 1190 | all subsequent PEERSTORE operations. |
1191 | |||
1192 | @menu | ||
1193 | * Storing records:: | ||
1194 | * Retrieving records:: | ||
1195 | * Monitoring records:: | ||
1196 | * Disconnecting from PEERSTORE:: | ||
1197 | @end menu | ||
902 | 1198 | ||
903 | @node Storing records | 1199 | @node Storing records |
904 | @subsection Storing records | 1200 | @subsection Storing records |
905 | 1201 | ||
906 | To store a new record, use the following function: | 1202 | To store a new record, use the following function: |
1203 | |||
907 | @example | 1204 | @example |
908 | @verbatiminclude tutorial-examples/013.c | 1205 | @verbatiminclude tutorial-examples/013.c |
909 | @end example | 1206 | @end example |
910 | 1207 | ||
911 | The @code{options} parameter can either be @code{GNUNET_PEERSTORE_STOREOPTION_MULTIPLE} | 1208 | @noindent |
912 | which means that multiple values can be stored under the same key combination (subsystem, peerid, key), | 1209 | The @code{options} parameter can either be |
913 | or @code{GNUNET_PEERSTORE_STOREOPTION_REPLACE} which means that PEERSTORE will replace any | 1210 | @code{GNUNET_PEERSTORE_STOREOPTION_MULTIPLE} which means that multiple |
914 | existing values under the given key combination (subsystem, peerid, key) with the new given value. | 1211 | values can be stored under the same key combination |
1212 | (subsystem, peerid, key), or @code{GNUNET_PEERSTORE_STOREOPTION_REPLACE} | ||
1213 | which means that PEERSTORE will replace any existing values under the | ||
1214 | given key combination (subsystem, peerid, key) with the new given value. | ||
915 | 1215 | ||
916 | The continuation function @code{cont} will be called after the store request is successfully | 1216 | The continuation function @code{cont} will be called after the store |
917 | sent to the PEERSTORE service. This does not guarantee that the record is successfully stored, only | 1217 | request is successfully sent to the PEERSTORE service. This does not |
918 | that it was received by the service. | 1218 | guarantee that the record is successfully stored, only that it was |
1219 | received by the service. | ||
1220 | |||
1221 | The @code{GNUNET_PEERSTORE_store} function returns a handle to the store | ||
1222 | operation. This handle can be used to cancel the store operation only | ||
1223 | before the continuation function is called: | ||
919 | 1224 | ||
920 | The @code{GNUNET_PEERSTORE_store} function returns a handle to the store operation. This handle | ||
921 | can be used to cancel the store operation only before the continuation function is called: | ||
922 | @example | 1225 | @example |
923 | void | 1226 | @verbatiminclude tutorial-examples/013.1.c |
924 | GNUNET_PEERSTORE_store_cancel (struct GNUNET_PEERSTORE_StoreContext *sc); | ||
925 | @end example | 1227 | @end example |
926 | 1228 | ||
927 | @node Retrieving records | 1229 | @node Retrieving records |
928 | @subsection Retrieving records | 1230 | @subsection Retrieving records |
929 | 1231 | ||
930 | To retrieve stored records, use the following function: | 1232 | To retrieve stored records, use the following function: |
1233 | |||
931 | @example | 1234 | @example |
932 | @verbatiminclude tutorial-examples/014.c | 1235 | @verbatiminclude tutorial-examples/014.c |
933 | @end example | 1236 | @end example |
934 | 1237 | ||
935 | The values of @code{peer} and @code{key} can be @code{NULL}. This allows the | 1238 | @noindent |
936 | iteration over values stored under any of the following key combinations: | 1239 | The values of @code{peer} and @code{key} can be @code{NULL}. This |
1240 | allows the iteration over values stored under any of the following | ||
1241 | key combinations: | ||
1242 | |||
937 | @itemize | 1243 | @itemize |
938 | @item (subsystem) | 1244 | @item (subsystem) |
939 | @item (subsystem, peerid) | 1245 | @item (subsystem, peerid) |
@@ -941,25 +1247,32 @@ iteration over values stored under any of the following key combinations: | |||
941 | @item (subsystem, peerid, key) | 1247 | @item (subsystem, peerid, key) |
942 | @end itemize | 1248 | @end itemize |
943 | 1249 | ||
944 | The @code{callback} function will be called once with each retrieved record and once | 1250 | The @code{callback} function will be called once with each retrieved |
945 | more with a @code{NULL} record to signal the end of results. | 1251 | record and once more with a @code{NULL} record to signal the end of |
1252 | results. | ||
946 | 1253 | ||
947 | The @code{GNUNET_PEERSTORE_iterate} function returns a handle to the iterate operation. This | 1254 | The @code{GNUNET_PEERSTORE_iterate} function returns a handle to the |
948 | handle can be used to cancel the iterate operation only before the callback function is called with | 1255 | iterate operation. This handle can be used to cancel the iterate |
949 | a @code{NULL} record. | 1256 | operation only before the callback function is called with a |
1257 | @code{NULL} record. | ||
950 | 1258 | ||
951 | @node Monitoring records | 1259 | @node Monitoring records |
952 | @subsection Monitoring records | 1260 | @subsection Monitoring records |
953 | 1261 | ||
954 | PEERSTORE offers the functionality of monitoring for new records stored under a specific key | 1262 | PEERSTORE offers the functionality of monitoring for new records |
955 | combination (subsystem, peerid, key). To start the monitoring, use the following function: | 1263 | stored under a specific key combination (subsystem, peerid, key). |
1264 | To start the monitoring, use the following function: | ||
1265 | |||
956 | @example | 1266 | @example |
957 | @verbatiminclude tutorial-examples/015.c | 1267 | @verbatiminclude tutorial-examples/015.c |
958 | @end example | 1268 | @end example |
959 | 1269 | ||
960 | Whenever a new record is stored under the given key combination, the @code{callback} function | 1270 | @noindent |
961 | will be called with this new record. This will continue until the connection to the PEERSTORE service | 1271 | Whenever a new record is stored under the given key combination, |
962 | is broken or the watch operation is canceled: | 1272 | the @code{callback} function will be called with this new |
1273 | record. This will continue until the connection to the PEERSTORE | ||
1274 | service is broken or the watch operation is canceled: | ||
1275 | |||
963 | @example | 1276 | @example |
964 | @verbatiminclude tutorial-examples/016.c | 1277 | @verbatiminclude tutorial-examples/016.c |
965 | @end example | 1278 | @end example |
@@ -967,15 +1280,18 @@ is broken or the watch operation is canceled: | |||
967 | @node Disconnecting from PEERSTORE | 1280 | @node Disconnecting from PEERSTORE |
968 | @subsection Disconnecting from PEERSTORE | 1281 | @subsection Disconnecting from PEERSTORE |
969 | 1282 | ||
970 | When the connection to the PEERSTORE service is no longer needed, disconnect using the following | 1283 | When the connection to the PEERSTORE service is no longer needed, |
971 | function: | 1284 | disconnect using the following function: |
1285 | |||
972 | @example | 1286 | @example |
973 | @verbatiminclude tutorial-examples/017.c | 1287 | @verbatiminclude tutorial-examples/017.c |
974 | @end example | 1288 | @end example |
975 | 1289 | ||
976 | If the @code{sync_first} flag is set to @code{GNUNET_YES}, the API will delay the | 1290 | @noindent |
977 | disconnection until all store requests are received by the PEERSTORE service. Otherwise, | 1291 | If the @code{sync_first} flag is set to @code{GNUNET_YES}, |
978 | it will disconnect immediately. | 1292 | the API will delay the disconnection until all store requests |
1293 | are received by the PEERSTORE service. Otherwise, it will | ||
1294 | disconnect immediately. | ||
979 | 1295 | ||
980 | @node Using the DHT | 1296 | @node Using the DHT |
981 | @section Using the DHT | 1297 | @section Using the DHT |
@@ -984,39 +1300,53 @@ The DHT allows to store data so other peers in the P2P network can | |||
984 | access it and retrieve data stored by any peers in the network. | 1300 | access it and retrieve data stored by any peers in the network. |
985 | This section will explain how to use the DHT. Of course, the first | 1301 | This section will explain how to use the DHT. Of course, the first |
986 | thing to do is to connect to the DHT service: | 1302 | thing to do is to connect to the DHT service: |
1303 | |||
987 | @example | 1304 | @example |
988 | @verbatiminclude tutorial-examples/018.c | 1305 | @verbatiminclude tutorial-examples/018.c |
989 | @end example | 1306 | @end example |
990 | 1307 | ||
1308 | @noindent | ||
991 | The second parameter indicates how many requests in parallel to expect. | 1309 | The second parameter indicates how many requests in parallel to expect. |
992 | It is not a hard limit, but a good approximation will make the DHT more | 1310 | It is not a hard limit, but a good approximation will make the DHT more |
993 | efficient. | 1311 | efficient. |
994 | 1312 | ||
1313 | @menu | ||
1314 | * Storing data in the DHT:: | ||
1315 | * Obtaining data from the DHT:: | ||
1316 | * Implementing a block plugin:: | ||
1317 | * Monitoring the DHT:: | ||
1318 | @end menu | ||
1319 | |||
995 | @node Storing data in the DHT | 1320 | @node Storing data in the DHT |
996 | @subsection Storing data in the DHT | 1321 | @subsection Storing data in the DHT |
997 | Since the DHT is a dynamic environment (peers join and leave frequently) | 1322 | Since the DHT is a dynamic environment (peers join and leave frequently) |
998 | the data that we put in the DHT does not stay there indefinitely. It is | 1323 | the data that we put in the DHT does not stay there indefinitely. It is |
999 | important to ``refresh'' the data periodically by simply storing it again, | 1324 | important to ``refresh'' the data periodically by simply storing it |
1000 | in order to make sure other peers can access it. | 1325 | again, in order to make sure other peers can access it. |
1001 | 1326 | ||
1002 | The put API call offers a callback to signal that the PUT request has been | 1327 | The put API call offers a callback to signal that the PUT request has been |
1003 | sent. This does not guarantee that the data is accessible to others peers, | 1328 | sent. This does not guarantee that the data is accessible to others peers, |
1004 | or even that is has been stored, only that the service has requested to | 1329 | or even that is has been stored, only that the service has requested to |
1005 | a neighboring peer the retransmission of the PUT request towards its final | 1330 | a neighboring peer the retransmission of the PUT request towards its final |
1006 | destination. Currently there is no feedback about whether or not the data | 1331 | destination. Currently there is no feedback about whether or not the data |
1007 | has been sucessfully stored or where it has been stored. In order to improve | 1332 | has been sucessfully stored or where it has been stored. In order to |
1008 | the availablilty of the data and to compensate for possible errors, peers leaving | 1333 | improve the availablilty of the data and to compensate for possible |
1009 | and other unfavorable events, just make several PUT requests! | 1334 | errors, peers leaving and other unfavorable events, just make several |
1335 | PUT requests! | ||
1336 | |||
1010 | @example | 1337 | @example |
1011 | @verbatiminclude tutorial-examples/019.c | 1338 | @verbatiminclude tutorial-examples/019.c |
1012 | @end example | 1339 | @end example |
1013 | 1340 | ||
1014 | Exercise: Store a value in the DHT periodically to make sure it is available | 1341 | @noindent |
1015 | over time. You might consider using the function @code{GNUNET\_SCHEDULER\_add\_delayed} | 1342 | Exercise: Store a value in the DHT periodically to make sure it |
1016 | and call @code{GNUNET\_DHT\_put} from inside a helper function. | 1343 | is available over time. You might consider using the function |
1344 | @code{GNUNET\_SCHEDULER\_add\_delayed} and call | ||
1345 | @code{GNUNET\_DHT\_put} from inside a helper function. | ||
1017 | 1346 | ||
1018 | @node Obtaining data from the DHT | 1347 | @node Obtaining data from the DHT |
1019 | @subsection Obtaining data from the DHT | 1348 | @subsection Obtaining data from the DHT |
1349 | |||
1020 | As we saw in the previous example, the DHT works in an asynchronous mode. | 1350 | As we saw in the previous example, the DHT works in an asynchronous mode. |
1021 | Each request to the DHT is executed ``in the background'' and the API | 1351 | Each request to the DHT is executed ``in the background'' and the API |
1022 | calls return immediately. In order to receive results from the DHT, the | 1352 | calls return immediately. In order to receive results from the DHT, the |
@@ -1026,16 +1356,20 @@ duplicates) until the timeout expires or we explicitly stop the request. | |||
1026 | It is possible to give a ``forever'' timeout with | 1356 | It is possible to give a ``forever'' timeout with |
1027 | @code{GNUNET\_TIME\_UNIT\_FOREVER\_REL}. | 1357 | @code{GNUNET\_TIME\_UNIT\_FOREVER\_REL}. |
1028 | 1358 | ||
1029 | If we give a route option @code{GNUNET\_DHT\_RO\_RECORD\_ROUTE} the callback | 1359 | If we give a route option @code{GNUNET\_DHT\_RO\_RECORD\_ROUTE} |
1030 | will get a list of all the peers the data has travelled, both on the PUT | 1360 | the callback will get a list of all the peers the data has travelled, |
1031 | path and on the GET path. | 1361 | both on the PUT path and on the GET path. |
1362 | |||
1032 | @example | 1363 | @example |
1033 | @verbatiminclude tutorial-examples/020.c | 1364 | @verbatiminclude tutorial-examples/020.c |
1034 | @end example | 1365 | @end example |
1035 | 1366 | ||
1036 | Exercise: Store a value in the DHT and after a while retrieve it. Show the IDs of all | 1367 | @noindent |
1037 | the peers the requests have gone through. In order to convert a peer ID to a string, use | 1368 | Exercise: Store a value in the DHT and after a while retrieve it. |
1038 | the function @code{GNUNET\_i2s}. Pay attention to the route option parameters in both calls! | 1369 | Show the IDs of all the peers the requests have gone through. |
1370 | In order to convert a peer ID to a string, use the function | ||
1371 | @code{GNUNET\_i2s}. Pay attention to the route option parameters | ||
1372 | in both calls! | ||
1039 | 1373 | ||
1040 | @node Implementing a block plugin | 1374 | @node Implementing a block plugin |
1041 | @subsection Implementing a block plugin | 1375 | @subsection Implementing a block plugin |
@@ -1049,6 +1383,14 @@ in the service's respective directory. The | |||
1049 | mandatory functions that need to be implemented for a block plugin are | 1383 | mandatory functions that need to be implemented for a block plugin are |
1050 | described in the following sections. | 1384 | described in the following sections. |
1051 | 1385 | ||
1386 | @menu | ||
1387 | * Validating requests and replies:: | ||
1388 | * Deriving a key from a reply:: | ||
1389 | * Initialization of the plugin:: | ||
1390 | * Shutdown of the plugin:: | ||
1391 | * Integration of the plugin with the build system:: | ||
1392 | @end menu | ||
1393 | |||
1052 | @node Validating requests and replies | 1394 | @node Validating requests and replies |
1053 | @subsubsection Validating requests and replies | 1395 | @subsubsection Validating requests and replies |
1054 | 1396 | ||
@@ -1062,10 +1404,12 @@ the @code{xquery} argument is application-specific. Applications that | |||
1062 | do not use an extended query should check that the @code{xquery\_size} | 1404 | do not use an extended query should check that the @code{xquery\_size} |
1063 | is zero. The block group is typically used to filter duplicate | 1405 | is zero. The block group is typically used to filter duplicate |
1064 | replies. | 1406 | replies. |
1407 | |||
1065 | @example | 1408 | @example |
1066 | @verbatiminclude tutorial-examples/021.c | 1409 | @verbatiminclude tutorial-examples/021.c |
1067 | @end example | 1410 | @end example |
1068 | 1411 | ||
1412 | @noindent | ||
1069 | Note that it is mandatory to detect duplicate replies in this function | 1413 | Note that it is mandatory to detect duplicate replies in this function |
1070 | and return the respective status code. Duplicate detection is | 1414 | and return the respective status code. Duplicate detection is |
1071 | typically done using the Bloom filter block group provided by | 1415 | typically done using the Bloom filter block group provided by |
@@ -1081,6 +1425,7 @@ function is used to obtain the key of a block --- for example, by | |||
1081 | means of hashing. If deriving the key is not possible, the function | 1425 | means of hashing. If deriving the key is not possible, the function |
1082 | should simply return @code{GNUNET\_SYSERR} (the DHT will still work | 1426 | should simply return @code{GNUNET\_SYSERR} (the DHT will still work |
1083 | just fine with such blocks). | 1427 | just fine with such blocks). |
1428 | |||
1084 | @example | 1429 | @example |
1085 | @verbatiminclude tutorial-examples/022.c | 1430 | @verbatiminclude tutorial-examples/022.c |
1086 | @end example | 1431 | @end example |
@@ -1093,6 +1438,7 @@ an initialization function which should initialize the plugin. The | |||
1093 | initialization function specifies what block types the plugin cares | 1438 | initialization function specifies what block types the plugin cares |
1094 | about and returns a struct with the functions that are to be used for | 1439 | about and returns a struct with the functions that are to be used for |
1095 | validation and obtaining keys (the ones just defined above). | 1440 | validation and obtaining keys (the ones just defined above). |
1441 | |||
1096 | @example | 1442 | @example |
1097 | @verbatiminclude tutorial-examples/023.c | 1443 | @verbatiminclude tutorial-examples/023.c |
1098 | @end example | 1444 | @end example |
@@ -1103,6 +1449,7 @@ validation and obtaining keys (the ones just defined above). | |||
1103 | Following GNUnet's general plugin API concept, the plugin must | 1449 | Following GNUnet's general plugin API concept, the plugin must |
1104 | export a second function for cleaning up. It usually does very | 1450 | export a second function for cleaning up. It usually does very |
1105 | little. | 1451 | little. |
1452 | |||
1106 | @example | 1453 | @example |
1107 | @verbatiminclude tutorial-examples/024.c | 1454 | @verbatiminclude tutorial-examples/024.c |
1108 | @end example | 1455 | @end example |
@@ -1112,28 +1459,34 @@ little. | |||
1112 | 1459 | ||
1113 | In order to compile the plugin, the @file{Makefile.am} file for the | 1460 | In order to compile the plugin, the @file{Makefile.am} file for the |
1114 | service SERVICE should contain a rule similar to this: | 1461 | service SERVICE should contain a rule similar to this: |
1115 | @c* Actually this is a Makefile not C. But the whole structure of examples | 1462 | @c Actually this is a Makefile not C. But the whole structure of examples |
1116 | @c* must be improved. | 1463 | @c must be improved. |
1464 | |||
1117 | @example | 1465 | @example |
1118 | @verbatiminclude tutorial-examples/025.c | 1466 | @verbatiminclude tutorial-examples/025.c |
1119 | @end example | 1467 | @end example |
1120 | 1468 | ||
1469 | @noindent | ||
1121 | Exercise: Write a block plugin that accepts all queries | 1470 | Exercise: Write a block plugin that accepts all queries |
1122 | and all replies but prints information about queries and replies | 1471 | and all replies but prints information about queries and replies |
1123 | when the respective validation hooks are called. | 1472 | when the respective validation hooks are called. |
1124 | 1473 | ||
1125 | @node Monitoring the DHT | 1474 | @node Monitoring the DHT |
1126 | @subsection Monitoring the DHT | 1475 | @subsection Monitoring the DHT |
1127 | It is possible to monitor the functioning of the local DHT service. When monitoring | 1476 | |
1128 | the DHT, the service will alert the monitoring program of any events, | 1477 | It is possible to monitor the functioning of the local |
1129 | both started locally or received for routing from another peer. The are three different | 1478 | DHT service. When monitoring the DHT, the service will |
1130 | types of events possible: a GET request, a PUT request or a response (a reply to | 1479 | alert the monitoring program of any events, both started |
1131 | a GET). | 1480 | locally or received for routing from another peer. |
1132 | 1481 | The are three different types of events possible: a | |
1133 | Since the different events have different associated data, the API gets 3 | 1482 | GET request, a PUT request or a response (a reply to a GET). |
1134 | different callbacks (one for each message type) and optional type and key parameters, | 1483 | |
1135 | to allow for filtering of messages. When an event happens, the appropiate callback | 1484 | Since the different events have different associated data, |
1136 | is called with all the information about the event. | 1485 | the API gets 3 different callbacks (one for each message type) |
1486 | and optional type and key parameters, to allow for filtering of | ||
1487 | messages. When an event happens, the appropiate callback is | ||
1488 | called with all the information about the event. | ||
1489 | |||
1137 | @example | 1490 | @example |
1138 | @verbatiminclude tutorial-examples/026.c | 1491 | @verbatiminclude tutorial-examples/026.c |
1139 | @end example | 1492 | @end example |
@@ -1141,17 +1494,20 @@ is called with all the information about the event. | |||
1141 | @node Debugging with gnunet-arm | 1494 | @node Debugging with gnunet-arm |
1142 | @section Debugging with gnunet-arm | 1495 | @section Debugging with gnunet-arm |
1143 | 1496 | ||
1144 | Even if services are managed by @command{gnunet-arm}, you can start them with | 1497 | Even if services are managed by @command{gnunet-arm}, you can |
1145 | @command{gdb} or @command{valgrind}. For example, you could add the following lines | 1498 | start them with @command{gdb} or @command{valgrind}. For |
1146 | to your configuration file to start the DHT service in a @command{gdb} session in a | 1499 | example, you could add the following lines to your |
1147 | fresh @command{xterm}: | 1500 | configuration file to start the DHT service in a @command{gdb} |
1501 | session in a fresh @command{xterm}: | ||
1148 | 1502 | ||
1149 | @example | 1503 | @example |
1150 | [dht] | 1504 | [dht] |
1151 | PREFIX=xterm -e gdb --args | 1505 | PREFIX=xterm -e gdb --args |
1152 | @end example | 1506 | @end example |
1153 | 1507 | ||
1154 | Alternatively, you can stop a service that was started via ARM and run it manually: | 1508 | @noindent |
1509 | Alternatively, you can stop a service that was started via | ||
1510 | ARM and run it manually: | ||
1155 | 1511 | ||
1156 | @example | 1512 | @example |
1157 | $ gnunet-arm -k dht | 1513 | $ gnunet-arm -k dht |
@@ -1159,20 +1515,23 @@ $ gdb --args gnunet-service-dht -L DEBUG | |||
1159 | $ valgrind gnunet-service-dht -L DEBUG | 1515 | $ valgrind gnunet-service-dht -L DEBUG |
1160 | @end example | 1516 | @end example |
1161 | 1517 | ||
1162 | Assuming other services are well-written, they will automatically re-integrate the | 1518 | @noindent |
1163 | restarted service with the peer. | 1519 | Assuming other services are well-written, they will automatically |
1520 | re-integrate the restarted service with the peer. | ||
1164 | 1521 | ||
1165 | GNUnet provides a powerful logging mechanism providing log levels @code{ERROR}, | 1522 | GNUnet provides a powerful logging mechanism providing log |
1166 | @code{WARNING}, @code{INFO} and @code{DEBUG}. The current log level is | 1523 | levels @code{ERROR}, @code{WARNING}, @code{INFO} and @code{DEBUG}. |
1167 | configured using the @code{$GNUNET_FORCE_LOG} environmental variable. | 1524 | The current log level is configured using the @code{$GNUNET_FORCE_LOG} |
1168 | The @code{DEBUG} level is only available if @command{--enable-logging=verbose} was used when | 1525 | environmental variable. The @code{DEBUG} level is only available if |
1169 | running @command{configure}. More details about logging can be found under | 1526 | @command{--enable-logging=verbose} was used when running |
1527 | @command{configure}. More details about logging can be found under | ||
1170 | @uref{https://gnunet.org/logging}. | 1528 | @uref{https://gnunet.org/logging}. |
1171 | 1529 | ||
1172 | You should also probably enable the creation of core files, by setting | 1530 | You should also probably enable the creation of core files, by setting |
1173 | @code{ulimit}, and echo'ing @code{1} into @file{/proc/sys/kernel/core\_uses\_pid}. | 1531 | @code{ulimit}, and echo'ing @code{1} into |
1174 | Then you can investigate the core dumps with @command{gdb}, which is often | 1532 | @file{/proc/sys/kernel/core\_uses\_pid}. Then you can investigate the |
1175 | the fastest method to find simple errors. | 1533 | core dumps with @command{gdb}, which is often the fastest method to |
1534 | find simple errors. | ||
1176 | 1535 | ||
1177 | Exercise: Add a memory leak to your service and obtain a trace | 1536 | Exercise: Add a memory leak to your service and obtain a trace |
1178 | pointing to the leak using @command{valgrind} while running the service | 1537 | pointing to the leak using @command{valgrind} while running the service |
diff --git a/doc/gnunet.texi b/doc/documentation/gnunet.texi index 7803e52d3..35eed54b6 100644 --- a/doc/gnunet.texi +++ b/doc/documentation/gnunet.texi | |||
@@ -5,15 +5,17 @@ | |||
5 | @setfilename gnunet.info | 5 | @setfilename gnunet.info |
6 | @documentencoding UTF-8 | 6 | @documentencoding UTF-8 |
7 | @settitle GNUnet Reference Manual | 7 | @settitle GNUnet Reference Manual |
8 | @exampleindent 2 | ||
9 | @urefbreakstyle before | ||
8 | @c %**end of header | 10 | @c %**end of header |
9 | 11 | ||
10 | @include version.texi | 12 | @include version.texi |
11 | 13 | ||
12 | @c Set Versions which might be used in more than one place: | 14 | @c Set Versions which might be used in more than one place: |
13 | @set GNUNET-DIST-URL https://ftp.gnu.org/gnu/gnunet/ | 15 | @set GNUFTP-URL https://ftp.gnu.org/gnu/gnunet |
14 | @set GNUNET-VERSION 0.10.1 | 16 | @set PYPI-URL https://pypi.python.org/packages/source |
15 | @set GNURL-VERSION-CURRENT 7.55.1 | 17 | @set GNURL-VERSION-CURRENT 7.55.1 |
16 | @set GNURL-DIST-URL https://gnunet.org/sites/default/files/ | 18 | @set GNUNET-DIST-URL https://gnunet.org/sites/default/files/ |
17 | @c @set OPENPGP-SIGNING-KEY-ID | 19 | @c @set OPENPGP-SIGNING-KEY-ID |
18 | 20 | ||
19 | @copying | 21 | @copying |
@@ -38,6 +40,13 @@ A copy of the license is also available from the Free Software | |||
38 | Foundation Web site at @url{http://www.gnu.org/licenses/gpl.html}. | 40 | Foundation Web site at @url{http://www.gnu.org/licenses/gpl.html}. |
39 | @end copying | 41 | @end copying |
40 | 42 | ||
43 | @c TODO: Improve this and improve https://directory.fsf.org/wiki/Gnunet | ||
44 | |||
45 | @dircategory Networking | ||
46 | @direntry | ||
47 | * GNUnet: (gnunet). Framework for secure peer-to-peer networking | ||
48 | @end direntry | ||
49 | |||
41 | @titlepage | 50 | @titlepage |
42 | @title GNUnet Reference Manual | 51 | @title GNUnet Reference Manual |
43 | @subtitle Installing, configuring, using and contributing to GNUnet | 52 | @subtitle Installing, configuring, using and contributing to GNUnet |
@@ -51,60 +60,27 @@ Edition @value{EDITION} @* | |||
51 | @insertcopying | 60 | @insertcopying |
52 | @end titlepage | 61 | @end titlepage |
53 | 62 | ||
63 | @summarycontents | ||
54 | @contents | 64 | @contents |
55 | @c ********************************************************************* | ||
56 | @node Top | ||
57 | @top Contributing to GNUnet | ||
58 | @c ********************************************************************* | ||
59 | |||
60 | This document describes GNUnet version @value{VERSION}. | ||
61 | |||
62 | |||
63 | GNUnet is a @uref{http://www.gnu.org/, GNU} package. All code contributions | ||
64 | must thus be put under the | ||
65 | @uref{http://www.gnu.org/copyleft/gpl.html, GNU Public License (GPL)}. | ||
66 | All documentation should be put under FSF approved licenses | ||
67 | (see @uref{http://www.gnu.org/copyleft/fdl.html, fdl}). | ||
68 | |||
69 | By submitting documentation, translations, comments and other content to this | ||
70 | website you automatically grant the right to publish code under the | ||
71 | GNU Public License and documentation under either or both the GNU | ||
72 | Public License or the GNU Free Documentation License. When contributing | ||
73 | to the GNUnet project, GNU standards and the | ||
74 | @uref{http://www.gnu.org/philosophy/philosophy.html, GNU philosophy} | ||
75 | should be adhered to. | ||
76 | |||
77 | Note that we do now require a formal copyright assignment for GNUnet | ||
78 | contributors to GNUnet e.V.; nevertheless, we do allow pseudonymous | ||
79 | contributions. By signing the copyright agreement and submitting your | ||
80 | code (or documentation) to us, you agree to share the rights to your | ||
81 | code with GNUnet e.V.; GNUnet e.V. receives non-exclusive ownership | ||
82 | rights, and in particular is allowed to dual-license the code. You | ||
83 | retain non-exclusive rights to your contributions, so you can also | ||
84 | share your contributions freely with other projects. | ||
85 | |||
86 | GNUnet e.V. will publish all accepted contributions under the GPLv3 or any | ||
87 | later version. The association may decide to publish contributions under | ||
88 | additional licenses (dual-licensing). | ||
89 | |||
90 | We do not intentionally remove your name from your contributions; however, | ||
91 | due to extensive editing it is not always trivial to attribute contributors | ||
92 | properly. If you find that you significantly contributed to a file (or the | ||
93 | project as a whole) and are not listed in the respective authors file or | ||
94 | section, please do let us know. | ||
95 | 65 | ||
66 | @node Top | ||
67 | @top Introduction | ||
96 | 68 | ||
69 | This document is the Reference Manual for GNUnet version @value{VERSION}. | ||
97 | 70 | ||
98 | @menu | 71 | @menu |
99 | 72 | ||
100 | * Philosophy:: About GNUnet | 73 | * Philosophy:: About GNUnet |
74 | * Vocabulary:: Vocabulary | ||
101 | * GNUnet Installation Handbook:: How to install GNUnet | 75 | * GNUnet Installation Handbook:: How to install GNUnet |
102 | * Using GNUnet:: Using GNUnet | 76 | * Using GNUnet:: Using GNUnet |
77 | * Configuration Handbook:: Configuring GNUnet | ||
78 | * GNUnet Contributors Handbook:: Contributing to GNUnet | ||
103 | * GNUnet Developer Handbook:: Developing GNUnet | 79 | * GNUnet Developer Handbook:: Developing GNUnet |
104 | * GNU Free Documentation License:: The license of this manual. | 80 | * GNU Free Documentation License:: The license of this manual |
105 | * GNU General Public License:: The license of this manual. | 81 | * GNU General Public License:: The license of this manual |
106 | * Concept Index:: Concepts. | 82 | * Concept Index:: Concepts |
107 | * Programming Index:: Data types, functions, and variables. | 83 | * Programming Index:: Data types, functions, and variables |
108 | 84 | ||
109 | @detailmenu | 85 | @detailmenu |
110 | --- The Detailed Node Listing --- | 86 | --- The Detailed Node Listing --- |
@@ -112,7 +88,7 @@ section, please do let us know. | |||
112 | Philosophy | 88 | Philosophy |
113 | 89 | ||
114 | * Design Goals:: | 90 | * Design Goals:: |
115 | * Security & Privacy:: | 91 | * Security and Privacy:: |
116 | * Versatility:: | 92 | * Versatility:: |
117 | * Practicality:: | 93 | * Practicality:: |
118 | * Key Concepts:: | 94 | * Key Concepts:: |
@@ -127,10 +103,86 @@ Philosophy | |||
127 | * Backup of Identities and Egos:: | 103 | * Backup of Identities and Egos:: |
128 | * Revocation:: | 104 | * Revocation:: |
129 | 105 | ||
130 | Installation | 106 | Vocabulary |
107 | |||
108 | * Definitions abbreviations and acronyms:: | ||
109 | * Words and characters:: | ||
110 | * Technical Assumptions:: | ||
111 | |||
112 | GNUnet Installation Handbook | ||
131 | 113 | ||
132 | * Dependencies:: | 114 | * Dependencies:: |
133 | * External dependencies:: | 115 | * Pre-installation notes:: |
116 | * Generic installation instructions:: | ||
117 | * Build instructions for Ubuntu 12.04 using Git:: | ||
118 | * Build instructions for software builds from source:: | ||
119 | * Build Instructions for Microsoft Windows Platforms:: | ||
120 | * Build instructions for Debian 7.5:: | ||
121 | * Installing GNUnet from Git on Ubuntu 14.4:: | ||
122 | * Build instructions for Debian 8:: | ||
123 | * Outdated build instructions for previous revisions:: | ||
124 | @c * Portable GNUnet:: | ||
125 | * The graphical configuration interface:: | ||
126 | * How to start and stop a GNUnet peer:: | ||
127 | |||
128 | Using GNUnet | ||
129 | |||
130 | * Checking the Installation:: | ||
131 | * First steps - File-sharing:: | ||
132 | * First steps - Using the GNU Name System:: | ||
133 | * First steps - Using GNUnet Conversation:: | ||
134 | * First steps - Using the GNUnet VPN:: | ||
135 | * File-sharing:: | ||
136 | * The GNU Name System:: | ||
137 | * Using the Virtual Public Network:: | ||
138 | |||
139 | Configuration Handbook | ||
140 | |||
141 | GNUnet Contributors Handbook | ||
142 | |||
143 | * Contributing to GNUnet:: | ||
144 | * Licenses of contributions:: | ||
145 | * Copyright Assignment:: | ||
146 | * Contributing to the Reference Manual:: | ||
147 | |||
148 | GNUnet Developer Handbook | ||
149 | |||
150 | * Developer Introduction:: | ||
151 | * Code overview:: | ||
152 | * System Architecture:: | ||
153 | * Subsystem stability:: | ||
154 | * Naming conventions and coding style guide:: | ||
155 | * Build-system:: | ||
156 | * Developing extensions for GNUnet using the gnunet-ext template:: | ||
157 | * Writing testcases:: | ||
158 | * TESTING library:: | ||
159 | * Performance regression analysis with Gauger:: | ||
160 | * TESTBED Subsystem:: | ||
161 | * libgnunetutil:: | ||
162 | * Automatic Restart Manager (ARM):: | ||
163 | * TRANSPORT Subsystem:: | ||
164 | * NAT library:: | ||
165 | * Distance-Vector plugin:: | ||
166 | * SMTP plugin:: | ||
167 | * Bluetooth plugin:: | ||
168 | * WLAN plugin:: | ||
169 | * ATS Subsystem:: | ||
170 | * CORE Subsystem:: | ||
171 | * CADET Subsystem:: | ||
172 | * NSE Subsystem:: | ||
173 | * HOSTLIST Subsystem:: | ||
174 | * IDENTITY Subsystem:: | ||
175 | * NAMESTORE Subsystem:: | ||
176 | * PEERINFO Subsystem:: | ||
177 | * PEERSTORE Subsystem:: | ||
178 | * SET Subsystem:: | ||
179 | * STATISTICS Subsystem:: | ||
180 | * Distributed Hash Table (DHT):: | ||
181 | * GNU Name System (GNS):: | ||
182 | * GNS Namecache:: | ||
183 | * REVOCATION Subsystem:: | ||
184 | * File-sharing (FS) Subsystem:: | ||
185 | * REGEX Subsystem:: | ||
134 | 186 | ||
135 | @end detailmenu | 187 | @end detailmenu |
136 | @end menu | 188 | @end menu |
@@ -139,6 +191,8 @@ Installation | |||
139 | @include chapters/philosophy.texi | 191 | @include chapters/philosophy.texi |
140 | @c ********************************************************************* | 192 | @c ********************************************************************* |
141 | 193 | ||
194 | @include chapters/vocabulary.texi | ||
195 | |||
142 | @c ********************************************************************* | 196 | @c ********************************************************************* |
143 | @include chapters/installation.texi | 197 | @include chapters/installation.texi |
144 | @c ********************************************************************* | 198 | @c ********************************************************************* |
@@ -147,11 +201,15 @@ Installation | |||
147 | @include chapters/user.texi | 201 | @include chapters/user.texi |
148 | @c ********************************************************************* | 202 | @c ********************************************************************* |
149 | 203 | ||
204 | @include chapters/configuration.texi | ||
205 | |||
206 | @include chapters/contributing.texi | ||
207 | |||
150 | @c ********************************************************************* | 208 | @c ********************************************************************* |
151 | @include chapters/developer.texi | 209 | @include chapters/developer.texi |
210 | @c @include gnunet-c-tutorial.texi | ||
152 | @c ********************************************************************* | 211 | @c ********************************************************************* |
153 | 212 | ||
154 | |||
155 | @c ********************************************************************* | 213 | @c ********************************************************************* |
156 | @node GNU Free Documentation License | 214 | @node GNU Free Documentation License |
157 | @appendix GNU Free Documentation License | 215 | @appendix GNU Free Documentation License |
diff --git a/doc/gpl-3.0.texi b/doc/documentation/gpl-3.0.texi index 0e2e212ac..0e2e212ac 100644 --- a/doc/gpl-3.0.texi +++ b/doc/documentation/gpl-3.0.texi | |||
diff --git a/doc/documentation/htmlxref.cnf b/doc/documentation/htmlxref.cnf new file mode 100644 index 000000000..a4928f6fe --- /dev/null +++ b/doc/documentation/htmlxref.cnf | |||
@@ -0,0 +1,668 @@ | |||
1 | # htmlxref.cnf - reference file for free Texinfo manuals on the web. | ||
2 | # Modified by Ludovic Courtès <ludo@gnu.org> for the GNU Guix manual. | ||
3 | # Modified by ng0 <ng0@gnunet.org> for the GNUnet manual. | ||
4 | |||
5 | htmlxrefversion=2017-10-26.06; # UTC | ||
6 | |||
7 | # Copyright 2010, 2011, 2012, 2013, 2014, 2015 Free Software Foundation, Inc. | ||
8 | # | ||
9 | # Copying and distribution of this file, with or without modification, | ||
10 | # are permitted in any medium without royalty provided the copyright | ||
11 | # notice and this notice are preserved. | ||
12 | # | ||
13 | # The latest version of this file is available at | ||
14 | # http://ftpmirror.gnu.org/texinfo/htmlxref.cnf. | ||
15 | # Email corrections or additions to bug-texinfo@gnu.org. | ||
16 | # The primary goal is to list all relevant GNU manuals; | ||
17 | # other free manuals are also welcome. | ||
18 | # | ||
19 | # To be included in this list, a manual must: | ||
20 | # | ||
21 | # - have a generic url, e.g., no version numbers; | ||
22 | # - have a unique file name (e.g., manual identifier), i.e., be related to the | ||
23 | # package name. Things like "refman" or "tutorial" don't work. | ||
24 | # - follow the naming convention for nodes described at | ||
25 | # http://www.gnu.org/software/texinfo/manual/texinfo/html_node/HTML-Xref.html | ||
26 | # This is what makeinfo and texi2html implement. | ||
27 | # | ||
28 | # Unless the above criteria are met, it's not possible to generate | ||
29 | # reliable cross-manual references. | ||
30 | # | ||
31 | # For information on automatically generating all the useful formats for | ||
32 | # a manual to put on the web, see | ||
33 | # http://www.gnu.org/prep/maintain/html_node/Manuals-on-Web-Pages.html. | ||
34 | |||
35 | # For people editing this file: when a manual named foo is related to a | ||
36 | # package named bar, the url should contain a variable reference ${BAR}. | ||
37 | # Otherwise, the gnumaint scripts have no way of knowing they are | ||
38 | # associated, and thus gnu.org/manual can't include them. | ||
39 | |||
40 | # shorten references to manuals on www.gnu.org. | ||
41 | G = https://www.gnu.org | ||
42 | GS = ${G}/software | ||
43 | |||
44 | 3dldf mono ${GS}/3dldf/manual/user_ref/3DLDF.html | ||
45 | 3dldf node ${GS}/3dldf/manual/user_ref/ | ||
46 | |||
47 | alive mono ${GS}/alive/manual/alive.html | ||
48 | alive node ${GS}/alive/manual/html_node/ | ||
49 | |||
50 | anubis chapter ${GS}/anubis/manual/html_chapter/ | ||
51 | anubis section ${GS}/anubis/manual/html_section/ | ||
52 | anubis node ${GS}/anubis/manual/html_node/ | ||
53 | |||
54 | artanis mono ${GS}/artanis/manual/artanis.html | ||
55 | artanis node ${GS}/artanis/manual/html_node/ | ||
56 | |||
57 | aspell section http://aspell.net/man-html/index.html | ||
58 | |||
59 | auctex mono ${GS}/auctex/manual/auctex.html | ||
60 | auctex node ${GS}/auctex/manual/auctex/ | ||
61 | |||
62 | autoconf mono ${GS}/autoconf/manual/autoconf.html | ||
63 | autoconf node ${GS}/autoconf/manual/html_node/ | ||
64 | |||
65 | autogen mono ${GS}/autogen/manual/html_mono/autogen.html | ||
66 | autogen chapter ${GS}/autogen/manual/html_chapter/ | ||
67 | autogen node ${GS}/autoconf/manual/html_node/ | ||
68 | |||
69 | automake mono ${GS}/automake/manual/automake.html | ||
70 | automake node ${GS}/automake/manual/html_node/ | ||
71 | |||
72 | avl node http://www.stanford.edu/~blp/avl/libavl.html/ | ||
73 | |||
74 | bash mono ${GS}/bash/manual/bash.html | ||
75 | bash node ${GS}/bash/manual/html_node/ | ||
76 | |||
77 | BINUTILS = http://sourceware.org/binutils/docs | ||
78 | binutils node ${BINUTILS}/binutils/ | ||
79 | as node ${BINUTILS}/as/ | ||
80 | bfd node ${BINUTILS}/bfd/ | ||
81 | gprof node ${BINUTILS}/gprof/ | ||
82 | ld node ${BINUTILS}/ld/ | ||
83 | |||
84 | bison mono ${GS}/bison/manual/bison.html | ||
85 | bison node ${GS}/bison/manual/html_node/ | ||
86 | |||
87 | bpel2owfn mono ${GS}/bpel2owfn/manual/2.0.x/bpel2owfn.html | ||
88 | |||
89 | ccd2cue mono ${GS}/ccd2cue/manual/ccd2cue.html | ||
90 | ccd2cue node ${GS}/ccd2cue/manual/html_node/ | ||
91 | |||
92 | cflow mono ${GS}/cflow/manual/cflow.html | ||
93 | cflow node ${GS}/cflow/manual/html_node/ | ||
94 | |||
95 | chess mono ${GS}/chess/manual/gnuchess.html | ||
96 | chess node ${GS}/chess/manual/html_node/ | ||
97 | |||
98 | combine mono ${GS}/combine/manual/combine.html | ||
99 | combine chapter ${GS}/combine/manual/html_chapter/ | ||
100 | combine section ${GS}/combine/manual/html_section/ | ||
101 | combine node ${GS}/combine/manual/html_node/ | ||
102 | |||
103 | complexity mono ${GS}/complexity/manual/complexity.html | ||
104 | complexity node ${GS}/complexity/manual/html_node/ | ||
105 | |||
106 | coreutils mono ${GS}/coreutils/manual/coreutils | ||
107 | coreutils node ${GS}/coreutils/manual/html_node/ | ||
108 | |||
109 | cpio mono ${GS}/cpio/manual/cpio | ||
110 | cpio node ${GS}/cpio/manual/html_node/ | ||
111 | |||
112 | cssc node ${GS}/cssc/manual/ | ||
113 | |||
114 | #cvs cannot be handled here; see http://ximbiot.com/cvs/manual. | ||
115 | |||
116 | ddd mono ${GS}/ddd/manual/html_mono/ddd.html | ||
117 | |||
118 | ddrescue mono ${GS}/ddrescue/manual/ddrescue_manual.html | ||
119 | |||
120 | DICO = http://puszcza.gnu.org.ua/software/dico/manual | ||
121 | dico mono ${DICO}/dico.html | ||
122 | dico chapter ${DICO}/html_chapter/ | ||
123 | dico section ${DICO}/html_section/ | ||
124 | dico node ${DICO}/html_node/ | ||
125 | |||
126 | diffutils mono ${GS}/diffutils/manual/diffutils | ||
127 | diffutils node ${GS}/diffutils/manual/html_node/ | ||
128 | |||
129 | ed mono ${GS}/ed/manual/ed_manual.html | ||
130 | |||
131 | EMACS = ${GS}/emacs/manual | ||
132 | emacs mono ${EMACS}/html_mono/emacs.html | ||
133 | emacs node ${EMACS}/html_node/emacs/ | ||
134 | # | ||
135 | ada-mode mono ${EMACS}/html_mono/ada-mode.html | ||
136 | ada-mode node ${EMACS}/html_node/ada-mode/ | ||
137 | # | ||
138 | autotype mono ${EMACS}/html_mono/autotype.html | ||
139 | autotype node ${EMACS}/html_node/autotype/ | ||
140 | # | ||
141 | ccmode mono ${EMACS}/html_mono/ccmode.html | ||
142 | ccmode node ${EMACS}/html_node/ccmode/ | ||
143 | # | ||
144 | cl mono ${EMACS}/html_mono/cl.html | ||
145 | cl node ${EMACS}/html_node/cl/ | ||
146 | # | ||
147 | ebrowse mono ${EMACS}/html_mono/ebrowse.html | ||
148 | ebrowse node ${EMACS}/html_node/ebrowse/ | ||
149 | # | ||
150 | ediff mono ${EMACS}/html_mono/ediff.html | ||
151 | ediff node ${EMACS}/html_node/ediff/ | ||
152 | # | ||
153 | eieio mono ${EMACS}/html_mono/eieio.html | ||
154 | eieio node ${EMACS}/html_node/eieio/ | ||
155 | # | ||
156 | elisp mono ${EMACS}/html_mono/elisp.html | ||
157 | elisp node ${EMACS}/html_node/elisp/ | ||
158 | # | ||
159 | epa mono ${EMACS}/html_mono/epa.html | ||
160 | epa node ${EMACS}/html_node/epa/ | ||
161 | # | ||
162 | erc mono ${EMACS}/html_mono/erc.html | ||
163 | erc node ${EMACS}/html_node/erc/ | ||
164 | # | ||
165 | dired-x mono ${EMACS}/html_mono/dired-x.html | ||
166 | dired-x node ${EMACS}/html_node/dired-x/ | ||
167 | # | ||
168 | eshell mono ${EMACS}/html_mono/eshell.html | ||
169 | eshell node ${EMACS}/html_node/eshell/ | ||
170 | # | ||
171 | flymake mono ${EMACS}/html_mono/flymake.html | ||
172 | flymake node ${EMACS}/html_node/flymake/ | ||
173 | # | ||
174 | gnus mono ${EMACS}/html_mono/gnus.html | ||
175 | gnus node ${EMACS}/html_node/gnus/ | ||
176 | # | ||
177 | idlwave mono ${EMACS}/html_mono/idlwave.html | ||
178 | idlwave node ${EMACS}/html_node/idlwave/ | ||
179 | # | ||
180 | message mono ${EMACS}/html_mono/message.html | ||
181 | message node ${EMACS}/html_node/message/ | ||
182 | # | ||
183 | mh-e mono ${EMACS}/html_mono/mh-e.html | ||
184 | mh-e node ${EMACS}/html_node/mh-e/ | ||
185 | # | ||
186 | nxml-mode mono ${EMACS}/html_mono/nxml-mode.html | ||
187 | nxml-mode node ${EMACS}/html_node/nxml-mode/ | ||
188 | # | ||
189 | org mono ${EMACS}/html_mono/org.html | ||
190 | org node ${EMACS}/html_node/org/ | ||
191 | # | ||
192 | pcl-cvs mono ${EMACS}/html_mono/pcl-cvs.html | ||
193 | pcl-cvs node ${EMACS}/html_node/pcl-cvs/ | ||
194 | # | ||
195 | rcirc mono ${EMACS}/html_mono/rcirc.html | ||
196 | rcirc node ${EMACS}/html_node/rcirc/ | ||
197 | # | ||
198 | semantic mono ${EMACS}/html_mono/semantic.html | ||
199 | semantic node ${EMACS}/html_node/semantic/ | ||
200 | # | ||
201 | smtp mono ${EMACS}/html_mono/smtpmail.html | ||
202 | smtp node ${EMACS}/html_node/smtpmail/ | ||
203 | # | ||
204 | speedbar mono ${EMACS}/html_mono/speedbar.html | ||
205 | speedbar node ${EMACS}/html_node/speedbar/ | ||
206 | # | ||
207 | tramp mono ${EMACS}/html_mono/tramp.html | ||
208 | tramp node ${EMACS}/html_node/tramp/ | ||
209 | # | ||
210 | vip mono ${EMACS}/html_mono/vip.html | ||
211 | vip node ${EMACS}/html_node/vip/ | ||
212 | # | ||
213 | viper mono ${EMACS}/html_mono/viper.html | ||
214 | viper node ${EMACS}/html_node/viper/ | ||
215 | # | ||
216 | woman mono ${EMACS}/html_mono/woman.html | ||
217 | woman node ${EMACS}/html_node/woman/ | ||
218 | # (end emacs manuals) | ||
219 | |||
220 | easejs mono ${GS}/easejs/manual/easejs.html | ||
221 | easejs node ${GS}/easejs/manual/ | ||
222 | |||
223 | EMACS_GUIX = https://alezost.github.io/guix.el/manual/latest | ||
224 | emacs-guix mono ${EMACS_GUIX}/emacs-guix.html | ||
225 | emacs-guix node ${EMACS_GUIX}/html_node/ | ||
226 | |||
227 | emacs-muse node ${GS}/emacs-muse/manual/muse.html | ||
228 | emacs-muse node ${GS}/emacs-muse/manual/html_node/ | ||
229 | |||
230 | emms node ${GS}/emms/manual/ | ||
231 | |||
232 | # The file is called 'find.info' but the package is 'findutils'. | ||
233 | find mono ${GS}/findutils/manual/html_mono/find.html | ||
234 | find node ${GS}/findutils/manual/html_node/find_html | ||
235 | findutils mono ${GS}/findutils/manual/html_mono/find.html | ||
236 | findutils node ${GS}/findutils/manual/html_node/find_html | ||
237 | |||
238 | FLEX = http://flex.sourceforge.net | ||
239 | flex node ${FLEX}/manual/ | ||
240 | |||
241 | gama mono ${GS}/gama/manual/gama.html | ||
242 | gama node ${GS}/gama/manual/html_node/ | ||
243 | |||
244 | GAWK = ${GS}/gawk/manual | ||
245 | gawk mono ${GAWK}/gawk.html | ||
246 | gawk node ${GAWK}/html_node/ | ||
247 | gawkinet mono ${GAWK}/gawkinet/gawkinet.html | ||
248 | gawkinet node ${GAWK}/gawkinet/html_node/ | ||
249 | |||
250 | gcal mono ${GS}/gcal/manual/gcal.html | ||
251 | gcal node ${GS}/gcal/manual/html_node/ | ||
252 | |||
253 | GCC = http://gcc.gnu.org/onlinedocs | ||
254 | gcc node ${GCC}/gcc/ | ||
255 | cpp node ${GCC}/cpp/ | ||
256 | gcj node ${GCC}/gcj/ | ||
257 | gfortran node ${GCC}/gfortran/ | ||
258 | gnat_rm node ${GCC}/gnat_rm/ | ||
259 | gnat_ugn_unw node ${GCC}/gnat_ugn_unw/ | ||
260 | libgomp node ${GCC}/libgomp/ | ||
261 | libstdc++ node ${GCC}/libstdc++/ | ||
262 | # | ||
263 | gccint node ${GCC}/gccint/ | ||
264 | cppinternals node ${GCC}/cppinternals/ | ||
265 | gfc-internals node ${GCC}/gfc-internals/ | ||
266 | gnat-style node ${GCC}/gnat-style/ | ||
267 | libiberty node ${GCC}/libiberty/ | ||
268 | |||
269 | GDB = http://sourceware.org/gdb/current/onlinedocs | ||
270 | gdb node ${GDB}/gdb/ | ||
271 | stabs node ${GDB}/stabs/ | ||
272 | |||
273 | GDBM = http://www.gnu.org.ua/software/gdbm/manual | ||
274 | gdbm mono ${GDBM}/gdbm.html | ||
275 | gdbm chapter ${GDBM}/html_chapter/ | ||
276 | gdbm section ${GDBM}/html_section/ | ||
277 | gdbm node ${GDBM}/html_node/ | ||
278 | |||
279 | gettext mono ${GS}/gettext/manual/gettext.html | ||
280 | gettext node ${GS}/gettext/manual/html_node/ | ||
281 | |||
282 | gforth node http://www.complang.tuwien.ac.at/forth/gforth/Docs-html/ | ||
283 | |||
284 | global mono ${GS}/global/manual/global.html | ||
285 | |||
286 | gmediaserver node ${GS}/gmediaserver/manual/ | ||
287 | |||
288 | gmp node http://www.gmplib.org/manual/ | ||
289 | |||
290 | gnu-arch node ${GS}/gnu-arch/tutorial/ | ||
291 | |||
292 | gnu-c-manual mono ${GS}/gnu-c-manual/gnu-c-manual.html | ||
293 | |||
294 | gnu-crypto node ${GS}/gnu-crypto/manual/ | ||
295 | |||
296 | gnubg mono ${GS}/gnubg/manual/gnubg.html | ||
297 | gnubg node ${GS}/gnubg/manual/html_node/ | ||
298 | |||
299 | gnubik mono ${GS}/gnubik/manual/gnubik.html | ||
300 | gnubik node ${GS}/gnubik/manual/html_node/ | ||
301 | |||
302 | gnulib mono ${GS}/gnulib/manual/gnulib.html | ||
303 | gnulib node ${GS}/gnulib/manual/html_node/ | ||
304 | |||
305 | GNUN = ${GS}/trans-coord/manual | ||
306 | gnun mono ${GNUN}/gnun/gnun.html | ||
307 | gnun node ${GNUN}/gnun/html_node/ | ||
308 | web-trans mono ${GNUN}/web-trans/web-trans.html | ||
309 | web-trans node ${GNUN}/web-trans/html_node/ | ||
310 | |||
311 | GNUNET = https://docs.gnunet.org/manuals | ||
312 | gnunet node ${GNUNET}/gnunet/ | ||
313 | gnunet-c-tutorial node ${GNUNET}/gnunet-c-tutorial/ | ||
314 | gnunet-java-tutorial node ${GNUNET}/gnunet-java-tutorial/ | ||
315 | |||
316 | GNUPG = http://www.gnupg.org/documentation/manuals | ||
317 | gnupg node ${GNUPG}/gnupg/ | ||
318 | dirmngr node ${GNUPG}/dirmngr/ | ||
319 | gcrypt node ${GNUPG}/gcrypt/ | ||
320 | libgcrypt node ${GNUPG}/gcrypt/ | ||
321 | ksba node ${GNUPG}/ksba/ | ||
322 | assuan node ${GNUPG}/assuan/ | ||
323 | gpgme node ${GNUPG}/gpgme/ | ||
324 | |||
325 | gnuprologjava node ${GS}/gnuprologjava/manual/ | ||
326 | |||
327 | gnuschool mono ${GS}/gnuschool/gnuschool.html | ||
328 | |||
329 | GNUSTANDARDS = ${G}/prep | ||
330 | maintain mono ${GNUSTANDARDS}/maintain/maintain.html | ||
331 | maintain node ${GNUSTANDARDS}/maintain/html_node/ | ||
332 | # | ||
333 | standards mono ${GNUSTANDARDS}/standards/standards.html | ||
334 | standards node ${GNUSTANDARDS}/standards/html_node/ | ||
335 | |||
336 | gnutls mono http://gnutls.org/manual/gnutls.html | ||
337 | gnutls node http://gnutls.org/manual/html_node/ | ||
338 | |||
339 | gnutls-guile mono http://gnutls.org/manual/gnutls-guile.html | ||
340 | gnutls-guile node http://gnutls.org/manual/gnutls-guile/ | ||
341 | |||
342 | gperf mono ${GS}/gperf/manual/gperf.html | ||
343 | gperf node ${GS}/gperf/manual/html_node/ | ||
344 | |||
345 | grep mono ${GS}/grep/manual/grep.html | ||
346 | grep node ${GS}/grep/manual/html_node/ | ||
347 | |||
348 | groff node ${GS}/groff/manual/html_node/ | ||
349 | |||
350 | GRUB = ${GS}/grub/manual | ||
351 | grub mono ${GRUB}/grub.html | ||
352 | grub node ${GRUB}/html_node/ | ||
353 | # | ||
354 | multiboot mono ${GRUB}/multiboot/multiboot.html | ||
355 | multiboot node ${GRUB}/multiboot/html_node/ | ||
356 | |||
357 | gsasl mono ${GS}/gsasl/manual/gsasl.html | ||
358 | gsasl node ${GS}/gsasl/manual/html_node/ | ||
359 | |||
360 | gsl node ${GS}/gsl/manual/html_node/ | ||
361 | |||
362 | gsrc mono ${GS}/gsrc/manual/gsrc.html | ||
363 | gsrc node ${GS}/gsrc/manual/html_node/ | ||
364 | |||
365 | gss mono ${GS}/gss/manual/gss.html | ||
366 | gss node ${GS}/gss/manual/html_node/ | ||
367 | |||
368 | gtypist mono ${GS}/gtypist/doc/ | ||
369 | |||
370 | guile mono ${GS}/guile/manual/guile.html | ||
371 | guile node ${GS}/guile/manual/html_node/ | ||
372 | |||
373 | guile-avahi mono http://nongnu.org/guile-avahi/doc/guile-avahi.html | ||
374 | |||
375 | GUILE_GNOME = ${GS}/guile-gnome/docs | ||
376 | gobject node ${GUILE_GNOME}/gobject/html/ | ||
377 | glib node ${GUILE_GNOME}/glib/html/ | ||
378 | atk node ${GUILE_GNOME}/atk/html/ | ||
379 | pango node ${GUILE_GNOME}/pango/html/ | ||
380 | pangocairo node ${GUILE_GNOME}/pangocairo/html/ | ||
381 | gdk node ${GUILE_GNOME}/gdk/html/ | ||
382 | gtk node ${GUILE_GNOME}/gtk/html/ | ||
383 | libglade node ${GUILE_GNOME}/libglade/html/ | ||
384 | gnome-vfs node ${GUILE_GNOME}/gnome-vfs/html/ | ||
385 | libgnomecanvas node ${GUILE_GNOME}/libgnomecanvas/html/ | ||
386 | gconf node ${GUILE_GNOME}/gconf/html/ | ||
387 | libgnome node ${GUILE_GNOME}/libgnome/html/ | ||
388 | libgnomeui node ${GUILE_GNOME}/libgnomeui/html/ | ||
389 | corba node ${GUILE_GNOME}/corba/html/ | ||
390 | clutter node ${GUILE_GNOME}/clutter/html/ | ||
391 | clutter-glx node ${GUILE_GNOME}/clutter-glx/html/ | ||
392 | |||
393 | guile-gtk node ${GS}/guile-gtk/docs/guile-gtk/ | ||
394 | |||
395 | guile-rpc mono ${GS}/guile-rpc/manual/guile-rpc.html | ||
396 | guile-rpc node ${GS}/guile-rpc/manual/html_node/ | ||
397 | |||
398 | guix mono ${GS}/guix/manual/guix.html | ||
399 | guix node ${GS}/guix/manual/html_node/ | ||
400 | |||
401 | gv mono ${GS}/gv/manual/gv.html | ||
402 | gv node ${GS}/gv/manual/html_node/ | ||
403 | |||
404 | gzip mono ${GS}/gzip/manual/gzip.html | ||
405 | gzip node ${GS}/gzip/manual/html_node/ | ||
406 | |||
407 | hello mono ${GS}/hello/manual/hello.html | ||
408 | hello node ${GS}/hello/manual/html_node/ | ||
409 | |||
410 | help2man mono ${GS}/help2man/help2man.html | ||
411 | |||
412 | idutils mono ${GS}/idutils/manual/idutils.html | ||
413 | idutils node ${GS}/idutils/manual/html_node/ | ||
414 | |||
415 | inetutils mono ${GS}/inetutils/manual/inetutils.html | ||
416 | inetutils node ${GS}/inetutils/manual/html_node/ | ||
417 | |||
418 | jwhois mono ${GS}/jwhois/manual/jwhois.html | ||
419 | jwhois node ${GS}/jwhois/manual/html_node/ | ||
420 | |||
421 | libc mono ${GS}/libc/manual/html_mono/libc.html | ||
422 | libc node ${GS}/libc/manual/html_node/ | ||
423 | |||
424 | LIBCDIO = ${GS}/libcdio | ||
425 | libcdio mono ${LIBCDIO}/libcdio.html | ||
426 | cd-text mono ${LIBCDIO}/cd-text-format.html | ||
427 | |||
428 | libextractor mono ${GS}/libextractor/manual/libextractor.html | ||
429 | libextractor node ${GS}/libextractor/manual/html_node/ | ||
430 | |||
431 | libidn mono ${GS}/libidn/manual/libidn.html | ||
432 | libidn node ${GS}/libidn/manual/html_node/ | ||
433 | |||
434 | librejs mono ${GS}/librejs/manual/librejs.html | ||
435 | librejs node ${GS}/librejs/manual/html_node/ | ||
436 | |||
437 | libmatheval mono ${GS}/libmatheval/manual/libmatheval.html | ||
438 | |||
439 | LIBMICROHTTPD = ${GS}/libmicrohttpd | ||
440 | libmicrohttpd mono ${LIBMICROHTTPD}/manual/libmicrohttpd.html | ||
441 | libmicrohttpd node ${LIBMICROHTTPD}/manual/html_node/ | ||
442 | microhttpd-tutorial mono ${LIBMICROHTTPD}/tutorial.html | ||
443 | |||
444 | libtasn1 mono ${GS}/libtasn1/manual/libtasn1.html | ||
445 | libtasn1 node ${GS}/libtasn1/manual/html_node/ | ||
446 | |||
447 | libtool mono ${GS}/libtool/manual/libtool.html | ||
448 | libtool node ${GS}/libtool/manual/html_node/ | ||
449 | |||
450 | lightning mono ${GS}/lightning/manual/lightning.html | ||
451 | lightning node ${GS}/lightning/manual/html_node/ | ||
452 | |||
453 | # The stable/ url redirects immediately, but that's ok. | ||
454 | # The .html extension is omitted on their web site, but it works if given. | ||
455 | LILYPOND = http://lilypond.org/doc/stable/Documentation | ||
456 | lilypond-internals node ${LILYPOND}/internals/ | ||
457 | lilypond-learning node ${LILYPOND}/learning/ | ||
458 | lilypond-notation node ${LILYPOND}/notation/ | ||
459 | lilypond-snippets node ${LILYPOND}/snippets/ | ||
460 | lilypond-usage node ${LILYPOND}/usage/ | ||
461 | lilypond-web node ${LILYPOND}/web/ | ||
462 | music-glossary node ${LILYPOND}/music-glossary/ | ||
463 | |||
464 | liquidwar6 mono ${GS}/liquidwar6/manual/liquidwar6.html | ||
465 | liquidwar6 node ${GS}/liquidwar6/manual/html_node/ | ||
466 | |||
467 | lispintro mono ${GS}/emacs/emacs-lisp-intro/html_mono/emacs-lisp-intro.html | ||
468 | lispintro node ${GS}/emacs/emacs-lisp-intro/html_node/index.html | ||
469 | |||
470 | LSH = http://www.lysator.liu.se/~nisse/lsh | ||
471 | lsh mono ${LSH}/lsh.html | ||
472 | |||
473 | m4 mono ${GS}/m4/manual/m4.html | ||
474 | m4 node ${GS}/m4/manual/html_node/ | ||
475 | |||
476 | mailutils mono ${GS}/mailutils/manual/mailutils.html | ||
477 | mailutils chapter ${GS}/mailutils/manual/html_chapter/ | ||
478 | mailutils section ${GS}/mailutils/manual/html_section/ | ||
479 | mailutils node ${GS}/mailutils/manual/html_node/ | ||
480 | |||
481 | make mono ${GS}/make/manual/make.html | ||
482 | make node ${GS}/make/manual/html_node/ | ||
483 | |||
484 | mcron mono ${GS}/mcron/manual/mcron.html | ||
485 | mcron node ${GS}/mcron/manual/html_node/ | ||
486 | |||
487 | mdk mono ${GS}/mdk/manual/mdk.html | ||
488 | mdk node ${GS}/mdk/manual/html_node/ | ||
489 | |||
490 | METAEXCHANGE = http://ftp.gwdg.de/pub/gnu2/iwfmdh/doc/texinfo | ||
491 | iwf_mh node ${METAEXCHANGE}/iwf_mh.html | ||
492 | scantest node ${METAEXCHANGE}/scantest.html | ||
493 | |||
494 | MIT_SCHEME = ${GS}/mit-scheme/documentation | ||
495 | mit-scheme-ref node ${MIT_SCHEME}/mit-scheme-ref/ | ||
496 | mit-scheme-user node ${MIT_SCHEME}/mit-scheme-user/ | ||
497 | sos node ${MIT_SCHEME}/mit-scheme-sos/ | ||
498 | mit-scheme-imail node ${MIT_SCHEME}/mit-scheme-imail/ | ||
499 | |||
500 | moe mono ${GS}/moe/manual/moe_manual.html | ||
501 | |||
502 | motti node ${GS}/motti/manual/ | ||
503 | |||
504 | mpc node http://www.multiprecision.org/index.php?prog=mpc&page=html | ||
505 | |||
506 | mpfr mono http://www.mpfr.org/mpfr-current/mpfr.html | ||
507 | |||
508 | mtools mono ${GS}/mtools/manual/mtools.html | ||
509 | |||
510 | myserver node http://www.myserverproject.net/documentation/ | ||
511 | |||
512 | nano mono http://www.nano-editor.org/dist/latest/nano.html | ||
513 | |||
514 | nettle chapter http://www.lysator.liu.se/~nisse/nettle/nettle.html | ||
515 | |||
516 | ocrad mono ${GS}/ocrad/manual/ocrad_manual.html | ||
517 | |||
518 | parted mono ${GS}/parted/manual/parted.html | ||
519 | parted node ${GS}/parted/manual/html_node/ | ||
520 | |||
521 | pascal mono http://www.gnu-pascal.de/gpc/ | ||
522 | |||
523 | # can't use pcb since url's contain dates --30nov10 | ||
524 | |||
525 | perl mono ${GS}/perl/manual/perldoc-all.html | ||
526 | |||
527 | PIES = http://www.gnu.org.ua/software/pies/manual | ||
528 | pies mono ${PIES}/pies.html | ||
529 | pies chapter ${PIES}/html_chapter/ | ||
530 | pies section ${PIES}/html_section/ | ||
531 | pies node ${PIES}/html_node/ | ||
532 | |||
533 | plotutils mono ${GS}/plotutils/manual/en/plotutils.html | ||
534 | plotutils node ${GS}/plotutils/manual/en/html_node/ | ||
535 | |||
536 | proxyknife mono ${GS}/proxyknife/manual/proxyknife.html | ||
537 | proxyknife node ${GS}/proxyknife/manual/html_node/ | ||
538 | |||
539 | pspp mono ${GS}/pspp/manual/pspp.html | ||
540 | pspp node ${GS}/pspp/manual/html_node/ | ||
541 | |||
542 | pyconfigure mono ${GS}/pyconfigure/manual/pyconfigure.html | ||
543 | pyconfigure node ${GS}/pyconfigure/manual/html_node/ | ||
544 | |||
545 | R = http://cran.r-project.org/doc/manuals | ||
546 | R-intro mono ${R}/R-intro.html | ||
547 | R-lang mono ${R}/R-lang.html | ||
548 | R-exts mono ${R}/R-exts.html | ||
549 | R-data mono ${R}/R-data.html | ||
550 | R-admin mono ${R}/R-admin.html | ||
551 | R-ints mono ${R}/R-ints.html | ||
552 | |||
553 | rcs mono ${GS}/rcs/manual/rcs.html | ||
554 | rcs node ${GS}/rcs/manual/html_node/ | ||
555 | |||
556 | READLINE = http://cnswww.cns.cwru.edu/php/chet/readline | ||
557 | readline mono ${READLINE}/readline.html | ||
558 | rluserman mono ${READLINE}/rluserman.html | ||
559 | history mono ${READLINE}/history.html | ||
560 | |||
561 | recode mono http://recode.progiciels-bpi.ca/manual/index.html | ||
562 | |||
563 | recutils mono ${GS}/recutils/manual/recutils.html | ||
564 | recutils node ${GS}/recutils/manual/html_node/ | ||
565 | |||
566 | reftex mono ${GS}/auctex/manual/reftex.html | ||
567 | reftex node ${GS}/auctex/manual/reftex/ | ||
568 | |||
569 | remotecontrol mono ${GS}/remotecontrol/manual/remotecontrol.html | ||
570 | remotecontrol node ${GS}/remotecontrol/manual/html_node/ | ||
571 | |||
572 | rottlog mono ${GS}/rottlog/manual/rottlog.html | ||
573 | rottlog node ${GS}/rottlog/manual/html_node/ | ||
574 | |||
575 | RUSH = http://www.gnu.org.ua/software/rush/manual | ||
576 | rush mono ${RUSH}/rush.html | ||
577 | rush chapter ${RUSH}/html_chapter/ | ||
578 | rush section ${RUSH}/html_section/ | ||
579 | rush node ${RUSH}/html_node/ | ||
580 | |||
581 | screen mono ${GS}/screen/manual/screen.html | ||
582 | screen node ${GS}/screen/manual/html_node/ | ||
583 | |||
584 | sed mono ${GS}/sed/manual/sed.html | ||
585 | sed node ${GS}/sed/manual/html_node/ | ||
586 | |||
587 | sharutils mono ${GS}/sharutils/manual/html_mono/sharutils.html | ||
588 | sharutils chapter ${GS}/sharutils/manual/html_chapter/ | ||
589 | sharutils node ${GS}/sharutils/manual/html_node/ | ||
590 | |||
591 | shepherd mono ${GS}/shepherd/manual/shepherd.html | ||
592 | shepherd node ${GS}/shepherd/manual/html_node/ | ||
593 | |||
594 | # can't use mono files since they have generic names | ||
595 | SMALLTALK = ${GS}/smalltalk | ||
596 | smalltalk node ${SMALLTALK}/manual/html_node/ | ||
597 | smalltalk-base node ${SMALLTALK}/manual-base/html_node/ | ||
598 | smalltalk-libs node ${SMALLTALK}/manual-libs/html_node/ | ||
599 | |||
600 | sourceinstall mono ${GS}/sourceinstall/manual/sourceinstall.html | ||
601 | sourceinstall node ${GS}/sourceinstall/manual/html_node/ | ||
602 | |||
603 | sqltutor mono ${GS}/sqltutor/manual/sqltutor.html | ||
604 | sqltutor node ${GS}/sqltutor/manual/html_node/ | ||
605 | |||
606 | src-highlite mono ${GS}/src-highlite/source-highlight.html | ||
607 | |||
608 | swbis mono ${GS}/swbis/manual.html | ||
609 | |||
610 | tar mono ${GS}/tar/manual/tar.html | ||
611 | tar chapter ${GS}/tar/manual/html_chapter/ | ||
612 | tar section ${GS}/tar/manual/html_section/ | ||
613 | tar node ${GS}/autoconf/manual/html_node/ | ||
614 | |||
615 | teseq mono ${GS}/teseq/teseq.html | ||
616 | teseq node ${GS}/teseq/html_node/ | ||
617 | |||
618 | TEXINFO = ${GS}/texinfo/manual | ||
619 | texinfo mono ${TEXINFO}/texinfo/texinfo.html | ||
620 | texinfo node ${TEXINFO}/texinfo/html_node/ | ||
621 | # | ||
622 | info mono ${TEXINFO}/info/info.html | ||
623 | info node ${TEXINFO}/info/html_node/ | ||
624 | # | ||
625 | info-stnd mono ${TEXINFO}/info-stnd/info-stnd.html | ||
626 | info-stnd node ${TEXINFO}/info-stnd/html_node/ | ||
627 | |||
628 | thales node ${GS}/thales/manual/ | ||
629 | |||
630 | units mono ${GS}/units/manual/units.html | ||
631 | units node ${GS}/units/manual/html_node/ | ||
632 | |||
633 | vc-dwim mono ${GS}/vc-dwim/manual/vc-dwim.html | ||
634 | vc-dwim node ${GS}/vc-dwim/manual/html_node/ | ||
635 | |||
636 | wdiff mono ${GS}/wdiff/manual/wdiff.html | ||
637 | wdiff node ${GS}/wdiff/manual/html_node/ | ||
638 | |||
639 | websocket4j mono ${GS}/websocket4j/manual/websocket4j.html | ||
640 | websocket4j node ${GS}/websocket4j/manual/html_node/ | ||
641 | |||
642 | wget mono ${GS}/wget/manual/wget.html | ||
643 | wget node ${GS}/wget/manual/html_node/ | ||
644 | |||
645 | xboard mono ${GS}/xboard/manual/xboard.html | ||
646 | xboard node ${GS}/xboard/manual/html_node/ | ||
647 | |||
648 | # emacs-page | ||
649 | # Free TeX-related Texinfo manuals on tug.org. | ||
650 | |||
651 | T = http://tug.org/texinfohtml | ||
652 | |||
653 | dvipng mono ${T}/dvipng.html | ||
654 | dvips mono ${T}/dvips.html | ||
655 | eplain mono ${T}/eplain.html | ||
656 | kpathsea mono ${T}/kpathsea.html | ||
657 | latex2e mono ${T}/latex2e.html | ||
658 | tlbuild mono ${T}/tlbuild.html | ||
659 | web2c mono ${T}/web2c.html | ||
660 | |||
661 | |||
662 | # Local Variables: | ||
663 | # eval: (add-hook 'write-file-hooks 'time-stamp) | ||
664 | # time-stamp-start: "htmlxrefversion=" | ||
665 | # time-stamp-format: "%:y-%02m-%02d.%02H" | ||
666 | # time-stamp-time-zone: "UTC" | ||
667 | # time-stamp-end: "; # UTC" | ||
668 | # End: | ||
diff --git a/doc/images/daemon_lego_block.png b/doc/documentation/images/daemon_lego_block.png index 5a088b532..5a088b532 100644 --- a/doc/images/daemon_lego_block.png +++ b/doc/documentation/images/daemon_lego_block.png | |||
Binary files differ | |||
diff --git a/doc/images/daemon_lego_block.svg b/doc/documentation/images/daemon_lego_block.svg index 38ad90d13..38ad90d13 100644 --- a/doc/images/daemon_lego_block.svg +++ b/doc/documentation/images/daemon_lego_block.svg | |||
diff --git a/doc/images/gnunet-0-10-peerinfo.png b/doc/documentation/images/gnunet-0-10-peerinfo.png index c5e711aff..c5e711aff 100644 --- a/doc/images/gnunet-0-10-peerinfo.png +++ b/doc/documentation/images/gnunet-0-10-peerinfo.png | |||
Binary files differ | |||
diff --git a/doc/images/gnunet-fs-gtk-0-10-star-tab.png b/doc/documentation/images/gnunet-fs-gtk-0-10-star-tab.png index d7993cc46..d7993cc46 100644 --- a/doc/images/gnunet-fs-gtk-0-10-star-tab.png +++ b/doc/documentation/images/gnunet-fs-gtk-0-10-star-tab.png | |||
Binary files differ | |||
diff --git a/doc/images/gnunet-gtk-0-10-download-area.png b/doc/documentation/images/gnunet-gtk-0-10-download-area.png index 8500d46c9..8500d46c9 100644 --- a/doc/images/gnunet-gtk-0-10-download-area.png +++ b/doc/documentation/images/gnunet-gtk-0-10-download-area.png | |||
Binary files differ | |||
diff --git a/doc/images/gnunet-gtk-0-10-fs-menu.png b/doc/documentation/images/gnunet-gtk-0-10-fs-menu.png index dc20c45a9..dc20c45a9 100644 --- a/doc/images/gnunet-gtk-0-10-fs-menu.png +++ b/doc/documentation/images/gnunet-gtk-0-10-fs-menu.png | |||
Binary files differ | |||
diff --git a/doc/images/gnunet-gtk-0-10-fs-publish-editing.png b/doc/documentation/images/gnunet-gtk-0-10-fs-publish-editing.png index 6f9f75ea6..6f9f75ea6 100644 --- a/doc/images/gnunet-gtk-0-10-fs-publish-editing.png +++ b/doc/documentation/images/gnunet-gtk-0-10-fs-publish-editing.png | |||
Binary files differ | |||
diff --git a/doc/images/gnunet-gtk-0-10-fs-publish-select.png b/doc/documentation/images/gnunet-gtk-0-10-fs-publish-select.png index 50672e379..50672e379 100644 --- a/doc/images/gnunet-gtk-0-10-fs-publish-select.png +++ b/doc/documentation/images/gnunet-gtk-0-10-fs-publish-select.png | |||
Binary files differ | |||
diff --git a/doc/images/gnunet-gtk-0-10-fs-publish-with-file.png b/doc/documentation/images/gnunet-gtk-0-10-fs-publish-with-file.png index b46542563..b46542563 100644 --- a/doc/images/gnunet-gtk-0-10-fs-publish-with-file.png +++ b/doc/documentation/images/gnunet-gtk-0-10-fs-publish-with-file.png | |||
Binary files differ | |||
diff --git a/doc/images/gnunet-gtk-0-10-fs-publish-with-file_0.png b/doc/documentation/images/gnunet-gtk-0-10-fs-publish-with-file_0.png index b46542563..b46542563 100644 --- a/doc/images/gnunet-gtk-0-10-fs-publish-with-file_0.png +++ b/doc/documentation/images/gnunet-gtk-0-10-fs-publish-with-file_0.png | |||
Binary files differ | |||
diff --git a/doc/images/gnunet-gtk-0-10-fs-publish.png b/doc/documentation/images/gnunet-gtk-0-10-fs-publish.png index 033b38fa5..033b38fa5 100644 --- a/doc/images/gnunet-gtk-0-10-fs-publish.png +++ b/doc/documentation/images/gnunet-gtk-0-10-fs-publish.png | |||
Binary files differ | |||
diff --git a/doc/images/gnunet-gtk-0-10-fs-published.png b/doc/documentation/images/gnunet-gtk-0-10-fs-published.png index fbd3dd6a3..fbd3dd6a3 100644 --- a/doc/images/gnunet-gtk-0-10-fs-published.png +++ b/doc/documentation/images/gnunet-gtk-0-10-fs-published.png | |||
Binary files differ | |||
diff --git a/doc/images/gnunet-gtk-0-10-fs-search.png b/doc/documentation/images/gnunet-gtk-0-10-fs-search.png index bb64ab92e..bb64ab92e 100644 --- a/doc/images/gnunet-gtk-0-10-fs-search.png +++ b/doc/documentation/images/gnunet-gtk-0-10-fs-search.png | |||
Binary files differ | |||
diff --git a/doc/images/gnunet-gtk-0-10-fs.png b/doc/documentation/images/gnunet-gtk-0-10-fs.png index c7a294878..c7a294878 100644 --- a/doc/images/gnunet-gtk-0-10-fs.png +++ b/doc/documentation/images/gnunet-gtk-0-10-fs.png | |||
Binary files differ | |||
diff --git a/doc/images/gnunet-gtk-0-10-gns-a-done.png b/doc/documentation/images/gnunet-gtk-0-10-gns-a-done.png index f8231b3a6..f8231b3a6 100644 --- a/doc/images/gnunet-gtk-0-10-gns-a-done.png +++ b/doc/documentation/images/gnunet-gtk-0-10-gns-a-done.png | |||
Binary files differ | |||
diff --git a/doc/images/gnunet-gtk-0-10-gns-a.png b/doc/documentation/images/gnunet-gtk-0-10-gns-a.png index 39858d72c..39858d72c 100644 --- a/doc/images/gnunet-gtk-0-10-gns-a.png +++ b/doc/documentation/images/gnunet-gtk-0-10-gns-a.png | |||
Binary files differ | |||
diff --git a/doc/images/gnunet-gtk-0-10-gns.png b/doc/documentation/images/gnunet-gtk-0-10-gns.png index c71a2bd7b..c71a2bd7b 100644 --- a/doc/images/gnunet-gtk-0-10-gns.png +++ b/doc/documentation/images/gnunet-gtk-0-10-gns.png | |||
Binary files differ | |||
diff --git a/doc/images/gnunet-gtk-0-10-identity.png b/doc/documentation/images/gnunet-gtk-0-10-identity.png index d0b426098..d0b426098 100644 --- a/doc/images/gnunet-gtk-0-10-identity.png +++ b/doc/documentation/images/gnunet-gtk-0-10-identity.png | |||
Binary files differ | |||
diff --git a/doc/images/gnunet-gtk-0-10-search-selected.png b/doc/documentation/images/gnunet-gtk-0-10-search-selected.png index da1ad4d31..da1ad4d31 100644 --- a/doc/images/gnunet-gtk-0-10-search-selected.png +++ b/doc/documentation/images/gnunet-gtk-0-10-search-selected.png | |||
Binary files differ | |||
diff --git a/doc/images/gnunet-gtk-0-10-traffic.png b/doc/documentation/images/gnunet-gtk-0-10-traffic.png index 76458f717..76458f717 100644 --- a/doc/images/gnunet-gtk-0-10-traffic.png +++ b/doc/documentation/images/gnunet-gtk-0-10-traffic.png | |||
Binary files differ | |||
diff --git a/doc/images/gnunet-gtk-0-10.png b/doc/documentation/images/gnunet-gtk-0-10.png index 3615849a7..3615849a7 100644 --- a/doc/images/gnunet-gtk-0-10.png +++ b/doc/documentation/images/gnunet-gtk-0-10.png | |||
Binary files differ | |||
diff --git a/doc/images/gnunet-namestore-gtk-phone.png b/doc/documentation/images/gnunet-namestore-gtk-phone.png index 3bb859629..3bb859629 100644 --- a/doc/images/gnunet-namestore-gtk-phone.png +++ b/doc/documentation/images/gnunet-namestore-gtk-phone.png | |||
Binary files differ | |||
diff --git a/doc/images/gnunet-namestore-gtk-vpn.png b/doc/documentation/images/gnunet-namestore-gtk-vpn.png index c716729ba..c716729ba 100644 --- a/doc/images/gnunet-namestore-gtk-vpn.png +++ b/doc/documentation/images/gnunet-namestore-gtk-vpn.png | |||
Binary files differ | |||
diff --git a/doc/images/gnunet-setup-exit.png b/doc/documentation/images/gnunet-setup-exit.png index 66bd972bc..66bd972bc 100644 --- a/doc/images/gnunet-setup-exit.png +++ b/doc/documentation/images/gnunet-setup-exit.png | |||
Binary files differ | |||
diff --git a/doc/images/gnunet-tutorial-service.png b/doc/documentation/images/gnunet-tutorial-service.png index 6daed2f35..6daed2f35 100644 --- a/doc/images/gnunet-tutorial-service.png +++ b/doc/documentation/images/gnunet-tutorial-service.png | |||
Binary files differ | |||
diff --git a/doc/images/gnunet-tutorial-system.png b/doc/documentation/images/gnunet-tutorial-system.png index 8b54e16cf..8b54e16cf 100644 --- a/doc/images/gnunet-tutorial-system.png +++ b/doc/documentation/images/gnunet-tutorial-system.png | |||
Binary files differ | |||
diff --git a/doc/images/iceweasel-preferences.png b/doc/documentation/images/iceweasel-preferences.png index e62c2c4d9..e62c2c4d9 100644 --- a/doc/images/iceweasel-preferences.png +++ b/doc/documentation/images/iceweasel-preferences.png | |||
Binary files differ | |||
diff --git a/doc/images/iceweasel-proxy.png b/doc/documentation/images/iceweasel-proxy.png index 9caad4508..9caad4508 100644 --- a/doc/images/iceweasel-proxy.png +++ b/doc/documentation/images/iceweasel-proxy.png | |||
Binary files differ | |||
diff --git a/doc/images/lego_stack.svg b/doc/documentation/images/lego_stack.svg index a0e8017c3..a0e8017c3 100644 --- a/doc/images/lego_stack.svg +++ b/doc/documentation/images/lego_stack.svg | |||
diff --git a/doc/images/service_lego_block.png b/doc/documentation/images/service_lego_block.png index 56caf6b9c..56caf6b9c 100644 --- a/doc/images/service_lego_block.png +++ b/doc/documentation/images/service_lego_block.png | |||
Binary files differ | |||
diff --git a/doc/images/service_lego_block.svg b/doc/documentation/images/service_lego_block.svg index ef0d0234f..ef0d0234f 100644 --- a/doc/images/service_lego_block.svg +++ b/doc/documentation/images/service_lego_block.svg | |||
diff --git a/doc/images/service_stack.png b/doc/documentation/images/service_stack.png index 747d087b2..747d087b2 100644 --- a/doc/images/service_stack.png +++ b/doc/documentation/images/service_stack.png | |||
Binary files differ | |||
diff --git a/doc/structure.dot b/doc/documentation/images/structure.dot index a53db90b8..a53db90b8 100644 --- a/doc/structure.dot +++ b/doc/documentation/images/structure.dot | |||
diff --git a/doc/documentation/index.html b/doc/documentation/index.html new file mode 100644 index 000000000..0c3b04e9d --- /dev/null +++ b/doc/documentation/index.html | |||
@@ -0,0 +1,35 @@ | |||
1 | <title>GNUnet - GNUnet Manuals and Handbooks</title> | ||
2 | <h2>GNUnet - GNUnet Manuals and Handbooks</h2> | ||
3 | |||
4 | <address>GNUnet e.V.</address> | ||
5 | <address>Fakultät für Informatik -- I8</address> | ||
6 | <address>Technische Universität München</address> | ||
7 | <address>Boltzmannstraße 3</address> | ||
8 | <address>85748 Garching</address> | ||
9 | <address>GERMANY</address> | ||
10 | |||
11 | <p>The following handbooks and manuals are available:</p> | ||
12 | |||
13 | <ul> | ||
14 | <li><a href="gnunet/index.html">GNUnet Reference Manual</li> | ||
15 | <li><a href="gnunet-c-tutorial/index.html">GNUnet C Tutorial</li> | ||
16 | </ul> | ||
17 | |||
18 | <div id="footer"> | ||
19 | <div class="unprintable"> | ||
20 | |||
21 | <p>Please send general FSF & GNU inquiries to | ||
22 | <a href="mailto:gnu@gnu.org"><gnu@gnu.org></a>. | ||
23 | There are also <a href="/contact/">other ways to contact</a> | ||
24 | the FSF. Broken links and other corrections or suggestions can be sent | ||
25 | to <a href="mailto:gnunet-developers@gnu.org"><gnunet-developers@gnu.org></a>.</p> | ||
26 | </div> | ||
27 | |||
28 | <p>Copyright © 2001 - 2017 GNUnet e.V.</p> | ||
29 | |||
30 | <p>This page is licensed under a FIXME License.</p> | ||
31 | |||
32 | </div> | ||
33 | </div> | ||
34 | </body> | ||
35 | </html> | ||
diff --git a/doc/documentation/run-gendocs.sh b/doc/documentation/run-gendocs.sh new file mode 100755 index 000000000..d02570177 --- /dev/null +++ b/doc/documentation/run-gendocs.sh | |||
@@ -0,0 +1,18 @@ | |||
1 | #!/bin/sh | ||
2 | |||
3 | make version.texi | ||
4 | make version2.texi | ||
5 | ./gendocs.sh --email gnunet-developers@gnu.org gnunet-c-tutorial "GNUnet C Tutorial" -o "manual/gnunet-c-tutorial" | ||
6 | #cd manual | ||
7 | #mkdir gnunet-c-tutorial | ||
8 | #mv * gnunet-c-tutorial/ | ||
9 | #cd .. | ||
10 | ./gendocs.sh --email gnunet-developers@gnu.org gnunet "GNUnet Reference Manual" -o "manual/gnunet" | ||
11 | #cd manual | ||
12 | #mkdir handbook | ||
13 | #mkdir ../tmp-gnunet | ||
14 | #mv gnunet ../tmp-gnunet | ||
15 | #mv * handbook/ | ||
16 | #mv ../tmp-gnunet gnunet | ||
17 | cp "index.html" manual/ | ||
18 | printf "Success" | ||
diff --git a/doc/testbed_test.c b/doc/documentation/testbed_test.c index 1696234b0..1696234b0 100644 --- a/doc/testbed_test.c +++ b/doc/documentation/testbed_test.c | |||
diff --git a/doc/tutorial-examples/001.c b/doc/documentation/tutorial-examples/001.c index 7f6699dd2..7f6699dd2 100644 --- a/doc/tutorial-examples/001.c +++ b/doc/documentation/tutorial-examples/001.c | |||
diff --git a/doc/tutorial-examples/002.c b/doc/documentation/tutorial-examples/002.c index 02233fd61..02233fd61 100644 --- a/doc/tutorial-examples/002.c +++ b/doc/documentation/tutorial-examples/002.c | |||
diff --git a/doc/documentation/tutorial-examples/003.c b/doc/documentation/tutorial-examples/003.c new file mode 100644 index 000000000..d158d7e75 --- /dev/null +++ b/doc/documentation/tutorial-examples/003.c | |||
@@ -0,0 +1,11 @@ | |||
1 | struct GNUNET_MQ_MessageHandlers handlers[] = { | ||
2 | // ... | ||
3 | GNUNET_MQ_handler_end () | ||
4 | }; | ||
5 | struct GNUNET_MQ_Handle *mq; | ||
6 | |||
7 | mq = GNUNET_CLIENT_connect (cfg, | ||
8 | "service-name", | ||
9 | handlers, | ||
10 | &error_cb, | ||
11 | NULL); | ||
diff --git a/doc/tutorial-examples/004.c b/doc/documentation/tutorial-examples/004.c index 0ef007907..0ef007907 100644 --- a/doc/tutorial-examples/004.c +++ b/doc/documentation/tutorial-examples/004.c | |||
diff --git a/doc/tutorial-examples/005.c b/doc/documentation/tutorial-examples/005.c index 0c459f509..0c459f509 100644 --- a/doc/tutorial-examples/005.c +++ b/doc/documentation/tutorial-examples/005.c | |||
diff --git a/doc/tutorial-examples/006.c b/doc/documentation/tutorial-examples/006.c index 944d2b18c..944d2b18c 100644 --- a/doc/tutorial-examples/006.c +++ b/doc/documentation/tutorial-examples/006.c | |||
diff --git a/doc/tutorial-examples/007.c b/doc/documentation/tutorial-examples/007.c index 096539e43..096539e43 100644 --- a/doc/tutorial-examples/007.c +++ b/doc/documentation/tutorial-examples/007.c | |||
diff --git a/doc/tutorial-examples/008.c b/doc/documentation/tutorial-examples/008.c index 2dffe2cf9..2dffe2cf9 100644 --- a/doc/tutorial-examples/008.c +++ b/doc/documentation/tutorial-examples/008.c | |||
diff --git a/doc/tutorial-examples/009.c b/doc/documentation/tutorial-examples/009.c index 26d918fb0..26d918fb0 100644 --- a/doc/tutorial-examples/009.c +++ b/doc/documentation/tutorial-examples/009.c | |||
diff --git a/doc/tutorial-examples/010.c b/doc/documentation/tutorial-examples/010.c index 33494490d..33494490d 100644 --- a/doc/tutorial-examples/010.c +++ b/doc/documentation/tutorial-examples/010.c | |||
diff --git a/doc/tutorial-examples/011.c b/doc/documentation/tutorial-examples/011.c index 23bc051de..23bc051de 100644 --- a/doc/tutorial-examples/011.c +++ b/doc/documentation/tutorial-examples/011.c | |||
diff --git a/doc/tutorial-examples/012.c b/doc/documentation/tutorial-examples/012.c index cb21d78ab..cb21d78ab 100644 --- a/doc/tutorial-examples/012.c +++ b/doc/documentation/tutorial-examples/012.c | |||
diff --git a/doc/documentation/tutorial-examples/013.1.c b/doc/documentation/tutorial-examples/013.1.c new file mode 100644 index 000000000..fa5212868 --- /dev/null +++ b/doc/documentation/tutorial-examples/013.1.c | |||
@@ -0,0 +1,3 @@ | |||
1 | void | ||
2 | GNUNET_PEERSTORE_store_cancel (struct GNUNET_PEERSTORE_StoreContext | ||
3 | *sc); | ||
diff --git a/doc/tutorial-examples/013.c b/doc/documentation/tutorial-examples/013.c index 6792417e1..6792417e1 100644 --- a/doc/tutorial-examples/013.c +++ b/doc/documentation/tutorial-examples/013.c | |||
diff --git a/doc/tutorial-examples/014.c b/doc/documentation/tutorial-examples/014.c index ce204f795..ce204f795 100644 --- a/doc/tutorial-examples/014.c +++ b/doc/documentation/tutorial-examples/014.c | |||
diff --git a/doc/tutorial-examples/015.c b/doc/documentation/tutorial-examples/015.c index 0dd267e8e..0dd267e8e 100644 --- a/doc/tutorial-examples/015.c +++ b/doc/documentation/tutorial-examples/015.c | |||
diff --git a/doc/tutorial-examples/016.c b/doc/documentation/tutorial-examples/016.c index d8db4b3b8..d169da16d 100644 --- a/doc/tutorial-examples/016.c +++ b/doc/documentation/tutorial-examples/016.c | |||
@@ -1,3 +1,4 @@ | |||
1 | void | 1 | void |
2 | GNUNET_PEERSTORE_watch_cancel (struct GNUNET_PEERSTORE_WatchContext *wc); | 2 | GNUNET_PEERSTORE_watch_cancel (struct GNUNET_PEERSTORE_WatchContext |
3 | *wc); | ||
3 | 4 | ||
diff --git a/doc/documentation/tutorial-examples/017.c b/doc/documentation/tutorial-examples/017.c new file mode 100644 index 000000000..c86fbcd1f --- /dev/null +++ b/doc/documentation/tutorial-examples/017.c | |||
@@ -0,0 +1,4 @@ | |||
1 | void | ||
2 | GNUNET_PEERSTORE_disconnect (struct GNUNET_PEERSTORE_Handle *h, | ||
3 | int sync_first); | ||
4 | |||
diff --git a/doc/tutorial-examples/018.c b/doc/documentation/tutorial-examples/018.c index 3fc22584c..3fc22584c 100644 --- a/doc/tutorial-examples/018.c +++ b/doc/documentation/tutorial-examples/018.c | |||
diff --git a/doc/tutorial-examples/019.c b/doc/documentation/tutorial-examples/019.c index d016d381b..aaf001516 100644 --- a/doc/tutorial-examples/019.c +++ b/doc/documentation/tutorial-examples/019.c | |||
@@ -1,4 +1,5 @@ | |||
1 | message_sent_cont (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) | 1 | message_sent_cont (void *cls, |
2 | const struct GNUNET_SCHEDULER_TaskContext *tc) | ||
2 | { | 3 | { |
3 | // Request has left local node | 4 | // Request has left local node |
4 | } | 5 | } |
@@ -8,7 +9,9 @@ GNUNET_DHT_put (struct GNUNET_DHT_Handle *handle, | |||
8 | const struct GNUNET_HashCode *key, | 9 | const struct GNUNET_HashCode *key, |
9 | uint32_t desired_replication_level, | 10 | uint32_t desired_replication_level, |
10 | enum GNUNET_DHT_RouteOption options, | 11 | enum GNUNET_DHT_RouteOption options, |
11 | enum GNUNET_BLOCK_Type type, size_t size, const void *data, | 12 | enum GNUNET_BLOCK_Type type, |
13 | size_t size, | ||
14 | const void *data, | ||
12 | struct GNUNET_TIME_Absolute exp, | 15 | struct GNUNET_TIME_Absolute exp, |
13 | struct GNUNET_TIME_Relative timeout, | 16 | struct GNUNET_TIME_Relative timeout, |
14 | GNUNET_DHT_PutContinuation cont, void *cont_cls) | 17 | GNUNET_DHT_PutContinuation cont, void *cont_cls) |
diff --git a/doc/tutorial-examples/020.c b/doc/documentation/tutorial-examples/020.c index 5ecba1c16..596db3069 100644 --- a/doc/tutorial-examples/020.c +++ b/doc/documentation/tutorial-examples/020.c | |||
@@ -5,7 +5,8 @@ get_result_iterator (void *cls, struct GNUNET_TIME_Absolute expiration, | |||
5 | unsigned int get_path_length, | 5 | unsigned int get_path_length, |
6 | const struct GNUNET_PeerIdentity *put_path, | 6 | const struct GNUNET_PeerIdentity *put_path, |
7 | unsigned int put_path_length, | 7 | unsigned int put_path_length, |
8 | enum GNUNET_BLOCK_Type type, size_t size, const void *data) | 8 | enum GNUNET_BLOCK_Type type, size_t size, |
9 | const void *data) | ||
9 | { | 10 | { |
10 | // Optionally: | 11 | // Optionally: |
11 | GNUNET_DHT_get_stop (get_handle); | 12 | GNUNET_DHT_get_stop (get_handle); |
diff --git a/doc/tutorial-examples/021.c b/doc/documentation/tutorial-examples/021.c index 688a31fe0..688a31fe0 100644 --- a/doc/tutorial-examples/021.c +++ b/doc/documentation/tutorial-examples/021.c | |||
diff --git a/doc/tutorial-examples/022.c b/doc/documentation/tutorial-examples/022.c index a373619bd..a373619bd 100644 --- a/doc/tutorial-examples/022.c +++ b/doc/documentation/tutorial-examples/022.c | |||
diff --git a/doc/tutorial-examples/023.c b/doc/documentation/tutorial-examples/023.c index 820c38b10..820c38b10 100644 --- a/doc/tutorial-examples/023.c +++ b/doc/documentation/tutorial-examples/023.c | |||
diff --git a/doc/tutorial-examples/024.c b/doc/documentation/tutorial-examples/024.c index 2e84b5905..2e84b5905 100644 --- a/doc/tutorial-examples/024.c +++ b/doc/documentation/tutorial-examples/024.c | |||
diff --git a/doc/tutorial-examples/025.c b/doc/documentation/tutorial-examples/025.c index 66d4f80ec..66d4f80ec 100644 --- a/doc/tutorial-examples/025.c +++ b/doc/documentation/tutorial-examples/025.c | |||
diff --git a/doc/tutorial-examples/026.c b/doc/documentation/tutorial-examples/026.c index 264e0b6b9..264e0b6b9 100644 --- a/doc/tutorial-examples/026.c +++ b/doc/documentation/tutorial-examples/026.c | |||
diff --git a/doc/hacks.el b/doc/hacks.el new file mode 100644 index 000000000..9f271b3af --- /dev/null +++ b/doc/hacks.el | |||
@@ -0,0 +1,17 @@ | |||
1 | ;;;; hacks.el --- a few functions to help me work on the manual | ||
2 | ;;;; Jim Blandy <jimb@red-bean.com> --- October 1998 | ||
3 | ;;;; -- imported from https://git.savannah.gnu.org/cgit/guile.git/tree/doc/hacks.el | ||
4 | |||
5 | (defun jh-exemplify-region (start end) | ||
6 | (interactive "r") | ||
7 | (save-excursion | ||
8 | (save-restriction | ||
9 | (narrow-to-region start end) | ||
10 | |||
11 | ;; Texinfo doesn't handle tabs well. | ||
12 | (untabify (point-min) (point-max)) | ||
13 | |||
14 | ;; Quote any characters special to texinfo. | ||
15 | (goto-char (point-min)) | ||
16 | (while (re-search-forward "[{}@]" nil t) | ||
17 | (replace-match "@\\&"))))) | ||
diff --git a/doc/man/gnunet-ecc.1 b/doc/man/gnunet-ecc.1 index a91a2ac2f..910687f1f 100644 --- a/doc/man/gnunet-ecc.1 +++ b/doc/man/gnunet-ecc.1 | |||
@@ -19,11 +19,11 @@ Create COUNT public-private key pairs and write them to FILENAME. Used for crea | |||
19 | .IP "\-p, \-\-print-public-key" | 19 | .IP "\-p, \-\-print-public-key" |
20 | Print the corresponding public key to stdout. This is the value used for PKEY records in GNS. | 20 | Print the corresponding public key to stdout. This is the value used for PKEY records in GNS. |
21 | .B | 21 | .B |
22 | .IP "\-p, \-\-print-hex" | 22 | .IP "\-P, \-\-print-private-key" |
23 | Print the corresponding public key to stdout in HEX format. Useful for comparing to Ed25519 keys in X.509 tools. | 23 | Print the corresponding private key to stdout. This is the value used for PKEY records in GNS. |
24 | .B | 24 | .B |
25 | .IP "\-P, \-\-print-peer-identity" | 25 | .IP "\-x, \-\-print-hex" |
26 | Print the corresponding peer identity (hash of the public key) to stdout. This hash is used for the name of peers. | 26 | Print the corresponding public key to stdout in HEX format. Useful for comparing to Ed25519 keys in X.509 tools. |
27 | .B | 27 | .B |
28 | .IP "\-c FILENAME, \-\-config=FILENAME" | 28 | .IP "\-c FILENAME, \-\-config=FILENAME" |
29 | Use the configuration file FILENAME. | 29 | Use the configuration file FILENAME. |
diff --git a/doc/man/gnunet-identity.1 b/doc/man/gnunet-identity.1 index b552ad6e3..50efa74d4 100644 --- a/doc/man/gnunet-identity.1 +++ b/doc/man/gnunet-identity.1 | |||
@@ -30,7 +30,7 @@ Print help page. | |||
30 | 30 | ||
31 | .TP | 31 | .TP |
32 | \fB\-d\fR, \fB\-\-display\fR | 32 | \fB\-d\fR, \fB\-\-display\fR |
33 | display all ouf our egos | 33 | display all of our egos |
34 | 34 | ||
35 | .TP | 35 | .TP |
36 | \fB\-m\fR, \fB\-\-monitor\fR | 36 | \fB\-m\fR, \fB\-\-monitor\fR |
diff --git a/doc/tutorial-examples/003.c b/doc/tutorial-examples/003.c deleted file mode 100644 index d13681ca6..000000000 --- a/doc/tutorial-examples/003.c +++ /dev/null | |||
@@ -1,7 +0,0 @@ | |||
1 | struct GNUNET_MQ_MessageHandlers handlers[] = { | ||
2 | // ... | ||
3 | GNUNET_MQ_handler_end () | ||
4 | }; | ||
5 | struct GNUNET_MQ_Handle *mq; | ||
6 | |||
7 | mq = GNUNET_CLIENT_connect (cfg, "service-name", handlers, &error_cb, NULL); | ||
diff --git a/doc/tutorial-examples/017.c b/doc/tutorial-examples/017.c deleted file mode 100644 index c4acbc088..000000000 --- a/doc/tutorial-examples/017.c +++ /dev/null | |||
@@ -1,3 +0,0 @@ | |||
1 | void | ||
2 | GNUNET_PEERSTORE_disconnect (struct GNUNET_PEERSTORE_Handle *h, int sync_first); | ||
3 | |||
diff --git a/po/POTFILES.in b/po/POTFILES.in index 43a5dcb08..4a4cfbd96 100644 --- a/po/POTFILES.in +++ b/po/POTFILES.in | |||
@@ -306,7 +306,6 @@ src/revocation/revocation_api.c | |||
306 | src/rps/gnunet-rps.c | 306 | src/rps/gnunet-rps.c |
307 | src/rps/gnunet-service-rps.c | 307 | src/rps/gnunet-service-rps.c |
308 | src/rps/gnunet-service-rps_custommap.c | 308 | src/rps/gnunet-service-rps_custommap.c |
309 | src/rps/gnunet-service-rps_peers.c | ||
310 | src/rps/gnunet-service-rps_sampler.c | 309 | src/rps/gnunet-service-rps_sampler.c |
311 | src/rps/gnunet-service-rps_sampler_elem.c | 310 | src/rps/gnunet-service-rps_sampler_elem.c |
312 | src/rps/gnunet-service-rps_view.c | 311 | src/rps/gnunet-service-rps_view.c |
diff --git a/src/arm/Makefile.am b/src/arm/Makefile.am index 373847fde..b1706a479 100644 --- a/src/arm/Makefile.am +++ b/src/arm/Makefile.am | |||
@@ -92,7 +92,8 @@ test_gnunet_service_arm_SOURCES = \ | |||
92 | 92 | ||
93 | do_subst = $(SED) -e 's,[@]PYTHON[@],$(PYTHON),g' | 93 | do_subst = $(SED) -e 's,[@]PYTHON[@],$(PYTHON),g' |
94 | 94 | ||
95 | %.py: %.py.in Makefile | 95 | SUFFIXES = .py.in .py |
96 | .py.in.py: | ||
96 | $(do_subst) < $(srcdir)/$< > $@ | 97 | $(do_subst) < $(srcdir)/$< > $@ |
97 | chmod +x $@ | 98 | chmod +x $@ |
98 | 99 | ||
diff --git a/src/arm/gnunet-service-arm.c b/src/arm/gnunet-service-arm.c index 2db2ba0d1..bf5982d10 100644 --- a/src/arm/gnunet-service-arm.c +++ b/src/arm/gnunet-service-arm.c | |||
@@ -1915,7 +1915,6 @@ setup_service (void *cls, | |||
1915 | struct sockaddr **addrs; | 1915 | struct sockaddr **addrs; |
1916 | socklen_t *addr_lens; | 1916 | socklen_t *addr_lens; |
1917 | int ret; | 1917 | int ret; |
1918 | unsigned int i; | ||
1919 | 1918 | ||
1920 | if (0 == strcasecmp (section, | 1919 | if (0 == strcasecmp (section, |
1921 | "arm")) | 1920 | "arm")) |
@@ -2027,7 +2026,7 @@ setup_service (void *cls, | |||
2027 | &addr_lens))) | 2026 | &addr_lens))) |
2028 | return; | 2027 | return; |
2029 | /* this will free (or capture) addrs[i] */ | 2028 | /* this will free (or capture) addrs[i] */ |
2030 | for (i = 0; i < ret; i++) | 2029 | for (unsigned int i = 0; i < ret; i++) |
2031 | create_listen_socket (addrs[i], | 2030 | create_listen_socket (addrs[i], |
2032 | addr_lens[i], | 2031 | addr_lens[i], |
2033 | sl); | 2032 | sl); |
diff --git a/src/ats-tool/gnunet-ats.c b/src/ats-tool/gnunet-ats.c index 5ec7693b1..d88e6d523 100644 --- a/src/ats-tool/gnunet-ats.c +++ b/src/ats-tool/gnunet-ats.c | |||
@@ -872,8 +872,8 @@ run (void *cls, | |||
872 | 872 | ||
873 | for (c = 0; c < strlen (opt_type_str); c++) | 873 | for (c = 0; c < strlen (opt_type_str); c++) |
874 | { | 874 | { |
875 | if (isupper (opt_type_str[c])) | 875 | if (isupper ((unsigned char) opt_type_str[c])) |
876 | opt_type_str[c] = tolower (opt_type_str[c]); | 876 | opt_type_str[c] = tolower ((unsigned char) opt_type_str[c]); |
877 | } | 877 | } |
878 | 878 | ||
879 | if (0 == strcasecmp ("latency", opt_type_str)) | 879 | if (0 == strcasecmp ("latency", opt_type_str)) |
@@ -974,7 +974,7 @@ main (int argc, | |||
974 | gettext_noop ("set preference for the given peer"), | 974 | gettext_noop ("set preference for the given peer"), |
975 | &opt_set_pref), | 975 | &opt_set_pref), |
976 | 976 | ||
977 | GNUNET_GETOPT_option_flag ('q', | 977 | GNUNET_GETOPT_option_flag ('q', |
978 | "quotas", | 978 | "quotas", |
979 | gettext_noop ("print all configured quotas"), | 979 | gettext_noop ("print all configured quotas"), |
980 | &opt_print_quotas), | 980 | &opt_print_quotas), |
diff --git a/src/cadet/cadet_api.c b/src/cadet/cadet_api.c index 00a482452..366e31405 100644 --- a/src/cadet/cadet_api.c +++ b/src/cadet/cadet_api.c | |||
@@ -824,6 +824,29 @@ handle_mq_error (void *cls, | |||
824 | 824 | ||
825 | 825 | ||
826 | /** | 826 | /** |
827 | * Check that message received from CADET service is well-formed. | ||
828 | * | ||
829 | * @param cls the `struct GNUNET_CADET_Handle` | ||
830 | * @param message the message we got | ||
831 | * @return #GNUNET_OK if the message is well-formed, | ||
832 | * #GNUNET_SYSERR otherwise | ||
833 | */ | ||
834 | static int | ||
835 | check_get_peers (void *cls, | ||
836 | const struct GNUNET_MessageHeader *message) | ||
837 | { | ||
838 | size_t esize; | ||
839 | |||
840 | esize = ntohs (message->size); | ||
841 | if (sizeof (struct GNUNET_CADET_LocalInfoPeer) == esize) | ||
842 | return GNUNET_OK; | ||
843 | if (sizeof (struct GNUNET_MessageHeader) == esize) | ||
844 | return GNUNET_OK; | ||
845 | return GNUNET_SYSERR; | ||
846 | } | ||
847 | |||
848 | |||
849 | /** | ||
827 | * Process a local reply about info on all tunnels, pass info to the user. | 850 | * Process a local reply about info on all tunnels, pass info to the user. |
828 | * | 851 | * |
829 | * @param cls Closure (Cadet handle). | 852 | * @param cls Closure (Cadet handle). |
@@ -831,17 +854,26 @@ handle_mq_error (void *cls, | |||
831 | */ | 854 | */ |
832 | static void | 855 | static void |
833 | handle_get_peers (void *cls, | 856 | handle_get_peers (void *cls, |
834 | const struct GNUNET_CADET_LocalInfoPeer *msg) | 857 | const struct GNUNET_MessageHeader *msg) |
835 | { | 858 | { |
836 | struct GNUNET_CADET_Handle *h = cls; | 859 | struct GNUNET_CADET_Handle *h = cls; |
860 | const struct GNUNET_CADET_LocalInfoPeer *info = | ||
861 | (const struct GNUNET_CADET_LocalInfoPeer *) msg; | ||
837 | 862 | ||
838 | if (NULL == h->info_cb.peers_cb) | 863 | if (NULL == h->info_cb.peers_cb) |
839 | return; | 864 | return; |
840 | h->info_cb.peers_cb (h->info_cls, | 865 | if (sizeof (struct GNUNET_CADET_LocalInfoPeer) == ntohs (msg->size)) |
841 | &msg->destination, | 866 | h->info_cb.peers_cb (h->info_cls, |
842 | (int) ntohs (msg->tunnel), | 867 | &info->destination, |
843 | (unsigned int) ntohs (msg->paths), | 868 | (int) ntohs (info->tunnel), |
844 | 0); | 869 | (unsigned int) ntohs (info->paths), |
870 | 0); | ||
871 | else | ||
872 | h->info_cb.peers_cb (h->info_cls, | ||
873 | NULL, | ||
874 | 0, | ||
875 | 0, | ||
876 | 0); | ||
845 | } | 877 | } |
846 | 878 | ||
847 | 879 | ||
@@ -946,6 +978,29 @@ handle_get_peer (void *cls, | |||
946 | 978 | ||
947 | 979 | ||
948 | /** | 980 | /** |
981 | * Check that message received from CADET service is well-formed. | ||
982 | * | ||
983 | * @param cls the `struct GNUNET_CADET_Handle` | ||
984 | * @param message the message we got | ||
985 | * @return #GNUNET_OK if the message is well-formed, | ||
986 | * #GNUNET_SYSERR otherwise | ||
987 | */ | ||
988 | static int | ||
989 | check_get_tunnels (void *cls, | ||
990 | const struct GNUNET_MessageHeader *message) | ||
991 | { | ||
992 | size_t esize; | ||
993 | |||
994 | esize = ntohs (message->size); | ||
995 | if (sizeof (struct GNUNET_CADET_LocalInfoTunnel) == esize) | ||
996 | return GNUNET_OK; | ||
997 | if (sizeof (struct GNUNET_MessageHeader) == esize) | ||
998 | return GNUNET_OK; | ||
999 | return GNUNET_SYSERR; | ||
1000 | } | ||
1001 | |||
1002 | |||
1003 | /** | ||
949 | * Process a local reply about info on all tunnels, pass info to the user. | 1004 | * Process a local reply about info on all tunnels, pass info to the user. |
950 | * | 1005 | * |
951 | * @param cls Closure (Cadet handle). | 1006 | * @param cls Closure (Cadet handle). |
@@ -953,19 +1008,28 @@ handle_get_peer (void *cls, | |||
953 | */ | 1008 | */ |
954 | static void | 1009 | static void |
955 | handle_get_tunnels (void *cls, | 1010 | handle_get_tunnels (void *cls, |
956 | const struct GNUNET_CADET_LocalInfoTunnel *msg) | 1011 | const struct GNUNET_MessageHeader *msg) |
957 | { | 1012 | { |
958 | struct GNUNET_CADET_Handle *h = cls; | 1013 | struct GNUNET_CADET_Handle *h = cls; |
1014 | const struct GNUNET_CADET_LocalInfoTunnel *info = | ||
1015 | (const struct GNUNET_CADET_LocalInfoTunnel *) msg; | ||
959 | 1016 | ||
960 | if (NULL == h->info_cb.tunnels_cb) | 1017 | if (NULL == h->info_cb.tunnels_cb) |
961 | return; | 1018 | return; |
962 | h->info_cb.tunnels_cb (h->info_cls, | 1019 | if (sizeof (struct GNUNET_CADET_LocalInfoTunnel) == ntohs (msg->size)) |
963 | &msg->destination, | 1020 | h->info_cb.tunnels_cb (h->info_cls, |
964 | ntohl (msg->channels), | 1021 | &info->destination, |
965 | ntohl (msg->connections), | 1022 | ntohl (info->channels), |
966 | ntohs (msg->estate), | 1023 | ntohl (info->connections), |
967 | ntohs (msg->cstate)); | 1024 | ntohs (info->estate), |
968 | 1025 | ntohs (info->cstate)); | |
1026 | else | ||
1027 | h->info_cb.tunnels_cb (h->info_cls, | ||
1028 | NULL, | ||
1029 | 0, | ||
1030 | 0, | ||
1031 | 0, | ||
1032 | 0); | ||
969 | } | 1033 | } |
970 | 1034 | ||
971 | 1035 | ||
@@ -1075,18 +1139,18 @@ reconnect (struct GNUNET_CADET_Handle *h) | |||
1075 | GNUNET_MESSAGE_TYPE_CADET_LOCAL_ACK, | 1139 | GNUNET_MESSAGE_TYPE_CADET_LOCAL_ACK, |
1076 | struct GNUNET_CADET_LocalAck, | 1140 | struct GNUNET_CADET_LocalAck, |
1077 | h), | 1141 | h), |
1078 | GNUNET_MQ_hd_fixed_size (get_peers, | 1142 | GNUNET_MQ_hd_var_size (get_peers, |
1079 | GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEERS, | 1143 | GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEERS, |
1080 | struct GNUNET_CADET_LocalInfoPeer, | 1144 | struct GNUNET_MessageHeader, |
1081 | h), | 1145 | h), |
1082 | GNUNET_MQ_hd_var_size (get_peer, | 1146 | GNUNET_MQ_hd_var_size (get_peer, |
1083 | GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEER, | 1147 | GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEER, |
1084 | struct GNUNET_CADET_LocalInfoPeer, | 1148 | struct GNUNET_CADET_LocalInfoPeer, |
1085 | h), | 1149 | h), |
1086 | GNUNET_MQ_hd_fixed_size (get_tunnels, | 1150 | GNUNET_MQ_hd_var_size (get_tunnels, |
1087 | GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNELS, | 1151 | GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNELS, |
1088 | struct GNUNET_CADET_LocalInfoTunnel, | 1152 | struct GNUNET_MessageHeader, |
1089 | h), | 1153 | h), |
1090 | GNUNET_MQ_hd_var_size (get_tunnel, | 1154 | GNUNET_MQ_hd_var_size (get_tunnel, |
1091 | GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNEL, | 1155 | GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNEL, |
1092 | struct GNUNET_CADET_LocalInfoTunnel, | 1156 | struct GNUNET_CADET_LocalInfoTunnel, |
@@ -1094,6 +1158,7 @@ reconnect (struct GNUNET_CADET_Handle *h) | |||
1094 | GNUNET_MQ_handler_end () | 1158 | GNUNET_MQ_handler_end () |
1095 | }; | 1159 | }; |
1096 | 1160 | ||
1161 | GNUNET_assert (NULL == h->mq); | ||
1097 | h->mq = GNUNET_CLIENT_connect (h->cfg, | 1162 | h->mq = GNUNET_CLIENT_connect (h->cfg, |
1098 | "cadet", | 1163 | "cadet", |
1099 | handlers, | 1164 | handlers, |
@@ -1600,7 +1665,10 @@ GNUNET_CADET_open_port (struct GNUNET_CADET_Handle *h, | |||
1600 | 1665 | ||
1601 | GNUNET_assert (NULL != connects); | 1666 | GNUNET_assert (NULL != connects); |
1602 | GNUNET_assert (NULL != disconnects); | 1667 | GNUNET_assert (NULL != disconnects); |
1603 | 1668 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | |
1669 | "Listening to CADET port %s\n", | ||
1670 | GNUNET_h2s (port)); | ||
1671 | |||
1604 | p = GNUNET_new (struct GNUNET_CADET_Port); | 1672 | p = GNUNET_new (struct GNUNET_CADET_Port); |
1605 | p->cadet = h; | 1673 | p->cadet = h; |
1606 | p->id = *port; | 1674 | p->id = *port; |
@@ -1663,6 +1731,10 @@ GNUNET_CADET_channel_create (struct GNUNET_CADET_Handle *h, | |||
1663 | struct GNUNET_MQ_Envelope *env; | 1731 | struct GNUNET_MQ_Envelope *env; |
1664 | 1732 | ||
1665 | GNUNET_assert (NULL != disconnects); | 1733 | GNUNET_assert (NULL != disconnects); |
1734 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
1735 | "Creating channel to peer %s at port %s\n", | ||
1736 | GNUNET_i2s (destination), | ||
1737 | GNUNET_h2s (port)); | ||
1666 | ch = create_channel (h, | 1738 | ch = create_channel (h, |
1667 | NULL); | 1739 | NULL); |
1668 | ch->ctx = channel_cls; | 1740 | ch->ctx = channel_cls; |
diff --git a/src/cadet/gnunet-cadet.c b/src/cadet/gnunet-cadet.c index 675e7faf0..a9b02714b 100644 --- a/src/cadet/gnunet-cadet.c +++ b/src/cadet/gnunet-cadet.c | |||
@@ -693,6 +693,7 @@ show_tunnel (void *cls) | |||
693 | { | 693 | { |
694 | struct GNUNET_PeerIdentity pid; | 694 | struct GNUNET_PeerIdentity pid; |
695 | 695 | ||
696 | job = NULL; | ||
696 | if (GNUNET_OK != | 697 | if (GNUNET_OK != |
697 | GNUNET_CRYPTO_eddsa_public_key_from_string (tunnel_id, | 698 | GNUNET_CRYPTO_eddsa_public_key_from_string (tunnel_id, |
698 | strlen (tunnel_id), | 699 | strlen (tunnel_id), |
diff --git a/src/cadet/gnunet-service-cadet_core.c b/src/cadet/gnunet-service-cadet_core.c index ae03b4f35..99957d8a1 100644 --- a/src/cadet/gnunet-service-cadet_core.c +++ b/src/cadet/gnunet-service-cadet_core.c | |||
@@ -771,6 +771,33 @@ handle_connection_create (void *cls, | |||
771 | 771 | ||
772 | options = (enum GNUNET_CADET_ChannelOption) ntohl (msg->options); | 772 | options = (enum GNUNET_CADET_ChannelOption) ntohl (msg->options); |
773 | path_length = size / sizeof (struct GNUNET_PeerIdentity); | 773 | path_length = size / sizeof (struct GNUNET_PeerIdentity); |
774 | if (0 == path_length) | ||
775 | { | ||
776 | LOG (GNUNET_ERROR_TYPE_DEBUG, | ||
777 | "Dropping CADET_CONNECTION_CREATE with empty path\n"); | ||
778 | GNUNET_break_op (0); | ||
779 | return; | ||
780 | } | ||
781 | /* Check for loops */ | ||
782 | struct GNUNET_CONTAINER_MultiPeerMap *map; | ||
783 | map = GNUNET_CONTAINER_multipeermap_create (path_length * 2, | ||
784 | GNUNET_YES); | ||
785 | GNUNET_assert (NULL != map); | ||
786 | for (off = 0; off < path_length; off++) { | ||
787 | if (GNUNET_SYSERR == | ||
788 | GNUNET_CONTAINER_multipeermap_put (map, | ||
789 | &pids[off], | ||
790 | NULL, | ||
791 | GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)) { | ||
792 | /* bogus request */ | ||
793 | GNUNET_CONTAINER_multipeermap_destroy (map); | ||
794 | LOG (GNUNET_ERROR_TYPE_DEBUG, | ||
795 | "Dropping CADET_CONNECTION_CREATE with cyclic path\n"); | ||
796 | GNUNET_break_op (0); | ||
797 | return; | ||
798 | } | ||
799 | } | ||
800 | GNUNET_CONTAINER_multipeermap_destroy (map); | ||
774 | /* Initiator is at offset 0. */ | 801 | /* Initiator is at offset 0. */ |
775 | for (off=1;off<path_length;off++) | 802 | for (off=1;off<path_length;off++) |
776 | if (0 == memcmp (&my_full_id, | 803 | if (0 == memcmp (&my_full_id, |
@@ -779,7 +806,8 @@ handle_connection_create (void *cls, | |||
779 | break; | 806 | break; |
780 | if (off == path_length) | 807 | if (off == path_length) |
781 | { | 808 | { |
782 | /* We are not on the path, bogus request */ | 809 | LOG (GNUNET_ERROR_TYPE_DEBUG, |
810 | "Dropping CADET_CONNECTION_CREATE without us in the path\n"); | ||
783 | GNUNET_break_op (0); | 811 | GNUNET_break_op (0); |
784 | return; | 812 | return; |
785 | } | 813 | } |
@@ -787,7 +815,8 @@ handle_connection_create (void *cls, | |||
787 | if (sender != GCP_get (&pids[off - 1], | 815 | if (sender != GCP_get (&pids[off - 1], |
788 | GNUNET_NO)) | 816 | GNUNET_NO)) |
789 | { | 817 | { |
790 | /* sender is not on the path, not allowed */ | 818 | LOG (GNUNET_ERROR_TYPE_DEBUG, |
819 | "Dropping CADET_CONNECTION_CREATE without sender in the path\n"); | ||
791 | GNUNET_break_op (0); | 820 | GNUNET_break_op (0); |
792 | return; | 821 | return; |
793 | } | 822 | } |
diff --git a/src/cadet/gnunet-service-cadet_paths.c b/src/cadet/gnunet-service-cadet_paths.c index 13752643c..9dd6f1ddd 100644 --- a/src/cadet/gnunet-service-cadet_paths.c +++ b/src/cadet/gnunet-service-cadet_paths.c | |||
@@ -187,33 +187,51 @@ GCPP_del_connection (struct CadetPeerPath *path, | |||
187 | 187 | ||
188 | 188 | ||
189 | /** | 189 | /** |
190 | * This path is no longer needed, free resources. | 190 | * Tries to attach @a path to a peer, working backwards from the end |
191 | * and stopping at @a stop_at. If path->hn is NULL on return then the | ||
192 | * path was not attached and you can assume that path->entries_length | ||
193 | * is equal to @a stop_at. | ||
191 | * | 194 | * |
192 | * @param path path resources to free | 195 | * @param path the path to attach |
196 | * @param stop_at the path length at which to stop trying | ||
193 | */ | 197 | */ |
194 | static void | 198 | static void |
195 | path_destroy (struct CadetPeerPath *path) | 199 | attach_path (struct CadetPeerPath *path, unsigned int stop_at) |
196 | { | 200 | { |
197 | LOG (GNUNET_ERROR_TYPE_DEBUG, | 201 | GNUNET_assert (NULL == path->hn); |
198 | "Destroying path %s\n", | 202 | |
199 | GCPP_2s (path)); | 203 | /* Try to attach this path to a peer, working backwards from the end. */ |
200 | for (unsigned int i=0;i<path->entries_length;i++) | 204 | while (path->entries_length > stop_at) |
201 | { | 205 | { |
202 | struct CadetPeerPathEntry *entry = path->entries[i]; | 206 | unsigned int end = path->entries_length - 1; |
207 | struct CadetPeerPathEntry *entry = path->entries[end]; | ||
208 | int force = GNUNET_NO; | ||
203 | 209 | ||
210 | recalculate_path_desirability (path); | ||
211 | /* If the entry already has a connection using it, force attach. */ | ||
204 | if (NULL != entry->cc) | 212 | if (NULL != entry->cc) |
205 | { | 213 | force = GNUNET_YES; |
206 | struct CadetTConnection *ct; | 214 | path->hn = GCP_attach_path (entry->peer, |
215 | path, | ||
216 | end, | ||
217 | force); | ||
218 | if (NULL != path->hn) | ||
219 | break; | ||
207 | 220 | ||
208 | ct = GCC_get_ct (entry->cc); | 221 | /* Attach failed, trim this entry from the path. */ |
209 | if (NULL != ct) | 222 | GNUNET_assert (NULL == entry->cc); |
210 | GCT_connection_lost (ct); | 223 | GCP_path_entry_remove (entry->peer, |
211 | GCC_destroy_without_tunnel (entry->cc); | 224 | entry, |
212 | } | 225 | end); |
213 | GNUNET_free (entry); | 226 | GNUNET_free (entry); |
227 | path->entries[end] = NULL; | ||
228 | path->entries_length--; | ||
214 | } | 229 | } |
215 | GNUNET_free (path->entries); | 230 | |
216 | GNUNET_free (path); | 231 | /* Shrink array to actual path length. */ |
232 | GNUNET_array_grow (path->entries, | ||
233 | path->entries_length, | ||
234 | path->entries_length); | ||
217 | } | 235 | } |
218 | 236 | ||
219 | 237 | ||
@@ -228,7 +246,6 @@ void | |||
228 | GCPP_release (struct CadetPeerPath *path) | 246 | GCPP_release (struct CadetPeerPath *path) |
229 | { | 247 | { |
230 | struct CadetPeerPathEntry *entry; | 248 | struct CadetPeerPathEntry *entry; |
231 | int force; | ||
232 | 249 | ||
233 | LOG (GNUNET_ERROR_TYPE_DEBUG, | 250 | LOG (GNUNET_ERROR_TYPE_DEBUG, |
234 | "Owner releases path %s\n", | 251 | "Owner releases path %s\n", |
@@ -236,34 +253,23 @@ GCPP_release (struct CadetPeerPath *path) | |||
236 | path->hn = NULL; | 253 | path->hn = NULL; |
237 | entry = path->entries[path->entries_length - 1]; | 254 | entry = path->entries[path->entries_length - 1]; |
238 | GNUNET_assert (path == entry->path); | 255 | GNUNET_assert (path == entry->path); |
239 | while (1) | 256 | GNUNET_assert (NULL == entry->cc); |
257 | /* cut 'off' end of path */ | ||
258 | GCP_path_entry_remove (entry->peer, | ||
259 | entry, | ||
260 | path->entries_length - 1); | ||
261 | GNUNET_free (entry); | ||
262 | path->entries[path->entries_length - 1] = NULL; | ||
263 | path->entries_length--; | ||
264 | /* see if new peer at the end likes this path any better */ | ||
265 | attach_path (path, 0); | ||
266 | if (NULL == path->hn) | ||
240 | { | 267 | { |
241 | /* cut 'off' end of path */ | 268 | /* nobody wants us, discard the path */ |
242 | GNUNET_assert (NULL == entry->cc); | 269 | GNUNET_assert (0 == path->entries_length); |
243 | GCP_path_entry_remove (entry->peer, | 270 | GNUNET_assert (NULL == path->entries); |
244 | entry, | 271 | GNUNET_free (path); |
245 | path->entries_length - 1); | ||
246 | path->entries_length--; /* We don't bother shrinking the 'entries' array, | ||
247 | as it's probably not worth it. */ | ||
248 | GNUNET_free (entry); | ||
249 | if (0 == path->entries_length) | ||
250 | break; /* the end */ | ||
251 | |||
252 | /* see if new peer at the end likes this path any better */ | ||
253 | entry = path->entries[path->entries_length - 1]; | ||
254 | GNUNET_assert (path == entry->path); | ||
255 | force = (NULL == entry->cc) ? GNUNET_NO : GNUNET_YES; | ||
256 | path->hn = GCP_attach_path (entry->peer, | ||
257 | path, | ||
258 | path->entries_length - 1, | ||
259 | force); | ||
260 | if (NULL != path->hn) | ||
261 | return; /* yep, got attached, we are done. */ | ||
262 | GNUNET_assert (GNUNET_NO == force); | ||
263 | } | 272 | } |
264 | |||
265 | /* nobody wants us, discard the path */ | ||
266 | path_destroy (path); | ||
267 | } | 273 | } |
268 | 274 | ||
269 | 275 | ||
@@ -422,32 +428,13 @@ extend_path (struct CadetPeerPath *path, | |||
422 | path, | 428 | path, |
423 | path->hn); | 429 | path->hn); |
424 | path->hn = NULL; | 430 | path->hn = NULL; |
425 | for (i=num_peers-1;i>=0;i--) | 431 | path->entries_length = old_len + num_peers; |
426 | { | 432 | attach_path (path, old_len); |
427 | struct CadetPeerPathEntry *entry = path->entries[old_len + i]; | ||
428 | |||
429 | path->entries_length = old_len + i + 1; | ||
430 | recalculate_path_desirability (path); | ||
431 | path->hn = GCP_attach_path (peers[i], | ||
432 | path, | ||
433 | old_len + (unsigned int) i, | ||
434 | force); | ||
435 | if (NULL != path->hn) | ||
436 | break; | ||
437 | GNUNET_assert (NULL == entry->cc); | ||
438 | GCP_path_entry_remove (entry->peer, | ||
439 | entry, | ||
440 | old_len + i); | ||
441 | GNUNET_free (entry); | ||
442 | path->entries[old_len + i] = NULL; | ||
443 | } | ||
444 | if (NULL == path->hn) | 433 | if (NULL == path->hn) |
445 | { | 434 | { |
446 | /* none of the peers is interested in this path; | 435 | /* none of the peers is interested in this path; |
447 | shrink path back and re-attach. */ | 436 | re-attach. */ |
448 | GNUNET_array_grow (path->entries, | 437 | GNUNET_assert (old_len == path->entries_length); |
449 | path->entries_length, | ||
450 | old_len); | ||
451 | path->hn = GCP_attach_path (path->entries[old_len - 1]->peer, | 438 | path->hn = GCP_attach_path (path->entries[old_len - 1]->peer, |
452 | path, | 439 | path, |
453 | old_len - 1, | 440 | old_len - 1, |
@@ -482,7 +469,6 @@ GCPP_try_path_from_dht (const struct GNUNET_PeerIdentity *get_path, | |||
482 | struct CadetPeer *cpath[get_path_length + put_path_length]; | 469 | struct CadetPeer *cpath[get_path_length + put_path_length]; |
483 | struct CheckMatchContext cm_ctx; | 470 | struct CheckMatchContext cm_ctx; |
484 | struct CadetPeerPath *path; | 471 | struct CadetPeerPath *path; |
485 | struct GNUNET_CONTAINER_HeapNode *hn; | ||
486 | int i; | 472 | int i; |
487 | unsigned int skip; | 473 | unsigned int skip; |
488 | unsigned int total_len; | 474 | unsigned int total_len; |
@@ -498,8 +484,16 @@ GCPP_try_path_from_dht (const struct GNUNET_PeerIdentity *get_path, | |||
498 | const struct GNUNET_PeerIdentity *pid; | 484 | const struct GNUNET_PeerIdentity *pid; |
499 | 485 | ||
500 | pid = (off < get_path_length) | 486 | pid = (off < get_path_length) |
501 | ? &get_path[get_path_length - off] | 487 | ? &get_path[get_path_length - off - 1] |
502 | : &put_path[get_path_length + put_path_length - off]; | 488 | : &put_path[get_path_length + put_path_length - off - 1]; |
489 | /* Check that I am not in the path */ | ||
490 | if (0 == memcmp (&my_full_id, | ||
491 | pid, | ||
492 | sizeof (struct GNUNET_PeerIdentity))) | ||
493 | { | ||
494 | skip = off + 1; | ||
495 | continue; | ||
496 | } | ||
503 | cpath[off - skip] = GCP_get (pid, | 497 | cpath[off - skip] = GCP_get (pid, |
504 | GNUNET_YES); | 498 | GNUNET_YES); |
505 | /* Check that no peer is twice on the path */ | 499 | /* Check that no peer is twice on the path */ |
@@ -512,6 +506,12 @@ GCPP_try_path_from_dht (const struct GNUNET_PeerIdentity *get_path, | |||
512 | } | 506 | } |
513 | } | 507 | } |
514 | } | 508 | } |
509 | if (skip >= total_len) | ||
510 | { | ||
511 | LOG (GNUNET_ERROR_TYPE_DEBUG, | ||
512 | "Path discovered from DHT is one big cycle?\n"); | ||
513 | return; | ||
514 | } | ||
515 | total_len -= skip; | 515 | total_len -= skip; |
516 | 516 | ||
517 | /* First figure out if this path is a subset of an existing path, an | 517 | /* First figure out if this path is a subset of an existing path, an |
@@ -572,39 +572,17 @@ GCPP_try_path_from_dht (const struct GNUNET_PeerIdentity *get_path, | |||
572 | } | 572 | } |
573 | 573 | ||
574 | /* Finally, try to attach it */ | 574 | /* Finally, try to attach it */ |
575 | hn = NULL; | 575 | attach_path (path, 0); |
576 | for (i=total_len-1;i>=0;i--) | 576 | if (NULL == path->hn) |
577 | { | ||
578 | struct CadetPeerPathEntry *entry = path->entries[i]; | ||
579 | |||
580 | path->entries_length = i + 1; | ||
581 | recalculate_path_desirability (path); | ||
582 | hn = GCP_attach_path (cpath[i], | ||
583 | path, | ||
584 | (unsigned int) i, | ||
585 | GNUNET_NO); | ||
586 | if (NULL != hn) | ||
587 | break; | ||
588 | GCP_path_entry_remove (entry->peer, | ||
589 | entry, | ||
590 | i); | ||
591 | GNUNET_free (entry); | ||
592 | path->entries[i] = NULL; | ||
593 | } | ||
594 | if (NULL == hn) | ||
595 | { | 577 | { |
596 | /* None of the peers on the path care about it. */ | 578 | /* None of the peers on the path care about it. */ |
597 | LOG (GNUNET_ERROR_TYPE_DEBUG, | 579 | LOG (GNUNET_ERROR_TYPE_DEBUG, |
598 | "Path discovered from DHT is not interesting to us\n"); | 580 | "Path discovered from DHT is not interesting to us\n"); |
599 | GNUNET_free (path->entries); | 581 | GNUNET_assert (0 == path->entries_length); |
582 | GNUNET_assert (NULL == path->entries); | ||
600 | GNUNET_free (path); | 583 | GNUNET_free (path); |
601 | return; | 584 | return; |
602 | } | 585 | } |
603 | path->hn = hn; | ||
604 | /* Shrink path to actual useful length */ | ||
605 | GNUNET_array_grow (path->entries, | ||
606 | path->entries_length, | ||
607 | i + 1); | ||
608 | LOG (GNUNET_ERROR_TYPE_DEBUG, | 586 | LOG (GNUNET_ERROR_TYPE_DEBUG, |
609 | "Created new path %s based on information from DHT\n", | 587 | "Created new path %s based on information from DHT\n", |
610 | GCPP_2s (path)); | 588 | GCPP_2s (path)); |
diff --git a/src/cadet/gnunet-service-cadet_peer.c b/src/cadet/gnunet-service-cadet_peer.c index 71c7c67d0..da78a03c4 100644 --- a/src/cadet/gnunet-service-cadet_peer.c +++ b/src/cadet/gnunet-service-cadet_peer.c | |||
@@ -979,7 +979,7 @@ GCP_attach_path (struct CadetPeer *cp, | |||
979 | (desirability < root_desirability) ) | 979 | (desirability < root_desirability) ) |
980 | { | 980 | { |
981 | LOG (GNUNET_ERROR_TYPE_DEBUG, | 981 | LOG (GNUNET_ERROR_TYPE_DEBUG, |
982 | "Decided to not attach path %p to peer %s due to undesirability\n", | 982 | "Decided to not attach path %s to peer %s due to undesirability\n", |
983 | GCPP_2s (path), | 983 | GCPP_2s (path), |
984 | GCP_2s (cp)); | 984 | GCP_2s (cp)); |
985 | return NULL; | 985 | return NULL; |
diff --git a/src/cadet/test_cadet.c b/src/cadet/test_cadet.c index 72df2203c..6691a0573 100644 --- a/src/cadet/test_cadet.c +++ b/src/cadet/test_cadet.c | |||
@@ -960,16 +960,16 @@ main (int argc, char *argv[]) | |||
960 | char port_id[] = "test port"; | 960 | char port_id[] = "test port"; |
961 | struct GNUNET_GETOPT_CommandLineOption options[] = { | 961 | struct GNUNET_GETOPT_CommandLineOption options[] = { |
962 | GNUNET_GETOPT_option_relative_time ('t', | 962 | GNUNET_GETOPT_option_relative_time ('t', |
963 | "time", | 963 | "time", |
964 | "short_time", | 964 | "short_time", |
965 | gettext_noop ("set short timeout"), | 965 | gettext_noop ("set short timeout"), |
966 | &short_time), | 966 | &short_time), |
967 | 967 | ||
968 | GNUNET_GETOPT_option_uint ('m', | 968 | GNUNET_GETOPT_option_uint ('m', |
969 | "messages", | 969 | "messages", |
970 | "NUM_MESSAGES", | 970 | "NUM_MESSAGES", |
971 | gettext_noop ("set number of messages to send"), | 971 | gettext_noop ("set number of messages to send"), |
972 | &total_packets), | 972 | &total_packets), |
973 | 973 | ||
974 | GNUNET_GETOPT_OPTION_END | 974 | GNUNET_GETOPT_OPTION_END |
975 | }; | 975 | }; |
diff --git a/src/conversation/gnunet-conversation.c b/src/conversation/gnunet-conversation.c index 8f9ddec25..00ab65680 100644 --- a/src/conversation/gnunet-conversation.c +++ b/src/conversation/gnunet-conversation.c | |||
@@ -1091,7 +1091,7 @@ handle_command_string (char *message, | |||
1091 | strlen (commands[i].command)))) | 1091 | strlen (commands[i].command)))) |
1092 | i++; | 1092 | i++; |
1093 | ptr = &message[strlen (commands[i].command)]; | 1093 | ptr = &message[strlen (commands[i].command)]; |
1094 | while (isspace ((int) *ptr)) | 1094 | while (isspace ((unsigned char) *ptr)) |
1095 | ptr++; | 1095 | ptr++; |
1096 | if ('\0' == *ptr) | 1096 | if ('\0' == *ptr) |
1097 | ptr = NULL; | 1097 | ptr = NULL; |
diff --git a/src/core/gnunet-service-core_kx.c b/src/core/gnunet-service-core_kx.c index ae0ae508f..944d1e692 100644 --- a/src/core/gnunet-service-core_kx.c +++ b/src/core/gnunet-service-core_kx.c | |||
@@ -953,7 +953,7 @@ handle_ephemeral_key (void *cls, | |||
953 | kx->peer, | 953 | kx->peer, |
954 | sizeof (struct GNUNET_PeerIdentity))) | 954 | sizeof (struct GNUNET_PeerIdentity))) |
955 | { | 955 | { |
956 | GNUNET_log (GNUNET_ERROR_TYPE_ERROR, | 956 | GNUNET_log (GNUNET_ERROR_TYPE_WARNING, |
957 | "Received EPHEMERAL_KEY from %s, but expected %s\n", | 957 | "Received EPHEMERAL_KEY from %s, but expected %s\n", |
958 | GNUNET_i2s (&m->origin_identity), | 958 | GNUNET_i2s (&m->origin_identity), |
959 | GNUNET_i2s_full (kx->peer)); | 959 | GNUNET_i2s_full (kx->peer)); |
diff --git a/src/core/gnunet-service-core_sessions.c b/src/core/gnunet-service-core_sessions.c index 034f2e883..5d34b7c26 100644 --- a/src/core/gnunet-service-core_sessions.c +++ b/src/core/gnunet-service-core_sessions.c | |||
@@ -975,6 +975,7 @@ GSC_SESSIONS_set_typemap (const struct GNUNET_PeerIdentity *peer, | |||
975 | session = find_session (peer); | 975 | session = find_session (peer); |
976 | if (NULL == session) | 976 | if (NULL == session) |
977 | { | 977 | { |
978 | GSC_TYPEMAP_destroy (nmap); | ||
978 | GNUNET_break (0); | 979 | GNUNET_break (0); |
979 | return; | 980 | return; |
980 | } | 981 | } |
diff --git a/src/core/test_core_api_reliability.c b/src/core/test_core_api_reliability.c index 900c9f732..528093c99 100644 --- a/src/core/test_core_api_reliability.c +++ b/src/core/test_core_api_reliability.c | |||
@@ -381,6 +381,7 @@ process_hello (void *cls, | |||
381 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 381 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, |
382 | "Received (my) `%s' from transport service\n", "HELLO"); | 382 | "Received (my) `%s' from transport service\n", "HELLO"); |
383 | GNUNET_assert (message != NULL); | 383 | GNUNET_assert (message != NULL); |
384 | GNUNET_free_non_null (p->hello); | ||
384 | p->hello = GNUNET_copy_message (message); | 385 | p->hello = GNUNET_copy_message (message); |
385 | if ((p == &p1) && (NULL == p2.oh)) | 386 | if ((p == &p1) && (NULL == p2.oh)) |
386 | p2.oh = GNUNET_TRANSPORT_offer_hello (p2.cfg, | 387 | p2.oh = GNUNET_TRANSPORT_offer_hello (p2.cfg, |
@@ -518,6 +519,8 @@ main (int argc, | |||
518 | &ok); | 519 | &ok); |
519 | stop_arm (&p1); | 520 | stop_arm (&p1); |
520 | stop_arm (&p2); | 521 | stop_arm (&p2); |
522 | GNUNET_free_non_null (p1.hello); | ||
523 | GNUNET_free_non_null (p2.hello); | ||
521 | GNUNET_DISK_directory_remove ("/tmp/test-gnunet-core-peer-1"); | 524 | GNUNET_DISK_directory_remove ("/tmp/test-gnunet-core-peer-1"); |
522 | GNUNET_DISK_directory_remove ("/tmp/test-gnunet-core-peer-2"); | 525 | GNUNET_DISK_directory_remove ("/tmp/test-gnunet-core-peer-2"); |
523 | 526 | ||
diff --git a/src/datastore/datastore_api.c b/src/datastore/datastore_api.c index 31f7a997f..2ad864987 100644 --- a/src/datastore/datastore_api.c +++ b/src/datastore/datastore_api.c | |||
@@ -651,6 +651,46 @@ process_queue (struct GNUNET_DATASTORE_Handle *h) | |||
651 | } | 651 | } |
652 | 652 | ||
653 | 653 | ||
654 | /** | ||
655 | * Get the entry at the head of the message queue. | ||
656 | * | ||
657 | * @param h handle to the datastore | ||
658 | * @param response_type the expected response type | ||
659 | * @return the queue entry | ||
660 | */ | ||
661 | static struct GNUNET_DATASTORE_QueueEntry * | ||
662 | get_queue_head (struct GNUNET_DATASTORE_Handle *h, | ||
663 | uint16_t response_type) | ||
664 | { | ||
665 | struct GNUNET_DATASTORE_QueueEntry *qe; | ||
666 | |||
667 | if (h->skip_next_messages > 0) | ||
668 | { | ||
669 | h->skip_next_messages--; | ||
670 | process_queue (h); | ||
671 | return NULL; | ||
672 | } | ||
673 | qe = h->queue_head; | ||
674 | if (NULL == qe) | ||
675 | { | ||
676 | GNUNET_break (0); | ||
677 | do_disconnect (h); | ||
678 | return NULL; | ||
679 | } | ||
680 | if (NULL != qe->env) | ||
681 | { | ||
682 | GNUNET_break (0); | ||
683 | do_disconnect (h); | ||
684 | return NULL; | ||
685 | } | ||
686 | if (response_type != qe->response_type) | ||
687 | { | ||
688 | GNUNET_break (0); | ||
689 | do_disconnect (h); | ||
690 | return NULL; | ||
691 | } | ||
692 | return qe; | ||
693 | } | ||
654 | 694 | ||
655 | 695 | ||
656 | /** | 696 | /** |
@@ -702,30 +742,10 @@ handle_status (void *cls, | |||
702 | const char *emsg; | 742 | const char *emsg; |
703 | int32_t status = ntohl (sm->status); | 743 | int32_t status = ntohl (sm->status); |
704 | 744 | ||
705 | if (h->skip_next_messages > 0) | 745 | qe = get_queue_head (h, |
706 | { | 746 | GNUNET_MESSAGE_TYPE_DATASTORE_STATUS); |
707 | h->skip_next_messages--; | 747 | if (NULL == qe) |
708 | process_queue (h); | ||
709 | return; | ||
710 | } | ||
711 | if (NULL == (qe = h->queue_head)) | ||
712 | { | ||
713 | GNUNET_break (0); | ||
714 | do_disconnect (h); | ||
715 | return; | ||
716 | } | ||
717 | if (NULL != qe->env) | ||
718 | { | ||
719 | GNUNET_break (0); | ||
720 | do_disconnect (h); | ||
721 | return; | ||
722 | } | ||
723 | if (GNUNET_MESSAGE_TYPE_DATASTORE_STATUS != qe->response_type) | ||
724 | { | ||
725 | GNUNET_break (0); | ||
726 | do_disconnect (h); | ||
727 | return; | 748 | return; |
728 | } | ||
729 | rc = qe->qc.sc; | 749 | rc = qe->qc.sc; |
730 | free_queue_entry (qe); | 750 | free_queue_entry (qe); |
731 | if (ntohs (sm->header.size) > sizeof (struct StatusMessage)) | 751 | if (ntohs (sm->header.size) > sizeof (struct StatusMessage)) |
@@ -785,30 +805,10 @@ handle_data (void *cls, | |||
785 | struct GNUNET_DATASTORE_QueueEntry *qe; | 805 | struct GNUNET_DATASTORE_QueueEntry *qe; |
786 | struct ResultContext rc; | 806 | struct ResultContext rc; |
787 | 807 | ||
788 | if (h->skip_next_messages > 0) | 808 | qe = get_queue_head (h, |
789 | { | 809 | GNUNET_MESSAGE_TYPE_DATASTORE_DATA); |
790 | process_queue (h); | ||
791 | return; | ||
792 | } | ||
793 | qe = h->queue_head; | ||
794 | if (NULL == qe) | 810 | if (NULL == qe) |
795 | { | ||
796 | GNUNET_break (0); | ||
797 | do_disconnect (h); | ||
798 | return; | ||
799 | } | ||
800 | if (NULL != qe->env) | ||
801 | { | ||
802 | GNUNET_break (0); | ||
803 | do_disconnect (h); | ||
804 | return; | ||
805 | } | ||
806 | if (GNUNET_MESSAGE_TYPE_DATASTORE_DATA != qe->response_type) | ||
807 | { | ||
808 | GNUNET_break (0); | ||
809 | do_disconnect (h); | ||
810 | return; | 811 | return; |
811 | } | ||
812 | #if INSANE_STATISTICS | 812 | #if INSANE_STATISTICS |
813 | GNUNET_STATISTICS_update (h->stats, | 813 | GNUNET_STATISTICS_update (h->stats, |
814 | gettext_noop ("# Results received"), | 814 | gettext_noop ("# Results received"), |
@@ -854,31 +854,10 @@ handle_data_end (void *cls, | |||
854 | struct GNUNET_DATASTORE_QueueEntry *qe; | 854 | struct GNUNET_DATASTORE_QueueEntry *qe; |
855 | struct ResultContext rc; | 855 | struct ResultContext rc; |
856 | 856 | ||
857 | if (h->skip_next_messages > 0) | 857 | qe = get_queue_head (h, |
858 | { | 858 | GNUNET_MESSAGE_TYPE_DATASTORE_DATA); |
859 | h->skip_next_messages--; | ||
860 | process_queue (h); | ||
861 | return; | ||
862 | } | ||
863 | qe = h->queue_head; | ||
864 | if (NULL == qe) | 859 | if (NULL == qe) |
865 | { | ||
866 | GNUNET_break (0); | ||
867 | do_disconnect (h); | ||
868 | return; | 860 | return; |
869 | } | ||
870 | if (NULL != qe->env) | ||
871 | { | ||
872 | GNUNET_break (0); | ||
873 | do_disconnect (h); | ||
874 | return; | ||
875 | } | ||
876 | if (GNUNET_MESSAGE_TYPE_DATASTORE_DATA != qe->response_type) | ||
877 | { | ||
878 | GNUNET_break (0); | ||
879 | do_disconnect (h); | ||
880 | return; | ||
881 | } | ||
882 | rc = qe->qc.rc; | 861 | rc = qe->qc.rc; |
883 | free_queue_entry (qe); | 862 | free_queue_entry (qe); |
884 | LOG (GNUNET_ERROR_TYPE_DEBUG, | 863 | LOG (GNUNET_ERROR_TYPE_DEBUG, |
diff --git a/src/datastore/plugin_datastore_sqlite.c b/src/datastore/plugin_datastore_sqlite.c index 77b8409cd..cc56f5959 100644 --- a/src/datastore/plugin_datastore_sqlite.c +++ b/src/datastore/plugin_datastore_sqlite.c | |||
@@ -185,21 +185,22 @@ static void | |||
185 | create_indices (sqlite3 * dbh) | 185 | create_indices (sqlite3 * dbh) |
186 | { | 186 | { |
187 | /* create indices */ | 187 | /* create indices */ |
188 | if ((SQLITE_OK != | 188 | if (0 != |
189 | (SQLITE_OK != | ||
189 | sqlite3_exec (dbh, "CREATE INDEX IF NOT EXISTS idx_hash ON gn091 (hash)", | 190 | sqlite3_exec (dbh, "CREATE INDEX IF NOT EXISTS idx_hash ON gn091 (hash)", |
190 | NULL, NULL, NULL)) || | 191 | NULL, NULL, NULL)) + |
191 | (SQLITE_OK != | 192 | (SQLITE_OK != |
192 | sqlite3_exec (dbh, | 193 | sqlite3_exec (dbh, |
193 | "CREATE INDEX IF NOT EXISTS idx_anon_type ON gn091 (anonLevel ASC,type)", | 194 | "CREATE INDEX IF NOT EXISTS idx_anon_type ON gn091 (anonLevel ASC,type)", |
194 | NULL, NULL, NULL)) || | 195 | NULL, NULL, NULL)) + |
195 | (SQLITE_OK != | 196 | (SQLITE_OK != |
196 | sqlite3_exec (dbh, | 197 | sqlite3_exec (dbh, |
197 | "CREATE INDEX IF NOT EXISTS idx_expire ON gn091 (expire ASC)", | 198 | "CREATE INDEX IF NOT EXISTS idx_expire ON gn091 (expire ASC)", |
198 | NULL, NULL, NULL)) || | 199 | NULL, NULL, NULL)) + |
199 | (SQLITE_OK != | 200 | (SQLITE_OK != |
200 | sqlite3_exec (dbh, | 201 | sqlite3_exec (dbh, |
201 | "CREATE INDEX IF NOT EXISTS idx_repl_rvalue ON gn091 (repl,rvalue)", | 202 | "CREATE INDEX IF NOT EXISTS idx_repl_rvalue ON gn091 (repl,rvalue)", |
202 | NULL, NULL, NULL))) | 203 | NULL, NULL, NULL)) ) |
203 | GNUNET_log_from (GNUNET_ERROR_TYPE_ERROR, "sqlite", | 204 | GNUNET_log_from (GNUNET_ERROR_TYPE_ERROR, "sqlite", |
204 | "Failed to create indices: %s\n", sqlite3_errmsg (dbh)); | 205 | "Failed to create indices: %s\n", sqlite3_errmsg (dbh)); |
205 | } | 206 | } |
@@ -354,40 +355,24 @@ database_setup (const struct GNUNET_CONFIGURATION_Handle *cfg, | |||
354 | (SQLITE_OK != | 355 | (SQLITE_OK != |
355 | sq_prepare (plugin->dbh, | 356 | sq_prepare (plugin->dbh, |
356 | "SELECT " RESULT_COLUMNS " FROM gn091 " | 357 | "SELECT " RESULT_COLUMNS " FROM gn091 " |
357 | #if SQLITE_VERSION_NUMBER >= 3007000 | ||
358 | "INDEXED BY idx_repl_rvalue " | ||
359 | #endif | ||
360 | "WHERE repl=?2 AND " " (rvalue>=?1 OR " | 358 | "WHERE repl=?2 AND " " (rvalue>=?1 OR " |
361 | " NOT EXISTS (SELECT 1 FROM gn091 " | 359 | " NOT EXISTS (SELECT 1 FROM gn091 " |
362 | #if SQLITE_VERSION_NUMBER >= 3007000 | ||
363 | "INDEXED BY idx_repl_rvalue " | ||
364 | #endif | ||
365 | "WHERE repl=?2 AND rvalue>=?1 LIMIT 1) ) " | 360 | "WHERE repl=?2 AND rvalue>=?1 LIMIT 1) ) " |
366 | "ORDER BY rvalue ASC LIMIT 1", | 361 | "ORDER BY rvalue ASC LIMIT 1", |
367 | &plugin->selRepl)) || | 362 | &plugin->selRepl)) || |
368 | (SQLITE_OK != | 363 | (SQLITE_OK != |
369 | sq_prepare (plugin->dbh, | 364 | sq_prepare (plugin->dbh, |
370 | "SELECT MAX(repl) FROM gn091" | 365 | "SELECT MAX(repl) FROM gn091", |
371 | #if SQLITE_VERSION_NUMBER >= 3007000 | ||
372 | " INDEXED BY idx_repl_rvalue" | ||
373 | #endif | ||
374 | "", | ||
375 | &plugin->maxRepl)) || | 366 | &plugin->maxRepl)) || |
376 | (SQLITE_OK != | 367 | (SQLITE_OK != |
377 | sq_prepare (plugin->dbh, | 368 | sq_prepare (plugin->dbh, |
378 | "SELECT " RESULT_COLUMNS " FROM gn091 " | 369 | "SELECT " RESULT_COLUMNS " FROM gn091 " |
379 | #if SQLITE_VERSION_NUMBER >= 3007000 | ||
380 | "INDEXED BY idx_expire " | ||
381 | #endif | ||
382 | "WHERE NOT EXISTS (SELECT 1 FROM gn091 WHERE expire < ?1 LIMIT 1) OR (expire < ?1) " | 370 | "WHERE NOT EXISTS (SELECT 1 FROM gn091 WHERE expire < ?1 LIMIT 1) OR (expire < ?1) " |
383 | "ORDER BY expire ASC LIMIT 1", | 371 | "ORDER BY expire ASC LIMIT 1", |
384 | &plugin->selExpi)) || | 372 | &plugin->selExpi)) || |
385 | (SQLITE_OK != | 373 | (SQLITE_OK != |
386 | sq_prepare (plugin->dbh, | 374 | sq_prepare (plugin->dbh, |
387 | "SELECT " RESULT_COLUMNS " FROM gn091 " | 375 | "SELECT " RESULT_COLUMNS " FROM gn091 " |
388 | #if SQLITE_VERSION_NUMBER >= 3007000 | ||
389 | "INDEXED BY idx_anon_type " | ||
390 | #endif | ||
391 | "WHERE _ROWID_ >= ? AND " | 376 | "WHERE _ROWID_ >= ? AND " |
392 | "anonLevel = 0 AND " | 377 | "anonLevel = 0 AND " |
393 | "type = ? " | 378 | "type = ? " |
diff --git a/src/dht/Makefile.am b/src/dht/Makefile.am index 00ce0e934..4a78ea4c7 100644 --- a/src/dht/Makefile.am +++ b/src/dht/Makefile.am | |||
@@ -213,7 +213,8 @@ endif | |||
213 | 213 | ||
214 | do_subst = $(SED) -e 's,[@]PYTHON[@],$(PYTHON),g' -e 's,[@]bindir[@],$(bindir),g' | 214 | do_subst = $(SED) -e 's,[@]PYTHON[@],$(PYTHON),g' -e 's,[@]bindir[@],$(bindir),g' |
215 | 215 | ||
216 | %.py: %.py.in Makefile | 216 | SUFFIXES = .py.in .py |
217 | .py.in.py: | ||
217 | $(do_subst) < $(srcdir)/$< > $@ | 218 | $(do_subst) < $(srcdir)/$< > $@ |
218 | chmod +x $@ | 219 | chmod +x $@ |
219 | 220 | ||
diff --git a/src/dns/dnsparser.c b/src/dns/dnsparser.c index 36b4c36f1..30d9245ff 100644 --- a/src/dns/dnsparser.c +++ b/src/dns/dnsparser.c | |||
@@ -1278,8 +1278,8 @@ GNUNET_DNSPARSER_hex_to_bin (const char *hex, | |||
1278 | in[2] = '\0'; | 1278 | in[2] = '\0'; |
1279 | for (off = 0; off < data_size; off++) | 1279 | for (off = 0; off < data_size; off++) |
1280 | { | 1280 | { |
1281 | in[0] = tolower ((int) hex[off * 2]); | 1281 | in[0] = tolower ((unsigned char) hex[off * 2]); |
1282 | in[1] = tolower ((int) hex[off * 2 + 1]); | 1282 | in[1] = tolower ((unsigned char) hex[off * 2 + 1]); |
1283 | if (1 != sscanf (in, "%x", &h)) | 1283 | if (1 != sscanf (in, "%x", &h)) |
1284 | return off; | 1284 | return off; |
1285 | idata[off] = (uint8_t) h; | 1285 | idata[off] = (uint8_t) h; |
diff --git a/src/fs/fs_misc.c b/src/fs/fs_misc.c index bcb8620cf..b26de431c 100644 --- a/src/fs/fs_misc.c +++ b/src/fs/fs_misc.c | |||
@@ -1,6 +1,6 @@ | |||
1 | /* | 1 | /* |
2 | This file is part of GNUnet. | 2 | This file is part of GNUnet. |
3 | Copyright (C) 2010, 2011 GNUnet e.V. | 3 | Copyright (C) 2010, 2011, 2017 GNUnet e.V. |
4 | 4 | ||
5 | GNUnet is free software; you can redistribute it and/or modify | 5 | GNUnet is free software; you can redistribute it and/or modify |
6 | it under the terms of the GNU General Public License as published | 6 | it under the terms of the GNU General Public License as published |
@@ -43,6 +43,8 @@ GNUNET_FS_meta_data_suggest_filename (const struct GNUNET_CONTAINER_MetaData | |||
43 | {"application/gnunet-directory", ".gnd"}, | 43 | {"application/gnunet-directory", ".gnd"}, |
44 | {"application/java", ".class"}, | 44 | {"application/java", ".class"}, |
45 | {"application/msword", ".doc"}, | 45 | {"application/msword", ".doc"}, |
46 | {"application/nar", ".nar"}, | ||
47 | {"application/narinfo", ".narinfo"}, | ||
46 | {"application/ogg", ".ogg"}, | 48 | {"application/ogg", ".ogg"}, |
47 | {"application/pdf", ".pdf"}, | 49 | {"application/pdf", ".pdf"}, |
48 | {"application/pgp-keys", ".key"}, | 50 | {"application/pgp-keys", ".key"}, |
@@ -53,8 +55,8 @@ GNUNET_FS_meta_data_suggest_filename (const struct GNUNET_CONTAINER_MetaData | |||
53 | {"application/xml", ".xml"}, | 55 | {"application/xml", ".xml"}, |
54 | {"application/x-debian-package", ".deb"}, | 56 | {"application/x-debian-package", ".deb"}, |
55 | {"application/x-dvi", ".dvi"}, | 57 | {"application/x-dvi", ".dvi"}, |
56 | {"applixation/x-flac", ".flac"}, | 58 | {"application/x-flac", ".flac"}, |
57 | {"applixation/x-gzip", ".gz"}, | 59 | {"application/x-gzip", ".gz"}, |
58 | {"application/x-java-archive", ".jar"}, | 60 | {"application/x-java-archive", ".jar"}, |
59 | {"application/x-java-vm", ".class"}, | 61 | {"application/x-java-vm", ".class"}, |
60 | {"application/x-python-code", ".pyc"}, | 62 | {"application/x-python-code", ".pyc"}, |
diff --git a/src/fs/fs_publish_ublock.c b/src/fs/fs_publish_ublock.c index e21443ccb..189a6909a 100644 --- a/src/fs/fs_publish_ublock.c +++ b/src/fs/fs_publish_ublock.c | |||
@@ -301,6 +301,7 @@ GNUNET_FS_publish_ublock_ (struct GNUNET_FS_Handle *h, | |||
301 | uc->task = GNUNET_SCHEDULER_add_now (&run_cont, | 301 | uc->task = GNUNET_SCHEDULER_add_now (&run_cont, |
302 | uc); | 302 | uc); |
303 | } | 303 | } |
304 | GNUNET_free (ub_enc); | ||
304 | return uc; | 305 | return uc; |
305 | } | 306 | } |
306 | 307 | ||
diff --git a/src/fs/plugin_block_fs.c b/src/fs/plugin_block_fs.c index 902519f15..c762835ce 100644 --- a/src/fs/plugin_block_fs.c +++ b/src/fs/plugin_block_fs.c | |||
@@ -71,17 +71,22 @@ block_plugin_fs_create_group (void *cls, | |||
71 | return NULL; | 71 | return NULL; |
72 | case GNUNET_BLOCK_TYPE_FS_UBLOCK: | 72 | case GNUNET_BLOCK_TYPE_FS_UBLOCK: |
73 | guard = va_arg (va, const char *); | 73 | guard = va_arg (va, const char *); |
74 | if (0 != strcmp (guard, | 74 | if (0 == strcmp (guard, |
75 | "seen-set-size")) | 75 | "seen-set-size")) |
76 | { | 76 | { |
77 | /* va-args invalid! bad bug, complain! */ | 77 | size = GNUNET_BLOCK_GROUP_compute_bloomfilter_size (va_arg (va, unsigned int), |
78 | GNUNET_break (0); | 78 | BLOOMFILTER_K); |
79 | size = 8; | 79 | } |
80 | else if (0 == strcmp (guard, | ||
81 | "filter-size")) | ||
82 | { | ||
83 | size = va_arg (va, unsigned int); | ||
80 | } | 84 | } |
81 | else | 85 | else |
82 | { | 86 | { |
83 | size = GNUNET_BLOCK_GROUP_compute_bloomfilter_size (va_arg (va, unsigned int), | 87 | /* va-args invalid! bad bug, complain! */ |
84 | BLOOMFILTER_K); | 88 | GNUNET_break (0); |
89 | size = 8; | ||
85 | } | 90 | } |
86 | if (0 == size) | 91 | if (0 == size) |
87 | size = raw_data_size; /* not for us to determine, use what we got! */ | 92 | size = raw_data_size; /* not for us to determine, use what we got! */ |
diff --git a/src/gns/gnunet-gns-proxy.c b/src/gns/gnunet-gns-proxy.c index 2a6de1c30..3af571eba 100644 --- a/src/gns/gnunet-gns-proxy.c +++ b/src/gns/gnunet-gns-proxy.c | |||
@@ -786,6 +786,8 @@ cleanup_s5r (struct Socks5Request *s5r) | |||
786 | 786 | ||
787 | /* ************************* HTTP handling with cURL *********************** */ | 787 | /* ************************* HTTP handling with cURL *********************** */ |
788 | 788 | ||
789 | static void | ||
790 | curl_download_prepare (); | ||
789 | 791 | ||
790 | /** | 792 | /** |
791 | * Callback for MHD response generation. This function is called from | 793 | * Callback for MHD response generation. This function is called from |
@@ -824,6 +826,11 @@ mhd_content_cb (void *cls, | |||
824 | { | 826 | { |
825 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 827 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, |
826 | "Pausing MHD download, no data available\n"); | 828 | "Pausing MHD download, no data available\n"); |
829 | if (NULL != s5r->curl) | ||
830 | { | ||
831 | curl_easy_pause (s5r->curl, CURLPAUSE_CONT); | ||
832 | curl_download_prepare (); | ||
833 | } | ||
827 | return 0; /* more data later */ | 834 | return 0; /* more data later */ |
828 | } | 835 | } |
829 | if ( (0 == bytes_to_copy) && | 836 | if ( (0 == bytes_to_copy) && |
@@ -833,6 +840,8 @@ mhd_content_cb (void *cls, | |||
833 | "Completed MHD download\n"); | 840 | "Completed MHD download\n"); |
834 | return MHD_CONTENT_READER_END_OF_STREAM; | 841 | return MHD_CONTENT_READER_END_OF_STREAM; |
835 | } | 842 | } |
843 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
844 | "Writing %lu/%lu bytes\n", bytes_to_copy, s5r->io_len); | ||
836 | GNUNET_memcpy (buf, s5r->io_buf, bytes_to_copy); | 845 | GNUNET_memcpy (buf, s5r->io_buf, bytes_to_copy); |
837 | memmove (s5r->io_buf, | 846 | memmove (s5r->io_buf, |
838 | &s5r->io_buf[bytes_to_copy], | 847 | &s5r->io_buf[bytes_to_copy], |
@@ -865,7 +874,7 @@ check_ssl_certificate (struct Socks5Request *s5r) | |||
865 | const char *name; | 874 | const char *name; |
866 | 875 | ||
867 | s5r->ssl_checked = GNUNET_YES; | 876 | s5r->ssl_checked = GNUNET_YES; |
868 | GNUNET_log (GNUNET_ERROR_TYPE_ERROR, | 877 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, |
869 | "Checking SSL certificate\n"); | 878 | "Checking SSL certificate\n"); |
870 | if (CURLE_OK != | 879 | if (CURLE_OK != |
871 | curl_easy_getinfo (s5r->curl, | 880 | curl_easy_getinfo (s5r->curl, |
@@ -1249,7 +1258,8 @@ curl_download_cb (void *ptr, size_t size, size_t nmemb, void* ctx) | |||
1249 | if (sizeof (s5r->io_buf) - s5r->io_len < total) | 1258 | if (sizeof (s5r->io_buf) - s5r->io_len < total) |
1250 | { | 1259 | { |
1251 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 1260 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, |
1252 | "Pausing CURL download, not enough space\n"); | 1261 | "Pausing CURL download, not enough space %lu %lu %lu\n", sizeof (s5r->io_buf), |
1262 | s5r->io_len, total); | ||
1253 | return CURL_WRITEFUNC_PAUSE; /* not enough space */ | 1263 | return CURL_WRITEFUNC_PAUSE; /* not enough space */ |
1254 | } | 1264 | } |
1255 | GNUNET_memcpy (&s5r->io_buf[s5r->io_len], | 1265 | GNUNET_memcpy (&s5r->io_buf[s5r->io_len], |
@@ -1833,7 +1843,7 @@ mhd_completed_cb (void *cls, | |||
1833 | for (header = s5r->header_head; header != NULL; header = s5r->header_head) | 1843 | for (header = s5r->header_head; header != NULL; header = s5r->header_head) |
1834 | { | 1844 | { |
1835 | GNUNET_CONTAINER_DLL_remove (s5r->header_head, | 1845 | GNUNET_CONTAINER_DLL_remove (s5r->header_head, |
1836 | s5r->header_head, | 1846 | s5r->header_tail, |
1837 | header); | 1847 | header); |
1838 | GNUNET_free (header->type); | 1848 | GNUNET_free (header->type); |
1839 | GNUNET_free (header->value); | 1849 | GNUNET_free (header->value); |
@@ -2414,6 +2424,8 @@ do_write (void *cls) | |||
2414 | if (len <= 0) | 2424 | if (len <= 0) |
2415 | { | 2425 | { |
2416 | /* write error: connection closed, shutdown, etc.; just clean up */ | 2426 | /* write error: connection closed, shutdown, etc.; just clean up */ |
2427 | GNUNET_log (GNUNET_ERROR_TYPE_ERROR, | ||
2428 | "Write Error\n"); | ||
2417 | cleanup_s5r (s5r); | 2429 | cleanup_s5r (s5r); |
2418 | return; | 2430 | return; |
2419 | } | 2431 | } |
diff --git a/src/hello/hello.c b/src/hello/hello.c index 27580275f..690a0961a 100644 --- a/src/hello/hello.c +++ b/src/hello/hello.c | |||
@@ -271,7 +271,10 @@ GNUNET_HELLO_iterate_addresses (const struct GNUNET_HELLO_Message *msg, | |||
271 | msize = GNUNET_HELLO_size (msg); | 271 | msize = GNUNET_HELLO_size (msg); |
272 | if ((msize < sizeof (struct GNUNET_HELLO_Message)) || | 272 | if ((msize < sizeof (struct GNUNET_HELLO_Message)) || |
273 | (ntohs (msg->header.type) != GNUNET_MESSAGE_TYPE_HELLO)) | 273 | (ntohs (msg->header.type) != GNUNET_MESSAGE_TYPE_HELLO)) |
274 | { | ||
275 | GNUNET_break_op (0); | ||
274 | return NULL; | 276 | return NULL; |
277 | } | ||
275 | ret = NULL; | 278 | ret = NULL; |
276 | if (return_modified) | 279 | if (return_modified) |
277 | { | 280 | { |
@@ -285,6 +288,10 @@ GNUNET_HELLO_iterate_addresses (const struct GNUNET_HELLO_Message *msg, | |||
285 | wpos = 0; | 288 | wpos = 0; |
286 | woff = (NULL != ret) ? (char *) &ret[1] : NULL; | 289 | woff = (NULL != ret) ? (char *) &ret[1] : NULL; |
287 | address.peer.public_key = msg->publicKey; | 290 | address.peer.public_key = msg->publicKey; |
291 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
292 | "HELLO has %u bytes of address data\n", | ||
293 | (unsigned int) insize); | ||
294 | |||
288 | while (insize > 0) | 295 | while (insize > 0) |
289 | { | 296 | { |
290 | esize = get_hello_address_size (inptr, | 297 | esize = get_hello_address_size (inptr, |
diff --git a/src/include/gnunet_crypto_lib.h b/src/include/gnunet_crypto_lib.h index 07cade0e3..e886a561c 100644 --- a/src/include/gnunet_crypto_lib.h +++ b/src/include/gnunet_crypto_lib.h | |||
@@ -1110,6 +1110,16 @@ GNUNET_CRYPTO_ecdsa_public_key_to_string (const struct GNUNET_CRYPTO_EcdsaPublic | |||
1110 | 1110 | ||
1111 | 1111 | ||
1112 | /** | 1112 | /** |
1113 | * Convert a private key to a string. | ||
1114 | * | ||
1115 | * @param priv key to convert | ||
1116 | * @return string representing @a pub | ||
1117 | */ | ||
1118 | char * | ||
1119 | GNUNET_CRYPTO_eddsa_private_key_to_string (const struct GNUNET_CRYPTO_EddsaPrivateKey *priv); | ||
1120 | |||
1121 | |||
1122 | /** | ||
1113 | * Convert a public key to a string. | 1123 | * Convert a public key to a string. |
1114 | * | 1124 | * |
1115 | * @param pub key to convert | 1125 | * @param pub key to convert |
@@ -2016,13 +2026,14 @@ GNUNET_CRYPTO_rsa_public_key_cmp (struct GNUNET_CRYPTO_RsaPublicKey *p1, | |||
2016 | * @param pkey the public key of the signer | 2026 | * @param pkey the public key of the signer |
2017 | * @param[out] buf set to a buffer with the blinded message to be signed | 2027 | * @param[out] buf set to a buffer with the blinded message to be signed |
2018 | * @param[out] buf_size number of bytes stored in @a buf | 2028 | * @param[out] buf_size number of bytes stored in @a buf |
2019 | * @return GNUNET_YES if successful, GNUNET_NO if RSA key is malicious | 2029 | * @return #GNUNET_YES if successful, #GNUNET_NO if RSA key is malicious |
2020 | */ | 2030 | */ |
2021 | int | 2031 | int |
2022 | GNUNET_CRYPTO_rsa_blind (const struct GNUNET_HashCode *hash, | 2032 | GNUNET_CRYPTO_rsa_blind (const struct GNUNET_HashCode *hash, |
2023 | const struct GNUNET_CRYPTO_RsaBlindingKeySecret *bks, | 2033 | const struct GNUNET_CRYPTO_RsaBlindingKeySecret *bks, |
2024 | struct GNUNET_CRYPTO_RsaPublicKey *pkey, | 2034 | struct GNUNET_CRYPTO_RsaPublicKey *pkey, |
2025 | char **buf, size_t *buf_size); | 2035 | char **buf, |
2036 | size_t *buf_size); | ||
2026 | 2037 | ||
2027 | 2038 | ||
2028 | /** | 2039 | /** |
@@ -2035,7 +2046,8 @@ GNUNET_CRYPTO_rsa_blind (const struct GNUNET_HashCode *hash, | |||
2035 | */ | 2046 | */ |
2036 | struct GNUNET_CRYPTO_RsaSignature * | 2047 | struct GNUNET_CRYPTO_RsaSignature * |
2037 | GNUNET_CRYPTO_rsa_sign_blinded (const struct GNUNET_CRYPTO_RsaPrivateKey *key, | 2048 | GNUNET_CRYPTO_rsa_sign_blinded (const struct GNUNET_CRYPTO_RsaPrivateKey *key, |
2038 | const void *msg, size_t msg_len); | 2049 | const void *msg, |
2050 | size_t msg_len); | ||
2039 | 2051 | ||
2040 | 2052 | ||
2041 | /** | 2053 | /** |
@@ -2105,7 +2117,7 @@ GNUNET_CRYPTO_rsa_signature_dup (const struct GNUNET_CRYPTO_RsaSignature *sig); | |||
2105 | * @return unblinded signature on success, NULL if RSA key is bad or malicious. | 2117 | * @return unblinded signature on success, NULL if RSA key is bad or malicious. |
2106 | */ | 2118 | */ |
2107 | struct GNUNET_CRYPTO_RsaSignature * | 2119 | struct GNUNET_CRYPTO_RsaSignature * |
2108 | GNUNET_CRYPTO_rsa_unblind (struct GNUNET_CRYPTO_RsaSignature *sig, | 2120 | GNUNET_CRYPTO_rsa_unblind (const struct GNUNET_CRYPTO_RsaSignature *sig, |
2109 | const struct GNUNET_CRYPTO_RsaBlindingKeySecret *bks, | 2121 | const struct GNUNET_CRYPTO_RsaBlindingKeySecret *bks, |
2110 | struct GNUNET_CRYPTO_RsaPublicKey *pkey); | 2122 | struct GNUNET_CRYPTO_RsaPublicKey *pkey); |
2111 | 2123 | ||
diff --git a/src/include/gnunet_getopt_lib.h b/src/include/gnunet_getopt_lib.h index f707bb091..e38925f14 100644 --- a/src/include/gnunet_getopt_lib.h +++ b/src/include/gnunet_getopt_lib.h | |||
@@ -230,11 +230,11 @@ GNUNET_GETOPT_option_filename (char shortName, | |||
230 | */ | 230 | */ |
231 | struct GNUNET_GETOPT_CommandLineOption | 231 | struct GNUNET_GETOPT_CommandLineOption |
232 | GNUNET_GETOPT_option_base32_fixed_size (char shortName, | 232 | GNUNET_GETOPT_option_base32_fixed_size (char shortName, |
233 | const char *name, | 233 | const char *name, |
234 | const char *argumentHelp, | 234 | const char *argumentHelp, |
235 | const char *description, | 235 | const char *description, |
236 | void *val, | 236 | void *val, |
237 | size_t val_size); | 237 | size_t val_size); |
238 | 238 | ||
239 | 239 | ||
240 | /** | 240 | /** |
@@ -264,9 +264,9 @@ GNUNET_GETOPT_option_base32_fixed_size (char shortName, | |||
264 | */ | 264 | */ |
265 | struct GNUNET_GETOPT_CommandLineOption | 265 | struct GNUNET_GETOPT_CommandLineOption |
266 | GNUNET_GETOPT_option_flag (char shortName, | 266 | GNUNET_GETOPT_option_flag (char shortName, |
267 | const char *name, | 267 | const char *name, |
268 | const char *description, | 268 | const char *description, |
269 | int *val); | 269 | int *val); |
270 | 270 | ||
271 | 271 | ||
272 | /** | 272 | /** |
@@ -280,10 +280,10 @@ GNUNET_GETOPT_option_flag (char shortName, | |||
280 | */ | 280 | */ |
281 | struct GNUNET_GETOPT_CommandLineOption | 281 | struct GNUNET_GETOPT_CommandLineOption |
282 | GNUNET_GETOPT_option_uint (char shortName, | 282 | GNUNET_GETOPT_option_uint (char shortName, |
283 | const char *name, | 283 | const char *name, |
284 | const char *argumentHelp, | 284 | const char *argumentHelp, |
285 | const char *description, | 285 | const char *description, |
286 | unsigned int *val); | 286 | unsigned int *val); |
287 | 287 | ||
288 | 288 | ||
289 | /** | 289 | /** |
@@ -297,10 +297,10 @@ GNUNET_GETOPT_option_uint (char shortName, | |||
297 | */ | 297 | */ |
298 | struct GNUNET_GETOPT_CommandLineOption | 298 | struct GNUNET_GETOPT_CommandLineOption |
299 | GNUNET_GETOPT_option_ulong (char shortName, | 299 | GNUNET_GETOPT_option_ulong (char shortName, |
300 | const char *name, | 300 | const char *name, |
301 | const char *argumentHelp, | 301 | const char *argumentHelp, |
302 | const char *description, | 302 | const char *description, |
303 | unsigned long long *val); | 303 | unsigned long long *val); |
304 | 304 | ||
305 | 305 | ||
306 | /** | 306 | /** |
@@ -315,10 +315,10 @@ GNUNET_GETOPT_option_ulong (char shortName, | |||
315 | */ | 315 | */ |
316 | struct GNUNET_GETOPT_CommandLineOption | 316 | struct GNUNET_GETOPT_CommandLineOption |
317 | GNUNET_GETOPT_option_relative_time (char shortName, | 317 | GNUNET_GETOPT_option_relative_time (char shortName, |
318 | const char *name, | 318 | const char *name, |
319 | const char *argumentHelp, | 319 | const char *argumentHelp, |
320 | const char *description, | 320 | const char *description, |
321 | struct GNUNET_TIME_Relative *val); | 321 | struct GNUNET_TIME_Relative *val); |
322 | 322 | ||
323 | 323 | ||
324 | /** | 324 | /** |
@@ -333,10 +333,10 @@ GNUNET_GETOPT_option_relative_time (char shortName, | |||
333 | */ | 333 | */ |
334 | struct GNUNET_GETOPT_CommandLineOption | 334 | struct GNUNET_GETOPT_CommandLineOption |
335 | GNUNET_GETOPT_option_absolute_time (char shortName, | 335 | GNUNET_GETOPT_option_absolute_time (char shortName, |
336 | const char *name, | 336 | const char *name, |
337 | const char *argumentHelp, | 337 | const char *argumentHelp, |
338 | const char *description, | 338 | const char *description, |
339 | struct GNUNET_TIME_Absolute *val); | 339 | struct GNUNET_TIME_Absolute *val); |
340 | 340 | ||
341 | 341 | ||
342 | /** | 342 | /** |
@@ -350,9 +350,9 @@ GNUNET_GETOPT_option_absolute_time (char shortName, | |||
350 | */ | 350 | */ |
351 | struct GNUNET_GETOPT_CommandLineOption | 351 | struct GNUNET_GETOPT_CommandLineOption |
352 | GNUNET_GETOPT_option_increment_uint (char shortName, | 352 | GNUNET_GETOPT_option_increment_uint (char shortName, |
353 | const char *name, | 353 | const char *name, |
354 | const char *description, | 354 | const char *description, |
355 | unsigned int *val); | 355 | unsigned int *val); |
356 | 356 | ||
357 | 357 | ||
358 | /** | 358 | /** |
diff --git a/src/include/gnunet_json_lib.h b/src/include/gnunet_json_lib.h index f2682bea7..c12badcd9 100644 --- a/src/include/gnunet_json_lib.h +++ b/src/include/gnunet_json_lib.h | |||
@@ -343,6 +343,16 @@ GNUNET_JSON_from_time_abs (struct GNUNET_TIME_Absolute stamp); | |||
343 | 343 | ||
344 | 344 | ||
345 | /** | 345 | /** |
346 | * Convert absolute timestamp to a json string. | ||
347 | * | ||
348 | * @param stamp the time stamp | ||
349 | * @return a json string with the timestamp in @a stamp | ||
350 | */ | ||
351 | json_t * | ||
352 | GNUNET_JSON_from_time_abs_nbo (struct GNUNET_TIME_AbsoluteNBO stamp); | ||
353 | |||
354 | |||
355 | /** | ||
346 | * Convert relative timestamp to a json string. | 356 | * Convert relative timestamp to a json string. |
347 | * | 357 | * |
348 | * @param stamp the time stamp | 358 | * @param stamp the time stamp |
diff --git a/src/include/gnunet_scheduler_lib.h b/src/include/gnunet_scheduler_lib.h index 875f5043a..a855ab8ab 100644 --- a/src/include/gnunet_scheduler_lib.h +++ b/src/include/gnunet_scheduler_lib.h | |||
@@ -400,6 +400,22 @@ void | |||
400 | GNUNET_SCHEDULER_run (GNUNET_SCHEDULER_TaskCallback task, | 400 | GNUNET_SCHEDULER_run (GNUNET_SCHEDULER_TaskCallback task, |
401 | void *task_cls); | 401 | void *task_cls); |
402 | 402 | ||
403 | /** | ||
404 | * Initialize and run scheduler. This function will return when all | ||
405 | * tasks have completed. When @ install_signals is GNUNET_YES, then | ||
406 | * this function behaves in the same was as GNUNET_SCHEDULER_run does. | ||
407 | * If @ install_signals is GNUNET_NO then no signal handlers are | ||
408 | * installed. | ||
409 | * | ||
410 | * @param install_signals whether to install signals (GNUNET_YES/NO) | ||
411 | * @param task task to run first (and immediately) | ||
412 | * @param task_cls closure of @a task | ||
413 | */ | ||
414 | void | ||
415 | GNUNET_SCHEDULER_run_with_optional_signals (int install_signals, | ||
416 | GNUNET_SCHEDULER_TaskCallback task, | ||
417 | void *task_cls); | ||
418 | |||
403 | 419 | ||
404 | /** | 420 | /** |
405 | * Request the shutdown of a scheduler. Marks all tasks | 421 | * Request the shutdown of a scheduler. Marks all tasks |
diff --git a/src/include/gnunet_set_service.h b/src/include/gnunet_set_service.h index 6e822d82e..a2999aebc 100644 --- a/src/include/gnunet_set_service.h +++ b/src/include/gnunet_set_service.h | |||
@@ -452,6 +452,8 @@ GNUNET_SET_listen (const struct GNUNET_CONFIGURATION_Handle *cfg, | |||
452 | /** | 452 | /** |
453 | * Cancel the given listen operation. After calling cancel, the | 453 | * Cancel the given listen operation. After calling cancel, the |
454 | * listen callback for this listen handle will not be called again. | 454 | * listen callback for this listen handle will not be called again. |
455 | * Note that cancelling a listen operation will automatically reject | ||
456 | * all operations that have not yet been accepted. | ||
455 | * | 457 | * |
456 | * @param lh handle for the listen operation | 458 | * @param lh handle for the listen operation |
457 | */ | 459 | */ |
@@ -556,7 +558,8 @@ GNUNET_SET_element_dup (const struct GNUNET_SET_Element *element); | |||
556 | * should be stored | 558 | * should be stored |
557 | */ | 559 | */ |
558 | void | 560 | void |
559 | GNUNET_SET_element_hash (const struct GNUNET_SET_Element *element, struct GNUNET_HashCode *ret_hash); | 561 | GNUNET_SET_element_hash (const struct GNUNET_SET_Element *element, |
562 | struct GNUNET_HashCode *ret_hash); | ||
560 | 563 | ||
561 | 564 | ||
562 | #if 0 /* keep Emacsens' auto-indent happy */ | 565 | #if 0 /* keep Emacsens' auto-indent happy */ |
diff --git a/src/integration-tests/Makefile.am b/src/integration-tests/Makefile.am index 6fff0b407..368980064 100644 --- a/src/integration-tests/Makefile.am +++ b/src/integration-tests/Makefile.am | |||
@@ -42,7 +42,8 @@ endif | |||
42 | 42 | ||
43 | do_subst = $(SED) -e 's,[@]PYTHON[@],$(PYTHON),g' | 43 | do_subst = $(SED) -e 's,[@]PYTHON[@],$(PYTHON),g' |
44 | 44 | ||
45 | %.py: %.py.in Makefile | 45 | SUFFIXES = .py.in .py |
46 | .py.in.py: | ||
46 | $(do_subst) < $(srcdir)/$< > $@ | 47 | $(do_subst) < $(srcdir)/$< > $@ |
47 | chmod +x $@ | 48 | chmod +x $@ |
48 | 49 | ||
diff --git a/src/json/json_generator.c b/src/json/json_generator.c index e660e10c5..98f7163bc 100644 --- a/src/json/json_generator.c +++ b/src/json/json_generator.c | |||
@@ -73,6 +73,19 @@ GNUNET_JSON_from_time_abs (struct GNUNET_TIME_Absolute stamp) | |||
73 | 73 | ||
74 | 74 | ||
75 | /** | 75 | /** |
76 | * Convert absolute timestamp to a json string. | ||
77 | * | ||
78 | * @param stamp the time stamp | ||
79 | * @return a json string with the timestamp in @a stamp | ||
80 | */ | ||
81 | json_t * | ||
82 | GNUNET_JSON_from_time_abs_nbo (struct GNUNET_TIME_AbsoluteNBO stamp) | ||
83 | { | ||
84 | return GNUNET_JSON_from_time_abs (GNUNET_TIME_absolute_ntoh (stamp)); | ||
85 | } | ||
86 | |||
87 | |||
88 | /** | ||
76 | * Convert relative timestamp to a json string. | 89 | * Convert relative timestamp to a json string. |
77 | * | 90 | * |
78 | * @param stamp the time stamp | 91 | * @param stamp the time stamp |
diff --git a/src/multicast/.gitignore b/src/multicast/.gitignore index 43752ec4b..a97844e81 100644 --- a/src/multicast/.gitignore +++ b/src/multicast/.gitignore | |||
@@ -3,3 +3,5 @@ gnunet-multicast | |||
3 | test_multicast | 3 | test_multicast |
4 | test_multicast_multipeer | 4 | test_multicast_multipeer |
5 | test_multicast_2peers | 5 | test_multicast_2peers |
6 | test_multicast_multipeer_line | ||
7 | test_multicast_multipeer_star | ||
diff --git a/src/peerinfo/gnunet-service-peerinfo.c b/src/peerinfo/gnunet-service-peerinfo.c index 731c24bf1..af1eb2d1d 100644 --- a/src/peerinfo/gnunet-service-peerinfo.c +++ b/src/peerinfo/gnunet-service-peerinfo.c | |||
@@ -987,9 +987,13 @@ discard_hosts_helper (void *cls, | |||
987 | int write_pos; | 987 | int write_pos; |
988 | unsigned int cnt; | 988 | unsigned int cnt; |
989 | char *writebuffer; | 989 | char *writebuffer; |
990 | uint64_t fsize; | ||
990 | 991 | ||
992 | GNUNET_DISK_file_size (fn, &fsize, GNUNET_YES, GNUNET_YES); | ||
991 | read_size = GNUNET_DISK_fn_read (fn, buffer, sizeof (buffer)); | 993 | read_size = GNUNET_DISK_fn_read (fn, buffer, sizeof (buffer)); |
992 | if (read_size < (int) sizeof (struct GNUNET_MessageHeader)) | 994 | |
995 | if ((read_size < (int) sizeof (struct GNUNET_MessageHeader)) || | ||
996 | (fsize > GNUNET_MAX_MESSAGE_SIZE)) | ||
993 | { | 997 | { |
994 | if (0 != UNLINK (fn)) | 998 | if (0 != UNLINK (fn)) |
995 | GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING | | 999 | GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING | |
diff --git a/src/peerinfo/peerinfo_api.c b/src/peerinfo/peerinfo_api.c index b75d4e291..8b47ed444 100644 --- a/src/peerinfo/peerinfo_api.c +++ b/src/peerinfo/peerinfo_api.c | |||
@@ -334,11 +334,12 @@ handle_info (void *cls, | |||
334 | { | 334 | { |
335 | struct GNUNET_PEERINFO_Handle *h = cls; | 335 | struct GNUNET_PEERINFO_Handle *h = cls; |
336 | struct GNUNET_PEERINFO_IteratorContext *ic = h->ic_head; | 336 | struct GNUNET_PEERINFO_IteratorContext *ic = h->ic_head; |
337 | const struct GNUNET_HELLO_Message *hello; | 337 | const struct GNUNET_HELLO_Message *hello = NULL; |
338 | uint16_t ms; | 338 | uint16_t ms; |
339 | 339 | ||
340 | ms = ntohs (im->header.size); | 340 | ms = ntohs (im->header.size); |
341 | hello = (0 == ms) ? NULL : (const struct GNUNET_HELLO_Message *) &im[1]; | 341 | if (ms > sizeof (struct InfoMessage)) |
342 | hello = (const struct GNUNET_HELLO_Message *) &im[1]; | ||
342 | if (NULL != ic->callback) | 343 | if (NULL != ic->callback) |
343 | ic->callback (ic->callback_cls, | 344 | ic->callback (ic->callback_cls, |
344 | &im->peer, | 345 | &im->peer, |
diff --git a/src/revocation/gnunet-service-revocation.c b/src/revocation/gnunet-service-revocation.c index 9d077f874..8281e9a16 100644 --- a/src/revocation/gnunet-service-revocation.c +++ b/src/revocation/gnunet-service-revocation.c | |||
@@ -509,6 +509,7 @@ transmit_task_cb (void *cls) | |||
509 | "Starting set exchange with peer `%s'\n", | 509 | "Starting set exchange with peer `%s'\n", |
510 | GNUNET_i2s (&peer_entry->id)); | 510 | GNUNET_i2s (&peer_entry->id)); |
511 | peer_entry->transmit_task = NULL; | 511 | peer_entry->transmit_task = NULL; |
512 | GNUNET_assert (NULL == peer_entry->so); | ||
512 | peer_entry->so = GNUNET_SET_prepare (&peer_entry->id, | 513 | peer_entry->so = GNUNET_SET_prepare (&peer_entry->id, |
513 | &revocation_set_union_app_id, | 514 | &revocation_set_union_app_id, |
514 | NULL, | 515 | NULL, |
@@ -758,6 +759,7 @@ handle_revocation_union_request (void *cls, | |||
758 | { | 759 | { |
759 | peer_entry = new_peer_entry (other_peer); | 760 | peer_entry = new_peer_entry (other_peer); |
760 | } | 761 | } |
762 | GNUNET_assert (NULL == peer_entry->so); | ||
761 | peer_entry->so = GNUNET_SET_accept (request, | 763 | peer_entry->so = GNUNET_SET_accept (request, |
762 | GNUNET_SET_RESULT_ADDED, | 764 | GNUNET_SET_RESULT_ADDED, |
763 | (struct GNUNET_SET_Option[]) {{ 0 }}, | 765 | (struct GNUNET_SET_Option[]) {{ 0 }}, |
diff --git a/src/rps/.gitignore b/src/rps/.gitignore index 0d460e62c..c8068912b 100644 --- a/src/rps/.gitignore +++ b/src/rps/.gitignore | |||
@@ -1,3 +1,14 @@ | |||
1 | gnunet-service-rps | 1 | gnunet-service-rps |
2 | gnunet-rps | 2 | gnunet-rps |
3 | gnunet-rps-profiler | 3 | gnunet-rps-profiler |
4 | test_rps_malicious_1 | ||
5 | test_rps_malicious_2 | ||
6 | test_rps_malicious_3 | ||
7 | test_rps_req_cancel | ||
8 | test_rps_seed_big | ||
9 | test_rps_seed_request | ||
10 | test_rps_single_req | ||
11 | test_service_rps_custommap | ||
12 | test_service_rps_sampler_elem | ||
13 | test_service_rps_view | ||
14 | test_rps_churn | ||
diff --git a/src/rps/Makefile.am b/src/rps/Makefile.am index e6c8cd929..de3469254 100644 --- a/src/rps/Makefile.am +++ b/src/rps/Makefile.am | |||
@@ -49,7 +49,6 @@ endif | |||
49 | gnunet_service_rps_SOURCES = \ | 49 | gnunet_service_rps_SOURCES = \ |
50 | gnunet-service-rps_sampler_elem.h gnunet-service-rps_sampler_elem.c \ | 50 | gnunet-service-rps_sampler_elem.h gnunet-service-rps_sampler_elem.c \ |
51 | gnunet-service-rps_sampler.h gnunet-service-rps_sampler.c \ | 51 | gnunet-service-rps_sampler.h gnunet-service-rps_sampler.c \ |
52 | gnunet-service-rps_peers.h gnunet-service-rps_peers.c \ | ||
53 | gnunet-service-rps_custommap.h gnunet-service-rps_custommap.c \ | 52 | gnunet-service-rps_custommap.h gnunet-service-rps_custommap.c \ |
54 | gnunet-service-rps_view.h gnunet-service-rps_view.c \ | 53 | gnunet-service-rps_view.h gnunet-service-rps_view.c \ |
55 | rps-test_util.h rps-test_util.c \ | 54 | rps-test_util.h rps-test_util.c \ |
@@ -73,7 +72,6 @@ if HAVE_TESTING | |||
73 | check_PROGRAMS = \ | 72 | check_PROGRAMS = \ |
74 | test_service_rps_view \ | 73 | test_service_rps_view \ |
75 | test_service_rps_custommap \ | 74 | test_service_rps_custommap \ |
76 | test_service_rps_peers \ | ||
77 | test_service_rps_sampler_elem \ | 75 | test_service_rps_sampler_elem \ |
78 | test_rps_malicious_1 \ | 76 | test_rps_malicious_1 \ |
79 | test_rps_malicious_2 \ | 77 | test_rps_malicious_2 \ |
@@ -81,7 +79,8 @@ check_PROGRAMS = \ | |||
81 | test_rps_seed_request \ | 79 | test_rps_seed_request \ |
82 | test_rps_single_req \ | 80 | test_rps_single_req \ |
83 | test_rps_req_cancel \ | 81 | test_rps_req_cancel \ |
84 | test_rps_seed_big | 82 | test_rps_seed_big \ |
83 | test_rps_churn | ||
85 | endif | 84 | endif |
86 | 85 | ||
87 | ld_rps_test_lib = \ | 86 | ld_rps_test_lib = \ |
@@ -105,13 +104,6 @@ test_service_rps_view_SOURCES = \ | |||
105 | test_service_rps_view.c | 104 | test_service_rps_view.c |
106 | test_service_rps_view_LDADD = $(top_builddir)/src/util/libgnunetutil.la | 105 | test_service_rps_view_LDADD = $(top_builddir)/src/util/libgnunetutil.la |
107 | 106 | ||
108 | test_service_rps_peers_SOURCES = \ | ||
109 | gnunet-service-rps_peers.h gnunet-service-rps_peers.c \ | ||
110 | test_service_rps_peers.c | ||
111 | test_service_rps_peers_LDADD = \ | ||
112 | $(top_builddir)/src/util/libgnunetutil.la \ | ||
113 | $(top_builddir)/src/cadet/libgnunetcadet.la | ||
114 | |||
115 | test_service_rps_custommap_SOURCES = \ | 107 | test_service_rps_custommap_SOURCES = \ |
116 | gnunet-service-rps_custommap.h gnunet-service-rps_custommap.c \ | 108 | gnunet-service-rps_custommap.h gnunet-service-rps_custommap.c \ |
117 | test_service_rps_custommap.c | 109 | test_service_rps_custommap.c |
@@ -145,6 +137,9 @@ test_rps_req_cancel_LDADD = $(ld_rps_test_lib) | |||
145 | test_rps_seed_big_SOURCES = $(rps_test_src) | 137 | test_rps_seed_big_SOURCES = $(rps_test_src) |
146 | test_rps_seed_big_LDADD = $(ld_rps_test_lib) | 138 | test_rps_seed_big_LDADD = $(ld_rps_test_lib) |
147 | 139 | ||
140 | test_rps_churn_SOURCES = $(rps_test_src) | ||
141 | test_rps_churn_LDADD = $(ld_rps_test_lib) | ||
142 | |||
148 | gnunet_rps_profiler_SOURCES = $(rps_test_src) | 143 | gnunet_rps_profiler_SOURCES = $(rps_test_src) |
149 | gnunet_rps_profiler_LDADD = $(ld_rps_test_lib) | 144 | gnunet_rps_profiler_LDADD = $(ld_rps_test_lib) |
150 | 145 | ||
diff --git a/src/rps/gnunet-service-rps.c b/src/rps/gnunet-service-rps.c index 0a4543b30..ec70075cf 100644 --- a/src/rps/gnunet-service-rps.c +++ b/src/rps/gnunet-service-rps.c | |||
@@ -33,7 +33,6 @@ | |||
33 | #include "rps-test_util.h" | 33 | #include "rps-test_util.h" |
34 | #include "gnunet-service-rps_sampler.h" | 34 | #include "gnunet-service-rps_sampler.h" |
35 | #include "gnunet-service-rps_custommap.h" | 35 | #include "gnunet-service-rps_custommap.h" |
36 | #include "gnunet-service-rps_peers.h" | ||
37 | #include "gnunet-service-rps_view.h" | 36 | #include "gnunet-service-rps_view.h" |
38 | 37 | ||
39 | #include <math.h> | 38 | #include <math.h> |
@@ -66,6 +65,1728 @@ static const struct GNUNET_CONFIGURATION_Handle *cfg; | |||
66 | static struct GNUNET_PeerIdentity own_identity; | 65 | static struct GNUNET_PeerIdentity own_identity; |
67 | 66 | ||
68 | 67 | ||
68 | |||
69 | /*********************************************************************** | ||
70 | * Old gnunet-service-rps_peers.c | ||
71 | ***********************************************************************/ | ||
72 | |||
73 | /** | ||
74 | * Set a peer flag of given peer context. | ||
75 | */ | ||
76 | #define set_peer_flag(peer_ctx, mask) ((peer_ctx->peer_flags) |= (mask)) | ||
77 | |||
78 | /** | ||
79 | * Get peer flag of given peer context. | ||
80 | */ | ||
81 | #define check_peer_flag_set(peer_ctx, mask)\ | ||
82 | ((peer_ctx->peer_flags) & (mask) ? GNUNET_YES : GNUNET_NO) | ||
83 | |||
84 | /** | ||
85 | * Unset flag of given peer context. | ||
86 | */ | ||
87 | #define unset_peer_flag(peer_ctx, mask) ((peer_ctx->peer_flags) &= ~(mask)) | ||
88 | |||
89 | /** | ||
90 | * Set a channel flag of given channel context. | ||
91 | */ | ||
92 | #define set_channel_flag(channel_flags, mask) ((*channel_flags) |= (mask)) | ||
93 | |||
94 | /** | ||
95 | * Get channel flag of given channel context. | ||
96 | */ | ||
97 | #define check_channel_flag_set(channel_flags, mask)\ | ||
98 | ((*channel_flags) & (mask) ? GNUNET_YES : GNUNET_NO) | ||
99 | |||
100 | /** | ||
101 | * Unset flag of given channel context. | ||
102 | */ | ||
103 | #define unset_channel_flag(channel_flags, mask) ((*channel_flags) &= ~(mask)) | ||
104 | |||
105 | |||
106 | |||
107 | /** | ||
108 | * Pending operation on peer consisting of callback and closure | ||
109 | * | ||
110 | * When an operation cannot be executed right now this struct is used to store | ||
111 | * the callback and closure for later execution. | ||
112 | */ | ||
113 | struct PeerPendingOp | ||
114 | { | ||
115 | /** | ||
116 | * Callback | ||
117 | */ | ||
118 | PeerOp op; | ||
119 | |||
120 | /** | ||
121 | * Closure | ||
122 | */ | ||
123 | void *op_cls; | ||
124 | }; | ||
125 | |||
126 | /** | ||
127 | * List containing all messages that are yet to be send | ||
128 | * | ||
129 | * This is used to keep track of all messages that have not been sent yet. When | ||
130 | * a peer is to be removed the pending messages can be removed properly. | ||
131 | */ | ||
132 | struct PendingMessage | ||
133 | { | ||
134 | /** | ||
135 | * DLL next, prev | ||
136 | */ | ||
137 | struct PendingMessage *next; | ||
138 | struct PendingMessage *prev; | ||
139 | |||
140 | /** | ||
141 | * The envelope to the corresponding message | ||
142 | */ | ||
143 | struct GNUNET_MQ_Envelope *ev; | ||
144 | |||
145 | /** | ||
146 | * The corresponding context | ||
147 | */ | ||
148 | struct PeerContext *peer_ctx; | ||
149 | |||
150 | /** | ||
151 | * The message type | ||
152 | */ | ||
153 | const char *type; | ||
154 | }; | ||
155 | |||
156 | /** | ||
157 | * Struct used to keep track of other peer's status | ||
158 | * | ||
159 | * This is stored in a multipeermap. | ||
160 | * It contains information such as cadet channels, a message queue for sending, | ||
161 | * status about the channels, the pending operations on this peer and some flags | ||
162 | * about the status of the peer itself. (live, valid, ...) | ||
163 | */ | ||
164 | struct PeerContext | ||
165 | { | ||
166 | /** | ||
167 | * Message queue open to client | ||
168 | */ | ||
169 | struct GNUNET_MQ_Handle *mq; | ||
170 | |||
171 | /** | ||
172 | * Channel open to client. | ||
173 | */ | ||
174 | struct GNUNET_CADET_Channel *send_channel; | ||
175 | |||
176 | /** | ||
177 | * Flags to the sending channel | ||
178 | */ | ||
179 | uint32_t *send_channel_flags; | ||
180 | |||
181 | /** | ||
182 | * Channel open from client. | ||
183 | */ | ||
184 | struct GNUNET_CADET_Channel *recv_channel; // unneeded? | ||
185 | |||
186 | /** | ||
187 | * Flags to the receiving channel | ||
188 | */ | ||
189 | uint32_t *recv_channel_flags; | ||
190 | |||
191 | /** | ||
192 | * Array of pending operations on this peer. | ||
193 | */ | ||
194 | struct PeerPendingOp *pending_ops; | ||
195 | |||
196 | /** | ||
197 | * Handle to the callback given to cadet_ntfy_tmt_rdy() | ||
198 | * | ||
199 | * To be canceled on shutdown. | ||
200 | */ | ||
201 | struct PendingMessage *liveliness_check_pending; | ||
202 | |||
203 | /** | ||
204 | * Number of pending operations. | ||
205 | */ | ||
206 | unsigned int num_pending_ops; | ||
207 | |||
208 | /** | ||
209 | * Identity of the peer | ||
210 | */ | ||
211 | struct GNUNET_PeerIdentity peer_id; | ||
212 | |||
213 | /** | ||
214 | * Flags indicating status of peer | ||
215 | */ | ||
216 | uint32_t peer_flags; | ||
217 | |||
218 | /** | ||
219 | * Last time we received something from that peer. | ||
220 | */ | ||
221 | struct GNUNET_TIME_Absolute last_message_recv; | ||
222 | |||
223 | /** | ||
224 | * Last time we received a keepalive message. | ||
225 | */ | ||
226 | struct GNUNET_TIME_Absolute last_keepalive; | ||
227 | |||
228 | /** | ||
229 | * DLL with all messages that are yet to be sent | ||
230 | */ | ||
231 | struct PendingMessage *pending_messages_head; | ||
232 | struct PendingMessage *pending_messages_tail; | ||
233 | |||
234 | /** | ||
235 | * This is pobably followed by 'statistical' data (when we first saw | ||
236 | * him, how did we get his ID, how many pushes (in a timeinterval), | ||
237 | * ...) | ||
238 | */ | ||
239 | }; | ||
240 | |||
241 | /** | ||
242 | * @brief Closure to #valid_peer_iterator | ||
243 | */ | ||
244 | struct PeersIteratorCls | ||
245 | { | ||
246 | /** | ||
247 | * Iterator function | ||
248 | */ | ||
249 | PeersIterator iterator; | ||
250 | |||
251 | /** | ||
252 | * Closure to iterator | ||
253 | */ | ||
254 | void *cls; | ||
255 | }; | ||
256 | |||
257 | /** | ||
258 | * @brief Hashmap of valid peers. | ||
259 | */ | ||
260 | static struct GNUNET_CONTAINER_MultiPeerMap *valid_peers; | ||
261 | |||
262 | /** | ||
263 | * @brief Maximum number of valid peers to keep. | ||
264 | * TODO read from config | ||
265 | */ | ||
266 | static uint32_t num_valid_peers_max = UINT32_MAX; | ||
267 | |||
268 | /** | ||
269 | * @brief Filename of the file that stores the valid peers persistently. | ||
270 | */ | ||
271 | static char *filename_valid_peers; | ||
272 | |||
273 | /** | ||
274 | * Set of all peers to keep track of them. | ||
275 | */ | ||
276 | static struct GNUNET_CONTAINER_MultiPeerMap *peer_map; | ||
277 | |||
278 | /** | ||
279 | * Cadet handle. | ||
280 | */ | ||
281 | static struct GNUNET_CADET_Handle *cadet_handle; | ||
282 | |||
283 | |||
284 | |||
285 | /** | ||
286 | * @brief Get the #PeerContext associated with a peer | ||
287 | * | ||
288 | * @param peer the peer id | ||
289 | * | ||
290 | * @return the #PeerContext | ||
291 | */ | ||
292 | static struct PeerContext * | ||
293 | get_peer_ctx (const struct GNUNET_PeerIdentity *peer) | ||
294 | { | ||
295 | struct PeerContext *ctx; | ||
296 | int ret; | ||
297 | |||
298 | ret = GNUNET_CONTAINER_multipeermap_contains (peer_map, peer); | ||
299 | GNUNET_assert (GNUNET_YES == ret); | ||
300 | ctx = GNUNET_CONTAINER_multipeermap_get (peer_map, peer); | ||
301 | GNUNET_assert (NULL != ctx); | ||
302 | return ctx; | ||
303 | } | ||
304 | |||
305 | int | ||
306 | Peers_check_peer_known (const struct GNUNET_PeerIdentity *peer); | ||
307 | |||
308 | /** | ||
309 | * @brief Create a new #PeerContext and insert it into the peer map | ||
310 | * | ||
311 | * @param peer the peer to create the #PeerContext for | ||
312 | * | ||
313 | * @return the #PeerContext | ||
314 | */ | ||
315 | static struct PeerContext * | ||
316 | create_peer_ctx (const struct GNUNET_PeerIdentity *peer) | ||
317 | { | ||
318 | struct PeerContext *ctx; | ||
319 | int ret; | ||
320 | |||
321 | GNUNET_assert (GNUNET_NO == Peers_check_peer_known (peer)); | ||
322 | |||
323 | ctx = GNUNET_new (struct PeerContext); | ||
324 | ctx->peer_id = *peer; | ||
325 | ctx->send_channel_flags = GNUNET_new (uint32_t); | ||
326 | ctx->recv_channel_flags = GNUNET_new (uint32_t); | ||
327 | ret = GNUNET_CONTAINER_multipeermap_put (peer_map, peer, ctx, | ||
328 | GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY); | ||
329 | GNUNET_assert (GNUNET_OK == ret); | ||
330 | return ctx; | ||
331 | } | ||
332 | |||
333 | |||
334 | /** | ||
335 | * @brief Create or get a #PeerContext | ||
336 | * | ||
337 | * @param peer the peer to get the associated context to | ||
338 | * | ||
339 | * @return the context | ||
340 | */ | ||
341 | static struct PeerContext * | ||
342 | create_or_get_peer_ctx (const struct GNUNET_PeerIdentity *peer) | ||
343 | { | ||
344 | if (GNUNET_NO == Peers_check_peer_known (peer)) | ||
345 | { | ||
346 | return create_peer_ctx (peer); | ||
347 | } | ||
348 | return get_peer_ctx (peer); | ||
349 | } | ||
350 | |||
351 | void | ||
352 | Peers_unset_peer_flag (const struct GNUNET_PeerIdentity *peer, enum Peers_PeerFlags flags); | ||
353 | |||
354 | void | ||
355 | Peers_set_peer_flag (const struct GNUNET_PeerIdentity *peer, enum Peers_PeerFlags flags); | ||
356 | |||
357 | /** | ||
358 | * @brief Check whether we have a connection to this @a peer | ||
359 | * | ||
360 | * Also sets the #Peers_ONLINE flag accordingly | ||
361 | * | ||
362 | * @param peer the peer in question | ||
363 | * | ||
364 | * @return #GNUNET_YES if we are connected | ||
365 | * #GNUNET_NO otherwise | ||
366 | */ | ||
367 | int | ||
368 | Peers_check_connected (const struct GNUNET_PeerIdentity *peer) | ||
369 | { | ||
370 | const struct PeerContext *peer_ctx; | ||
371 | |||
372 | /* If we don't know about this peer we don't know whether it's online */ | ||
373 | if (GNUNET_NO == Peers_check_peer_known (peer)) | ||
374 | { | ||
375 | return GNUNET_NO; | ||
376 | } | ||
377 | /* Get the context */ | ||
378 | peer_ctx = get_peer_ctx (peer); | ||
379 | /* If we have no channel to this peer we don't know whether it's online */ | ||
380 | if ( (NULL == peer_ctx->send_channel) && | ||
381 | (NULL == peer_ctx->recv_channel) ) | ||
382 | { | ||
383 | Peers_unset_peer_flag (peer, Peers_ONLINE); | ||
384 | return GNUNET_NO; | ||
385 | } | ||
386 | /* Otherwise (if we have a channel, we know that it's online */ | ||
387 | Peers_set_peer_flag (peer, Peers_ONLINE); | ||
388 | return GNUNET_YES; | ||
389 | } | ||
390 | |||
391 | |||
392 | /** | ||
393 | * @brief The closure to #get_rand_peer_iterator. | ||
394 | */ | ||
395 | struct GetRandPeerIteratorCls | ||
396 | { | ||
397 | /** | ||
398 | * @brief The index of the peer to return. | ||
399 | * Will be decreased until 0. | ||
400 | * Then current peer is returned. | ||
401 | */ | ||
402 | uint32_t index; | ||
403 | |||
404 | /** | ||
405 | * @brief Pointer to peer to return. | ||
406 | */ | ||
407 | const struct GNUNET_PeerIdentity *peer; | ||
408 | }; | ||
409 | |||
410 | |||
411 | /** | ||
412 | * @brief Iterator function for #get_random_peer_from_peermap. | ||
413 | * | ||
414 | * Implements #GNUNET_CONTAINER_PeerMapIterator. | ||
415 | * Decreases the index until the index is null. | ||
416 | * Then returns the current peer. | ||
417 | * | ||
418 | * @param cls the #GetRandPeerIteratorCls containing index and peer | ||
419 | * @param peer current peer | ||
420 | * @param value unused | ||
421 | * | ||
422 | * @return #GNUNET_YES if we should continue to | ||
423 | * iterate, | ||
424 | * #GNUNET_NO if not. | ||
425 | */ | ||
426 | static int | ||
427 | get_rand_peer_iterator (void *cls, | ||
428 | const struct GNUNET_PeerIdentity *peer, | ||
429 | void *value) | ||
430 | { | ||
431 | struct GetRandPeerIteratorCls *iterator_cls = cls; | ||
432 | if (0 >= iterator_cls->index) | ||
433 | { | ||
434 | iterator_cls->peer = peer; | ||
435 | return GNUNET_NO; | ||
436 | } | ||
437 | iterator_cls->index--; | ||
438 | return GNUNET_YES; | ||
439 | } | ||
440 | |||
441 | |||
442 | /** | ||
443 | * @brief Get a random peer from @a peer_map | ||
444 | * | ||
445 | * @param peer_map the peer_map to get the peer from | ||
446 | * | ||
447 | * @return a random peer | ||
448 | */ | ||
449 | static const struct GNUNET_PeerIdentity * | ||
450 | get_random_peer_from_peermap (const struct | ||
451 | GNUNET_CONTAINER_MultiPeerMap *peer_map) | ||
452 | { | ||
453 | struct GetRandPeerIteratorCls *iterator_cls; | ||
454 | const struct GNUNET_PeerIdentity *ret; | ||
455 | |||
456 | iterator_cls = GNUNET_new (struct GetRandPeerIteratorCls); | ||
457 | iterator_cls->index = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, | ||
458 | GNUNET_CONTAINER_multipeermap_size (peer_map)); | ||
459 | (void) GNUNET_CONTAINER_multipeermap_iterate (valid_peers, | ||
460 | get_rand_peer_iterator, | ||
461 | iterator_cls); | ||
462 | ret = iterator_cls->peer; | ||
463 | GNUNET_free (iterator_cls); | ||
464 | return ret; | ||
465 | } | ||
466 | |||
467 | |||
468 | /** | ||
469 | * @brief Add a given @a peer to valid peers. | ||
470 | * | ||
471 | * If valid peers are already #num_valid_peers_max, delete a peer previously. | ||
472 | * | ||
473 | * @param peer the peer that is added to the valid peers. | ||
474 | * | ||
475 | * @return #GNUNET_YES if no other peer had to be removed | ||
476 | * #GNUNET_NO otherwise | ||
477 | */ | ||
478 | static int | ||
479 | add_valid_peer (const struct GNUNET_PeerIdentity *peer) | ||
480 | { | ||
481 | const struct GNUNET_PeerIdentity *rand_peer; | ||
482 | int ret; | ||
483 | |||
484 | ret = GNUNET_YES; | ||
485 | while (GNUNET_CONTAINER_multipeermap_size (valid_peers) >= num_valid_peers_max) | ||
486 | { | ||
487 | rand_peer = get_random_peer_from_peermap (valid_peers); | ||
488 | GNUNET_CONTAINER_multipeermap_remove_all (valid_peers, rand_peer); | ||
489 | ret = GNUNET_NO; | ||
490 | } | ||
491 | (void) GNUNET_CONTAINER_multipeermap_put (valid_peers, peer, NULL, | ||
492 | GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY); | ||
493 | return ret; | ||
494 | } | ||
495 | |||
496 | |||
497 | /** | ||
498 | * @brief Set the peer flag to living and | ||
499 | * call the pending operations on this peer. | ||
500 | * | ||
501 | * Also adds peer to #valid_peers. | ||
502 | * | ||
503 | * @param peer_ctx the #PeerContext of the peer to set live | ||
504 | */ | ||
505 | static void | ||
506 | set_peer_live (struct PeerContext *peer_ctx) | ||
507 | { | ||
508 | struct GNUNET_PeerIdentity *peer; | ||
509 | unsigned int i; | ||
510 | |||
511 | peer = &peer_ctx->peer_id; | ||
512 | LOG (GNUNET_ERROR_TYPE_DEBUG, | ||
513 | "Peer %s is live and valid, calling %i pending operations on it\n", | ||
514 | GNUNET_i2s (peer), | ||
515 | peer_ctx->num_pending_ops); | ||
516 | |||
517 | if (NULL != peer_ctx->liveliness_check_pending) | ||
518 | { | ||
519 | LOG (GNUNET_ERROR_TYPE_DEBUG, | ||
520 | "Removing pending liveliness check for peer %s\n", | ||
521 | GNUNET_i2s (&peer_ctx->peer_id)); | ||
522 | // TODO wait until cadet sets mq->cancel_impl | ||
523 | //GNUNET_MQ_send_cancel (peer_ctx->liveliness_check_pending->ev); | ||
524 | GNUNET_free (peer_ctx->liveliness_check_pending); | ||
525 | peer_ctx->liveliness_check_pending = NULL; | ||
526 | } | ||
527 | |||
528 | (void) add_valid_peer (peer); | ||
529 | set_peer_flag (peer_ctx, Peers_ONLINE); | ||
530 | |||
531 | /* Call pending operations */ | ||
532 | for (i = 0; i < peer_ctx->num_pending_ops; i++) | ||
533 | { | ||
534 | peer_ctx->pending_ops[i].op (peer_ctx->pending_ops[i].op_cls, peer); | ||
535 | } | ||
536 | GNUNET_array_grow (peer_ctx->pending_ops, peer_ctx->num_pending_ops, 0); | ||
537 | } | ||
538 | |||
539 | static void | ||
540 | cleanup_destroyed_channel (void *cls, | ||
541 | const struct GNUNET_CADET_Channel *channel); | ||
542 | |||
543 | /* Declaration of handlers */ | ||
544 | static void | ||
545 | handle_peer_check (void *cls, | ||
546 | const struct GNUNET_MessageHeader *msg); | ||
547 | |||
548 | static void | ||
549 | handle_peer_push (void *cls, | ||
550 | const struct GNUNET_MessageHeader *msg); | ||
551 | |||
552 | static void | ||
553 | handle_peer_pull_request (void *cls, | ||
554 | const struct GNUNET_MessageHeader *msg); | ||
555 | |||
556 | static int | ||
557 | check_peer_pull_reply (void *cls, | ||
558 | const struct GNUNET_RPS_P2P_PullReplyMessage *msg); | ||
559 | |||
560 | static void | ||
561 | handle_peer_pull_reply (void *cls, | ||
562 | const struct GNUNET_RPS_P2P_PullReplyMessage *msg); | ||
563 | |||
564 | /* End declaration of handlers */ | ||
565 | |||
566 | |||
567 | /** | ||
568 | * @brief Get the channel of a peer. If not existing, create. | ||
569 | * | ||
570 | * @param peer the peer id | ||
571 | * @return the #GNUNET_CADET_Channel used to send data to @a peer | ||
572 | */ | ||
573 | struct GNUNET_CADET_Channel * | ||
574 | get_channel (const struct GNUNET_PeerIdentity *peer) | ||
575 | { | ||
576 | struct PeerContext *peer_ctx; | ||
577 | struct GNUNET_HashCode port; | ||
578 | /* There exists a copy-paste-clone in run() */ | ||
579 | struct GNUNET_MQ_MessageHandler cadet_handlers[] = { | ||
580 | GNUNET_MQ_hd_fixed_size (peer_check, | ||
581 | GNUNET_MESSAGE_TYPE_RPS_PP_CHECK_LIVE, | ||
582 | struct GNUNET_MessageHeader, | ||
583 | NULL), | ||
584 | GNUNET_MQ_hd_fixed_size (peer_push, | ||
585 | GNUNET_MESSAGE_TYPE_RPS_PP_PUSH, | ||
586 | struct GNUNET_MessageHeader, | ||
587 | NULL), | ||
588 | GNUNET_MQ_hd_fixed_size (peer_pull_request, | ||
589 | GNUNET_MESSAGE_TYPE_RPS_PP_PULL_REQUEST, | ||
590 | struct GNUNET_MessageHeader, | ||
591 | NULL), | ||
592 | GNUNET_MQ_hd_var_size (peer_pull_reply, | ||
593 | GNUNET_MESSAGE_TYPE_RPS_PP_PULL_REPLY, | ||
594 | struct GNUNET_RPS_P2P_PullReplyMessage, | ||
595 | NULL), | ||
596 | GNUNET_MQ_handler_end () | ||
597 | }; | ||
598 | |||
599 | |||
600 | peer_ctx = get_peer_ctx (peer); | ||
601 | if (NULL == peer_ctx->send_channel) | ||
602 | { | ||
603 | LOG (GNUNET_ERROR_TYPE_DEBUG, | ||
604 | "Trying to establish channel to peer %s\n", | ||
605 | GNUNET_i2s (peer)); | ||
606 | GNUNET_CRYPTO_hash (GNUNET_APPLICATION_PORT_RPS, | ||
607 | strlen (GNUNET_APPLICATION_PORT_RPS), | ||
608 | &port); | ||
609 | peer_ctx->send_channel = | ||
610 | GNUNET_CADET_channel_create (cadet_handle, | ||
611 | (struct GNUNET_PeerIdentity *) peer, /* context */ | ||
612 | peer, | ||
613 | &port, | ||
614 | GNUNET_CADET_OPTION_RELIABLE, | ||
615 | NULL, /* WindowSize handler */ | ||
616 | cleanup_destroyed_channel, /* Disconnect handler */ | ||
617 | cadet_handlers); | ||
618 | } | ||
619 | GNUNET_assert (NULL != peer_ctx->send_channel); | ||
620 | return peer_ctx->send_channel; | ||
621 | } | ||
622 | |||
623 | |||
624 | /** | ||
625 | * Get the message queue (#GNUNET_MQ_Handle) of a specific peer. | ||
626 | * | ||
627 | * If we already have a message queue open to this client, | ||
628 | * simply return it, otherways create one. | ||
629 | * | ||
630 | * @param peer the peer to get the mq to | ||
631 | * @return the #GNUNET_MQ_Handle | ||
632 | */ | ||
633 | static struct GNUNET_MQ_Handle * | ||
634 | get_mq (const struct GNUNET_PeerIdentity *peer) | ||
635 | { | ||
636 | struct PeerContext *peer_ctx; | ||
637 | |||
638 | peer_ctx = get_peer_ctx (peer); | ||
639 | |||
640 | if (NULL == peer_ctx->mq) | ||
641 | { | ||
642 | (void) get_channel (peer); | ||
643 | peer_ctx->mq = GNUNET_CADET_get_mq (peer_ctx->send_channel); | ||
644 | } | ||
645 | return peer_ctx->mq; | ||
646 | } | ||
647 | |||
648 | |||
649 | /** | ||
650 | * @brief This is called in response to the first message we sent as a | ||
651 | * liveliness check. | ||
652 | * | ||
653 | * @param cls #PeerContext of peer with pending liveliness check | ||
654 | */ | ||
655 | static void | ||
656 | mq_liveliness_check_successful (void *cls) | ||
657 | { | ||
658 | struct PeerContext *peer_ctx = cls; | ||
659 | |||
660 | if (NULL != peer_ctx->liveliness_check_pending) | ||
661 | { | ||
662 | LOG (GNUNET_ERROR_TYPE_DEBUG, | ||
663 | "Liveliness check for peer %s was successfull\n", | ||
664 | GNUNET_i2s (&peer_ctx->peer_id)); | ||
665 | GNUNET_free (peer_ctx->liveliness_check_pending); | ||
666 | peer_ctx->liveliness_check_pending = NULL; | ||
667 | set_peer_live (peer_ctx); | ||
668 | } | ||
669 | } | ||
670 | |||
671 | /** | ||
672 | * Issue a check whether peer is live | ||
673 | * | ||
674 | * @param peer_ctx the context of the peer | ||
675 | */ | ||
676 | static void | ||
677 | check_peer_live (struct PeerContext *peer_ctx) | ||
678 | { | ||
679 | LOG (GNUNET_ERROR_TYPE_DEBUG, | ||
680 | "Get informed about peer %s getting live\n", | ||
681 | GNUNET_i2s (&peer_ctx->peer_id)); | ||
682 | |||
683 | struct GNUNET_MQ_Handle *mq; | ||
684 | struct GNUNET_MQ_Envelope *ev; | ||
685 | |||
686 | ev = GNUNET_MQ_msg_header (GNUNET_MESSAGE_TYPE_RPS_PP_CHECK_LIVE); | ||
687 | peer_ctx->liveliness_check_pending = GNUNET_new (struct PendingMessage); | ||
688 | peer_ctx->liveliness_check_pending->ev = ev; | ||
689 | peer_ctx->liveliness_check_pending->peer_ctx = peer_ctx; | ||
690 | peer_ctx->liveliness_check_pending->type = "Check liveliness"; | ||
691 | mq = get_mq (&peer_ctx->peer_id); | ||
692 | GNUNET_MQ_notify_sent (ev, | ||
693 | mq_liveliness_check_successful, | ||
694 | peer_ctx); | ||
695 | GNUNET_MQ_send (mq, ev); | ||
696 | } | ||
697 | |||
698 | /** | ||
699 | * @brief Add an envelope to a message passed to mq to list of pending messages | ||
700 | * | ||
701 | * @param peer peer the message was sent to | ||
702 | * @param ev envelope to the message | ||
703 | * @param type type of the message to be sent | ||
704 | * @return pointer to pending message | ||
705 | */ | ||
706 | static struct PendingMessage * | ||
707 | insert_pending_message (const struct GNUNET_PeerIdentity *peer, | ||
708 | struct GNUNET_MQ_Envelope *ev, | ||
709 | const char *type) | ||
710 | { | ||
711 | struct PendingMessage *pending_msg; | ||
712 | struct PeerContext *peer_ctx; | ||
713 | |||
714 | peer_ctx = get_peer_ctx (peer); | ||
715 | pending_msg = GNUNET_new (struct PendingMessage); | ||
716 | pending_msg->ev = ev; | ||
717 | pending_msg->peer_ctx = peer_ctx; | ||
718 | pending_msg->type = type; | ||
719 | GNUNET_CONTAINER_DLL_insert (peer_ctx->pending_messages_head, | ||
720 | peer_ctx->pending_messages_tail, | ||
721 | pending_msg); | ||
722 | return pending_msg; | ||
723 | } | ||
724 | |||
725 | |||
726 | /** | ||
727 | * @brief Remove a pending message from the respective DLL | ||
728 | * | ||
729 | * @param pending_msg the pending message to remove | ||
730 | * @param cancel cancel the pending message, too | ||
731 | */ | ||
732 | static void | ||
733 | remove_pending_message (struct PendingMessage *pending_msg, int cancel) | ||
734 | { | ||
735 | struct PeerContext *peer_ctx; | ||
736 | |||
737 | peer_ctx = pending_msg->peer_ctx; | ||
738 | GNUNET_assert (NULL != peer_ctx); | ||
739 | GNUNET_CONTAINER_DLL_remove (peer_ctx->pending_messages_head, | ||
740 | peer_ctx->pending_messages_tail, | ||
741 | pending_msg); | ||
742 | // TODO wait for the cadet implementation of message cancellation | ||
743 | //if (GNUNET_YES == cancel) | ||
744 | //{ | ||
745 | // GNUNET_MQ_send_cancel (pending_msg->ev); | ||
746 | //} | ||
747 | GNUNET_free (pending_msg); | ||
748 | } | ||
749 | |||
750 | |||
751 | /** | ||
752 | * @brief Check whether function of type #PeerOp was already scheduled | ||
753 | * | ||
754 | * The array with pending operations will probably never grow really big, so | ||
755 | * iterating over it should be ok. | ||
756 | * | ||
757 | * @param peer the peer to check | ||
758 | * @param peer_op the operation (#PeerOp) on the peer | ||
759 | * | ||
760 | * @return #GNUNET_YES if this operation is scheduled on that peer | ||
761 | * #GNUNET_NO otherwise | ||
762 | */ | ||
763 | static int | ||
764 | check_operation_scheduled (const struct GNUNET_PeerIdentity *peer, | ||
765 | const PeerOp peer_op) | ||
766 | { | ||
767 | const struct PeerContext *peer_ctx; | ||
768 | unsigned int i; | ||
769 | |||
770 | peer_ctx = get_peer_ctx (peer); | ||
771 | for (i = 0; i < peer_ctx->num_pending_ops; i++) | ||
772 | if (peer_op == peer_ctx->pending_ops[i].op) | ||
773 | return GNUNET_YES; | ||
774 | return GNUNET_NO; | ||
775 | } | ||
776 | |||
777 | int | ||
778 | Peers_remove_peer (const struct GNUNET_PeerIdentity *peer); | ||
779 | |||
780 | /** | ||
781 | * Iterator over hash map entries. Deletes all contexts of peers. | ||
782 | * | ||
783 | * @param cls closure | ||
784 | * @param key current public key | ||
785 | * @param value value in the hash map | ||
786 | * @return #GNUNET_YES if we should continue to iterate, | ||
787 | * #GNUNET_NO if not. | ||
788 | */ | ||
789 | static int | ||
790 | peermap_clear_iterator (void *cls, | ||
791 | const struct GNUNET_PeerIdentity *key, | ||
792 | void *value) | ||
793 | { | ||
794 | Peers_remove_peer (key); | ||
795 | return GNUNET_YES; | ||
796 | } | ||
797 | |||
798 | |||
799 | /** | ||
800 | * @brief This is called once a message is sent. | ||
801 | * | ||
802 | * Removes the pending message | ||
803 | * | ||
804 | * @param cls type of the message that was sent | ||
805 | */ | ||
806 | static void | ||
807 | mq_notify_sent_cb (void *cls) | ||
808 | { | ||
809 | struct PendingMessage *pending_msg = (struct PendingMessage *) cls; | ||
810 | LOG (GNUNET_ERROR_TYPE_DEBUG, | ||
811 | "%s was sent.\n", | ||
812 | pending_msg->type); | ||
813 | /* Do not cancle message */ | ||
814 | remove_pending_message (pending_msg, GNUNET_NO); | ||
815 | } | ||
816 | |||
817 | |||
818 | /** | ||
819 | * @brief Iterator function for #store_valid_peers. | ||
820 | * | ||
821 | * Implements #GNUNET_CONTAINER_PeerMapIterator. | ||
822 | * Writes single peer to disk. | ||
823 | * | ||
824 | * @param cls the file handle to write to. | ||
825 | * @param peer current peer | ||
826 | * @param value unused | ||
827 | * | ||
828 | * @return #GNUNET_YES if we should continue to | ||
829 | * iterate, | ||
830 | * #GNUNET_NO if not. | ||
831 | */ | ||
832 | static int | ||
833 | store_peer_presistently_iterator (void *cls, | ||
834 | const struct GNUNET_PeerIdentity *peer, | ||
835 | void *value) | ||
836 | { | ||
837 | const struct GNUNET_DISK_FileHandle *fh = cls; | ||
838 | char peer_string[128]; | ||
839 | int size; | ||
840 | ssize_t ret; | ||
841 | |||
842 | if (NULL == peer) | ||
843 | { | ||
844 | return GNUNET_YES; | ||
845 | } | ||
846 | size = GNUNET_snprintf (peer_string, | ||
847 | sizeof (peer_string), | ||
848 | "%s\n", | ||
849 | GNUNET_i2s_full (peer)); | ||
850 | GNUNET_assert (53 == size); | ||
851 | ret = GNUNET_DISK_file_write (fh, | ||
852 | peer_string, | ||
853 | size); | ||
854 | GNUNET_assert (size == ret); | ||
855 | return GNUNET_YES; | ||
856 | } | ||
857 | |||
858 | |||
859 | /** | ||
860 | * @brief Store the peers currently in #valid_peers to disk. | ||
861 | */ | ||
862 | static void | ||
863 | store_valid_peers () | ||
864 | { | ||
865 | struct GNUNET_DISK_FileHandle *fh; | ||
866 | uint32_t number_written_peers; | ||
867 | int ret; | ||
868 | |||
869 | if (0 == strncmp ("DISABLE", filename_valid_peers, 7)) | ||
870 | { | ||
871 | return; | ||
872 | } | ||
873 | |||
874 | ret = GNUNET_DISK_directory_create_for_file (filename_valid_peers); | ||
875 | if (GNUNET_SYSERR == ret) | ||
876 | { | ||
877 | LOG (GNUNET_ERROR_TYPE_WARNING, | ||
878 | "Not able to create directory for file `%s'\n", | ||
879 | filename_valid_peers); | ||
880 | GNUNET_break (0); | ||
881 | } | ||
882 | else if (GNUNET_NO == ret) | ||
883 | { | ||
884 | LOG (GNUNET_ERROR_TYPE_WARNING, | ||
885 | "Directory for file `%s' exists but is not writable for us\n", | ||
886 | filename_valid_peers); | ||
887 | GNUNET_break (0); | ||
888 | } | ||
889 | fh = GNUNET_DISK_file_open (filename_valid_peers, | ||
890 | GNUNET_DISK_OPEN_WRITE | | ||
891 | GNUNET_DISK_OPEN_CREATE, | ||
892 | GNUNET_DISK_PERM_USER_READ | | ||
893 | GNUNET_DISK_PERM_USER_WRITE); | ||
894 | if (NULL == fh) | ||
895 | { | ||
896 | LOG (GNUNET_ERROR_TYPE_WARNING, | ||
897 | "Not able to write valid peers to file `%s'\n", | ||
898 | filename_valid_peers); | ||
899 | return; | ||
900 | } | ||
901 | LOG (GNUNET_ERROR_TYPE_DEBUG, | ||
902 | "Writing %u valid peers to disk\n", | ||
903 | GNUNET_CONTAINER_multipeermap_size (valid_peers)); | ||
904 | number_written_peers = | ||
905 | GNUNET_CONTAINER_multipeermap_iterate (valid_peers, | ||
906 | store_peer_presistently_iterator, | ||
907 | fh); | ||
908 | GNUNET_assert (GNUNET_OK == GNUNET_DISK_file_close (fh)); | ||
909 | GNUNET_assert (number_written_peers == | ||
910 | GNUNET_CONTAINER_multipeermap_size (valid_peers)); | ||
911 | } | ||
912 | |||
913 | |||
914 | /** | ||
915 | * @brief Convert string representation of peer id to peer id. | ||
916 | * | ||
917 | * Counterpart to #GNUNET_i2s_full. | ||
918 | * | ||
919 | * @param string_repr The string representation of the peer id | ||
920 | * | ||
921 | * @return The peer id | ||
922 | */ | ||
923 | static const struct GNUNET_PeerIdentity * | ||
924 | s2i_full (const char *string_repr) | ||
925 | { | ||
926 | struct GNUNET_PeerIdentity *peer; | ||
927 | size_t len; | ||
928 | int ret; | ||
929 | |||
930 | peer = GNUNET_new (struct GNUNET_PeerIdentity); | ||
931 | len = strlen (string_repr); | ||
932 | if (52 > len) | ||
933 | { | ||
934 | LOG (GNUNET_ERROR_TYPE_WARNING, | ||
935 | "Not able to convert string representation of PeerID to PeerID\n" | ||
936 | "Sting representation: %s (len %lu) - too short\n", | ||
937 | string_repr, | ||
938 | len); | ||
939 | GNUNET_break (0); | ||
940 | } | ||
941 | else if (52 < len) | ||
942 | { | ||
943 | len = 52; | ||
944 | } | ||
945 | ret = GNUNET_CRYPTO_eddsa_public_key_from_string (string_repr, | ||
946 | len, | ||
947 | &peer->public_key); | ||
948 | if (GNUNET_OK != ret) | ||
949 | { | ||
950 | LOG (GNUNET_ERROR_TYPE_WARNING, | ||
951 | "Not able to convert string representation of PeerID to PeerID\n" | ||
952 | "Sting representation: %s\n", | ||
953 | string_repr); | ||
954 | GNUNET_break (0); | ||
955 | } | ||
956 | return peer; | ||
957 | } | ||
958 | |||
959 | |||
960 | /** | ||
961 | * @brief Restore the peers on disk to #valid_peers. | ||
962 | */ | ||
963 | static void | ||
964 | restore_valid_peers () | ||
965 | { | ||
966 | off_t file_size; | ||
967 | uint32_t num_peers; | ||
968 | struct GNUNET_DISK_FileHandle *fh; | ||
969 | char *buf; | ||
970 | ssize_t size_read; | ||
971 | char *iter_buf; | ||
972 | char *str_repr; | ||
973 | const struct GNUNET_PeerIdentity *peer; | ||
974 | |||
975 | if (0 == strncmp ("DISABLE", filename_valid_peers, 7)) | ||
976 | { | ||
977 | return; | ||
978 | } | ||
979 | |||
980 | if (GNUNET_OK != GNUNET_DISK_file_test (filename_valid_peers)) | ||
981 | { | ||
982 | return; | ||
983 | } | ||
984 | fh = GNUNET_DISK_file_open (filename_valid_peers, | ||
985 | GNUNET_DISK_OPEN_READ, | ||
986 | GNUNET_DISK_PERM_NONE); | ||
987 | GNUNET_assert (NULL != fh); | ||
988 | GNUNET_assert (GNUNET_OK == GNUNET_DISK_file_handle_size (fh, &file_size)); | ||
989 | num_peers = file_size / 53; | ||
990 | buf = GNUNET_malloc (file_size); | ||
991 | size_read = GNUNET_DISK_file_read (fh, buf, file_size); | ||
992 | GNUNET_assert (size_read == file_size); | ||
993 | LOG (GNUNET_ERROR_TYPE_DEBUG, | ||
994 | "Restoring %" PRIu32 " peers from file `%s'\n", | ||
995 | num_peers, | ||
996 | filename_valid_peers); | ||
997 | for (iter_buf = buf; iter_buf < buf + file_size - 1; iter_buf += 53) | ||
998 | { | ||
999 | str_repr = GNUNET_strndup (iter_buf, 53); | ||
1000 | peer = s2i_full (str_repr); | ||
1001 | GNUNET_free (str_repr); | ||
1002 | add_valid_peer (peer); | ||
1003 | LOG (GNUNET_ERROR_TYPE_DEBUG, | ||
1004 | "Restored valid peer %s from disk\n", | ||
1005 | GNUNET_i2s_full (peer)); | ||
1006 | } | ||
1007 | iter_buf = NULL; | ||
1008 | GNUNET_free (buf); | ||
1009 | LOG (GNUNET_ERROR_TYPE_DEBUG, | ||
1010 | "num_peers: %" PRIu32 ", _size (valid_peers): %u\n", | ||
1011 | num_peers, | ||
1012 | GNUNET_CONTAINER_multipeermap_size (valid_peers)); | ||
1013 | if (num_peers != GNUNET_CONTAINER_multipeermap_size (valid_peers)) | ||
1014 | { | ||
1015 | LOG (GNUNET_ERROR_TYPE_WARNING, | ||
1016 | "Number of restored peers does not match file size. Have probably duplicates.\n"); | ||
1017 | } | ||
1018 | GNUNET_assert (GNUNET_OK == GNUNET_DISK_file_close (fh)); | ||
1019 | LOG (GNUNET_ERROR_TYPE_DEBUG, | ||
1020 | "Restored %u valid peers from disk\n", | ||
1021 | GNUNET_CONTAINER_multipeermap_size (valid_peers)); | ||
1022 | } | ||
1023 | |||
1024 | |||
1025 | /** | ||
1026 | * @brief Initialise storage of peers | ||
1027 | * | ||
1028 | * @param fn_valid_peers filename of the file used to store valid peer ids | ||
1029 | * @param cadet_h cadet handle | ||
1030 | * @param disconnect_handler Disconnect handler | ||
1031 | * @param own_id own peer identity | ||
1032 | */ | ||
1033 | void | ||
1034 | Peers_initialise (char* fn_valid_peers, | ||
1035 | struct GNUNET_CADET_Handle *cadet_h, | ||
1036 | GNUNET_CADET_DisconnectEventHandler disconnect_handler, | ||
1037 | const struct GNUNET_PeerIdentity *own_id) | ||
1038 | { | ||
1039 | filename_valid_peers = GNUNET_strdup (fn_valid_peers); | ||
1040 | cadet_handle = cadet_h; | ||
1041 | own_identity = *own_id; | ||
1042 | peer_map = GNUNET_CONTAINER_multipeermap_create (4, GNUNET_NO); | ||
1043 | valid_peers = GNUNET_CONTAINER_multipeermap_create (4, GNUNET_NO); | ||
1044 | restore_valid_peers (); | ||
1045 | } | ||
1046 | |||
1047 | |||
1048 | /** | ||
1049 | * @brief Delete storage of peers that was created with #Peers_initialise () | ||
1050 | */ | ||
1051 | void | ||
1052 | Peers_terminate () | ||
1053 | { | ||
1054 | if (GNUNET_SYSERR == | ||
1055 | GNUNET_CONTAINER_multipeermap_iterate (peer_map, | ||
1056 | peermap_clear_iterator, | ||
1057 | NULL)) | ||
1058 | { | ||
1059 | LOG (GNUNET_ERROR_TYPE_WARNING, | ||
1060 | "Iteration destroying peers was aborted.\n"); | ||
1061 | } | ||
1062 | GNUNET_CONTAINER_multipeermap_destroy (peer_map); | ||
1063 | store_valid_peers (); | ||
1064 | GNUNET_free (filename_valid_peers); | ||
1065 | GNUNET_CONTAINER_multipeermap_destroy (valid_peers); | ||
1066 | } | ||
1067 | |||
1068 | |||
1069 | /** | ||
1070 | * Iterator over #valid_peers hash map entries. | ||
1071 | * | ||
1072 | * @param cls closure - unused | ||
1073 | * @param peer current peer id | ||
1074 | * @param value value in the hash map - unused | ||
1075 | * @return #GNUNET_YES if we should continue to | ||
1076 | * iterate, | ||
1077 | * #GNUNET_NO if not. | ||
1078 | */ | ||
1079 | static int | ||
1080 | valid_peer_iterator (void *cls, | ||
1081 | const struct GNUNET_PeerIdentity *peer, | ||
1082 | void *value) | ||
1083 | { | ||
1084 | struct PeersIteratorCls *it_cls = cls; | ||
1085 | |||
1086 | return it_cls->iterator (it_cls->cls, | ||
1087 | peer); | ||
1088 | } | ||
1089 | |||
1090 | |||
1091 | /** | ||
1092 | * @brief Get all currently known, valid peer ids. | ||
1093 | * | ||
1094 | * @param it function to call on each peer id | ||
1095 | * @param it_cls extra argument to @a it | ||
1096 | * @return the number of key value pairs processed, | ||
1097 | * #GNUNET_SYSERR if it aborted iteration | ||
1098 | */ | ||
1099 | int | ||
1100 | Peers_get_valid_peers (PeersIterator iterator, | ||
1101 | void *it_cls) | ||
1102 | { | ||
1103 | struct PeersIteratorCls *cls; | ||
1104 | int ret; | ||
1105 | |||
1106 | cls = GNUNET_new (struct PeersIteratorCls); | ||
1107 | cls->iterator = iterator; | ||
1108 | cls->cls = it_cls; | ||
1109 | ret = GNUNET_CONTAINER_multipeermap_iterate (valid_peers, | ||
1110 | valid_peer_iterator, | ||
1111 | cls); | ||
1112 | GNUNET_free (cls); | ||
1113 | return ret; | ||
1114 | } | ||
1115 | |||
1116 | |||
1117 | /** | ||
1118 | * @brief Add peer to known peers. | ||
1119 | * | ||
1120 | * This function is called on new peer_ids from 'external' sources | ||
1121 | * (client seed, cadet get_peers(), ...) | ||
1122 | * | ||
1123 | * @param peer the new #GNUNET_PeerIdentity | ||
1124 | * | ||
1125 | * @return #GNUNET_YES if peer was inserted | ||
1126 | * #GNUNET_NO otherwise (if peer was already known or | ||
1127 | * peer was #own_identity) | ||
1128 | */ | ||
1129 | int | ||
1130 | Peers_insert_peer (const struct GNUNET_PeerIdentity *peer) | ||
1131 | { | ||
1132 | if ( (GNUNET_YES == Peers_check_peer_known (peer)) || | ||
1133 | (0 == GNUNET_CRYPTO_cmp_peer_identity (peer, &own_identity)) ) | ||
1134 | { | ||
1135 | return GNUNET_NO; /* We already know this peer - nothing to do */ | ||
1136 | } | ||
1137 | (void) create_peer_ctx (peer); | ||
1138 | return GNUNET_YES; | ||
1139 | } | ||
1140 | |||
1141 | int | ||
1142 | Peers_check_peer_flag (const struct GNUNET_PeerIdentity *peer, enum Peers_PeerFlags flags); | ||
1143 | |||
1144 | /** | ||
1145 | * @brief Try connecting to a peer to see whether it is online | ||
1146 | * | ||
1147 | * If not known yet, insert into known peers | ||
1148 | * | ||
1149 | * @param peer the peer whose liveliness is to be checked | ||
1150 | * @return #GNUNET_YES if peer had to be inserted | ||
1151 | * #GNUNET_NO otherwise (if peer was already known or | ||
1152 | * peer was #own_identity) | ||
1153 | */ | ||
1154 | int | ||
1155 | Peers_issue_peer_liveliness_check (const struct GNUNET_PeerIdentity *peer) | ||
1156 | { | ||
1157 | struct PeerContext *peer_ctx; | ||
1158 | int ret; | ||
1159 | |||
1160 | if (0 == GNUNET_CRYPTO_cmp_peer_identity (peer, &own_identity)) | ||
1161 | { | ||
1162 | return GNUNET_NO; | ||
1163 | } | ||
1164 | ret = Peers_insert_peer (peer); | ||
1165 | peer_ctx = get_peer_ctx (peer); | ||
1166 | if (GNUNET_NO == Peers_check_peer_flag (peer, Peers_ONLINE)) | ||
1167 | { | ||
1168 | check_peer_live (peer_ctx); | ||
1169 | } | ||
1170 | return ret; | ||
1171 | } | ||
1172 | |||
1173 | |||
1174 | /** | ||
1175 | * @brief Check if peer is removable. | ||
1176 | * | ||
1177 | * Check if | ||
1178 | * - a recv channel exists | ||
1179 | * - there are pending messages | ||
1180 | * - there is no pending pull reply | ||
1181 | * | ||
1182 | * @param peer the peer in question | ||
1183 | * @return #GNUNET_YES if peer is removable | ||
1184 | * #GNUNET_NO if peer is NOT removable | ||
1185 | * #GNUNET_SYSERR if peer is not known | ||
1186 | */ | ||
1187 | int | ||
1188 | Peers_check_removable (const struct GNUNET_PeerIdentity *peer) | ||
1189 | { | ||
1190 | struct PeerContext *peer_ctx; | ||
1191 | |||
1192 | if (GNUNET_NO == GNUNET_CONTAINER_multipeermap_contains (peer_map, peer)) | ||
1193 | { | ||
1194 | return GNUNET_SYSERR; | ||
1195 | } | ||
1196 | |||
1197 | peer_ctx = get_peer_ctx (peer); | ||
1198 | if ( (NULL != peer_ctx->recv_channel) || | ||
1199 | (NULL != peer_ctx->pending_messages_head) || | ||
1200 | (GNUNET_NO == check_peer_flag_set (peer_ctx, Peers_PULL_REPLY_PENDING)) ) | ||
1201 | { | ||
1202 | return GNUNET_NO; | ||
1203 | } | ||
1204 | return GNUNET_YES; | ||
1205 | } | ||
1206 | |||
1207 | uint32_t * | ||
1208 | Peers_get_channel_flag (const struct GNUNET_PeerIdentity *peer, | ||
1209 | enum Peers_ChannelRole role); | ||
1210 | |||
1211 | int | ||
1212 | Peers_check_channel_flag (uint32_t *channel_flags, enum Peers_ChannelFlags flags); | ||
1213 | |||
1214 | /** | ||
1215 | * @brief Remove peer | ||
1216 | * | ||
1217 | * @param peer the peer to clean | ||
1218 | * @return #GNUNET_YES if peer was removed | ||
1219 | * #GNUNET_NO otherwise | ||
1220 | */ | ||
1221 | int | ||
1222 | Peers_remove_peer (const struct GNUNET_PeerIdentity *peer) | ||
1223 | { | ||
1224 | struct PeerContext *peer_ctx; | ||
1225 | uint32_t *channel_flag; | ||
1226 | |||
1227 | if (GNUNET_NO == GNUNET_CONTAINER_multipeermap_contains (peer_map, peer)) | ||
1228 | { | ||
1229 | return GNUNET_NO; | ||
1230 | } | ||
1231 | |||
1232 | peer_ctx = get_peer_ctx (peer); | ||
1233 | set_peer_flag (peer_ctx, Peers_TO_DESTROY); | ||
1234 | LOG (GNUNET_ERROR_TYPE_DEBUG, | ||
1235 | "Going to remove peer %s\n", | ||
1236 | GNUNET_i2s (&peer_ctx->peer_id)); | ||
1237 | Peers_unset_peer_flag (peer, Peers_ONLINE); | ||
1238 | |||
1239 | GNUNET_array_grow (peer_ctx->pending_ops, peer_ctx->num_pending_ops, 0); | ||
1240 | while (NULL != peer_ctx->pending_messages_head) | ||
1241 | { | ||
1242 | LOG (GNUNET_ERROR_TYPE_DEBUG, | ||
1243 | "Removing unsent %s\n", | ||
1244 | peer_ctx->pending_messages_head->type); | ||
1245 | /* Cancle pending message, too */ | ||
1246 | remove_pending_message (peer_ctx->pending_messages_head, GNUNET_YES); | ||
1247 | } | ||
1248 | /* If we are still waiting for notification whether this peer is live | ||
1249 | * cancel the according task */ | ||
1250 | if (NULL != peer_ctx->liveliness_check_pending) | ||
1251 | { | ||
1252 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
1253 | "Removing pending liveliness check for peer %s\n", | ||
1254 | GNUNET_i2s (&peer_ctx->peer_id)); | ||
1255 | // TODO wait until cadet sets mq->cancel_impl | ||
1256 | //GNUNET_MQ_send_cancel (peer_ctx->liveliness_check_pending->ev); | ||
1257 | GNUNET_free (peer_ctx->liveliness_check_pending); | ||
1258 | peer_ctx->liveliness_check_pending = NULL; | ||
1259 | } | ||
1260 | channel_flag = Peers_get_channel_flag (peer, Peers_CHANNEL_ROLE_SENDING); | ||
1261 | if (NULL != peer_ctx->send_channel && | ||
1262 | GNUNET_YES != Peers_check_channel_flag (channel_flag, Peers_CHANNEL_DESTROING)) | ||
1263 | { | ||
1264 | LOG (GNUNET_ERROR_TYPE_DEBUG, | ||
1265 | "Destroying send channel\n"); | ||
1266 | GNUNET_CADET_channel_destroy (peer_ctx->send_channel); | ||
1267 | peer_ctx->send_channel = NULL; | ||
1268 | } | ||
1269 | channel_flag = Peers_get_channel_flag (peer, Peers_CHANNEL_ROLE_RECEIVING); | ||
1270 | if (NULL != peer_ctx->recv_channel && | ||
1271 | GNUNET_YES != Peers_check_channel_flag (channel_flag, Peers_CHANNEL_DESTROING)) | ||
1272 | { | ||
1273 | LOG (GNUNET_ERROR_TYPE_DEBUG, | ||
1274 | "Destroying recv channel\n"); | ||
1275 | GNUNET_CADET_channel_destroy (peer_ctx->recv_channel); | ||
1276 | peer_ctx->recv_channel = NULL; | ||
1277 | } | ||
1278 | |||
1279 | GNUNET_free (peer_ctx->send_channel_flags); | ||
1280 | GNUNET_free (peer_ctx->recv_channel_flags); | ||
1281 | |||
1282 | if (GNUNET_YES != GNUNET_CONTAINER_multipeermap_remove_all (peer_map, &peer_ctx->peer_id)) | ||
1283 | { | ||
1284 | LOG (GNUNET_ERROR_TYPE_WARNING, "removing peer from peer_map failed\n"); | ||
1285 | } | ||
1286 | GNUNET_free (peer_ctx); | ||
1287 | return GNUNET_YES; | ||
1288 | } | ||
1289 | |||
1290 | |||
1291 | /** | ||
1292 | * @brief set flags on a given peer. | ||
1293 | * | ||
1294 | * @param peer the peer to set flags on | ||
1295 | * @param flags the flags | ||
1296 | */ | ||
1297 | void | ||
1298 | Peers_set_peer_flag (const struct GNUNET_PeerIdentity *peer, enum Peers_PeerFlags flags) | ||
1299 | { | ||
1300 | struct PeerContext *peer_ctx; | ||
1301 | |||
1302 | peer_ctx = get_peer_ctx (peer); | ||
1303 | set_peer_flag (peer_ctx, flags); | ||
1304 | } | ||
1305 | |||
1306 | |||
1307 | /** | ||
1308 | * @brief unset flags on a given peer. | ||
1309 | * | ||
1310 | * @param peer the peer to unset flags on | ||
1311 | * @param flags the flags | ||
1312 | */ | ||
1313 | void | ||
1314 | Peers_unset_peer_flag (const struct GNUNET_PeerIdentity *peer, enum Peers_PeerFlags flags) | ||
1315 | { | ||
1316 | struct PeerContext *peer_ctx; | ||
1317 | |||
1318 | peer_ctx = get_peer_ctx (peer); | ||
1319 | unset_peer_flag (peer_ctx, flags); | ||
1320 | } | ||
1321 | |||
1322 | |||
1323 | /** | ||
1324 | * @brief Check whether flags on a peer are set. | ||
1325 | * | ||
1326 | * @param peer the peer to check the flag of | ||
1327 | * @param flags the flags to check | ||
1328 | * | ||
1329 | * @return #GNUNET_SYSERR if peer is not known | ||
1330 | * #GNUNET_YES if all given flags are set | ||
1331 | * #GNUNET_NO otherwise | ||
1332 | */ | ||
1333 | int | ||
1334 | Peers_check_peer_flag (const struct GNUNET_PeerIdentity *peer, enum Peers_PeerFlags flags) | ||
1335 | { | ||
1336 | struct PeerContext *peer_ctx; | ||
1337 | |||
1338 | if (GNUNET_NO == Peers_check_peer_known (peer)) | ||
1339 | { | ||
1340 | return GNUNET_SYSERR; | ||
1341 | } | ||
1342 | peer_ctx = get_peer_ctx (peer); | ||
1343 | return check_peer_flag_set (peer_ctx, flags); | ||
1344 | } | ||
1345 | |||
1346 | |||
1347 | /** | ||
1348 | * @brief set flags on a given channel. | ||
1349 | * | ||
1350 | * @param channel the channel to set flags on | ||
1351 | * @param flags the flags | ||
1352 | */ | ||
1353 | void | ||
1354 | Peers_set_channel_flag (uint32_t *channel_flags, enum Peers_ChannelFlags flags) | ||
1355 | { | ||
1356 | set_channel_flag (channel_flags, flags); | ||
1357 | } | ||
1358 | |||
1359 | |||
1360 | /** | ||
1361 | * @brief unset flags on a given channel. | ||
1362 | * | ||
1363 | * @param channel the channel to unset flags on | ||
1364 | * @param flags the flags | ||
1365 | */ | ||
1366 | void | ||
1367 | Peers_unset_channel_flag (uint32_t *channel_flags, enum Peers_ChannelFlags flags) | ||
1368 | { | ||
1369 | unset_channel_flag (channel_flags, flags); | ||
1370 | } | ||
1371 | |||
1372 | |||
1373 | /** | ||
1374 | * @brief Check whether flags on a channel are set. | ||
1375 | * | ||
1376 | * @param channel the channel to check the flag of | ||
1377 | * @param flags the flags to check | ||
1378 | * | ||
1379 | * @return #GNUNET_YES if all given flags are set | ||
1380 | * #GNUNET_NO otherwise | ||
1381 | */ | ||
1382 | int | ||
1383 | Peers_check_channel_flag (uint32_t *channel_flags, enum Peers_ChannelFlags flags) | ||
1384 | { | ||
1385 | return check_channel_flag_set (channel_flags, flags); | ||
1386 | } | ||
1387 | |||
1388 | /** | ||
1389 | * @brief Get the flags for the channel in @a role for @a peer. | ||
1390 | * | ||
1391 | * @param peer Peer to get the channel flags for. | ||
1392 | * @param role Role of channel to get flags for | ||
1393 | * | ||
1394 | * @return The flags. | ||
1395 | */ | ||
1396 | uint32_t * | ||
1397 | Peers_get_channel_flag (const struct GNUNET_PeerIdentity *peer, | ||
1398 | enum Peers_ChannelRole role) | ||
1399 | { | ||
1400 | const struct PeerContext *peer_ctx; | ||
1401 | |||
1402 | peer_ctx = get_peer_ctx (peer); | ||
1403 | if (Peers_CHANNEL_ROLE_SENDING == role) | ||
1404 | { | ||
1405 | return peer_ctx->send_channel_flags; | ||
1406 | } | ||
1407 | else if (Peers_CHANNEL_ROLE_RECEIVING == role) | ||
1408 | { | ||
1409 | return peer_ctx->recv_channel_flags; | ||
1410 | } | ||
1411 | else | ||
1412 | { | ||
1413 | GNUNET_assert (0); | ||
1414 | } | ||
1415 | } | ||
1416 | |||
1417 | /** | ||
1418 | * @brief Check whether we have information about the given peer. | ||
1419 | * | ||
1420 | * FIXME probably deprecated. Make this the new _online. | ||
1421 | * | ||
1422 | * @param peer peer in question | ||
1423 | * | ||
1424 | * @return #GNUNET_YES if peer is known | ||
1425 | * #GNUNET_NO if peer is not knwon | ||
1426 | */ | ||
1427 | int | ||
1428 | Peers_check_peer_known (const struct GNUNET_PeerIdentity *peer) | ||
1429 | { | ||
1430 | return GNUNET_CONTAINER_multipeermap_contains (peer_map, peer); | ||
1431 | } | ||
1432 | |||
1433 | |||
1434 | /** | ||
1435 | * @brief Check whether @a peer is actually a peer. | ||
1436 | * | ||
1437 | * A valid peer is a peer that we know exists eg. we were connected to once. | ||
1438 | * | ||
1439 | * @param peer peer in question | ||
1440 | * | ||
1441 | * @return #GNUNET_YES if peer is valid | ||
1442 | * #GNUNET_NO if peer is not valid | ||
1443 | */ | ||
1444 | int | ||
1445 | Peers_check_peer_valid (const struct GNUNET_PeerIdentity *peer) | ||
1446 | { | ||
1447 | return GNUNET_CONTAINER_multipeermap_contains (valid_peers, peer); | ||
1448 | } | ||
1449 | |||
1450 | |||
1451 | /** | ||
1452 | * @brief Indicate that we want to send to the other peer | ||
1453 | * | ||
1454 | * This establishes a sending channel | ||
1455 | * | ||
1456 | * @param peer the peer to establish channel to | ||
1457 | */ | ||
1458 | void | ||
1459 | Peers_indicate_sending_intention (const struct GNUNET_PeerIdentity *peer) | ||
1460 | { | ||
1461 | GNUNET_assert (GNUNET_YES == Peers_check_peer_known (peer)); | ||
1462 | (void) get_channel (peer); | ||
1463 | } | ||
1464 | |||
1465 | |||
1466 | /** | ||
1467 | * @brief Check whether other peer has the intention to send/opened channel | ||
1468 | * towars us | ||
1469 | * | ||
1470 | * @param peer the peer in question | ||
1471 | * | ||
1472 | * @return #GNUNET_YES if peer has the intention to send | ||
1473 | * #GNUNET_NO otherwise | ||
1474 | */ | ||
1475 | int | ||
1476 | Peers_check_peer_send_intention (const struct GNUNET_PeerIdentity *peer) | ||
1477 | { | ||
1478 | const struct PeerContext *peer_ctx; | ||
1479 | |||
1480 | peer_ctx = get_peer_ctx (peer); | ||
1481 | if (NULL != peer_ctx->recv_channel) | ||
1482 | { | ||
1483 | return GNUNET_YES; | ||
1484 | } | ||
1485 | return GNUNET_NO; | ||
1486 | } | ||
1487 | |||
1488 | |||
1489 | /** | ||
1490 | * Handle the channel a peer opens to us. | ||
1491 | * | ||
1492 | * @param cls The closure | ||
1493 | * @param channel The channel the peer wants to establish | ||
1494 | * @param initiator The peer's peer ID | ||
1495 | * | ||
1496 | * @return initial channel context for the channel | ||
1497 | * (can be NULL -- that's not an error) | ||
1498 | */ | ||
1499 | void * | ||
1500 | Peers_handle_inbound_channel (void *cls, | ||
1501 | struct GNUNET_CADET_Channel *channel, | ||
1502 | const struct GNUNET_PeerIdentity *initiator) | ||
1503 | { | ||
1504 | struct PeerContext *peer_ctx; | ||
1505 | |||
1506 | LOG (GNUNET_ERROR_TYPE_DEBUG, | ||
1507 | "New channel was established to us (Peer %s).\n", | ||
1508 | GNUNET_i2s (initiator)); | ||
1509 | GNUNET_assert (NULL != channel); /* according to cadet API */ | ||
1510 | /* Make sure we 'know' about this peer */ | ||
1511 | peer_ctx = create_or_get_peer_ctx (initiator); | ||
1512 | set_peer_live (peer_ctx); | ||
1513 | /* We only accept one incoming channel per peer */ | ||
1514 | if (GNUNET_YES == Peers_check_peer_send_intention (initiator)) | ||
1515 | { | ||
1516 | set_channel_flag (peer_ctx->recv_channel_flags, | ||
1517 | Peers_CHANNEL_ESTABLISHED_TWICE); | ||
1518 | GNUNET_CADET_channel_destroy (channel); | ||
1519 | /* return the channel context */ | ||
1520 | return &peer_ctx->peer_id; | ||
1521 | } | ||
1522 | peer_ctx->recv_channel = channel; | ||
1523 | return &peer_ctx->peer_id; | ||
1524 | } | ||
1525 | |||
1526 | |||
1527 | /** | ||
1528 | * @brief Check whether a sending channel towards the given peer exists | ||
1529 | * | ||
1530 | * @param peer the peer to check for | ||
1531 | * | ||
1532 | * @return #GNUNET_YES if a sending channel towards that peer exists | ||
1533 | * #GNUNET_NO otherwise | ||
1534 | */ | ||
1535 | int | ||
1536 | Peers_check_sending_channel_exists (const struct GNUNET_PeerIdentity *peer) | ||
1537 | { | ||
1538 | struct PeerContext *peer_ctx; | ||
1539 | |||
1540 | if (GNUNET_NO == Peers_check_peer_known (peer)) | ||
1541 | { /* If no such peer exists, there is no channel */ | ||
1542 | return GNUNET_NO; | ||
1543 | } | ||
1544 | peer_ctx = get_peer_ctx (peer); | ||
1545 | if (NULL == peer_ctx->send_channel) | ||
1546 | { | ||
1547 | return GNUNET_NO; | ||
1548 | } | ||
1549 | return GNUNET_YES; | ||
1550 | } | ||
1551 | |||
1552 | |||
1553 | /** | ||
1554 | * @brief check whether the given channel is the sending channel of the given | ||
1555 | * peer | ||
1556 | * | ||
1557 | * @param peer the peer in question | ||
1558 | * @param channel the channel to check for | ||
1559 | * @param role either #Peers_CHANNEL_ROLE_SENDING, or | ||
1560 | * #Peers_CHANNEL_ROLE_RECEIVING | ||
1561 | * | ||
1562 | * @return #GNUNET_YES if the given chennel is the sending channel of the peer | ||
1563 | * #GNUNET_NO otherwise | ||
1564 | */ | ||
1565 | int | ||
1566 | Peers_check_channel_role (const struct GNUNET_PeerIdentity *peer, | ||
1567 | const struct GNUNET_CADET_Channel *channel, | ||
1568 | enum Peers_ChannelRole role) | ||
1569 | { | ||
1570 | const struct PeerContext *peer_ctx; | ||
1571 | |||
1572 | if (GNUNET_NO == Peers_check_peer_known (peer)) | ||
1573 | { | ||
1574 | return GNUNET_NO; | ||
1575 | } | ||
1576 | peer_ctx = get_peer_ctx (peer); | ||
1577 | if ( (Peers_CHANNEL_ROLE_SENDING == role) && | ||
1578 | (channel == peer_ctx->send_channel) ) | ||
1579 | { | ||
1580 | return GNUNET_YES; | ||
1581 | } | ||
1582 | if ( (Peers_CHANNEL_ROLE_RECEIVING == role) && | ||
1583 | (channel == peer_ctx->recv_channel) ) | ||
1584 | { | ||
1585 | return GNUNET_YES; | ||
1586 | } | ||
1587 | return GNUNET_NO; | ||
1588 | } | ||
1589 | |||
1590 | |||
1591 | /** | ||
1592 | * @brief Destroy the send channel of a peer e.g. stop indicating a sending | ||
1593 | * intention to another peer | ||
1594 | * | ||
1595 | * If there is also no channel to receive messages from that peer, remove it | ||
1596 | * from the peermap. | ||
1597 | * TODO really? | ||
1598 | * | ||
1599 | * @peer the peer identity of the peer whose sending channel to destroy | ||
1600 | * @return #GNUNET_YES if channel was destroyed | ||
1601 | * #GNUNET_NO otherwise | ||
1602 | */ | ||
1603 | int | ||
1604 | Peers_destroy_sending_channel (const struct GNUNET_PeerIdentity *peer) | ||
1605 | { | ||
1606 | struct PeerContext *peer_ctx; | ||
1607 | |||
1608 | if (GNUNET_NO == Peers_check_peer_known (peer)) | ||
1609 | { | ||
1610 | return GNUNET_NO; | ||
1611 | } | ||
1612 | peer_ctx = get_peer_ctx (peer); | ||
1613 | if (NULL != peer_ctx->send_channel) | ||
1614 | { | ||
1615 | set_channel_flag (peer_ctx->send_channel_flags, Peers_CHANNEL_CLEAN); | ||
1616 | GNUNET_CADET_channel_destroy (peer_ctx->send_channel); | ||
1617 | peer_ctx->send_channel = NULL; | ||
1618 | (void) Peers_check_connected (peer); | ||
1619 | return GNUNET_YES; | ||
1620 | } | ||
1621 | return GNUNET_NO; | ||
1622 | } | ||
1623 | |||
1624 | /** | ||
1625 | * This is called when a channel is destroyed. | ||
1626 | * | ||
1627 | * @param cls The closure | ||
1628 | * @param channel The channel being closed | ||
1629 | * @param channel_ctx The context associated with this channel | ||
1630 | */ | ||
1631 | void | ||
1632 | Peers_cleanup_destroyed_channel (void *cls, | ||
1633 | const struct GNUNET_CADET_Channel *channel) | ||
1634 | { | ||
1635 | struct GNUNET_PeerIdentity *peer = cls; | ||
1636 | struct PeerContext *peer_ctx; | ||
1637 | |||
1638 | if (GNUNET_NO == Peers_check_peer_known (peer)) | ||
1639 | {/* We don't want to implicitly create a context that we're about to kill */ | ||
1640 | LOG (GNUNET_ERROR_TYPE_DEBUG, | ||
1641 | "channel (%s) without associated context was destroyed\n", | ||
1642 | GNUNET_i2s (peer)); | ||
1643 | return; | ||
1644 | } | ||
1645 | peer_ctx = get_peer_ctx (peer); | ||
1646 | |||
1647 | /* If our peer issued the destruction of the channel, the #Peers_TO_DESTROY | ||
1648 | * flag will be set. In this case simply make sure that the channels are | ||
1649 | * cleaned. */ | ||
1650 | /* FIXME This distinction seems to be redundant */ | ||
1651 | if (Peers_check_peer_flag (peer, Peers_TO_DESTROY)) | ||
1652 | {/* We initiatad the destruction of this particular peer */ | ||
1653 | if (channel == peer_ctx->send_channel) | ||
1654 | peer_ctx->send_channel = NULL; | ||
1655 | else if (channel == peer_ctx->recv_channel) | ||
1656 | peer_ctx->recv_channel = NULL; | ||
1657 | |||
1658 | if (NULL != peer_ctx->send_channel) | ||
1659 | { | ||
1660 | GNUNET_CADET_channel_destroy (peer_ctx->send_channel); | ||
1661 | peer_ctx->send_channel = NULL; | ||
1662 | } | ||
1663 | if (NULL != peer_ctx->recv_channel) | ||
1664 | { | ||
1665 | GNUNET_CADET_channel_destroy (peer_ctx->recv_channel); | ||
1666 | peer_ctx->recv_channel = NULL; | ||
1667 | } | ||
1668 | /* Set the #Peers_ONLINE flag accordingly */ | ||
1669 | (void) Peers_check_connected (peer); | ||
1670 | return; | ||
1671 | } | ||
1672 | |||
1673 | else | ||
1674 | { /* We did not initiate the destruction of this peer */ | ||
1675 | if (channel == peer_ctx->send_channel) | ||
1676 | { /* Something (but us) killd the channel - clean up peer */ | ||
1677 | LOG (GNUNET_ERROR_TYPE_DEBUG, | ||
1678 | "send channel (%s) was destroyed - cleaning up\n", | ||
1679 | GNUNET_i2s (peer)); | ||
1680 | peer_ctx->send_channel = NULL; | ||
1681 | } | ||
1682 | else if (channel == peer_ctx->recv_channel) | ||
1683 | { /* Other peer doesn't want to send us messages anymore */ | ||
1684 | LOG (GNUNET_ERROR_TYPE_DEBUG, | ||
1685 | "Peer %s destroyed recv channel - cleaning up channel\n", | ||
1686 | GNUNET_i2s (peer)); | ||
1687 | peer_ctx->recv_channel = NULL; | ||
1688 | } | ||
1689 | else | ||
1690 | { | ||
1691 | LOG (GNUNET_ERROR_TYPE_WARNING, | ||
1692 | "unknown channel (%s) was destroyed\n", | ||
1693 | GNUNET_i2s (peer)); | ||
1694 | } | ||
1695 | } | ||
1696 | (void) Peers_check_connected (peer); | ||
1697 | } | ||
1698 | |||
1699 | /** | ||
1700 | * @brief Send a message to another peer. | ||
1701 | * | ||
1702 | * Keeps track about pending messages so they can be properly removed when the | ||
1703 | * peer is destroyed. | ||
1704 | * | ||
1705 | * @param peer receeiver of the message | ||
1706 | * @param ev envelope of the message | ||
1707 | * @param type type of the message | ||
1708 | */ | ||
1709 | void | ||
1710 | Peers_send_message (const struct GNUNET_PeerIdentity *peer, | ||
1711 | struct GNUNET_MQ_Envelope *ev, | ||
1712 | const char *type) | ||
1713 | { | ||
1714 | struct PendingMessage *pending_msg; | ||
1715 | struct GNUNET_MQ_Handle *mq; | ||
1716 | |||
1717 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
1718 | "Sending message to %s of type %s\n", | ||
1719 | GNUNET_i2s (peer), | ||
1720 | type); | ||
1721 | pending_msg = insert_pending_message (peer, ev, type); | ||
1722 | mq = get_mq (peer); | ||
1723 | GNUNET_MQ_notify_sent (ev, | ||
1724 | mq_notify_sent_cb, | ||
1725 | pending_msg); | ||
1726 | GNUNET_MQ_send (mq, ev); | ||
1727 | } | ||
1728 | |||
1729 | /** | ||
1730 | * @brief Schedule a operation on given peer | ||
1731 | * | ||
1732 | * Avoids scheduling an operation twice. | ||
1733 | * | ||
1734 | * @param peer the peer we want to schedule the operation for once it gets live | ||
1735 | * | ||
1736 | * @return #GNUNET_YES if the operation was scheduled | ||
1737 | * #GNUNET_NO otherwise | ||
1738 | */ | ||
1739 | int | ||
1740 | Peers_schedule_operation (const struct GNUNET_PeerIdentity *peer, | ||
1741 | const PeerOp peer_op) | ||
1742 | { | ||
1743 | struct PeerPendingOp pending_op; | ||
1744 | struct PeerContext *peer_ctx; | ||
1745 | |||
1746 | if (0 == GNUNET_CRYPTO_cmp_peer_identity (peer, &own_identity)) | ||
1747 | { | ||
1748 | return GNUNET_NO; | ||
1749 | } | ||
1750 | GNUNET_assert (GNUNET_YES == Peers_check_peer_known (peer)); | ||
1751 | |||
1752 | //TODO if LIVE/ONLINE execute immediately | ||
1753 | |||
1754 | if (GNUNET_NO == check_operation_scheduled (peer, peer_op)) | ||
1755 | { | ||
1756 | peer_ctx = get_peer_ctx (peer); | ||
1757 | pending_op.op = peer_op; | ||
1758 | pending_op.op_cls = NULL; | ||
1759 | GNUNET_array_append (peer_ctx->pending_ops, | ||
1760 | peer_ctx->num_pending_ops, | ||
1761 | pending_op); | ||
1762 | return GNUNET_YES; | ||
1763 | } | ||
1764 | return GNUNET_NO; | ||
1765 | } | ||
1766 | |||
1767 | /** | ||
1768 | * @brief Get the recv_channel of @a peer. | ||
1769 | * Needed to correctly handle (call #GNUNET_CADET_receive_done()) incoming | ||
1770 | * messages. | ||
1771 | * | ||
1772 | * @param peer The peer to get the recv_channel from. | ||
1773 | * | ||
1774 | * @return The recv_channel. | ||
1775 | */ | ||
1776 | struct GNUNET_CADET_Channel * | ||
1777 | Peers_get_recv_channel (const struct GNUNET_PeerIdentity *peer) | ||
1778 | { | ||
1779 | struct PeerContext *peer_ctx; | ||
1780 | |||
1781 | GNUNET_assert (GNUNET_YES == Peers_check_peer_known (peer)); | ||
1782 | peer_ctx = get_peer_ctx (peer); | ||
1783 | return peer_ctx->recv_channel; | ||
1784 | } | ||
1785 | /*********************************************************************** | ||
1786 | * /Old gnunet-service-rps_peers.c | ||
1787 | ***********************************************************************/ | ||
1788 | |||
1789 | |||
69 | /*********************************************************************** | 1790 | /*********************************************************************** |
70 | * Housekeeping with clients | 1791 | * Housekeeping with clients |
71 | ***********************************************************************/ | 1792 | ***********************************************************************/ |
@@ -847,6 +2568,7 @@ cleanup_destroyed_channel (void *cls, | |||
847 | { | 2568 | { |
848 | struct GNUNET_PeerIdentity *peer = cls; | 2569 | struct GNUNET_PeerIdentity *peer = cls; |
849 | uint32_t *channel_flag; | 2570 | uint32_t *channel_flag; |
2571 | struct PeerContext *peer_ctx; | ||
850 | 2572 | ||
851 | if (GNUNET_NO == Peers_check_peer_known (peer)) | 2573 | if (GNUNET_NO == Peers_check_peer_known (peer)) |
852 | { /* We don't know a context to that peer */ | 2574 | { /* We don't know a context to that peer */ |
@@ -856,6 +2578,15 @@ cleanup_destroyed_channel (void *cls, | |||
856 | return; | 2578 | return; |
857 | } | 2579 | } |
858 | 2580 | ||
2581 | peer_ctx = get_peer_ctx (peer); | ||
2582 | if (GNUNET_YES == Peers_check_channel_role (peer, channel, Peers_CHANNEL_ROLE_RECEIVING)) | ||
2583 | { | ||
2584 | set_channel_flag (peer_ctx->recv_channel_flags, Peers_CHANNEL_DESTROING); | ||
2585 | } else if (GNUNET_YES == Peers_check_channel_role (peer, channel, Peers_CHANNEL_ROLE_SENDING)) | ||
2586 | { | ||
2587 | set_channel_flag (peer_ctx->send_channel_flags, Peers_CHANNEL_DESTROING); | ||
2588 | } | ||
2589 | |||
859 | if (GNUNET_YES == Peers_check_peer_flag (peer, Peers_TO_DESTROY)) | 2590 | if (GNUNET_YES == Peers_check_peer_flag (peer, Peers_TO_DESTROY)) |
860 | { /* We are in the middle of removing that peer from our knowledge. In this | 2591 | { /* We are in the middle of removing that peer from our knowledge. In this |
861 | case simply make sure that the channels are cleaned. */ | 2592 | case simply make sure that the channels are cleaned. */ |
@@ -1121,7 +2852,6 @@ handle_client_request_cancel (void *cls, | |||
1121 | (rep_cls->id != ntohl (msg->id)) ) | 2852 | (rep_cls->id != ntohl (msg->id)) ) |
1122 | rep_cls = rep_cls->next; | 2853 | rep_cls = rep_cls->next; |
1123 | GNUNET_assert (rep_cls->id == ntohl (msg->id)); | 2854 | GNUNET_assert (rep_cls->id == ntohl (msg->id)); |
1124 | RPS_sampler_request_cancel (rep_cls->req_handle); | ||
1125 | destroy_reply_cls (rep_cls); | 2855 | destroy_reply_cls (rep_cls); |
1126 | GNUNET_SERVICE_client_continue (cli_ctx->client); | 2856 | GNUNET_SERVICE_client_continue (cli_ctx->client); |
1127 | } | 2857 | } |
@@ -1239,24 +2969,24 @@ handle_peer_push (void *cls, | |||
1239 | (3 == mal_type) ) | 2969 | (3 == mal_type) ) |
1240 | { /* Try to maximise representation */ | 2970 | { /* Try to maximise representation */ |
1241 | tmp_att_peer = GNUNET_new (struct AttackedPeer); | 2971 | tmp_att_peer = GNUNET_new (struct AttackedPeer); |
1242 | GNUNET_memcpy (&tmp_att_peer->peer_id, peer, sizeof (struct GNUNET_PeerIdentity)); | 2972 | tmp_att_peer->peer_id = *peer; |
1243 | if (NULL == att_peer_set) | 2973 | if (NULL == att_peer_set) |
1244 | att_peer_set = GNUNET_CONTAINER_multipeermap_create (1, GNUNET_NO); | 2974 | att_peer_set = GNUNET_CONTAINER_multipeermap_create (1, GNUNET_NO); |
1245 | if (GNUNET_NO == GNUNET_CONTAINER_multipeermap_contains (att_peer_set, | 2975 | if (GNUNET_NO == |
1246 | peer)) | 2976 | GNUNET_CONTAINER_multipeermap_contains (att_peer_set, |
2977 | peer)) | ||
1247 | { | 2978 | { |
1248 | GNUNET_CONTAINER_DLL_insert (att_peers_head, | 2979 | GNUNET_CONTAINER_DLL_insert (att_peers_head, |
1249 | att_peers_tail, | 2980 | att_peers_tail, |
1250 | tmp_att_peer); | 2981 | tmp_att_peer); |
1251 | add_peer_array_to_set (peer, 1, att_peer_set); | 2982 | add_peer_array_to_set (peer, 1, att_peer_set); |
1252 | } | 2983 | } |
1253 | GNUNET_CADET_receive_done (Peers_get_recv_channel (peer)); | ||
1254 | } | 2984 | } |
1255 | 2985 | ||
1256 | 2986 | ||
1257 | else if (2 == mal_type) | 2987 | else if (2 == mal_type) |
1258 | { /* We attack one single well-known peer - simply ignore */ | 2988 | { |
1259 | GNUNET_CADET_receive_done (Peers_get_recv_channel (peer)); | 2989 | /* We attack one single well-known peer - simply ignore */ |
1260 | } | 2990 | } |
1261 | #endif /* ENABLE_MALICIOUS */ | 2991 | #endif /* ENABLE_MALICIOUS */ |
1262 | 2992 | ||
@@ -1289,7 +3019,6 @@ handle_peer_pull_request (void *cls, | |||
1289 | || 3 == mal_type) | 3019 | || 3 == mal_type) |
1290 | { /* Try to maximise representation */ | 3020 | { /* Try to maximise representation */ |
1291 | send_pull_reply (peer, mal_peers, num_mal_peers); | 3021 | send_pull_reply (peer, mal_peers, num_mal_peers); |
1292 | GNUNET_CADET_receive_done (Peers_get_recv_channel (peer)); | ||
1293 | } | 3022 | } |
1294 | 3023 | ||
1295 | else if (2 == mal_type) | 3024 | else if (2 == mal_type) |
@@ -1298,7 +3027,6 @@ handle_peer_pull_request (void *cls, | |||
1298 | { | 3027 | { |
1299 | send_pull_reply (peer, mal_peers, num_mal_peers); | 3028 | send_pull_reply (peer, mal_peers, num_mal_peers); |
1300 | } | 3029 | } |
1301 | GNUNET_CADET_receive_done (Peers_get_recv_channel (peer)); | ||
1302 | } | 3030 | } |
1303 | #endif /* ENABLE_MALICIOUS */ | 3031 | #endif /* ENABLE_MALICIOUS */ |
1304 | 3032 | ||
@@ -1360,7 +3088,7 @@ static void | |||
1360 | handle_peer_pull_reply (void *cls, | 3088 | handle_peer_pull_reply (void *cls, |
1361 | const struct GNUNET_RPS_P2P_PullReplyMessage *msg) | 3089 | const struct GNUNET_RPS_P2P_PullReplyMessage *msg) |
1362 | { | 3090 | { |
1363 | struct GNUNET_PeerIdentity *peers; | 3091 | const struct GNUNET_PeerIdentity *peers; |
1364 | struct GNUNET_PeerIdentity *sender = cls; | 3092 | struct GNUNET_PeerIdentity *sender = cls; |
1365 | uint32_t i; | 3093 | uint32_t i; |
1366 | #ifdef ENABLE_MALICIOUS | 3094 | #ifdef ENABLE_MALICIOUS |
@@ -1373,12 +3101,11 @@ handle_peer_pull_reply (void *cls, | |||
1373 | // We shouldn't even receive pull replies as we're not sending | 3101 | // We shouldn't even receive pull replies as we're not sending |
1374 | if (2 == mal_type) | 3102 | if (2 == mal_type) |
1375 | { | 3103 | { |
1376 | GNUNET_CADET_receive_done (Peers_get_recv_channel (sender)); | ||
1377 | } | 3104 | } |
1378 | #endif /* ENABLE_MALICIOUS */ | 3105 | #endif /* ENABLE_MALICIOUS */ |
1379 | 3106 | ||
1380 | /* Do actual logic */ | 3107 | /* Do actual logic */ |
1381 | peers = (struct GNUNET_PeerIdentity *) &msg[1]; | 3108 | peers = (const struct GNUNET_PeerIdentity *) &msg[1]; |
1382 | 3109 | ||
1383 | LOG (GNUNET_ERROR_TYPE_DEBUG, | 3110 | LOG (GNUNET_ERROR_TYPE_DEBUG, |
1384 | "PULL REPLY received, got following %u peers:\n", | 3111 | "PULL REPLY received, got following %u peers:\n", |
@@ -2132,7 +3859,6 @@ shutdown_task (void *cls) | |||
2132 | reply_cls); | 3859 | reply_cls); |
2133 | GNUNET_free (reply_cls); | 3860 | GNUNET_free (reply_cls); |
2134 | } | 3861 | } |
2135 | GNUNET_MQ_destroy (client_ctx->mq); | ||
2136 | GNUNET_CONTAINER_DLL_remove (cli_ctx_head, cli_ctx_tail, client_ctx); | 3862 | GNUNET_CONTAINER_DLL_remove (cli_ctx_head, cli_ctx_tail, client_ctx); |
2137 | GNUNET_free (client_ctx); | 3863 | GNUNET_free (client_ctx); |
2138 | } | 3864 | } |
@@ -2150,6 +3876,7 @@ shutdown_task (void *cls) | |||
2150 | GNUNET_NSE_disconnect (nse); | 3876 | GNUNET_NSE_disconnect (nse); |
2151 | RPS_sampler_destroy (prot_sampler); | 3877 | RPS_sampler_destroy (prot_sampler); |
2152 | RPS_sampler_destroy (client_sampler); | 3878 | RPS_sampler_destroy (client_sampler); |
3879 | GNUNET_CADET_close_port (cadet_port); | ||
2153 | GNUNET_CADET_disconnect (cadet_handle); | 3880 | GNUNET_CADET_disconnect (cadet_handle); |
2154 | View_destroy (); | 3881 | View_destroy (); |
2155 | CustomPeerMap_destroy (push_map); | 3882 | CustomPeerMap_destroy (push_map); |
@@ -2240,26 +3967,6 @@ run (void *cls, | |||
2240 | const struct GNUNET_CONFIGURATION_Handle *c, | 3967 | const struct GNUNET_CONFIGURATION_Handle *c, |
2241 | struct GNUNET_SERVICE_Handle *service) | 3968 | struct GNUNET_SERVICE_Handle *service) |
2242 | { | 3969 | { |
2243 | struct GNUNET_MQ_MessageHandler cadet_handlers[] = { | ||
2244 | GNUNET_MQ_hd_fixed_size (peer_check, | ||
2245 | GNUNET_MESSAGE_TYPE_RPS_PP_CHECK_LIVE, | ||
2246 | struct GNUNET_MessageHeader, | ||
2247 | NULL), | ||
2248 | GNUNET_MQ_hd_fixed_size (peer_push, | ||
2249 | GNUNET_MESSAGE_TYPE_RPS_PP_PUSH, | ||
2250 | struct GNUNET_MessageHeader, | ||
2251 | NULL), | ||
2252 | GNUNET_MQ_hd_fixed_size (peer_pull_request, | ||
2253 | GNUNET_MESSAGE_TYPE_RPS_PP_PULL_REQUEST, | ||
2254 | struct GNUNET_MessageHeader, | ||
2255 | NULL), | ||
2256 | GNUNET_MQ_hd_var_size (peer_pull_reply, | ||
2257 | GNUNET_MESSAGE_TYPE_RPS_PP_PULL_REPLY, | ||
2258 | struct GNUNET_RPS_P2P_PullReplyMessage, | ||
2259 | NULL), | ||
2260 | GNUNET_MQ_handler_end () | ||
2261 | }; | ||
2262 | |||
2263 | int size; | 3970 | int size; |
2264 | int out_size; | 3971 | int out_size; |
2265 | char* fn_valid_peers; | 3972 | char* fn_valid_peers; |
@@ -2349,6 +4056,27 @@ run (void *cls, | |||
2349 | 4056 | ||
2350 | 4057 | ||
2351 | /* Initialise cadet */ | 4058 | /* Initialise cadet */ |
4059 | /* There exists a copy-paste-clone in get_channel() */ | ||
4060 | struct GNUNET_MQ_MessageHandler cadet_handlers[] = { | ||
4061 | GNUNET_MQ_hd_fixed_size (peer_check, | ||
4062 | GNUNET_MESSAGE_TYPE_RPS_PP_CHECK_LIVE, | ||
4063 | struct GNUNET_MessageHeader, | ||
4064 | NULL), | ||
4065 | GNUNET_MQ_hd_fixed_size (peer_push, | ||
4066 | GNUNET_MESSAGE_TYPE_RPS_PP_PUSH, | ||
4067 | struct GNUNET_MessageHeader, | ||
4068 | NULL), | ||
4069 | GNUNET_MQ_hd_fixed_size (peer_pull_request, | ||
4070 | GNUNET_MESSAGE_TYPE_RPS_PP_PULL_REQUEST, | ||
4071 | struct GNUNET_MessageHeader, | ||
4072 | NULL), | ||
4073 | GNUNET_MQ_hd_var_size (peer_pull_reply, | ||
4074 | GNUNET_MESSAGE_TYPE_RPS_PP_PULL_REPLY, | ||
4075 | struct GNUNET_RPS_P2P_PullReplyMessage, | ||
4076 | NULL), | ||
4077 | GNUNET_MQ_handler_end () | ||
4078 | }; | ||
4079 | |||
2352 | cadet_handle = GNUNET_CADET_connect (cfg); | 4080 | cadet_handle = GNUNET_CADET_connect (cfg); |
2353 | GNUNET_assert (NULL != cadet_handle); | 4081 | GNUNET_assert (NULL != cadet_handle); |
2354 | GNUNET_CRYPTO_hash (GNUNET_APPLICATION_PORT_RPS, | 4082 | GNUNET_CRYPTO_hash (GNUNET_APPLICATION_PORT_RPS, |
@@ -2365,7 +4093,7 @@ run (void *cls, | |||
2365 | 4093 | ||
2366 | peerinfo_handle = GNUNET_PEERINFO_connect (cfg); | 4094 | peerinfo_handle = GNUNET_PEERINFO_connect (cfg); |
2367 | Peers_initialise (fn_valid_peers, cadet_handle, cleanup_destroyed_channel, | 4095 | Peers_initialise (fn_valid_peers, cadet_handle, cleanup_destroyed_channel, |
2368 | cadet_handlers, &own_identity); | 4096 | &own_identity); |
2369 | GNUNET_free (fn_valid_peers); | 4097 | GNUNET_free (fn_valid_peers); |
2370 | 4098 | ||
2371 | /* Initialise sampler */ | 4099 | /* Initialise sampler */ |
diff --git a/src/rps/gnunet-service-rps_peers.c b/src/rps/gnunet-service-rps_peers.c deleted file mode 100644 index 58aa84ccf..000000000 --- a/src/rps/gnunet-service-rps_peers.c +++ /dev/null | |||
@@ -1,1694 +0,0 @@ | |||
1 | /* | ||
2 | This file is part of GNUnet. | ||
3 | Copyright (C) | ||
4 | |||
5 | GNUnet is free software; you can redistribute it and/or modify | ||
6 | it under the terms of the GNU General Public License as published | ||
7 | by the Free Software Foundation; either version 3, or (at your | ||
8 | option) any later version. | ||
9 | |||
10 | GNUnet is distributed in the hope that it will be useful, but | ||
11 | WITHOUT ANY WARRANTY; without even the implied warranty of | ||
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
13 | General Public License for more details. | ||
14 | |||
15 | You should have received a copy of the GNU General Public License | ||
16 | along with GNUnet; see the file COPYING. If not, write to the | ||
17 | Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, | ||
18 | Boston, MA 02110-1301, USA. | ||
19 | */ | ||
20 | |||
21 | /** | ||
22 | * @file rps/gnunet-service-rps_peers.c | ||
23 | * @brief utilities for managing (information about) peers | ||
24 | * @author Julius Bünger | ||
25 | */ | ||
26 | #include "platform.h" | ||
27 | #include "gnunet_applications.h" | ||
28 | #include "gnunet_util_lib.h" | ||
29 | #include "gnunet_cadet_service.h" | ||
30 | #include <inttypes.h> | ||
31 | #include "rps.h" | ||
32 | #include "gnunet-service-rps_peers.h" | ||
33 | |||
34 | |||
35 | |||
36 | #define LOG(kind, ...) GNUNET_log_from(kind,"rps-peers",__VA_ARGS__) | ||
37 | |||
38 | |||
39 | /** | ||
40 | * Set a peer flag of given peer context. | ||
41 | */ | ||
42 | #define set_peer_flag(peer_ctx, mask) ((peer_ctx->peer_flags) |= (mask)) | ||
43 | |||
44 | /** | ||
45 | * Get peer flag of given peer context. | ||
46 | */ | ||
47 | #define check_peer_flag_set(peer_ctx, mask)\ | ||
48 | ((peer_ctx->peer_flags) & (mask) ? GNUNET_YES : GNUNET_NO) | ||
49 | |||
50 | /** | ||
51 | * Unset flag of given peer context. | ||
52 | */ | ||
53 | #define unset_peer_flag(peer_ctx, mask) ((peer_ctx->peer_flags) &= ~(mask)) | ||
54 | |||
55 | /** | ||
56 | * Set a channel flag of given channel context. | ||
57 | */ | ||
58 | #define set_channel_flag(channel_flags, mask) ((*channel_flags) |= (mask)) | ||
59 | |||
60 | /** | ||
61 | * Get channel flag of given channel context. | ||
62 | */ | ||
63 | #define check_channel_flag_set(channel_flags, mask)\ | ||
64 | ((*channel_flags) & (mask) ? GNUNET_YES : GNUNET_NO) | ||
65 | |||
66 | /** | ||
67 | * Unset flag of given channel context. | ||
68 | */ | ||
69 | #define unset_channel_flag(channel_flags, mask) ((*channel_flags) &= ~(mask)) | ||
70 | |||
71 | |||
72 | |||
73 | /** | ||
74 | * Pending operation on peer consisting of callback and closure | ||
75 | * | ||
76 | * When an operation cannot be executed right now this struct is used to store | ||
77 | * the callback and closure for later execution. | ||
78 | */ | ||
79 | struct PeerPendingOp | ||
80 | { | ||
81 | /** | ||
82 | * Callback | ||
83 | */ | ||
84 | PeerOp op; | ||
85 | |||
86 | /** | ||
87 | * Closure | ||
88 | */ | ||
89 | void *op_cls; | ||
90 | }; | ||
91 | |||
92 | /** | ||
93 | * List containing all messages that are yet to be send | ||
94 | * | ||
95 | * This is used to keep track of all messages that have not been sent yet. When | ||
96 | * a peer is to be removed the pending messages can be removed properly. | ||
97 | */ | ||
98 | struct PendingMessage | ||
99 | { | ||
100 | /** | ||
101 | * DLL next, prev | ||
102 | */ | ||
103 | struct PendingMessage *next; | ||
104 | struct PendingMessage *prev; | ||
105 | |||
106 | /** | ||
107 | * The envelope to the corresponding message | ||
108 | */ | ||
109 | struct GNUNET_MQ_Envelope *ev; | ||
110 | |||
111 | /** | ||
112 | * The corresponding context | ||
113 | */ | ||
114 | struct PeerContext *peer_ctx; | ||
115 | |||
116 | /** | ||
117 | * The message type | ||
118 | */ | ||
119 | const char *type; | ||
120 | }; | ||
121 | |||
122 | /** | ||
123 | * Struct used to keep track of other peer's status | ||
124 | * | ||
125 | * This is stored in a multipeermap. | ||
126 | * It contains information such as cadet channels, a message queue for sending, | ||
127 | * status about the channels, the pending operations on this peer and some flags | ||
128 | * about the status of the peer itself. (live, valid, ...) | ||
129 | */ | ||
130 | struct PeerContext | ||
131 | { | ||
132 | /** | ||
133 | * Message queue open to client | ||
134 | */ | ||
135 | struct GNUNET_MQ_Handle *mq; | ||
136 | |||
137 | /** | ||
138 | * Channel open to client. | ||
139 | */ | ||
140 | struct GNUNET_CADET_Channel *send_channel; | ||
141 | |||
142 | /** | ||
143 | * Flags to the sending channel | ||
144 | */ | ||
145 | uint32_t *send_channel_flags; | ||
146 | |||
147 | /** | ||
148 | * Channel open from client. | ||
149 | */ | ||
150 | struct GNUNET_CADET_Channel *recv_channel; // unneeded? | ||
151 | |||
152 | /** | ||
153 | * Flags to the receiving channel | ||
154 | */ | ||
155 | uint32_t *recv_channel_flags; | ||
156 | |||
157 | /** | ||
158 | * Array of pending operations on this peer. | ||
159 | */ | ||
160 | struct PeerPendingOp *pending_ops; | ||
161 | |||
162 | /** | ||
163 | * Handle to the callback given to cadet_ntfy_tmt_rdy() | ||
164 | * | ||
165 | * To be canceled on shutdown. | ||
166 | */ | ||
167 | struct PendingMessage *liveliness_check_pending; | ||
168 | |||
169 | /** | ||
170 | * Number of pending operations. | ||
171 | */ | ||
172 | unsigned int num_pending_ops; | ||
173 | |||
174 | /** | ||
175 | * Identity of the peer | ||
176 | */ | ||
177 | struct GNUNET_PeerIdentity peer_id; | ||
178 | |||
179 | /** | ||
180 | * Flags indicating status of peer | ||
181 | */ | ||
182 | uint32_t peer_flags; | ||
183 | |||
184 | /** | ||
185 | * Last time we received something from that peer. | ||
186 | */ | ||
187 | struct GNUNET_TIME_Absolute last_message_recv; | ||
188 | |||
189 | /** | ||
190 | * Last time we received a keepalive message. | ||
191 | */ | ||
192 | struct GNUNET_TIME_Absolute last_keepalive; | ||
193 | |||
194 | /** | ||
195 | * DLL with all messages that are yet to be sent | ||
196 | */ | ||
197 | struct PendingMessage *pending_messages_head; | ||
198 | struct PendingMessage *pending_messages_tail; | ||
199 | |||
200 | /** | ||
201 | * This is pobably followed by 'statistical' data (when we first saw | ||
202 | * him, how did we get his ID, how many pushes (in a timeinterval), | ||
203 | * ...) | ||
204 | */ | ||
205 | }; | ||
206 | |||
207 | /** | ||
208 | * @brief Closure to #valid_peer_iterator | ||
209 | */ | ||
210 | struct PeersIteratorCls | ||
211 | { | ||
212 | /** | ||
213 | * Iterator function | ||
214 | */ | ||
215 | PeersIterator iterator; | ||
216 | |||
217 | /** | ||
218 | * Closure to iterator | ||
219 | */ | ||
220 | void *cls; | ||
221 | }; | ||
222 | |||
223 | /** | ||
224 | * @brief Hashmap of valid peers. | ||
225 | */ | ||
226 | static struct GNUNET_CONTAINER_MultiPeerMap *valid_peers; | ||
227 | |||
228 | /** | ||
229 | * @brief Maximum number of valid peers to keep. | ||
230 | * TODO read from config | ||
231 | */ | ||
232 | static uint32_t num_valid_peers_max = UINT32_MAX; | ||
233 | |||
234 | /** | ||
235 | * @brief Filename of the file that stores the valid peers persistently. | ||
236 | */ | ||
237 | static char *filename_valid_peers; | ||
238 | |||
239 | /** | ||
240 | * Set of all peers to keep track of them. | ||
241 | */ | ||
242 | static struct GNUNET_CONTAINER_MultiPeerMap *peer_map; | ||
243 | |||
244 | /** | ||
245 | * Own #GNUNET_PeerIdentity. | ||
246 | */ | ||
247 | static const struct GNUNET_PeerIdentity *own_identity; | ||
248 | |||
249 | /** | ||
250 | * Cadet handle. | ||
251 | */ | ||
252 | static struct GNUNET_CADET_Handle *cadet_handle; | ||
253 | |||
254 | /** | ||
255 | * @brief Disconnect handler | ||
256 | */ | ||
257 | static GNUNET_CADET_DisconnectEventHandler cleanup_destroyed_channel; | ||
258 | |||
259 | /** | ||
260 | * @brief cadet handlers | ||
261 | */ | ||
262 | static const struct GNUNET_MQ_MessageHandler *cadet_handlers; | ||
263 | |||
264 | |||
265 | |||
266 | /** | ||
267 | * @brief Get the #PeerContext associated with a peer | ||
268 | * | ||
269 | * @param peer the peer id | ||
270 | * | ||
271 | * @return the #PeerContext | ||
272 | */ | ||
273 | static struct PeerContext * | ||
274 | get_peer_ctx (const struct GNUNET_PeerIdentity *peer) | ||
275 | { | ||
276 | struct PeerContext *ctx; | ||
277 | int ret; | ||
278 | |||
279 | ret = GNUNET_CONTAINER_multipeermap_contains (peer_map, peer); | ||
280 | GNUNET_assert (GNUNET_YES == ret); | ||
281 | ctx = GNUNET_CONTAINER_multipeermap_get (peer_map, peer); | ||
282 | GNUNET_assert (NULL != ctx); | ||
283 | return ctx; | ||
284 | } | ||
285 | |||
286 | |||
287 | /** | ||
288 | * @brief Create a new #PeerContext and insert it into the peer map | ||
289 | * | ||
290 | * @param peer the peer to create the #PeerContext for | ||
291 | * | ||
292 | * @return the #PeerContext | ||
293 | */ | ||
294 | static struct PeerContext * | ||
295 | create_peer_ctx (const struct GNUNET_PeerIdentity *peer) | ||
296 | { | ||
297 | struct PeerContext *ctx; | ||
298 | int ret; | ||
299 | |||
300 | GNUNET_assert (GNUNET_NO == Peers_check_peer_known (peer)); | ||
301 | |||
302 | ctx = GNUNET_new (struct PeerContext); | ||
303 | ctx->peer_id = *peer; | ||
304 | ctx->send_channel_flags = GNUNET_new (uint32_t); | ||
305 | ctx->recv_channel_flags = GNUNET_new (uint32_t); | ||
306 | ret = GNUNET_CONTAINER_multipeermap_put (peer_map, peer, ctx, | ||
307 | GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY); | ||
308 | GNUNET_assert (GNUNET_OK == ret); | ||
309 | return ctx; | ||
310 | } | ||
311 | |||
312 | |||
313 | /** | ||
314 | * @brief Create or get a #PeerContext | ||
315 | * | ||
316 | * @param peer the peer to get the associated context to | ||
317 | * | ||
318 | * @return the context | ||
319 | */ | ||
320 | static struct PeerContext * | ||
321 | create_or_get_peer_ctx (const struct GNUNET_PeerIdentity *peer) | ||
322 | { | ||
323 | if (GNUNET_NO == Peers_check_peer_known (peer)) | ||
324 | { | ||
325 | return create_peer_ctx (peer); | ||
326 | } | ||
327 | return get_peer_ctx (peer); | ||
328 | } | ||
329 | |||
330 | |||
331 | /** | ||
332 | * @brief Check whether we have a connection to this @a peer | ||
333 | * | ||
334 | * Also sets the #Peers_ONLINE flag accordingly | ||
335 | * | ||
336 | * @param peer the peer in question | ||
337 | * | ||
338 | * @return #GNUNET_YES if we are connected | ||
339 | * #GNUNET_NO otherwise | ||
340 | */ | ||
341 | int | ||
342 | Peers_check_connected (const struct GNUNET_PeerIdentity *peer) | ||
343 | { | ||
344 | const struct PeerContext *peer_ctx; | ||
345 | |||
346 | /* If we don't know about this peer we don't know whether it's online */ | ||
347 | if (GNUNET_NO == Peers_check_peer_known (peer)) | ||
348 | { | ||
349 | return GNUNET_NO; | ||
350 | } | ||
351 | /* Get the context */ | ||
352 | peer_ctx = get_peer_ctx (peer); | ||
353 | /* If we have no channel to this peer we don't know whether it's online */ | ||
354 | if ( (NULL == peer_ctx->send_channel) && | ||
355 | (NULL == peer_ctx->recv_channel) ) | ||
356 | { | ||
357 | Peers_unset_peer_flag (peer, Peers_ONLINE); | ||
358 | return GNUNET_NO; | ||
359 | } | ||
360 | /* Otherwise (if we have a channel, we know that it's online */ | ||
361 | Peers_set_peer_flag (peer, Peers_ONLINE); | ||
362 | return GNUNET_YES; | ||
363 | } | ||
364 | |||
365 | |||
366 | /** | ||
367 | * @brief The closure to #get_rand_peer_iterator. | ||
368 | */ | ||
369 | struct GetRandPeerIteratorCls | ||
370 | { | ||
371 | /** | ||
372 | * @brief The index of the peer to return. | ||
373 | * Will be decreased until 0. | ||
374 | * Then current peer is returned. | ||
375 | */ | ||
376 | uint32_t index; | ||
377 | |||
378 | /** | ||
379 | * @brief Pointer to peer to return. | ||
380 | */ | ||
381 | const struct GNUNET_PeerIdentity *peer; | ||
382 | }; | ||
383 | |||
384 | |||
385 | /** | ||
386 | * @brief Iterator function for #get_random_peer_from_peermap. | ||
387 | * | ||
388 | * Implements #GNUNET_CONTAINER_PeerMapIterator. | ||
389 | * Decreases the index until the index is null. | ||
390 | * Then returns the current peer. | ||
391 | * | ||
392 | * @param cls the #GetRandPeerIteratorCls containing index and peer | ||
393 | * @param peer current peer | ||
394 | * @param value unused | ||
395 | * | ||
396 | * @return #GNUNET_YES if we should continue to | ||
397 | * iterate, | ||
398 | * #GNUNET_NO if not. | ||
399 | */ | ||
400 | static int | ||
401 | get_rand_peer_iterator (void *cls, | ||
402 | const struct GNUNET_PeerIdentity *peer, | ||
403 | void *value) | ||
404 | { | ||
405 | struct GetRandPeerIteratorCls *iterator_cls = cls; | ||
406 | if (0 >= iterator_cls->index) | ||
407 | { | ||
408 | iterator_cls->peer = peer; | ||
409 | return GNUNET_NO; | ||
410 | } | ||
411 | iterator_cls->index--; | ||
412 | return GNUNET_YES; | ||
413 | } | ||
414 | |||
415 | |||
416 | /** | ||
417 | * @brief Get a random peer from @a peer_map | ||
418 | * | ||
419 | * @param peer_map the peer_map to get the peer from | ||
420 | * | ||
421 | * @return a random peer | ||
422 | */ | ||
423 | static const struct GNUNET_PeerIdentity * | ||
424 | get_random_peer_from_peermap (const struct | ||
425 | GNUNET_CONTAINER_MultiPeerMap *peer_map) | ||
426 | { | ||
427 | struct GetRandPeerIteratorCls *iterator_cls; | ||
428 | const struct GNUNET_PeerIdentity *ret; | ||
429 | |||
430 | iterator_cls = GNUNET_new (struct GetRandPeerIteratorCls); | ||
431 | iterator_cls->index = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, | ||
432 | GNUNET_CONTAINER_multipeermap_size (peer_map)); | ||
433 | (void) GNUNET_CONTAINER_multipeermap_iterate (valid_peers, | ||
434 | get_rand_peer_iterator, | ||
435 | iterator_cls); | ||
436 | ret = iterator_cls->peer; | ||
437 | GNUNET_free (iterator_cls); | ||
438 | return ret; | ||
439 | } | ||
440 | |||
441 | |||
442 | /** | ||
443 | * @brief Add a given @a peer to valid peers. | ||
444 | * | ||
445 | * If valid peers are already #num_valid_peers_max, delete a peer previously. | ||
446 | * | ||
447 | * @param peer the peer that is added to the valid peers. | ||
448 | * | ||
449 | * @return #GNUNET_YES if no other peer had to be removed | ||
450 | * #GNUNET_NO otherwise | ||
451 | */ | ||
452 | static int | ||
453 | add_valid_peer (const struct GNUNET_PeerIdentity *peer) | ||
454 | { | ||
455 | const struct GNUNET_PeerIdentity *rand_peer; | ||
456 | int ret; | ||
457 | |||
458 | ret = GNUNET_YES; | ||
459 | while (GNUNET_CONTAINER_multipeermap_size (valid_peers) >= num_valid_peers_max) | ||
460 | { | ||
461 | rand_peer = get_random_peer_from_peermap (valid_peers); | ||
462 | GNUNET_CONTAINER_multipeermap_remove_all (valid_peers, rand_peer); | ||
463 | ret = GNUNET_NO; | ||
464 | } | ||
465 | (void) GNUNET_CONTAINER_multipeermap_put (valid_peers, peer, NULL, | ||
466 | GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY); | ||
467 | return ret; | ||
468 | } | ||
469 | |||
470 | |||
471 | /** | ||
472 | * @brief Set the peer flag to living and | ||
473 | * call the pending operations on this peer. | ||
474 | * | ||
475 | * Also adds peer to #valid_peers. | ||
476 | * | ||
477 | * @param peer_ctx the #PeerContext of the peer to set live | ||
478 | */ | ||
479 | static void | ||
480 | set_peer_live (struct PeerContext *peer_ctx) | ||
481 | { | ||
482 | struct GNUNET_PeerIdentity *peer; | ||
483 | unsigned int i; | ||
484 | |||
485 | peer = &peer_ctx->peer_id; | ||
486 | LOG (GNUNET_ERROR_TYPE_DEBUG, | ||
487 | "Peer %s is live and valid, calling %i pending operations on it\n", | ||
488 | GNUNET_i2s (peer), | ||
489 | peer_ctx->num_pending_ops); | ||
490 | |||
491 | if (NULL != peer_ctx->liveliness_check_pending) | ||
492 | { | ||
493 | LOG (GNUNET_ERROR_TYPE_DEBUG, | ||
494 | "Removing pending liveliness check for peer %s\n", | ||
495 | GNUNET_i2s (&peer_ctx->peer_id)); | ||
496 | // TODO wait until cadet sets mq->cancel_impl | ||
497 | //GNUNET_MQ_send_cancel (peer_ctx->liveliness_check_pending->ev); | ||
498 | GNUNET_free (peer_ctx->liveliness_check_pending); | ||
499 | peer_ctx->liveliness_check_pending = NULL; | ||
500 | } | ||
501 | |||
502 | (void) add_valid_peer (peer); | ||
503 | set_peer_flag (peer_ctx, Peers_ONLINE); | ||
504 | |||
505 | /* Call pending operations */ | ||
506 | for (i = 0; i < peer_ctx->num_pending_ops; i++) | ||
507 | { | ||
508 | peer_ctx->pending_ops[i].op (peer_ctx->pending_ops[i].op_cls, peer); | ||
509 | } | ||
510 | GNUNET_array_grow (peer_ctx->pending_ops, peer_ctx->num_pending_ops, 0); | ||
511 | } | ||
512 | |||
513 | |||
514 | /** | ||
515 | * @brief Get the channel of a peer. If not existing, create. | ||
516 | * | ||
517 | * @param peer the peer id | ||
518 | * @return the #GNUNET_CADET_Channel used to send data to @a peer | ||
519 | */ | ||
520 | struct GNUNET_CADET_Channel * | ||
521 | get_channel (const struct GNUNET_PeerIdentity *peer) | ||
522 | { | ||
523 | struct PeerContext *peer_ctx; | ||
524 | struct GNUNET_HashCode port; | ||
525 | |||
526 | peer_ctx = get_peer_ctx (peer); | ||
527 | if (NULL == peer_ctx->send_channel) | ||
528 | { | ||
529 | LOG (GNUNET_ERROR_TYPE_DEBUG, | ||
530 | "Trying to establish channel to peer %s\n", | ||
531 | GNUNET_i2s (peer)); | ||
532 | GNUNET_CRYPTO_hash (GNUNET_APPLICATION_PORT_RPS, | ||
533 | strlen (GNUNET_APPLICATION_PORT_RPS), | ||
534 | &port); | ||
535 | peer_ctx->send_channel = | ||
536 | GNUNET_CADET_channel_create (cadet_handle, | ||
537 | (struct GNUNET_PeerIdentity *) peer, /* context */ | ||
538 | peer, | ||
539 | &port, | ||
540 | GNUNET_CADET_OPTION_RELIABLE, | ||
541 | NULL, /* WindowSize handler */ | ||
542 | cleanup_destroyed_channel, /* Disconnect handler */ | ||
543 | cadet_handlers); | ||
544 | } | ||
545 | GNUNET_assert (NULL != peer_ctx->send_channel); | ||
546 | return peer_ctx->send_channel; | ||
547 | } | ||
548 | |||
549 | |||
550 | /** | ||
551 | * Get the message queue (#GNUNET_MQ_Handle) of a specific peer. | ||
552 | * | ||
553 | * If we already have a message queue open to this client, | ||
554 | * simply return it, otherways create one. | ||
555 | * | ||
556 | * @param peer the peer to get the mq to | ||
557 | * @return the #GNUNET_MQ_Handle | ||
558 | */ | ||
559 | static struct GNUNET_MQ_Handle * | ||
560 | get_mq (const struct GNUNET_PeerIdentity *peer) | ||
561 | { | ||
562 | struct PeerContext *peer_ctx; | ||
563 | |||
564 | peer_ctx = get_peer_ctx (peer); | ||
565 | |||
566 | if (NULL == peer_ctx->mq) | ||
567 | { | ||
568 | (void) get_channel (peer); | ||
569 | peer_ctx->mq = GNUNET_CADET_get_mq (peer_ctx->send_channel); | ||
570 | } | ||
571 | return peer_ctx->mq; | ||
572 | } | ||
573 | |||
574 | |||
575 | /** | ||
576 | * @brief This is called in response to the first message we sent as a | ||
577 | * liveliness check. | ||
578 | * | ||
579 | * @param cls #PeerContext of peer with pending liveliness check | ||
580 | */ | ||
581 | static void | ||
582 | mq_liveliness_check_successful (void *cls) | ||
583 | { | ||
584 | struct PeerContext *peer_ctx = cls; | ||
585 | |||
586 | if (NULL != peer_ctx->liveliness_check_pending) | ||
587 | { | ||
588 | LOG (GNUNET_ERROR_TYPE_DEBUG, | ||
589 | "Liveliness check for peer %s was successfull\n", | ||
590 | GNUNET_i2s (&peer_ctx->peer_id)); | ||
591 | GNUNET_free (peer_ctx->liveliness_check_pending); | ||
592 | peer_ctx->liveliness_check_pending = NULL; | ||
593 | set_peer_live (peer_ctx); | ||
594 | } | ||
595 | } | ||
596 | |||
597 | /** | ||
598 | * Issue a check whether peer is live | ||
599 | * | ||
600 | * @param peer_ctx the context of the peer | ||
601 | */ | ||
602 | static void | ||
603 | check_peer_live (struct PeerContext *peer_ctx) | ||
604 | { | ||
605 | LOG (GNUNET_ERROR_TYPE_DEBUG, | ||
606 | "Get informed about peer %s getting live\n", | ||
607 | GNUNET_i2s (&peer_ctx->peer_id)); | ||
608 | |||
609 | struct GNUNET_MQ_Handle *mq; | ||
610 | struct GNUNET_MQ_Envelope *ev; | ||
611 | |||
612 | ev = GNUNET_MQ_msg_header (GNUNET_MESSAGE_TYPE_RPS_PP_CHECK_LIVE); | ||
613 | peer_ctx->liveliness_check_pending = GNUNET_new (struct PendingMessage); | ||
614 | peer_ctx->liveliness_check_pending->ev = ev; | ||
615 | peer_ctx->liveliness_check_pending->peer_ctx = peer_ctx; | ||
616 | peer_ctx->liveliness_check_pending->type = "Check liveliness"; | ||
617 | mq = get_mq (&peer_ctx->peer_id); | ||
618 | GNUNET_MQ_notify_sent (ev, | ||
619 | mq_liveliness_check_successful, | ||
620 | peer_ctx); | ||
621 | GNUNET_MQ_send (mq, ev); | ||
622 | } | ||
623 | |||
624 | /** | ||
625 | * @brief Add an envelope to a message passed to mq to list of pending messages | ||
626 | * | ||
627 | * @param peer peer the message was sent to | ||
628 | * @param ev envelope to the message | ||
629 | * @param type type of the message to be sent | ||
630 | * @return pointer to pending message | ||
631 | */ | ||
632 | static struct PendingMessage * | ||
633 | insert_pending_message (const struct GNUNET_PeerIdentity *peer, | ||
634 | struct GNUNET_MQ_Envelope *ev, | ||
635 | const char *type) | ||
636 | { | ||
637 | struct PendingMessage *pending_msg; | ||
638 | struct PeerContext *peer_ctx; | ||
639 | |||
640 | peer_ctx = get_peer_ctx (peer); | ||
641 | pending_msg = GNUNET_new (struct PendingMessage); | ||
642 | pending_msg->ev = ev; | ||
643 | pending_msg->peer_ctx = peer_ctx; | ||
644 | pending_msg->type = type; | ||
645 | GNUNET_CONTAINER_DLL_insert (peer_ctx->pending_messages_head, | ||
646 | peer_ctx->pending_messages_tail, | ||
647 | pending_msg); | ||
648 | return pending_msg; | ||
649 | } | ||
650 | |||
651 | |||
652 | /** | ||
653 | * @brief Remove a pending message from the respective DLL | ||
654 | * | ||
655 | * @param pending_msg the pending message to remove | ||
656 | */ | ||
657 | static void | ||
658 | remove_pending_message (struct PendingMessage *pending_msg) | ||
659 | { | ||
660 | struct PeerContext *peer_ctx; | ||
661 | |||
662 | peer_ctx = pending_msg->peer_ctx; | ||
663 | GNUNET_CONTAINER_DLL_remove (peer_ctx->pending_messages_head, | ||
664 | peer_ctx->pending_messages_tail, | ||
665 | pending_msg); | ||
666 | GNUNET_MQ_send_cancel (peer_ctx->pending_messages_head->ev); | ||
667 | GNUNET_free (pending_msg); | ||
668 | } | ||
669 | |||
670 | |||
671 | /** | ||
672 | * @brief Check whether function of type #PeerOp was already scheduled | ||
673 | * | ||
674 | * The array with pending operations will probably never grow really big, so | ||
675 | * iterating over it should be ok. | ||
676 | * | ||
677 | * @param peer the peer to check | ||
678 | * @param peer_op the operation (#PeerOp) on the peer | ||
679 | * | ||
680 | * @return #GNUNET_YES if this operation is scheduled on that peer | ||
681 | * #GNUNET_NO otherwise | ||
682 | */ | ||
683 | static int | ||
684 | check_operation_scheduled (const struct GNUNET_PeerIdentity *peer, | ||
685 | const PeerOp peer_op) | ||
686 | { | ||
687 | const struct PeerContext *peer_ctx; | ||
688 | unsigned int i; | ||
689 | |||
690 | peer_ctx = get_peer_ctx (peer); | ||
691 | for (i = 0; i < peer_ctx->num_pending_ops; i++) | ||
692 | if (peer_op == peer_ctx->pending_ops[i].op) | ||
693 | return GNUNET_YES; | ||
694 | return GNUNET_NO; | ||
695 | } | ||
696 | |||
697 | |||
698 | /** | ||
699 | * Iterator over hash map entries. Deletes all contexts of peers. | ||
700 | * | ||
701 | * @param cls closure | ||
702 | * @param key current public key | ||
703 | * @param value value in the hash map | ||
704 | * @return #GNUNET_YES if we should continue to iterate, | ||
705 | * #GNUNET_NO if not. | ||
706 | */ | ||
707 | static int | ||
708 | peermap_clear_iterator (void *cls, | ||
709 | const struct GNUNET_PeerIdentity *key, | ||
710 | void *value) | ||
711 | { | ||
712 | Peers_remove_peer (key); | ||
713 | return GNUNET_YES; | ||
714 | } | ||
715 | |||
716 | |||
717 | /** | ||
718 | * @brief This is called once a message is sent. | ||
719 | * | ||
720 | * Removes the pending message | ||
721 | * | ||
722 | * @param cls type of the message that was sent | ||
723 | */ | ||
724 | static void | ||
725 | mq_notify_sent_cb (void *cls) | ||
726 | { | ||
727 | struct PendingMessage *pending_msg = (struct PendingMessage *) cls; | ||
728 | LOG (GNUNET_ERROR_TYPE_DEBUG, | ||
729 | "%s was sent.\n", | ||
730 | pending_msg->type); | ||
731 | remove_pending_message (pending_msg); | ||
732 | } | ||
733 | |||
734 | |||
735 | /** | ||
736 | * @brief Iterator function for #store_valid_peers. | ||
737 | * | ||
738 | * Implements #GNUNET_CONTAINER_PeerMapIterator. | ||
739 | * Writes single peer to disk. | ||
740 | * | ||
741 | * @param cls the file handle to write to. | ||
742 | * @param peer current peer | ||
743 | * @param value unused | ||
744 | * | ||
745 | * @return #GNUNET_YES if we should continue to | ||
746 | * iterate, | ||
747 | * #GNUNET_NO if not. | ||
748 | */ | ||
749 | static int | ||
750 | store_peer_presistently_iterator (void *cls, | ||
751 | const struct GNUNET_PeerIdentity *peer, | ||
752 | void *value) | ||
753 | { | ||
754 | const struct GNUNET_DISK_FileHandle *fh = cls; | ||
755 | char peer_string[128]; | ||
756 | int size; | ||
757 | ssize_t ret; | ||
758 | |||
759 | if (NULL == peer) | ||
760 | { | ||
761 | return GNUNET_YES; | ||
762 | } | ||
763 | size = GNUNET_snprintf (peer_string, | ||
764 | sizeof (peer_string), | ||
765 | "%s\n", | ||
766 | GNUNET_i2s_full (peer)); | ||
767 | GNUNET_assert (53 == size); | ||
768 | ret = GNUNET_DISK_file_write (fh, | ||
769 | peer_string, | ||
770 | size); | ||
771 | GNUNET_assert (size == ret); | ||
772 | return GNUNET_YES; | ||
773 | } | ||
774 | |||
775 | |||
776 | /** | ||
777 | * @brief Store the peers currently in #valid_peers to disk. | ||
778 | */ | ||
779 | static void | ||
780 | store_valid_peers () | ||
781 | { | ||
782 | struct GNUNET_DISK_FileHandle *fh; | ||
783 | uint32_t number_written_peers; | ||
784 | int ret; | ||
785 | |||
786 | if (0 == strncmp ("DISABLE", filename_valid_peers, 7)) | ||
787 | { | ||
788 | return; | ||
789 | } | ||
790 | |||
791 | ret = GNUNET_DISK_directory_create_for_file (filename_valid_peers); | ||
792 | if (GNUNET_SYSERR == ret) | ||
793 | { | ||
794 | LOG (GNUNET_ERROR_TYPE_WARNING, | ||
795 | "Not able to create directory for file `%s'\n", | ||
796 | filename_valid_peers); | ||
797 | GNUNET_break (0); | ||
798 | } | ||
799 | else if (GNUNET_NO == ret) | ||
800 | { | ||
801 | LOG (GNUNET_ERROR_TYPE_WARNING, | ||
802 | "Directory for file `%s' exists but is not writable for us\n", | ||
803 | filename_valid_peers); | ||
804 | GNUNET_break (0); | ||
805 | } | ||
806 | fh = GNUNET_DISK_file_open (filename_valid_peers, | ||
807 | GNUNET_DISK_OPEN_WRITE | | ||
808 | GNUNET_DISK_OPEN_CREATE, | ||
809 | GNUNET_DISK_PERM_USER_READ | | ||
810 | GNUNET_DISK_PERM_USER_WRITE); | ||
811 | if (NULL == fh) | ||
812 | { | ||
813 | LOG (GNUNET_ERROR_TYPE_WARNING, | ||
814 | "Not able to write valid peers to file `%s'\n", | ||
815 | filename_valid_peers); | ||
816 | return; | ||
817 | } | ||
818 | LOG (GNUNET_ERROR_TYPE_DEBUG, | ||
819 | "Writing %u valid peers to disk\n", | ||
820 | GNUNET_CONTAINER_multipeermap_size (valid_peers)); | ||
821 | number_written_peers = | ||
822 | GNUNET_CONTAINER_multipeermap_iterate (valid_peers, | ||
823 | store_peer_presistently_iterator, | ||
824 | fh); | ||
825 | GNUNET_assert (GNUNET_OK == GNUNET_DISK_file_close (fh)); | ||
826 | GNUNET_assert (number_written_peers == | ||
827 | GNUNET_CONTAINER_multipeermap_size (valid_peers)); | ||
828 | } | ||
829 | |||
830 | |||
831 | /** | ||
832 | * @brief Convert string representation of peer id to peer id. | ||
833 | * | ||
834 | * Counterpart to #GNUNET_i2s_full. | ||
835 | * | ||
836 | * @param string_repr The string representation of the peer id | ||
837 | * | ||
838 | * @return The peer id | ||
839 | */ | ||
840 | static const struct GNUNET_PeerIdentity * | ||
841 | s2i_full (const char *string_repr) | ||
842 | { | ||
843 | struct GNUNET_PeerIdentity *peer; | ||
844 | size_t len; | ||
845 | int ret; | ||
846 | |||
847 | peer = GNUNET_new (struct GNUNET_PeerIdentity); | ||
848 | len = strlen (string_repr); | ||
849 | if (52 > len) | ||
850 | { | ||
851 | LOG (GNUNET_ERROR_TYPE_WARNING, | ||
852 | "Not able to convert string representation of PeerID to PeerID\n" | ||
853 | "Sting representation: %s (len %u) - too short\n", | ||
854 | string_repr, | ||
855 | len); | ||
856 | GNUNET_break (0); | ||
857 | } | ||
858 | else if (52 < len) | ||
859 | { | ||
860 | len = 52; | ||
861 | } | ||
862 | ret = GNUNET_CRYPTO_eddsa_public_key_from_string (string_repr, | ||
863 | len, | ||
864 | &peer->public_key); | ||
865 | if (GNUNET_OK != ret) | ||
866 | { | ||
867 | LOG (GNUNET_ERROR_TYPE_WARNING, | ||
868 | "Not able to convert string representation of PeerID to PeerID\n" | ||
869 | "Sting representation: %s\n", | ||
870 | string_repr); | ||
871 | GNUNET_break (0); | ||
872 | } | ||
873 | return peer; | ||
874 | } | ||
875 | |||
876 | |||
877 | /** | ||
878 | * @brief Restore the peers on disk to #valid_peers. | ||
879 | */ | ||
880 | static void | ||
881 | restore_valid_peers () | ||
882 | { | ||
883 | off_t file_size; | ||
884 | uint32_t num_peers; | ||
885 | struct GNUNET_DISK_FileHandle *fh; | ||
886 | char *buf; | ||
887 | ssize_t size_read; | ||
888 | char *iter_buf; | ||
889 | char *str_repr; | ||
890 | const struct GNUNET_PeerIdentity *peer; | ||
891 | |||
892 | if (0 == strncmp ("DISABLE", filename_valid_peers, 7)) | ||
893 | { | ||
894 | return; | ||
895 | } | ||
896 | |||
897 | if (GNUNET_OK != GNUNET_DISK_file_test (filename_valid_peers)) | ||
898 | { | ||
899 | return; | ||
900 | } | ||
901 | fh = GNUNET_DISK_file_open (filename_valid_peers, | ||
902 | GNUNET_DISK_OPEN_READ, | ||
903 | GNUNET_DISK_PERM_NONE); | ||
904 | GNUNET_assert (NULL != fh); | ||
905 | GNUNET_assert (GNUNET_OK == GNUNET_DISK_file_handle_size (fh, &file_size)); | ||
906 | num_peers = file_size / 53; | ||
907 | buf = GNUNET_malloc (file_size); | ||
908 | size_read = GNUNET_DISK_file_read (fh, buf, file_size); | ||
909 | GNUNET_assert (size_read == file_size); | ||
910 | LOG (GNUNET_ERROR_TYPE_DEBUG, | ||
911 | "Restoring %" PRIu32 " peers from file `%s'\n", | ||
912 | num_peers, | ||
913 | filename_valid_peers); | ||
914 | for (iter_buf = buf; iter_buf < buf + file_size - 1; iter_buf += 53) | ||
915 | { | ||
916 | str_repr = GNUNET_strndup (iter_buf, 53); | ||
917 | peer = s2i_full (str_repr); | ||
918 | GNUNET_free (str_repr); | ||
919 | add_valid_peer (peer); | ||
920 | LOG (GNUNET_ERROR_TYPE_DEBUG, | ||
921 | "Restored valid peer %s from disk\n", | ||
922 | GNUNET_i2s_full (peer)); | ||
923 | } | ||
924 | iter_buf = NULL; | ||
925 | GNUNET_free (buf); | ||
926 | LOG (GNUNET_ERROR_TYPE_DEBUG, | ||
927 | "num_peers: %" PRIu32 ", _size (valid_peers): %u\n", | ||
928 | num_peers, | ||
929 | GNUNET_CONTAINER_multipeermap_size (valid_peers)); | ||
930 | if (num_peers != GNUNET_CONTAINER_multipeermap_size (valid_peers)) | ||
931 | { | ||
932 | LOG (GNUNET_ERROR_TYPE_WARNING, | ||
933 | "Number of restored peers does not match file size. Have probably duplicates.\n"); | ||
934 | } | ||
935 | GNUNET_assert (GNUNET_OK == GNUNET_DISK_file_close (fh)); | ||
936 | LOG (GNUNET_ERROR_TYPE_DEBUG, | ||
937 | "Restored %u valid peers from disk\n", | ||
938 | GNUNET_CONTAINER_multipeermap_size (valid_peers)); | ||
939 | } | ||
940 | |||
941 | |||
942 | /** | ||
943 | * @brief Initialise storage of peers | ||
944 | * | ||
945 | * @param fn_valid_peers filename of the file used to store valid peer ids | ||
946 | * @param cadet_h cadet handle | ||
947 | * @param disconnect_handler Disconnect handler | ||
948 | * @param c_handlers cadet handlers | ||
949 | * @param own_id own peer identity | ||
950 | */ | ||
951 | void | ||
952 | Peers_initialise (char* fn_valid_peers, | ||
953 | struct GNUNET_CADET_Handle *cadet_h, | ||
954 | GNUNET_CADET_DisconnectEventHandler disconnect_handler, | ||
955 | const struct GNUNET_MQ_MessageHandler *c_handlers, | ||
956 | const struct GNUNET_PeerIdentity *own_id) | ||
957 | { | ||
958 | filename_valid_peers = GNUNET_strdup (fn_valid_peers); | ||
959 | cadet_handle = cadet_h; | ||
960 | cleanup_destroyed_channel = disconnect_handler; | ||
961 | cadet_handlers = c_handlers; | ||
962 | own_identity = own_id; | ||
963 | peer_map = GNUNET_CONTAINER_multipeermap_create (4, GNUNET_NO); | ||
964 | valid_peers = GNUNET_CONTAINER_multipeermap_create (4, GNUNET_NO); | ||
965 | restore_valid_peers (); | ||
966 | } | ||
967 | |||
968 | |||
969 | /** | ||
970 | * @brief Delete storage of peers that was created with #Peers_initialise () | ||
971 | */ | ||
972 | void | ||
973 | Peers_terminate () | ||
974 | { | ||
975 | if (GNUNET_SYSERR == | ||
976 | GNUNET_CONTAINER_multipeermap_iterate (peer_map, | ||
977 | peermap_clear_iterator, | ||
978 | NULL)) | ||
979 | { | ||
980 | LOG (GNUNET_ERROR_TYPE_WARNING, | ||
981 | "Iteration destroying peers was aborted.\n"); | ||
982 | } | ||
983 | GNUNET_CONTAINER_multipeermap_destroy (peer_map); | ||
984 | store_valid_peers (); | ||
985 | GNUNET_free (filename_valid_peers); | ||
986 | GNUNET_CONTAINER_multipeermap_destroy (valid_peers); | ||
987 | } | ||
988 | |||
989 | |||
990 | /** | ||
991 | * Iterator over #valid_peers hash map entries. | ||
992 | * | ||
993 | * @param cls closure - unused | ||
994 | * @param peer current peer id | ||
995 | * @param value value in the hash map - unused | ||
996 | * @return #GNUNET_YES if we should continue to | ||
997 | * iterate, | ||
998 | * #GNUNET_NO if not. | ||
999 | */ | ||
1000 | static int | ||
1001 | valid_peer_iterator (void *cls, | ||
1002 | const struct GNUNET_PeerIdentity *peer, | ||
1003 | void *value) | ||
1004 | { | ||
1005 | struct PeersIteratorCls *it_cls = cls; | ||
1006 | |||
1007 | return it_cls->iterator (it_cls->cls, | ||
1008 | peer); | ||
1009 | } | ||
1010 | |||
1011 | |||
1012 | /** | ||
1013 | * @brief Get all currently known, valid peer ids. | ||
1014 | * | ||
1015 | * @param it function to call on each peer id | ||
1016 | * @param it_cls extra argument to @a it | ||
1017 | * @return the number of key value pairs processed, | ||
1018 | * #GNUNET_SYSERR if it aborted iteration | ||
1019 | */ | ||
1020 | int | ||
1021 | Peers_get_valid_peers (PeersIterator iterator, | ||
1022 | void *it_cls) | ||
1023 | { | ||
1024 | struct PeersIteratorCls *cls; | ||
1025 | int ret; | ||
1026 | |||
1027 | cls = GNUNET_new (struct PeersIteratorCls); | ||
1028 | cls->iterator = iterator; | ||
1029 | cls->cls = it_cls; | ||
1030 | ret = GNUNET_CONTAINER_multipeermap_iterate (valid_peers, | ||
1031 | valid_peer_iterator, | ||
1032 | cls); | ||
1033 | GNUNET_free (cls); | ||
1034 | return ret; | ||
1035 | } | ||
1036 | |||
1037 | |||
1038 | /** | ||
1039 | * @brief Add peer to known peers. | ||
1040 | * | ||
1041 | * This function is called on new peer_ids from 'external' sources | ||
1042 | * (client seed, cadet get_peers(), ...) | ||
1043 | * | ||
1044 | * @param peer the new #GNUNET_PeerIdentity | ||
1045 | * | ||
1046 | * @return #GNUNET_YES if peer was inserted | ||
1047 | * #GNUNET_NO otherwise (if peer was already known or | ||
1048 | * peer was #own_identity) | ||
1049 | */ | ||
1050 | int | ||
1051 | Peers_insert_peer (const struct GNUNET_PeerIdentity *peer) | ||
1052 | { | ||
1053 | if ( (GNUNET_YES == Peers_check_peer_known (peer)) || | ||
1054 | (0 == GNUNET_CRYPTO_cmp_peer_identity (peer, own_identity)) ) | ||
1055 | { | ||
1056 | return GNUNET_NO; /* We already know this peer - nothing to do */ | ||
1057 | } | ||
1058 | (void) create_peer_ctx (peer); | ||
1059 | return GNUNET_YES; | ||
1060 | } | ||
1061 | |||
1062 | |||
1063 | /** | ||
1064 | * @brief Try connecting to a peer to see whether it is online | ||
1065 | * | ||
1066 | * If not known yet, insert into known peers | ||
1067 | * | ||
1068 | * @param peer the peer whose liveliness is to be checked | ||
1069 | * @return #GNUNET_YES if peer had to be inserted | ||
1070 | * #GNUNET_NO otherwise (if peer was already known or | ||
1071 | * peer was #own_identity) | ||
1072 | */ | ||
1073 | int | ||
1074 | Peers_issue_peer_liveliness_check (const struct GNUNET_PeerIdentity *peer) | ||
1075 | { | ||
1076 | struct PeerContext *peer_ctx; | ||
1077 | int ret; | ||
1078 | |||
1079 | if (0 == GNUNET_CRYPTO_cmp_peer_identity (peer, own_identity)) | ||
1080 | { | ||
1081 | return GNUNET_NO; | ||
1082 | } | ||
1083 | ret = Peers_insert_peer (peer); | ||
1084 | peer_ctx = get_peer_ctx (peer); | ||
1085 | if (GNUNET_NO == Peers_check_peer_flag (peer, Peers_ONLINE)) | ||
1086 | { | ||
1087 | check_peer_live (peer_ctx); | ||
1088 | } | ||
1089 | return ret; | ||
1090 | } | ||
1091 | |||
1092 | |||
1093 | /** | ||
1094 | * @brief Check if peer is removable. | ||
1095 | * | ||
1096 | * Check if | ||
1097 | * - a recv channel exists | ||
1098 | * - there are pending messages | ||
1099 | * - there is no pending pull reply | ||
1100 | * | ||
1101 | * @param peer the peer in question | ||
1102 | * @return #GNUNET_YES if peer is removable | ||
1103 | * #GNUNET_NO if peer is NOT removable | ||
1104 | * #GNUNET_SYSERR if peer is not known | ||
1105 | */ | ||
1106 | int | ||
1107 | Peers_check_removable (const struct GNUNET_PeerIdentity *peer) | ||
1108 | { | ||
1109 | struct PeerContext *peer_ctx; | ||
1110 | |||
1111 | if (GNUNET_NO == GNUNET_CONTAINER_multipeermap_contains (peer_map, peer)) | ||
1112 | { | ||
1113 | return GNUNET_SYSERR; | ||
1114 | } | ||
1115 | |||
1116 | peer_ctx = get_peer_ctx (peer); | ||
1117 | if ( (NULL != peer_ctx->recv_channel) || | ||
1118 | (NULL != peer_ctx->pending_messages_head) || | ||
1119 | (GNUNET_NO == check_peer_flag_set (peer_ctx, Peers_PULL_REPLY_PENDING)) ) | ||
1120 | { | ||
1121 | return GNUNET_NO; | ||
1122 | } | ||
1123 | return GNUNET_YES; | ||
1124 | } | ||
1125 | |||
1126 | |||
1127 | /** | ||
1128 | * @brief Remove peer | ||
1129 | * | ||
1130 | * @param peer the peer to clean | ||
1131 | * @return #GNUNET_YES if peer was removed | ||
1132 | * #GNUNET_NO otherwise | ||
1133 | */ | ||
1134 | int | ||
1135 | Peers_remove_peer (const struct GNUNET_PeerIdentity *peer) | ||
1136 | { | ||
1137 | struct PeerContext *peer_ctx; | ||
1138 | |||
1139 | if (GNUNET_NO == GNUNET_CONTAINER_multipeermap_contains (peer_map, peer)) | ||
1140 | { | ||
1141 | return GNUNET_NO; | ||
1142 | } | ||
1143 | |||
1144 | peer_ctx = get_peer_ctx (peer); | ||
1145 | set_peer_flag (peer_ctx, Peers_TO_DESTROY); | ||
1146 | LOG (GNUNET_ERROR_TYPE_DEBUG, | ||
1147 | "Going to remove peer %s\n", | ||
1148 | GNUNET_i2s (&peer_ctx->peer_id)); | ||
1149 | Peers_unset_peer_flag (peer, Peers_ONLINE); | ||
1150 | |||
1151 | GNUNET_array_grow (peer_ctx->pending_ops, peer_ctx->num_pending_ops, 0); | ||
1152 | while (NULL != peer_ctx->pending_messages_head) | ||
1153 | { | ||
1154 | LOG (GNUNET_ERROR_TYPE_DEBUG, | ||
1155 | "Removing unsent %s\n", | ||
1156 | peer_ctx->pending_messages_head->type); | ||
1157 | remove_pending_message (peer_ctx->pending_messages_head); | ||
1158 | } | ||
1159 | /* If we are still waiting for notification whether this peer is live | ||
1160 | * cancel the according task */ | ||
1161 | if (NULL != peer_ctx->liveliness_check_pending) | ||
1162 | { | ||
1163 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
1164 | "Removing pending liveliness check for peer %s\n", | ||
1165 | GNUNET_i2s (&peer_ctx->peer_id)); | ||
1166 | // TODO wait until cadet sets mq->cancel_impl | ||
1167 | //GNUNET_MQ_send_cancel (peer_ctx->liveliness_check_pending->ev); | ||
1168 | GNUNET_free (peer_ctx->liveliness_check_pending); | ||
1169 | peer_ctx->liveliness_check_pending = NULL; | ||
1170 | } | ||
1171 | if (NULL != peer_ctx->send_channel) | ||
1172 | { | ||
1173 | LOG (GNUNET_ERROR_TYPE_DEBUG, | ||
1174 | "Destroying send channel\n"); | ||
1175 | GNUNET_CADET_channel_destroy (peer_ctx->send_channel); | ||
1176 | peer_ctx->send_channel = NULL; | ||
1177 | } | ||
1178 | if (NULL != peer_ctx->recv_channel) | ||
1179 | { | ||
1180 | LOG (GNUNET_ERROR_TYPE_DEBUG, | ||
1181 | "Destroying recv channel\n"); | ||
1182 | GNUNET_CADET_channel_destroy (peer_ctx->recv_channel); | ||
1183 | peer_ctx->recv_channel = NULL; | ||
1184 | } | ||
1185 | if (NULL != peer_ctx->mq) | ||
1186 | { | ||
1187 | GNUNET_MQ_destroy (peer_ctx->mq); | ||
1188 | peer_ctx->mq = NULL; | ||
1189 | } | ||
1190 | |||
1191 | GNUNET_free (peer_ctx->send_channel_flags); | ||
1192 | GNUNET_free (peer_ctx->recv_channel_flags); | ||
1193 | |||
1194 | if (GNUNET_YES != GNUNET_CONTAINER_multipeermap_remove_all (peer_map, &peer_ctx->peer_id)) | ||
1195 | { | ||
1196 | LOG (GNUNET_ERROR_TYPE_WARNING, "removing peer from peer_map failed\n"); | ||
1197 | } | ||
1198 | GNUNET_free (peer_ctx); | ||
1199 | return GNUNET_YES; | ||
1200 | } | ||
1201 | |||
1202 | |||
1203 | /** | ||
1204 | * @brief set flags on a given peer. | ||
1205 | * | ||
1206 | * @param peer the peer to set flags on | ||
1207 | * @param flags the flags | ||
1208 | */ | ||
1209 | void | ||
1210 | Peers_set_peer_flag (const struct GNUNET_PeerIdentity *peer, enum Peers_PeerFlags flags) | ||
1211 | { | ||
1212 | struct PeerContext *peer_ctx; | ||
1213 | |||
1214 | peer_ctx = get_peer_ctx (peer); | ||
1215 | set_peer_flag (peer_ctx, flags); | ||
1216 | } | ||
1217 | |||
1218 | |||
1219 | /** | ||
1220 | * @brief unset flags on a given peer. | ||
1221 | * | ||
1222 | * @param peer the peer to unset flags on | ||
1223 | * @param flags the flags | ||
1224 | */ | ||
1225 | void | ||
1226 | Peers_unset_peer_flag (const struct GNUNET_PeerIdentity *peer, enum Peers_PeerFlags flags) | ||
1227 | { | ||
1228 | struct PeerContext *peer_ctx; | ||
1229 | |||
1230 | peer_ctx = get_peer_ctx (peer); | ||
1231 | unset_peer_flag (peer_ctx, flags); | ||
1232 | } | ||
1233 | |||
1234 | |||
1235 | /** | ||
1236 | * @brief Check whether flags on a peer are set. | ||
1237 | * | ||
1238 | * @param peer the peer to check the flag of | ||
1239 | * @param flags the flags to check | ||
1240 | * | ||
1241 | * @return #GNUNET_SYSERR if peer is not known | ||
1242 | * #GNUNET_YES if all given flags are set | ||
1243 | * #GNUNET_NO otherwise | ||
1244 | */ | ||
1245 | int | ||
1246 | Peers_check_peer_flag (const struct GNUNET_PeerIdentity *peer, enum Peers_PeerFlags flags) | ||
1247 | { | ||
1248 | struct PeerContext *peer_ctx; | ||
1249 | |||
1250 | if (GNUNET_NO == Peers_check_peer_known (peer)) | ||
1251 | { | ||
1252 | return GNUNET_SYSERR; | ||
1253 | } | ||
1254 | peer_ctx = get_peer_ctx (peer); | ||
1255 | return check_peer_flag_set (peer_ctx, flags); | ||
1256 | } | ||
1257 | |||
1258 | |||
1259 | /** | ||
1260 | * @brief set flags on a given channel. | ||
1261 | * | ||
1262 | * @param channel the channel to set flags on | ||
1263 | * @param flags the flags | ||
1264 | */ | ||
1265 | void | ||
1266 | Peers_set_channel_flag (uint32_t *channel_flags, enum Peers_ChannelFlags flags) | ||
1267 | { | ||
1268 | set_channel_flag (channel_flags, flags); | ||
1269 | } | ||
1270 | |||
1271 | |||
1272 | /** | ||
1273 | * @brief unset flags on a given channel. | ||
1274 | * | ||
1275 | * @param channel the channel to unset flags on | ||
1276 | * @param flags the flags | ||
1277 | */ | ||
1278 | void | ||
1279 | Peers_unset_channel_flag (uint32_t *channel_flags, enum Peers_ChannelFlags flags) | ||
1280 | { | ||
1281 | unset_channel_flag (channel_flags, flags); | ||
1282 | } | ||
1283 | |||
1284 | |||
1285 | /** | ||
1286 | * @brief Check whether flags on a channel are set. | ||
1287 | * | ||
1288 | * @param channel the channel to check the flag of | ||
1289 | * @param flags the flags to check | ||
1290 | * | ||
1291 | * @return #GNUNET_YES if all given flags are set | ||
1292 | * #GNUNET_NO otherwise | ||
1293 | */ | ||
1294 | int | ||
1295 | Peers_check_channel_flag (uint32_t *channel_flags, enum Peers_ChannelFlags flags) | ||
1296 | { | ||
1297 | return check_channel_flag_set (channel_flags, flags); | ||
1298 | } | ||
1299 | |||
1300 | /** | ||
1301 | * @brief Get the flags for the channel in @a role for @a peer. | ||
1302 | * | ||
1303 | * @param peer Peer to get the channel flags for. | ||
1304 | * @param role Role of channel to get flags for | ||
1305 | * | ||
1306 | * @return The flags. | ||
1307 | */ | ||
1308 | uint32_t * | ||
1309 | Peers_get_channel_flag (const struct GNUNET_PeerIdentity *peer, | ||
1310 | enum Peers_ChannelRole role) | ||
1311 | { | ||
1312 | const struct PeerContext *peer_ctx; | ||
1313 | |||
1314 | peer_ctx = get_peer_ctx (peer); | ||
1315 | if (Peers_CHANNEL_ROLE_SENDING == role) | ||
1316 | { | ||
1317 | return peer_ctx->send_channel_flags; | ||
1318 | } | ||
1319 | else if (Peers_CHANNEL_ROLE_RECEIVING == role) | ||
1320 | { | ||
1321 | return peer_ctx->recv_channel_flags; | ||
1322 | } | ||
1323 | else | ||
1324 | { | ||
1325 | GNUNET_assert (0); | ||
1326 | } | ||
1327 | } | ||
1328 | |||
1329 | /** | ||
1330 | * @brief Check whether we have information about the given peer. | ||
1331 | * | ||
1332 | * FIXME probably deprecated. Make this the new _online. | ||
1333 | * | ||
1334 | * @param peer peer in question | ||
1335 | * | ||
1336 | * @return #GNUNET_YES if peer is known | ||
1337 | * #GNUNET_NO if peer is not knwon | ||
1338 | */ | ||
1339 | int | ||
1340 | Peers_check_peer_known (const struct GNUNET_PeerIdentity *peer) | ||
1341 | { | ||
1342 | return GNUNET_CONTAINER_multipeermap_contains (peer_map, peer); | ||
1343 | } | ||
1344 | |||
1345 | |||
1346 | /** | ||
1347 | * @brief Check whether @a peer is actually a peer. | ||
1348 | * | ||
1349 | * A valid peer is a peer that we know exists eg. we were connected to once. | ||
1350 | * | ||
1351 | * @param peer peer in question | ||
1352 | * | ||
1353 | * @return #GNUNET_YES if peer is valid | ||
1354 | * #GNUNET_NO if peer is not valid | ||
1355 | */ | ||
1356 | int | ||
1357 | Peers_check_peer_valid (const struct GNUNET_PeerIdentity *peer) | ||
1358 | { | ||
1359 | return GNUNET_CONTAINER_multipeermap_contains (valid_peers, peer); | ||
1360 | } | ||
1361 | |||
1362 | |||
1363 | /** | ||
1364 | * @brief Indicate that we want to send to the other peer | ||
1365 | * | ||
1366 | * This establishes a sending channel | ||
1367 | * | ||
1368 | * @param peer the peer to establish channel to | ||
1369 | */ | ||
1370 | void | ||
1371 | Peers_indicate_sending_intention (const struct GNUNET_PeerIdentity *peer) | ||
1372 | { | ||
1373 | GNUNET_assert (GNUNET_YES == Peers_check_peer_known (peer)); | ||
1374 | (void) get_channel (peer); | ||
1375 | } | ||
1376 | |||
1377 | |||
1378 | /** | ||
1379 | * @brief Check whether other peer has the intention to send/opened channel | ||
1380 | * towars us | ||
1381 | * | ||
1382 | * @param peer the peer in question | ||
1383 | * | ||
1384 | * @return #GNUNET_YES if peer has the intention to send | ||
1385 | * #GNUNET_NO otherwise | ||
1386 | */ | ||
1387 | int | ||
1388 | Peers_check_peer_send_intention (const struct GNUNET_PeerIdentity *peer) | ||
1389 | { | ||
1390 | const struct PeerContext *peer_ctx; | ||
1391 | |||
1392 | peer_ctx = get_peer_ctx (peer); | ||
1393 | if (NULL != peer_ctx->recv_channel) | ||
1394 | { | ||
1395 | return GNUNET_YES; | ||
1396 | } | ||
1397 | return GNUNET_NO; | ||
1398 | } | ||
1399 | |||
1400 | |||
1401 | /** | ||
1402 | * Handle the channel a peer opens to us. | ||
1403 | * | ||
1404 | * @param cls The closure | ||
1405 | * @param channel The channel the peer wants to establish | ||
1406 | * @param initiator The peer's peer ID | ||
1407 | * | ||
1408 | * @return initial channel context for the channel | ||
1409 | * (can be NULL -- that's not an error) | ||
1410 | */ | ||
1411 | void * | ||
1412 | Peers_handle_inbound_channel (void *cls, | ||
1413 | struct GNUNET_CADET_Channel *channel, | ||
1414 | const struct GNUNET_PeerIdentity *initiator) | ||
1415 | { | ||
1416 | struct PeerContext *peer_ctx; | ||
1417 | |||
1418 | LOG (GNUNET_ERROR_TYPE_DEBUG, | ||
1419 | "New channel was established to us (Peer %s).\n", | ||
1420 | GNUNET_i2s (initiator)); | ||
1421 | GNUNET_assert (NULL != channel); /* according to cadet API */ | ||
1422 | /* Make sure we 'know' about this peer */ | ||
1423 | peer_ctx = create_or_get_peer_ctx (initiator); | ||
1424 | set_peer_live (peer_ctx); | ||
1425 | /* We only accept one incoming channel per peer */ | ||
1426 | if (GNUNET_YES == Peers_check_peer_send_intention (initiator)) | ||
1427 | { | ||
1428 | set_channel_flag (peer_ctx->recv_channel_flags, | ||
1429 | Peers_CHANNEL_ESTABLISHED_TWICE); | ||
1430 | GNUNET_CADET_channel_destroy (channel); | ||
1431 | /* return the channel context */ | ||
1432 | return &peer_ctx->peer_id; | ||
1433 | } | ||
1434 | peer_ctx->recv_channel = channel; | ||
1435 | return &peer_ctx->peer_id; | ||
1436 | } | ||
1437 | |||
1438 | |||
1439 | /** | ||
1440 | * @brief Check whether a sending channel towards the given peer exists | ||
1441 | * | ||
1442 | * @param peer the peer to check for | ||
1443 | * | ||
1444 | * @return #GNUNET_YES if a sending channel towards that peer exists | ||
1445 | * #GNUNET_NO otherwise | ||
1446 | */ | ||
1447 | int | ||
1448 | Peers_check_sending_channel_exists (const struct GNUNET_PeerIdentity *peer) | ||
1449 | { | ||
1450 | struct PeerContext *peer_ctx; | ||
1451 | |||
1452 | if (GNUNET_NO == Peers_check_peer_known (peer)) | ||
1453 | { /* If no such peer exists, there is no channel */ | ||
1454 | return GNUNET_NO; | ||
1455 | } | ||
1456 | peer_ctx = get_peer_ctx (peer); | ||
1457 | if (NULL == peer_ctx->send_channel) | ||
1458 | { | ||
1459 | return GNUNET_NO; | ||
1460 | } | ||
1461 | return GNUNET_YES; | ||
1462 | } | ||
1463 | |||
1464 | |||
1465 | /** | ||
1466 | * @brief check whether the given channel is the sending channel of the given | ||
1467 | * peer | ||
1468 | * | ||
1469 | * @param peer the peer in question | ||
1470 | * @param channel the channel to check for | ||
1471 | * @param role either #Peers_CHANNEL_ROLE_SENDING, or | ||
1472 | * #Peers_CHANNEL_ROLE_RECEIVING | ||
1473 | * | ||
1474 | * @return #GNUNET_YES if the given chennel is the sending channel of the peer | ||
1475 | * #GNUNET_NO otherwise | ||
1476 | */ | ||
1477 | int | ||
1478 | Peers_check_channel_role (const struct GNUNET_PeerIdentity *peer, | ||
1479 | const struct GNUNET_CADET_Channel *channel, | ||
1480 | enum Peers_ChannelRole role) | ||
1481 | { | ||
1482 | const struct PeerContext *peer_ctx; | ||
1483 | |||
1484 | if (GNUNET_NO == Peers_check_peer_known (peer)) | ||
1485 | { | ||
1486 | return GNUNET_NO; | ||
1487 | } | ||
1488 | peer_ctx = get_peer_ctx (peer); | ||
1489 | if ( (Peers_CHANNEL_ROLE_SENDING == role) && | ||
1490 | (channel == peer_ctx->send_channel) ) | ||
1491 | { | ||
1492 | return GNUNET_YES; | ||
1493 | } | ||
1494 | if ( (Peers_CHANNEL_ROLE_RECEIVING == role) && | ||
1495 | (channel == peer_ctx->recv_channel) ) | ||
1496 | { | ||
1497 | return GNUNET_YES; | ||
1498 | } | ||
1499 | return GNUNET_NO; | ||
1500 | } | ||
1501 | |||
1502 | |||
1503 | /** | ||
1504 | * @brief Destroy the send channel of a peer e.g. stop indicating a sending | ||
1505 | * intention to another peer | ||
1506 | * | ||
1507 | * If there is also no channel to receive messages from that peer, remove it | ||
1508 | * from the peermap. | ||
1509 | * TODO really? | ||
1510 | * | ||
1511 | * @peer the peer identity of the peer whose sending channel to destroy | ||
1512 | * @return #GNUNET_YES if channel was destroyed | ||
1513 | * #GNUNET_NO otherwise | ||
1514 | */ | ||
1515 | int | ||
1516 | Peers_destroy_sending_channel (const struct GNUNET_PeerIdentity *peer) | ||
1517 | { | ||
1518 | struct PeerContext *peer_ctx; | ||
1519 | |||
1520 | if (GNUNET_NO == Peers_check_peer_known (peer)) | ||
1521 | { | ||
1522 | return GNUNET_NO; | ||
1523 | } | ||
1524 | peer_ctx = get_peer_ctx (peer); | ||
1525 | if (NULL != peer_ctx->send_channel) | ||
1526 | { | ||
1527 | set_channel_flag (peer_ctx->send_channel_flags, Peers_CHANNEL_CLEAN); | ||
1528 | GNUNET_CADET_channel_destroy (peer_ctx->send_channel); | ||
1529 | peer_ctx->send_channel = NULL; | ||
1530 | (void) Peers_check_connected (peer); | ||
1531 | return GNUNET_YES; | ||
1532 | } | ||
1533 | return GNUNET_NO; | ||
1534 | } | ||
1535 | |||
1536 | /** | ||
1537 | * This is called when a channel is destroyed. | ||
1538 | * | ||
1539 | * @param cls The closure | ||
1540 | * @param channel The channel being closed | ||
1541 | * @param channel_ctx The context associated with this channel | ||
1542 | */ | ||
1543 | void | ||
1544 | Peers_cleanup_destroyed_channel (void *cls, | ||
1545 | const struct GNUNET_CADET_Channel *channel) | ||
1546 | { | ||
1547 | struct GNUNET_PeerIdentity *peer = cls; | ||
1548 | struct PeerContext *peer_ctx; | ||
1549 | |||
1550 | if (GNUNET_NO == Peers_check_peer_known (peer)) | ||
1551 | {/* We don't want to implicitly create a context that we're about to kill */ | ||
1552 | LOG (GNUNET_ERROR_TYPE_DEBUG, | ||
1553 | "channel (%s) without associated context was destroyed\n", | ||
1554 | GNUNET_i2s (peer)); | ||
1555 | return; | ||
1556 | } | ||
1557 | peer_ctx = get_peer_ctx (peer); | ||
1558 | |||
1559 | /* If our peer issued the destruction of the channel, the #Peers_TO_DESTROY | ||
1560 | * flag will be set. In this case simply make sure that the channels are | ||
1561 | * cleaned. */ | ||
1562 | /* FIXME This distinction seems to be redundant */ | ||
1563 | if (Peers_check_peer_flag (peer, Peers_TO_DESTROY)) | ||
1564 | {/* We initiatad the destruction of this particular peer */ | ||
1565 | if (channel == peer_ctx->send_channel) | ||
1566 | peer_ctx->send_channel = NULL; | ||
1567 | else if (channel == peer_ctx->recv_channel) | ||
1568 | peer_ctx->recv_channel = NULL; | ||
1569 | |||
1570 | if (NULL != peer_ctx->send_channel) | ||
1571 | { | ||
1572 | GNUNET_CADET_channel_destroy (peer_ctx->send_channel); | ||
1573 | peer_ctx->send_channel = NULL; | ||
1574 | } | ||
1575 | if (NULL != peer_ctx->recv_channel) | ||
1576 | { | ||
1577 | GNUNET_CADET_channel_destroy (peer_ctx->recv_channel); | ||
1578 | peer_ctx->recv_channel = NULL; | ||
1579 | } | ||
1580 | /* Set the #Peers_ONLINE flag accordingly */ | ||
1581 | (void) Peers_check_connected (peer); | ||
1582 | return; | ||
1583 | } | ||
1584 | |||
1585 | else | ||
1586 | { /* We did not initiate the destruction of this peer */ | ||
1587 | if (channel == peer_ctx->send_channel) | ||
1588 | { /* Something (but us) killd the channel - clean up peer */ | ||
1589 | LOG (GNUNET_ERROR_TYPE_DEBUG, | ||
1590 | "send channel (%s) was destroyed - cleaning up\n", | ||
1591 | GNUNET_i2s (peer)); | ||
1592 | peer_ctx->send_channel = NULL; | ||
1593 | } | ||
1594 | else if (channel == peer_ctx->recv_channel) | ||
1595 | { /* Other peer doesn't want to send us messages anymore */ | ||
1596 | LOG (GNUNET_ERROR_TYPE_DEBUG, | ||
1597 | "Peer %s destroyed recv channel - cleaning up channel\n", | ||
1598 | GNUNET_i2s (peer)); | ||
1599 | peer_ctx->recv_channel = NULL; | ||
1600 | } | ||
1601 | else | ||
1602 | { | ||
1603 | LOG (GNUNET_ERROR_TYPE_WARNING, | ||
1604 | "unknown channel (%s) was destroyed\n", | ||
1605 | GNUNET_i2s (peer)); | ||
1606 | } | ||
1607 | } | ||
1608 | (void) Peers_check_connected (peer); | ||
1609 | } | ||
1610 | |||
1611 | /** | ||
1612 | * @brief Send a message to another peer. | ||
1613 | * | ||
1614 | * Keeps track about pending messages so they can be properly removed when the | ||
1615 | * peer is destroyed. | ||
1616 | * | ||
1617 | * @param peer receeiver of the message | ||
1618 | * @param ev envelope of the message | ||
1619 | * @param type type of the message | ||
1620 | */ | ||
1621 | void | ||
1622 | Peers_send_message (const struct GNUNET_PeerIdentity *peer, | ||
1623 | struct GNUNET_MQ_Envelope *ev, | ||
1624 | const char *type) | ||
1625 | { | ||
1626 | struct PendingMessage *pending_msg; | ||
1627 | struct GNUNET_MQ_Handle *mq; | ||
1628 | |||
1629 | pending_msg = insert_pending_message (peer, ev, type); | ||
1630 | mq = get_mq (peer); | ||
1631 | GNUNET_MQ_notify_sent (ev, | ||
1632 | mq_notify_sent_cb, | ||
1633 | pending_msg); | ||
1634 | GNUNET_MQ_send (mq, ev); | ||
1635 | } | ||
1636 | |||
1637 | /** | ||
1638 | * @brief Schedule a operation on given peer | ||
1639 | * | ||
1640 | * Avoids scheduling an operation twice. | ||
1641 | * | ||
1642 | * @param peer the peer we want to schedule the operation for once it gets live | ||
1643 | * | ||
1644 | * @return #GNUNET_YES if the operation was scheduled | ||
1645 | * #GNUNET_NO otherwise | ||
1646 | */ | ||
1647 | int | ||
1648 | Peers_schedule_operation (const struct GNUNET_PeerIdentity *peer, | ||
1649 | const PeerOp peer_op) | ||
1650 | { | ||
1651 | struct PeerPendingOp pending_op; | ||
1652 | struct PeerContext *peer_ctx; | ||
1653 | |||
1654 | if (0 == GNUNET_CRYPTO_cmp_peer_identity (peer, own_identity)) | ||
1655 | { | ||
1656 | return GNUNET_NO; | ||
1657 | } | ||
1658 | GNUNET_assert (GNUNET_YES == Peers_check_peer_known (peer)); | ||
1659 | |||
1660 | //TODO if LIVE/ONLINE execute immediately | ||
1661 | |||
1662 | if (GNUNET_NO == check_operation_scheduled (peer, peer_op)) | ||
1663 | { | ||
1664 | peer_ctx = get_peer_ctx (peer); | ||
1665 | pending_op.op = peer_op; | ||
1666 | pending_op.op_cls = NULL; | ||
1667 | GNUNET_array_append (peer_ctx->pending_ops, | ||
1668 | peer_ctx->num_pending_ops, | ||
1669 | pending_op); | ||
1670 | return GNUNET_YES; | ||
1671 | } | ||
1672 | return GNUNET_NO; | ||
1673 | } | ||
1674 | |||
1675 | /** | ||
1676 | * @brief Get the recv_channel of @a peer. | ||
1677 | * Needed to correctly handle (call #GNUNET_CADET_receive_done()) incoming | ||
1678 | * messages. | ||
1679 | * | ||
1680 | * @param peer The peer to get the recv_channel from. | ||
1681 | * | ||
1682 | * @return The recv_channel. | ||
1683 | */ | ||
1684 | struct GNUNET_CADET_Channel * | ||
1685 | Peers_get_recv_channel (const struct GNUNET_PeerIdentity *peer) | ||
1686 | { | ||
1687 | struct PeerContext *peer_ctx; | ||
1688 | |||
1689 | GNUNET_assert (GNUNET_YES == Peers_check_peer_known (peer)); | ||
1690 | peer_ctx = get_peer_ctx (peer); | ||
1691 | return peer_ctx->recv_channel; | ||
1692 | } | ||
1693 | |||
1694 | /* end of gnunet-service-rps_peers.c */ | ||
diff --git a/src/rps/gnunet-service-rps_peers.h b/src/rps/gnunet-service-rps_peers.h deleted file mode 100644 index 15970a7ce..000000000 --- a/src/rps/gnunet-service-rps_peers.h +++ /dev/null | |||
@@ -1,437 +0,0 @@ | |||
1 | /* | ||
2 | This file is part of GNUnet. | ||
3 | Copyright (C) | ||
4 | |||
5 | GNUnet is free software; you can redistribute it and/or modify | ||
6 | it under the terms of the GNU General Public License as published | ||
7 | by the Free Software Foundation; either version 3, or (at your | ||
8 | option) any later version. | ||
9 | |||
10 | GNUnet is distributed in the hope that it will be useful, but | ||
11 | WITHOUT ANY WARRANTY; without even the implied warranty of | ||
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
13 | General Public License for more details. | ||
14 | |||
15 | You should have received a copy of the GNU General Public License | ||
16 | along with GNUnet; see the file COPYING. If not, write to the | ||
17 | Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, | ||
18 | Boston, MA 02110-1301, USA. | ||
19 | */ | ||
20 | |||
21 | /** | ||
22 | * @file rps/gnunet-service-rps_peers.h | ||
23 | * @brief utilities for managing (information about) peers | ||
24 | * @author Julius Bünger | ||
25 | */ | ||
26 | #include "gnunet_util_lib.h" | ||
27 | #include <inttypes.h> | ||
28 | #include "gnunet_cadet_service.h" | ||
29 | |||
30 | |||
31 | /** | ||
32 | * Different flags indicating the status of another peer. | ||
33 | */ | ||
34 | enum Peers_PeerFlags | ||
35 | { | ||
36 | /** | ||
37 | * If we are waiting for a reply from that peer (sent a pull request). | ||
38 | */ | ||
39 | Peers_PULL_REPLY_PENDING = 0x01, | ||
40 | |||
41 | /* IN_OTHER_GOSSIP_LIST = 0x02, unneeded? */ | ||
42 | /* IN_OWN_SAMPLER_LIST = 0x04, unneeded? */ | ||
43 | /* IN_OWN_GOSSIP_LIST = 0x08, unneeded? */ | ||
44 | |||
45 | /** | ||
46 | * We set this bit when we know the peer is online. | ||
47 | */ | ||
48 | Peers_ONLINE = 0x20, | ||
49 | |||
50 | /** | ||
51 | * We set this bit when we are going to destroy the channel to this peer. | ||
52 | * When cleanup_channel is called, we know that we wanted to destroy it. | ||
53 | * Otherwise the channel to the other peer was destroyed. | ||
54 | */ | ||
55 | Peers_TO_DESTROY = 0x40, | ||
56 | }; | ||
57 | |||
58 | /** | ||
59 | * Keep track of the status of a channel. | ||
60 | * | ||
61 | * This is needed in order to know what to do with a channel when it's | ||
62 | * destroyed. | ||
63 | */ | ||
64 | enum Peers_ChannelFlags | ||
65 | { | ||
66 | /** | ||
67 | * We destroyed the channel because the other peer established a second one. | ||
68 | */ | ||
69 | Peers_CHANNEL_ESTABLISHED_TWICE = 0x1, | ||
70 | |||
71 | /** | ||
72 | * The channel was removed because it was not needed any more. This should be | ||
73 | * the sending channel. | ||
74 | */ | ||
75 | Peers_CHANNEL_CLEAN = 0x2, | ||
76 | }; | ||
77 | |||
78 | /** | ||
79 | * @brief The role of a channel. Sending or receiving. | ||
80 | */ | ||
81 | enum Peers_ChannelRole | ||
82 | { | ||
83 | /** | ||
84 | * Channel is used for sending | ||
85 | */ | ||
86 | Peers_CHANNEL_ROLE_SENDING = 0x01, | ||
87 | |||
88 | /** | ||
89 | * Channel is used for receiving | ||
90 | */ | ||
91 | Peers_CHANNEL_ROLE_RECEIVING = 0x02, | ||
92 | }; | ||
93 | |||
94 | /** | ||
95 | * @brief Functions of this type can be used to be stored at a peer for later execution. | ||
96 | * | ||
97 | * @param cls closure | ||
98 | * @param peer peer to execute function on | ||
99 | */ | ||
100 | typedef void (* PeerOp) (void *cls, const struct GNUNET_PeerIdentity *peer); | ||
101 | |||
102 | /** | ||
103 | * @brief Iterator over valid peers. | ||
104 | * | ||
105 | * @param cls closure | ||
106 | * @param peer current public peer id | ||
107 | * @return #GNUNET_YES if we should continue to | ||
108 | * iterate, | ||
109 | * #GNUNET_NO if not. | ||
110 | */ | ||
111 | typedef int | ||
112 | (*PeersIterator) (void *cls, | ||
113 | const struct GNUNET_PeerIdentity *peer); | ||
114 | |||
115 | /** | ||
116 | * @brief Initialise storage of peers | ||
117 | * | ||
118 | * @param fn_valid_peers filename of the file used to store valid peer ids | ||
119 | * @param cadet_h cadet handle | ||
120 | * @param disconnect_handler Disconnect handler | ||
121 | * @param c_handlers cadet handlers | ||
122 | * @param own_id own peer identity | ||
123 | */ | ||
124 | void | ||
125 | Peers_initialise (char* fn_valid_peers, | ||
126 | struct GNUNET_CADET_Handle *cadet_h, | ||
127 | GNUNET_CADET_DisconnectEventHandler disconnect_handler, | ||
128 | const struct GNUNET_MQ_MessageHandler *c_handlers, | ||
129 | const struct GNUNET_PeerIdentity *own_id); | ||
130 | |||
131 | /** | ||
132 | * @brief Delete storage of peers that was created with #Peers_initialise () | ||
133 | */ | ||
134 | void | ||
135 | Peers_terminate (); | ||
136 | |||
137 | |||
138 | /** | ||
139 | * @brief Get all currently known, valid peer ids. | ||
140 | * | ||
141 | * @param it function to call on each peer id | ||
142 | * @param it_cls extra argument to @a it | ||
143 | * @return the number of key value pairs processed, | ||
144 | * #GNUNET_SYSERR if it aborted iteration | ||
145 | */ | ||
146 | int | ||
147 | Peers_get_valid_peers (PeersIterator iterator, | ||
148 | void *it_cls); | ||
149 | |||
150 | /** | ||
151 | * @brief Add peer to known peers. | ||
152 | * | ||
153 | * This function is called on new peer_ids from 'external' sources | ||
154 | * (client seed, cadet get_peers(), ...) | ||
155 | * | ||
156 | * @param peer the new #GNUNET_PeerIdentity | ||
157 | * | ||
158 | * @return #GNUNET_YES if peer was inserted | ||
159 | * #GNUNET_NO otherwise (if peer was already known or | ||
160 | * peer was #own_identity) | ||
161 | */ | ||
162 | int | ||
163 | Peers_insert_peer (const struct GNUNET_PeerIdentity *peer); | ||
164 | |||
165 | /** | ||
166 | * @brief Try connecting to a peer to see whether it is online | ||
167 | * | ||
168 | * If not known yet, insert into known peers | ||
169 | * | ||
170 | * @param peer the peer whose liveliness is to be checked | ||
171 | * @return #GNUNET_YES if peer had to be inserted | ||
172 | * #GNUNET_NO otherwise (if peer was already known or | ||
173 | * peer was #own_identity) | ||
174 | */ | ||
175 | int | ||
176 | Peers_issue_peer_liveliness_check (const struct GNUNET_PeerIdentity *peer); | ||
177 | |||
178 | /** | ||
179 | * @brief Check if peer is removable. | ||
180 | * | ||
181 | * Check if | ||
182 | * - a recv channel exists | ||
183 | * - there are pending messages | ||
184 | * - there is no pending pull reply | ||
185 | * | ||
186 | * @param peer the peer in question | ||
187 | * @return #GNUNET_YES if peer is removable | ||
188 | * #GNUNET_NO if peer is NOT removable | ||
189 | * #GNUNET_SYSERR if peer is not known | ||
190 | */ | ||
191 | int | ||
192 | Peers_check_removable (const struct GNUNET_PeerIdentity *peer); | ||
193 | |||
194 | /** | ||
195 | * @brief Remove peer | ||
196 | * | ||
197 | * @param peer the peer to clean | ||
198 | * @return #GNUNET_YES if peer was removed | ||
199 | * #GNUNET_NO otherwise | ||
200 | */ | ||
201 | int | ||
202 | Peers_remove_peer (const struct GNUNET_PeerIdentity *peer); | ||
203 | |||
204 | /** | ||
205 | * @brief set flags on a given peer. | ||
206 | * | ||
207 | * @param peer the peer to set flags on | ||
208 | * @param flags the flags | ||
209 | */ | ||
210 | void | ||
211 | Peers_set_peer_flag (const struct GNUNET_PeerIdentity *peer, enum Peers_PeerFlags flags); | ||
212 | |||
213 | /** | ||
214 | * @brief unset flags on a given peer. | ||
215 | * | ||
216 | * @param peer the peer to unset flags on | ||
217 | * @param flags the flags | ||
218 | */ | ||
219 | void | ||
220 | Peers_unset_peer_flag (const struct GNUNET_PeerIdentity *peer, enum Peers_PeerFlags flags); | ||
221 | |||
222 | /** | ||
223 | * @brief Check whether flags on a peer are set. | ||
224 | * | ||
225 | * @param peer the peer to check the flag of | ||
226 | * @param flags the flags to check | ||
227 | * | ||
228 | * @return #GNUNET_YES if all given flags are set | ||
229 | * ##GNUNET_NO otherwise | ||
230 | */ | ||
231 | int | ||
232 | Peers_check_peer_flag (const struct GNUNET_PeerIdentity *peer, enum Peers_PeerFlags flags); | ||
233 | |||
234 | |||
235 | /** | ||
236 | * @brief set flags on a given channel. | ||
237 | * | ||
238 | * @param channel the channel to set flags on | ||
239 | * @param flags the flags | ||
240 | */ | ||
241 | void | ||
242 | Peers_set_channel_flag (uint32_t *channel_flags, enum Peers_ChannelFlags flags); | ||
243 | |||
244 | /** | ||
245 | * @brief unset flags on a given channel. | ||
246 | * | ||
247 | * @param channel the channel to unset flags on | ||
248 | * @param flags the flags | ||
249 | */ | ||
250 | void | ||
251 | Peers_unset_channel_flag (uint32_t *channel_flags, enum Peers_ChannelFlags flags); | ||
252 | |||
253 | /** | ||
254 | * @brief Check whether flags on a channel are set. | ||
255 | * | ||
256 | * @param channel the channel to check the flag of | ||
257 | * @param flags the flags to check | ||
258 | * | ||
259 | * @return #GNUNET_YES if all given flags are set | ||
260 | * #GNUNET_NO otherwise | ||
261 | */ | ||
262 | int | ||
263 | Peers_check_channel_flag (uint32_t *channel_flags, enum Peers_ChannelFlags flags); | ||
264 | |||
265 | /** | ||
266 | * @brief Get the flags for the channel in @a role for @a peer. | ||
267 | * | ||
268 | * @param peer Peer to get the channel flags for. | ||
269 | * @param role Role of channel to get flags for | ||
270 | * | ||
271 | * @return The flags. | ||
272 | */ | ||
273 | uint32_t * | ||
274 | Peers_get_channel_flag (const struct GNUNET_PeerIdentity *peer, | ||
275 | enum Peers_ChannelRole role); | ||
276 | |||
277 | /** | ||
278 | * @brief Check whether we have information about the given peer. | ||
279 | * | ||
280 | * FIXME probably deprecated. Make this the new _online. | ||
281 | * | ||
282 | * @param peer peer in question | ||
283 | * | ||
284 | * @return #GNUNET_YES if peer is known | ||
285 | * #GNUNET_NO if peer is not knwon | ||
286 | */ | ||
287 | int | ||
288 | Peers_check_peer_known (const struct GNUNET_PeerIdentity *peer); | ||
289 | |||
290 | /** | ||
291 | * @brief Check whether @a peer is actually a peer. | ||
292 | * | ||
293 | * A valid peer is a peer that we know exists eg. we were connected to once. | ||
294 | * | ||
295 | * @param peer peer in question | ||
296 | * | ||
297 | * @return #GNUNET_YES if peer is valid | ||
298 | * #GNUNET_NO if peer is not valid | ||
299 | */ | ||
300 | int | ||
301 | Peers_check_peer_valid (const struct GNUNET_PeerIdentity *peer); | ||
302 | |||
303 | /** | ||
304 | * @brief Indicate that we want to send to the other peer | ||
305 | * | ||
306 | * This establishes a sending channel | ||
307 | * | ||
308 | * @param peer the peer to establish channel to | ||
309 | */ | ||
310 | void | ||
311 | Peers_indicate_sending_intention (const struct GNUNET_PeerIdentity *peer); | ||
312 | |||
313 | /** | ||
314 | * @brief Check whether other peer has the intention to send/opened channel | ||
315 | * towars us | ||
316 | * | ||
317 | * @param peer the peer in question | ||
318 | * | ||
319 | * @return #GNUNET_YES if peer has the intention to send | ||
320 | * #GNUNET_NO otherwise | ||
321 | */ | ||
322 | int | ||
323 | Peers_check_peer_send_intention (const struct GNUNET_PeerIdentity *peer); | ||
324 | |||
325 | /** | ||
326 | * Handle the channel a peer opens to us. | ||
327 | * | ||
328 | * @param cls The closure | ||
329 | * @param channel The channel the peer wants to establish | ||
330 | * @param initiator The peer's peer ID | ||
331 | * | ||
332 | * @return initial channel context for the channel | ||
333 | * (can be NULL -- that's not an error) | ||
334 | */ | ||
335 | void * | ||
336 | Peers_handle_inbound_channel (void *cls, | ||
337 | struct GNUNET_CADET_Channel *channel, | ||
338 | const struct GNUNET_PeerIdentity *initiator); | ||
339 | |||
340 | /** | ||
341 | * @brief Check whether a sending channel towards the given peer exists | ||
342 | * | ||
343 | * @param peer the peer to check for | ||
344 | * | ||
345 | * @return #GNUNET_YES if a sending channel towards that peer exists | ||
346 | * #GNUNET_NO otherwise | ||
347 | */ | ||
348 | int | ||
349 | Peers_check_sending_channel_exists (const struct GNUNET_PeerIdentity *peer); | ||
350 | |||
351 | /** | ||
352 | * @brief check whether the given channel is the sending channel of the given | ||
353 | * peer | ||
354 | * | ||
355 | * @param peer the peer in question | ||
356 | * @param channel the channel to check for | ||
357 | * @param role either #Peers_CHANNEL_ROLE_SENDING, or | ||
358 | * #Peers_CHANNEL_ROLE_RECEIVING | ||
359 | * | ||
360 | * @return #GNUNET_YES if the given chennel is the sending channel of the peer | ||
361 | * #GNUNET_NO otherwise | ||
362 | */ | ||
363 | int | ||
364 | Peers_check_channel_role (const struct GNUNET_PeerIdentity *peer, | ||
365 | const struct GNUNET_CADET_Channel *channel, | ||
366 | enum Peers_ChannelRole role); | ||
367 | |||
368 | /** | ||
369 | * @brief Destroy the send channel of a peer e.g. stop indicating a sending | ||
370 | * intention to another peer | ||
371 | * | ||
372 | * If there is also no channel to receive messages from that peer, remove it | ||
373 | * from the peermap. | ||
374 | * | ||
375 | * @peer the peer identity of the peer whose sending channel to destroy | ||
376 | * @return #GNUNET_YES if channel was destroyed | ||
377 | * #GNUNET_NO otherwise | ||
378 | */ | ||
379 | int | ||
380 | Peers_destroy_sending_channel (const struct GNUNET_PeerIdentity *peer); | ||
381 | |||
382 | /** | ||
383 | * This is called when a channel is destroyed. | ||
384 | * | ||
385 | * Removes peer completely from our knowledge if the send_channel was destroyed | ||
386 | * Otherwise simply delete the recv_channel | ||
387 | * | ||
388 | * @param cls The closure | ||
389 | * @param channel The channel being closed | ||
390 | * @param channel_ctx The context associated with this channel | ||
391 | */ | ||
392 | void | ||
393 | Peers_cleanup_destroyed_channel (void *cls, | ||
394 | const struct GNUNET_CADET_Channel *channel); | ||
395 | |||
396 | /** | ||
397 | * @brief Send a message to another peer. | ||
398 | * | ||
399 | * Keeps track about pending messages so they can be properly removed when the | ||
400 | * peer is destroyed. | ||
401 | * | ||
402 | * @param peer receeiver of the message | ||
403 | * @param ev envelope of the message | ||
404 | * @param type type of the message | ||
405 | */ | ||
406 | void | ||
407 | Peers_send_message (const struct GNUNET_PeerIdentity *peer, | ||
408 | struct GNUNET_MQ_Envelope *ev, | ||
409 | const char *type); | ||
410 | |||
411 | /** | ||
412 | * @brief Schedule a operation on given peer | ||
413 | * | ||
414 | * Avoids scheduling an operation twice. | ||
415 | * | ||
416 | * @param peer the peer we want to schedule the operation for once it gets live | ||
417 | * | ||
418 | * @return #GNUNET_YES if the operation was scheduled | ||
419 | * #GNUNET_NO otherwise | ||
420 | */ | ||
421 | int | ||
422 | Peers_schedule_operation (const struct GNUNET_PeerIdentity *peer, | ||
423 | const PeerOp peer_op); | ||
424 | |||
425 | /** | ||
426 | * @brief Get the recv_channel of @a peer. | ||
427 | * Needed to correctly handle (call #GNUNET_CADET_receive_done()) incoming | ||
428 | * messages. | ||
429 | * | ||
430 | * @param peer The peer to get the recv_channel from. | ||
431 | * | ||
432 | * @return The recv_channel. | ||
433 | */ | ||
434 | struct GNUNET_CADET_Channel * | ||
435 | Peers_get_recv_channel (const struct GNUNET_PeerIdentity *peer); | ||
436 | |||
437 | /* end of gnunet-service-rps_peers.h */ | ||
diff --git a/src/rps/rps.h b/src/rps/rps.h index 3037e2190..f5cc2e8d1 100644 --- a/src/rps/rps.h +++ b/src/rps/rps.h | |||
@@ -175,4 +175,100 @@ struct GNUNET_RPS_CS_ActMaliciousMessage | |||
175 | }; | 175 | }; |
176 | #endif /* ENABLE_MALICIOUS */ | 176 | #endif /* ENABLE_MALICIOUS */ |
177 | 177 | ||
178 | |||
179 | /*********************************************************************** | ||
180 | * Defines from old gnunet-service-rps_peers.h | ||
181 | ***********************************************************************/ | ||
182 | |||
183 | /** | ||
184 | * Different flags indicating the status of another peer. | ||
185 | */ | ||
186 | enum Peers_PeerFlags | ||
187 | { | ||
188 | /** | ||
189 | * If we are waiting for a reply from that peer (sent a pull request). | ||
190 | */ | ||
191 | Peers_PULL_REPLY_PENDING = 0x01, | ||
192 | |||
193 | /* IN_OTHER_GOSSIP_LIST = 0x02, unneeded? */ | ||
194 | /* IN_OWN_SAMPLER_LIST = 0x04, unneeded? */ | ||
195 | /* IN_OWN_GOSSIP_LIST = 0x08, unneeded? */ | ||
196 | |||
197 | /** | ||
198 | * We set this bit when we know the peer is online. | ||
199 | */ | ||
200 | Peers_ONLINE = 0x20, | ||
201 | |||
202 | /** | ||
203 | * We set this bit when we are going to destroy the channel to this peer. | ||
204 | * When cleanup_channel is called, we know that we wanted to destroy it. | ||
205 | * Otherwise the channel to the other peer was destroyed. | ||
206 | */ | ||
207 | Peers_TO_DESTROY = 0x40, | ||
208 | }; | ||
209 | |||
210 | /** | ||
211 | * Keep track of the status of a channel. | ||
212 | * | ||
213 | * This is needed in order to know what to do with a channel when it's | ||
214 | * destroyed. | ||
215 | */ | ||
216 | enum Peers_ChannelFlags | ||
217 | { | ||
218 | /** | ||
219 | * We destroyed the channel because the other peer established a second one. | ||
220 | */ | ||
221 | Peers_CHANNEL_ESTABLISHED_TWICE = 0x1, | ||
222 | |||
223 | /** | ||
224 | * The channel was removed because it was not needed any more. This should be | ||
225 | * the sending channel. | ||
226 | */ | ||
227 | Peers_CHANNEL_CLEAN = 0x2, | ||
228 | |||
229 | /** | ||
230 | * We destroyed the channel because the other peer established a second one. | ||
231 | */ | ||
232 | Peers_CHANNEL_DESTROING = 0x4, | ||
233 | }; | ||
234 | |||
235 | |||
236 | /** | ||
237 | * @brief The role of a channel. Sending or receiving. | ||
238 | */ | ||
239 | enum Peers_ChannelRole | ||
240 | { | ||
241 | /** | ||
242 | * Channel is used for sending | ||
243 | */ | ||
244 | Peers_CHANNEL_ROLE_SENDING = 0x01, | ||
245 | |||
246 | /** | ||
247 | * Channel is used for receiving | ||
248 | */ | ||
249 | Peers_CHANNEL_ROLE_RECEIVING = 0x02, | ||
250 | }; | ||
251 | |||
252 | /** | ||
253 | * @brief Functions of this type can be used to be stored at a peer for later execution. | ||
254 | * | ||
255 | * @param cls closure | ||
256 | * @param peer peer to execute function on | ||
257 | */ | ||
258 | typedef void (* PeerOp) (void *cls, const struct GNUNET_PeerIdentity *peer); | ||
259 | |||
260 | /** | ||
261 | * @brief Iterator over valid peers. | ||
262 | * | ||
263 | * @param cls closure | ||
264 | * @param peer current public peer id | ||
265 | * @return #GNUNET_YES if we should continue to | ||
266 | * iterate, | ||
267 | * #GNUNET_NO if not. | ||
268 | */ | ||
269 | typedef int | ||
270 | (*PeersIterator) (void *cls, | ||
271 | const struct GNUNET_PeerIdentity *peer); | ||
272 | |||
273 | |||
178 | GNUNET_NETWORK_STRUCT_END | 274 | GNUNET_NETWORK_STRUCT_END |
diff --git a/src/rps/test_rps.c b/src/rps/test_rps.c index acd3a165d..0114a19fe 100644 --- a/src/rps/test_rps.c +++ b/src/rps/test_rps.c | |||
@@ -18,7 +18,7 @@ | |||
18 | Boston, MA 02110-1301, USA. | 18 | Boston, MA 02110-1301, USA. |
19 | */ | 19 | */ |
20 | /** | 20 | /** |
21 | * @file rps/test_rps_multipeer.c | 21 | * @file rps/test_rps.c |
22 | * @brief Testcase for the random peer sampling service. Starts | 22 | * @brief Testcase for the random peer sampling service. Starts |
23 | * a peergroup with a given number of peers, then waits to | 23 | * a peergroup with a given number of peers, then waits to |
24 | * receive size pushes/pulls from each peer. Expects to wait | 24 | * receive size pushes/pulls from each peer. Expects to wait |
@@ -344,6 +344,11 @@ struct SingleTestRun | |||
344 | * Number of Requests to make. | 344 | * Number of Requests to make. |
345 | */ | 345 | */ |
346 | uint32_t num_requests; | 346 | uint32_t num_requests; |
347 | |||
348 | /** | ||
349 | * Run with churn | ||
350 | */ | ||
351 | int have_churn; | ||
347 | } cur_test_run; | 352 | } cur_test_run; |
348 | 353 | ||
349 | /** | 354 | /** |
@@ -1023,6 +1028,33 @@ req_cancel_cb (struct RPSPeer *rps_peer) | |||
1023 | } | 1028 | } |
1024 | 1029 | ||
1025 | /*********************************** | 1030 | /*********************************** |
1031 | * CHURN | ||
1032 | ***********************************/ | ||
1033 | |||
1034 | static void | ||
1035 | churn (void *cls); | ||
1036 | |||
1037 | static void | ||
1038 | churn_test_cb (struct RPSPeer *rps_peer) | ||
1039 | { | ||
1040 | /* Start churn */ | ||
1041 | if (GNUNET_YES == cur_test_run.have_churn && NULL == churn_task) | ||
1042 | { | ||
1043 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
1044 | "Starting churn task\n"); | ||
1045 | churn_task = GNUNET_SCHEDULER_add_delayed ( | ||
1046 | GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 5), | ||
1047 | churn, | ||
1048 | NULL); | ||
1049 | } else { | ||
1050 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
1051 | "Not starting churn task\n"); | ||
1052 | } | ||
1053 | |||
1054 | schedule_missing_requests (rps_peer); | ||
1055 | } | ||
1056 | |||
1057 | /*********************************** | ||
1026 | * PROFILER | 1058 | * PROFILER |
1027 | ***********************************/ | 1059 | ***********************************/ |
1028 | 1060 | ||
@@ -1148,6 +1180,9 @@ churn (void *cls) | |||
1148 | double portion_go_online; | 1180 | double portion_go_online; |
1149 | double portion_go_offline; | 1181 | double portion_go_offline; |
1150 | 1182 | ||
1183 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
1184 | "Churn function executing\n"); | ||
1185 | |||
1151 | /* Compute the probability for an online peer to go offline | 1186 | /* Compute the probability for an online peer to go offline |
1152 | * this round */ | 1187 | * this round */ |
1153 | portion_online = num_peers_online * 1.0 / num_peers; | 1188 | portion_online = num_peers_online * 1.0 / num_peers; |
@@ -1256,12 +1291,17 @@ static void | |||
1256 | profiler_cb (struct RPSPeer *rps_peer) | 1291 | profiler_cb (struct RPSPeer *rps_peer) |
1257 | { | 1292 | { |
1258 | /* Start churn */ | 1293 | /* Start churn */ |
1259 | if (NULL == churn_task) | 1294 | if (GNUNET_YES == cur_test_run.have_churn && NULL == churn_task) |
1260 | { | 1295 | { |
1296 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
1297 | "Starting churn task\n"); | ||
1261 | churn_task = GNUNET_SCHEDULER_add_delayed ( | 1298 | churn_task = GNUNET_SCHEDULER_add_delayed ( |
1262 | GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 5), | 1299 | GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 5), |
1263 | churn, | 1300 | churn, |
1264 | NULL); | 1301 | NULL); |
1302 | } else { | ||
1303 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
1304 | "Not starting churn task\n"); | ||
1265 | } | 1305 | } |
1266 | 1306 | ||
1267 | /* Only request peer ids at one peer. | 1307 | /* Only request peer ids at one peer. |
@@ -1353,6 +1393,24 @@ run (void *cls, | |||
1353 | struct OpListEntry *entry; | 1393 | struct OpListEntry *entry; |
1354 | uint32_t num_mal_peers; | 1394 | uint32_t num_mal_peers; |
1355 | 1395 | ||
1396 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "RUN was called\n"); | ||
1397 | |||
1398 | /* Check whether we timed out */ | ||
1399 | if (n_peers != num_peers || | ||
1400 | NULL == peers || | ||
1401 | 0 == links_succeeded) | ||
1402 | { | ||
1403 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Going down due to args (eg. timeout)\n"); | ||
1404 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "\tn_peers: %u\n", n_peers); | ||
1405 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "\tnum_peers: %" PRIu32 "\n", num_peers); | ||
1406 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "\tpeers: %p\n", peers); | ||
1407 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "\tlinks_succeeded: %u\n", links_succeeded); | ||
1408 | GNUNET_SCHEDULER_shutdown (); | ||
1409 | return; | ||
1410 | } | ||
1411 | |||
1412 | |||
1413 | /* Initialize peers */ | ||
1356 | testbed_peers = peers; | 1414 | testbed_peers = peers; |
1357 | num_peers_online = 0; | 1415 | num_peers_online = 0; |
1358 | for (i = 0; i < num_peers; i++) | 1416 | for (i = 0; i < num_peers; i++) |
@@ -1412,6 +1470,7 @@ main (int argc, char *argv[]) | |||
1412 | cur_test_run.pre_test = NULL; | 1470 | cur_test_run.pre_test = NULL; |
1413 | cur_test_run.reply_handle = default_reply_handle; | 1471 | cur_test_run.reply_handle = default_reply_handle; |
1414 | cur_test_run.eval_cb = default_eval_cb; | 1472 | cur_test_run.eval_cb = default_eval_cb; |
1473 | cur_test_run.have_churn = GNUNET_YES; | ||
1415 | churn_task = NULL; | 1474 | churn_task = NULL; |
1416 | timeout = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 30); | 1475 | timeout = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 30); |
1417 | 1476 | ||
@@ -1446,6 +1505,7 @@ main (int argc, char *argv[]) | |||
1446 | GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Test single request\n"); | 1505 | GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Test single request\n"); |
1447 | cur_test_run.name = "test-rps-single-req"; | 1506 | cur_test_run.name = "test-rps-single-req"; |
1448 | cur_test_run.main_test = single_req_cb; | 1507 | cur_test_run.main_test = single_req_cb; |
1508 | cur_test_run.have_churn = GNUNET_NO; | ||
1449 | } | 1509 | } |
1450 | 1510 | ||
1451 | else if (strstr (argv[0], "_delayed_reqs") != NULL) | 1511 | else if (strstr (argv[0], "_delayed_reqs") != NULL) |
@@ -1453,6 +1513,7 @@ main (int argc, char *argv[]) | |||
1453 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Test delayed requests\n"); | 1513 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Test delayed requests\n"); |
1454 | cur_test_run.name = "test-rps-delayed-reqs"; | 1514 | cur_test_run.name = "test-rps-delayed-reqs"; |
1455 | cur_test_run.main_test = delay_req_cb; | 1515 | cur_test_run.main_test = delay_req_cb; |
1516 | cur_test_run.have_churn = GNUNET_NO; | ||
1456 | } | 1517 | } |
1457 | 1518 | ||
1458 | else if (strstr (argv[0], "_seed_big") != NULL) | 1519 | else if (strstr (argv[0], "_seed_big") != NULL) |
@@ -1462,6 +1523,7 @@ main (int argc, char *argv[]) | |||
1462 | cur_test_run.name = "test-rps-seed-big"; | 1523 | cur_test_run.name = "test-rps-seed-big"; |
1463 | cur_test_run.main_test = seed_big_cb; | 1524 | cur_test_run.main_test = seed_big_cb; |
1464 | cur_test_run.eval_cb = no_eval; | 1525 | cur_test_run.eval_cb = no_eval; |
1526 | cur_test_run.have_churn = GNUNET_NO; | ||
1465 | timeout = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 10); | 1527 | timeout = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 10); |
1466 | } | 1528 | } |
1467 | 1529 | ||
@@ -1470,6 +1532,7 @@ main (int argc, char *argv[]) | |||
1470 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Test seeding and requesting on a single peer\n"); | 1532 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Test seeding and requesting on a single peer\n"); |
1471 | cur_test_run.name = "test-rps-single-peer-seed"; | 1533 | cur_test_run.name = "test-rps-single-peer-seed"; |
1472 | cur_test_run.main_test = single_peer_seed_cb; | 1534 | cur_test_run.main_test = single_peer_seed_cb; |
1535 | cur_test_run.have_churn = GNUNET_NO; | ||
1473 | } | 1536 | } |
1474 | 1537 | ||
1475 | else if (strstr (argv[0], "_seed_request") != NULL) | 1538 | else if (strstr (argv[0], "_seed_request") != NULL) |
@@ -1477,6 +1540,7 @@ main (int argc, char *argv[]) | |||
1477 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Test seeding and requesting on multiple peers\n"); | 1540 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Test seeding and requesting on multiple peers\n"); |
1478 | cur_test_run.name = "test-rps-seed-request"; | 1541 | cur_test_run.name = "test-rps-seed-request"; |
1479 | cur_test_run.main_test = seed_req_cb; | 1542 | cur_test_run.main_test = seed_req_cb; |
1543 | cur_test_run.have_churn = GNUNET_NO; | ||
1480 | } | 1544 | } |
1481 | 1545 | ||
1482 | else if (strstr (argv[0], "_seed") != NULL) | 1546 | else if (strstr (argv[0], "_seed") != NULL) |
@@ -1485,6 +1549,7 @@ main (int argc, char *argv[]) | |||
1485 | cur_test_run.name = "test-rps-seed"; | 1549 | cur_test_run.name = "test-rps-seed"; |
1486 | cur_test_run.main_test = seed_cb; | 1550 | cur_test_run.main_test = seed_cb; |
1487 | cur_test_run.eval_cb = no_eval; | 1551 | cur_test_run.eval_cb = no_eval; |
1552 | cur_test_run.have_churn = GNUNET_NO; | ||
1488 | } | 1553 | } |
1489 | 1554 | ||
1490 | else if (strstr (argv[0], "_req_cancel") != NULL) | 1555 | else if (strstr (argv[0], "_req_cancel") != NULL) |
@@ -1494,6 +1559,20 @@ main (int argc, char *argv[]) | |||
1494 | num_peers = 1; | 1559 | num_peers = 1; |
1495 | cur_test_run.main_test = req_cancel_cb; | 1560 | cur_test_run.main_test = req_cancel_cb; |
1496 | cur_test_run.eval_cb = no_eval; | 1561 | cur_test_run.eval_cb = no_eval; |
1562 | cur_test_run.have_churn = GNUNET_NO; | ||
1563 | timeout = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 10); | ||
1564 | } | ||
1565 | |||
1566 | else if (strstr (argv[0], "_churn") != NULL) | ||
1567 | { | ||
1568 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Test churn\n"); | ||
1569 | cur_test_run.name = "test-rps-churn"; | ||
1570 | num_peers = 5; | ||
1571 | cur_test_run.init_peer = default_init_peer; | ||
1572 | cur_test_run.main_test = churn_test_cb; | ||
1573 | cur_test_run.reply_handle = default_reply_handle; | ||
1574 | cur_test_run.eval_cb = default_eval_cb; | ||
1575 | cur_test_run.have_churn = GNUNET_YES; | ||
1497 | timeout = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 10); | 1576 | timeout = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 10); |
1498 | } | 1577 | } |
1499 | 1578 | ||
@@ -1510,6 +1589,7 @@ main (int argc, char *argv[]) | |||
1510 | cur_test_run.eval_cb = profiler_eval; | 1589 | cur_test_run.eval_cb = profiler_eval; |
1511 | cur_test_run.request_interval = 2; | 1590 | cur_test_run.request_interval = 2; |
1512 | cur_test_run.num_requests = 5; | 1591 | cur_test_run.num_requests = 5; |
1592 | cur_test_run.have_churn = GNUNET_YES; | ||
1513 | timeout = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 90); | 1593 | timeout = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 90); |
1514 | 1594 | ||
1515 | /* 'Clean' directory */ | 1595 | /* 'Clean' directory */ |
@@ -1542,4 +1622,4 @@ main (int argc, char *argv[]) | |||
1542 | return ret_value; | 1622 | return ret_value; |
1543 | } | 1623 | } |
1544 | 1624 | ||
1545 | /* end of test_rps_multipeer.c */ | 1625 | /* end of test_rps.c */ |
diff --git a/src/rps/test_rps.conf b/src/rps/test_rps.conf index 7da91ccf0..fce07c945 100644 --- a/src/rps/test_rps.conf +++ b/src/rps/test_rps.conf | |||
@@ -24,16 +24,16 @@ INITSIZE = 4 | |||
24 | [testbed] | 24 | [testbed] |
25 | HOSTNAME = localhost | 25 | HOSTNAME = localhost |
26 | 26 | ||
27 | OPERATION_TIMEOUT = 60 s | 27 | # OPERATION_TIMEOUT = 60 s |
28 | 28 | ||
29 | MAX_PARALLEL_TOPOLOGY_CONFIG_OPERATIONS = 1 | 29 | # MAX_PARALLEL_TOPOLOGY_CONFIG_OPERATIONS = 100 |
30 | #OVERLAY_TOPOLOGY = CLIQUE | 30 | OVERLAY_TOPOLOGY = CLIQUE |
31 | OVERLAY_TOPOLOGY = SMALL_WORLD | 31 | #OVERLAY_TOPOLOGY = SMALL_WORLD |
32 | #SCALE_FREE_TOPOLOGY_CAP = | 32 | #SCALE_FREE_TOPOLOGY_CAP = |
33 | 33 | ||
34 | OVERLAY_RANDOM_LINKS = 25 | 34 | # OVERLAY_RANDOM_LINKS = 25 |
35 | 35 | ||
36 | SETUP_TIMEOUT = 2 m | 36 | # SETUP_TIMEOUT = 2 m |
37 | 37 | ||
38 | [nse] | 38 | [nse] |
39 | WORKBITS = 0 | 39 | WORKBITS = 0 |
@@ -46,7 +46,27 @@ USE_LOCALADDR = YES | |||
46 | RETURN_LOCAL_ADDRESSES = YES | 46 | RETURN_LOCAL_ADDRESSES = YES |
47 | 47 | ||
48 | [transport] | 48 | [transport] |
49 | PLUGINS = unix | 49 | PLUGINS = udp |
50 | |||
51 | [ats] | ||
52 | # Network specific inbound/outbound quotas | ||
53 | UNSPECIFIED_QUOTA_IN = unlimited | ||
54 | UNSPECIFIED_QUOTA_OUT = unlimited | ||
55 | # LOOPBACK | ||
56 | LOOPBACK_QUOTA_IN = unlimited | ||
57 | LOOPBACK_QUOTA_OUT = unlimited | ||
58 | # LAN | ||
59 | LAN_QUOTA_IN = unlimited | ||
60 | LAN_QUOTA_OUT = unlimited | ||
61 | #WAN | ||
62 | WAN_QUOTA_OUT = unlimited | ||
63 | WAN_QUOTA_IN = unlimited | ||
64 | # WLAN | ||
65 | WLAN_QUOTA_IN = unlimited | ||
66 | WLAN_QUOTA_OUT = unlimited | ||
67 | # BLUETOOTH | ||
68 | BLUETOOTH_QUOTA_IN = unlimited | ||
69 | BLUETOOTH_QUOTA_OUT = unlimited | ||
50 | 70 | ||
51 | [dht] | 71 | [dht] |
52 | DISABLE_TRY_CONNECT = YES | 72 | DISABLE_TRY_CONNECT = YES |
@@ -69,6 +89,10 @@ NO_IO = YES | |||
69 | FORCESTART = NO | 89 | FORCESTART = NO |
70 | AUTOSTART = NO | 90 | AUTOSTART = NO |
71 | 91 | ||
92 | [zonemaster] | ||
93 | FORCESTART = NO | ||
94 | AUTOSTART = NO | ||
95 | |||
72 | [namecache] | 96 | [namecache] |
73 | FORCESTART = NO | 97 | FORCESTART = NO |
74 | AUTOSTART = NO | 98 | AUTOSTART = NO |
diff --git a/src/rps/test_service_rps_peers.c b/src/rps/test_service_rps_peers.c deleted file mode 100644 index 9cd677fef..000000000 --- a/src/rps/test_service_rps_peers.c +++ /dev/null | |||
@@ -1,137 +0,0 @@ | |||
1 | /* | ||
2 | This file is part of GNUnet. | ||
3 | Copyright (C) | ||
4 | |||
5 | GNUnet is free software; you can redistribute it and/or modify | ||
6 | it under the terms of the GNU General Public License as published | ||
7 | by the Free Software Foundation; either version 3, or (at your | ||
8 | option) any later version. | ||
9 | |||
10 | GNUnet is distributed in the hope that it will be useful, but | ||
11 | WITHOUT ANY WARRANTY; without even the implied warranty of | ||
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
13 | General Public License for more details. | ||
14 | |||
15 | You should have received a copy of the GNU General Public License | ||
16 | along with GNUnet; see the file COPYING. If not, write to the | ||
17 | Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, | ||
18 | Boston, MA 02110-1301, USA. | ||
19 | */ | ||
20 | /** | ||
21 | * @file rps/test_service_rps_peers.c | ||
22 | * @brief testcase for gnunet-service-rps_peers.c | ||
23 | */ | ||
24 | #include <platform.h> | ||
25 | #include "gnunet-service-rps_peers.h" | ||
26 | |||
27 | #define ABORT() { fprintf(stderr, "Error at %s:%d\n", __FILE__, __LINE__); Peers_terminate (); return 1; } | ||
28 | #define CHECK(c) { if (! (c)) ABORT(); } | ||
29 | |||
30 | #define FN_VALID_PEERS "DISABLE" | ||
31 | |||
32 | /** | ||
33 | * @brief Dummy implementation of #PeerOp (Operation on peer) | ||
34 | * | ||
35 | * @param cls closure | ||
36 | * @param peer peer | ||
37 | */ | ||
38 | void peer_op (void *cls, const struct GNUNET_PeerIdentity *peer) | ||
39 | { | ||
40 | GNUNET_assert (NULL != peer); | ||
41 | } | ||
42 | |||
43 | /** | ||
44 | * @brief Function that is called on a peer for later execution | ||
45 | * | ||
46 | * @param cls closure | ||
47 | * @param peer peer to execute function upon | ||
48 | */ | ||
49 | void | ||
50 | peer_op (void *cls, const struct GNUNET_PeerIdentity *peer); | ||
51 | |||
52 | static int | ||
53 | check () | ||
54 | { | ||
55 | struct GNUNET_PeerIdentity k1; | ||
56 | struct GNUNET_PeerIdentity own_id; | ||
57 | |||
58 | memset (&k1, 0, sizeof (k1)); | ||
59 | memset (&own_id, 1, sizeof (own_id)); | ||
60 | |||
61 | /* Do nothing */ | ||
62 | Peers_initialise (FN_VALID_PEERS, NULL, NULL, NULL, &own_id); | ||
63 | Peers_terminate (); | ||
64 | |||
65 | |||
66 | /* Create peer */ | ||
67 | Peers_initialise (FN_VALID_PEERS, NULL, NULL, NULL, &own_id); | ||
68 | CHECK (GNUNET_YES == Peers_insert_peer (&k1)); | ||
69 | Peers_terminate (); | ||
70 | |||
71 | |||
72 | /* Create peer */ | ||
73 | Peers_initialise (FN_VALID_PEERS, NULL, NULL, NULL, &own_id); | ||
74 | CHECK (GNUNET_YES == Peers_insert_peer (&k1)); | ||
75 | CHECK (GNUNET_YES == Peers_remove_peer (&k1)); | ||
76 | Peers_terminate (); | ||
77 | |||
78 | |||
79 | /* Insertion and Removal */ | ||
80 | Peers_initialise (FN_VALID_PEERS, NULL, NULL, NULL, &own_id); | ||
81 | CHECK (GNUNET_NO == Peers_check_peer_known (&k1)); | ||
82 | |||
83 | CHECK (GNUNET_YES == Peers_insert_peer (&k1)); | ||
84 | CHECK (GNUNET_NO == Peers_insert_peer (&k1)); | ||
85 | CHECK (GNUNET_YES == Peers_check_peer_known (&k1)); | ||
86 | |||
87 | CHECK (GNUNET_YES == Peers_remove_peer (&k1)); | ||
88 | CHECK (GNUNET_NO == Peers_remove_peer (&k1)); | ||
89 | CHECK (GNUNET_NO == Peers_check_peer_known (&k1)); | ||
90 | |||
91 | |||
92 | /* Flags */ | ||
93 | Peers_insert_peer (&k1); | ||
94 | |||
95 | CHECK (GNUNET_NO == Peers_check_peer_flag (&k1, Peers_PULL_REPLY_PENDING)); | ||
96 | CHECK (GNUNET_NO == Peers_check_peer_flag (&k1, Peers_ONLINE)); | ||
97 | CHECK (GNUNET_NO == Peers_check_peer_flag (&k1, Peers_TO_DESTROY)); | ||
98 | |||
99 | CHECK (GNUNET_NO == Peers_check_peer_flag (&k1, Peers_ONLINE)); | ||
100 | |||
101 | Peers_set_peer_flag (&k1, Peers_ONLINE); | ||
102 | CHECK (GNUNET_YES == Peers_check_peer_flag (&k1, Peers_ONLINE)); | ||
103 | CHECK (GNUNET_NO == Peers_check_peer_flag (&k1, Peers_TO_DESTROY)); | ||
104 | CHECK (GNUNET_YES == Peers_check_peer_flag (&k1, Peers_ONLINE)); | ||
105 | CHECK (GNUNET_NO == Peers_check_peer_flag (&k1, Peers_TO_DESTROY)); | ||
106 | |||
107 | /* Check send intention */ | ||
108 | CHECK (GNUNET_NO == Peers_check_peer_send_intention (&k1)); | ||
109 | |||
110 | /* Check existence of sending channel */ | ||
111 | CHECK (GNUNET_NO == Peers_check_sending_channel_exists (&k1)); | ||
112 | |||
113 | /* Check role of channels */ | ||
114 | CHECK (GNUNET_YES == Peers_check_channel_role (&k1, | ||
115 | NULL, | ||
116 | Peers_CHANNEL_ROLE_SENDING)); | ||
117 | CHECK (GNUNET_YES == Peers_check_channel_role (&k1, | ||
118 | NULL, | ||
119 | Peers_CHANNEL_ROLE_RECEIVING)); | ||
120 | |||
121 | CHECK (GNUNET_YES == Peers_schedule_operation (&k1, peer_op)); | ||
122 | |||
123 | Peers_terminate (); | ||
124 | return 0; | ||
125 | } | ||
126 | |||
127 | |||
128 | int | ||
129 | main (int argc, char *argv[]) | ||
130 | { | ||
131 | GNUNET_log_setup ("test_service_rps_peers", | ||
132 | "WARNING", | ||
133 | NULL); | ||
134 | return check (); | ||
135 | } | ||
136 | |||
137 | /* end of test_service_rps_peers.c */ | ||
diff --git a/src/scalarproduct/gnunet-service-scalarproduct-ecc_alice.c b/src/scalarproduct/gnunet-service-scalarproduct-ecc_alice.c index c0b33f8ef..57f275c81 100644 --- a/src/scalarproduct/gnunet-service-scalarproduct-ecc_alice.c +++ b/src/scalarproduct/gnunet-service-scalarproduct-ecc_alice.c | |||
@@ -260,8 +260,6 @@ destroy_service_session (struct AliceServiceSession *s) | |||
260 | } | 260 | } |
261 | if (NULL != s->intersection_listen) | 261 | if (NULL != s->intersection_listen) |
262 | { | 262 | { |
263 | LOG (GNUNET_ERROR_TYPE_DEBUG, | ||
264 | "Set intersection, listen still up!\n"); | ||
265 | GNUNET_SET_listen_cancel (s->intersection_listen); | 263 | GNUNET_SET_listen_cancel (s->intersection_listen); |
266 | s->intersection_listen = NULL; | 264 | s->intersection_listen = NULL; |
267 | } | 265 | } |
@@ -274,8 +272,6 @@ destroy_service_session (struct AliceServiceSession *s) | |||
274 | } | 272 | } |
275 | if (NULL != s->intersection_set) | 273 | if (NULL != s->intersection_set) |
276 | { | 274 | { |
277 | LOG (GNUNET_ERROR_TYPE_DEBUG, | ||
278 | "Set intersection, set still there!\n"); | ||
279 | GNUNET_SET_destroy (s->intersection_set); | 275 | GNUNET_SET_destroy (s->intersection_set); |
280 | s->intersection_set = NULL; | 276 | s->intersection_set = NULL; |
281 | } | 277 | } |
@@ -809,10 +805,6 @@ cb_intersection_request_alice (void *cls, | |||
809 | prepare_client_end_notification (s); | 805 | prepare_client_end_notification (s); |
810 | return; | 806 | return; |
811 | } | 807 | } |
812 | GNUNET_SET_destroy (s->intersection_set); | ||
813 | s->intersection_set = NULL; | ||
814 | GNUNET_SET_listen_cancel (s->intersection_listen); | ||
815 | s->intersection_listen = NULL; | ||
816 | } | 808 | } |
817 | 809 | ||
818 | 810 | ||
diff --git a/src/scalarproduct/gnunet-service-scalarproduct_alice.c b/src/scalarproduct/gnunet-service-scalarproduct_alice.c index a55d03900..fcb1ce032 100644 --- a/src/scalarproduct/gnunet-service-scalarproduct_alice.c +++ b/src/scalarproduct/gnunet-service-scalarproduct_alice.c | |||
@@ -243,8 +243,6 @@ free_element_cb (void *cls, | |||
243 | static void | 243 | static void |
244 | destroy_service_session (struct AliceServiceSession *s) | 244 | destroy_service_session (struct AliceServiceSession *s) |
245 | { | 245 | { |
246 | unsigned int i; | ||
247 | |||
248 | if (GNUNET_YES == s->in_destroy) | 246 | if (GNUNET_YES == s->in_destroy) |
249 | return; | 247 | return; |
250 | s->in_destroy = GNUNET_YES; | 248 | s->in_destroy = GNUNET_YES; |
@@ -285,7 +283,7 @@ destroy_service_session (struct AliceServiceSession *s) | |||
285 | } | 283 | } |
286 | if (NULL != s->sorted_elements) | 284 | if (NULL != s->sorted_elements) |
287 | { | 285 | { |
288 | for (i=0;i<s->used_element_count;i++) | 286 | for (unsigned int i=0;i<s->used_element_count;i++) |
289 | gcry_mpi_release (s->sorted_elements[i].value); | 287 | gcry_mpi_release (s->sorted_elements[i].value); |
290 | GNUNET_free (s->sorted_elements); | 288 | GNUNET_free (s->sorted_elements); |
291 | s->sorted_elements = NULL; | 289 | s->sorted_elements = NULL; |
@@ -1043,10 +1041,6 @@ cb_intersection_request_alice (void *cls, | |||
1043 | prepare_client_end_notification (s); | 1041 | prepare_client_end_notification (s); |
1044 | return; | 1042 | return; |
1045 | } | 1043 | } |
1046 | GNUNET_SET_destroy (s->intersection_set); | ||
1047 | s->intersection_set = NULL; | ||
1048 | GNUNET_SET_listen_cancel (s->intersection_listen); | ||
1049 | s->intersection_listen = NULL; | ||
1050 | } | 1044 | } |
1051 | 1045 | ||
1052 | 1046 | ||
diff --git a/src/secretsharing/Makefile.am b/src/secretsharing/Makefile.am index 5ab8739af..c808e8200 100644 --- a/src/secretsharing/Makefile.am +++ b/src/secretsharing/Makefile.am | |||
@@ -47,7 +47,7 @@ libgnunetsecretsharing_la_SOURCES = \ | |||
47 | secretsharing_api.c \ | 47 | secretsharing_api.c \ |
48 | secretsharing_common.c \ | 48 | secretsharing_common.c \ |
49 | secretsharing.h | 49 | secretsharing.h |
50 | libgnunetsecretsharing_la_LIBADD = \ | 50 | libgnunetsecretsharing_la_LIBADD = \ |
51 | $(top_builddir)/src/util/libgnunetutil.la \ | 51 | $(top_builddir)/src/util/libgnunetutil.la \ |
52 | $(LIBGCRYPT_LIBS) \ | 52 | $(LIBGCRYPT_LIBS) \ |
53 | $(LTLIBINTL) | 53 | $(LTLIBINTL) |
diff --git a/src/set/gnunet-service-set.c b/src/set/gnunet-service-set.c index f98d43a7d..42d06b275 100644 --- a/src/set/gnunet-service-set.c +++ b/src/set/gnunet-service-set.c | |||
@@ -155,6 +155,17 @@ static struct Listener *listener_head; | |||
155 | static struct Listener *listener_tail; | 155 | static struct Listener *listener_tail; |
156 | 156 | ||
157 | /** | 157 | /** |
158 | * Number of active clients. | ||
159 | */ | ||
160 | static unsigned int num_clients; | ||
161 | |||
162 | /** | ||
163 | * Are we in shutdown? if #GNUNET_YES and the number of clients | ||
164 | * drops to zero, disconnect from CADET. | ||
165 | */ | ||
166 | static int in_shutdown; | ||
167 | |||
168 | /** | ||
158 | * Counter for allocating unique IDs for clients, used to identify | 169 | * Counter for allocating unique IDs for clients, used to identify |
159 | * incoming operation requests from remote peers, that the client can | 170 | * incoming operation requests from remote peers, that the client can |
160 | * choose to accept or refuse. 0 must not be used (reserved for | 171 | * choose to accept or refuse. 0 must not be used (reserved for |
@@ -485,6 +496,7 @@ client_connect_cb (void *cls, | |||
485 | { | 496 | { |
486 | struct ClientState *cs; | 497 | struct ClientState *cs; |
487 | 498 | ||
499 | num_clients++; | ||
488 | cs = GNUNET_new (struct ClientState); | 500 | cs = GNUNET_new (struct ClientState); |
489 | cs->client = c; | 501 | cs->client = c; |
490 | cs->mq = mq; | 502 | cs->mq = mq; |
@@ -616,13 +628,29 @@ client_disconnect_cb (void *cls, | |||
616 | GNUNET_CADET_close_port (listener->open_port); | 628 | GNUNET_CADET_close_port (listener->open_port); |
617 | listener->open_port = NULL; | 629 | listener->open_port = NULL; |
618 | while (NULL != (op = listener->op_head)) | 630 | while (NULL != (op = listener->op_head)) |
631 | { | ||
632 | GNUNET_log (GNUNET_ERROR_TYPE_INFO, | ||
633 | "Destroying incoming operation `%u' from peer `%s'\n", | ||
634 | (unsigned int) op->client_request_id, | ||
635 | GNUNET_i2s (&op->peer)); | ||
619 | incoming_destroy (op); | 636 | incoming_destroy (op); |
637 | } | ||
620 | GNUNET_CONTAINER_DLL_remove (listener_head, | 638 | GNUNET_CONTAINER_DLL_remove (listener_head, |
621 | listener_tail, | 639 | listener_tail, |
622 | listener); | 640 | listener); |
623 | GNUNET_free (listener); | 641 | GNUNET_free (listener); |
624 | } | 642 | } |
625 | GNUNET_free (cs); | 643 | GNUNET_free (cs); |
644 | num_clients--; | ||
645 | if ( (GNUNET_YES == in_shutdown) && | ||
646 | (0 == num_clients) ) | ||
647 | { | ||
648 | if (NULL != cadet) | ||
649 | { | ||
650 | GNUNET_CADET_disconnect (cadet); | ||
651 | cadet = NULL; | ||
652 | } | ||
653 | } | ||
626 | } | 654 | } |
627 | 655 | ||
628 | 656 | ||
@@ -1299,6 +1327,7 @@ handle_client_listen (void *cls, | |||
1299 | } | 1327 | } |
1300 | listener = GNUNET_new (struct Listener); | 1328 | listener = GNUNET_new (struct Listener); |
1301 | listener->cs = cs; | 1329 | listener->cs = cs; |
1330 | cs->listener = listener; | ||
1302 | listener->app_id = msg->app_id; | 1331 | listener->app_id = msg->app_id; |
1303 | listener->operation = (enum GNUNET_SET_OperationType) ntohl (msg->operation); | 1332 | listener->operation = (enum GNUNET_SET_OperationType) ntohl (msg->operation); |
1304 | GNUNET_CONTAINER_DLL_insert (listener_head, | 1333 | GNUNET_CONTAINER_DLL_insert (listener_head, |
@@ -1917,10 +1946,14 @@ static void | |||
1917 | shutdown_task (void *cls) | 1946 | shutdown_task (void *cls) |
1918 | { | 1947 | { |
1919 | /* Delay actual shutdown to allow service to disconnect clients */ | 1948 | /* Delay actual shutdown to allow service to disconnect clients */ |
1920 | if (NULL != cadet) | 1949 | in_shutdown = GNUNET_YES; |
1950 | if (0 == num_clients) | ||
1921 | { | 1951 | { |
1922 | GNUNET_CADET_disconnect (cadet); | 1952 | if (NULL != cadet) |
1923 | cadet = NULL; | 1953 | { |
1954 | GNUNET_CADET_disconnect (cadet); | ||
1955 | cadet = NULL; | ||
1956 | } | ||
1924 | } | 1957 | } |
1925 | GNUNET_STATISTICS_destroy (_GSS_statistics, | 1958 | GNUNET_STATISTICS_destroy (_GSS_statistics, |
1926 | GNUNET_YES); | 1959 | GNUNET_YES); |
diff --git a/src/set/gnunet-service-set_union.c b/src/set/gnunet-service-set_union.c index b2983545f..9586dcf27 100644 --- a/src/set/gnunet-service-set_union.c +++ b/src/set/gnunet-service-set_union.c | |||
@@ -364,7 +364,7 @@ fail_union_operation (struct Operation *op) | |||
364 | struct GNUNET_MQ_Envelope *ev; | 364 | struct GNUNET_MQ_Envelope *ev; |
365 | struct GNUNET_SET_ResultMessage *msg; | 365 | struct GNUNET_SET_ResultMessage *msg; |
366 | 366 | ||
367 | LOG (GNUNET_ERROR_TYPE_ERROR, | 367 | LOG (GNUNET_ERROR_TYPE_WARNING, |
368 | "union operation failed\n"); | 368 | "union operation failed\n"); |
369 | ev = GNUNET_MQ_msg (msg, GNUNET_MESSAGE_TYPE_SET_RESULT); | 369 | ev = GNUNET_MQ_msg (msg, GNUNET_MESSAGE_TYPE_SET_RESULT); |
370 | msg->result_status = htons (GNUNET_SET_STATUS_FAILURE); | 370 | msg->result_status = htons (GNUNET_SET_STATUS_FAILURE); |
@@ -739,11 +739,10 @@ get_order_from_difference (unsigned int diff) | |||
739 | unsigned int ibf_order; | 739 | unsigned int ibf_order; |
740 | 740 | ||
741 | ibf_order = 2; | 741 | ibf_order = 2; |
742 | while ( (1<<ibf_order) < (IBF_ALPHA * diff) || | 742 | while ( ( (1<<ibf_order) < (IBF_ALPHA * diff) || |
743 | ((1<<ibf_order) < SE_IBF_HASH_NUM) ) | 743 | ((1<<ibf_order) < SE_IBF_HASH_NUM) ) && |
744 | (ibf_order < MAX_IBF_ORDER) ) | ||
744 | ibf_order++; | 745 | ibf_order++; |
745 | if (ibf_order > MAX_IBF_ORDER) | ||
746 | ibf_order = MAX_IBF_ORDER; | ||
747 | // add one for correction | 746 | // add one for correction |
748 | return ibf_order + 1; | 747 | return ibf_order + 1; |
749 | } | 748 | } |
@@ -1405,7 +1404,7 @@ send_client_done (void *cls) | |||
1405 | } | 1404 | } |
1406 | 1405 | ||
1407 | if (PHASE_DONE != op->state->phase) { | 1406 | if (PHASE_DONE != op->state->phase) { |
1408 | LOG (GNUNET_ERROR_TYPE_ERROR, | 1407 | LOG (GNUNET_ERROR_TYPE_WARNING, |
1409 | "union operation failed\n"); | 1408 | "union operation failed\n"); |
1410 | ev = GNUNET_MQ_msg (rm, GNUNET_MESSAGE_TYPE_SET_RESULT); | 1409 | ev = GNUNET_MQ_msg (rm, GNUNET_MESSAGE_TYPE_SET_RESULT); |
1411 | rm->result_status = htons (GNUNET_SET_STATUS_FAILURE); | 1410 | rm->result_status = htons (GNUNET_SET_STATUS_FAILURE); |
diff --git a/src/set/gnunet-service-set_union_strata_estimator.c b/src/set/gnunet-service-set_union_strata_estimator.c index e3d6bfaec..0bd22c92e 100644 --- a/src/set/gnunet-service-set_union_strata_estimator.c +++ b/src/set/gnunet-service-set_union_strata_estimator.c | |||
@@ -228,21 +228,19 @@ unsigned int | |||
228 | strata_estimator_difference (const struct StrataEstimator *se1, | 228 | strata_estimator_difference (const struct StrataEstimator *se1, |
229 | const struct StrataEstimator *se2) | 229 | const struct StrataEstimator *se2) |
230 | { | 230 | { |
231 | int i; | ||
232 | unsigned int count; | 231 | unsigned int count; |
233 | 232 | ||
234 | GNUNET_assert (se1->strata_count == se2->strata_count); | 233 | GNUNET_assert (se1->strata_count == se2->strata_count); |
235 | count = 0; | 234 | count = 0; |
236 | for (i = se1->strata_count - 1; i >= 0; i--) | 235 | for (int i = se1->strata_count - 1; i >= 0; i--) |
237 | { | 236 | { |
238 | struct InvertibleBloomFilter *diff; | 237 | struct InvertibleBloomFilter *diff; |
239 | /* number of keys decoded from the ibf */ | 238 | /* number of keys decoded from the ibf */ |
240 | int ibf_count; | ||
241 | 239 | ||
242 | /* FIXME: implement this without always allocating new IBFs */ | 240 | /* FIXME: implement this without always allocating new IBFs */ |
243 | diff = ibf_dup (se1->strata[i]); | 241 | diff = ibf_dup (se1->strata[i]); |
244 | ibf_subtract (diff, se2->strata[i]); | 242 | ibf_subtract (diff, se2->strata[i]); |
245 | for (ibf_count = 0; GNUNET_YES; ibf_count++) | 243 | for (int ibf_count = 0; GNUNET_YES; ibf_count++) |
246 | { | 244 | { |
247 | int more; | 245 | int more; |
248 | 246 | ||
diff --git a/src/set/test_set_intersection_result_full.c b/src/set/test_set_intersection_result_full.c index a36aae4d5..16de983cf 100644 --- a/src/set/test_set_intersection_result_full.c +++ b/src/set/test_set_intersection_result_full.c | |||
@@ -131,8 +131,6 @@ listen_cb (void *cls, | |||
131 | "starting intersection by accepting and committing\n"); | 131 | "starting intersection by accepting and committing\n"); |
132 | GNUNET_assert (NULL != context_msg); | 132 | GNUNET_assert (NULL != context_msg); |
133 | GNUNET_assert (ntohs (context_msg->type) == GNUNET_MESSAGE_TYPE_DUMMY); | 133 | GNUNET_assert (ntohs (context_msg->type) == GNUNET_MESSAGE_TYPE_DUMMY); |
134 | GNUNET_SET_listen_cancel (listen_handle); | ||
135 | listen_handle = NULL; | ||
136 | oh2 = GNUNET_SET_accept (request, | 134 | oh2 = GNUNET_SET_accept (request, |
137 | GNUNET_SET_RESULT_FULL, | 135 | GNUNET_SET_RESULT_FULL, |
138 | (struct GNUNET_SET_Option[]) { 0 }, | 136 | (struct GNUNET_SET_Option[]) { 0 }, |
diff --git a/src/set/test_set_union_result_symmetric.c b/src/set/test_set_union_result_symmetric.c index f81c7b8f7..3008e5aac 100644 --- a/src/set/test_set_union_result_symmetric.c +++ b/src/set/test_set_union_result_symmetric.c | |||
@@ -182,8 +182,6 @@ listen_cb (void *cls, | |||
182 | GNUNET_assert (ntohs (context_msg->type) == GNUNET_MESSAGE_TYPE_DUMMY); | 182 | GNUNET_assert (ntohs (context_msg->type) == GNUNET_MESSAGE_TYPE_DUMMY); |
183 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 183 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, |
184 | "listen cb called\n"); | 184 | "listen cb called\n"); |
185 | GNUNET_SET_listen_cancel (listen_handle); | ||
186 | listen_handle = NULL; | ||
187 | oh2 = GNUNET_SET_accept (request, | 185 | oh2 = GNUNET_SET_accept (request, |
188 | GNUNET_SET_RESULT_SYMMETRIC, | 186 | GNUNET_SET_RESULT_SYMMETRIC, |
189 | (struct GNUNET_SET_Option[]) { 0 }, | 187 | (struct GNUNET_SET_Option[]) { 0 }, |
diff --git a/src/social/gnunet-social.c b/src/social/gnunet-social.c index 0de8809ff..12c5bf2e1 100644 --- a/src/social/gnunet-social.c +++ b/src/social/gnunet-social.c | |||
@@ -281,7 +281,7 @@ exit_fail () | |||
281 | * This also indicates the end of the connection to the service. | 281 | * This also indicates the end of the connection to the service. |
282 | */ | 282 | */ |
283 | static void | 283 | static void |
284 | host_left () | 284 | host_left (void *cls) |
285 | { | 285 | { |
286 | GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE, | 286 | GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE, |
287 | "The host has left the place.\n"); | 287 | "The host has left the place.\n"); |
diff --git a/src/social/social_api.c b/src/social/social_api.c index 96ddfe912..89843831b 100644 --- a/src/social/social_api.c +++ b/src/social/social_api.c | |||
@@ -2773,6 +2773,8 @@ GNUNET_SOCIAL_app_disconnect (struct GNUNET_SOCIAL_App *app, | |||
2773 | GNUNET_ContinuationCallback disconnect_cb, | 2773 | GNUNET_ContinuationCallback disconnect_cb, |
2774 | void *disconnect_cls) | 2774 | void *disconnect_cls) |
2775 | { | 2775 | { |
2776 | if (NULL == app) return; | ||
2777 | |||
2776 | app->disconnect_cb = disconnect_cb; | 2778 | app->disconnect_cb = disconnect_cb; |
2777 | app->disconnect_cls = disconnect_cls; | 2779 | app->disconnect_cls = disconnect_cls; |
2778 | 2780 | ||
diff --git a/src/sq/sq_result_helper.c b/src/sq/sq_result_helper.c index 9579863b2..f2986a053 100644 --- a/src/sq/sq_result_helper.c +++ b/src/sq/sq_result_helper.c | |||
@@ -620,7 +620,7 @@ extract_uint16 (void *cls, | |||
620 | void *dst) | 620 | void *dst) |
621 | { | 621 | { |
622 | uint64_t v; | 622 | uint64_t v; |
623 | uint32_t *u = dst; | 623 | uint16_t *u = dst; |
624 | 624 | ||
625 | GNUNET_assert (sizeof (uint16_t) == *dst_size); | 625 | GNUNET_assert (sizeof (uint16_t) == *dst_size); |
626 | if (SQLITE_INTEGER != | 626 | if (SQLITE_INTEGER != |
diff --git a/src/statistics/Makefile.am b/src/statistics/Makefile.am index b2e256960..16a1ea2d0 100644 --- a/src/statistics/Makefile.am +++ b/src/statistics/Makefile.am | |||
@@ -90,7 +90,8 @@ endif | |||
90 | 90 | ||
91 | do_subst = $(SED) -e 's,[@]PYTHON[@],$(PYTHON),g' | 91 | do_subst = $(SED) -e 's,[@]PYTHON[@],$(PYTHON),g' |
92 | 92 | ||
93 | %.py: %.py.in Makefile | 93 | SUFFIXES = .py.in .py |
94 | .py.in.py: | ||
94 | $(do_subst) < $(srcdir)/$< > $@ | 95 | $(do_subst) < $(srcdir)/$< > $@ |
95 | chmod +x $@ | 96 | chmod +x $@ |
96 | 97 | ||
diff --git a/src/testbed/gnunet-daemon-testbed-underlay.c b/src/testbed/gnunet-daemon-testbed-underlay.c index 0b6c44710..3605e0384 100644 --- a/src/testbed/gnunet-daemon-testbed-underlay.c +++ b/src/testbed/gnunet-daemon-testbed-underlay.c | |||
@@ -165,7 +165,8 @@ check_access (void *cls, const struct GNUNET_PeerIdentity * pid) | |||
165 | 165 | ||
166 | 166 | ||
167 | static int | 167 | static int |
168 | get_identity (unsigned int offset, struct GNUNET_PeerIdentity *id) | 168 | get_identity (unsigned int offset, |
169 | struct GNUNET_PeerIdentity *id) | ||
169 | { | 170 | { |
170 | struct GNUNET_CRYPTO_EddsaPrivateKey private_key; | 171 | struct GNUNET_CRYPTO_EddsaPrivateKey private_key; |
171 | 172 | ||
@@ -174,7 +175,8 @@ get_identity (unsigned int offset, struct GNUNET_PeerIdentity *id) | |||
174 | GNUNET_memcpy (&private_key, | 175 | GNUNET_memcpy (&private_key, |
175 | hostkeys_data + (offset * GNUNET_TESTING_HOSTKEYFILESIZE), | 176 | hostkeys_data + (offset * GNUNET_TESTING_HOSTKEYFILESIZE), |
176 | GNUNET_TESTING_HOSTKEYFILESIZE); | 177 | GNUNET_TESTING_HOSTKEYFILESIZE); |
177 | GNUNET_CRYPTO_eddsa_key_get_public (&private_key, &id->public_key); | 178 | GNUNET_CRYPTO_eddsa_key_get_public (&private_key, |
179 | &id->public_key); | ||
178 | return GNUNET_OK; | 180 | return GNUNET_OK; |
179 | } | 181 | } |
180 | 182 | ||
diff --git a/src/testbed/gnunet-service-testbed_oc.c b/src/testbed/gnunet-service-testbed_oc.c index 09849797c..11c45a0f5 100644 --- a/src/testbed/gnunet-service-testbed_oc.c +++ b/src/testbed/gnunet-service-testbed_oc.c | |||
@@ -261,7 +261,7 @@ struct OverlayConnectContext | |||
261 | enum OverlayConnectContextType type; | 261 | enum OverlayConnectContextType type; |
262 | 262 | ||
263 | /** | 263 | /** |
264 | * The id of the second peer which is has to connect to the first peer | 264 | * The id of the second peer which has to connect to the first peer |
265 | */ | 265 | */ |
266 | uint32_t other_peer_id; | 266 | uint32_t other_peer_id; |
267 | }; | 267 | }; |
@@ -930,10 +930,10 @@ send_hello (void *cls) | |||
930 | other_peer_str); | 930 | other_peer_str); |
931 | GNUNET_free (other_peer_str); | 931 | GNUNET_free (other_peer_str); |
932 | lp2c->ohh = | 932 | lp2c->ohh = |
933 | GNUNET_TRANSPORT_offer_hello (lp2c->tcc.cfg, | 933 | GNUNET_TRANSPORT_offer_hello (lp2c->tcc.cfg, |
934 | occ->hello, | 934 | occ->hello, |
935 | &occ_hello_sent_cb, | 935 | &occ_hello_sent_cb, |
936 | occ); | 936 | occ); |
937 | if (NULL == lp2c->ohh) | 937 | if (NULL == lp2c->ohh) |
938 | { | 938 | { |
939 | GNUNET_break (0); | 939 | GNUNET_break (0); |
@@ -1001,6 +1001,11 @@ p2_transport_connect (struct OverlayConnectContext *occ) | |||
1001 | { | 1001 | { |
1002 | struct Peer *peer2; | 1002 | struct Peer *peer2; |
1003 | 1003 | ||
1004 | /* HUH? Why to *obtain* HELLO? Seems we use this to *SEND* the | ||
1005 | HELLO! */ | ||
1006 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
1007 | "Connecting to transport of peer %s to obtain HELLO\n", | ||
1008 | GNUNET_i2s (&occ->other_peer_identity)); | ||
1004 | GNUNET_assert (NULL == occ->emsg); | 1009 | GNUNET_assert (NULL == occ->emsg); |
1005 | GNUNET_assert (NULL != occ->hello); | 1010 | GNUNET_assert (NULL != occ->hello); |
1006 | GNUNET_assert (NULL == occ->ghh); | 1011 | GNUNET_assert (NULL == occ->ghh); |
diff --git a/src/testbed/gnunet-testbed-profiler.c b/src/testbed/gnunet-testbed-profiler.c index 9829bbf0d..0fa6d8172 100644 --- a/src/testbed/gnunet-testbed-profiler.c +++ b/src/testbed/gnunet-testbed-profiler.c | |||
@@ -175,9 +175,7 @@ controller_event_cb (void *cls, | |||
175 | { | 175 | { |
176 | printf ("\nAborting due to very high failure rate\n"); | 176 | printf ("\nAborting due to very high failure rate\n"); |
177 | print_overlay_links_summary (); | 177 | print_overlay_links_summary (); |
178 | if (NULL != abort_task) | 178 | GNUNET_SCHEDULER_shutdown (); |
179 | GNUNET_SCHEDULER_cancel (abort_task); | ||
180 | abort_task = GNUNET_SCHEDULER_add_now (&do_abort, NULL); | ||
181 | return; | 179 | return; |
182 | } | 180 | } |
183 | } | 181 | } |
@@ -260,11 +258,12 @@ run (void *cls, char *const *args, const char *cfgfile, | |||
260 | event_mask = 0; | 258 | event_mask = 0; |
261 | event_mask |= (1LL << GNUNET_TESTBED_ET_CONNECT); | 259 | event_mask |= (1LL << GNUNET_TESTBED_ET_CONNECT); |
262 | event_mask |= (1LL << GNUNET_TESTBED_ET_OPERATION_FINISHED); | 260 | event_mask |= (1LL << GNUNET_TESTBED_ET_OPERATION_FINISHED); |
263 | GNUNET_TESTBED_run (hosts_file, cfg, num_peers, event_mask, controller_event_cb, | 261 | GNUNET_TESTBED_run (hosts_file, cfg, num_peers, event_mask, |
264 | NULL, &test_run, NULL); | 262 | &controller_event_cb, NULL, |
263 | &test_run, NULL); | ||
265 | abort_task = | 264 | abort_task = |
266 | GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_FOREVER_REL, &do_abort, | 265 | GNUNET_SCHEDULER_add_shutdown (&do_abort, |
267 | NULL); | 266 | NULL); |
268 | } | 267 | } |
269 | 268 | ||
270 | 269 | ||
@@ -310,6 +309,8 @@ main (int argc, char *const *argv) | |||
310 | const char *binaryHelp = "gnunet-testbed-profiler [OPTIONS]"; | 309 | const char *binaryHelp = "gnunet-testbed-profiler [OPTIONS]"; |
311 | int ret; | 310 | int ret; |
312 | 311 | ||
312 | unsetenv ("XDG_DATA_HOME"); | ||
313 | unsetenv ("XDG_CONFIG_HOME"); | ||
313 | if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv)) | 314 | if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv)) |
314 | return 2; | 315 | return 2; |
315 | result = GNUNET_SYSERR; | 316 | result = GNUNET_SYSERR; |
diff --git a/src/testbed/testbed_api_topology.c b/src/testbed/testbed_api_topology.c index 7bc36d1b4..7d0ccd269 100644 --- a/src/testbed/testbed_api_topology.c +++ b/src/testbed/testbed_api_topology.c | |||
@@ -1051,7 +1051,7 @@ gen_topo_from_file (struct TopologyContext *tc, | |||
1051 | state = PEER_INDEX; | 1051 | state = PEER_INDEX; |
1052 | while (offset < fs) | 1052 | while (offset < fs) |
1053 | { | 1053 | { |
1054 | if (0 != isspace (data[offset])) | 1054 | if (0 != isspace ((unsigned char) data[offset])) |
1055 | { | 1055 | { |
1056 | offset++; | 1056 | offset++; |
1057 | continue; | 1057 | continue; |
diff --git a/src/topology/friends.c b/src/topology/friends.c index a960fad17..65f2700bb 100644 --- a/src/topology/friends.c +++ b/src/topology/friends.c | |||
@@ -95,7 +95,7 @@ GNUNET_FRIENDS_parse (const struct GNUNET_CONFIGURATION_Handle *cfg, | |||
95 | pos = 0; | 95 | pos = 0; |
96 | while (pos < fsize) | 96 | while (pos < fsize) |
97 | { | 97 | { |
98 | while ((pos < fsize) && (! isspace ((int) data[pos]))) | 98 | while ((pos < fsize) && (! isspace ((unsigned char) data[pos]))) |
99 | pos++; | 99 | pos++; |
100 | if (GNUNET_OK != | 100 | if (GNUNET_OK != |
101 | GNUNET_CRYPTO_eddsa_public_key_from_string (&data[start], | 101 | GNUNET_CRYPTO_eddsa_public_key_from_string (&data[start], |
diff --git a/src/transport/gnunet-service-transport.c b/src/transport/gnunet-service-transport.c index ec4d82164..124260c41 100644 --- a/src/transport/gnunet-service-transport.c +++ b/src/transport/gnunet-service-transport.c | |||
@@ -541,6 +541,13 @@ client_disconnect_cb (void *cls, | |||
541 | GNUNET_CONTAINER_multipeermap_iterate (active_stccs, | 541 | GNUNET_CONTAINER_multipeermap_iterate (active_stccs, |
542 | &mark_match_down, | 542 | &mark_match_down, |
543 | tc); | 543 | tc); |
544 | for (struct AddressToStringContext *cur = a2s_head; | ||
545 | NULL != cur; | ||
546 | cur = cur->next) | ||
547 | { | ||
548 | if (cur->tc == tc) | ||
549 | cur->tc = NULL; | ||
550 | } | ||
544 | GNUNET_CONTAINER_DLL_remove (clients_head, | 551 | GNUNET_CONTAINER_DLL_remove (clients_head, |
545 | clients_tail, | 552 | clients_tail, |
546 | tc); | 553 | tc); |
@@ -681,6 +688,8 @@ handle_client_hello (void *cls, | |||
681 | { | 688 | { |
682 | struct TransportClient *tc = cls; | 689 | struct TransportClient *tc = cls; |
683 | 690 | ||
691 | GNUNET_log (GNUNET_ERROR_TYPE_INFO, | ||
692 | "Received HELLO message\n"); | ||
684 | GST_validation_handle_hello (message); | 693 | GST_validation_handle_hello (message); |
685 | GNUNET_SERVICE_client_continue (tc->client); | 694 | GNUNET_SERVICE_client_continue (tc->client); |
686 | } | 695 | } |
@@ -864,6 +873,8 @@ transmit_address_to_client (void *cls, | |||
864 | 873 | ||
865 | GNUNET_assert ( (GNUNET_OK == res) || | 874 | GNUNET_assert ( (GNUNET_OK == res) || |
866 | (GNUNET_SYSERR == res) ); | 875 | (GNUNET_SYSERR == res) ); |
876 | if (NULL == actx->tc) | ||
877 | return; | ||
867 | if (NULL == buf) | 878 | if (NULL == buf) |
868 | { | 879 | { |
869 | env = GNUNET_MQ_msg (atsm, | 880 | env = GNUNET_MQ_msg (atsm, |
@@ -878,6 +889,7 @@ transmit_address_to_client (void *cls, | |||
878 | GNUNET_CONTAINER_DLL_remove (a2s_head, | 889 | GNUNET_CONTAINER_DLL_remove (a2s_head, |
879 | a2s_tail, | 890 | a2s_tail, |
880 | actx); | 891 | actx); |
892 | GNUNET_free (actx); | ||
881 | return; | 893 | return; |
882 | } | 894 | } |
883 | if (GNUNET_SYSERR == res) | 895 | if (GNUNET_SYSERR == res) |
@@ -2792,7 +2804,7 @@ run (void *cls, | |||
2792 | GNUNET_assert (NULL != GST_my_private_key); | 2804 | GNUNET_assert (NULL != GST_my_private_key); |
2793 | 2805 | ||
2794 | GNUNET_log(GNUNET_ERROR_TYPE_INFO, | 2806 | GNUNET_log(GNUNET_ERROR_TYPE_INFO, |
2795 | "My identity is `%4s'\n", | 2807 | "My identity is `%s'\n", |
2796 | GNUNET_i2s_full (&GST_my_identity)); | 2808 | GNUNET_i2s_full (&GST_my_identity)); |
2797 | 2809 | ||
2798 | GNUNET_SCHEDULER_add_shutdown (&shutdown_task, | 2810 | GNUNET_SCHEDULER_add_shutdown (&shutdown_task, |
diff --git a/src/transport/gnunet-service-transport_ats.c b/src/transport/gnunet-service-transport_ats.c index a20c998b3..01e115bfc 100644 --- a/src/transport/gnunet-service-transport_ats.c +++ b/src/transport/gnunet-service-transport_ats.c | |||
@@ -337,15 +337,10 @@ GST_ats_block_address (const struct GNUNET_HELLO_Address *address, | |||
337 | return; /* our own, ignore! */ | 337 | return; /* our own, ignore! */ |
338 | ai = find_ai (address, | 338 | ai = find_ai (address, |
339 | session); | 339 | session); |
340 | if (NULL == ai) | 340 | if (NULL == ai || NULL == ai->ar) |
341 | { | 341 | { |
342 | GNUNET_assert (0); | 342 | /* The address is already gone/blocked, this can happen during a blacklist |
343 | return; | 343 | * callback. */ |
344 | } | ||
345 | if (NULL == ai->ar) | ||
346 | { | ||
347 | /* already blocked, how did it get used!? */ | ||
348 | GNUNET_break (0); | ||
349 | return; | 344 | return; |
350 | } | 345 | } |
351 | ai->back_off = GNUNET_TIME_STD_BACKOFF (ai->back_off); | 346 | ai->back_off = GNUNET_TIME_STD_BACKOFF (ai->back_off); |
diff --git a/src/transport/gnunet-service-transport_neighbours.c b/src/transport/gnunet-service-transport_neighbours.c index 19f5fd081..ac72a667c 100644 --- a/src/transport/gnunet-service-transport_neighbours.c +++ b/src/transport/gnunet-service-transport_neighbours.c | |||
@@ -2433,7 +2433,10 @@ switch_address_bl_check_cont (void *cls, | |||
2433 | goto cleanup; | 2433 | goto cleanup; |
2434 | 2434 | ||
2435 | papi = GST_plugins_find (address->transport_name); | 2435 | papi = GST_plugins_find (address->transport_name); |
2436 | GNUNET_assert (NULL != papi); | 2436 | if (NULL == papi) { |
2437 | /* This can happen during shutdown. */ | ||
2438 | goto cleanup; | ||
2439 | } | ||
2437 | 2440 | ||
2438 | if (GNUNET_NO == result) | 2441 | if (GNUNET_NO == result) |
2439 | { | 2442 | { |
diff --git a/src/transport/gnunet-service-transport_validation.c b/src/transport/gnunet-service-transport_validation.c index 4a6d427be..cd5aeb5e2 100644 --- a/src/transport/gnunet-service-transport_validation.c +++ b/src/transport/gnunet-service-transport_validation.c | |||
@@ -784,15 +784,24 @@ revalidate_address (void *cls) | |||
784 | GNUNET_STATISTICS_update (GST_stats, | 784 | GNUNET_STATISTICS_update (GST_stats, |
785 | gettext_noop ("# address revalidations started"), 1, | 785 | gettext_noop ("# address revalidations started"), 1, |
786 | GNUNET_NO); | 786 | GNUNET_NO); |
787 | if (NULL != ve->bc) | ||
788 | { | ||
789 | GST_blacklist_test_cancel (ve->bc); | ||
790 | ve->bc = NULL; | ||
791 | } | ||
787 | bc = GST_blacklist_test_allowed (&ve->address->peer, | 792 | bc = GST_blacklist_test_allowed (&ve->address->peer, |
788 | ve->address->transport_name, | 793 | ve->address->transport_name, |
789 | &transmit_ping_if_allowed, | 794 | &transmit_ping_if_allowed, |
790 | ve, | 795 | ve, |
791 | NULL, | 796 | NULL, |
792 | NULL); | 797 | NULL); |
793 | if (NULL != bc) | 798 | if (NULL != bc) |
794 | ve->bc = bc; /* only set 'bc' if 'transmit_ping_if_allowed' was not already | 799 | { |
795 | * called... */ | 800 | /* If transmit_ping_if_allowed was already called it may have freed ve, |
801 | * so only set ve->bc if it has not been called. | ||
802 | */ | ||
803 | ve->bc = bc; | ||
804 | } | ||
796 | } | 805 | } |
797 | 806 | ||
798 | 807 | ||
@@ -1338,6 +1347,9 @@ GST_validation_handle_address (const struct GNUNET_HELLO_Address *address) | |||
1338 | if (NULL == papi) | 1347 | if (NULL == papi) |
1339 | { | 1348 | { |
1340 | /* This plugin is currently unvailable ... ignore */ | 1349 | /* This plugin is currently unvailable ... ignore */ |
1350 | GNUNET_log (GNUNET_ERROR_TYPE_INFO, | ||
1351 | "No plugin available for %s\n", | ||
1352 | address->transport_name); | ||
1341 | return; | 1353 | return; |
1342 | } | 1354 | } |
1343 | ve = find_validation_entry (address); | 1355 | ve = find_validation_entry (address); |
@@ -1349,6 +1361,13 @@ GST_validation_handle_address (const struct GNUNET_HELLO_Address *address) | |||
1349 | GNUNET_i2s (&ve->address->peer)); | 1361 | GNUNET_i2s (&ve->address->peer)); |
1350 | ve->revalidation_task = GNUNET_SCHEDULER_add_now (&revalidate_address, ve); | 1362 | ve->revalidation_task = GNUNET_SCHEDULER_add_now (&revalidate_address, ve); |
1351 | } | 1363 | } |
1364 | else | ||
1365 | { | ||
1366 | GNUNET_log (GNUNET_ERROR_TYPE_INFO, | ||
1367 | "Validation already running for address `%s' of %s\n", | ||
1368 | GST_plugins_a2s (ve->address), | ||
1369 | GNUNET_i2s (&ve->address->peer)); | ||
1370 | } | ||
1352 | } | 1371 | } |
1353 | 1372 | ||
1354 | 1373 | ||
@@ -1648,6 +1667,9 @@ GST_validation_handle_hello (const struct GNUNET_MessageHeader *hello) | |||
1648 | sizeof (struct GNUNET_PeerIdentity))) | 1667 | sizeof (struct GNUNET_PeerIdentity))) |
1649 | { | 1668 | { |
1650 | /* got our own HELLO, how boring */ | 1669 | /* got our own HELLO, how boring */ |
1670 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
1671 | "Validation received our own HELLO (%s), ignoring\n", | ||
1672 | GNUNET_i2s (&pid)); | ||
1651 | return GNUNET_OK; | 1673 | return GNUNET_OK; |
1652 | } | 1674 | } |
1653 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 1675 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, |
diff --git a/src/transport/test_transport_testing_restart.c b/src/transport/test_transport_testing_restart.c index 595177e03..06275055d 100644 --- a/src/transport/test_transport_testing_restart.c +++ b/src/transport/test_transport_testing_restart.c | |||
@@ -71,7 +71,8 @@ restart_cb (void *cls) | |||
71 | p->no, | 71 | p->no, |
72 | GNUNET_i2s (&p->id)); | 72 | GNUNET_i2s (&p->id)); |
73 | ret = 0; | 73 | ret = 0; |
74 | end (); | 74 | GNUNET_SCHEDULER_add_now (&end, |
75 | NULL); | ||
75 | } | 76 | } |
76 | 77 | ||
77 | 78 | ||
diff --git a/src/transport/test_transport_testing_startstop.c b/src/transport/test_transport_testing_startstop.c index 6ac0250cc..931e922c4 100644 --- a/src/transport/test_transport_testing_startstop.c +++ b/src/transport/test_transport_testing_startstop.c | |||
@@ -71,7 +71,8 @@ start_cb (void *cls) | |||
71 | p->no, | 71 | p->no, |
72 | GNUNET_i2s (&p->id)); | 72 | GNUNET_i2s (&p->id)); |
73 | ret = 0; | 73 | ret = 0; |
74 | end (); | 74 | GNUNET_SCHEDULER_add_now (&end, |
75 | NULL); | ||
75 | } | 76 | } |
76 | 77 | ||
77 | 78 | ||
diff --git a/src/transport/transport-testing.c b/src/transport/transport-testing.c index 2aa6cdbb0..68cda3bd7 100644 --- a/src/transport/transport-testing.c +++ b/src/transport/transport-testing.c | |||
@@ -384,7 +384,7 @@ GNUNET_TRANSPORT_TESTING_start_peer (struct GNUNET_TRANSPORT_TESTING_Handle *tth | |||
384 | { | 384 | { |
385 | char *emsg = NULL; | 385 | char *emsg = NULL; |
386 | struct GNUNET_TRANSPORT_TESTING_PeerContext *p; | 386 | struct GNUNET_TRANSPORT_TESTING_PeerContext *p; |
387 | struct GNUNET_PeerIdentity *dummy; | 387 | struct GNUNET_PeerIdentity dummy; |
388 | unsigned int i; | 388 | unsigned int i; |
389 | 389 | ||
390 | if (GNUNET_NO == GNUNET_DISK_file_test (cfgname)) | 390 | if (GNUNET_NO == GNUNET_DISK_file_test (cfgname)) |
@@ -678,6 +678,11 @@ GNUNET_TRANSPORT_TESTING_stop_peer (struct GNUNET_TRANSPORT_TESTING_PeerContext | |||
678 | GNUNET_CONFIGURATION_destroy (p->cfg); | 678 | GNUNET_CONFIGURATION_destroy (p->cfg); |
679 | p->cfg = NULL; | 679 | p->cfg = NULL; |
680 | } | 680 | } |
681 | if (NULL != p->handlers) | ||
682 | { | ||
683 | GNUNET_free (p->handlers); | ||
684 | p->handlers = NULL; | ||
685 | } | ||
681 | GNUNET_CONTAINER_DLL_remove (tth->p_head, | 686 | GNUNET_CONTAINER_DLL_remove (tth->p_head, |
682 | tth->p_tail, | 687 | tth->p_tail, |
683 | p); | 688 | p); |
diff --git a/src/util/.gitignore b/src/util/.gitignore index 3576a2134..d32a66833 100644 --- a/src/util/.gitignore +++ b/src/util/.gitignore | |||
@@ -67,3 +67,4 @@ test_socks.nc | |||
67 | perf_crypto_asymmetric | 67 | perf_crypto_asymmetric |
68 | perf_crypto_hash | 68 | perf_crypto_hash |
69 | perf_crypto_symmetric | 69 | perf_crypto_symmetric |
70 | perf_crypto_rsa | ||
diff --git a/src/util/crypto_ecc.c b/src/util/crypto_ecc.c index eaa49a991..7845932ee 100644 --- a/src/util/crypto_ecc.c +++ b/src/util/crypto_ecc.c | |||
@@ -354,6 +354,37 @@ GNUNET_CRYPTO_eddsa_public_key_to_string (const struct GNUNET_CRYPTO_EddsaPublic | |||
354 | 354 | ||
355 | 355 | ||
356 | /** | 356 | /** |
357 | * Convert a private key to a string. | ||
358 | * | ||
359 | * @param priv key to convert | ||
360 | * @return string representing @a pub | ||
361 | */ | ||
362 | char * | ||
363 | GNUNET_CRYPTO_eddsa_private_key_to_string (const struct GNUNET_CRYPTO_EddsaPrivateKey *priv) | ||
364 | { | ||
365 | char *privkeybuf; | ||
366 | size_t keylen = (sizeof (struct GNUNET_CRYPTO_EddsaPrivateKey)) * 8; | ||
367 | char *end; | ||
368 | |||
369 | if (keylen % 5 > 0) | ||
370 | keylen += 5 - keylen % 5; | ||
371 | keylen /= 5; | ||
372 | privkeybuf = GNUNET_malloc (keylen + 1); | ||
373 | end = GNUNET_STRINGS_data_to_string ((unsigned char *) priv, | ||
374 | sizeof (struct GNUNET_CRYPTO_EddsaPrivateKey), | ||
375 | privkeybuf, | ||
376 | keylen); | ||
377 | if (NULL == end) | ||
378 | { | ||
379 | GNUNET_free (privkeybuf); | ||
380 | return NULL; | ||
381 | } | ||
382 | *end = '\0'; | ||
383 | return privkeybuf; | ||
384 | } | ||
385 | |||
386 | |||
387 | /** | ||
357 | * Convert a string representing a public key to a public key. | 388 | * Convert a string representing a public key to a public key. |
358 | * | 389 | * |
359 | * @param enc encoded public key | 390 | * @param enc encoded public key |
@@ -374,9 +405,10 @@ GNUNET_CRYPTO_ecdsa_public_key_from_string (const char *enc, | |||
374 | if (enclen != keylen) | 405 | if (enclen != keylen) |
375 | return GNUNET_SYSERR; | 406 | return GNUNET_SYSERR; |
376 | 407 | ||
377 | if (GNUNET_OK != GNUNET_STRINGS_string_to_data (enc, enclen, | 408 | if (GNUNET_OK != |
378 | pub, | 409 | GNUNET_STRINGS_string_to_data (enc, enclen, |
379 | sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey))) | 410 | pub, |
411 | sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey))) | ||
380 | return GNUNET_SYSERR; | 412 | return GNUNET_SYSERR; |
381 | return GNUNET_OK; | 413 | return GNUNET_OK; |
382 | } | 414 | } |
@@ -403,9 +435,10 @@ GNUNET_CRYPTO_eddsa_public_key_from_string (const char *enc, | |||
403 | if (enclen != keylen) | 435 | if (enclen != keylen) |
404 | return GNUNET_SYSERR; | 436 | return GNUNET_SYSERR; |
405 | 437 | ||
406 | if (GNUNET_OK != GNUNET_STRINGS_string_to_data (enc, enclen, | 438 | if (GNUNET_OK != |
407 | pub, | 439 | GNUNET_STRINGS_string_to_data (enc, enclen, |
408 | sizeof (struct GNUNET_CRYPTO_EddsaPublicKey))) | 440 | pub, |
441 | sizeof (struct GNUNET_CRYPTO_EddsaPublicKey))) | ||
409 | return GNUNET_SYSERR; | 442 | return GNUNET_SYSERR; |
410 | return GNUNET_OK; | 443 | return GNUNET_OK; |
411 | } | 444 | } |
diff --git a/src/util/crypto_paillier.c b/src/util/crypto_paillier.c index 3ed025a2a..530a2957f 100644 --- a/src/util/crypto_paillier.c +++ b/src/util/crypto_paillier.c | |||
@@ -370,9 +370,11 @@ GNUNET_CRYPTO_paillier_decrypt (const struct GNUNET_CRYPTO_PaillierPrivateKey *p | |||
370 | /* mod = cmum1 / n (mod n) */ | 370 | /* mod = cmum1 / n (mod n) */ |
371 | GNUNET_assert (0 != (mod = gcry_mpi_new (0))); | 371 | GNUNET_assert (0 != (mod = gcry_mpi_new (0))); |
372 | gcry_mpi_div (mod, NULL, cmum1, n, 0); | 372 | gcry_mpi_div (mod, NULL, cmum1, n, 0); |
373 | gcry_mpi_release (cmum1); | ||
373 | 374 | ||
374 | /* m = mod * mu mod n */ | 375 | /* m = mod * mu mod n */ |
375 | gcry_mpi_mulm (m, mod, mu, n); | 376 | gcry_mpi_mulm (m, mod, mu, n); |
377 | gcry_mpi_release (mod); | ||
376 | gcry_mpi_release (mu); | 378 | gcry_mpi_release (mu); |
377 | gcry_mpi_release (n); | 379 | gcry_mpi_release (n); |
378 | } | 380 | } |
diff --git a/src/util/crypto_rsa.c b/src/util/crypto_rsa.c index 7a108c21b..a985d8e59 100644 --- a/src/util/crypto_rsa.c +++ b/src/util/crypto_rsa.c | |||
@@ -1046,7 +1046,7 @@ GNUNET_CRYPTO_rsa_public_key_dup (const struct GNUNET_CRYPTO_RsaPublicKey *key) | |||
1046 | * @return unblinded signature on success, NULL if RSA key is bad or malicious. | 1046 | * @return unblinded signature on success, NULL if RSA key is bad or malicious. |
1047 | */ | 1047 | */ |
1048 | struct GNUNET_CRYPTO_RsaSignature * | 1048 | struct GNUNET_CRYPTO_RsaSignature * |
1049 | GNUNET_CRYPTO_rsa_unblind (struct GNUNET_CRYPTO_RsaSignature *sig, | 1049 | GNUNET_CRYPTO_rsa_unblind (const struct GNUNET_CRYPTO_RsaSignature *sig, |
1050 | const struct GNUNET_CRYPTO_RsaBlindingKeySecret *bks, | 1050 | const struct GNUNET_CRYPTO_RsaBlindingKeySecret *bks, |
1051 | struct GNUNET_CRYPTO_RsaPublicKey *pkey) | 1051 | struct GNUNET_CRYPTO_RsaPublicKey *pkey) |
1052 | { | 1052 | { |
diff --git a/src/util/gnunet-ecc.c b/src/util/gnunet-ecc.c index 42ecc2101..66a4bd3e9 100644 --- a/src/util/gnunet-ecc.c +++ b/src/util/gnunet-ecc.c | |||
@@ -49,6 +49,11 @@ static unsigned int list_keys_count; | |||
49 | static int print_public_key; | 49 | static int print_public_key; |
50 | 50 | ||
51 | /** | 51 | /** |
52 | * Flag for printing private key. | ||
53 | */ | ||
54 | static int print_private_key; | ||
55 | |||
56 | /** | ||
52 | * Flag for printing public key in hex. | 57 | * Flag for printing public key in hex. |
53 | */ | 58 | */ |
54 | static int print_public_key_hex; | 59 | static int print_public_key_hex; |
@@ -377,7 +382,7 @@ run (void *cls, char *const *args, const char *cfgfile, | |||
377 | create_keys (args[0], args[1]); | 382 | create_keys (args[0], args[1]); |
378 | return; | 383 | return; |
379 | } | 384 | } |
380 | if (print_public_key || print_public_key_hex) | 385 | if (print_public_key || print_public_key_hex || print_private_key) |
381 | { | 386 | { |
382 | char *str; | 387 | char *str; |
383 | struct GNUNET_DISK_FileHandle *keyfile; | 388 | struct GNUNET_DISK_FileHandle *keyfile; |
@@ -388,19 +393,26 @@ run (void *cls, char *const *args, const char *cfgfile, | |||
388 | GNUNET_DISK_PERM_NONE); | 393 | GNUNET_DISK_PERM_NONE); |
389 | if (NULL == keyfile) | 394 | if (NULL == keyfile) |
390 | return; | 395 | return; |
391 | while (sizeof (pk) == GNUNET_DISK_file_read (keyfile, &pk, sizeof (pk))) | 396 | while (sizeof (pk) == |
397 | GNUNET_DISK_file_read (keyfile, &pk, sizeof (pk))) | ||
392 | { | 398 | { |
393 | GNUNET_CRYPTO_eddsa_key_get_public (&pk, &pub); | 399 | GNUNET_CRYPTO_eddsa_key_get_public (&pk, &pub); |
394 | if (print_public_key_hex) | 400 | if (print_public_key_hex) |
395 | { | 401 | { |
396 | print_hex ("HEX:", &pub, sizeof (pub)); | 402 | print_hex ("HEX:", &pub, sizeof (pub)); |
397 | } | 403 | } |
398 | else | 404 | else if (print_public_key) |
399 | { | 405 | { |
400 | str = GNUNET_CRYPTO_eddsa_public_key_to_string (&pub); | 406 | str = GNUNET_CRYPTO_eddsa_public_key_to_string (&pub); |
401 | FPRINTF (stdout, "%s\n", str); | 407 | FPRINTF (stdout, "%s\n", str); |
402 | GNUNET_free (str); | 408 | GNUNET_free (str); |
403 | } | 409 | } |
410 | else if (print_private_key) | ||
411 | { | ||
412 | str = GNUNET_CRYPTO_eddsa_private_key_to_string (&pk); | ||
413 | FPRINTF (stdout, "%s\n", str); | ||
414 | GNUNET_free (str); | ||
415 | } | ||
404 | } | 416 | } |
405 | GNUNET_DISK_file_close (keyfile); | 417 | GNUNET_DISK_file_close (keyfile); |
406 | } | 418 | } |
@@ -438,6 +450,10 @@ main (int argc, | |||
438 | "print-public-key", | 450 | "print-public-key", |
439 | gettext_noop ("print the public key in ASCII format"), | 451 | gettext_noop ("print the public key in ASCII format"), |
440 | &print_public_key), | 452 | &print_public_key), |
453 | GNUNET_GETOPT_option_flag ('P', | ||
454 | "print-private-key", | ||
455 | gettext_noop ("print the private key in ASCII format"), | ||
456 | &print_private_key), | ||
441 | GNUNET_GETOPT_option_flag ('x', | 457 | GNUNET_GETOPT_option_flag ('x', |
442 | "print-hex", | 458 | "print-hex", |
443 | gettext_noop ("print the public key in HEX format"), | 459 | gettext_noop ("print the public key in HEX format"), |
diff --git a/src/util/network.c b/src/util/network.c index 66a468e45..942288613 100644 --- a/src/util/network.c +++ b/src/util/network.c | |||
@@ -1793,10 +1793,18 @@ GNUNET_NETWORK_socket_select (struct GNUNET_NETWORK_FDSet *rfds, | |||
1793 | _("Fatal internal logic error, process hangs in `%s' (abort with CTRL-C)!\n"), | 1793 | _("Fatal internal logic error, process hangs in `%s' (abort with CTRL-C)!\n"), |
1794 | "select"); | 1794 | "select"); |
1795 | } | 1795 | } |
1796 | tv.tv_sec = timeout.rel_value_us / GNUNET_TIME_UNIT_SECONDS.rel_value_us; | 1796 | if (timeout.rel_value_us / GNUNET_TIME_UNIT_SECONDS.rel_value_us > (unsigned long long) LONG_MAX) |
1797 | tv.tv_usec = | 1797 | { |
1798 | (timeout.rel_value_us - | 1798 | tv.tv_sec = LONG_MAX; |
1799 | (tv.tv_sec * GNUNET_TIME_UNIT_SECONDS.rel_value_us)); | 1799 | tv.tv_usec = 999999L; |
1800 | } | ||
1801 | else | ||
1802 | { | ||
1803 | tv.tv_sec = (long) (timeout.rel_value_us / GNUNET_TIME_UNIT_SECONDS.rel_value_us); | ||
1804 | tv.tv_usec = | ||
1805 | (timeout.rel_value_us - | ||
1806 | (tv.tv_sec * GNUNET_TIME_UNIT_SECONDS.rel_value_us)); | ||
1807 | } | ||
1800 | return select (nfds, | 1808 | return select (nfds, |
1801 | (NULL != rfds) ? &rfds->sds : NULL, | 1809 | (NULL != rfds) ? &rfds->sds : NULL, |
1802 | (NULL != wfds) ? &wfds->sds : NULL, | 1810 | (NULL != wfds) ? &wfds->sds : NULL, |
diff --git a/src/util/resolver_api.c b/src/util/resolver_api.c index 33a340729..11b8134d6 100644 --- a/src/util/resolver_api.c +++ b/src/util/resolver_api.c | |||
@@ -469,6 +469,7 @@ handle_response (void *cls, | |||
469 | uint16_t size; | 469 | uint16_t size; |
470 | char *nret; | 470 | char *nret; |
471 | 471 | ||
472 | GNUNET_assert (NULL != rh); | ||
472 | size = ntohs (msg->size); | 473 | size = ntohs (msg->size); |
473 | if (size == sizeof (struct GNUNET_MessageHeader)) | 474 | if (size == sizeof (struct GNUNET_MessageHeader)) |
474 | { | 475 | { |
diff --git a/src/util/scheduler.c b/src/util/scheduler.c index b07c51811..4615ecee9 100644 --- a/src/util/scheduler.c +++ b/src/util/scheduler.c | |||
@@ -785,6 +785,14 @@ void | |||
785 | GNUNET_SCHEDULER_run (GNUNET_SCHEDULER_TaskCallback task, | 785 | GNUNET_SCHEDULER_run (GNUNET_SCHEDULER_TaskCallback task, |
786 | void *task_cls) | 786 | void *task_cls) |
787 | { | 787 | { |
788 | GNUNET_SCHEDULER_run_with_optional_signals(GNUNET_YES, task, task_cls); | ||
789 | } | ||
790 | |||
791 | void | ||
792 | GNUNET_SCHEDULER_run_with_optional_signals (int install_signals, | ||
793 | GNUNET_SCHEDULER_TaskCallback task, | ||
794 | void *task_cls) | ||
795 | { | ||
788 | struct GNUNET_NETWORK_FDSet *rs; | 796 | struct GNUNET_NETWORK_FDSet *rs; |
789 | struct GNUNET_NETWORK_FDSet *ws; | 797 | struct GNUNET_NETWORK_FDSet *ws; |
790 | struct GNUNET_TIME_Relative timeout; | 798 | struct GNUNET_TIME_Relative timeout; |
@@ -818,24 +826,29 @@ GNUNET_SCHEDULER_run (GNUNET_SCHEDULER_TaskCallback task, | |||
818 | GNUNET_DISK_PIPE_END_READ); | 826 | GNUNET_DISK_PIPE_END_READ); |
819 | GNUNET_assert (NULL != pr); | 827 | GNUNET_assert (NULL != pr); |
820 | my_pid = getpid (); | 828 | my_pid = getpid (); |
821 | LOG (GNUNET_ERROR_TYPE_DEBUG, | 829 | |
822 | "Registering signal handlers\n"); | 830 | if (GNUNET_YES == install_signals) |
823 | shc_int = GNUNET_SIGNAL_handler_install (SIGINT, | 831 | { |
832 | LOG (GNUNET_ERROR_TYPE_DEBUG, | ||
833 | "Registering signal handlers\n"); | ||
834 | shc_int = GNUNET_SIGNAL_handler_install (SIGINT, | ||
835 | &sighandler_shutdown); | ||
836 | shc_term = GNUNET_SIGNAL_handler_install (SIGTERM, | ||
824 | &sighandler_shutdown); | 837 | &sighandler_shutdown); |
825 | shc_term = GNUNET_SIGNAL_handler_install (SIGTERM, | ||
826 | &sighandler_shutdown); | ||
827 | #if (SIGTERM != GNUNET_TERM_SIG) | 838 | #if (SIGTERM != GNUNET_TERM_SIG) |
828 | shc_gterm = GNUNET_SIGNAL_handler_install (GNUNET_TERM_SIG, | 839 | shc_gterm = GNUNET_SIGNAL_handler_install (GNUNET_TERM_SIG, |
829 | &sighandler_shutdown); | 840 | &sighandler_shutdown); |
830 | #endif | 841 | #endif |
831 | #ifndef MINGW | 842 | #ifndef MINGW |
832 | shc_pipe = GNUNET_SIGNAL_handler_install (SIGPIPE, | 843 | shc_pipe = GNUNET_SIGNAL_handler_install (SIGPIPE, |
833 | &sighandler_pipe); | 844 | &sighandler_pipe); |
834 | shc_quit = GNUNET_SIGNAL_handler_install (SIGQUIT, | 845 | shc_quit = GNUNET_SIGNAL_handler_install (SIGQUIT, |
835 | &sighandler_shutdown); | 846 | &sighandler_shutdown); |
836 | shc_hup = GNUNET_SIGNAL_handler_install (SIGHUP, | 847 | shc_hup = GNUNET_SIGNAL_handler_install (SIGHUP, |
837 | &sighandler_shutdown); | 848 | &sighandler_shutdown); |
838 | #endif | 849 | #endif |
850 | } | ||
851 | |||
839 | current_priority = GNUNET_SCHEDULER_PRIORITY_DEFAULT; | 852 | current_priority = GNUNET_SCHEDULER_PRIORITY_DEFAULT; |
840 | current_lifeness = GNUNET_YES; | 853 | current_lifeness = GNUNET_YES; |
841 | GNUNET_SCHEDULER_add_with_reason_and_priority (task, | 854 | GNUNET_SCHEDULER_add_with_reason_and_priority (task, |
@@ -951,16 +964,21 @@ GNUNET_SCHEDULER_run (GNUNET_SCHEDULER_TaskCallback task, | |||
951 | busy_wait_warning = 0; | 964 | busy_wait_warning = 0; |
952 | } | 965 | } |
953 | } | 966 | } |
954 | GNUNET_SIGNAL_handler_uninstall (shc_int); | 967 | |
955 | GNUNET_SIGNAL_handler_uninstall (shc_term); | 968 | if (GNUNET_YES == install_signals) |
969 | { | ||
970 | GNUNET_SIGNAL_handler_uninstall (shc_int); | ||
971 | GNUNET_SIGNAL_handler_uninstall (shc_term); | ||
956 | #if (SIGTERM != GNUNET_TERM_SIG) | 972 | #if (SIGTERM != GNUNET_TERM_SIG) |
957 | GNUNET_SIGNAL_handler_uninstall (shc_gterm); | 973 | GNUNET_SIGNAL_handler_uninstall (shc_gterm); |
958 | #endif | 974 | #endif |
959 | #ifndef MINGW | 975 | #ifndef MINGW |
960 | GNUNET_SIGNAL_handler_uninstall (shc_pipe); | 976 | GNUNET_SIGNAL_handler_uninstall (shc_pipe); |
961 | GNUNET_SIGNAL_handler_uninstall (shc_quit); | 977 | GNUNET_SIGNAL_handler_uninstall (shc_quit); |
962 | GNUNET_SIGNAL_handler_uninstall (shc_hup); | 978 | GNUNET_SIGNAL_handler_uninstall (shc_hup); |
963 | #endif | 979 | #endif |
980 | } | ||
981 | |||
964 | GNUNET_DISK_pipe_close (shutdown_pipe_handle); | 982 | GNUNET_DISK_pipe_close (shutdown_pipe_handle); |
965 | shutdown_pipe_handle = NULL; | 983 | shutdown_pipe_handle = NULL; |
966 | GNUNET_NETWORK_fdset_destroy (rs); | 984 | GNUNET_NETWORK_fdset_destroy (rs); |
diff --git a/src/util/test_crypto_paillier.c b/src/util/test_crypto_paillier.c index 9950978c1..1e7e0b301 100644 --- a/src/util/test_crypto_paillier.c +++ b/src/util/test_crypto_paillier.c | |||
@@ -37,6 +37,7 @@ test_crypto () | |||
37 | struct GNUNET_CRYPTO_PaillierCiphertext ciphertext; | 37 | struct GNUNET_CRYPTO_PaillierCiphertext ciphertext; |
38 | struct GNUNET_CRYPTO_PaillierPublicKey public_key; | 38 | struct GNUNET_CRYPTO_PaillierPublicKey public_key; |
39 | struct GNUNET_CRYPTO_PaillierPrivateKey private_key; | 39 | struct GNUNET_CRYPTO_PaillierPrivateKey private_key; |
40 | int ret = 0; | ||
40 | 41 | ||
41 | GNUNET_CRYPTO_paillier_create (&public_key, | 42 | GNUNET_CRYPTO_paillier_create (&public_key, |
42 | &private_key); | 43 | &private_key); |
@@ -54,7 +55,6 @@ test_crypto () | |||
54 | &public_key, | 55 | &public_key, |
55 | &ciphertext, | 56 | &ciphertext, |
56 | plaintext_result); | 57 | plaintext_result); |
57 | |||
58 | if (0 != gcry_mpi_cmp (plaintext, | 58 | if (0 != gcry_mpi_cmp (plaintext, |
59 | plaintext_result)) | 59 | plaintext_result)) |
60 | { | 60 | { |
@@ -65,9 +65,11 @@ test_crypto () | |||
65 | plaintext); | 65 | plaintext); |
66 | gcry_log_debugmpi ("\n", | 66 | gcry_log_debugmpi ("\n", |
67 | plaintext_result); | 67 | plaintext_result); |
68 | return 1; | 68 | ret = 1; |
69 | } | 69 | } |
70 | return 0; | 70 | gcry_mpi_release (plaintext); |
71 | gcry_mpi_release (plaintext_result); | ||
72 | return ret; | ||
71 | } | 73 | } |
72 | 74 | ||
73 | 75 | ||
@@ -84,6 +86,7 @@ test_hom_simple (unsigned int a, | |||
84 | struct GNUNET_CRYPTO_PaillierCiphertext c_result; | 86 | struct GNUNET_CRYPTO_PaillierCiphertext c_result; |
85 | struct GNUNET_CRYPTO_PaillierPublicKey public_key; | 87 | struct GNUNET_CRYPTO_PaillierPublicKey public_key; |
86 | struct GNUNET_CRYPTO_PaillierPrivateKey private_key; | 88 | struct GNUNET_CRYPTO_PaillierPrivateKey private_key; |
89 | int ret = 0; | ||
87 | 90 | ||
88 | GNUNET_CRYPTO_paillier_create (&public_key, | 91 | GNUNET_CRYPTO_paillier_create (&public_key, |
89 | &private_key); | 92 | &private_key); |
@@ -119,9 +122,13 @@ test_hom_simple (unsigned int a, | |||
119 | "GNUNET_CRYPTO_paillier failed simple math!\n"); | 122 | "GNUNET_CRYPTO_paillier failed simple math!\n"); |
120 | gcry_log_debugmpi ("got ", hom_result); | 123 | gcry_log_debugmpi ("got ", hom_result); |
121 | gcry_log_debugmpi ("wanted ", result); | 124 | gcry_log_debugmpi ("wanted ", result); |
122 | return 1; | 125 | ret = 1; |
123 | } | 126 | } |
124 | return 0; | 127 | gcry_mpi_release (m1); |
128 | gcry_mpi_release (m2); | ||
129 | gcry_mpi_release (result); | ||
130 | gcry_mpi_release (hom_result); | ||
131 | return ret; | ||
125 | } | 132 | } |
126 | 133 | ||
127 | 134 | ||
@@ -168,7 +175,8 @@ test_hom () | |||
168 | fprintf (stderr, | 175 | fprintf (stderr, |
169 | "GNUNET_CRYPTO_paillier_encrypt 1 failed, should return 1 allowed operation, got %d!\n", | 176 | "GNUNET_CRYPTO_paillier_encrypt 1 failed, should return 1 allowed operation, got %d!\n", |
170 | ret); | 177 | ret); |
171 | return 1; | 178 | ret = 1; |
179 | goto out; | ||
172 | } | 180 | } |
173 | if (2 != (ret = GNUNET_CRYPTO_paillier_encrypt (&public_key, | 181 | if (2 != (ret = GNUNET_CRYPTO_paillier_encrypt (&public_key, |
174 | m2, | 182 | m2, |
@@ -178,7 +186,8 @@ test_hom () | |||
178 | fprintf (stderr, | 186 | fprintf (stderr, |
179 | "GNUNET_CRYPTO_paillier_encrypt 2 failed, should return 2 allowed operation, got %d!\n", | 187 | "GNUNET_CRYPTO_paillier_encrypt 2 failed, should return 2 allowed operation, got %d!\n", |
180 | ret); | 188 | ret); |
181 | return 1; | 189 | ret = 1; |
190 | goto out; | ||
182 | } | 191 | } |
183 | 192 | ||
184 | if (0 != (ret = GNUNET_CRYPTO_paillier_hom_add (&public_key, | 193 | if (0 != (ret = GNUNET_CRYPTO_paillier_hom_add (&public_key, |
@@ -189,7 +198,8 @@ test_hom () | |||
189 | fprintf (stderr, | 198 | fprintf (stderr, |
190 | "GNUNET_CRYPTO_paillier_hom_add failed, expected 0 remaining operations, got %d!\n", | 199 | "GNUNET_CRYPTO_paillier_hom_add failed, expected 0 remaining operations, got %d!\n", |
191 | ret); | 200 | ret); |
192 | return 1; | 201 | ret = 1; |
202 | goto out; | ||
193 | } | 203 | } |
194 | 204 | ||
195 | GNUNET_CRYPTO_paillier_decrypt (&private_key, | 205 | GNUNET_CRYPTO_paillier_decrypt (&private_key, |
@@ -203,9 +213,14 @@ test_hom () | |||
203 | "GNUNET_CRYPTO_paillier miscalculated with large numbers!\n"); | 213 | "GNUNET_CRYPTO_paillier miscalculated with large numbers!\n"); |
204 | gcry_log_debugmpi ("got", hom_result); | 214 | gcry_log_debugmpi ("got", hom_result); |
205 | gcry_log_debugmpi ("wanted", result); | 215 | gcry_log_debugmpi ("wanted", result); |
206 | return 1; | 216 | ret = 1; |
207 | } | 217 | } |
208 | return 0; | 218 | out: |
219 | gcry_mpi_release (m1); | ||
220 | gcry_mpi_release (m2); | ||
221 | gcry_mpi_release (result); | ||
222 | gcry_mpi_release (hom_result); | ||
223 | return ret; | ||
209 | } | 224 | } |
210 | 225 | ||
211 | 226 | ||
diff --git a/src/util/test_mq.c b/src/util/test_mq.c index 442c110db..9e8fc844e 100644 --- a/src/util/test_mq.c +++ b/src/util/test_mq.c | |||
@@ -51,6 +51,7 @@ test1 () | |||
51 | GNUNET_assert (NULL != mm); | 51 | GNUNET_assert (NULL != mm); |
52 | GNUNET_assert (42 == ntohs (mm->header.type)); | 52 | GNUNET_assert (42 == ntohs (mm->header.type)); |
53 | GNUNET_assert (sizeof (struct MyMessage) == ntohs (mm->header.size)); | 53 | GNUNET_assert (sizeof (struct MyMessage) == ntohs (mm->header.size)); |
54 | GNUNET_MQ_discard (mqm); | ||
54 | } | 55 | } |
55 | 56 | ||
56 | 57 | ||