aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--.gitignore3
-rw-r--r--contrib/timeout_watchdog.c7
-rw-r--r--doc/documentation/chapters/installation.texi47
-rw-r--r--doc/man/Makefile.am4
-rw-r--r--doc/man/gnunet-fs.17
-rw-r--r--doc/man/gnunet-gns-import.117
-rw-r--r--doc/man/gnunet-namestore-fcfsd.12
-rw-r--r--doc/man/gnunet-zoneimport.147
-rw-r--r--src/credential/gnunet-service-credential.c4
-rw-r--r--src/dht/dht.h29
-rw-r--r--src/dht/dht_api.c55
-rw-r--r--src/dht/dht_test_lib.c14
-rw-r--r--src/dht/gnunet-dht-put.c81
-rw-r--r--src/dht/gnunet-service-dht_clients.c8
-rw-r--r--src/dht/test_dht_api.c4
-rw-r--r--src/dht/test_dht_topo.c9
-rw-r--r--src/dns/dnsstub.c683
-rw-r--r--src/dns/gnunet-service-dns.c43
-rw-r--r--src/dns/gnunet-zoneimport.c24
-rw-r--r--src/exit/gnunet-daemon-exit.c42
-rw-r--r--src/fs/gnunet-service-fs_put.c7
-rw-r--r--src/gns/gns.conf.in3
-rw-r--r--src/gns/gns_tld_api.c14
-rw-r--r--src/gns/gnunet-dns2gns.c19
-rw-r--r--src/gns/gnunet-gns-proxy.c14
-rw-r--r--src/gns/gnunet-service-gns.c10
-rw-r--r--src/gns/gnunet-service-gns_resolver.c1099
-rw-r--r--src/gns/test_gns_lookup.conf2
-rw-r--r--src/gnsrecord/gnsrecord_crypto.c177
-rw-r--r--src/gnsrecord/gnsrecord_misc.c18
-rw-r--r--src/gnsrecord/test_gnsrecord_crypto.c79
-rw-r--r--src/identity-provider/gnunet-service-identity-provider.c21
-rw-r--r--src/identity-provider/plugin_rest_openid_connect.c7
-rw-r--r--src/include/gnunet_dht_service.h18
-rw-r--r--src/include/gnunet_dnsstub_lib.h85
-rw-r--r--src/include/gnunet_gns_service.h9
-rw-r--r--src/include/gnunet_gnsrecord_lib.h26
-rw-r--r--src/include/gnunet_namestore_plugin.h16
-rw-r--r--src/include/gnunet_namestore_service.h5
-rw-r--r--src/include/gnunet_protocols.h5
-rw-r--r--src/namecache/namecache.conf.in5
-rw-r--r--src/namecache/plugin_namecache_sqlite.c75
-rw-r--r--src/namestore/.gitignore48
-rw-r--r--src/namestore/Makefile.am466
-rw-r--r--src/namestore/gnunet-namestore-fcfsd.c21
-rw-r--r--src/namestore/gnunet-namestore.c14
-rw-r--r--src/namestore/gnunet-service-namestore.c372
-rw-r--r--src/namestore/gnunet-zoneimport.c977
-rw-r--r--src/namestore/namestore.conf.in11
-rw-r--r--src/namestore/namestore.h12
-rw-r--r--src/namestore/namestore_api.c11
-rw-r--r--src/namestore/perf_namestore_api_flat.conf7
-rw-r--r--src/namestore/perf_namestore_api_postgres.conf7
-rw-r--r--src/namestore/perf_namestore_api_sqlite.conf4
-rw-r--r--src/namestore/perf_namestore_api_zone_iteration.c388
-rw-r--r--src/namestore/plugin_namestore_flat.c192
-rw-r--r--src/namestore/plugin_namestore_postgres.c133
-rw-r--r--src/namestore/plugin_namestore_sqlite.c283
-rw-r--r--src/namestore/plugin_rest_namestore.c8
-rw-r--r--src/namestore/test_namestore_api.conf11
-rw-r--r--src/namestore/test_namestore_api_flat.conf4
-rw-r--r--src/namestore/test_namestore_api_lookup_nick.c16
-rw-r--r--src/namestore/test_namestore_api_lookup_private.c14
-rw-r--r--src/namestore/test_namestore_api_lookup_public.c10
-rw-r--r--src/namestore/test_namestore_api_lookup_shadow.c16
-rw-r--r--src/namestore/test_namestore_api_lookup_shadow_filter.c12
-rw-r--r--src/namestore/test_namestore_api_monitoring.c34
-rw-r--r--src/namestore/test_namestore_api_monitoring_existing.c41
-rw-r--r--src/namestore/test_namestore_api_postgres.conf4
-rw-r--r--src/namestore/test_namestore_api_remove.c30
-rw-r--r--src/namestore/test_namestore_api_remove_not_existing_record.c12
-rw-r--r--src/namestore/test_namestore_api_sqlite.conf4
-rw-r--r--src/namestore/test_namestore_api_store.c10
-rw-r--r--src/namestore/test_namestore_api_store_update.c10
-rw-r--r--src/namestore/test_namestore_api_zone_iteration.c13
-rw-r--r--src/namestore/test_namestore_api_zone_iteration_nick.c15
-rw-r--r--src/namestore/test_namestore_api_zone_iteration_specific_zone.c15
-rw-r--r--src/namestore/test_namestore_api_zone_iteration_stop.c13
-rw-r--r--src/namestore/test_namestore_api_zone_to_name.c17
-rw-r--r--src/namestore/test_plugin_namestore.c100
-rw-r--r--src/util/container_multihashmap.c55
-rw-r--r--src/util/mq.c8
-rw-r--r--src/util/resolver_api.c10
-rw-r--r--src/zonemaster/gnunet-service-zonemaster.c608
84 files changed, 4711 insertions, 2169 deletions
diff --git a/.gitignore b/.gitignore
index 3c739546f..b7dd58739 100644
--- a/.gitignore
+++ b/.gitignore
@@ -46,3 +46,6 @@ src/credential/gnunet-credential
46src/credential/gnunet-service-credential 46src/credential/gnunet-service-credential
47src/identity-provider/gnunet-idp 47src/identity-provider/gnunet-idp
48*.patch 48*.patch
49src/namestore/perf_namestore_api_zone_iteration_flat
50src/namestore/perf_namestore_api_zone_iteration_postgres
51src/namestore/perf_namestore_api_zone_iteration_sqlite
diff --git a/contrib/timeout_watchdog.c b/contrib/timeout_watchdog.c
index 7a0376b9f..fc61a7cc7 100644
--- a/contrib/timeout_watchdog.c
+++ b/contrib/timeout_watchdog.c
@@ -33,12 +33,14 @@
33 33
34static pid_t child; 34static pid_t child;
35 35
36
36static void 37static void
37sigchld_handler (int val) 38sigchld_handler (int val)
38{ 39{
39 int status = 0; 40 int status = 0;
40 int ret = 0; 41 int ret = 0;
41 42
43 (void) val;
42 waitpid (child, &status, 0); 44 waitpid (child, &status, 0);
43 if (WIFEXITED (status) != 0) 45 if (WIFEXITED (status) != 0)
44 { 46 {
@@ -53,6 +55,7 @@ sigchld_handler (int val)
53 exit (ret); 55 exit (ret);
54} 56}
55 57
58
56static void 59static void
57sigint_handler (int val) 60sigint_handler (int val)
58{ 61{
@@ -60,8 +63,10 @@ sigint_handler (int val)
60 exit (val); 63 exit (val);
61} 64}
62 65
66
63int 67int
64main (int argc, char *argv[]) 68main (int argc,
69 char *argv[])
65{ 70{
66 int timeout = 0; 71 int timeout = 0;
67 pid_t gpid = 0; 72 pid_t gpid = 0;
diff --git a/doc/documentation/chapters/installation.texi b/doc/documentation/chapters/installation.texi
index eeb14473b..3a76fb162 100644
--- a/doc/documentation/chapters/installation.texi
+++ b/doc/documentation/chapters/installation.texi
@@ -294,7 +294,7 @@ If you want to be able to enable DEBUG-level log messages, add
294@command{./configure} command. 294@command{./configure} command.
295@code{DEBUG}-level log messages are in English only and 295@code{DEBUG}-level log messages are in English only and
296should only be useful for developers (or for filing 296should only be useful for developers (or for filing
297really detailed bug reports). 297really detailed bug reports).
298 298
299Finally, you probably want to compile @command{gnunet-gtk}, which 299Finally, you probably want to compile @command{gnunet-gtk}, which
300includes @command{gnunet-setup} (a graphical tool for 300includes @command{gnunet-setup} (a graphical tool for
@@ -389,10 +389,9 @@ For those services, systems with more than one user may require each user
389to specify a different port number in their personal configuration file. 389to specify a different port number in their personal configuration file.
390 390
391Finally, the user should perform the basic initial setup for the GNU Name 391Finally, the user should perform the basic initial setup for the GNU Name
392System (GNS). This is done by running two commands: 392System (GNS) certificate authority. This is done by running:
393 393
394@example 394@example
395$ gnunet-gns-import.sh
396$ gnunet-gns-proxy-setup-ca 395$ gnunet-gns-proxy-setup-ca
397@end example 396@end example
398 397
@@ -420,7 +419,7 @@ hosts: files gns [NOTFOUND=return] mdns4_minimal \
420 419
421@c FIXME: Document new behavior. 420@c FIXME: Document new behavior.
422You might want to make sure that @file{/lib/libnss_gns.so.2} exists on 421You might want to make sure that @file{/lib/libnss_gns.so.2} exists on
423your system, it should have been created during the installation. 422your system, it should have been created during the installation.
424 423
425@node Build instructions for Ubuntu 12.04 using Git 424@node Build instructions for Ubuntu 12.04 using Git
426@section Build instructions for Ubuntu 12.04 using Git 425@section Build instructions for Ubuntu 12.04 using Git
@@ -802,7 +801,7 @@ $ gpg --keyserver pgp.mit.edu --recv-keys A88C8ADD129828D7EAC02E52E22F9BBFEE3485
802@end example 801@end example
803 802
804@noindent 803@noindent
805and rerun the gpg --verify command. 804and rerun the gpg --verify command.
806 805
807@example 806@example
808$ tar xvf gnurl-7.57.0.tar.xz 807$ tar xvf gnurl-7.57.0.tar.xz
@@ -1418,7 +1417,7 @@ as a normal user.
1418We begin by installing a few Debian packages from stable: 1417We begin by installing a few Debian packages from stable:
1419 1418
1420@example 1419@example
1421# apt-get install gcc make python-zbar libltdl-dev libsqlite3-dev \ 1420# apt-get install gcc make python-zbar libltdl-dev libsqlite3-dev \
1422libunistring-dev libopus-dev libpulse-dev openssl libglpk-dev texlive \ 1421libunistring-dev libopus-dev libpulse-dev openssl libglpk-dev texlive \
1423libidn11-dev libmysqlclient-dev libpq-dev libarchive-dev libbz2-dev \ 1422libidn11-dev libmysqlclient-dev libpq-dev libarchive-dev libbz2-dev \
1424libflac-dev libgif-dev libglib2.0-dev libgtk-3-dev libmpeg2-4-dev \ 1423libflac-dev libgif-dev libglib2.0-dev libgtk-3-dev libmpeg2-4-dev \
@@ -1633,7 +1632,7 @@ have to compile it from source:
1633@item Unzip the downloaded source tarball using your favourite 1632@item Unzip the downloaded source tarball using your favourite
1634unzipper application In the MSYS shell 1633unzipper application In the MSYS shell
1635 1634
1636@item change to the respective directory 1635@item change to the respective directory
1637 1636
1638@item Configure glpk for "i686-pc-mingw32": 1637@item Configure glpk for "i686-pc-mingw32":
1639 1638
@@ -1781,7 +1780,7 @@ $ make install
1781This document is a @b{DEPRECATED} installation guide for GNUnet on 1780This document is a @b{DEPRECATED} installation guide for GNUnet on
1782Windows. 1781Windows.
1783It will not work for recent GNUnet versions, but maybe it will be of 1782It will not work for recent GNUnet versions, but maybe it will be of
1784some use if problems arise. 1783some use if problems arise.
1785 1784
1786The Windows build uses a UNIX emulator for Windows, 1785The Windows build uses a UNIX emulator for Windows,
1787@uref{http://www.mingw.org/, MinGW}, to build the executable modules. 1786@uref{http://www.mingw.org/, MinGW}, to build the executable modules.
@@ -1799,7 +1798,7 @@ We regret any inconvenience, and if you have problems, please report them.
1799* Installer:: 1798* Installer::
1800* Source:: 1799* Source::
1801@end menu 1800@end menu
1802 1801
1803@node Hardware and OS requirements 1802@node Hardware and OS requirements
1804@subsubsection Hardware and OS requirements 1803@subsubsection Hardware and OS requirements
1805 1804
@@ -1878,7 +1877,7 @@ This batch file opens a shell which is used to invoke the build
1878processes. 1877processes.
1879MinGW's standard shell (@command{msys.bat}) is not suitable 1878MinGW's standard shell (@command{msys.bat}) is not suitable
1880because it opens a separate console window. 1879because it opens a separate console window.
1881On Vista, @command{bash.bat} needs to be run as Administrator. 1880On Vista, @command{bash.bat} needs to be run as Administrator.
1882 1881
1883@item 1882@item
1884Start @command{bash.sh} and rename 1883Start @command{bash.sh} and rename
@@ -1948,7 +1947,7 @@ directory (@file{c:\mingw\mingw}).
1948 1947
1949@item @strong{SQLite}@ 1948@item @strong{SQLite}@
1950GNUnet uses the SQLite database to store data. 1949GNUnet uses the SQLite database to store data.
1951Get the prebuilt binary from here and unpack it to your MinGW directory. 1950Get the prebuilt binary from here and unpack it to your MinGW directory.
1952 1951
1953@item @strong{MySQL}@ 1952@item @strong{MySQL}@
1954As an alternative to SQLite, GNUnet also supports MySQL. 1953As an alternative to SQLite, GNUnet also supports MySQL.
@@ -1984,7 +1983,7 @@ dlltool --input-def ../include/libmySQL.def \
1984--output-lib libmysqlclient.a -k 1983--output-lib libmysqlclient.a -k
1985@end example 1984@end example
1986 1985
1987@item Copy include\* to include\mysql\ 1986@item Copy include\* to include\mysql\
1988 1987
1989@item Pass @code{--with-mysql=/c/mysql} to 1988@item Pass @code{--with-mysql=/c/mysql} to
1990@command{./configure} and copy @file{libmysql.dll} 1989@command{./configure} and copy @file{libmysql.dll}
@@ -2082,7 +2081,7 @@ debug information and are quite large. To compile release versions
2082(small and fast) set the variable @code{CFLAGS}: 2081(small and fast) set the variable @code{CFLAGS}:
2083 2082
2084@example 2083@example
2085export CFLAGS='-O2 -march=pentium -fomit-frame-pointer' 2084export CFLAGS='-O2 -march=pentium -fomit-frame-pointer'
2086./configure --prefix=$HOME --with-extractor=$HOME 2085./configure --prefix=$HOME --with-extractor=$HOME
2087@end example 2086@end example
2088 2087
@@ -2097,7 +2096,7 @@ located in @file{contrib\win} in the GNUnet source tree.
2097@subsubsection Source 2096@subsubsection Source
2098 2097
2099@c FIXME: URL 2098@c FIXME: URL
2100The sources of all dependencies are available here. 2099The sources of all dependencies are available here.
2101 2100
2102@c @node Portable GNUnet 2101@c @node Portable GNUnet
2103@c @section Portable GNUnet 2102@c @section Portable GNUnet
@@ -2372,7 +2371,7 @@ The hostlist client supports the following proxy types at the moment:
2372@end itemize 2371@end itemize
2373 2372
2374In addition authentication at the proxy with username and password can be 2373In addition authentication at the proxy with username and password can be
2375configured. 2374configured.
2376 2375
2377To configure proxy support for the hostlist client in the 2376To configure proxy support for the hostlist client in the
2378@command{gnunet-setup} tool, select the "hostlist" tab and select 2377@command{gnunet-setup} tool, select the "hostlist" tab and select
@@ -2521,7 +2520,7 @@ inconsistencies. Some of the other databases do not support repair.
2521@item Access mysql as root: 2520@item Access mysql as root:
2522 2521
2523@example 2522@example
2524$ mysql -u root -p 2523$ mysql -u root -p
2525@end example 2524@end example
2526 2525
2527@noindent 2526@noindent
@@ -3324,19 +3323,6 @@ steps.
3324First of all, GNS needs to be integrated with the operating system. Most 3323First of all, GNS needs to be integrated with the operating system. Most
3325of this section is about the operating system level integration. 3324of this section is about the operating system level integration.
3326 3325
3327Additionally, each individual user who wants to use the system must also
3328initialize their GNS zones. This can be done by running (after starting
3329GNUnet)
3330
3331@example
3332$ gnunet-gns-import.sh
3333@end example
3334
3335@noindent
3336after the local GNUnet peer has been started. Note that the namestore (in
3337particular the namestore database backend) should not be reconfigured
3338afterwards (as records are not automatically migrated between backends).
3339
3340The remainder of this chapter will detail the various methods for 3326The remainder of this chapter will detail the various methods for
3341configuring the use of GNS with your operating system. 3327configuring the use of GNS with your operating system.
3342 3328
@@ -3528,7 +3514,7 @@ gnurl --version
3528curl 7.56.0 (x86_64-unknown-linux-gnu) libcurl/7.56.0 \ 3514curl 7.56.0 (x86_64-unknown-linux-gnu) libcurl/7.56.0 \
3529GnuTLS/3.5.13 zlib/1.2.11 libidn2/2.0.4 3515GnuTLS/3.5.13 zlib/1.2.11 libidn2/2.0.4
3530Release-Date: 2017-10-08 3516Release-Date: 2017-10-08
3531Protocols: http https 3517Protocols: http https
3532Features: AsynchDNS IDN IPv6 Largefile NTLM SSL libz \ 3518Features: AsynchDNS IDN IPv6 Largefile NTLM SSL libz \
3533TLS-SRP UnixSockets HTTPS-proxy 3519TLS-SRP UnixSockets HTTPS-proxy
3534@end example 3520@end example
@@ -4125,4 +4111,3 @@ Furthermore, 'make install' will silently fail to set the DNS binaries to
4125be owned by group "gnunetdns" unless that group already exists (!). 4111be owned by group "gnunetdns" unless that group already exists (!).
4126An alternative name for the "gnunetdns" group can be specified using the 4112An alternative name for the "gnunetdns" group can be specified using the
4127@code{--with-gnunetdns=GRPNAME} configure option. 4113@code{--with-gnunetdns=GRPNAME} configure option.
4128
diff --git a/doc/man/Makefile.am b/doc/man/Makefile.am
index c40363b59..a6a116dca 100644
--- a/doc/man/Makefile.am
+++ b/doc/man/Makefile.am
@@ -17,7 +17,6 @@ man_MANS = \
17 gnunet-ecc.1 \ 17 gnunet-ecc.1 \
18 gnunet-fs.1 \ 18 gnunet-fs.1 \
19 gnunet-gns.1 \ 19 gnunet-gns.1 \
20 gnunet-gns-import.1 \
21 gnunet-gns-proxy.1 \ 20 gnunet-gns-proxy.1 \
22 gnunet-identity.1 \ 21 gnunet-identity.1 \
23 gnunet-cadet.1 \ 22 gnunet-cadet.1 \
@@ -42,6 +41,7 @@ man_MANS = \
42 gnunet-transport-certificate-creation.1 \ 41 gnunet-transport-certificate-creation.1 \
43 gnunet-unindex.1 \ 42 gnunet-unindex.1 \
44 gnunet-uri.1 \ 43 gnunet-uri.1 \
45 gnunet-vpn.1 44 gnunet-vpn.1 \
45 gnunet-zoneimport.1
46 46
47EXTRA_DIST = ${man_MANS} 47EXTRA_DIST = ${man_MANS}
diff --git a/doc/man/gnunet-fs.1 b/doc/man/gnunet-fs.1
index 53a900b46..65f104d61 100644
--- a/doc/man/gnunet-fs.1
+++ b/doc/man/gnunet-fs.1
@@ -29,10 +29,9 @@ print the version number
29\fB\-V\fR, \fB\-\-verbose\fR 29\fB\-V\fR, \fB\-\-verbose\fR
30be verbose 30be verbose
31 31
32.SH NOTES
33 32
34 33.SH BUGS
35.SH "REPORTING BUGS"
36Report bugs by using mantis <https://gnunet.org/bugs/> or by sending electronic mail to <gnunet\-developers@gnu.org> 34Report bugs by using mantis <https://gnunet.org/bugs/> or by sending electronic mail to <gnunet\-developers@gnu.org>
37.SH "SEE ALSO" 35
36.SH SEE ALSO
38\fBgnunet\-publish\fP(1) 37\fBgnunet\-publish\fP(1)
diff --git a/doc/man/gnunet-gns-import.1 b/doc/man/gnunet-gns-import.1
deleted file mode 100644
index f90f7f375..000000000
--- a/doc/man/gnunet-gns-import.1
+++ /dev/null
@@ -1,17 +0,0 @@
1.TH GNUNET\-GNS\-IMPORT 1 "Jan 31, 2014" "GNUnet"
2
3.SH NAME
4gnunet\-gns\-import \- import some GNS authorities into your GNS namestore
5
6.SH SYNOPSIS
7.B gnunet\-gns\-import
8.br
9
10.SH DESCRIPTION
11\fBgnunet\-gns\-import\fP calls gnunet-namestore to fetch some GNS authorities into your GNS namestore.
12
13.SH BUGS
14Report bugs by using Mantis <https://gnunet.org/bugs/> or by sending electronic mail to <gnunet\-developers@gnu.org>
15
16.SH SEE ALSO
17gnunet\-gns(1), gnunet\-namestore(1)
diff --git a/doc/man/gnunet-namestore-fcfsd.1 b/doc/man/gnunet-namestore-fcfsd.1
index 3160910d4..8039e796f 100644
--- a/doc/man/gnunet-namestore-fcfsd.1
+++ b/doc/man/gnunet-namestore-fcfsd.1
@@ -13,7 +13,7 @@ Most users will not want to run an FCFS\-zone and thus will not need this progra
13 13
14\fBgnunet\-gns-fcfsd\fP runs a web server where users can register names to be mapped to their GNS zone. Names are made available on a First Come First Served basis (hence fcfs). Registered names do not expire. The HTTP server is run on the port that is specified in the configuration file in section "[fcfsd]" under the name "HTTPPORT". The key of the zone in which the names are registered must be specified under the name "ZONEKEY" in the same section. It is possible to manage gnunet\-gns\-fcfsd using gnunet\-(service\-arm) by starting the daemon using "gnunet\-arm \-i fcfsd" or by setting "FORCESTART=YES" in the "fcfds" section of your configuration. 14\fBgnunet\-gns-fcfsd\fP runs a web server where users can register names to be mapped to their GNS zone. Names are made available on a First Come First Served basis (hence fcfs). Registered names do not expire. The HTTP server is run on the port that is specified in the configuration file in section "[fcfsd]" under the name "HTTPPORT". The key of the zone in which the names are registered must be specified under the name "ZONEKEY" in the same section. It is possible to manage gnunet\-gns\-fcfsd using gnunet\-(service\-arm) by starting the daemon using "gnunet\-arm \-i fcfsd" or by setting "FORCESTART=YES" in the "fcfds" section of your configuration.
15 15
16An FCFS\-zone is run at http://gnunet.org/fcfs/. The respective zone key can be imported into an individual user's zone using "gnunet-gns-import.sh". GNS users are encouraged to register their zone with the gnunet.org FCFS authority. 16An FCFS\-zone is run at http://gnunet.org/fcfs/. GNS users are encouraged to register their zone with the gnunet.org FCFS authority.
17 17
18If you want to run your own FCFS registrar, you need to first create a pseudonym (using "gnunet\-identity \-C NAME"), and then assign it to be used for the "fcfsd" service using "gnunet\-identity \-e NAME \-s fcfsd". After that, you can start the FCFSD service (possibly using gnunet\-arm). 18If you want to run your own FCFS registrar, you need to first create a pseudonym (using "gnunet\-identity \-C NAME"), and then assign it to be used for the "fcfsd" service using "gnunet\-identity \-e NAME \-s fcfsd". After that, you can start the FCFSD service (possibly using gnunet\-arm).
19 19
diff --git a/doc/man/gnunet-zoneimport.1 b/doc/man/gnunet-zoneimport.1
new file mode 100644
index 000000000..06b3a6bcf
--- /dev/null
+++ b/doc/man/gnunet-zoneimport.1
@@ -0,0 +1,47 @@
1.TH GNUNET\-ZONEIMPORT 1 "Apr 23, 2018" "GNUnet"
2
3.SH NAME
4gnunet\-zoneimport \- import DNS zone into GNS zone
5
6.SH SYNOPSIS
7.B gnunet\-zoneimport [IP]+
8.br
9
10.SH DESCRIPTION
11\fBgnunet\-zoneimport\fP reads a list of domain names (FQDN) from stdin and issues DNS queries for each of the domain names given. It then checks if a local ego with a name matching the domain exists. Specifically, if the domain name is "example.fr", it will check if an ego "fr" exists, while for a domain "example.com.fr" it will look for an ego called "com.fr"). If so, it will convert the DNS records into GNS records (in particular converting NS records and glue records to GNS2DNS records) and add them to the namestore under the label ("example" in the examples above).
12
13The arguments given to gnunet\-zoneimport is a list of IP addresses of DNS servers to query.
14
15gnunet\-zoneimport will usually never terminate: it will check when DNS records expire, and re-issue requests when the old DNS records have expired so that GNS always has the latest data.
16
17gnunet\-zoneimport will issue many DNS queries in parallel, but is rate-limited in various ways, so most DNS servers should easily handle the load. gnunet\-zoneimport will perform a limited number of retries if queries fail.
18
19gnunet\-zoneimport operates incrementally. It will check if the namestore already has (non-expired) records stored for a given name in the respective zone and not issue those requests again. Thus, it is fine to restart gnunet\-zoneimport whenever the list of domain names changes.
20
21Finally, gnunet\-zoneimport keeps information for each domain name in memory. This consumes about 200 bytes per domain name, or 1 GB for 5 million labels.
22
23.SH OPTIONS
24.B
25.IP "\-c FILENAME, \-\-config=FILENAME"
26Use the configuration file FILENAME.
27.B
28.IP "\-h, \-\-help"
29Print short help on options.
30.B
31.IP "\-s MAPSIZE, \-\-size=MAPSIZE"
32Specifies the size (in number of entries) to use for the main hash map. The value provided should be at least twice the number of domain names that will be given to the tool. This option is required for very large zones where the number of records encountered is too large for the automatic growth mechanism to work (that one is limited to at most 16 MB allocations for security reasons). Do not worry about this unless you are importing millions of domain names from a zone.
33
34.SH NOTES
35
36Typical invocaton would be:
37
38$ gnunet\-zoneimport 1.2.3.4 < names.txt
39
40
41
42
43.SH BUGS
44Report bugs by using Mantis <https://gnunet.org/bugs/> or by sending electronic mail to <gnunet\-developers@gnu.org>
45
46.SH SEE ALSO
47gnunet\-gns(1), gnunet\-namestore(1)
diff --git a/src/credential/gnunet-service-credential.c b/src/credential/gnunet-service-credential.c
index a2c339363..399371a2e 100644
--- a/src/credential/gnunet-service-credential.c
+++ b/src/credential/gnunet-service-credential.c
@@ -961,9 +961,11 @@ collect_next (void *cls)
961 struct VerifyRequestHandle *vrh = cls; 961 struct VerifyRequestHandle *vrh = cls;
962 vrh->collect_next_task = NULL; 962 vrh->collect_next_task = NULL;
963 GNUNET_assert (NULL != vrh->cred_collection_iter); 963 GNUNET_assert (NULL != vrh->cred_collection_iter);
964 GNUNET_NAMESTORE_zone_iterator_next (vrh->cred_collection_iter); 964 GNUNET_NAMESTORE_zone_iterator_next (vrh->cred_collection_iter,
965 1);
965} 966}
966 967
968
967static void 969static void
968handle_cred_collection_cb (void *cls, 970handle_cred_collection_cb (void *cls,
969 const struct GNUNET_CRYPTO_EcdsaPrivateKey *key, 971 const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
diff --git a/src/dht/dht.h b/src/dht/dht.h
index 4c994f93a..95ffa33ca 100644
--- a/src/dht/dht.h
+++ b/src/dht/dht.h
@@ -214,11 +214,6 @@ struct GNUNET_DHT_ClientPutMessage
214 uint32_t desired_replication_level GNUNET_PACKED; 214 uint32_t desired_replication_level GNUNET_PACKED;
215 215
216 /** 216 /**
217 * Unique ID for the PUT message.
218 */
219 uint64_t unique_id GNUNET_PACKED;
220
221 /**
222 * How long should this data persist? 217 * How long should this data persist?
223 */ 218 */
224 struct GNUNET_TIME_AbsoluteNBO expiration; 219 struct GNUNET_TIME_AbsoluteNBO expiration;
@@ -234,30 +229,6 @@ struct GNUNET_DHT_ClientPutMessage
234 229
235 230
236/** 231/**
237 * Message to confirming receipt of PUT, sent from DHT service to clients.
238 */
239struct GNUNET_DHT_ClientPutConfirmationMessage
240{
241 /**
242 * Type: #GNUNET_MESSAGE_TYPE_DHT_CLIENT_PUT_OK
243 */
244 struct GNUNET_MessageHeader header;
245
246 /**
247 * Always zero.
248 */
249 uint32_t reserved GNUNET_PACKED;
250
251 /**
252 * Unique ID from the PUT message that is being confirmed.
253 */
254 uint64_t unique_id GNUNET_PACKED;
255
256};
257
258
259
260/**
261 * Message to monitor put requests going through peer, DHT service -> clients. 232 * Message to monitor put requests going through peer, DHT service -> clients.
262 */ 233 */
263struct GNUNET_DHT_MonitorPutMessage 234struct GNUNET_DHT_MonitorPutMessage
diff --git a/src/dht/dht_api.c b/src/dht/dht_api.c
index 42ddc7b60..af0dafbf3 100644
--- a/src/dht/dht_api.c
+++ b/src/dht/dht_api.c
@@ -55,7 +55,7 @@ struct GNUNET_DHT_PutHandle
55 /** 55 /**
56 * Continuation to call when done. 56 * Continuation to call when done.
57 */ 57 */
58 GNUNET_DHT_PutContinuation cont; 58 GNUNET_SCHEDULER_TaskCallback cont;
59 59
60 /** 60 /**
61 * Main handle to this DHT api 61 * Main handle to this DHT api
@@ -68,9 +68,9 @@ struct GNUNET_DHT_PutHandle
68 void *cont_cls; 68 void *cont_cls;
69 69
70 /** 70 /**
71 * Unique ID for the PUT operation. 71 * Envelope from the PUT operation.
72 */ 72 */
73 uint64_t unique_id; 73 struct GNUNET_MQ_Envelope *env;
74 74
75}; 75};
76 76
@@ -440,7 +440,7 @@ static void
440do_disconnect (struct GNUNET_DHT_Handle *h) 440do_disconnect (struct GNUNET_DHT_Handle *h)
441{ 441{
442 struct GNUNET_DHT_PutHandle *ph; 442 struct GNUNET_DHT_PutHandle *ph;
443 GNUNET_DHT_PutContinuation cont; 443 GNUNET_SCHEDULER_TaskCallback cont;
444 void *cont_cls; 444 void *cont_cls;
445 445
446 if (NULL == h->mq) 446 if (NULL == h->mq)
@@ -456,10 +456,10 @@ do_disconnect (struct GNUNET_DHT_Handle *h)
456 { 456 {
457 cont = ph->cont; 457 cont = ph->cont;
458 cont_cls = ph->cont_cls; 458 cont_cls = ph->cont_cls;
459 ph->env = NULL;
459 GNUNET_DHT_put_cancel (ph); 460 GNUNET_DHT_put_cancel (ph);
460 if (NULL != cont) 461 if (NULL != cont)
461 cont (cont_cls, 462 cont (cont_cls);
462 GNUNET_SYSERR);
463 } 463 }
464 GNUNET_assert (NULL == h->reconnect_task); 464 GNUNET_assert (NULL == h->reconnect_task);
465 h->reconnect_task 465 h->reconnect_task
@@ -818,31 +818,23 @@ handle_client_result (void *cls,
818 818
819 819
820/** 820/**
821 * Process a put confirmation message from the service. 821 * Process a MQ PUT transmission notification.
822 * 822 *
823 * @param cls The DHT handle. 823 * @param cls The DHT handle.
824 * @param msg confirmation message from the service.
825 */ 824 */
826static void 825static void
827handle_put_confirmation (void *cls, 826handle_put_cont (void *cls)
828 const struct GNUNET_DHT_ClientPutConfirmationMessage *msg)
829{ 827{
830 struct GNUNET_DHT_Handle *handle = cls; 828 struct GNUNET_DHT_PutHandle *ph = cls;
831 struct GNUNET_DHT_PutHandle *ph; 829 GNUNET_SCHEDULER_TaskCallback cont;
832 GNUNET_DHT_PutContinuation cont;
833 void *cont_cls; 830 void *cont_cls;
834 831
835 for (ph = handle->put_head; NULL != ph; ph = ph->next)
836 if (ph->unique_id == msg->unique_id)
837 break;
838 if (NULL == ph)
839 return;
840 cont = ph->cont; 832 cont = ph->cont;
841 cont_cls = ph->cont_cls; 833 cont_cls = ph->cont_cls;
834 ph->env = NULL;
842 GNUNET_DHT_put_cancel (ph); 835 GNUNET_DHT_put_cancel (ph);
843 if (NULL != cont) 836 if (NULL != cont)
844 cont (cont_cls, 837 cont (cont_cls);
845 GNUNET_OK);
846} 838}
847 839
848 840
@@ -872,10 +864,6 @@ try_connect (struct GNUNET_DHT_Handle *h)
872 GNUNET_MESSAGE_TYPE_DHT_CLIENT_RESULT, 864 GNUNET_MESSAGE_TYPE_DHT_CLIENT_RESULT,
873 struct GNUNET_DHT_ClientResultMessage, 865 struct GNUNET_DHT_ClientResultMessage,
874 h), 866 h),
875 GNUNET_MQ_hd_fixed_size (put_confirmation,
876 GNUNET_MESSAGE_TYPE_DHT_CLIENT_PUT_OK,
877 struct GNUNET_DHT_ClientPutConfirmationMessage,
878 h),
879 GNUNET_MQ_handler_end () 867 GNUNET_MQ_handler_end ()
880 }; 868 };
881 if (NULL != h->mq) 869 if (NULL != h->mq)
@@ -941,8 +929,7 @@ GNUNET_DHT_disconnect (struct GNUNET_DHT_Handle *handle)
941 while (NULL != (ph = handle->put_head)) 929 while (NULL != (ph = handle->put_head))
942 { 930 {
943 if (NULL != ph->cont) 931 if (NULL != ph->cont)
944 ph->cont (ph->cont_cls, 932 ph->cont (ph->cont_cls);
945 GNUNET_SYSERR);
946 GNUNET_DHT_put_cancel (ph); 933 GNUNET_DHT_put_cancel (ph);
947 } 934 }
948 if (NULL != handle->mq) 935 if (NULL != handle->mq)
@@ -989,7 +976,7 @@ GNUNET_DHT_put (struct GNUNET_DHT_Handle *handle,
989 size_t size, 976 size_t size,
990 const void *data, 977 const void *data,
991 struct GNUNET_TIME_Absolute exp, 978 struct GNUNET_TIME_Absolute exp,
992 GNUNET_DHT_PutContinuation cont, 979 GNUNET_SCHEDULER_TaskCallback cont,
993 void *cont_cls) 980 void *cont_cls)
994{ 981{
995 struct GNUNET_MQ_Envelope *env; 982 struct GNUNET_MQ_Envelope *env;
@@ -1014,22 +1001,24 @@ GNUNET_DHT_put (struct GNUNET_DHT_Handle *handle,
1014 ph->dht_handle = handle; 1001 ph->dht_handle = handle;
1015 ph->cont = cont; 1002 ph->cont = cont;
1016 ph->cont_cls = cont_cls; 1003 ph->cont_cls = cont_cls;
1017 ph->unique_id = ++handle->uid_gen;
1018 GNUNET_CONTAINER_DLL_insert_tail (handle->put_head, 1004 GNUNET_CONTAINER_DLL_insert_tail (handle->put_head,
1019 handle->put_tail, 1005 handle->put_tail,
1020 ph); 1006 ph);
1021 env = GNUNET_MQ_msg_extra (put_msg, 1007 env = GNUNET_MQ_msg_extra (put_msg,
1022 size, 1008 size,
1023 GNUNET_MESSAGE_TYPE_DHT_CLIENT_PUT); 1009 GNUNET_MESSAGE_TYPE_DHT_CLIENT_PUT);
1010 GNUNET_MQ_notify_sent (env,
1011 &handle_put_cont,
1012 ph);
1013 ph->env = env;
1024 put_msg->type = htonl ((uint32_t) type); 1014 put_msg->type = htonl ((uint32_t) type);
1025 put_msg->options = htonl ((uint32_t) options); 1015 put_msg->options = htonl ((uint32_t) options);
1026 put_msg->desired_replication_level = htonl (desired_replication_level); 1016 put_msg->desired_replication_level = htonl (desired_replication_level);
1027 put_msg->unique_id = ph->unique_id;
1028 put_msg->expiration = GNUNET_TIME_absolute_hton (exp); 1017 put_msg->expiration = GNUNET_TIME_absolute_hton (exp);
1029 put_msg->key = *key; 1018 put_msg->key = *key;
1030 GNUNET_memcpy (&put_msg[1], 1019 GNUNET_memcpy (&put_msg[1],
1031 data, 1020 data,
1032 size); 1021 size);
1033 GNUNET_MQ_send (handle->mq, 1022 GNUNET_MQ_send (handle->mq,
1034 env); 1023 env);
1035 return ph; 1024 return ph;
@@ -1052,6 +1041,10 @@ GNUNET_DHT_put_cancel (struct GNUNET_DHT_PutHandle *ph)
1052{ 1041{
1053 struct GNUNET_DHT_Handle *handle = ph->dht_handle; 1042 struct GNUNET_DHT_Handle *handle = ph->dht_handle;
1054 1043
1044 if (NULL != ph->env)
1045 GNUNET_MQ_notify_sent (ph->env,
1046 NULL,
1047 NULL);
1055 GNUNET_CONTAINER_DLL_remove (handle->put_head, 1048 GNUNET_CONTAINER_DLL_remove (handle->put_head,
1056 handle->put_tail, 1049 handle->put_tail,
1057 ph); 1050 ph);
diff --git a/src/dht/dht_test_lib.c b/src/dht/dht_test_lib.c
index 4c1bd3057..52d5a3731 100644
--- a/src/dht/dht_test_lib.c
+++ b/src/dht/dht_test_lib.c
@@ -114,7 +114,6 @@ dht_connect_cb (void *cls,
114 const char *emsg) 114 const char *emsg)
115{ 115{
116 struct GNUNET_DHT_TEST_Context *ctx = cls; 116 struct GNUNET_DHT_TEST_Context *ctx = cls;
117 unsigned int i;
118 117
119 if (NULL != emsg) 118 if (NULL != emsg)
120 { 119 {
@@ -124,10 +123,10 @@ dht_connect_cb (void *cls,
124 GNUNET_SCHEDULER_shutdown (); 123 GNUNET_SCHEDULER_shutdown ();
125 return; 124 return;
126 } 125 }
127 for (i=0;i<ctx->num_peers;i++) 126 for (unsigned int i=0;i<ctx->num_peers;i++)
128 if (op == ctx->ops[i]) 127 if (op == ctx->ops[i])
129 ctx->dhts[i] = ca_result; 128 ctx->dhts[i] = ca_result;
130 for (i=0;i<ctx->num_peers;i++) 129 for (unsigned int i=0;i<ctx->num_peers;i++)
131 if (NULL == ctx->dhts[i]) 130 if (NULL == ctx->dhts[i])
132 return; /* still some DHT connections missing */ 131 return; /* still some DHT connections missing */
133 /* all DHT connections ready! */ 132 /* all DHT connections ready! */
@@ -147,9 +146,7 @@ dht_connect_cb (void *cls,
147void 146void
148GNUNET_DHT_TEST_cleanup (struct GNUNET_DHT_TEST_Context *ctx) 147GNUNET_DHT_TEST_cleanup (struct GNUNET_DHT_TEST_Context *ctx)
149{ 148{
150 unsigned int i; 149 for (unsigned int i=0;i<ctx->num_peers;i++)
151
152 for (i=0;i<ctx->num_peers;i++)
153 GNUNET_TESTBED_operation_done (ctx->ops[i]); 150 GNUNET_TESTBED_operation_done (ctx->ops[i]);
154 GNUNET_free (ctx->ops); 151 GNUNET_free (ctx->ops);
155 GNUNET_free (ctx->dhts); 152 GNUNET_free (ctx->dhts);
@@ -160,18 +157,17 @@ GNUNET_DHT_TEST_cleanup (struct GNUNET_DHT_TEST_Context *ctx)
160 157
161static void 158static void
162dht_test_run (void *cls, 159dht_test_run (void *cls,
163 struct GNUNET_TESTBED_RunHandle *h, 160 struct GNUNET_TESTBED_RunHandle *h,
164 unsigned int num_peers, 161 unsigned int num_peers,
165 struct GNUNET_TESTBED_Peer **peers, 162 struct GNUNET_TESTBED_Peer **peers,
166 unsigned int links_succeeded, 163 unsigned int links_succeeded,
167 unsigned int links_failed) 164 unsigned int links_failed)
168{ 165{
169 struct GNUNET_DHT_TEST_Context *ctx = cls; 166 struct GNUNET_DHT_TEST_Context *ctx = cls;
170 unsigned int i;
171 167
172 GNUNET_assert (num_peers == ctx->num_peers); 168 GNUNET_assert (num_peers == ctx->num_peers);
173 ctx->peers = peers; 169 ctx->peers = peers;
174 for (i=0;i<num_peers;i++) 170 for (unsigned int i=0;i<num_peers;i++)
175 ctx->ops[i] = GNUNET_TESTBED_service_connect (ctx, 171 ctx->ops[i] = GNUNET_TESTBED_service_connect (ctx,
176 peers[i], 172 peers[i],
177 "dht", 173 "dht",
diff --git a/src/dht/gnunet-dht-put.c b/src/dht/gnunet-dht-put.c
index f183fe588..db4d04681 100644
--- a/src/dht/gnunet-dht-put.c
+++ b/src/dht/gnunet-dht-put.c
@@ -103,34 +103,12 @@ shutdown_task (void *cls)
103 * Signature of the main function of a task. 103 * Signature of the main function of a task.
104 * 104 *
105 * @param cls closure 105 * @param cls closure
106 * @param success #GNUNET_OK if the PUT was transmitted,
107 * #GNUNET_NO on timeout,
108 * #GNUNET_SYSERR on disconnect from service
109 * after the PUT message was transmitted
110 * (so we don't know if it was received or not)
111 */ 106 */
112static void 107static void
113message_sent_cont (void *cls, int success) 108message_sent_cont (void *cls)
114{ 109{
115 if (verbose) 110 GNUNET_SCHEDULER_add_now (&shutdown_task,
116 { 111 NULL);
117 switch (success)
118 {
119 case GNUNET_OK:
120 FPRINTF (stderr, "%s `%s'!\n", _("PUT request sent with key"), GNUNET_h2s_full(&key));
121 break;
122 case GNUNET_NO:
123 FPRINTF (stderr, "%s", _("Timeout sending PUT request!\n"));
124 break;
125 case GNUNET_SYSERR:
126 FPRINTF (stderr, "%s", _("PUT request not confirmed!\n"));
127 break;
128 default:
129 GNUNET_break (0);
130 break;
131 }
132 }
133 GNUNET_SCHEDULER_add_now (&shutdown_task, NULL);
134} 112}
135 113
136 114
@@ -160,7 +138,8 @@ run (void *cls,
160 138
161 if (NULL == (dht_handle = GNUNET_DHT_connect (cfg, 1))) 139 if (NULL == (dht_handle = GNUNET_DHT_connect (cfg, 1)))
162 { 140 {
163 FPRINTF (stderr, _("Could not connect to %s service!\n"), "DHT"); 141 FPRINTF (stderr,
142 _("Could not connect to DHT service!\n"));
164 ret = 1; 143 ret = 1;
165 return; 144 return;
166 } 145 }
@@ -203,55 +182,47 @@ main (int argc, char *const *argv)
203{ 182{
204 183
205 struct GNUNET_GETOPT_CommandLineOption options[] = { 184 struct GNUNET_GETOPT_CommandLineOption options[] = {
206
207 GNUNET_GETOPT_option_string ('d', 185 GNUNET_GETOPT_option_string ('d',
208 "data", 186 "data",
209 "DATA", 187 "DATA",
210 gettext_noop ("the data to insert under the key"), 188 gettext_noop ("the data to insert under the key"),
211 &data), 189 &data),
212
213 GNUNET_GETOPT_option_relative_time ('e', 190 GNUNET_GETOPT_option_relative_time ('e',
214 "expiration", 191 "expiration",
215 "EXPIRATION", 192 "EXPIRATION",
216 gettext_noop ("how long to store this entry in the dht (in seconds)"), 193 gettext_noop ("how long to store this entry in the dht (in seconds)"),
217 &expiration), 194 &expiration),
218
219 GNUNET_GETOPT_option_string ('k', 195 GNUNET_GETOPT_option_string ('k',
220 "key", 196 "key",
221 "KEY", 197 "KEY",
222 gettext_noop ("the query key"), 198 gettext_noop ("the query key"),
223 &query_key), 199 &query_key),
224
225 GNUNET_GETOPT_option_flag ('x', 200 GNUNET_GETOPT_option_flag ('x',
226 "demultiplex", 201 "demultiplex",
227 gettext_noop ("use DHT's demultiplex everywhere option"), 202 gettext_noop ("use DHT's demultiplex everywhere option"),
228 &demultixplex_everywhere), 203 &demultixplex_everywhere),
229
230 GNUNET_GETOPT_option_uint ('r', 204 GNUNET_GETOPT_option_uint ('r',
231 "replication", 205 "replication",
232 "LEVEL", 206 "LEVEL",
233 gettext_noop ("how many replicas to create"), 207 gettext_noop ("how many replicas to create"),
234 &replication), 208 &replication),
235
236 GNUNET_GETOPT_option_flag ('R', 209 GNUNET_GETOPT_option_flag ('R',
237 "record", 210 "record",
238 gettext_noop ("use DHT's record route option"), 211 gettext_noop ("use DHT's record route option"),
239 &record_route), 212 &record_route),
240
241 GNUNET_GETOPT_option_uint ('t', 213 GNUNET_GETOPT_option_uint ('t',
242 "type", 214 "type",
243 "TYPE", 215 "TYPE",
244 gettext_noop ("the type to insert data as"), 216 gettext_noop ("the type to insert data as"),
245 &query_type), 217 &query_type),
246
247 GNUNET_GETOPT_option_verbose (&verbose), 218 GNUNET_GETOPT_option_verbose (&verbose),
248
249 GNUNET_GETOPT_OPTION_END 219 GNUNET_GETOPT_OPTION_END
250 }; 220 };
251 221
252 222
253 if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, 223 if (GNUNET_OK !=
254 &argc, &argv)) 224 GNUNET_STRINGS_get_utf8_args (argc, argv,
225 &argc, &argv))
255 return 2; 226 return 2;
256 expiration = GNUNET_TIME_UNIT_HOURS; 227 expiration = GNUNET_TIME_UNIT_HOURS;
257 return (GNUNET_OK == 228 return (GNUNET_OK ==
diff --git a/src/dht/gnunet-service-dht_clients.c b/src/dht/gnunet-service-dht_clients.c
index cb155c484..503d7867b 100644
--- a/src/dht/gnunet-service-dht_clients.c
+++ b/src/dht/gnunet-service-dht_clients.c
@@ -477,8 +477,6 @@ handle_dht_local_put (void *cls,
477 struct ClientHandle *ch = cls; 477 struct ClientHandle *ch = cls;
478 struct GNUNET_CONTAINER_BloomFilter *peer_bf; 478 struct GNUNET_CONTAINER_BloomFilter *peer_bf;
479 uint16_t size; 479 uint16_t size;
480 struct GNUNET_MQ_Envelope *env;
481 struct GNUNET_DHT_ClientPutConfirmationMessage *conf;
482 480
483 size = ntohs (dht_msg->header.size); 481 size = ntohs (dht_msg->header.size);
484 GNUNET_STATISTICS_update (GDS_stats, 482 GNUNET_STATISTICS_update (GDS_stats,
@@ -537,12 +535,6 @@ handle_dht_local_put (void *cls,
537 &dht_msg[1], 535 &dht_msg[1],
538 size - sizeof (struct GNUNET_DHT_ClientPutMessage)); 536 size - sizeof (struct GNUNET_DHT_ClientPutMessage));
539 GNUNET_CONTAINER_bloomfilter_free (peer_bf); 537 GNUNET_CONTAINER_bloomfilter_free (peer_bf);
540 env = GNUNET_MQ_msg (conf,
541 GNUNET_MESSAGE_TYPE_DHT_CLIENT_PUT_OK);
542 conf->reserved = htonl (0);
543 conf->unique_id = dht_msg->unique_id;
544 GNUNET_MQ_send (ch->mq,
545 env);
546 GNUNET_SERVICE_client_continue (ch->client); 538 GNUNET_SERVICE_client_continue (ch->client);
547} 539}
548 540
diff --git a/src/dht/test_dht_api.c b/src/dht/test_dht_api.c
index 8f4e0ed31..62d121306 100644
--- a/src/dht/test_dht_api.c
+++ b/src/dht/test_dht_api.c
@@ -105,11 +105,9 @@ test_get_iterator (void *cls,
105 * Signature of the main function of a task. 105 * Signature of the main function of a task.
106 * 106 *
107 * @param cls closure 107 * @param cls closure
108 * @param success result of PUT
109 */ 108 */
110static void 109static void
111test_get (void *cls, 110test_get (void *cls)
112 int success)
113{ 111{
114 struct GNUNET_HashCode hash; 112 struct GNUNET_HashCode hash;
115 113
diff --git a/src/dht/test_dht_topo.c b/src/dht/test_dht_topo.c
index 8be3064f7..79edb2b0c 100644
--- a/src/dht/test_dht_topo.c
+++ b/src/dht/test_dht_topo.c
@@ -332,19 +332,17 @@ dht_get_handler (void *cls,
332 "Get successful\n"); 332 "Get successful\n");
333#if 0 333#if 0
334 { 334 {
335 int i;
336
337 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 335 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
338 "PATH: (get %u, put %u)\n", 336 "PATH: (get %u, put %u)\n",
339 get_path_length, 337 get_path_length,
340 put_path_length); 338 put_path_length);
341 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 339 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
342 " LOCAL\n"); 340 " LOCAL\n");
343 for (i = get_path_length - 1; i >= 0; i--) 341 for (int i = get_path_length - 1; i >= 0; i--)
344 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 342 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
345 " %s\n", 343 " %s\n",
346 GNUNET_i2s (&get_path[i])); 344 GNUNET_i2s (&get_path[i]));
347 for (i = put_path_length - 1; i >= 0; i--) 345 for (int i = put_path_length - 1; i >= 0; i--)
348 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 346 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
349 " %s\n", 347 " %s\n",
350 GNUNET_i2s (&put_path[i])); 348 GNUNET_i2s (&put_path[i]));
@@ -384,12 +382,11 @@ do_puts (void *cls)
384 struct GNUNET_DHT_Handle **hs = cls; 382 struct GNUNET_DHT_Handle **hs = cls;
385 struct GNUNET_HashCode key; 383 struct GNUNET_HashCode key;
386 struct GNUNET_HashCode value; 384 struct GNUNET_HashCode value;
387 unsigned int i;
388 385
389 put_task = NULL; 386 put_task = NULL;
390 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 387 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
391 "Putting values into DHT\n"); 388 "Putting values into DHT\n");
392 for (i = 0; i < NUM_PEERS; i++) 389 for (unsigned int i = 0; i < NUM_PEERS; i++)
393 { 390 {
394 GNUNET_CRYPTO_hash (&i, 391 GNUNET_CRYPTO_hash (&i,
395 sizeof (i), 392 sizeof (i),
diff --git a/src/dns/dnsstub.c b/src/dns/dnsstub.c
index 6aa2d7b8f..6eb3612c2 100644
--- a/src/dns/dnsstub.c
+++ b/src/dns/dnsstub.c
@@ -1,6 +1,6 @@
1/* 1/*
2 This file is part of GNUnet. 2 This file is part of GNUnet.
3 Copyright (C) 2012 GNUnet e.V. 3 Copyright (C) 2012, 2018 GNUnet e.V.
4 4
5 GNUnet is free software; you can redistribute it and/or modify 5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published 6 it under the terms of the GNU General Public License as published
@@ -28,20 +28,15 @@
28#include "gnunet_dnsstub_lib.h" 28#include "gnunet_dnsstub_lib.h"
29 29
30/** 30/**
31 * Timeout for an external (Internet-DNS) DNS resolution
32 */
33#define REQUEST_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 5)
34
35/**
36 * Timeout for retrying DNS queries. 31 * Timeout for retrying DNS queries.
37 */ 32 */
38#define DNS_RETRANSMIT_DELAY GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS, 250) 33#define DNS_RETRANSMIT_DELAY GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS, 250)
39 34
35
40/** 36/**
41 * How many DNS sockets do we open at most at the same time? 37 * DNS Server used for resolution.
42 * (technical socket maximum is this number x2 for IPv4+IPv6)
43 */ 38 */
44#define DNS_SOCKET_MAX 128 39struct DnsServer;
45 40
46 41
47/** 42/**
@@ -81,19 +76,14 @@ struct GNUNET_DNSSTUB_RequestSocket
81 struct GNUNET_SCHEDULER_Task *retry_task; 76 struct GNUNET_SCHEDULER_Task *retry_task;
82 77
83 /** 78 /**
84 * When should this request time out? 79 * Next address we sent the DNS request to.
85 */
86 struct GNUNET_TIME_Absolute timeout;
87
88 /**
89 * Address we sent the DNS request to.
90 */ 80 */
91 struct sockaddr_storage addr; 81 struct DnsServer *ds_pos;
92 82
93 /** 83 /**
94 * Number of bytes in @e addr. 84 * Context this request executes in.
95 */ 85 */
96 socklen_t addrlen; 86 struct GNUNET_DNSSTUB_Context *ctx;
97 87
98 /** 88 /**
99 * Query we sent to @e addr. 89 * Query we sent to @e addr.
@@ -109,6 +99,29 @@ struct GNUNET_DNSSTUB_RequestSocket
109 99
110 100
111/** 101/**
102 * DNS Server used for resolution.
103 */
104struct DnsServer
105{
106
107 /**
108 * Kept in a DLL.
109 */
110 struct DnsServer *next;
111
112 /**
113 * Kept in a DLL.
114 */
115 struct DnsServer *prev;
116
117 /**
118 * IP address of the DNS resolver.
119 */
120 struct sockaddr_storage ss;
121};
122
123
124/**
112 * Handle to the stub resolver. 125 * Handle to the stub resolver.
113 */ 126 */
114struct GNUNET_DNSSTUB_Context 127struct GNUNET_DNSSTUB_Context
@@ -117,13 +130,28 @@ struct GNUNET_DNSSTUB_Context
117 /** 130 /**
118 * Array of all open sockets for DNS requests. 131 * Array of all open sockets for DNS requests.
119 */ 132 */
120 struct GNUNET_DNSSTUB_RequestSocket sockets[DNS_SOCKET_MAX]; 133 struct GNUNET_DNSSTUB_RequestSocket *sockets;
134
135 /**
136 * DLL of DNS resolvers we use.
137 */
138 struct DnsServer *dns_head;
139
140 /**
141 * DLL of DNS resolvers we use.
142 */
143 struct DnsServer *dns_tail;
144
145 /**
146 * How frequently do we retry requests?
147 */
148 struct GNUNET_TIME_Relative retry_freq;
121 149
122 /** 150 /**
123 * IP address to use for the DNS server if we are a DNS exit service 151 * Length of @e sockets array.
124 * (for VPN via cadet); otherwise NULL.
125 */ 152 */
126 char *dns_exit; 153 unsigned int num_sockets;
154
127}; 155};
128 156
129 157
@@ -212,33 +240,21 @@ open_socket (int af)
212 240
213 241
214/** 242/**
215 * Read a DNS response from the (unhindered) UDP-Socket
216 *
217 * @param cls socket to read from
218 */
219static void
220read_response (void *cls);
221
222
223/**
224 * Get a socket of the specified address family to send out a 243 * Get a socket of the specified address family to send out a
225 * UDP DNS request to the Internet. 244 * UDP DNS request to the Internet.
226 * 245 *
227 * @param ctx the DNSSTUB context 246 * @param ctx the DNSSTUB context
228 * @param af desired address family 247 * @return NULL on error
229 * @return NULL on error (given AF not "supported")
230 */ 248 */
231static struct GNUNET_DNSSTUB_RequestSocket * 249static struct GNUNET_DNSSTUB_RequestSocket *
232get_request_socket (struct GNUNET_DNSSTUB_Context *ctx, 250get_request_socket (struct GNUNET_DNSSTUB_Context *ctx)
233 int af)
234{ 251{
235 struct GNUNET_DNSSTUB_RequestSocket *rs; 252 struct GNUNET_DNSSTUB_RequestSocket *rs;
236 struct GNUNET_NETWORK_FDSet *rset;
237 253
238 for (unsigned int i=0;i<256;i++) 254 for (unsigned int i=0;i<256;i++)
239 { 255 {
240 rs = &ctx->sockets[GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE, 256 rs = &ctx->sockets[GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE,
241 DNS_SOCKET_MAX)]; 257 ctx->num_sockets)];
242 if (NULL == rs->rc) 258 if (NULL == rs->rc)
243 break; 259 break;
244 } 260 }
@@ -246,25 +262,10 @@ get_request_socket (struct GNUNET_DNSSTUB_Context *ctx,
246 { 262 {
247 /* signal "failure" */ 263 /* signal "failure" */
248 rs->rc (rs->rc_cls, 264 rs->rc (rs->rc_cls,
249 rs,
250 NULL, 265 NULL,
251 0); 266 0);
252 rs->rc = NULL; 267 rs->rc = NULL;
253 } 268 }
254 rs->timeout = GNUNET_TIME_relative_to_absolute (REQUEST_TIMEOUT);
255 switch (af)
256 {
257 case AF_INET:
258 if (NULL == rs->dnsout4)
259 rs->dnsout4 = open_socket (AF_INET);
260 break;
261 case AF_INET6:
262 if (NULL == rs->dnsout6)
263 rs->dnsout6 = open_socket (AF_INET6);
264 break;
265 default:
266 return NULL;
267 }
268 if (NULL != rs->read_task) 269 if (NULL != rs->read_task)
269 { 270 {
270 GNUNET_SCHEDULER_cancel (rs->read_task); 271 GNUNET_SCHEDULER_cancel (rs->read_task);
@@ -280,194 +281,7 @@ get_request_socket (struct GNUNET_DNSSTUB_Context *ctx,
280 GNUNET_free (rs->request); 281 GNUNET_free (rs->request);
281 rs->request = NULL; 282 rs->request = NULL;
282 } 283 }
283 if ( (NULL == rs->dnsout4) && 284 rs->ctx = ctx;
284 (NULL == rs->dnsout6) )
285 return NULL;
286 rset = GNUNET_NETWORK_fdset_create ();
287 if (NULL != rs->dnsout4)
288 GNUNET_NETWORK_fdset_set (rset,
289 rs->dnsout4);
290 if (NULL != rs->dnsout6)
291 GNUNET_NETWORK_fdset_set (rset,
292 rs->dnsout6);
293 rs->read_task = GNUNET_SCHEDULER_add_select (GNUNET_SCHEDULER_PRIORITY_DEFAULT,
294 REQUEST_TIMEOUT,
295 rset,
296 NULL,
297 &read_response,
298 rs);
299 GNUNET_NETWORK_fdset_destroy (rset);
300 return rs;
301}
302
303
304/**
305 * Task to (re)transmit the DNS query, possibly repeatedly until
306 * we succeed.
307 *
308 * @param cls our `struct GNUNET_DNSSTUB_RequestSocket *`
309 */
310static void
311transmit_query (void *cls)
312{
313 struct GNUNET_DNSSTUB_RequestSocket *rs = cls;
314 struct GNUNET_NETWORK_Handle *ret;
315
316 rs->retry_task = NULL;
317 ret = (NULL != rs->dnsout4) ? rs->dnsout4 : rs->dnsout6;
318 GNUNET_assert (NULL != ret);
319 if (GNUNET_SYSERR ==
320 GNUNET_NETWORK_socket_sendto (ret,
321 rs->request,
322 rs->request_len,
323 (struct sockaddr *) &rs->addr,
324 rs->addrlen))
325 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
326 _("Failed to send DNS request to %s\n"),
327 GNUNET_a2s ((struct sockaddr *) &rs->addr,
328 rs->addrlen));
329 else
330 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
331 _("Sent DNS request to %s\n"),
332 GNUNET_a2s ((struct sockaddr *) &rs->addr,
333 rs->addrlen));
334 rs->retry_task = GNUNET_SCHEDULER_add_delayed (DNS_RETRANSMIT_DELAY,
335 &transmit_query,
336 rs);
337}
338
339
340/**
341 * Perform DNS resolution.
342 *
343 * @param ctx stub resolver to use
344 * @param sa the socket address
345 * @param sa_len the length of @a sa
346 * @param request DNS request to transmit
347 * @param request_len number of bytes in @a request
348 * @param rc function to call with result
349 * @param rc_cls closure for @a rc
350 * @return socket used for the request, NULL on error
351 */
352struct GNUNET_DNSSTUB_RequestSocket *
353GNUNET_DNSSTUB_resolve (struct GNUNET_DNSSTUB_Context *ctx,
354 const struct sockaddr *sa,
355 socklen_t sa_len,
356 const void *request,
357 size_t request_len,
358 GNUNET_DNSSTUB_ResultCallback rc,
359 void *rc_cls)
360{
361 struct GNUNET_DNSSTUB_RequestSocket *rs;
362
363 if (NULL == (rs = get_request_socket (ctx,
364 sa->sa_family)))
365 return NULL;
366 GNUNET_assert (NULL == rs->rc);
367 GNUNET_memcpy (&rs->addr,
368 sa,
369 sa_len);
370 rs->addrlen = sa_len;
371 rs->rc = rc;
372 rs->rc_cls = rc_cls;
373 rs->request = GNUNET_memdup (request,
374 request_len);
375 rs->request_len = request_len;
376 rs->retry_task = GNUNET_SCHEDULER_add_now (&transmit_query,
377 rs);
378 return rs;
379}
380
381
382/**
383 * Perform DNS resolution using our default IP from init.
384 *
385 * @param ctx stub resolver to use
386 * @param request DNS request to transmit
387 * @param request_len number of bytes in msg
388 * @param rc function to call with result
389 * @param rc_cls closure for 'rc'
390 * @return socket used for the request, NULL on error
391 */
392struct GNUNET_DNSSTUB_RequestSocket *
393GNUNET_DNSSTUB_resolve2 (struct GNUNET_DNSSTUB_Context *ctx,
394 const void *request,
395 size_t request_len,
396 GNUNET_DNSSTUB_ResultCallback rc,
397 void *rc_cls)
398{
399 int af;
400 struct sockaddr_in v4;
401 struct sockaddr_in6 v6;
402 struct sockaddr *sa;
403 socklen_t salen;
404 struct GNUNET_NETWORK_Handle *dnsout;
405 struct GNUNET_DNSSTUB_RequestSocket *rs;
406
407 memset (&v4, 0, sizeof (v4));
408 memset (&v6, 0, sizeof (v6));
409 if (1 == inet_pton (AF_INET,
410 ctx->dns_exit,
411 &v4.sin_addr))
412 {
413 salen = sizeof (v4);
414 v4.sin_family = AF_INET;
415 v4.sin_port = htons (53);
416#if HAVE_SOCKADDR_IN_SIN_LEN
417 v4.sin_len = (u_char) salen;
418#endif
419 sa = (struct sockaddr *) &v4;
420 af = AF_INET;
421 }
422 else if (1 == inet_pton (AF_INET6,
423 ctx->dns_exit,
424 &v6.sin6_addr))
425 {
426 salen = sizeof (v6);
427 v6.sin6_family = AF_INET6;
428 v6.sin6_port = htons (53);
429#if HAVE_SOCKADDR_IN_SIN_LEN
430 v6.sin6_len = (u_char) salen;
431#endif
432 sa = (struct sockaddr *) &v6;
433 af = AF_INET6;
434 }
435 else
436 {
437 GNUNET_break (0);
438 return NULL;
439 }
440 if (NULL == (rs = get_request_socket (ctx,
441 af)))
442 return NULL;
443 GNUNET_assert (NULL == rs->rc);
444 if (NULL != rs->dnsout4)
445 dnsout = rs->dnsout4;
446 else
447 dnsout = rs->dnsout6;
448 if (NULL == dnsout)
449 {
450 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
451 _("Configured DNS exit `%s' is not working / valid.\n"),
452 ctx->dns_exit);
453 return NULL;
454 }
455 GNUNET_memcpy (&rs->addr,
456 sa,
457 salen);
458 rs->addrlen = salen;
459 rs->rc = rc;
460 rs->rc_cls = rc_cls;
461 if (GNUNET_SYSERR ==
462 GNUNET_NETWORK_socket_sendto (dnsout,
463 request,
464 request_len,
465 sa,
466 salen))
467 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
468 _("Failed to send DNS request to %s\n"),
469 GNUNET_a2s (sa, salen));
470 rs->timeout = GNUNET_TIME_relative_to_absolute (REQUEST_TIMEOUT);
471 return rs; 285 return rs;
472} 286}
473 287
@@ -484,9 +298,7 @@ static int
484do_dns_read (struct GNUNET_DNSSTUB_RequestSocket *rs, 298do_dns_read (struct GNUNET_DNSSTUB_RequestSocket *rs,
485 struct GNUNET_NETWORK_Handle *dnsout) 299 struct GNUNET_NETWORK_Handle *dnsout)
486{ 300{
487 struct sockaddr_storage addr; 301 struct GNUNET_DNSSTUB_Context *ctx = rs->ctx;
488 socklen_t addrlen;
489 struct GNUNET_TUN_DnsHeader *dns;
490 ssize_t r; 302 ssize_t r;
491 int len; 303 int len;
492 304
@@ -507,9 +319,15 @@ do_dns_read (struct GNUNET_DNSSTUB_RequestSocket *rs,
507 len); 319 len);
508 { 320 {
509 unsigned char buf[len] GNUNET_ALIGN; 321 unsigned char buf[len] GNUNET_ALIGN;
322 int found;
323 struct sockaddr_storage addr;
324 socklen_t addrlen;
325 struct GNUNET_TUN_DnsHeader *dns;
510 326
511 addrlen = sizeof (addr); 327 addrlen = sizeof (addr);
512 memset (&addr, 0, sizeof (addr)); 328 memset (&addr,
329 0,
330 sizeof (addr));
513 r = GNUNET_NETWORK_socket_recvfrom (dnsout, 331 r = GNUNET_NETWORK_socket_recvfrom (dnsout,
514 buf, 332 buf,
515 sizeof (buf), 333 sizeof (buf),
@@ -522,6 +340,24 @@ do_dns_read (struct GNUNET_DNSSTUB_RequestSocket *rs,
522 GNUNET_NETWORK_socket_close (dnsout); 340 GNUNET_NETWORK_socket_close (dnsout);
523 return GNUNET_SYSERR; 341 return GNUNET_SYSERR;
524 } 342 }
343 found = GNUNET_NO;
344 for (struct DnsServer *ds = ctx->dns_head; NULL != ds; ds = ds->next)
345 {
346 if (0 == memcmp (&addr,
347 &ds->ss,
348 GNUNET_MIN (sizeof (struct sockaddr_storage),
349 addrlen)))
350 {
351 found = GNUNET_YES;
352 break;
353 }
354 }
355 if (GNUNET_NO == found)
356 {
357 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
358 "Received DNS response from server we never asked (ignored)");
359 return GNUNET_NO;
360 }
525 if (sizeof (struct GNUNET_TUN_DnsHeader) > r) 361 if (sizeof (struct GNUNET_TUN_DnsHeader) > r)
526 { 362 {
527 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 363 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
@@ -530,22 +366,15 @@ do_dns_read (struct GNUNET_DNSSTUB_RequestSocket *rs,
530 return GNUNET_NO; 366 return GNUNET_NO;
531 } 367 }
532 dns = (struct GNUNET_TUN_DnsHeader *) buf; 368 dns = (struct GNUNET_TUN_DnsHeader *) buf;
533 if ( (addrlen != rs->addrlen) || 369 if (NULL == rs->rc)
534 (GNUNET_YES !=
535 GNUNET_TUN_sockaddr_cmp ((struct sockaddr *) &rs->addr,
536 (struct sockaddr *) &addr,
537 GNUNET_YES)) ||
538 (0 == GNUNET_TIME_absolute_get_remaining (rs->timeout).rel_value_us) )
539 { 370 {
540 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 371 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
541 "Request timeout or invalid sender address; ignoring reply\n"); 372 "Request timeout or cancelled; ignoring reply\n");
542 return GNUNET_NO; 373 return GNUNET_NO;
543 } 374 }
544 if (NULL != rs->rc) 375 rs->rc (rs->rc_cls,
545 rs->rc (rs->rc_cls, 376 dns,
546 rs, 377 r);
547 dns,
548 r);
549 } 378 }
550 return GNUNET_OK; 379 return GNUNET_OK;
551} 380}
@@ -557,44 +386,21 @@ do_dns_read (struct GNUNET_DNSSTUB_RequestSocket *rs,
557 * @param cls socket to read from 386 * @param cls socket to read from
558 */ 387 */
559static void 388static void
560read_response (void *cls) 389read_response (void *cls);
390
391
392/**
393 * Schedule #read_response() task for @a rs.
394 *
395 * @param rs request to schedule read operation for
396 */
397static void
398schedule_read (struct GNUNET_DNSSTUB_RequestSocket *rs)
561{ 399{
562 struct GNUNET_DNSSTUB_RequestSocket *rs = cls;
563 struct GNUNET_NETWORK_FDSet *rset; 400 struct GNUNET_NETWORK_FDSet *rset;
564 const struct GNUNET_SCHEDULER_TaskContext *tc;
565
566 rs->read_task = NULL;
567 tc = GNUNET_SCHEDULER_get_task_context ();
568 if (0 == (tc->reason & GNUNET_SCHEDULER_REASON_READ_READY))
569 {
570 /* signal "failure" (from timeout) */
571 if (NULL != rs->rc)
572 {
573 rs->rc (rs->rc_cls,
574 rs,
575 NULL,
576 0);
577 rs->rc = NULL;
578 }
579 /* timeout */
580 cleanup_rs (rs);
581 return;
582 }
583 /* read and process ready sockets */
584 if ((NULL != rs->dnsout4) &&
585 (GNUNET_NETWORK_fdset_isset (tc->read_ready,
586 rs->dnsout4)) &&
587 (GNUNET_SYSERR == do_dns_read (rs,
588 rs->dnsout4)))
589 rs->dnsout4 = NULL;
590 if ((NULL != rs->dnsout6) &&
591 (GNUNET_NETWORK_fdset_isset (tc->read_ready,
592 rs->dnsout6)) &&
593 (GNUNET_SYSERR == do_dns_read (rs,
594 rs->dnsout6)))
595 rs->dnsout6 = NULL;
596 401
597 /* re-schedule read task */ 402 if (NULL != rs->read_task)
403 GNUNET_SCHEDULER_cancel (rs->read_task);
598 rset = GNUNET_NETWORK_fdset_create (); 404 rset = GNUNET_NETWORK_fdset_create ();
599 if (NULL != rs->dnsout4) 405 if (NULL != rs->dnsout4)
600 GNUNET_NETWORK_fdset_set (rset, 406 GNUNET_NETWORK_fdset_set (rset,
@@ -603,7 +409,7 @@ read_response (void *cls)
603 GNUNET_NETWORK_fdset_set (rset, 409 GNUNET_NETWORK_fdset_set (rset,
604 rs->dnsout6); 410 rs->dnsout6);
605 rs->read_task = GNUNET_SCHEDULER_add_select (GNUNET_SCHEDULER_PRIORITY_DEFAULT, 411 rs->read_task = GNUNET_SCHEDULER_add_select (GNUNET_SCHEDULER_PRIORITY_DEFAULT,
606 GNUNET_TIME_absolute_get_remaining (rs->timeout), 412 GNUNET_TIME_UNIT_FOREVER_REL,
607 rset, 413 rset,
608 NULL, 414 NULL,
609 &read_response, 415 &read_response,
@@ -613,6 +419,151 @@ read_response (void *cls)
613 419
614 420
615/** 421/**
422 * Read a DNS response from the (unhindered) UDP-Socket
423 *
424 * @param cls `struct GNUNET_DNSSTUB_RequestSocket` to read from
425 */
426static void
427read_response (void *cls)
428{
429 struct GNUNET_DNSSTUB_RequestSocket *rs = cls;
430 const struct GNUNET_SCHEDULER_TaskContext *tc;
431
432 rs->read_task = NULL;
433 tc = GNUNET_SCHEDULER_get_task_context ();
434 /* read and process ready sockets */
435 if ( (NULL != rs->dnsout4) &&
436 (GNUNET_NETWORK_fdset_isset (tc->read_ready,
437 rs->dnsout4)) &&
438 (GNUNET_SYSERR ==
439 do_dns_read (rs,
440 rs->dnsout4)) )
441 rs->dnsout4 = NULL;
442 if ( (NULL != rs->dnsout6) &&
443 (GNUNET_NETWORK_fdset_isset (tc->read_ready,
444 rs->dnsout6)) &&
445 (GNUNET_SYSERR ==
446 do_dns_read (rs,
447 rs->dnsout6)) )
448 rs->dnsout6 = NULL;
449 /* re-schedule read task */
450 schedule_read (rs);
451}
452
453
454/**
455 * Task to (re)transmit the DNS query, possibly repeatedly until
456 * we succeed.
457 *
458 * @param cls our `struct GNUNET_DNSSTUB_RequestSocket *`
459 */
460static void
461transmit_query (void *cls)
462{
463 struct GNUNET_DNSSTUB_RequestSocket *rs = cls;
464 struct GNUNET_DNSSTUB_Context *ctx = rs->ctx;
465 const struct sockaddr *sa;
466 socklen_t salen;
467 struct DnsServer *ds;
468 struct GNUNET_NETWORK_Handle *dnsout;
469
470 rs->retry_task = GNUNET_SCHEDULER_add_delayed (ctx->retry_freq,
471 &transmit_query,
472 rs);
473 ds = rs->ds_pos;
474 rs->ds_pos = ds->next;
475 if (NULL == rs->ds_pos)
476 rs->ds_pos = ctx->dns_head;
477 GNUNET_assert (NULL != ds);
478 dnsout = NULL;
479 switch (ds->ss.ss_family)
480 {
481 case AF_INET:
482 if (NULL == rs->dnsout4)
483 rs->dnsout4 = open_socket (AF_INET);
484 dnsout = rs->dnsout4;
485 sa = (const struct sockaddr *) &ds->ss;
486 salen = sizeof (struct sockaddr_in);
487 break;
488 case AF_INET6:
489 if (NULL == rs->dnsout6)
490 rs->dnsout6 = open_socket (AF_INET6);
491 dnsout = rs->dnsout6;
492 sa = (const struct sockaddr *) &ds->ss;
493 salen = sizeof (struct sockaddr_in6);
494 break;
495 default:
496 return;
497 }
498 if (NULL == dnsout)
499 {
500 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
501 "Unable to use configure DNS server, skipping\n");
502 return;
503 }
504 if (GNUNET_SYSERR ==
505 GNUNET_NETWORK_socket_sendto (dnsout,
506 rs->request,
507 rs->request_len,
508 sa,
509 salen))
510 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
511 _("Failed to send DNS request to %s\n"),
512 GNUNET_a2s (sa,
513 salen));
514 else
515 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
516 _("Sent DNS request to %s\n"),
517 GNUNET_a2s (sa,
518 salen));
519 schedule_read (rs);
520}
521
522
523/**
524 * Perform DNS resolution using our default IP from init.
525 *
526 * @param ctx stub resolver to use
527 * @param request DNS request to transmit
528 * @param request_len number of bytes in msg
529 * @param rc function to call with result
530 * @param rc_cls closure for 'rc'
531 * @return socket used for the request, NULL on error
532 */
533struct GNUNET_DNSSTUB_RequestSocket *
534GNUNET_DNSSTUB_resolve (struct GNUNET_DNSSTUB_Context *ctx,
535 const void *request,
536 size_t request_len,
537 GNUNET_DNSSTUB_ResultCallback rc,
538 void *rc_cls)
539{
540 struct GNUNET_DNSSTUB_RequestSocket *rs;
541
542 if (NULL == ctx->dns_head)
543 {
544 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
545 "No DNS server configured for resolution\n");
546 return NULL;
547 }
548 if (NULL == (rs = get_request_socket (ctx)))
549 {
550 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
551 "No request socket available for DNS resolution\n");
552 return NULL;
553 }
554 rs->ds_pos = ctx->dns_head;
555 rs->rc = rc;
556 rs->rc_cls = rc_cls;
557 rs->request = GNUNET_memdup (request,
558 request_len);
559 rs->request_len = request_len;
560 rs->retry_task = GNUNET_SCHEDULER_add_now (&transmit_query,
561 rs);
562 return rs;
563}
564
565
566/**
616 * Cancel DNS resolution. 567 * Cancel DNS resolution.
617 * 568 *
618 * @param rs resolution to cancel 569 * @param rs resolution to cancel
@@ -626,28 +577,153 @@ GNUNET_DNSSTUB_resolve_cancel (struct GNUNET_DNSSTUB_RequestSocket *rs)
626 GNUNET_SCHEDULER_cancel (rs->retry_task); 577 GNUNET_SCHEDULER_cancel (rs->retry_task);
627 rs->retry_task = NULL; 578 rs->retry_task = NULL;
628 } 579 }
580 if (NULL != rs->read_task)
581 {
582 GNUNET_SCHEDULER_cancel (rs->read_task);
583 rs->read_task = NULL;
584 }
629} 585}
630 586
631 587
632/** 588/**
633 * Start a DNS stub resolver. 589 * Start a DNS stub resolver.
634 * 590 *
635 * @param dns_ip target IP address to use 591 * @param num_sockets how many sockets should we open
592 * in parallel for DNS queries for this stub?
636 * @return NULL on error 593 * @return NULL on error
637 */ 594 */
638struct GNUNET_DNSSTUB_Context * 595struct GNUNET_DNSSTUB_Context *
639GNUNET_DNSSTUB_start (const char *dns_ip) 596GNUNET_DNSSTUB_start (unsigned int num_sockets)
640{ 597{
641 struct GNUNET_DNSSTUB_Context *ctx; 598 struct GNUNET_DNSSTUB_Context *ctx;
642 599
600 if (0 == num_sockets)
601 {
602 GNUNET_break (0);
603 return NULL;
604 }
643 ctx = GNUNET_new (struct GNUNET_DNSSTUB_Context); 605 ctx = GNUNET_new (struct GNUNET_DNSSTUB_Context);
644 if (NULL != dns_ip) 606 ctx->num_sockets = num_sockets;
645 ctx->dns_exit = GNUNET_strdup (dns_ip); 607 ctx->sockets = GNUNET_new_array (num_sockets,
608 struct GNUNET_DNSSTUB_RequestSocket);
609 ctx->retry_freq = DNS_RETRANSMIT_DELAY;
646 return ctx; 610 return ctx;
647} 611}
648 612
649 613
650/** 614/**
615 * Add nameserver for use by the DNSSTUB. We will use
616 * all provided nameservers for resolution (round-robin).
617 *
618 * @param ctx resolver context to modify
619 * @param dns_ip target IP address to use (as string)
620 * @return #GNUNET_OK on success
621 */
622int
623GNUNET_DNSSTUB_add_dns_ip (struct GNUNET_DNSSTUB_Context *ctx,
624 const char *dns_ip)
625{
626 struct DnsServer *ds;
627 struct in_addr i4;
628 struct in6_addr i6;
629
630 ds = GNUNET_new (struct DnsServer);
631 if (1 == inet_pton (AF_INET,
632 dns_ip,
633 &i4))
634 {
635 struct sockaddr_in *s4 = (struct sockaddr_in *) &ds->ss;
636
637 s4->sin_family = AF_INET;
638 s4->sin_port = htons (53);
639 s4->sin_addr = i4;
640#if HAVE_SOCKADDR_IN_SIN_LEN
641 s4->sin_len = (u_char) sizeof (struct sockaddr_in);
642#endif
643 }
644 else if (1 == inet_pton (AF_INET6,
645 dns_ip,
646 &i6))
647 {
648 struct sockaddr_in6 *s6 = (struct sockaddr_in6 *) &ds->ss;
649
650 s6->sin6_family = AF_INET6;
651 s6->sin6_port = htons (53);
652 s6->sin6_addr = i6;
653#if HAVE_SOCKADDR_IN_SIN_LEN
654 s6->sin6_len = (u_char) sizeof (struct sockaddr_in6);
655#endif
656 }
657 else
658 {
659 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
660 "Malformed IP address `%s' for DNS server\n",
661 dns_ip);
662 GNUNET_free (ds);
663 return GNUNET_SYSERR;
664 }
665 GNUNET_CONTAINER_DLL_insert (ctx->dns_head,
666 ctx->dns_tail,
667 ds);
668 return GNUNET_OK;
669}
670
671
672/**
673 * Add nameserver for use by the DNSSTUB. We will use
674 * all provided nameservers for resolution (round-robin).
675 *
676 * @param ctx resolver context to modify
677 * @param sa socket address of DNS resolver to use
678 * @return #GNUNET_OK on success
679 */
680int
681GNUNET_DNSSTUB_add_dns_sa (struct GNUNET_DNSSTUB_Context *ctx,
682 const struct sockaddr *sa)
683{
684 struct DnsServer *ds;
685
686 ds = GNUNET_new (struct DnsServer);
687 switch (sa->sa_family)
688 {
689 case AF_INET:
690 memcpy (&ds->ss,
691 sa,
692 sizeof (struct sockaddr_in));
693 break;
694 case AF_INET6:
695 memcpy (&ds->ss,
696 sa,
697 sizeof (struct sockaddr_in6));
698 break;
699 default:
700 GNUNET_break (0);
701 GNUNET_free (ds);
702 return GNUNET_SYSERR;
703 }
704 GNUNET_CONTAINER_DLL_insert (ctx->dns_head,
705 ctx->dns_tail,
706 ds);
707 return GNUNET_OK;
708}
709
710
711/**
712 * How long should we try requests before timing out?
713 * Only effective for requests issued after this call.
714 *
715 * @param ctx resolver context to modify
716 * @param retry_freq how long to wait between retries
717 */
718void
719GNUNET_DNSSTUB_set_retry (struct GNUNET_DNSSTUB_Context *ctx,
720 struct GNUNET_TIME_Relative retry_freq)
721{
722 ctx->retry_freq = retry_freq;
723}
724
725
726/**
651 * Cleanup DNSSTUB resolver. 727 * Cleanup DNSSTUB resolver.
652 * 728 *
653 * @param ctx stub resolver to clean up 729 * @param ctx stub resolver to clean up
@@ -655,15 +731,18 @@ GNUNET_DNSSTUB_start (const char *dns_ip)
655void 731void
656GNUNET_DNSSTUB_stop (struct GNUNET_DNSSTUB_Context *ctx) 732GNUNET_DNSSTUB_stop (struct GNUNET_DNSSTUB_Context *ctx)
657{ 733{
658 unsigned int i; 734 struct DnsServer *ds;
659 735
660 for (i=0;i<DNS_SOCKET_MAX;i++) 736 while (NULL != (ds = ctx->dns_head))
661 cleanup_rs (&ctx->sockets[i]);
662 if (NULL != ctx->dns_exit)
663 { 737 {
664 GNUNET_free (ctx->dns_exit); 738 GNUNET_CONTAINER_DLL_remove (ctx->dns_head,
665 ctx->dns_exit = NULL; 739 ctx->dns_tail,
740 ds);
741 GNUNET_free (ds);
666 } 742 }
743 for (unsigned int i=0;i<ctx->num_sockets;i++)
744 cleanup_rs (&ctx->sockets[i]);
745 GNUNET_free (ctx->sockets);
667 GNUNET_free (ctx); 746 GNUNET_free (ctx);
668} 747}
669 748
diff --git a/src/dns/gnunet-service-dns.c b/src/dns/gnunet-service-dns.c
index 9feaa8413..39ce7f6e5 100644
--- a/src/dns/gnunet-service-dns.c
+++ b/src/dns/gnunet-service-dns.c
@@ -508,13 +508,11 @@ send_request_to_client (struct RequestRecord *rr,
508 * succeeded. 508 * succeeded.
509 * 509 *
510 * @param cls NULL 510 * @param cls NULL
511 * @param rs the socket that received the response
512 * @param dns the response itself 511 * @param dns the response itself
513 * @param r number of bytes in dns 512 * @param r number of bytes in dns
514 */ 513 */
515static void 514static void
516process_dns_result (void *cls, 515process_dns_result (void *cls,
517 struct GNUNET_DNSSTUB_RequestSocket *rs,
518 const struct GNUNET_TUN_DnsHeader *dns, 516 const struct GNUNET_TUN_DnsHeader *dns,
519 size_t r); 517 size_t r);
520 518
@@ -530,7 +528,6 @@ next_phase (struct RequestRecord *rr)
530{ 528{
531 struct ClientRecord *cr; 529 struct ClientRecord *cr;
532 int nz; 530 int nz;
533 socklen_t salen;
534 531
535 if (rr->phase == RP_DROP) 532 if (rr->phase == RP_DROP)
536 { 533 {
@@ -582,22 +579,27 @@ next_phase (struct RequestRecord *rr)
582 next_phase (rr); 579 next_phase (rr);
583 return; 580 return;
584 case RP_QUERY: 581 case RP_QUERY:
582#if 0
583 /* TODO: optionally, use this to forward DNS requests to the
584 *original* DNS server instead of the one we have configured...
585 (but then we need to create a fresh dnsstub for each request
586 *and* manage the timeout) */
585 switch (rr->dst_addr.ss_family) 587 switch (rr->dst_addr.ss_family)
586 { 588 {
587 case AF_INET: 589 case AF_INET:
588 salen = sizeof (struct sockaddr_in); 590 salen = sizeof (struct sockaddr_in);
591 sa = (const struct sockaddr *) &rr->dst_addr;
589 break; 592 break;
590 case AF_INET6: 593 case AF_INET6:
591 salen = sizeof (struct sockaddr_in6); 594 salen = sizeof (struct sockaddr_in6);
595 sa = (const struct sockaddr *) &rr->dst_addr;
592 break; 596 break;
593 default: 597 default:
594 GNUNET_assert (0); 598 GNUNET_assert (0);
595 } 599 }
596 600#endif
597 rr->phase = RP_INTERNET_DNS; 601 rr->phase = RP_INTERNET_DNS;
598 rr->rs = GNUNET_DNSSTUB_resolve (dnsstub, 602 rr->rs = GNUNET_DNSSTUB_resolve (dnsstub,
599 (struct sockaddr*) &rr->dst_addr,
600 salen,
601 rr->payload, 603 rr->payload,
602 rr->payload_length, 604 rr->payload_length,
603 &process_dns_result, 605 &process_dns_result,
@@ -714,13 +716,11 @@ client_disconnect_cb (void *cls,
714 * succeeded. 716 * succeeded.
715 * 717 *
716 * @param cls NULL 718 * @param cls NULL
717 * @param rs the socket that received the response
718 * @param dns the response itself 719 * @param dns the response itself
719 * @param r number of bytes in dns 720 * @param r number of bytes in dns
720 */ 721 */
721static void 722static void
722process_dns_result (void *cls, 723process_dns_result (void *cls,
723 struct GNUNET_DNSSTUB_RequestSocket *rs,
724 const struct GNUNET_TUN_DnsHeader *dns, 724 const struct GNUNET_TUN_DnsHeader *dns,
725 size_t r) 725 size_t r)
726{ 726{
@@ -733,8 +733,7 @@ process_dns_result (void *cls,
733 return; /* ignore */ 733 return; /* ignore */
734 734
735 rr = &requests[dns->id]; 735 rr = &requests[dns->id];
736 if ( (rr->phase != RP_INTERNET_DNS) || 736 if (rr->phase != RP_INTERNET_DNS)
737 (rr->rs != rs) )
738 { 737 {
739 /* unexpected / bogus reply */ 738 /* unexpected / bogus reply */
740 GNUNET_STATISTICS_update (stats, 739 GNUNET_STATISTICS_update (stats,
@@ -1055,8 +1054,6 @@ run (void *cls,
1055 char *ipv4mask; 1054 char *ipv4mask;
1056 char *ipv6addr; 1055 char *ipv6addr;
1057 char *ipv6prefix; 1056 char *ipv6prefix;
1058 struct in_addr dns_exit4;
1059 struct in6_addr dns_exit6;
1060 char *dns_exit; 1057 char *dns_exit;
1061 char *binary; 1058 char *binary;
1062 int nortsetup; 1059 int nortsetup;
@@ -1065,24 +1062,26 @@ run (void *cls,
1065 stats = GNUNET_STATISTICS_create ("dns", cfg); 1062 stats = GNUNET_STATISTICS_create ("dns", cfg);
1066 GNUNET_SCHEDULER_add_shutdown (&cleanup_task, 1063 GNUNET_SCHEDULER_add_shutdown (&cleanup_task,
1067 cls); 1064 cls);
1065 dnsstub = GNUNET_DNSSTUB_start (128);
1066 /* TODO: support multiple DNS_EXIT servers being configured */
1067 /* TODO: see above TODO on using DNS server from original packet.
1068 Not sure which is best... */
1068 dns_exit = NULL; 1069 dns_exit = NULL;
1069 if ( ( (GNUNET_OK != 1070 if ( (GNUNET_OK !=
1070 GNUNET_CONFIGURATION_get_value_string (cfg, 1071 GNUNET_CONFIGURATION_get_value_string (cfg,
1071 "dns", 1072 "dns",
1072 "DNS_EXIT", 1073 "DNS_EXIT",
1073 &dns_exit)) || 1074 &dns_exit)) ||
1074 ( (1 != inet_pton (AF_INET, dns_exit, &dns_exit4)) && 1075 (GNUNET_OK !=
1075 (1 != inet_pton (AF_INET6, dns_exit, &dns_exit6)) ) ) ) 1076 GNUNET_DNSSTUB_add_dns_ip (dnsstub,
1077 dns_exit)) )
1076 { 1078 {
1077 GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_ERROR, 1079 GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_ERROR,
1078 "dns", 1080 "dns",
1079 "DNS_EXIT", 1081 "DNS_EXIT",
1080 _("need a valid IPv4 or IPv6 address\n")); 1082 _("need a valid IPv4 or IPv6 address\n"));
1081 GNUNET_free_non_null (dns_exit); 1083 GNUNET_free_non_null (dns_exit);
1082 dns_exit = NULL;
1083 } 1084 }
1084 dnsstub = GNUNET_DNSSTUB_start (dns_exit);
1085 GNUNET_free_non_null (dns_exit);
1086 binary = GNUNET_OS_get_libexec_binary_path ("gnunet-helper-dns"); 1085 binary = GNUNET_OS_get_libexec_binary_path ("gnunet-helper-dns");
1087 if (GNUNET_YES != 1086 if (GNUNET_YES !=
1088 GNUNET_OS_check_helper_binary (binary, 1087 GNUNET_OS_check_helper_binary (binary,
diff --git a/src/dns/gnunet-zoneimport.c b/src/dns/gnunet-zoneimport.c
index 914868af4..860672e7a 100644
--- a/src/dns/gnunet-zoneimport.c
+++ b/src/dns/gnunet-zoneimport.c
@@ -290,13 +290,11 @@ process_record (struct Request *req,
290 * Function called with the result of a DNS resolution. 290 * Function called with the result of a DNS resolution.
291 * 291 *
292 * @param cls closure with the `struct Request` 292 * @param cls closure with the `struct Request`
293 * @param rs socket that received the response
294 * @param dns dns response, never NULL 293 * @param dns dns response, never NULL
295 * @param dns_len number of bytes in @a dns 294 * @param dns_len number of bytes in @a dns
296 */ 295 */
297static void 296static void
298process_result (void *cls, 297process_result (void *cls,
299 struct GNUNET_DNSSTUB_RequestSocket *rs,
300 const struct GNUNET_TUN_DnsHeader *dns, 298 const struct GNUNET_TUN_DnsHeader *dns,
301 size_t dns_len) 299 size_t dns_len)
302{ 300{
@@ -407,11 +405,11 @@ submit_req (struct Request *req)
407 (pending >= THRESH) ) 405 (pending >= THRESH) )
408 return GNUNET_SYSERR; 406 return GNUNET_SYSERR;
409 GNUNET_assert (NULL == req->rs); 407 GNUNET_assert (NULL == req->rs);
410 req->rs = GNUNET_DNSSTUB_resolve2 (ctx, 408 req->rs = GNUNET_DNSSTUB_resolve (ctx,
411 req->raw, 409 req->raw,
412 req->raw_len, 410 req->raw_len,
413 &process_result, 411 &process_result,
414 req); 412 req);
415 GNUNET_assert (NULL != req->rs); 413 GNUNET_assert (NULL != req->rs);
416 req->issue_num++; 414 req->issue_num++;
417 last_request = now; 415 last_request = now;
@@ -561,13 +559,23 @@ main (int argc,
561 "Missing required configuration argument\n"); 559 "Missing required configuration argument\n");
562 return -1; 560 return -1;
563 } 561 }
564 ctx = GNUNET_DNSSTUB_start (argv[1]); 562 ctx = GNUNET_DNSSTUB_start (256);
565 if (NULL == ctx) 563 if (NULL == ctx)
566 { 564 {
567 fprintf (stderr, 565 fprintf (stderr,
568 "Failed to initialize GNUnet DNS STUB\n"); 566 "Failed to initialize GNUnet DNS STUB\n");
569 return 1; 567 return 1;
570 } 568 }
569 if (GNUNET_OK !=
570 GNUNET_DNSSTUB_add_dns_ip (ctx,
571 argv[1]))
572 {
573 fprintf (stderr,
574 "Failed to use `%s' for DNS resolver\n",
575 argv[1]);
576 return 1;
577 }
578
571 while (NULL != 579 while (NULL !=
572 fgets (hn, 580 fgets (hn,
573 sizeof (hn), 581 sizeof (hn),
diff --git a/src/exit/gnunet-daemon-exit.c b/src/exit/gnunet-daemon-exit.c
index 0b3cc505a..ae40feea0 100644
--- a/src/exit/gnunet-daemon-exit.c
+++ b/src/exit/gnunet-daemon-exit.c
@@ -458,13 +458,11 @@ GNUNET_NETWORK_STRUCT_END
458 * succeeded. 458 * succeeded.
459 * 459 *
460 * @param cls NULL 460 * @param cls NULL
461 * @param rs the socket that received the response
462 * @param dns the response itself 461 * @param dns the response itself
463 * @param r number of bytes in @a dns 462 * @param r number of bytes in @a dns
464 */ 463 */
465static void 464static void
466process_dns_result (void *cls, 465process_dns_result (void *cls,
467 struct GNUNET_DNSSTUB_RequestSocket *rs,
468 const struct GNUNET_TUN_DnsHeader *dns, 466 const struct GNUNET_TUN_DnsHeader *dns,
469 size_t r) 467 size_t r)
470{ 468{
@@ -479,8 +477,7 @@ process_dns_result (void *cls,
479 return; 477 return;
480 /* Handle case that this is a reply to a request from a CADET DNS channel */ 478 /* Handle case that this is a reply to a request from a CADET DNS channel */
481 ts = channels[dns->id]; 479 ts = channels[dns->id];
482 if ( (NULL == ts) || 480 if (NULL == ts)
483 (ts->specifics.dns.rs != rs) )
484 return; 481 return;
485 LOG (GNUNET_ERROR_TYPE_DEBUG, 482 LOG (GNUNET_ERROR_TYPE_DEBUG,
486 "Got a response from the stub resolver for DNS request received via CADET!\n"); 483 "Got a response from the stub resolver for DNS request received via CADET!\n");
@@ -557,11 +554,11 @@ handle_dns_request (void *cls,
557 dlen); 554 dlen);
558 dout = (struct GNUNET_TUN_DnsHeader *) buf; 555 dout = (struct GNUNET_TUN_DnsHeader *) buf;
559 dout->id = ts->specifics.dns.my_id; 556 dout->id = ts->specifics.dns.my_id;
560 ts->specifics.dns.rs = GNUNET_DNSSTUB_resolve2 (dnsstub, 557 ts->specifics.dns.rs = GNUNET_DNSSTUB_resolve (dnsstub,
561 buf, 558 buf,
562 dlen, 559 dlen,
563 &process_dns_result, 560 &process_dns_result,
564 NULL); 561 NULL);
565 if (NULL == ts->specifics.dns.rs) 562 if (NULL == ts->specifics.dns.rs)
566 { 563 {
567 GNUNET_break_op (0); 564 GNUNET_break_op (0);
@@ -3429,16 +3426,11 @@ do_dht_put (void *cls);
3429 * Schedules the next PUT. 3426 * Schedules the next PUT.
3430 * 3427 *
3431 * @param cls closure, NULL 3428 * @param cls closure, NULL
3432 * @param success #GNUNET_OK if the operation worked (unused)
3433 */ 3429 */
3434static void 3430static void
3435dht_put_cont (void *cls, 3431dht_put_cont (void *cls)
3436 int success)
3437{ 3432{
3438 dht_put = NULL; 3433 dht_put = NULL;
3439 dht_task = GNUNET_SCHEDULER_add_delayed (DHT_PUT_FREQUENCY,
3440 &do_dht_put,
3441 NULL);
3442} 3434}
3443 3435
3444 3436
@@ -3453,7 +3445,9 @@ do_dht_put (void *cls)
3453{ 3445{
3454 struct GNUNET_TIME_Absolute expiration; 3446 struct GNUNET_TIME_Absolute expiration;
3455 3447
3456 dht_task = NULL; 3448 dht_task = GNUNET_SCHEDULER_add_delayed (DHT_PUT_FREQUENCY,
3449 &do_dht_put,
3450 NULL);
3457 expiration = GNUNET_TIME_absolute_ntoh (dns_advertisement.expiration_time); 3451 expiration = GNUNET_TIME_absolute_ntoh (dns_advertisement.expiration_time);
3458 if (GNUNET_TIME_absolute_get_remaining (expiration).rel_value_us < 3452 if (GNUNET_TIME_absolute_get_remaining (expiration).rel_value_us <
3459 GNUNET_TIME_UNIT_HOURS.rel_value_us) 3453 GNUNET_TIME_UNIT_HOURS.rel_value_us)
@@ -3466,6 +3460,8 @@ do_dht_put (void *cls)
3466 &dns_advertisement.purpose, 3460 &dns_advertisement.purpose,
3467 &dns_advertisement.signature)); 3461 &dns_advertisement.signature));
3468 } 3462 }
3463 if (NULL != dht_put)
3464 GNUNET_DHT_put_cancel (dht_put);
3469 dht_put = GNUNET_DHT_put (dht, 3465 dht_put = GNUNET_DHT_put (dht,
3470 &dht_put_key, 3466 &dht_put_key,
3471 1 /* replication */, 3467 1 /* replication */,
@@ -3545,25 +3541,23 @@ advertise_dns_exit ()
3545 }; 3541 };
3546 char *dns_exit; 3542 char *dns_exit;
3547 struct GNUNET_HashCode port; 3543 struct GNUNET_HashCode port;
3548 struct in_addr dns_exit4;
3549 struct in6_addr dns_exit6;
3550 3544
3551 if (GNUNET_YES != 3545 if (GNUNET_YES !=
3552 GNUNET_CONFIGURATION_get_value_yesno (cfg, 3546 GNUNET_CONFIGURATION_get_value_yesno (cfg,
3553 "exit", 3547 "exit",
3554 "EXIT_DNS")) 3548 "EXIT_DNS"))
3555 return; 3549 return;
3550 GNUNET_assert (NULL != (dnsstub = GNUNET_DNSSTUB_start (128)));
3551 dns_exit = NULL;
3552 /* TODO: support using multiple DNS resolvers */
3556 if ( (GNUNET_OK != 3553 if ( (GNUNET_OK !=
3557 GNUNET_CONFIGURATION_get_value_string (cfg, 3554 GNUNET_CONFIGURATION_get_value_string (cfg,
3558 "exit", 3555 "exit",
3559 "DNS_RESOLVER", 3556 "DNS_RESOLVER",
3560 &dns_exit)) || 3557 &dns_exit)) ||
3561 ( (1 != inet_pton (AF_INET, 3558 (GNUNET_OK !=
3562 dns_exit, 3559 GNUNET_DNSSTUB_add_dns_ip (dnsstub,
3563 &dns_exit4)) && 3560 dns_exit)) )
3564 (1 != inet_pton (AF_INET6,
3565 dns_exit,
3566 &dns_exit6)) ) )
3567 { 3561 {
3568 GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_ERROR, 3562 GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_ERROR,
3569 "dns", 3563 "dns",
diff --git a/src/fs/gnunet-service-fs_put.c b/src/fs/gnunet-service-fs_put.c
index e8c7f586d..bb8560fff 100644
--- a/src/fs/gnunet-service-fs_put.c
+++ b/src/fs/gnunet-service-fs_put.c
@@ -135,14 +135,9 @@ schedule_next_put (struct PutOperator *po)
135 * Continuation called after DHT PUT operation has finished. 135 * Continuation called after DHT PUT operation has finished.
136 * 136 *
137 * @param cls type of blocks to gather 137 * @param cls type of blocks to gather
138 * @param success GNUNET_OK if the PUT was transmitted,
139 * GNUNET_NO on timeout,
140 * GNUNET_SYSERR on disconnect from service
141 * after the PUT message was transmitted
142 * (so we don't know if it was received or not)
143 */ 138 */
144static void 139static void
145delay_dht_put_blocks (void *cls, int success) 140delay_dht_put_blocks (void *cls)
146{ 141{
147 struct PutOperator *po = cls; 142 struct PutOperator *po = cls;
148 143
diff --git a/src/gns/gns.conf.in b/src/gns/gns.conf.in
index d48a213e9..2e49a4c60 100644
--- a/src/gns/gns.conf.in
+++ b/src/gns/gns.conf.in
@@ -22,9 +22,6 @@ MAX_PARALLEL_BACKGROUND_QUERIES = 1000
22# called via NSS or other mechanisms). 22# called via NSS or other mechanisms).
23INTERCEPT_DNS = YES 23INTERCEPT_DNS = YES
24 24
25# Using caching (start with namestore), or always ask DHT?
26USE_CACHE = YES
27
28# PREFIX = valgrind --leak-check=full --track-origins=yes 25# PREFIX = valgrind --leak-check=full --track-origins=yes
29 26
30# Zones 27# Zones
diff --git a/src/gns/gns_tld_api.c b/src/gns/gns_tld_api.c
index 293e37140..8a4d03b11 100644
--- a/src/gns/gns_tld_api.c
+++ b/src/gns/gns_tld_api.c
@@ -94,7 +94,7 @@ struct GNUNET_GNS_LookupWithTldRequest
94 * @return the part of @a name after the last ".", 94 * @return the part of @a name after the last ".",
95 * or @a name if @a name does not contain a "." 95 * or @a name if @a name does not contain a "."
96 */ 96 */
97static const char * 97static char *
98get_tld (const char *name) 98get_tld (const char *name)
99{ 99{
100 const char *tld; 100 const char *tld;
@@ -105,14 +105,14 @@ get_tld (const char *name)
105 tld = name; 105 tld = name;
106 else 106 else
107 tld++; /* skip the '.' */ 107 tld++; /* skip the '.' */
108 return tld; 108 return GNUNET_strdup (tld);
109} 109}
110 110
111 111
112/** 112/**
113 * Eat the TLD of the given @a name. 113 * Eat the TLD of the given @a name.
114 * 114 *
115 * @param name a name 115 * @param[in,out] name a name
116 */ 116 */
117static void 117static void
118eat_tld (char *name) 118eat_tld (char *name)
@@ -124,7 +124,7 @@ eat_tld (char *name)
124 (unsigned char) '.'); 124 (unsigned char) '.');
125 if (NULL == tld) 125 if (NULL == tld)
126 strcpy (name, 126 strcpy (name,
127 GNUNET_GNS_MASTERZONE_STR); 127 GNUNET_GNS_EMPTY_LABEL_AT);
128 else 128 else
129 *tld = '\0'; 129 *tld = '\0';
130} 130}
@@ -229,7 +229,7 @@ GNUNET_GNS_lookup_with_tld (struct GNUNET_GNS_Handle *handle,
229 void *proc_cls) 229 void *proc_cls)
230{ 230{
231 struct GNUNET_GNS_LookupWithTldRequest *ltr; 231 struct GNUNET_GNS_LookupWithTldRequest *ltr;
232 const char *tld; 232 char *tld;
233 char *dot_tld; 233 char *dot_tld;
234 char *zonestr; 234 char *zonestr;
235 struct GNUNET_CRYPTO_EcdsaPublicKey pkey; 235 struct GNUNET_CRYPTO_EcdsaPublicKey pkey;
@@ -251,6 +251,7 @@ GNUNET_GNS_lookup_with_tld (struct GNUNET_GNS_Handle *handle,
251 eat_tld (ltr->name); 251 eat_tld (ltr->name);
252 lookup_with_public_key (ltr, 252 lookup_with_public_key (ltr,
253 &pkey); 253 &pkey);
254 GNUNET_free (tld);
254 return ltr; 255 return ltr;
255 } 256 }
256 257
@@ -277,6 +278,7 @@ GNUNET_GNS_lookup_with_tld (struct GNUNET_GNS_Handle *handle,
277 GNUNET_free (dot_tld); 278 GNUNET_free (dot_tld);
278 GNUNET_free (ltr->name); 279 GNUNET_free (ltr->name);
279 GNUNET_free (ltr); 280 GNUNET_free (ltr);
281 GNUNET_free (tld);
280 return NULL; 282 return NULL;
281 } 283 }
282 GNUNET_free (dot_tld); 284 GNUNET_free (dot_tld);
@@ -284,6 +286,7 @@ GNUNET_GNS_lookup_with_tld (struct GNUNET_GNS_Handle *handle,
284 eat_tld (ltr->name); 286 eat_tld (ltr->name);
285 lookup_with_public_key (ltr, 287 lookup_with_public_key (ltr,
286 &pkey); 288 &pkey);
289 GNUNET_free (tld);
287 return ltr; 290 return ltr;
288 } 291 }
289 GNUNET_free (dot_tld); 292 GNUNET_free (dot_tld);
@@ -301,6 +304,7 @@ GNUNET_GNS_lookup_with_tld (struct GNUNET_GNS_Handle *handle,
301 tld, 304 tld,
302 &identity_zone_cb, 305 &identity_zone_cb,
303 ltr); 306 ltr);
307 GNUNET_free (tld);
304 if (NULL == ltr->id_op) 308 if (NULL == ltr->id_op)
305 { 309 {
306 GNUNET_free (ltr->name); 310 GNUNET_free (ltr->name);
diff --git a/src/gns/gnunet-dns2gns.c b/src/gns/gnunet-dns2gns.c
index 47cc6dde0..bc66f1325 100644
--- a/src/gns/gnunet-dns2gns.c
+++ b/src/gns/gnunet-dns2gns.c
@@ -247,19 +247,16 @@ do_timeout (void *cls)
247 * Iterator called on obtained result for a DNS lookup 247 * Iterator called on obtained result for a DNS lookup
248 * 248 *
249 * @param cls closure 249 * @param cls closure
250 * @param rs the request socket
251 * @param dns the DNS udp payload 250 * @param dns the DNS udp payload
252 * @param r size of the DNS payload 251 * @param r size of the DNS payload
253 */ 252 */
254static void 253static void
255dns_result_processor (void *cls, 254dns_result_processor (void *cls,
256 struct GNUNET_DNSSTUB_RequestSocket *rs,
257 const struct GNUNET_TUN_DnsHeader *dns, 255 const struct GNUNET_TUN_DnsHeader *dns,
258 size_t r) 256 size_t r)
259{ 257{
260 struct Request *request = cls; 258 struct Request *request = cls;
261 259
262 (void) rs;
263 if (NULL == dns) 260 if (NULL == dns)
264 { 261 {
265 /* DNSSTUB gave up, so we trigger timeout early */ 262 /* DNSSTUB gave up, so we trigger timeout early */
@@ -307,11 +304,11 @@ result_processor (void *cls,
307 request->original_request_id = request->packet->id; 304 request->original_request_id = request->packet->id;
308 GNUNET_DNSPARSER_free_packet (request->packet); 305 GNUNET_DNSPARSER_free_packet (request->packet);
309 request->packet = NULL; 306 request->packet = NULL;
310 request->dns_lookup = GNUNET_DNSSTUB_resolve2 (dns_stub, 307 request->dns_lookup = GNUNET_DNSSTUB_resolve (dns_stub,
311 request->udp_msg, 308 request->udp_msg,
312 request->udp_msg_size, 309 request->udp_msg_size,
313 &dns_result_processor, 310 &dns_result_processor,
314 request); 311 request);
315 return; 312 return;
316 } 313 }
317 packet = request->packet; 314 packet = request->packet;
@@ -594,8 +591,12 @@ run (void *cls,
594 NULL); 591 NULL);
595 if (NULL == (gns = GNUNET_GNS_connect (cfg))) 592 if (NULL == (gns = GNUNET_GNS_connect (cfg)))
596 return; 593 return;
597 if (NULL == (dns_stub = GNUNET_DNSSTUB_start (dns_ip))) 594 GNUNET_assert (NULL != (dns_stub = GNUNET_DNSSTUB_start (128)));
595 if (GNUNET_OK !=
596 GNUNET_DNSSTUB_add_dns_ip (dns_stub,
597 dns_ip))
598 { 598 {
599 GNUNET_DNSSTUB_stop (dns_stub);
599 GNUNET_GNS_disconnect (gns); 600 GNUNET_GNS_disconnect (gns);
600 gns = NULL; 601 gns = NULL;
601 return; 602 return;
diff --git a/src/gns/gnunet-gns-proxy.c b/src/gns/gnunet-gns-proxy.c
index 8b9aa599e..08663a57e 100644
--- a/src/gns/gnunet-gns-proxy.c
+++ b/src/gns/gnunet-gns-proxy.c
@@ -855,9 +855,9 @@ mhd_content_cb (void *cls,
855 return MHD_CONTENT_READER_END_OF_STREAM; 855 return MHD_CONTENT_READER_END_OF_STREAM;
856 } 856 }
857 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 857 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
858 "Writing %lu/%lu bytes\n", 858 "Writing %llu/%llu bytes\n",
859 bytes_to_copy, 859 (unsigned long long) bytes_to_copy,
860 s5r->io_len); 860 (unsigned long long) s5r->io_len);
861 GNUNET_memcpy (buf, 861 GNUNET_memcpy (buf,
862 s5r->io_buf, 862 s5r->io_buf,
863 bytes_to_copy); 863 bytes_to_copy);
@@ -1307,12 +1307,12 @@ curl_download_cb (void *ptr,
1307 if (sizeof (s5r->io_buf) - s5r->io_len < total) 1307 if (sizeof (s5r->io_buf) - s5r->io_len < total)
1308 { 1308 {
1309 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 1309 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1310 "Pausing CURL `%s%s' download, not enough space %lu %lu %lu\n", 1310 "Pausing CURL `%s%s' download, not enough space %llu %llu %llu\n",
1311 s5r->domain, 1311 s5r->domain,
1312 s5r->url, 1312 s5r->url,
1313 sizeof (s5r->io_buf), 1313 (unsigned long long) sizeof (s5r->io_buf),
1314 s5r->io_len, 1314 (unsigned long long) s5r->io_len,
1315 total); 1315 (unsigned long long) total);
1316 return CURL_WRITEFUNC_PAUSE; /* not enough space */ 1316 return CURL_WRITEFUNC_PAUSE; /* not enough space */
1317 } 1317 }
1318 GNUNET_memcpy (&s5r->io_buf[s5r->io_len], 1318 GNUNET_memcpy (&s5r->io_buf[s5r->io_len],
diff --git a/src/gns/gnunet-service-gns.c b/src/gns/gnunet-service-gns.c
index 90cd47e1d..c376ddfcc 100644
--- a/src/gns/gnunet-service-gns.c
+++ b/src/gns/gnunet-service-gns.c
@@ -229,14 +229,14 @@ GNS_get_tld (const char *name)
229/** 229/**
230 * Task run during shutdown. 230 * Task run during shutdown.
231 * 231 *
232 * @param cls unused 232 * @param cls unused, NULL
233 * @param tc unused
234 */ 233 */
235static void 234static void
236shutdown_task (void *cls) 235shutdown_task (void *cls)
237{ 236{
238 struct GNS_TopLevelDomain *tld; 237 struct GNS_TopLevelDomain *tld;
239 238
239 (void) cls;
240 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 240 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
241 "Shutting down!\n"); 241 "Shutting down!\n");
242 GNS_interceptor_done (); 242 GNS_interceptor_done ();
@@ -283,6 +283,7 @@ client_disconnect_cb (void *cls,
283 struct ClientLookupHandle *clh; 283 struct ClientLookupHandle *clh;
284 struct GnsClient *gc = app_ctx; 284 struct GnsClient *gc = app_ctx;
285 285
286 (void) cls;
286 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 287 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
287 "Client %p disconnected\n", 288 "Client %p disconnected\n",
288 client); 289 client);
@@ -314,6 +315,8 @@ client_connect_cb (void *cls,
314 struct GNUNET_MQ_Handle *mq) 315 struct GNUNET_MQ_Handle *mq)
315{ 316{
316 struct GnsClient *gc; 317 struct GnsClient *gc;
318
319 (void) cls;
317 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 320 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
318 "Client %p connected\n", 321 "Client %p connected\n",
319 client); 322 client);
@@ -383,6 +386,7 @@ check_lookup (void *cls,
383 size_t msg_size; 386 size_t msg_size;
384 const char* name; 387 const char* name;
385 388
389 (void) cls;
386 msg_size = ntohs (l_msg->header.size); 390 msg_size = ntohs (l_msg->header.size);
387 if (msg_size < sizeof (struct LookupMessage)) 391 if (msg_size < sizeof (struct LookupMessage))
388 { 392 {
@@ -479,6 +483,8 @@ read_service_conf (void *cls,
479 struct GNUNET_CRYPTO_EcdsaPublicKey pk; 483 struct GNUNET_CRYPTO_EcdsaPublicKey pk;
480 struct GNS_TopLevelDomain *tld; 484 struct GNS_TopLevelDomain *tld;
481 485
486 (void) cls;
487 (void) section;
482 if (option[0] != '.') 488 if (option[0] != '.')
483 return; 489 return;
484 if (GNUNET_OK != 490 if (GNUNET_OK !=
diff --git a/src/gns/gnunet-service-gns_resolver.c b/src/gns/gnunet-service-gns_resolver.c
index 94819b040..92e03bc69 100644
--- a/src/gns/gnunet-service-gns_resolver.c
+++ b/src/gns/gnunet-service-gns_resolver.c
@@ -72,6 +72,57 @@
72 * DLL to hold the authority chain we had to pass in the resolution 72 * DLL to hold the authority chain we had to pass in the resolution
73 * process. 73 * process.
74 */ 74 */
75struct AuthorityChain;
76
77
78/**
79 * Element of a resolution process for looking up the
80 * responsible DNS server hostname in a GNS2DNS recursive
81 * resolution.
82 */
83struct Gns2DnsPending
84{
85
86 /**
87 * Kept in a DLL.
88 */
89 struct Gns2DnsPending *next;
90
91 /**
92 * Kept in a DLL.
93 */
94 struct Gns2DnsPending *prev;
95
96 /**
97 * Context this activity belongs with.
98 */
99 struct AuthorityChain *ac;
100
101 /**
102 * Handle for the resolution of the IP part of the
103 * GNS2DNS record. Will return to us the addresses
104 * of the DNS resolver to use.
105 */
106 struct GNS_ResolverHandle *rh;
107
108 /**
109 * Handle for DNS resolution of the DNS nameserver.
110 */
111 struct GNUNET_RESOLVER_RequestHandle *dns_rh;
112};
113
114
115/**
116 * Handle to a currenty pending resolution. On result (positive or
117 * negative) the #GNS_ResultProcessor is called.
118 */
119struct GNS_ResolverHandle;
120
121
122/**
123 * DLL to hold the authority chain we had to pass in the resolution
124 * process.
125 */
75struct AuthorityChain 126struct AuthorityChain
76{ 127{
77 /** 128 /**
@@ -131,11 +182,32 @@ struct AuthorityChain
131 char name[GNUNET_DNSPARSER_MAX_NAME_LENGTH + 1]; 182 char name[GNUNET_DNSPARSER_MAX_NAME_LENGTH + 1];
132 183
133 /** 184 /**
134 * IP address of the DNS resolver that is authoritative. 185 * List of resolutions of the 'ip' of the name server that
135 * (this implementation currently only supports one 186 * are still pending.
136 * IP at a time). 187 */
188 struct Gns2DnsPending *gp_head;
189
190 /**
191 * Tail of list of resolutions of the 'ip' of the name server that
192 * are still pending.
193 */
194 struct Gns2DnsPending *gp_tail;
195
196 /**
197 * Handle to perform DNS lookups with this authority (in GNS2DNS handling).
198 */
199 struct GNUNET_DNSSTUB_Context *dns_handle;
200
201 /**
202 * Did we succeed in getting an IP address for *any* of the DNS servers listed?
203 * Once we do, we can start with DNS queries.
204 */
205 int found;
206
207 /**
208 * Did we start the recursive resolution via DNS?
137 */ 209 */
138 struct sockaddr_storage dns_ip; 210 int launched;
139 211
140 } dns_authority; 212 } dns_authority;
141 213
@@ -218,34 +290,6 @@ struct VpnContext
218 290
219 291
220/** 292/**
221 * Information we keep during the resolution of an
222 * IP address for a DNS server while handling a
223 * GNS2DNS record.
224 */
225struct Gns2DnsContext
226{
227
228 /**
229 * DNS domain in which the resolution will continue
230 * (first part of the GNS2DNS record).
231 */
232 char *ns;
233
234 /**
235 * Handle for the resolution of the IP part of the
236 * GNS2DNS record. Will return to us the addresses
237 * of the DNS resolver to use.
238 */
239 struct GNS_ResolverHandle *rh;
240
241 /**
242 * Handle for DNS resolution of the DNS nameserver.
243 */
244 struct GNUNET_RESOLVER_RequestHandle *dns_rh;
245};
246
247
248/**
249 * Handle to a currenty pending resolution. On result (positive or 293 * Handle to a currenty pending resolution. On result (positive or
250 * negative) the #GNS_ResultProcessor is called. 294 * negative) the #GNS_ResultProcessor is called.
251 */ 295 */
@@ -278,12 +322,6 @@ struct GNS_ResolverHandle
278 void* proc_cls; 322 void* proc_cls;
279 323
280 /** 324 /**
281 * Handle used during GNS2DNS resolution for looking up the
282 * IP address of the DNS server.
283 */
284 struct Gns2DnsContext *g2dc;
285
286 /**
287 * Handle for DHT lookups. should be NULL if no lookups are in progress 325 * Handle for DHT lookups. should be NULL if no lookups are in progress
288 */ 326 */
289 struct GNUNET_DHT_GetHandle *get_handle; 327 struct GNUNET_DHT_GetHandle *get_handle;
@@ -431,11 +469,6 @@ static struct GNUNET_VPN_Handle *vpn_handle;
431static struct GNUNET_DHT_Handle *dht_handle; 469static struct GNUNET_DHT_Handle *dht_handle;
432 470
433/** 471/**
434 * Handle to perform DNS lookups.
435 */
436static struct GNUNET_DNSSTUB_Context *dns_handle;
437
438/**
439 * Heap for limiting parallel DHT lookups 472 * Heap for limiting parallel DHT lookups
440 */ 473 */
441static struct GNUNET_CONTAINER_Heap *dht_lookup_heap; 474static struct GNUNET_CONTAINER_Heap *dht_lookup_heap;
@@ -468,7 +501,7 @@ static struct CacheOps *co_tail;
468/** 501/**
469 * Use namecache 502 * Use namecache
470 */ 503 */
471static int use_cache; 504static int disable_cache;
472 505
473/** 506/**
474 * Global configuration. 507 * Global configuration.
@@ -543,21 +576,53 @@ translate_dot_plus (struct GNS_ResolverHandle *rh,
543 576
544 577
545/** 578/**
546 * Task scheduled to asynchronously fail a resolution. 579 * Wrapper around #GNS_resolver_lookup_cancel() as a task.
580 * Used for delayed cleanup so we can unwind the stack first.
547 * 581 *
548 * @param cls the 'struct GNS_ResolverHandle' of the resolution to fail 582 * @param cls the `struct GNS_ResolverHandle`
549 */ 583 */
550static void 584static void
551fail_resolution (void *cls) 585GNS_resolver_lookup_cancel_ (void *cls)
552{ 586{
553 struct GNS_ResolverHandle *rh = cls; 587 struct GNS_ResolverHandle *rh = cls;
554 588
555 rh->task_id = NULL; 589 rh->task_id = NULL;
556 rh->proc (rh->proc_cls, 0, NULL);
557 GNS_resolver_lookup_cancel (rh); 590 GNS_resolver_lookup_cancel (rh);
558} 591}
559 592
560 593
594/**
595 * Function called to asynchronously fail a resolution.
596 *
597 * @param rh the resolution to fail
598 */
599static void
600fail_resolution (struct GNS_ResolverHandle *rh)
601{
602 rh->proc (rh->proc_cls,
603 0,
604 NULL);
605 GNUNET_assert (NULL == rh->task_id);
606 rh->task_id = GNUNET_SCHEDULER_add_now (&GNS_resolver_lookup_cancel_,
607 rh);
608}
609
610
611/**
612 * Function called when a resolution times out.
613 *
614 * @param cls the `struct GNS_ResolverHandle`
615 */
616static void
617timeout_resolution (void *cls)
618{
619 struct GNS_ResolverHandle *rh = cls;
620
621 rh->task_id = NULL;
622 fail_resolution (rh);
623}
624
625
561#if (defined WINDOWS) || (defined DARWIN) 626#if (defined WINDOWS) || (defined DARWIN)
562/* Don't have this on W32, here's a naive implementation 627/* Don't have this on W32, here's a naive implementation
563 * Was somehow removed on OS X ... */ 628 * Was somehow removed on OS X ... */
@@ -831,13 +896,11 @@ start_resolver_lookup (void *cls);
831 * 896 *
832 * @param cls the request handle of the resolution that 897 * @param cls the request handle of the resolution that
833 * we were attempting to make 898 * we were attempting to make
834 * @param rs socket that received the response
835 * @param dns dns response, never NULL 899 * @param dns dns response, never NULL
836 * @param dns_len number of bytes in @a dns 900 * @param dns_len number of bytes in @a dns
837 */ 901 */
838static void 902static void
839dns_result_parser (void *cls, 903dns_result_parser (void *cls,
840 struct GNUNET_DNSSTUB_RequestSocket *rs,
841 const struct GNUNET_TUN_DnsHeader *dns, 904 const struct GNUNET_TUN_DnsHeader *dns,
842 size_t dns_len) 905 size_t dns_len)
843{ 906{
@@ -845,18 +908,13 @@ dns_result_parser (void *cls,
845 struct GNUNET_DNSPARSER_Packet *p; 908 struct GNUNET_DNSPARSER_Packet *p;
846 const struct GNUNET_DNSPARSER_Record *rec; 909 const struct GNUNET_DNSPARSER_Record *rec;
847 unsigned int rd_count; 910 unsigned int rd_count;
848 unsigned int i;
849 911
850 (void) rs;
851 if (NULL == dns) 912 if (NULL == dns)
852 { 913 {
853 rh->dns_request = NULL; 914 rh->dns_request = NULL;
854 GNUNET_SCHEDULER_cancel (rh->task_id); 915 GNUNET_SCHEDULER_cancel (rh->task_id);
855 rh->task_id = NULL; 916 rh->task_id = NULL;
856 rh->proc (rh->proc_cls, 917 fail_resolution (rh);
857 0,
858 NULL);
859 GNS_resolver_lookup_cancel (rh);
860 return; 918 return;
861 } 919 }
862 if (rh->original_dns_id != dns->id) 920 if (rh->original_dns_id != dns->id)
@@ -872,6 +930,8 @@ dns_result_parser (void *cls,
872 _("Failed to parse DNS response\n")); 930 _("Failed to parse DNS response\n"));
873 return; 931 return;
874 } 932 }
933
934 /* We got a result from DNS */
875 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 935 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
876 "Received DNS response for `%s' with %u answers\n", 936 "Received DNS response for `%s' with %u answers\n",
877 rh->ac_tail->label, 937 rh->ac_tail->label,
@@ -920,8 +980,10 @@ dns_result_parser (void *cls,
920 980
921 buf_off = 0; 981 buf_off = 0;
922 skip = 0; 982 skip = 0;
923 memset (rd, 0, sizeof (rd)); 983 memset (rd,
924 for (i=0;i<rd_count;i++) 984 0,
985 sizeof (rd));
986 for (unsigned int i=0;i<rd_count;i++)
925 { 987 {
926 if (i < p->num_answers) 988 if (i < p->num_answers)
927 rec = &p->answers[i]; 989 rec = &p->answers[i];
@@ -1043,9 +1105,12 @@ dns_result_parser (void *cls,
1043 rh->proc (rh->proc_cls, 1105 rh->proc (rh->proc_cls,
1044 rd_count - skip, 1106 rd_count - skip,
1045 rd); 1107 rd);
1046 GNS_resolver_lookup_cancel (rh);
1047 } 1108 }
1048 GNUNET_DNSPARSER_free_packet (p); 1109 GNUNET_DNSPARSER_free_packet (p);
1110 if (NULL != rh->task_id)
1111 GNUNET_SCHEDULER_cancel (rh->task_id); /* should be timeout task */
1112 rh->task_id = GNUNET_SCHEDULER_add_now (&GNS_resolver_lookup_cancel_,
1113 rh);
1049} 1114}
1050 1115
1051 1116
@@ -1061,7 +1126,6 @@ static void
1061recursive_dns_resolution (struct GNS_ResolverHandle *rh) 1126recursive_dns_resolution (struct GNS_ResolverHandle *rh)
1062{ 1127{
1063 struct AuthorityChain *ac; 1128 struct AuthorityChain *ac;
1064 socklen_t sa_len;
1065 struct GNUNET_DNSPARSER_Query *query; 1129 struct GNUNET_DNSPARSER_Query *query;
1066 struct GNUNET_DNSPARSER_Packet *p; 1130 struct GNUNET_DNSPARSER_Packet *p;
1067 char *dns_request; 1131 char *dns_request;
@@ -1074,20 +1138,6 @@ recursive_dns_resolution (struct GNS_ResolverHandle *rh)
1074 "Starting DNS lookup for `%s'\n", 1138 "Starting DNS lookup for `%s'\n",
1075 ac->label); 1139 ac->label);
1076 GNUNET_assert (GNUNET_NO == ac->gns_authority); 1140 GNUNET_assert (GNUNET_NO == ac->gns_authority);
1077 switch (((const struct sockaddr *) &ac->authority_info.dns_authority.dns_ip)->sa_family)
1078 {
1079 case AF_INET:
1080 sa_len = sizeof (struct sockaddr_in);
1081 break;
1082 case AF_INET6:
1083 sa_len = sizeof (struct sockaddr_in6);
1084 break;
1085 default:
1086 GNUNET_break (0);
1087 rh->proc (rh->proc_cls, 0, NULL);
1088 GNS_resolver_lookup_cancel (rh);
1089 return;
1090 }
1091 query = GNUNET_new (struct GNUNET_DNSPARSER_Query); 1141 query = GNUNET_new (struct GNUNET_DNSPARSER_Query);
1092 query->name = GNUNET_strdup (ac->label); 1142 query->name = GNUNET_strdup (ac->label);
1093 query->type = rh->record_type; 1143 query->type = rh->record_type;
@@ -1109,20 +1159,22 @@ recursive_dns_resolution (struct GNS_ResolverHandle *rh)
1109 rh->proc (rh->proc_cls, 1159 rh->proc (rh->proc_cls,
1110 0, 1160 0,
1111 NULL); 1161 NULL);
1112 GNS_resolver_lookup_cancel (rh); 1162 GNUNET_assert (NULL == rh->task_id);
1163 rh->task_id = GNUNET_SCHEDULER_add_now (&GNS_resolver_lookup_cancel_,
1164 rh);
1113 } 1165 }
1114 else 1166 else
1115 { 1167 {
1116 rh->original_dns_id = p->id; 1168 rh->original_dns_id = p->id;
1117 rh->dns_request = GNUNET_DNSSTUB_resolve (dns_handle, 1169 GNUNET_assert (NULL != ac->authority_info.dns_authority.dns_handle);
1118 (const struct sockaddr *) &ac->authority_info.dns_authority.dns_ip, 1170 GNUNET_assert (NULL == rh->dns_request);
1119 sa_len, 1171 rh->dns_request = GNUNET_DNSSTUB_resolve (ac->authority_info.dns_authority.dns_handle,
1120 dns_request, 1172 dns_request,
1121 dns_request_length, 1173 dns_request_length,
1122 &dns_result_parser, 1174 &dns_result_parser,
1123 rh); 1175 rh);
1124 rh->task_id = GNUNET_SCHEDULER_add_delayed (DNS_LOOKUP_TIMEOUT, 1176 rh->task_id = GNUNET_SCHEDULER_add_delayed (DNS_LOOKUP_TIMEOUT,
1125 &fail_resolution, 1177 &timeout_resolution,
1126 rh); 1178 rh);
1127 } 1179 }
1128 if (GNUNET_SYSERR != ret) 1180 if (GNUNET_SYSERR != ret)
@@ -1293,15 +1345,47 @@ vpn_allocation_cb (void *cls,
1293 1345
1294 1346
1295/** 1347/**
1348 * We have resolved one or more of the nameservers for a
1349 * GNS2DNS lookup. Once we have some of them, begin using
1350 * the DNSSTUB resolver.
1351 *
1352 * @param ac context for GNS2DNS resolution
1353 */
1354static void
1355continue_with_gns2dns (struct AuthorityChain *ac)
1356{
1357 struct GNS_ResolverHandle *rh = ac->rh;
1358
1359 if ( (NULL != ac->authority_info.dns_authority.gp_head) &&
1360 (GNUNET_NO == ac->authority_info.dns_authority.found) )
1361 return; /* more pending and none found yet */
1362 if (GNUNET_NO == ac->authority_info.dns_authority.found)
1363 {
1364 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1365 "Failed to resolve DNS server for `%s' in GNS2DNS resolution\n",
1366 ac->authority_info.dns_authority.name);
1367 fail_resolution (rh);
1368 return;
1369 }
1370 if (GNUNET_NO != ac->authority_info.dns_authority.launched)
1371 return; /* already running, do not launch again! */
1372 /* recurse */
1373 ac->authority_info.dns_authority.launched = GNUNET_YES;
1374 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1375 "Will continue resolution using DNS to resolve `%s'\n",
1376 ac->label);
1377 GNUNET_assert (NULL == rh->task_id);
1378 rh->task_id = GNUNET_SCHEDULER_add_now (&recursive_resolution,
1379 rh);
1380
1381}
1382
1383
1384/**
1296 * We've resolved the IP address for the DNS resolver to use 1385 * We've resolved the IP address for the DNS resolver to use
1297 * after encountering a GNS2DNS record. 1386 * after encountering a GNS2DNS record.
1298 * 1387 *
1299 * TODO: Right now we only foward the request to ONE DNS resolver, 1388 * @param cls the `struct Gns2DnsPending` used for this request
1300 * even if we get multiple IP addresses back; a correct implementation
1301 * should try all DNS resolvers.
1302 *
1303 * @param cls the `struct GNS_ResolverHandle` where we encountered
1304 * the GNS2DNS record
1305 * @param rd_count number of records in @a rd 1389 * @param rd_count number of records in @a rd
1306 * @param rd addresses for the DNS resolver (presumably) 1390 * @param rd addresses for the DNS resolver (presumably)
1307 */ 1391 */
@@ -1310,136 +1394,96 @@ handle_gns2dns_result (void *cls,
1310 unsigned int rd_count, 1394 unsigned int rd_count,
1311 const struct GNUNET_GNSRECORD_Data *rd) 1395 const struct GNUNET_GNSRECORD_Data *rd)
1312{ 1396{
1313 struct GNS_ResolverHandle *rh = cls; 1397 struct Gns2DnsPending *gp = cls;
1314 struct AuthorityChain *ac; 1398 struct AuthorityChain *ac = gp->ac;
1315 struct sockaddr *sa; 1399
1316 struct sockaddr_in v4; 1400 GNUNET_CONTAINER_DLL_remove (ac->authority_info.dns_authority.gp_head,
1317 struct sockaddr_in6 v6; 1401 ac->authority_info.dns_authority.gp_tail,
1318 size_t sa_len; 1402 gp);
1319 1403 /* enable cleanup of 'rh' handle that automatically comes after we return,
1320 /* find suitable A/AAAA record */ 1404 and which expects 'rh' to be in the #rlh_head DLL. */
1321 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 1405 if (NULL != gp->rh)
1322 "Received %u results for IP address of DNS server for GNS2DNS transition\n",
1323 rd_count);
1324 /* enable cleanup of 'rh' handle that comes next... */
1325 if (NULL != rh->g2dc->rh)
1326 { 1406 {
1327 GNUNET_CONTAINER_DLL_insert (rlh_head, 1407 GNUNET_CONTAINER_DLL_insert (rlh_head,
1328 rlh_tail, 1408 rlh_tail,
1329 rh->g2dc->rh); 1409 gp->rh);
1330 rh->g2dc->rh = NULL; 1410 gp->rh = NULL;
1331 } 1411 }
1332 sa = NULL; 1412 GNUNET_free (gp);
1333 sa_len = 0; 1413 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1414 "Received %u results for IP address of DNS server for GNS2DNS transition\n",
1415 rd_count);
1416 /* find suitable A/AAAA record */
1334 for (unsigned int j=0;j<rd_count;j++) 1417 for (unsigned int j=0;j<rd_count;j++)
1335 { 1418 {
1336 switch (rd[j].record_type) 1419 switch (rd[j].record_type)
1337 { 1420 {
1338 case GNUNET_DNSPARSER_TYPE_A: 1421 case GNUNET_DNSPARSER_TYPE_A:
1339 if (sizeof (struct in_addr) != rd[j].data_size)
1340 { 1422 {
1341 GNUNET_break_op (0); 1423 struct sockaddr_in v4;
1342 rh->proc (rh->proc_cls, 0, NULL); 1424
1343 GNS_resolver_lookup_cancel (rh); 1425 if (sizeof (struct in_addr) != rd[j].data_size)
1344 return; 1426 {
1345 } 1427 GNUNET_break_op (0);
1346 /* FIXME: might want to check if we support IPv4 here, 1428 continue;
1347 and otherwise skip this one and hope we find another */ 1429 }
1348 memset (&v4, 0, sizeof (v4)); 1430 memset (&v4,
1349 sa_len = sizeof (v4); 1431 0,
1350 v4.sin_family = AF_INET; 1432 sizeof (v4));
1351 v4.sin_port = htons (53); 1433 v4.sin_family = AF_INET;
1434 v4.sin_port = htons (53);
1352#if HAVE_SOCKADDR_IN_SIN_LEN 1435#if HAVE_SOCKADDR_IN_SIN_LEN
1353 v4.sin_len = (u_char) sa_len; 1436 v4.sin_len = (u_char) sizeof (v4);
1354#endif 1437#endif
1355 GNUNET_memcpy (&v4.sin_addr, 1438 GNUNET_memcpy (&v4.sin_addr,
1356 rd[j].data, 1439 rd[j].data,
1357 sizeof (struct in_addr)); 1440 sizeof (struct in_addr));
1358 sa = (struct sockaddr *) &v4; 1441 if (GNUNET_OK ==
1359 break; 1442 GNUNET_DNSSTUB_add_dns_sa (ac->authority_info.dns_authority.dns_handle,
1443 (const struct sockaddr *) &v4))
1444 ac->authority_info.dns_authority.found = GNUNET_YES;
1445 break;
1446 }
1360 case GNUNET_DNSPARSER_TYPE_AAAA: 1447 case GNUNET_DNSPARSER_TYPE_AAAA:
1361 if (sizeof (struct in6_addr) != rd[j].data_size)
1362 { 1448 {
1363 GNUNET_break_op (0); 1449 struct sockaddr_in6 v6;
1364 rh->proc (rh->proc_cls, 0, NULL); 1450
1365 GNS_resolver_lookup_cancel (rh); 1451 if (sizeof (struct in6_addr) != rd[j].data_size)
1366 return; 1452 {
1367 } 1453 GNUNET_break_op (0);
1368 /* FIXME: might want to check if we support IPv6 here, 1454 continue;
1369 and otherwise skip this one and hope we find another */ 1455 }
1370 memset (&v6, 0, sizeof (v6)); 1456 /* FIXME: might want to check if we support IPv6 here,
1371 sa_len = sizeof (v6); 1457 and otherwise skip this one and hope we find another */
1372 v6.sin6_family = AF_INET6; 1458 memset (&v6,
1373 v6.sin6_port = htons (53); 1459 0,
1460 sizeof (v6));
1461 v6.sin6_family = AF_INET6;
1462 v6.sin6_port = htons (53);
1374#if HAVE_SOCKADDR_IN_SIN_LEN 1463#if HAVE_SOCKADDR_IN_SIN_LEN
1375 v6.sin6_len = (u_char) sa_len; 1464 v6.sin6_len = (u_char) sizeof (v6);
1376#endif 1465#endif
1377 GNUNET_memcpy (&v6.sin6_addr, 1466 GNUNET_memcpy (&v6.sin6_addr,
1378 rd[j].data, 1467 rd[j].data,
1379 sizeof (struct in6_addr)); 1468 sizeof (struct in6_addr));
1380 sa = (struct sockaddr *) &v6; 1469 if (GNUNET_OK ==
1381 break; 1470 GNUNET_DNSSTUB_add_dns_sa (ac->authority_info.dns_authority.dns_handle,
1471 (const struct sockaddr *) &v6))
1472 ac->authority_info.dns_authority.found = GNUNET_YES;
1473 break;
1474 }
1382 default: 1475 default:
1383 break; 1476 break;
1384 } 1477 }
1385 if (NULL != sa)
1386 break;
1387 }
1388 if (NULL == sa)
1389 {
1390 /* we cannot continue; NS without A/AAAA */
1391 rh->proc (rh->proc_cls, 0, NULL);
1392 GNS_resolver_lookup_cancel (rh);
1393 return;
1394 } 1478 }
1395 /* expand authority chain */ 1479 continue_with_gns2dns (ac);
1396 ac = GNUNET_new (struct AuthorityChain);
1397 ac->rh = rh;
1398 GNUNET_assert (strlen (rh->g2dc->ns) <= GNUNET_DNSPARSER_MAX_NAME_LENGTH);
1399 strcpy (ac->authority_info.dns_authority.name,
1400 rh->g2dc->ns);
1401 GNUNET_memcpy (&ac->authority_info.dns_authority.dns_ip,
1402 sa,
1403 sa_len);
1404 /* for DNS recursion, the label is the full DNS name,
1405 created from the remainder of the GNS name and the
1406 name in the NS record */
1407 GNUNET_asprintf (&ac->label,
1408 "%.*s%s%s",
1409 (int) rh->name_resolution_pos,
1410 rh->name,
1411 (0 != rh->name_resolution_pos) ? "." : "",
1412 rh->g2dc->ns);
1413 GNUNET_free (rh->g2dc->ns);
1414 GNUNET_free (rh->g2dc);
1415 rh->g2dc = NULL;
1416 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1417 "Will continue resolution using DNS server `%s' to resolve `%s'\n",
1418 GNUNET_a2s (sa,
1419 sa_len),
1420 ac->label);
1421 GNUNET_CONTAINER_DLL_insert_tail (rh->ac_head,
1422 rh->ac_tail,
1423 ac);
1424 if (strlen (ac->label) > GNUNET_DNSPARSER_MAX_NAME_LENGTH)
1425 {
1426 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1427 _("GNS lookup resulted in DNS name that is too long (`%s')\n"),
1428 ac->label);
1429 rh->proc (rh->proc_cls, 0, NULL);
1430 GNS_resolver_lookup_cancel (rh);
1431 return;
1432 }
1433 /* recurse */
1434 rh->task_id = GNUNET_SCHEDULER_add_now (&recursive_resolution,
1435 rh);
1436} 1480}
1437 1481
1438 1482
1439/** 1483/**
1440 * Function called by the resolver for each address obtained from DNS. 1484 * Function called by the resolver for each address obtained from DNS.
1441 * 1485 *
1442 * @param cls closure, a `struct Gns2DnsContext *` 1486 * @param cls closure, a `struct Gns2DnsPending *`
1443 * @param addr one of the addresses of the host, NULL for the last address 1487 * @param addr one of the addresses of the host, NULL for the last address
1444 * @param addrlen length of @a addr 1488 * @param addrlen length of @a addr
1445 */ 1489 */
@@ -1448,57 +1492,270 @@ handle_gns2dns_ip (void *cls,
1448 const struct sockaddr *addr, 1492 const struct sockaddr *addr,
1449 socklen_t addrlen) 1493 socklen_t addrlen)
1450{ 1494{
1451 struct Gns2DnsContext *g2dc = cls; 1495 struct Gns2DnsPending *gp = cls;
1452 struct GNUNET_GNSRECORD_Data rd; 1496 struct AuthorityChain *ac = gp->ac;
1453 1497
1498 GNUNET_RESOLVER_request_cancel (gp->dns_rh);
1499 GNUNET_CONTAINER_DLL_remove (ac->authority_info.dns_authority.gp_head,
1500 ac->authority_info.dns_authority.gp_tail,
1501 gp);
1502 GNUNET_free (gp);
1454 if (NULL == addr) 1503 if (NULL == addr)
1455 { 1504 {
1456 /* DNS resolution failed */ 1505 /* DNS resolution failed */
1457 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, 1506 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1458 "Failed to use DNS to resolve name of DNS resolver\n"); 1507 "Failed to use DNS to resolve name of DNS resolver\n");
1459 g2dc->rh->g2dc = NULL;
1460 fail_resolution (g2dc->rh);
1461 GNUNET_free (g2dc);
1462 } 1508 }
1463 switch (addr->sa_family) 1509 else
1464 { 1510 {
1465 case AF_INET: 1511 if (GNUNET_OK ==
1512 GNUNET_DNSSTUB_add_dns_sa (ac->authority_info.dns_authority.dns_handle,
1513 addr))
1514 ac->authority_info.dns_authority.found = GNUNET_YES;
1515 }
1516 continue_with_gns2dns (ac);
1517}
1518
1519
1520/**
1521 * We found a CNAME record, perform recursive resolution on it.
1522 *
1523 * @param rh resolution handle
1524 * @param rd record with CNAME to resolve recursively
1525 */
1526static void
1527recursive_cname_resolution (struct GNS_ResolverHandle *rh,
1528 const struct GNUNET_GNSRECORD_Data *rd)
1529{
1530 char *cname;
1531 size_t off;
1532
1533 off = 0;
1534 cname = GNUNET_DNSPARSER_parse_name (rd->data,
1535 rd->data_size,
1536 &off);
1537 if ( (NULL == cname) ||
1538 (off != rd->data_size) )
1539 {
1540 GNUNET_break_op (0); /* record not well-formed */
1541 GNUNET_free_non_null (cname);
1542 fail_resolution (rh);
1543 return;
1544 }
1545 handle_gns_cname_result (rh,
1546 cname);
1547 GNUNET_free (cname);
1548}
1549
1550
1551/**
1552 * We found a PKEY record, perform recursive resolution on it.
1553 *
1554 * @param rh resolution handle
1555 * @param rd record with PKEY to resolve recursively
1556 */
1557static void
1558recursive_pkey_resolution (struct GNS_ResolverHandle *rh,
1559 const struct GNUNET_GNSRECORD_Data *rd)
1560{
1561 struct AuthorityChain *ac;
1562
1563 /* delegation to another zone */
1564 if (sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey) !=
1565 rd->data_size)
1566 {
1567 GNUNET_break_op (0);
1568 fail_resolution (rh);
1569 return;
1570 }
1571 /* expand authority chain */
1572 ac = GNUNET_new (struct AuthorityChain);
1573 ac->rh = rh;
1574 ac->gns_authority = GNUNET_YES;
1575 ac->suggested_shortening_label = NULL;
1576 ac->shortening_started = GNUNET_NO;
1577 GNUNET_memcpy (&ac->authority_info.gns_authority,
1578 rd->data,
1579 sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey));
1580 ac->label = resolver_lookup_get_next_label (rh);
1581 /* add AC to tail */
1582 GNUNET_CONTAINER_DLL_insert_tail (rh->ac_head,
1583 rh->ac_tail,
1584 ac);
1585 /* recurse */
1586 rh->task_id = GNUNET_SCHEDULER_add_now (&recursive_resolution,
1587 rh);
1588}
1589
1590
1591/**
1592 * We found one or more GNS2DNS records, perform recursive resolution on it.
1593 * (to be precise, one or more records in @a rd is GNS2DNS, there may be others,
1594 * so this function still needs to check which ones are GNS2DNS).
1595 *
1596 * @param rh resolution handle
1597 * @param rd_count length of the @a rd array
1598 * @param rd record with PKEY to resolve recursively
1599 * @return #GNUNET_OK if this worked, #GNUNET_SYSERR if no GNS2DNS records were in @a rd
1600 */
1601static int
1602recursive_gns2dns_resolution (struct GNS_ResolverHandle *rh,
1603 unsigned int rd_count,
1604 const struct GNUNET_GNSRECORD_Data *rd)
1605{
1606 struct AuthorityChain *ac;
1607 const char *tld;
1608 char *ns;
1609
1610 ns = NULL;
1611 /* expand authority chain */
1612 ac = GNUNET_new (struct AuthorityChain);
1613 ac->rh = rh;
1614 ac->authority_info.dns_authority.dns_handle = GNUNET_DNSSTUB_start (4);
1615
1616 for (unsigned int i=0;i<rd_count;i++)
1617 {
1618 char *ip;
1619 char *n;
1620 size_t off;
1621 struct Gns2DnsPending *gp;
1622 struct GNUNET_CRYPTO_EcdsaPublicKey zone;
1623
1624 if (GNUNET_GNSRECORD_TYPE_GNS2DNS != rd[i].record_type)
1625 continue;
1626 off = 0;
1627 n = GNUNET_DNSPARSER_parse_name (rd[i].data,
1628 rd[i].data_size,
1629 &off);
1630 ip = GNUNET_DNSPARSER_parse_name (rd[i].data,
1631 rd[i].data_size,
1632 &off);
1633 if ( (NULL == n) ||
1634 (NULL == ip) ||
1635 (off != rd[i].data_size) )
1466 { 1636 {
1467 const struct sockaddr_in *v4 = (const struct sockaddr_in *) addr; 1637 GNUNET_break_op (0);
1468 1638 GNUNET_free_non_null (n);
1469 GNUNET_assert (sizeof (*v4) == addrlen); 1639 GNUNET_free_non_null (ip);
1470 rd.data = v4; 1640 continue;
1471 rd.data_size = sizeof (*v4);
1472 rd.expiration_time = UINT64_MAX;
1473 rd.record_type = GNUNET_DNSPARSER_TYPE_A;
1474 rd.flags = 0;
1475 break;
1476 } 1641 }
1477 case AF_INET6: 1642 /* resolve 'ip' to determine the IP(s) of the DNS
1643 resolver to use for lookup of 'ns' */
1644 if (NULL != ns)
1478 { 1645 {
1479 const struct sockaddr_in6 *v6 = (const struct sockaddr_in6 *) addr; 1646 if (0 != strcasecmp (ns,
1480 1647 n))
1481 GNUNET_assert (sizeof (*v6) == addrlen); 1648 {
1482 rd.data = v6; 1649 /* NS values must all be the same for all GNS2DNS records,
1483 rd.data_size = sizeof (v6); 1650 anything else leads to insanity */
1484 rd.expiration_time = UINT64_MAX; 1651 GNUNET_break_op (0);
1485 rd.record_type = GNUNET_DNSPARSER_TYPE_AAAA; 1652 GNUNET_free (n);
1486 rd.flags = 0; 1653 GNUNET_free (ip);
1487 break; 1654 continue;
1655 }
1656 GNUNET_free (n);
1657 }
1658 else
1659 {
1660 ns = n;
1488 } 1661 }
1489 default:
1490 return;
1491 }
1492 GNUNET_RESOLVER_request_cancel (g2dc->dns_rh);
1493 g2dc->dns_rh = NULL;
1494 handle_gns2dns_result (g2dc->rh,
1495 1,
1496 &rd);
1497 1662
1663 /* check if 'ip' is already an IPv4/IPv6 address */
1664 if (GNUNET_OK ==
1665 GNUNET_DNSSTUB_add_dns_ip (ac->authority_info.dns_authority.dns_handle,
1666 ip))
1667 {
1668 ac->authority_info.dns_authority.found = GNUNET_YES;
1669 GNUNET_free (ip);
1670 continue;
1671 }
1672 tld = GNS_get_tld (ip);
1673 if (0 != strcmp (tld,
1674 "+"))
1675 {
1676 /* 'ip' is a DNS name */
1677 gp = GNUNET_new (struct Gns2DnsPending);
1678 gp->ac = ac;
1679 GNUNET_CONTAINER_DLL_insert (ac->authority_info.dns_authority.gp_head,
1680 ac->authority_info.dns_authority.gp_tail,
1681 gp);
1682 gp->dns_rh = GNUNET_RESOLVER_ip_get (ip,
1683 AF_UNSPEC,
1684 GNUNET_TIME_UNIT_FOREVER_REL,
1685 &handle_gns2dns_ip,
1686 gp);
1687 GNUNET_free (ip);
1688 continue;
1689 }
1690 /* 'ip' should be a GNS name */
1691 gp = GNUNET_new (struct Gns2DnsPending);
1692 gp->ac = ac;
1693 GNUNET_CONTAINER_DLL_insert (ac->authority_info.dns_authority.gp_head,
1694 ac->authority_info.dns_authority.gp_tail,
1695 gp);
1696 gp->rh = GNUNET_new (struct GNS_ResolverHandle);
1697 ip = translate_dot_plus (rh,
1698 ip);
1699 tld = GNS_get_tld (ip);
1700 if (GNUNET_OK !=
1701 GNUNET_GNSRECORD_zkey_to_pkey (tld,
1702 &zone))
1703 {
1704 GNUNET_break_op (0);
1705 GNUNET_free (ip);
1706 continue;
1707 }
1708 gp->rh->authority_zone = zone;
1709 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1710 "Resolving `%s' to determine IP address of DNS server for GNS2DNS transition for `%s'\n",
1711 ip,
1712 ns);
1713 gp->rh->name = ip;
1714 gp->rh->name_resolution_pos = strlen (ip) - strlen (tld) - 1;
1715 gp->rh->proc = &handle_gns2dns_result;
1716 gp->rh->proc_cls = gp;
1717 gp->rh->record_type = GNUNET_GNSRECORD_TYPE_ANY;
1718 gp->rh->options = GNUNET_GNS_LO_DEFAULT;
1719 gp->rh->loop_limiter = rh->loop_limiter + 1;
1720 gp->rh->task_id
1721 = GNUNET_SCHEDULER_add_now (&start_resolver_lookup,
1722 gp->rh);
1723 } /* end 'for all records' */
1724
1725 if (NULL == ns)
1726 {
1727 /* not a single GNS2DNS record found */
1728 GNUNET_free (ac);
1729 return GNUNET_SYSERR;
1730 }
1731 GNUNET_assert (strlen (ns) <= GNUNET_DNSPARSER_MAX_NAME_LENGTH);
1732 strcpy (ac->authority_info.dns_authority.name,
1733 ns);
1734 /* for DNS recursion, the label is the full DNS name,
1735 created from the remainder of the GNS name and the
1736 name in the NS record */
1737 GNUNET_asprintf (&ac->label,
1738 "%.*s%s%s",
1739 (int) rh->name_resolution_pos,
1740 rh->name,
1741 (0 != rh->name_resolution_pos) ? "." : "",
1742 ns);
1743 GNUNET_free (ns);
1744 GNUNET_CONTAINER_DLL_insert_tail (rh->ac_head,
1745 rh->ac_tail,
1746 ac);
1747 if (strlen (ac->label) > GNUNET_DNSPARSER_MAX_NAME_LENGTH)
1748 {
1749 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1750 _("GNS lookup resulted in DNS name that is too long (`%s')\n"),
1751 ac->label);
1752 return GNUNET_SYSERR;
1753 }
1754 continue_with_gns2dns (ac);
1755 return GNUNET_OK;
1498} 1756}
1499 1757
1500 1758
1501
1502/** 1759/**
1503 * Process a records that were decrypted from a block. 1760 * Process a records that were decrypted from a block.
1504 * 1761 *
@@ -1512,7 +1769,6 @@ handle_gns_resolution_result (void *cls,
1512 const struct GNUNET_GNSRECORD_Data *rd) 1769 const struct GNUNET_GNSRECORD_Data *rd)
1513{ 1770{
1514 struct GNS_ResolverHandle *rh = cls; 1771 struct GNS_ResolverHandle *rh = cls;
1515 struct AuthorityChain *ac;
1516 struct AuthorityChain *shorten_ac; 1772 struct AuthorityChain *shorten_ac;
1517 char *cname; 1773 char *cname;
1518 struct VpnContext *vpn_ctx; 1774 struct VpnContext *vpn_ctx;
@@ -1532,6 +1788,14 @@ handle_gns_resolution_result (void *cls,
1532 rh->ac_tail->label, 1788 rh->ac_tail->label,
1533 GNUNET_GNSRECORD_z2s (&rh->ac_tail->authority_info.gns_authority), 1789 GNUNET_GNSRECORD_z2s (&rh->ac_tail->authority_info.gns_authority),
1534 rd_count); 1790 rd_count);
1791 if (0 == rd_count)
1792 {
1793 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1794 _("GNS lookup failed (zero records found)\n"));
1795 fail_resolution (rh);
1796 return;
1797 }
1798
1535 if (0 == rh->name_resolution_pos) 1799 if (0 == rh->name_resolution_pos)
1536 { 1800 {
1537 /* top-level match, are we done yet? */ 1801 /* top-level match, are we done yet? */
@@ -1547,9 +1811,8 @@ handle_gns_resolution_result (void *cls,
1547 (off != rd[0].data_size) ) 1811 (off != rd[0].data_size) )
1548 { 1812 {
1549 GNUNET_break_op (0); 1813 GNUNET_break_op (0);
1550 rh->proc (rh->proc_cls, 0, NULL);
1551 GNS_resolver_lookup_cancel (rh);
1552 GNUNET_free_non_null (cname); 1814 GNUNET_free_non_null (cname);
1815 fail_resolution (rh);
1553 return; 1816 return;
1554 } 1817 }
1555 handle_gns_cname_result (rh, 1818 handle_gns_cname_result (rh,
@@ -1573,8 +1836,7 @@ handle_gns_resolution_result (void *cls,
1573 rd[i].data_size) 1836 rd[i].data_size)
1574 { 1837 {
1575 GNUNET_break_op (0); 1838 GNUNET_break_op (0);
1576 rh->proc (rh->proc_cls, 0, NULL); 1839 fail_resolution (rh);
1577 GNS_resolver_lookup_cancel (rh);
1578 return; 1840 return;
1579 } 1841 }
1580 vpn = (const struct GNUNET_TUN_GnsVpnRecord *) rd[i].data; 1842 vpn = (const struct GNUNET_TUN_GnsVpnRecord *) rd[i].data;
@@ -1582,8 +1844,7 @@ handle_gns_resolution_result (void *cls,
1582 if ('\0' != vname[rd[i].data_size - 1 - sizeof (struct GNUNET_TUN_GnsVpnRecord)]) 1844 if ('\0' != vname[rd[i].data_size - 1 - sizeof (struct GNUNET_TUN_GnsVpnRecord)])
1583 { 1845 {
1584 GNUNET_break_op (0); 1846 GNUNET_break_op (0);
1585 rh->proc (rh->proc_cls, 0, NULL); 1847 fail_resolution (rh);
1586 GNS_resolver_lookup_cancel (rh);
1587 return; 1848 return;
1588 } 1849 }
1589 GNUNET_TUN_service_name_to_hash (vname, 1850 GNUNET_TUN_service_name_to_hash (vname,
@@ -1621,7 +1882,13 @@ handle_gns_resolution_result (void *cls,
1621 /* delegation to DNS */ 1882 /* delegation to DNS */
1622 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 1883 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1623 "Found GNS2DNS record, delegating to DNS!\n"); 1884 "Found GNS2DNS record, delegating to DNS!\n");
1624 goto do_recurse; 1885 if (GNUNET_OK ==
1886 recursive_gns2dns_resolution (rh,
1887 rd_count,
1888 rd))
1889 return;
1890 else
1891 goto fail;
1625 } 1892 }
1626 default: 1893 default:
1627 break; 1894 break;
@@ -1823,18 +2090,20 @@ handle_gns_resolution_result (void *cls,
1823 GNUNET_break_op (0); 2090 GNUNET_break_op (0);
1824 break; 2091 break;
1825 } 2092 }
1826 GNUNET_memcpy (&pub, rd[i].data, rd[i].data_size); 2093 GNUNET_memcpy (&pub,
2094 rd[i].data,
2095 rd[i].data_size);
1827 rd_off++; 2096 rd_off++;
1828 if (GNUNET_GNSRECORD_TYPE_PKEY != rh->record_type) 2097 if (GNUNET_GNSRECORD_TYPE_PKEY != rh->record_type)
1829 { 2098 {
1830 /* try to resolve "+" */ 2099 /* try to resolve "@" */
1831 struct AuthorityChain *ac; 2100 struct AuthorityChain *ac;
1832 2101
1833 ac = GNUNET_new (struct AuthorityChain); 2102 ac = GNUNET_new (struct AuthorityChain);
1834 ac->rh = rh; 2103 ac->rh = rh;
1835 ac->gns_authority = GNUNET_YES; 2104 ac->gns_authority = GNUNET_YES;
1836 ac->authority_info.gns_authority = pub; 2105 ac->authority_info.gns_authority = pub;
1837 ac->label = GNUNET_strdup (GNUNET_GNS_MASTERZONE_STR); 2106 ac->label = GNUNET_strdup (GNUNET_GNS_EMPTY_LABEL_AT);
1838 ac->suggested_shortening_label = NULL; 2107 ac->suggested_shortening_label = NULL;
1839 ac->shortening_started = GNUNET_NO; 2108 ac->shortening_started = GNUNET_NO;
1840 GNUNET_CONTAINER_DLL_insert_tail (rh->ac_head, 2109 GNUNET_CONTAINER_DLL_insert_tail (rh->ac_head,
@@ -1856,7 +2125,13 @@ handle_gns_resolution_result (void *cls,
1856 } 2125 }
1857 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 2126 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1858 "Found GNS2DNS record, delegating to DNS!\n"); 2127 "Found GNS2DNS record, delegating to DNS!\n");
1859 goto do_recurse; 2128 if (GNUNET_OK ==
2129 recursive_gns2dns_resolution (rh,
2130 rd_count,
2131 rd))
2132 return;
2133 else
2134 goto fail;
1860 } 2135 }
1861 case GNUNET_GNSRECORD_TYPE_BOX: 2136 case GNUNET_GNSRECORD_TYPE_BOX:
1862 { 2137 {
@@ -1901,200 +2176,35 @@ handle_gns_resolution_result (void *cls,
1901 rh->proc (rh->proc_cls, 2176 rh->proc (rh->proc_cls,
1902 rd_off, 2177 rd_off,
1903 rd_new); 2178 rd_new);
1904 GNS_resolver_lookup_cancel (rh); 2179 rh->task_id = GNUNET_SCHEDULER_add_now (&GNS_resolver_lookup_cancel_,
2180 rh);
1905 return; 2181 return;
1906 } 2182 }
1907 do_recurse: 2183
1908 /* need to recurse, check if we can */ 2184 switch (rd[0].record_type)
1909 for (unsigned int i=0;i<rd_count;i++)
1910 { 2185 {
1911 switch (rd[i].record_type) 2186 case GNUNET_DNSPARSER_TYPE_CNAME:
1912 { 2187 GNUNET_break_op (1 == rd_count); /* CNAME should be unique */
1913 case GNUNET_GNSRECORD_TYPE_PKEY: 2188 recursive_cname_resolution (rh,
1914 /* delegation to another zone */ 2189 &rd[0]);
1915 if (sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey) != 2190 return;
1916 rd[i].data_size) 2191 case GNUNET_GNSRECORD_TYPE_PKEY:
1917 { 2192 GNUNET_break_op (1 == rd_count); /* PKEY should be unique */
1918 GNUNET_break_op (0); 2193 recursive_pkey_resolution (rh,
1919 rh->proc (rh->proc_cls, 0, NULL); 2194 &rd[0]);
1920 GNS_resolver_lookup_cancel (rh); 2195 return;
1921 return; 2196 default:
1922 } 2197 if (GNUNET_OK ==
1923 /* expand authority chain */ 2198 recursive_gns2dns_resolution (rh,
1924 ac = GNUNET_new (struct AuthorityChain); 2199 rd_count,
1925 ac->rh = rh; 2200 rd))
1926 ac->gns_authority = GNUNET_YES;
1927 ac->suggested_shortening_label = NULL;
1928 ac->shortening_started = GNUNET_NO;
1929 GNUNET_memcpy (&ac->authority_info.gns_authority,
1930 rd[i].data,
1931 sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey));
1932 ac->label = resolver_lookup_get_next_label (rh);
1933 /* add AC to tail */
1934 GNUNET_CONTAINER_DLL_insert_tail (rh->ac_head,
1935 rh->ac_tail,
1936 ac);
1937 /* recurse */
1938 rh->task_id = GNUNET_SCHEDULER_add_now (&recursive_resolution,
1939 rh);
1940 return; 2201 return;
1941 case GNUNET_GNSRECORD_TYPE_GNS2DNS: 2202 break;
1942 {
1943 /* TODO: Right now we only foward the request to ONE DNS resolver,
1944 even if we get multiple IP addresses back; a correct implementation
1945 should try all DNS resolvers. */
1946 /* resolution continues within DNS */
1947 struct Gns2DnsContext *g2dc;
1948 char *ip;
1949 char *ns;
1950 const char *tld;
1951 struct GNUNET_CRYPTO_EcdsaPublicKey zone;
1952 struct in_addr v4;
1953 struct in6_addr v6;
1954
1955 off = 0;
1956 ns = GNUNET_DNSPARSER_parse_name (rd[i].data,
1957 rd[i].data_size,
1958 &off);
1959 ip = GNUNET_DNSPARSER_parse_name (rd[i].data,
1960 rd[i].data_size,
1961 &off);
1962 if ( (NULL == ns) ||
1963 (NULL == ip) ||
1964 (off != rd[i].data_size) )
1965 {
1966 GNUNET_break_op (0);
1967 GNUNET_free_non_null (ns);
1968 GNUNET_free_non_null (ip);
1969 fail_resolution (rh);
1970 return;
1971 }
1972 /* resolve 'ip' to determine the IP(s) of the DNS
1973 resolver to use for lookup of 'ns' */
1974 g2dc = GNUNET_new (struct Gns2DnsContext);
1975 g2dc->ns = ns;
1976 rh->g2dc = g2dc;
1977
1978 /* check if 'ip' is already an IPv4/IPv6 address */
1979 if (1 == inet_pton (AF_INET,
1980 ip,
1981 &v4))
1982 {
1983 /* name is IPv4 address, pretend it's an A record */
1984 struct GNUNET_GNSRECORD_Data rd;
1985
1986 GNUNET_free (ip);
1987 rd.data = &v4;
1988 rd.data_size = sizeof (v4);
1989 rd.expiration_time = UINT64_MAX;
1990 rd.record_type = GNUNET_DNSPARSER_TYPE_A;
1991 rd.flags = 0;
1992 handle_gns2dns_result (rh,
1993 1,
1994 &rd);
1995 return;
1996 }
1997 if (1 == inet_pton (AF_INET6,
1998 ip,
1999 &v6))
2000 {
2001 /* name is IPv6 address, pretend it's an AAAA record */
2002 struct GNUNET_GNSRECORD_Data rd;
2003
2004 GNUNET_free (ip);
2005 rd.data = &v6;
2006 rd.data_size = sizeof (v6);
2007 rd.expiration_time = UINT64_MAX;
2008 rd.record_type = GNUNET_DNSPARSER_TYPE_AAAA;
2009 rd.flags = 0;
2010 handle_gns2dns_result (rh,
2011 1,
2012 &rd);
2013 return;
2014 }
2015
2016 tld = GNS_get_tld (ip);
2017 if (0 != strcmp (tld,
2018 "+"))
2019 {
2020 /* 'ip' is a DNS name */
2021 g2dc->dns_rh = GNUNET_RESOLVER_ip_get (ip,
2022 AF_UNSPEC,
2023 GNUNET_TIME_UNIT_FOREVER_REL,
2024 &handle_gns2dns_ip,
2025 g2dc);
2026 GNUNET_free (ip);
2027 return;
2028 }
2029
2030 /* 'ip' should be a GNS name */
2031 g2dc->rh = GNUNET_new (struct GNS_ResolverHandle);
2032
2033 ip = translate_dot_plus (rh,
2034 ip);
2035 tld = GNS_get_tld (ip);
2036 if (GNUNET_OK !=
2037 GNUNET_GNSRECORD_zkey_to_pkey (tld,
2038 &zone))
2039 {
2040 GNUNET_break_op (0);
2041 GNUNET_free_non_null (ns);
2042 GNUNET_free_non_null (ip);
2043 GNUNET_free (g2dc);
2044 fail_resolution (rh);
2045 return;
2046 }
2047 g2dc->rh->authority_zone = zone;
2048 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2049 "Resolving `%s' to determine IP address of DNS server for GNS2DNS transition for `%s'\n",
2050 ip,
2051 ns);
2052 g2dc->rh->name = ip;
2053 g2dc->rh->name_resolution_pos = strlen (ip) - strlen (tld) - 1;
2054 g2dc->rh->proc = &handle_gns2dns_result;
2055 g2dc->rh->proc_cls = rh;
2056 g2dc->rh->record_type = GNUNET_GNSRECORD_TYPE_ANY;
2057 g2dc->rh->options = GNUNET_GNS_LO_DEFAULT;
2058 g2dc->rh->loop_limiter = rh->loop_limiter + 1;
2059 g2dc->rh->task_id
2060 = GNUNET_SCHEDULER_add_now (&start_resolver_lookup,
2061 g2dc->rh);
2062 return;
2063 }
2064 case GNUNET_DNSPARSER_TYPE_CNAME:
2065 {
2066 char *cname;
2067
2068 off = 0;
2069 cname = GNUNET_DNSPARSER_parse_name (rd[i].data,
2070 rd[i].data_size,
2071 &off);
2072 if ( (NULL == cname) ||
2073 (off != rd[i].data_size) )
2074 {
2075 GNUNET_break_op (0); /* record not well-formed */
2076 rh->proc (rh->proc_cls, 0, NULL);
2077 GNS_resolver_lookup_cancel (rh);
2078 GNUNET_free_non_null (cname);
2079 return;
2080 }
2081 handle_gns_cname_result (rh,
2082 cname);
2083 GNUNET_free (cname);
2084 return;
2085 }
2086 /* FIXME: handle DNAME */
2087 default:
2088 /* skip */
2089 break;
2090 }
2091 } 2203 }
2204 fail:
2092 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, 2205 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
2093 _("GNS lookup recursion failed (no delegation record found)\n")); 2206 _("GNS lookup recursion failed (no delegation record found)\n"));
2094 rh->proc (rh->proc_cls, 2207 fail_resolution (rh);
2095 0,
2096 NULL);
2097 GNS_resolver_lookup_cancel (rh);
2098} 2208}
2099 2209
2100 2210
@@ -2176,8 +2286,7 @@ handle_dht_response (void *cls,
2176 { 2286 {
2177 /* how did this pass DHT block validation!? */ 2287 /* how did this pass DHT block validation!? */
2178 GNUNET_break (0); 2288 GNUNET_break (0);
2179 rh->proc (rh->proc_cls, 0, NULL); 2289 fail_resolution (rh);
2180 GNS_resolver_lookup_cancel (rh);
2181 return; 2290 return;
2182 } 2291 }
2183 block = data; 2292 block = data;
@@ -2188,8 +2297,7 @@ handle_dht_response (void *cls,
2188 { 2297 {
2189 /* how did this pass DHT block validation!? */ 2298 /* how did this pass DHT block validation!? */
2190 GNUNET_break (0); 2299 GNUNET_break (0);
2191 rh->proc (rh->proc_cls, 0, NULL); 2300 fail_resolution (rh);
2192 GNS_resolver_lookup_cancel (rh);
2193 return; 2301 return;
2194 } 2302 }
2195 if (GNUNET_OK != 2303 if (GNUNET_OK !=
@@ -2200,8 +2308,7 @@ handle_dht_response (void *cls,
2200 rh)) 2308 rh))
2201 { 2309 {
2202 GNUNET_break_op (0); /* block was ill-formed */ 2310 GNUNET_break_op (0); /* block was ill-formed */
2203 rh->proc (rh->proc_cls, 0, NULL); 2311 fail_resolution (rh);
2204 GNS_resolver_lookup_cancel (rh);
2205 return; 2312 return;
2206 } 2313 }
2207 if (0 == GNUNET_TIME_absolute_get_remaining (GNUNET_TIME_absolute_ntoh (block->expiration_time)).rel_value_us) 2314 if (0 == GNUNET_TIME_absolute_get_remaining (GNUNET_TIME_absolute_ntoh (block->expiration_time)).rel_value_us)
@@ -2252,8 +2359,7 @@ start_dht_request (struct GNS_ResolverHandle *rh,
2252 /* fail longest-standing DHT request */ 2359 /* fail longest-standing DHT request */
2253 rx = GNUNET_CONTAINER_heap_peek (dht_lookup_heap); 2360 rx = GNUNET_CONTAINER_heap_peek (dht_lookup_heap);
2254 GNUNET_assert (NULL != rx); 2361 GNUNET_assert (NULL != rx);
2255 rx->proc (rx->proc_cls, 0, NULL); 2362 fail_resolution (rx);
2256 GNS_resolver_lookup_cancel (rx);
2257 } 2363 }
2258} 2364}
2259 2365
@@ -2324,8 +2430,7 @@ handle_namecache_block_response (void *cls,
2324 "Resolution failed for `%s' in zone %s (DHT lookup not permitted by configuration)\n", 2430 "Resolution failed for `%s' in zone %s (DHT lookup not permitted by configuration)\n",
2325 ac->label, 2431 ac->label,
2326 GNUNET_GNSRECORD_z2s (&ac->authority_info.gns_authority)); 2432 GNUNET_GNSRECORD_z2s (&ac->authority_info.gns_authority));
2327 rh->proc (rh->proc_cls, 0, NULL); 2433 fail_resolution (rh);
2328 GNS_resolver_lookup_cancel (rh);
2329 return; 2434 return;
2330 } 2435 }
2331 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 2436 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
@@ -2373,7 +2478,7 @@ recursive_gns_resolution_namecache (struct GNS_ResolverHandle *rh)
2373 GNUNET_GNSRECORD_query_from_public_key (&ac->authority_info.gns_authority, 2478 GNUNET_GNSRECORD_query_from_public_key (&ac->authority_info.gns_authority,
2374 ac->label, 2479 ac->label,
2375 &query); 2480 &query);
2376 if (GNUNET_YES == use_cache) 2481 if (GNUNET_YES != disable_cache)
2377 { 2482 {
2378 rh->namecache_qe 2483 rh->namecache_qe
2379 = GNUNET_NAMECACHE_lookup_block (namecache_handle, 2484 = GNUNET_NAMECACHE_lookup_block (namecache_handle,
@@ -2384,7 +2489,8 @@ recursive_gns_resolution_namecache (struct GNS_ResolverHandle *rh)
2384 } 2489 }
2385 else 2490 else
2386 { 2491 {
2387 start_dht_request (rh, &query); 2492 start_dht_request (rh,
2493 &query);
2388 } 2494 }
2389} 2495}
2390 2496
@@ -2408,8 +2514,7 @@ handle_revocation_result (void *cls,
2408 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, 2514 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
2409 _("Zone %s was revoked, resolution fails\n"), 2515 _("Zone %s was revoked, resolution fails\n"),
2410 GNUNET_GNSRECORD_z2s (&ac->authority_info.gns_authority)); 2516 GNUNET_GNSRECORD_z2s (&ac->authority_info.gns_authority));
2411 rh->proc (rh->proc_cls, 0, NULL); 2517 fail_resolution (rh);
2412 GNS_resolver_lookup_cancel (rh);
2413 return; 2518 return;
2414 } 2519 }
2415 recursive_gns_resolution_namecache (rh); 2520 recursive_gns_resolution_namecache (rh);
@@ -2453,8 +2558,7 @@ recursive_resolution (void *cls)
2453 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, 2558 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
2454 "Encountered unbounded recursion resolving `%s'\n", 2559 "Encountered unbounded recursion resolving `%s'\n",
2455 rh->name); 2560 rh->name);
2456 rh->proc (rh->proc_cls, 0, NULL); 2561 fail_resolution (rh);
2457 GNS_resolver_lookup_cancel (rh);
2458 return; 2562 return;
2459 } 2563 }
2460 if (GNUNET_YES == rh->ac_tail->gns_authority) 2564 if (GNUNET_YES == rh->ac_tail->gns_authority)
@@ -2491,8 +2595,12 @@ start_resolver_lookup (void *cls)
2491 rd.expiration_time = UINT64_MAX; 2595 rd.expiration_time = UINT64_MAX;
2492 rd.record_type = GNUNET_DNSPARSER_TYPE_A; 2596 rd.record_type = GNUNET_DNSPARSER_TYPE_A;
2493 rd.flags = 0; 2597 rd.flags = 0;
2494 rh->proc (rh->proc_cls, 1, &rd); 2598 rh->proc (rh->proc_cls,
2495 GNS_resolver_lookup_cancel (rh); 2599 1,
2600 &rd);
2601 GNUNET_assert (NULL == rh->task_id);
2602 rh->task_id = GNUNET_SCHEDULER_add_now (&GNS_resolver_lookup_cancel_,
2603 rh);
2496 return; 2604 return;
2497 } 2605 }
2498 if (1 == inet_pton (AF_INET6, 2606 if (1 == inet_pton (AF_INET6,
@@ -2507,8 +2615,12 @@ start_resolver_lookup (void *cls)
2507 rd.expiration_time = UINT64_MAX; 2615 rd.expiration_time = UINT64_MAX;
2508 rd.record_type = GNUNET_DNSPARSER_TYPE_AAAA; 2616 rd.record_type = GNUNET_DNSPARSER_TYPE_AAAA;
2509 rd.flags = 0; 2617 rd.flags = 0;
2510 rh->proc (rh->proc_cls, 1, &rd); 2618 rh->proc (rh->proc_cls,
2511 GNS_resolver_lookup_cancel (rh); 2619 1,
2620 &rd);
2621 GNUNET_assert (NULL == rh->task_id);
2622 rh->task_id = GNUNET_SCHEDULER_add_now (&GNS_resolver_lookup_cancel_,
2623 rh);
2512 return; 2624 return;
2513 } 2625 }
2514 2626
@@ -2518,8 +2630,8 @@ start_resolver_lookup (void *cls)
2518 ac->suggested_shortening_label = NULL; 2630 ac->suggested_shortening_label = NULL;
2519 if (NULL == ac->label) 2631 if (NULL == ac->label)
2520 /* name was just the "TLD", so we default to label 2632 /* name was just the "TLD", so we default to label
2521 #GNUNET_GNS_MASTERZONE_STR */ 2633 #GNUNET_GNS_EMPTY_LABEL_AT */
2522 ac->label = GNUNET_strdup (GNUNET_GNS_MASTERZONE_STR); 2634 ac->label = GNUNET_strdup (GNUNET_GNS_EMPTY_LABEL_AT);
2523 ac->gns_authority = GNUNET_YES; 2635 ac->gns_authority = GNUNET_YES;
2524 ac->authority_info.gns_authority = rh->authority_zone; 2636 ac->authority_info.gns_authority = rh->authority_zone;
2525 GNUNET_CONTAINER_DLL_insert_tail (rh->ac_head, 2637 GNUNET_CONTAINER_DLL_insert_tail (rh->ac_head,
@@ -2587,37 +2699,51 @@ GNS_resolver_lookup_cancel (struct GNS_ResolverHandle *rh)
2587 GNUNET_CONTAINER_DLL_remove (rlh_head, 2699 GNUNET_CONTAINER_DLL_remove (rlh_head,
2588 rlh_tail, 2700 rlh_tail,
2589 rh); 2701 rh);
2702 if (NULL != rh->dns_request)
2703 {
2704 GNUNET_DNSSTUB_resolve_cancel (rh->dns_request);
2705 rh->dns_request = NULL;
2706 }
2590 while (NULL != (ac = rh->ac_head)) 2707 while (NULL != (ac = rh->ac_head))
2591 { 2708 {
2592 GNUNET_CONTAINER_DLL_remove (rh->ac_head, 2709 GNUNET_CONTAINER_DLL_remove (rh->ac_head,
2593 rh->ac_tail, 2710 rh->ac_tail,
2594 ac); 2711 ac);
2712 if (GNUNET_NO == ac->gns_authority)
2713 {
2714 struct Gns2DnsPending *gp;
2715
2716 while (NULL != (gp = ac->authority_info.dns_authority.gp_head))
2717 {
2718 GNUNET_CONTAINER_DLL_remove (ac->authority_info.dns_authority.gp_head,
2719 ac->authority_info.dns_authority.gp_tail,
2720 gp);
2721 if (NULL != gp->rh)
2722 {
2723 /* rh->g2dc->rh is NOT in the DLL yet, so to enable us
2724 using GNS_resolver_lookup_cancel here, we need to
2725 add it first... */
2726 GNUNET_CONTAINER_DLL_insert (rlh_head,
2727 rlh_tail,
2728 gp->rh);
2729 GNUNET_assert (NULL == gp->rh->task_id);
2730 gp->rh->task_id = GNUNET_SCHEDULER_add_now (&GNS_resolver_lookup_cancel_,
2731 gp->rh);
2732 gp->rh = NULL;
2733 }
2734 if (NULL != gp->dns_rh)
2735 {
2736 GNUNET_RESOLVER_request_cancel (gp->dns_rh);
2737 gp->dns_rh = NULL;
2738 }
2739 GNUNET_free (gp);
2740 }
2741 GNUNET_DNSSTUB_stop (ac->authority_info.dns_authority.dns_handle);
2742 }
2595 GNUNET_free (ac->label); 2743 GNUNET_free (ac->label);
2596 GNUNET_free_non_null (ac->suggested_shortening_label); 2744 GNUNET_free_non_null (ac->suggested_shortening_label);
2597 GNUNET_free (ac); 2745 GNUNET_free (ac);
2598 } 2746 }
2599 if (NULL != rh->g2dc)
2600 {
2601 /* rh->g2dc->rh is NOT in the DLL yet, so to enable us
2602 using GNS_resolver_lookup_cancel here, we need to
2603 add it first... */
2604 if (NULL != rh->g2dc->rh)
2605 {
2606 GNUNET_CONTAINER_DLL_insert (rlh_head,
2607 rlh_tail,
2608 rh->g2dc->rh);
2609 GNS_resolver_lookup_cancel (rh->g2dc->rh);
2610 rh->g2dc->rh = NULL;
2611 }
2612 if (NULL != rh->g2dc->dns_rh)
2613 {
2614 GNUNET_RESOLVER_request_cancel (rh->g2dc->dns_rh);
2615 rh->g2dc->rh = NULL;
2616 }
2617 GNUNET_free (rh->g2dc->ns);
2618 GNUNET_free (rh->g2dc);
2619 rh->g2dc = NULL;
2620 }
2621 if (NULL != rh->task_id) 2747 if (NULL != rh->task_id)
2622 { 2748 {
2623 GNUNET_SCHEDULER_cancel (rh->task_id); 2749 GNUNET_SCHEDULER_cancel (rh->task_id);
@@ -2639,11 +2765,6 @@ GNS_resolver_lookup_cancel (struct GNS_ResolverHandle *rh)
2639 GNUNET_free (vpn_ctx->rd_data); 2765 GNUNET_free (vpn_ctx->rd_data);
2640 GNUNET_free (vpn_ctx); 2766 GNUNET_free (vpn_ctx);
2641 } 2767 }
2642 if (NULL != rh->dns_request)
2643 {
2644 GNUNET_DNSSTUB_resolve_cancel (rh->dns_request);
2645 rh->dns_request = NULL;
2646 }
2647 if (NULL != rh->namecache_qe) 2768 if (NULL != rh->namecache_qe)
2648 { 2769 {
2649 GNUNET_NAMECACHE_cancel (rh->namecache_qe); 2770 GNUNET_NAMECACHE_cancel (rh->namecache_qe);
@@ -2690,34 +2811,18 @@ GNS_resolver_init (struct GNUNET_NAMECACHE_Handle *nc,
2690 const struct GNUNET_CONFIGURATION_Handle *c, 2811 const struct GNUNET_CONFIGURATION_Handle *c,
2691 unsigned long long max_bg_queries) 2812 unsigned long long max_bg_queries)
2692{ 2813{
2693 char *dns_ip;
2694
2695 cfg = c; 2814 cfg = c;
2696 namecache_handle = nc; 2815 namecache_handle = nc;
2697 dht_handle = dht; 2816 dht_handle = dht;
2698 dht_lookup_heap = 2817 dht_lookup_heap =
2699 GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN); 2818 GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
2700 max_allowed_background_queries = max_bg_queries; 2819 max_allowed_background_queries = max_bg_queries;
2701 if (GNUNET_SYSERR == (use_cache = 2820 disable_cache = GNUNET_CONFIGURATION_get_value_yesno (cfg,
2702 GNUNET_CONFIGURATION_get_value_yesno (c, 2821 "namecache",
2703 "gns", 2822 "DISABLE");
2704 "USE_CACHE"))) 2823 if (GNUNET_YES == disable_cache)
2705 use_cache = GNUNET_YES; 2824 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
2706 if (GNUNET_NO == use_cache)
2707 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
2708 "Namecache disabled\n"); 2825 "Namecache disabled\n");
2709
2710 if (GNUNET_OK !=
2711 GNUNET_CONFIGURATION_get_value_string (c,
2712 "gns",
2713 "DNS_RESOLVER",
2714 &dns_ip))
2715 {
2716 /* user did not specify DNS resolver, use 8.8.8.8 */
2717 dns_ip = GNUNET_strdup ("8.8.8.8");
2718 }
2719 dns_handle = GNUNET_DNSSTUB_start (dns_ip);
2720 GNUNET_free (dns_ip);
2721 vpn_handle = GNUNET_VPN_connect (cfg); 2826 vpn_handle = GNUNET_VPN_connect (cfg);
2722} 2827}
2723 2828
@@ -2734,7 +2839,9 @@ GNS_resolver_done ()
2734 /* abort active resolutions */ 2839 /* abort active resolutions */
2735 while (NULL != (rh = rlh_head)) 2840 while (NULL != (rh = rlh_head))
2736 { 2841 {
2737 rh->proc (rh->proc_cls, 0, NULL); 2842 rh->proc (rh->proc_cls,
2843 0,
2844 NULL);
2738 GNS_resolver_lookup_cancel (rh); 2845 GNS_resolver_lookup_cancel (rh);
2739 } 2846 }
2740 while (NULL != (co = co_head)) 2847 while (NULL != (co = co_head))
@@ -2747,8 +2854,6 @@ GNS_resolver_done ()
2747 } 2854 }
2748 GNUNET_CONTAINER_heap_destroy (dht_lookup_heap); 2855 GNUNET_CONTAINER_heap_destroy (dht_lookup_heap);
2749 dht_lookup_heap = NULL; 2856 dht_lookup_heap = NULL;
2750 GNUNET_DNSSTUB_stop (dns_handle);
2751 dns_handle = NULL;
2752 GNUNET_VPN_disconnect (vpn_handle); 2857 GNUNET_VPN_disconnect (vpn_handle);
2753 vpn_handle = NULL; 2858 vpn_handle = NULL;
2754 dht_handle = NULL; 2859 dht_handle = NULL;
diff --git a/src/gns/test_gns_lookup.conf b/src/gns/test_gns_lookup.conf
index ea8c7c3fc..a9a2345c7 100644
--- a/src/gns/test_gns_lookup.conf
+++ b/src/gns/test_gns_lookup.conf
@@ -10,7 +10,7 @@ AUTOSTART = YES
10PLUGINS = 10PLUGINS =
11 11
12[gns] 12[gns]
13#PREFIX = valgrind --leak-check=full --track-origins=yes 13# PREFIX = valgrind --leak-check=full --track-origins=yes
14AUTOSTART = YES 14AUTOSTART = YES
15AUTO_IMPORT_PKEY = YES 15AUTO_IMPORT_PKEY = YES
16MAX_PARALLEL_BACKGROUND_QUERIES = 10 16MAX_PARALLEL_BACKGROUND_QUERIES = 10
diff --git a/src/gnsrecord/gnsrecord_crypto.c b/src/gnsrecord/gnsrecord_crypto.c
index 0fab97f06..cebc842f3 100644
--- a/src/gnsrecord/gnsrecord_crypto.c
+++ b/src/gnsrecord/gnsrecord_crypto.c
@@ -72,6 +72,7 @@ derive_block_aes_key (struct GNUNET_CRYPTO_SymmetricInitializationVector *iv,
72 * Sign name and records 72 * Sign name and records
73 * 73 *
74 * @param key the private key 74 * @param key the private key
75 * @param pkey associated public key
75 * @param expire block expiration 76 * @param expire block expiration
76 * @param label the name for the records 77 * @param label the name for the records
77 * @param rd record data 78 * @param rd record data
@@ -79,29 +80,29 @@ derive_block_aes_key (struct GNUNET_CRYPTO_SymmetricInitializationVector *iv,
79 * @return NULL on error (block too large) 80 * @return NULL on error (block too large)
80 */ 81 */
81struct GNUNET_GNSRECORD_Block * 82struct GNUNET_GNSRECORD_Block *
82GNUNET_GNSRECORD_block_create (const struct GNUNET_CRYPTO_EcdsaPrivateKey *key, 83block_create (const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
83 struct GNUNET_TIME_Absolute expire, 84 const struct GNUNET_CRYPTO_EcdsaPublicKey *pkey,
84 const char *label, 85 struct GNUNET_TIME_Absolute expire,
85 const struct GNUNET_GNSRECORD_Data *rd, 86 const char *label,
86 unsigned int rd_count) 87 const struct GNUNET_GNSRECORD_Data *rd,
88 unsigned int rd_count)
87{ 89{
88 size_t payload_len = GNUNET_GNSRECORD_records_get_size (rd_count, rd); 90 size_t payload_len = GNUNET_GNSRECORD_records_get_size (rd_count,
91 rd);
89 char payload[sizeof (uint32_t) + payload_len]; 92 char payload[sizeof (uint32_t) + payload_len];
90 struct GNUNET_GNSRECORD_Block *block; 93 struct GNUNET_GNSRECORD_Block *block;
91 struct GNUNET_CRYPTO_EcdsaPublicKey pkey;
92 struct GNUNET_CRYPTO_EcdsaPrivateKey *dkey; 94 struct GNUNET_CRYPTO_EcdsaPrivateKey *dkey;
93 struct GNUNET_CRYPTO_SymmetricInitializationVector iv; 95 struct GNUNET_CRYPTO_SymmetricInitializationVector iv;
94 struct GNUNET_CRYPTO_SymmetricSessionKey skey; 96 struct GNUNET_CRYPTO_SymmetricSessionKey skey;
95 struct GNUNET_GNSRECORD_Data rdc[rd_count]; 97 struct GNUNET_GNSRECORD_Data rdc[rd_count];
96 uint32_t rd_count_nbo; 98 uint32_t rd_count_nbo;
97 unsigned int i;
98 struct GNUNET_TIME_Absolute now; 99 struct GNUNET_TIME_Absolute now;
99 100
100 if (payload_len > GNUNET_GNSRECORD_MAX_BLOCK_SIZE) 101 if (payload_len > GNUNET_GNSRECORD_MAX_BLOCK_SIZE)
101 return NULL; 102 return NULL;
102 /* convert relative to absolute times */ 103 /* convert relative to absolute times */
103 now = GNUNET_TIME_absolute_get (); 104 now = GNUNET_TIME_absolute_get ();
104 for (i=0;i<rd_count;i++) 105 for (unsigned int i=0;i<rd_count;i++)
105 { 106 {
106 rdc[i] = rd[i]; 107 rdc[i] = rd[i];
107 if (0 != (rd[i].flags & GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION)) 108 if (0 != (rd[i].flags & GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION))
@@ -116,13 +117,19 @@ GNUNET_GNSRECORD_block_create (const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
116 } 117 }
117 /* serialize */ 118 /* serialize */
118 rd_count_nbo = htonl (rd_count); 119 rd_count_nbo = htonl (rd_count);
119 GNUNET_memcpy (payload, &rd_count_nbo, sizeof (uint32_t)); 120 GNUNET_memcpy (payload,
121 &rd_count_nbo,
122 sizeof (uint32_t));
120 GNUNET_assert (payload_len == 123 GNUNET_assert (payload_len ==
121 GNUNET_GNSRECORD_records_serialize (rd_count, rdc, 124 GNUNET_GNSRECORD_records_serialize (rd_count,
122 payload_len, &payload[sizeof (uint32_t)])); 125 rdc,
126 payload_len,
127 &payload[sizeof (uint32_t)]));
123 block = GNUNET_malloc (sizeof (struct GNUNET_GNSRECORD_Block) + 128 block = GNUNET_malloc (sizeof (struct GNUNET_GNSRECORD_Block) +
124 sizeof (uint32_t) + payload_len); 129 sizeof (uint32_t) +
125 block->purpose.size = htonl (sizeof (uint32_t) + payload_len + 130 payload_len);
131 block->purpose.size = htonl (sizeof (uint32_t) +
132 payload_len +
126 sizeof (struct GNUNET_CRYPTO_EccSignaturePurpose) + 133 sizeof (struct GNUNET_CRYPTO_EccSignaturePurpose) +
127 sizeof (struct GNUNET_TIME_AbsoluteNBO)); 134 sizeof (struct GNUNET_TIME_AbsoluteNBO));
128 block->purpose.purpose = htonl (GNUNET_SIGNATURE_PURPOSE_GNS_RECORD_SIGN); 135 block->purpose.purpose = htonl (GNUNET_SIGNATURE_PURPOSE_GNS_RECORD_SIGN);
@@ -132,18 +139,21 @@ GNUNET_GNSRECORD_block_create (const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
132 label, 139 label,
133 "gns"); 140 "gns");
134 GNUNET_CRYPTO_ecdsa_key_get_public (dkey, 141 GNUNET_CRYPTO_ecdsa_key_get_public (dkey,
135 &block->derived_key); 142 &block->derived_key);
136 GNUNET_CRYPTO_ecdsa_key_get_public (key, 143 derive_block_aes_key (&iv,
137 &pkey); 144 &skey,
138 derive_block_aes_key (&iv, &skey, label, &pkey); 145 label,
146 pkey);
139 GNUNET_break (payload_len + sizeof (uint32_t) == 147 GNUNET_break (payload_len + sizeof (uint32_t) ==
140 GNUNET_CRYPTO_symmetric_encrypt (payload, payload_len + sizeof (uint32_t), 148 GNUNET_CRYPTO_symmetric_encrypt (payload,
141 &skey, &iv, 149 payload_len + sizeof (uint32_t),
150 &skey,
151 &iv,
142 &block[1])); 152 &block[1]));
143 if (GNUNET_OK != 153 if (GNUNET_OK !=
144 GNUNET_CRYPTO_ecdsa_sign (dkey, 154 GNUNET_CRYPTO_ecdsa_sign (dkey,
145 &block->purpose, 155 &block->purpose,
146 &block->signature)) 156 &block->signature))
147 { 157 {
148 GNUNET_break (0); 158 GNUNET_break (0);
149 GNUNET_free (dkey); 159 GNUNET_free (dkey);
@@ -156,6 +166,97 @@ GNUNET_GNSRECORD_block_create (const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
156 166
157 167
158/** 168/**
169 * Sign name and records
170 *
171 * @param key the private key
172 * @param expire block expiration
173 * @param label the name for the records
174 * @param rd record data
175 * @param rd_count number of records
176 * @return NULL on error (block too large)
177 */
178struct GNUNET_GNSRECORD_Block *
179GNUNET_GNSRECORD_block_create (const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
180 struct GNUNET_TIME_Absolute expire,
181 const char *label,
182 const struct GNUNET_GNSRECORD_Data *rd,
183 unsigned int rd_count)
184{
185 struct GNUNET_CRYPTO_EcdsaPublicKey pkey;
186
187 GNUNET_CRYPTO_ecdsa_key_get_public (key,
188 &pkey);
189 return block_create (key,
190 &pkey,
191 expire,
192 label,
193 rd,
194 rd_count);
195}
196
197
198/**
199 * Line in cache mapping private keys to public keys.
200 */
201struct KeyCacheLine
202{
203 /**
204 * A private key.
205 */
206 struct GNUNET_CRYPTO_EcdsaPrivateKey key;
207
208 /**
209 * Associated public key.
210 */
211 struct GNUNET_CRYPTO_EcdsaPublicKey pkey;
212
213};
214
215
216/**
217 * Sign name and records, cache derived public key (also keeps the
218 * private key in static memory, so do not use this function if
219 * keeping the private key in the process'es RAM is a major issue).
220 *
221 * @param key the private key
222 * @param expire block expiration
223 * @param label the name for the records
224 * @param rd record data
225 * @param rd_count number of records
226 * @return NULL on error (block too large)
227 */
228struct GNUNET_GNSRECORD_Block *
229GNUNET_GNSRECORD_block_create2 (const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
230 struct GNUNET_TIME_Absolute expire,
231 const char *label,
232 const struct GNUNET_GNSRECORD_Data *rd,
233 unsigned int rd_count)
234{
235#define CSIZE 64
236 static struct KeyCacheLine cache[CSIZE];
237 struct KeyCacheLine *line;
238
239 line = &cache[(*(unsigned int *) key) % CSIZE];
240 if (0 != memcmp (&line->key,
241 key,
242 sizeof (*key)))
243 {
244 /* cache miss, recompute */
245 line->key = *key;
246 GNUNET_CRYPTO_ecdsa_key_get_public (key,
247 &line->pkey);
248 }
249 return block_create (key,
250 &line->pkey,
251 expire,
252 label,
253 rd,
254 rd_count);
255}
256
257
258
259/**
159 * Check if a signature is valid. This API is used by the GNS Block 260 * Check if a signature is valid. This API is used by the GNS Block
160 * to validate signatures received from the network. 261 * to validate signatures received from the network.
161 * 262 *
@@ -166,9 +267,9 @@ int
166GNUNET_GNSRECORD_block_verify (const struct GNUNET_GNSRECORD_Block *block) 267GNUNET_GNSRECORD_block_verify (const struct GNUNET_GNSRECORD_Block *block)
167{ 268{
168 return GNUNET_CRYPTO_ecdsa_verify (GNUNET_SIGNATURE_PURPOSE_GNS_RECORD_SIGN, 269 return GNUNET_CRYPTO_ecdsa_verify (GNUNET_SIGNATURE_PURPOSE_GNS_RECORD_SIGN,
169 &block->purpose, 270 &block->purpose,
170 &block->signature, 271 &block->signature,
171 &block->derived_key); 272 &block->derived_key);
172} 273}
173 274
174 275
@@ -224,9 +325,7 @@ GNUNET_GNSRECORD_block_decrypt (const struct GNUNET_GNSRECORD_Block *block,
224 } 325 }
225 { 326 {
226 struct GNUNET_GNSRECORD_Data rd[rd_count]; 327 struct GNUNET_GNSRECORD_Data rd[rd_count];
227 unsigned int i;
228 unsigned int j; 328 unsigned int j;
229 unsigned int k;
230 struct GNUNET_TIME_Absolute now; 329 struct GNUNET_TIME_Absolute now;
231 330
232 if (GNUNET_OK != 331 if (GNUNET_OK !=
@@ -241,7 +340,7 @@ GNUNET_GNSRECORD_block_decrypt (const struct GNUNET_GNSRECORD_Block *block,
241 /* hide expired records */ 340 /* hide expired records */
242 now = GNUNET_TIME_absolute_get (); 341 now = GNUNET_TIME_absolute_get ();
243 j = 0; 342 j = 0;
244 for (i=0;i<rd_count;i++) 343 for (unsigned int i=0;i<rd_count;i++)
245 { 344 {
246 if (0 != (rd[i].flags & GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION)) 345 if (0 != (rd[i].flags & GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION))
247 { 346 {
@@ -254,7 +353,7 @@ GNUNET_GNSRECORD_block_decrypt (const struct GNUNET_GNSRECORD_Block *block,
254 { 353 {
255 int include_record = GNUNET_YES; 354 int include_record = GNUNET_YES;
256 /* Shadow record, figure out if we have a not expired active record */ 355 /* Shadow record, figure out if we have a not expired active record */
257 for (k=0;k<rd_count;k++) 356 for (unsigned int k=0;k<rd_count;k++)
258 { 357 {
259 if (k == i) 358 if (k == i)
260 continue; 359 continue;
@@ -283,7 +382,9 @@ GNUNET_GNSRECORD_block_decrypt (const struct GNUNET_GNSRECORD_Block *block,
283 } 382 }
284 rd_count = j; 383 rd_count = j;
285 if (NULL != proc) 384 if (NULL != proc)
286 proc (proc_cls, rd_count, (0 != rd_count) ? rd : NULL); 385 proc (proc_cls,
386 rd_count,
387 (0 != rd_count) ? rd : NULL);
287 } 388 }
288 } 389 }
289 return GNUNET_OK; 390 return GNUNET_OK;
@@ -304,8 +405,11 @@ GNUNET_GNSRECORD_query_from_private_key (const struct GNUNET_CRYPTO_EcdsaPrivate
304{ 405{
305 struct GNUNET_CRYPTO_EcdsaPublicKey pub; 406 struct GNUNET_CRYPTO_EcdsaPublicKey pub;
306 407
307 GNUNET_CRYPTO_ecdsa_key_get_public (zone, &pub); 408 GNUNET_CRYPTO_ecdsa_key_get_public (zone,
308 GNUNET_GNSRECORD_query_from_public_key (&pub, label, query); 409 &pub);
410 GNUNET_GNSRECORD_query_from_public_key (&pub,
411 label,
412 query);
309} 413}
310 414
311 415
@@ -323,8 +427,13 @@ GNUNET_GNSRECORD_query_from_public_key (const struct GNUNET_CRYPTO_EcdsaPublicKe
323{ 427{
324 struct GNUNET_CRYPTO_EcdsaPublicKey pd; 428 struct GNUNET_CRYPTO_EcdsaPublicKey pd;
325 429
326 GNUNET_CRYPTO_ecdsa_public_key_derive (pub, label, "gns", &pd); 430 GNUNET_CRYPTO_ecdsa_public_key_derive (pub,
327 GNUNET_CRYPTO_hash (&pd, sizeof (pd), query); 431 label,
432 "gns",
433 &pd);
434 GNUNET_CRYPTO_hash (&pd,
435 sizeof (pd),
436 query);
328} 437}
329 438
330 439
diff --git a/src/gnsrecord/gnsrecord_misc.c b/src/gnsrecord/gnsrecord_misc.c
index 3c1ead437..05f56cf70 100644
--- a/src/gnsrecord/gnsrecord_misc.c
+++ b/src/gnsrecord/gnsrecord_misc.c
@@ -152,8 +152,6 @@ struct GNUNET_TIME_Absolute
152GNUNET_GNSRECORD_record_get_expiration_time (unsigned int rd_count, 152GNUNET_GNSRECORD_record_get_expiration_time (unsigned int rd_count,
153 const struct GNUNET_GNSRECORD_Data *rd) 153 const struct GNUNET_GNSRECORD_Data *rd)
154{ 154{
155 unsigned int c;
156 unsigned int c2;
157 struct GNUNET_TIME_Absolute expire; 155 struct GNUNET_TIME_Absolute expire;
158 struct GNUNET_TIME_Absolute at; 156 struct GNUNET_TIME_Absolute at;
159 struct GNUNET_TIME_Relative rt; 157 struct GNUNET_TIME_Relative rt;
@@ -163,7 +161,7 @@ GNUNET_GNSRECORD_record_get_expiration_time (unsigned int rd_count,
163 if (NULL == rd) 161 if (NULL == rd)
164 return GNUNET_TIME_UNIT_ZERO_ABS; 162 return GNUNET_TIME_UNIT_ZERO_ABS;
165 expire = GNUNET_TIME_UNIT_FOREVER_ABS; 163 expire = GNUNET_TIME_UNIT_FOREVER_ABS;
166 for (c = 0; c < rd_count; c++) 164 for (unsigned int c = 0; c < rd_count; c++)
167 { 165 {
168 if (0 != (rd[c].flags & GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION)) 166 if (0 != (rd[c].flags & GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION))
169 { 167 {
@@ -175,12 +173,12 @@ GNUNET_GNSRECORD_record_get_expiration_time (unsigned int rd_count,
175 at.abs_value_us = rd[c].expiration_time; 173 at.abs_value_us = rd[c].expiration_time;
176 } 174 }
177 175
178 for (c2 = 0; c2 < rd_count; c2++) 176 for (unsigned int c2 = 0; c2 < rd_count; c2++)
179 { 177 {
180 /* Check for shadow record */ 178 /* Check for shadow record */
181 if ((c == c2) || 179 if ( (c == c2) ||
182 (rd[c].record_type != rd[c2].record_type) || 180 (rd[c].record_type != rd[c2].record_type) ||
183 (0 == (rd[c2].flags & GNUNET_GNSRECORD_RF_SHADOW_RECORD))) 181 (0 == (rd[c2].flags & GNUNET_GNSRECORD_RF_SHADOW_RECORD)) )
184 continue; 182 continue;
185 /* We have a shadow record */ 183 /* We have a shadow record */
186 if (0 != (rd[c2].flags & GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION)) 184 if (0 != (rd[c2].flags & GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION))
@@ -192,9 +190,11 @@ GNUNET_GNSRECORD_record_get_expiration_time (unsigned int rd_count,
192 { 190 {
193 at_shadow.abs_value_us = rd[c2].expiration_time; 191 at_shadow.abs_value_us = rd[c2].expiration_time;
194 } 192 }
195 at = GNUNET_TIME_absolute_max (at, at_shadow); 193 at = GNUNET_TIME_absolute_max (at,
194 at_shadow);
196 } 195 }
197 expire = GNUNET_TIME_absolute_min (at, expire); 196 expire = GNUNET_TIME_absolute_min (at,
197 expire);
198 } 198 }
199 LOG (GNUNET_ERROR_TYPE_DEBUG, 199 LOG (GNUNET_ERROR_TYPE_DEBUG,
200 "Determined expiration time for block with %u records to be %s\n", 200 "Determined expiration time for block with %u records to be %s\n",
diff --git a/src/gnsrecord/test_gnsrecord_crypto.c b/src/gnsrecord/test_gnsrecord_crypto.c
index 24fe2ad36..1df3f3730 100644
--- a/src/gnsrecord/test_gnsrecord_crypto.c
+++ b/src/gnsrecord/test_gnsrecord_crypto.c
@@ -52,11 +52,10 @@ static int res;
52static struct GNUNET_GNSRECORD_Data * 52static struct GNUNET_GNSRECORD_Data *
53create_record (int count) 53create_record (int count)
54{ 54{
55 unsigned int c;
56 struct GNUNET_GNSRECORD_Data *rd; 55 struct GNUNET_GNSRECORD_Data *rd;
57 56
58 rd = GNUNET_malloc (count * sizeof (struct GNUNET_GNSRECORD_Data)); 57 rd = GNUNET_malloc (count * sizeof (struct GNUNET_GNSRECORD_Data));
59 for (c = 0; c < count; c++) 58 for (unsigned int c = 0; c < count; c++)
60 { 59 {
61 rd[c].expiration_time = GNUNET_TIME_absolute_get().abs_value_us + 1000000000; 60 rd[c].expiration_time = GNUNET_TIME_absolute_get().abs_value_us + 1000000000;
62 rd[c].record_type = TEST_RECORD_TYPE; 61 rd[c].record_type = TEST_RECORD_TYPE;
@@ -74,17 +73,19 @@ rd_decrypt_cb (void *cls,
74 const struct GNUNET_GNSRECORD_Data *rd) 73 const struct GNUNET_GNSRECORD_Data *rd)
75{ 74{
76 char rd_cmp_data[TEST_RECORD_DATALEN]; 75 char rd_cmp_data[TEST_RECORD_DATALEN];
77 int c;
78 76
79 GNUNET_assert (RECORDS == rd_count); 77 GNUNET_assert (RECORDS == rd_count);
80 GNUNET_assert (NULL != rd); 78 GNUNET_assert (NULL != rd);
81 79 memset (rd_cmp_data,
82 memset (rd_cmp_data, 'a', TEST_RECORD_DATALEN); 80 'a',
83 for (c = 0; c < rd_count; c++) 81 TEST_RECORD_DATALEN);
82 for (unsigned int c = 0; c < rd_count; c++)
84 { 83 {
85 GNUNET_assert (TEST_RECORD_TYPE == rd[c].record_type); 84 GNUNET_assert (TEST_RECORD_TYPE == rd[c].record_type);
86 GNUNET_assert (TEST_RECORD_DATALEN == rd[c].data_size); 85 GNUNET_assert (TEST_RECORD_DATALEN == rd[c].data_size);
87 GNUNET_assert (0 == memcmp (&rd_cmp_data, rd[c].data, TEST_RECORD_DATALEN)); 86 GNUNET_assert (0 == memcmp (&rd_cmp_data,
87 rd[c].data,
88 TEST_RECORD_DATALEN));
88 } 89 }
89 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 90 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
90 "Block was decrypted successfully \n"); 91 "Block was decrypted successfully \n");
@@ -93,37 +94,53 @@ rd_decrypt_cb (void *cls,
93} 94}
94 95
95static void 96static void
96run (void *cls, char *const *args, const char *cfgfile, 97run (void *cls,
98 char *const *args,
99 const char *cfgfile,
97 const struct GNUNET_CONFIGURATION_Handle *cfg) 100 const struct GNUNET_CONFIGURATION_Handle *cfg)
98{ 101{
99 struct GNUNET_GNSRECORD_Block *block; 102 struct GNUNET_GNSRECORD_Block *block;
100 struct GNUNET_CRYPTO_EcdsaPublicKey pubkey; 103 struct GNUNET_CRYPTO_EcdsaPublicKey pubkey;
104 struct GNUNET_HashCode query_pub;
105 struct GNUNET_HashCode query_priv;
101 106
102 /* load privat key */ 107 privkey = GNUNET_CRYPTO_ecdsa_key_create ();
103 char *hostkey_file; 108 GNUNET_assert (NULL != privkey);
104 GNUNET_asprintf(&hostkey_file,
105 "zonefiles%s%s",
106 DIR_SEPARATOR_STR,
107 "N0UJMP015AFUNR2BTNM3FKPBLG38913BL8IDMCO2H0A1LIB81960.zkey");
108 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
109 "Using zonekey file `%s'\n",
110 hostkey_file);
111 privkey = GNUNET_CRYPTO_ecdsa_key_create_from_file(hostkey_file);
112 GNUNET_free (hostkey_file);
113 GNUNET_assert (privkey != NULL);
114 struct GNUNET_TIME_Absolute expire = GNUNET_TIME_absolute_get(); 109 struct GNUNET_TIME_Absolute expire = GNUNET_TIME_absolute_get();
115 /* get public key */
116 GNUNET_CRYPTO_ecdsa_key_get_public(privkey, &pubkey);
117 110
111 /* get public key */
112 GNUNET_CRYPTO_ecdsa_key_get_public (privkey,
113 &pubkey);
114
115 /* test query derivation */
116 GNUNET_GNSRECORD_query_from_private_key (privkey,
117 "testlabel",
118 &query_priv);
119 GNUNET_GNSRECORD_query_from_public_key (&pubkey,
120 "testlabel",
121 &query_pub);
122 GNUNET_assert (0 == memcmp (&query_priv,
123 &query_pub,
124 sizeof (struct GNUNET_HashCode)));
118 /* create record */ 125 /* create record */
119 s_name = "DUMMY.dummy.gnunet"; 126 s_name = "DUMMY.dummy.gnunet";
120 s_rd = create_record (RECORDS); 127 s_rd = create_record (RECORDS);
121 128
122 /* Create block */ 129 /* Create block */
123 GNUNET_assert (NULL != (block = GNUNET_GNSRECORD_block_create (privkey, expire,s_name, s_rd, RECORDS))); 130 GNUNET_assert (NULL != (block =
124 GNUNET_assert (GNUNET_OK == GNUNET_GNSRECORD_block_verify (block)); 131 GNUNET_GNSRECORD_block_create (privkey,
125 GNUNET_assert (GNUNET_OK == GNUNET_GNSRECORD_block_decrypt (block, &pubkey, s_name, &rd_decrypt_cb, s_name)); 132 expire,
126 133 s_name,
134 s_rd,
135 RECORDS)));
136 GNUNET_assert (GNUNET_OK ==
137 GNUNET_GNSRECORD_block_verify (block));
138 GNUNET_assert (GNUNET_OK ==
139 GNUNET_GNSRECORD_block_decrypt (block,
140 &pubkey,
141 s_name,
142 &rd_decrypt_cb,
143 s_name));
127 GNUNET_free (block); 144 GNUNET_free (block);
128} 145}
129 146
@@ -131,7 +148,8 @@ run (void *cls, char *const *args, const char *cfgfile,
131int 148int
132main (int argc, char *argv[]) 149main (int argc, char *argv[])
133{ 150{
134 static char *const argvx[] = { "test-gnsrecord-crypto", 151 static char *const argvx[] = {
152 "test-gnsrecord-crypto",
135 NULL 153 NULL
136 }; 154 };
137 static struct GNUNET_GETOPT_CommandLineOption options[] = { 155 static struct GNUNET_GETOPT_CommandLineOption options[] = {
@@ -139,8 +157,11 @@ main (int argc, char *argv[])
139 }; 157 };
140 158
141 res = 1; 159 res = 1;
142 GNUNET_PROGRAM_run ((sizeof (argvx) / sizeof (char *)) - 1, argvx, "test-namestore-api", 160 GNUNET_PROGRAM_run ((sizeof (argvx) / sizeof (char *)) - 1,
143 "nohelp", options, &run, &res); 161 argvx,
162 "test-gnsrecord-crypto",
163 "nohelp", options,
164 &run, &res);
144 return res; 165 return res;
145} 166}
146 167
diff --git a/src/identity-provider/gnunet-service-identity-provider.c b/src/identity-provider/gnunet-service-identity-provider.c
index 572202c3f..6afb7bb05 100644
--- a/src/identity-provider/gnunet-service-identity-provider.c
+++ b/src/identity-provider/gnunet-service-identity-provider.c
@@ -2190,12 +2190,15 @@ attr_iter_cb (void *cls,
2190 2190
2191 if (rd_count != 1) 2191 if (rd_count != 1)
2192 { 2192 {
2193 GNUNET_NAMESTORE_zone_iterator_next (ai->ns_it); 2193 GNUNET_NAMESTORE_zone_iterator_next (ai->ns_it,
2194 1);
2194 return; 2195 return;
2195 } 2196 }
2196 2197
2197 if (GNUNET_GNSRECORD_TYPE_ID_ATTR != rd->record_type) { 2198 if (GNUNET_GNSRECORD_TYPE_ID_ATTR != rd->record_type)
2198 GNUNET_NAMESTORE_zone_iterator_next (ai->ns_it); 2199 {
2200 GNUNET_NAMESTORE_zone_iterator_next (ai->ns_it,
2201 1);
2199 return; 2202 return;
2200 } 2203 }
2201 attr_ver = ntohl(*((uint32_t*)rd->data)); 2204 attr_ver = ntohl(*((uint32_t*)rd->data));
@@ -2209,8 +2212,10 @@ attr_iter_cb (void *cls,
2209 rd->data_size-sizeof (uint32_t), 2212 rd->data_size-sizeof (uint32_t),
2210 key, 2213 key,
2211 (void**)&attr_ser); 2214 (void**)&attr_ser);
2212 if (GNUNET_SYSERR == msg_extra_len) { 2215 if (GNUNET_SYSERR == msg_extra_len)
2213 GNUNET_NAMESTORE_zone_iterator_next (ai->ns_it); 2216 {
2217 GNUNET_NAMESTORE_zone_iterator_next (ai->ns_it,
2218 1);
2214 return; 2219 return;
2215 } 2220 }
2216 2221
@@ -2253,13 +2258,15 @@ iterate_after_abe_bootstrap (void *cls,
2253 ai); 2258 ai);
2254} 2259}
2255 2260
2256void 2261
2262static void
2257iterate_next_after_abe_bootstrap (void *cls, 2263iterate_next_after_abe_bootstrap (void *cls,
2258 struct GNUNET_ABE_AbeMasterKey *abe_key) 2264 struct GNUNET_ABE_AbeMasterKey *abe_key)
2259{ 2265{
2260 struct AttributeIterator *ai = cls; 2266 struct AttributeIterator *ai = cls;
2261 ai->abe_key = abe_key; 2267 ai->abe_key = abe_key;
2262 GNUNET_NAMESTORE_zone_iterator_next (ai->ns_it); 2268 GNUNET_NAMESTORE_zone_iterator_next (ai->ns_it,
2269 1);
2263} 2270}
2264 2271
2265 2272
diff --git a/src/identity-provider/plugin_rest_openid_connect.c b/src/identity-provider/plugin_rest_openid_connect.c
index 7e8054e60..125bb98f6 100644
--- a/src/identity-provider/plugin_rest_openid_connect.c
+++ b/src/identity-provider/plugin_rest_openid_connect.c
@@ -1076,16 +1076,19 @@ namestore_iteration_callback (
1076 } 1076 }
1077 } 1077 }
1078 1078
1079 GNUNET_NAMESTORE_zone_iterator_next (handle->namestore_handle_it); 1079 GNUNET_NAMESTORE_zone_iterator_next (handle->namestore_handle_it,
1080 1);
1080} 1081}
1081 1082
1083
1082/** 1084/**
1083 * Iteration over all results finished, build final 1085 * Iteration over all results finished, build final
1084 * response. 1086 * response.
1085 * 1087 *
1086 * @param cls the `struct RequestHandle` 1088 * @param cls the `struct RequestHandle`
1087 */ 1089 */
1088static void namestore_iteration_finished (void *cls) 1090static void
1091namestore_iteration_finished (void *cls)
1089{ 1092{
1090 struct RequestHandle *handle = cls; 1093 struct RequestHandle *handle = cls;
1091 struct GNUNET_HashCode cache_key; 1094 struct GNUNET_HashCode cache_key;
diff --git a/src/include/gnunet_dht_service.h b/src/include/gnunet_dht_service.h
index d798482e3..a4c63889e 100644
--- a/src/include/gnunet_dht_service.h
+++ b/src/include/gnunet_dht_service.h
@@ -142,22 +142,6 @@ struct GNUNET_DHT_PutHandle;
142 142
143 143
144/** 144/**
145 * Type of a PUT continuation. You must not call
146 * #GNUNET_DHT_disconnect in this continuation.
147 *
148 * @param cls closure
149 * @param success #GNUNET_OK if the PUT was transmitted,
150 * #GNUNET_NO on timeout,
151 * #GNUNET_SYSERR on disconnect from service
152 * after the PUT message was transmitted
153 * (so we don't know if it was received or not)
154 */
155typedef void
156(*GNUNET_DHT_PutContinuation)(void *cls,
157 int success);
158
159
160/**
161 * Perform a PUT operation storing data in the DHT. 145 * Perform a PUT operation storing data in the DHT.
162 * 146 *
163 * @param handle handle to DHT service 147 * @param handle handle to DHT service
@@ -184,7 +168,7 @@ GNUNET_DHT_put (struct GNUNET_DHT_Handle *handle,
184 size_t size, 168 size_t size,
185 const void *data, 169 const void *data,
186 struct GNUNET_TIME_Absolute exp, 170 struct GNUNET_TIME_Absolute exp,
187 GNUNET_DHT_PutContinuation cont, 171 GNUNET_SCHEDULER_TaskCallback cont,
188 void *cont_cls); 172 void *cont_cls);
189 173
190 174
diff --git a/src/include/gnunet_dnsstub_lib.h b/src/include/gnunet_dnsstub_lib.h
index 1c3305f7b..41e30d044 100644
--- a/src/include/gnunet_dnsstub_lib.h
+++ b/src/include/gnunet_dnsstub_lib.h
@@ -1,6 +1,6 @@
1/* 1/*
2 This file is part of GNUnet 2 This file is part of GNUnet
3 Copyright (C) 2012 GNUnet e.V. 3 Copyright (C) 2012, 2018 GNUnet e.V.
4 4
5 GNUnet is free software; you can redistribute it and/or modify 5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published 6 it under the terms of the GNU General Public License as published
@@ -48,14 +48,52 @@ struct GNUNET_DNSSTUB_RequestSocket;
48/** 48/**
49 * Start a DNS stub resolver. 49 * Start a DNS stub resolver.
50 * 50 *
51 * @param dns_ip target IP address to use 51 * @param num_sockets how many sockets should we open
52 * in parallel for DNS queries for this stub?
52 * @return NULL on error 53 * @return NULL on error
53 */ 54 */
54struct GNUNET_DNSSTUB_Context * 55struct GNUNET_DNSSTUB_Context *
55GNUNET_DNSSTUB_start (const char *dns_ip); 56GNUNET_DNSSTUB_start (unsigned int num_sockets);
56 57
57 58
58/** 59/**
60 * Add nameserver for use by the DNSSTUB. We will use
61 * all provided nameservers for resolution (round-robin).
62 *
63 * @param ctx resolver context to modify
64 * @param dns_ip target IP address to use (as string)
65 * @return #GNUNET_OK on success
66 */
67int
68GNUNET_DNSSTUB_add_dns_ip (struct GNUNET_DNSSTUB_Context *ctx,
69 const char *dns_ip);
70
71
72/**
73 * Add nameserver for use by the DNSSTUB. We will use
74 * all provided nameservers for resolution (round-robin).
75 *
76 * @param ctx resolver context to modify
77 * @param sa socket address of DNS resolver to use
78 * @return #GNUNET_OK on success
79 */
80int
81GNUNET_DNSSTUB_add_dns_sa (struct GNUNET_DNSSTUB_Context *ctx,
82 const struct sockaddr *sa);
83
84
85/**
86 * How long should we try requests before timing out?
87 * Only effective for requests issued after this call.
88 *
89 * @param ctx resolver context to modify
90 * @param retry_frequ how long to wait between retries
91 */
92void
93GNUNET_DNSSTUB_set_retry (struct GNUNET_DNSSTUB_Context *ctx,
94 struct GNUNET_TIME_Relative retry_freq);
95
96/**
59 * Cleanup DNSSTUB resolver. 97 * Cleanup DNSSTUB resolver.
60 * 98 *
61 * @param ctx stub resolver to clean up 99 * @param ctx stub resolver to clean up
@@ -66,57 +104,36 @@ GNUNET_DNSSTUB_stop (struct GNUNET_DNSSTUB_Context *ctx);
66 104
67/** 105/**
68 * Function called with the result of a DNS resolution. 106 * Function called with the result of a DNS resolution.
107 * Once this function is called, the resolution request
108 * is automatically cancelled / cleaned up. In particular,
109 * the function will only be called once.
69 * 110 *
70 * @param cls closure 111 * @param cls closure
71 * @param rs socket that received the response 112 * @param dns dns response, NULL on hard error (i.e. timeout)
72 * @param dns dns response, never NULL
73 * @param dns_len number of bytes in @a dns 113 * @param dns_len number of bytes in @a dns
74 */ 114 */
75typedef void 115typedef void
76(*GNUNET_DNSSTUB_ResultCallback)(void *cls, 116(*GNUNET_DNSSTUB_ResultCallback)(void *cls,
77 struct GNUNET_DNSSTUB_RequestSocket *rs,
78 const struct GNUNET_TUN_DnsHeader *dns, 117 const struct GNUNET_TUN_DnsHeader *dns,
79 size_t dns_len); 118 size_t dns_len);
80 119
81 120
82/** 121/**
83 * Perform DNS resolution using given address.
84 *
85 * @param ctx stub resolver to use
86 * @param sa the socket address
87 * @param sa_len the socket length
88 * @param request DNS request to transmit
89 * @param request_len number of bytes in msg
90 * @param rc function to call with result
91 * @param rc_cls closure for @a rc
92 * @return socket used for the request, NULL on error
93 */
94struct GNUNET_DNSSTUB_RequestSocket *
95GNUNET_DNSSTUB_resolve (struct GNUNET_DNSSTUB_Context *ctx,
96 const struct sockaddr *sa,
97 socklen_t sa_len,
98 const void *request,
99 size_t request_len,
100 GNUNET_DNSSTUB_ResultCallback rc,
101 void *rc_cls);
102
103
104/**
105 * Perform DNS resolution using our default IP from init. 122 * Perform DNS resolution using our default IP from init.
106 * 123 *
107 * @param ctx stub resolver to use 124 * @param ctx stub resolver to use
108 * @param request DNS request to transmit 125 * @param request DNS request to transmit
109 * @param request_len number of bytes in msg 126 * @param request_len number of bytes in msg
110 * @param rc function to call with result 127 * @param rc function to call with result (once)
111 * @param rc_cls closure for @a rc 128 * @param rc_cls closure for @a rc
112 * @return socket used for the request, NULL on error 129 * @return socket used for the request, NULL on error
113 */ 130 */
114struct GNUNET_DNSSTUB_RequestSocket * 131struct GNUNET_DNSSTUB_RequestSocket *
115GNUNET_DNSSTUB_resolve2 (struct GNUNET_DNSSTUB_Context *ctx, 132GNUNET_DNSSTUB_resolve (struct GNUNET_DNSSTUB_Context *ctx,
116 const void *request, 133 const void *request,
117 size_t request_len, 134 size_t request_len,
118 GNUNET_DNSSTUB_ResultCallback rc, 135 GNUNET_DNSSTUB_ResultCallback rc,
119 void *rc_cls); 136 void *rc_cls);
120 137
121 138
122/** 139/**
diff --git a/src/include/gnunet_gns_service.h b/src/include/gnunet_gns_service.h
index 8c1f64783..ff3110406 100644
--- a/src/include/gnunet_gns_service.h
+++ b/src/include/gnunet_gns_service.h
@@ -48,13 +48,10 @@ extern "C"
48 48
49 49
50/** 50/**
51 * String we use to indicate the local master zone or a 51 * String we use to indicate an empty label (top-level
52 * root entry in the current zone. 52 * entry in the zone). DNS uses "@", so do we.
53 *
54 * FIXME: probably should be changed to "@" and renamed
55 * (this name is confusing!)
56 */ 53 */
57#define GNUNET_GNS_MASTERZONE_STR "+" 54#define GNUNET_GNS_EMPTY_LABEL_AT "@"
58 55
59/** 56/**
60 * Connection to the GNS service. 57 * Connection to the GNS service.
diff --git a/src/include/gnunet_gnsrecord_lib.h b/src/include/gnunet_gnsrecord_lib.h
index d03b4db3b..38346ada3 100644
--- a/src/include/gnunet_gnsrecord_lib.h
+++ b/src/include/gnunet_gnsrecord_lib.h
@@ -342,9 +342,10 @@ GNUNET_NETWORK_STRUCT_END
342 * @param rd_count number of entries in @a rd array 342 * @param rd_count number of entries in @a rd array
343 * @param rd array of records with data to store 343 * @param rd array of records with data to store
344 */ 344 */
345typedef void (*GNUNET_GNSRECORD_RecordCallback) (void *cls, 345typedef void
346 unsigned int rd_count, 346(*GNUNET_GNSRECORD_RecordCallback) (void *cls,
347 const struct GNUNET_GNSRECORD_Data *rd); 347 unsigned int rd_count,
348 const struct GNUNET_GNSRECORD_Data *rd);
348 349
349 350
350 351
@@ -557,6 +558,25 @@ GNUNET_GNSRECORD_block_create (const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
557 558
558 559
559/** 560/**
561 * Sign name and records, cache derived public key (also keeps the
562 * private key in static memory, so do not use this function if
563 * keeping the private key in the process'es RAM is a major issue).
564 *
565 * @param key the private key
566 * @param expire block expiration
567 * @param label the name for the records
568 * @param rd record data
569 * @param rd_count number of records in @a rd
570 */
571struct GNUNET_GNSRECORD_Block *
572GNUNET_GNSRECORD_block_create2 (const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
573 struct GNUNET_TIME_Absolute expire,
574 const char *label,
575 const struct GNUNET_GNSRECORD_Data *rd,
576 unsigned int rd_count);
577
578
579/**
560 * Check if a signature is valid. This API is used by the GNS Block 580 * Check if a signature is valid. This API is used by the GNS Block
561 * to validate signatures received from the network. 581 * to validate signatures received from the network.
562 * 582 *
diff --git a/src/include/gnunet_namestore_plugin.h b/src/include/gnunet_namestore_plugin.h
index 3ebf48987..e15068222 100644
--- a/src/include/gnunet_namestore_plugin.h
+++ b/src/include/gnunet_namestore_plugin.h
@@ -1,6 +1,6 @@
1/* 1/*
2 This file is part of GNUnet 2 This file is part of GNUnet
3 Copyright (C) 2012, 2013 GNUnet e.V. 3 Copyright (C) 2012, 2013, 2018 GNUnet e.V.
4 4
5 GNUnet is free software; you can redistribute it and/or modify 5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published 6 it under the terms of the GNU General Public License as published
@@ -47,6 +47,7 @@ extern "C"
47 * Function called for each matching record. 47 * Function called for each matching record.
48 * 48 *
49 * @param cls closure 49 * @param cls closure
50 * @param serial unique serial number of the record
50 * @param zone_key private key of the zone 51 * @param zone_key private key of the zone
51 * @param label name that is being mapped (at most 255 characters long) 52 * @param label name that is being mapped (at most 255 characters long)
52 * @param rd_count number of entries in @a rd array 53 * @param rd_count number of entries in @a rd array
@@ -54,6 +55,7 @@ extern "C"
54 */ 55 */
55typedef void 56typedef void
56(*GNUNET_NAMESTORE_RecordIterator) (void *cls, 57(*GNUNET_NAMESTORE_RecordIterator) (void *cls,
58 uint64_t serial,
57 const struct GNUNET_CRYPTO_EcdsaPrivateKey *private_key, 59 const struct GNUNET_CRYPTO_EcdsaPrivateKey *private_key,
58 const char *label, 60 const char *label,
59 unsigned int rd_count, 61 unsigned int rd_count,
@@ -97,7 +99,7 @@ struct GNUNET_NAMESTORE_PluginFunctions
97 * @param label name of the record in the zone 99 * @param label name of the record in the zone
98 * @param iter function to call with the result 100 * @param iter function to call with the result
99 * @param iter_cls closure for @a iter 101 * @param iter_cls closure for @a iter
100 * @return #GNUNET_OK on success, else #GNUNET_SYSERR 102 * @return #GNUNET_OK on success, #GNUNET_NO for no results, else #GNUNET_SYSERR
101 */ 103 */
102 int 104 int
103 (*lookup_records) (void *cls, 105 (*lookup_records) (void *cls,
@@ -109,19 +111,21 @@ struct GNUNET_NAMESTORE_PluginFunctions
109 111
110 /** 112 /**
111 * Iterate over the results for a particular zone in the 113 * Iterate over the results for a particular zone in the
112 * datastore. Will return at most one result to the iterator. 114 * datastore. Will return at most @a limit results to the iterator.
113 * 115 *
114 * @param cls closure (internal context for the plugin) 116 * @param cls closure (internal context for the plugin)
115 * @param zone private key of the zone, NULL for all zones 117 * @param zone private key of the zone, NULL for all zones
116 * @param offset offset in the list of all matching records 118 * @param serial serial (to exclude) in the list of matching records
119 * @param limit maximum number of results to return to @a iter
117 * @param iter function to call with the result 120 * @param iter function to call with the result
118 * @param iter_cls closure for @a iter 121 * @param iter_cls closure for @a iter
119 * @return #GNUNET_OK on success, #GNUNET_NO if there were no results, #GNUNET_SYSERR on error 122 * @return #GNUNET_OK on success, #GNUNET_NO if there were no more results, #GNUNET_SYSERR on error
120 */ 123 */
121 int 124 int
122 (*iterate_records) (void *cls, 125 (*iterate_records) (void *cls,
123 const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone, 126 const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone,
124 uint64_t offset, 127 uint64_t serial,
128 uint64_t limit,
125 GNUNET_NAMESTORE_RecordIterator iter, 129 GNUNET_NAMESTORE_RecordIterator iter,
126 void *iter_cls); 130 void *iter_cls);
127 131
diff --git a/src/include/gnunet_namestore_service.h b/src/include/gnunet_namestore_service.h
index f8c2eaf3b..4828f72ad 100644
--- a/src/include/gnunet_namestore_service.h
+++ b/src/include/gnunet_namestore_service.h
@@ -267,9 +267,12 @@ GNUNET_NAMESTORE_zone_iteration_start (struct GNUNET_NAMESTORE_Handle *h,
267 * for the next record. 267 * for the next record.
268 * 268 *
269 * @param it the iterator 269 * @param it the iterator
270 * @param limit number of records to return to the iterator in one shot
271 * (before #GNUNET_NAMESTORE_zone_iterator_next is to be called again)
270 */ 272 */
271void 273void
272GNUNET_NAMESTORE_zone_iterator_next (struct GNUNET_NAMESTORE_ZoneIterator *it); 274GNUNET_NAMESTORE_zone_iterator_next (struct GNUNET_NAMESTORE_ZoneIterator *it,
275 uint64_t limit);
273 276
274 277
275/** 278/**
diff --git a/src/include/gnunet_protocols.h b/src/include/gnunet_protocols.h
index d692b28ff..bf1b48679 100644
--- a/src/include/gnunet_protocols.h
+++ b/src/include/gnunet_protocols.h
@@ -624,11 +624,6 @@ extern "C"
624#define GNUNET_MESSAGE_TYPE_DHT_MONITOR_STOP 154 624#define GNUNET_MESSAGE_TYPE_DHT_MONITOR_STOP 154
625 625
626/** 626/**
627 * Acknowledge receiving PUT request
628 */
629#define GNUNET_MESSAGE_TYPE_DHT_CLIENT_PUT_OK 155
630
631/**
632 * Certain results are already known to the client, filter those. 627 * Certain results are already known to the client, filter those.
633 */ 628 */
634#define GNUNET_MESSAGE_TYPE_DHT_CLIENT_GET_RESULTS_KNOWN 156 629#define GNUNET_MESSAGE_TYPE_DHT_CLIENT_GET_RESULTS_KNOWN 156
diff --git a/src/namecache/namecache.conf.in b/src/namecache/namecache.conf.in
index 236cafecd..becd34187 100644
--- a/src/namecache/namecache.conf.in
+++ b/src/namecache/namecache.conf.in
@@ -11,6 +11,11 @@ ACCEPT_FROM = 127.0.0.1;
11ACCEPT_FROM6 = ::1; 11ACCEPT_FROM6 = ::1;
12DATABASE = sqlite 12DATABASE = sqlite
13 13
14# Disables use of caching by GNS. Useful for systems that
15# publish very large zones and are CPU bound, if they do not
16# also do a large number of lookups.
17DISABLE = NO
18
14[namecache-sqlite] 19[namecache-sqlite]
15FILENAME = $GNUNET_DATA_HOME/namecache/sqlite.db 20FILENAME = $GNUNET_DATA_HOME/namecache/sqlite.db
16 21
diff --git a/src/namecache/plugin_namecache_sqlite.c b/src/namecache/plugin_namecache_sqlite.c
index 37d6d3b62..e0f64a6b0 100644
--- a/src/namecache/plugin_namecache_sqlite.c
+++ b/src/namecache/plugin_namecache_sqlite.c
@@ -136,13 +136,16 @@ create_indices (sqlite3 * dbh)
136{ 136{
137 /* create indices */ 137 /* create indices */
138 if ( (SQLITE_OK != 138 if ( (SQLITE_OK !=
139 sqlite3_exec (dbh, "CREATE INDEX IF NOT EXISTS ir_query_hash ON ns096blocks (query,expiration_time)", 139 sqlite3_exec (dbh,
140 "CREATE INDEX IF NOT EXISTS ir_query_hash ON ns096blocks (query,expiration_time)",
140 NULL, NULL, NULL)) || 141 NULL, NULL, NULL)) ||
141 (SQLITE_OK != 142 (SQLITE_OK !=
142 sqlite3_exec (dbh, "CREATE INDEX IF NOT EXISTS ir_block_expiration ON ns096blocks (expiration_time)", 143 sqlite3_exec (dbh,
144 "CREATE INDEX IF NOT EXISTS ir_block_expiration ON ns096blocks (expiration_time)",
143 NULL, NULL, NULL)) ) 145 NULL, NULL, NULL)) )
144 LOG (GNUNET_ERROR_TYPE_ERROR, 146 LOG (GNUNET_ERROR_TYPE_ERROR,
145 "Failed to create indices: %s\n", sqlite3_errmsg (dbh)); 147 "Failed to create indices: %s\n",
148 sqlite3_errmsg (dbh));
146} 149}
147 150
148 151
@@ -208,28 +211,44 @@ database_setup (struct Plugin *plugin)
208 return GNUNET_SYSERR; 211 return GNUNET_SYSERR;
209 } 212 }
210 CHECK (SQLITE_OK == 213 CHECK (SQLITE_OK ==
211 sqlite3_exec (plugin->dbh, "PRAGMA temp_store=MEMORY", NULL, NULL, 214 sqlite3_exec (plugin->dbh,
215 "PRAGMA temp_store=MEMORY",
216 NULL, NULL,
212 ENULL)); 217 ENULL));
213 CHECK (SQLITE_OK == 218 CHECK (SQLITE_OK ==
214 sqlite3_exec (plugin->dbh, "PRAGMA synchronous=NORMAL", NULL, NULL, 219 sqlite3_exec (plugin->dbh,
220 "PRAGMA synchronous=NORMAL",
221 NULL, NULL,
215 ENULL)); 222 ENULL));
216 CHECK (SQLITE_OK == 223 CHECK (SQLITE_OK ==
217 sqlite3_exec (plugin->dbh, "PRAGMA legacy_file_format=OFF", NULL, NULL, 224 sqlite3_exec (plugin->dbh,
225 "PRAGMA legacy_file_format=OFF",
226 NULL, NULL,
218 ENULL)); 227 ENULL));
219 CHECK (SQLITE_OK == 228 CHECK (SQLITE_OK ==
220 sqlite3_exec (plugin->dbh, "PRAGMA auto_vacuum=INCREMENTAL", NULL, 229 sqlite3_exec (plugin->dbh,
221 NULL, ENULL)); 230 "PRAGMA auto_vacuum=INCREMENTAL",
231 NULL, NULL,
232 ENULL));
222 CHECK (SQLITE_OK == 233 CHECK (SQLITE_OK ==
223 sqlite3_exec (plugin->dbh, "PRAGMA encoding=\"UTF-8\"", NULL, 234 sqlite3_exec (plugin->dbh,
224 NULL, ENULL)); 235 "PRAGMA encoding=\"UTF-8\"",
236 NULL, NULL,
237 ENULL));
225 CHECK (SQLITE_OK == 238 CHECK (SQLITE_OK ==
226 sqlite3_exec (plugin->dbh, "PRAGMA locking_mode=EXCLUSIVE", NULL, NULL, 239 sqlite3_exec (plugin->dbh,
240 "PRAGMA locking_mode=EXCLUSIVE",
241 NULL, NULL,
227 ENULL)); 242 ENULL));
228 CHECK (SQLITE_OK == 243 CHECK (SQLITE_OK ==
229 sqlite3_exec (plugin->dbh, "PRAGMA page_size=4092", NULL, NULL, 244 sqlite3_exec (plugin->dbh,
245 "PRAGMA page_size=4092",
246 NULL, NULL,
230 ENULL)); 247 ENULL));
231 248
232 CHECK (SQLITE_OK == sqlite3_busy_timeout (plugin->dbh, BUSY_TIMEOUT_MS)); 249 CHECK (SQLITE_OK ==
250 sqlite3_busy_timeout (plugin->dbh,
251 BUSY_TIMEOUT_MS));
233 252
234 253
235 /* Create tables */ 254 /* Create tables */
@@ -237,17 +256,19 @@ database_setup (struct Plugin *plugin)
237 sq_prepare (plugin->dbh, 256 sq_prepare (plugin->dbh,
238 "SELECT 1 FROM sqlite_master WHERE tbl_name = 'ns096blocks'", 257 "SELECT 1 FROM sqlite_master WHERE tbl_name = 'ns096blocks'",
239 &stmt)); 258 &stmt));
240 if ((sqlite3_step (stmt) == SQLITE_DONE) && 259 if ( (sqlite3_step (stmt) == SQLITE_DONE) &&
241 (sqlite3_exec 260 (SQLITE_OK !=
242 (plugin->dbh, 261 sqlite3_exec (plugin->dbh,
243 "CREATE TABLE ns096blocks (" 262 "CREATE TABLE ns096blocks ("
244 " query BLOB NOT NULL," 263 " query BLOB NOT NULL,"
245 " block BLOB NOT NULL," 264 " block BLOB NOT NULL,"
246 " expiration_time INT8 NOT NULL" 265 " expiration_time INT8 NOT NULL"
247 ")", 266 ")",
248 NULL, NULL, NULL) != SQLITE_OK)) 267 NULL, NULL, NULL)) )
249 { 268 {
250 LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "sqlite3_exec"); 269 LOG_SQLITE (plugin,
270 GNUNET_ERROR_TYPE_ERROR,
271 "sqlite3_exec");
251 sqlite3_finalize (stmt); 272 sqlite3_finalize (stmt);
252 return GNUNET_SYSERR; 273 return GNUNET_SYSERR;
253 } 274 }
@@ -411,11 +432,11 @@ namecache_sqlite_cache_block (void *cls,
411 GNUNET_CRYPTO_hash (&block->derived_key, 432 GNUNET_CRYPTO_hash (&block->derived_key,
412 sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey), 433 sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey),
413 &query); 434 &query);
414 fprintf (stderr,
415 "Caching new version of block %s (expires %llu)\n",
416 GNUNET_h2s (&query),
417 (unsigned long long) expiration.abs_value_us);
418 expiration = GNUNET_TIME_absolute_ntoh (block->expiration_time); 435 expiration = GNUNET_TIME_absolute_ntoh (block->expiration_time);
436 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
437 "Caching new version of block %s (expires %s)\n",
438 GNUNET_h2s (&query),
439 GNUNET_STRINGS_absolute_time_to_string (expiration));
419 if (block_size > 64 * 65536) 440 if (block_size > 64 * 65536)
420 { 441 {
421 GNUNET_break (0); 442 GNUNET_break (0);
diff --git a/src/namestore/.gitignore b/src/namestore/.gitignore
index 9a4c615ba..e56271eac 100644
--- a/src/namestore/.gitignore
+++ b/src/namestore/.gitignore
@@ -20,3 +20,51 @@ test_plugin_namestore_postgres
20test_plugin_namestore_sqlite 20test_plugin_namestore_sqlite
21test_plugin_namestore_flat 21test_plugin_namestore_flat
22gnunet-zoneimport 22gnunet-zoneimport
23test_namestore_api_lookup_nick_flat
24test_namestore_api_lookup_nick_postgres
25test_namestore_api_lookup_nick_sqlite
26test_namestore_api_lookup_private_flat
27test_namestore_api_lookup_private_postgres
28test_namestore_api_lookup_private_sqlite
29test_namestore_api_lookup_public_flat
30test_namestore_api_lookup_public_postgres
31test_namestore_api_lookup_public_sqlite
32test_namestore_api_lookup_shadow_filter_flat
33test_namestore_api_lookup_shadow_filter_postgres
34test_namestore_api_lookup_shadow_filter_sqlite
35test_namestore_api_lookup_shadow_flat
36test_namestore_api_lookup_shadow_postgres
37test_namestore_api_lookup_shadow_sqlite
38test_namestore_api_monitoring_existing_flat
39test_namestore_api_monitoring_existing_postgres
40test_namestore_api_monitoring_existing_sqlite
41test_namestore_api_monitoring_flat
42test_namestore_api_monitoring_postgres
43test_namestore_api_monitoring_sqlite
44test_namestore_api_remove_flat
45test_namestore_api_remove_not_existing_record_flat
46test_namestore_api_remove_not_existing_record_postgres
47test_namestore_api_remove_not_existing_record_sqlite
48test_namestore_api_remove_postgres
49test_namestore_api_remove_sqlite
50test_namestore_api_store_flat
51test_namestore_api_store_postgres
52test_namestore_api_store_sqlite
53test_namestore_api_store_update_flat
54test_namestore_api_store_update_postgres
55test_namestore_api_store_update_sqlite
56test_namestore_api_zone_iteration_flat
57test_namestore_api_zone_iteration_nick_flat
58test_namestore_api_zone_iteration_nick_postgres
59test_namestore_api_zone_iteration_nick_sqlite
60test_namestore_api_zone_iteration_postgres
61test_namestore_api_zone_iteration_specific_zone_flat
62test_namestore_api_zone_iteration_specific_zone_postgres
63test_namestore_api_zone_iteration_specific_zone_sqlite
64test_namestore_api_zone_iteration_sqlite
65test_namestore_api_zone_iteration_stop_flat
66test_namestore_api_zone_iteration_stop_postgres
67test_namestore_api_zone_iteration_stop_sqlite
68test_namestore_api_zone_to_name_flat
69test_namestore_api_zone_to_name_postgres
70test_namestore_api_zone_to_name_sqlite
diff --git a/src/namestore/Makefile.am b/src/namestore/Makefile.am
index fd8f8054f..fa85cc060 100644
--- a/src/namestore/Makefile.am
+++ b/src/namestore/Makefile.am
@@ -20,74 +20,81 @@ if USE_COVERAGE
20 XLIBS = -lgcov 20 XLIBS = -lgcov
21endif 21endif
22 22
23if HAVE_EXPERIMENTAL
24FLAT_PLUGIN = libgnunet_plugin_namestore_flat.la 23FLAT_PLUGIN = libgnunet_plugin_namestore_flat.la
25if HAVE_TESTING 24if HAVE_TESTING
26FLAT_TESTS = test_plugin_namestore_flat 25FLAT_TESTS = test_plugin_namestore_flat \
27endif 26 test_namestore_api_store_flat \
27 test_namestore_api_store_update_flat \
28 test_namestore_api_remove_flat \
29 test_namestore_api_zone_iteration_flat \
30 test_namestore_api_lookup_nick_flat \
31 test_namestore_api_monitoring_flat \
32 test_namestore_api_lookup_public_flat \
33 test_namestore_api_lookup_private_flat \
34 test_namestore_api_lookup_shadow_flat \
35 test_namestore_api_lookup_shadow_filter_flat \
36 test_namestore_api_remove_not_existing_record_flat \
37 test_namestore_api_zone_iteration_nick_flat \
38 test_namestore_api_zone_iteration_specific_zone_flat \
39 test_namestore_api_zone_iteration_stop_flat \
40 test_namestore_api_monitoring_existing_flat \
41 test_namestore_api_zone_to_name_flat \
42 perf_namestore_api_zone_iteration_flat
28endif 43endif
29 44
30if HAVE_SQLITE 45if HAVE_SQLITE
31SQLITE_PLUGIN = libgnunet_plugin_namestore_sqlite.la 46SQLITE_PLUGIN = libgnunet_plugin_namestore_sqlite.la
32if HAVE_TESTING 47if HAVE_TESTING
33SQLITE_TESTS = test_plugin_namestore_sqlite 48SQLITE_TESTS = test_plugin_namestore_sqlite \
49 test_namestore_api_store_sqlite \
50 test_namestore_api_store_update_sqlite \
51 test_namestore_api_zone_iteration_sqlite \
52 test_namestore_api_remove_sqlite \
53 test_namestore_api_lookup_nick_sqlite \
54 test_namestore_api_monitoring_sqlite \
55 test_namestore_api_lookup_public_sqlite \
56 test_namestore_api_lookup_private_sqlite \
57 test_namestore_api_lookup_shadow_sqlite \
58 test_namestore_api_lookup_shadow_filter_sqlite \
59 test_namestore_api_remove_not_existing_record_sqlite \
60 test_namestore_api_zone_iteration_nick_sqlite \
61 test_namestore_api_zone_iteration_specific_zone_sqlite \
62 test_namestore_api_zone_iteration_stop_sqlite \
63 test_namestore_api_monitoring_existing_sqlite \
64 test_namestore_api_zone_to_name_sqlite \
65 perf_namestore_api_zone_iteration_sqlite
34endif 66endif
35endif 67endif
36 68
37if HAVE_POSTGRESQL 69if HAVE_POSTGRESQL
38# postgres doesn't even build yet; thus: experimental!
39POSTGRES_PLUGIN = libgnunet_plugin_namestore_postgres.la 70POSTGRES_PLUGIN = libgnunet_plugin_namestore_postgres.la
40if HAVE_TESTING 71if HAVE_TESTING
41POSTGRES_TESTS = test_plugin_namestore_postgres 72POSTGRES_TESTS = test_plugin_namestore_postgres \
42endif 73 test_namestore_api_store_postgres \
74 test_namestore_api_store_update_postgres \
75 test_namestore_api_remove_postgres \
76 test_namestore_api_zone_iteration_postgres \
77 test_namestore_api_lookup_nick_postgres \
78 test_namestore_api_monitoring_postgres \
79 test_namestore_api_lookup_public_postgres \
80 test_namestore_api_lookup_private_postgres \
81 test_namestore_api_lookup_shadow_postgres \
82 test_namestore_api_lookup_shadow_filter_postgres \
83 test_namestore_api_remove_not_existing_record_postgres \
84 test_namestore_api_zone_iteration_nick_postgres \
85 test_namestore_api_zone_iteration_specific_zone_postgres \
86 test_namestore_api_zone_iteration_stop_postgres \
87 test_namestore_api_monitoring_existing_postgres \
88 test_namestore_api_zone_to_name_postgres \
89 perf_namestore_api_zone_iteration_postgres
43endif 90endif
44
45# testcases do not even build yet; thus: experimental!
46if HAVE_TESTING
47TESTING_TESTS = \
48 test_namestore_api_store.nc \
49 test_namestore_api_store_update.nc \
50 test_namestore_api_lookup_public.nc \
51 test_namestore_api_lookup_private.nc \
52 test_namestore_api_lookup_nick.nc \
53 test_namestore_api_lookup_shadow.nc \
54 test_namestore_api_lookup_shadow_filter.nc \
55 test_namestore_api_remove.nc \
56 test_namestore_api_remove_not_existing_record.nc \
57 test_namestore_api_zone_iteration.nc \
58 test_namestore_api_zone_iteration_nick.nc \
59 test_namestore_api_zone_iteration_specific_zone.nc \
60 test_namestore_api_zone_iteration_stop.nc \
61 test_namestore_api_monitoring.nc \
62 test_namestore_api_monitoring_existing.nc \
63 test_namestore_api_zone_to_name
64endif 91endif
65 92
66# Declare .nc (NO-CONCURRENCY) as a test extension so that we can impart
67# sequential execution order for them
68TEST_EXTENSIONS = .nc
69test_namestore_api_store.log: test_namestore_api_store_update.log
70test_namestore_api_store_update.log: test_namestore_api_lookup_public.log
71test_namestore_api_lookup_public.log: test_namestore_api_lookup_private.log
72test_namestore_api_lookup_private.log: test_namestore_api_lookup_nick.log
73test_namestore_api_lookup_nick.log: test_namestore_api_lookup_shadow.log
74test_namestore_api_lookup_shadow.log: test_namestore_api_lookup_shadow_filter.log
75test_namestore_api_lookup_shadow_filter.log: test_namestore_api_remove.log
76test_namestore_api_remove.log: test_namestore_api_remove_not_existing_record.log
77test_namestore_api_remove_not_existing_record.log: test_namestore_api_zone_iteration.log
78test_namestore_api_zone_iteration.log: test_namestore_api_zone_iteration_nick.log
79test_namestore_api_zone_iteration_nick.log: test_namestore_api_zone_iteration_specific_zone.log
80test_namestore_api_zone_iteration_specific_zone.log: test_namestore_api_zone_iteration_stop.log
81test_namestore_api_zone_iteration_stop.log: test_namestore_api_monitoring.log
82test_namestore_api_monitoring.log: test_namestore_api_monitoring_existing.log
83
84
85if HAVE_SQLITE 93if HAVE_SQLITE
86check_PROGRAMS = \ 94check_PROGRAMS = \
87 $(SQLITE_TESTS) \ 95 $(SQLITE_TESTS) \
88 $(POSTGRES_TESTS) \ 96 $(POSTGRES_TESTS) \
89 $(FLAT_TESTS) \ 97 $(FLAT_TESTS)
90 $(TESTING_TESTS)
91endif 98endif
92 99
93if HAVE_MHD 100if HAVE_MHD
@@ -124,9 +131,7 @@ libexec_PROGRAMS = \
124 gnunet-service-namestore 131 gnunet-service-namestore
125 132
126bin_PROGRAMS = \ 133bin_PROGRAMS = \
127 gnunet-namestore 134 gnunet-namestore \
128
129noinst_PROGRAMS = \
130 gnunet-zoneimport 135 gnunet-zoneimport
131 136
132if HAVE_MHD 137if HAVE_MHD
@@ -139,6 +144,7 @@ gnunet_zoneimport_SOURCES = \
139 gnunet-zoneimport.c 144 gnunet-zoneimport.c
140gnunet_zoneimport_LDADD = \ 145gnunet_zoneimport_LDADD = \
141 libgnunetnamestore.la \ 146 libgnunetnamestore.la \
147 $(top_builddir)/src/statistics/libgnunetstatistics.la \
142 $(top_builddir)/src/identity/libgnunetidentity.la \ 148 $(top_builddir)/src/identity/libgnunetidentity.la \
143 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \ 149 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
144 $(top_builddir)/src/dns/libgnunetdnsparser.la \ 150 $(top_builddir)/src/dns/libgnunetdnsparser.la \
@@ -234,135 +240,423 @@ libgnunet_plugin_rest_namestore_la_LDFLAGS = \
234 $(GN_PLUGIN_LDFLAGS) 240 $(GN_PLUGIN_LDFLAGS)
235 241
236 242
237test_namestore_api_store_nc_SOURCES = \ 243test_namestore_api_store_flat_SOURCES = \
244 test_namestore_api_store.c
245test_namestore_api_store_flat_LDADD = \
246 $(top_builddir)/src/testing/libgnunettesting.la \
247 $(top_builddir)/src/util/libgnunetutil.la \
248 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
249 libgnunetnamestore.la
250
251test_namestore_api_store_sqlite_SOURCES = \
252 test_namestore_api_store.c
253test_namestore_api_store_sqlite_LDADD = \
254 $(top_builddir)/src/testing/libgnunettesting.la \
255 $(top_builddir)/src/util/libgnunetutil.la \
256 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
257 libgnunetnamestore.la
258
259test_namestore_api_store_postgres_SOURCES = \
238 test_namestore_api_store.c 260 test_namestore_api_store.c
239test_namestore_api_store_nc_LDADD = \ 261test_namestore_api_store_postgres_LDADD = \
262 $(top_builddir)/src/testing/libgnunettesting.la \
263 $(top_builddir)/src/util/libgnunetutil.la \
264 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
265 libgnunetnamestore.la
266
267test_namestore_api_store_update_flat_SOURCES = \
268 test_namestore_api_store_update.c
269test_namestore_api_store_update_flat_LDADD = \
270 $(top_builddir)/src/testing/libgnunettesting.la \
271 $(top_builddir)/src/util/libgnunetutil.la \
272 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
273 $(top_builddir)/src/namecache/libgnunetnamecache.la \
274 libgnunetnamestore.la
275
276test_namestore_api_store_update_sqlite_SOURCES = \
277 test_namestore_api_store_update.c
278test_namestore_api_store_update_sqlite_LDADD = \
240 $(top_builddir)/src/testing/libgnunettesting.la \ 279 $(top_builddir)/src/testing/libgnunettesting.la \
241 $(top_builddir)/src/util/libgnunetutil.la \ 280 $(top_builddir)/src/util/libgnunetutil.la \
242 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \ 281 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
282 $(top_builddir)/src/namecache/libgnunetnamecache.la \
243 libgnunetnamestore.la 283 libgnunetnamestore.la
244 284
245test_namestore_api_store_update_nc_SOURCES = \ 285test_namestore_api_store_update_postgres_SOURCES = \
246 test_namestore_api_store_update.c 286 test_namestore_api_store_update.c
247test_namestore_api_store_update_nc_LDADD = \ 287test_namestore_api_store_update_postgres_LDADD = \
288 $(top_builddir)/src/testing/libgnunettesting.la \
289 $(top_builddir)/src/util/libgnunetutil.la \
290 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
291 $(top_builddir)/src/namecache/libgnunetnamecache.la \
292 libgnunetnamestore.la
293
294test_namestore_api_lookup_public_flat_SOURCES = \
295 test_namestore_api_lookup_public.c
296test_namestore_api_lookup_public_flat_LDADD = \
297 $(top_builddir)/src/testing/libgnunettesting.la \
298 $(top_builddir)/src/util/libgnunetutil.la \
299 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
300 $(top_builddir)/src/namecache/libgnunetnamecache.la \
301 libgnunetnamestore.la
302
303test_namestore_api_lookup_public_sqlite_SOURCES = \
304 test_namestore_api_lookup_public.c
305test_namestore_api_lookup_public_sqlite_LDADD = \
248 $(top_builddir)/src/testing/libgnunettesting.la \ 306 $(top_builddir)/src/testing/libgnunettesting.la \
249 $(top_builddir)/src/util/libgnunetutil.la \ 307 $(top_builddir)/src/util/libgnunetutil.la \
250 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \ 308 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
251 $(top_builddir)/src/namecache/libgnunetnamecache.la \ 309 $(top_builddir)/src/namecache/libgnunetnamecache.la \
252 libgnunetnamestore.la 310 libgnunetnamestore.la
253 311
254test_namestore_api_lookup_public_nc_SOURCES = \ 312test_namestore_api_lookup_public_postgres_SOURCES = \
255 test_namestore_api_lookup_public.c 313 test_namestore_api_lookup_public.c
256test_namestore_api_lookup_public_nc_LDADD = \ 314test_namestore_api_lookup_public_postgres_LDADD = \
315 $(top_builddir)/src/testing/libgnunettesting.la \
316 $(top_builddir)/src/util/libgnunetutil.la \
317 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
318 $(top_builddir)/src/namecache/libgnunetnamecache.la \
319 libgnunetnamestore.la
320
321test_namestore_api_lookup_nick_sqlite_SOURCES = \
322 test_namestore_api_lookup_nick.c
323test_namestore_api_lookup_nick_sqlite_LDADD = \
324 $(top_builddir)/src/testing/libgnunettesting.la \
325 $(top_builddir)/src/util/libgnunetutil.la \
326 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
327 $(top_builddir)/src/namecache/libgnunetnamecache.la \
328 libgnunetnamestore.la
329
330test_namestore_api_lookup_nick_postgres_SOURCES = \
331 test_namestore_api_lookup_nick.c
332test_namestore_api_lookup_nick_postgres_LDADD = \
257 $(top_builddir)/src/testing/libgnunettesting.la \ 333 $(top_builddir)/src/testing/libgnunettesting.la \
258 $(top_builddir)/src/util/libgnunetutil.la \ 334 $(top_builddir)/src/util/libgnunetutil.la \
259 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \ 335 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
260 $(top_builddir)/src/namecache/libgnunetnamecache.la \ 336 $(top_builddir)/src/namecache/libgnunetnamecache.la \
261 libgnunetnamestore.la 337 libgnunetnamestore.la
262 338
263test_namestore_api_lookup_nick_nc_SOURCES = \ 339test_namestore_api_lookup_nick_flat_SOURCES = \
264 test_namestore_api_lookup_nick.c 340 test_namestore_api_lookup_nick.c
265test_namestore_api_lookup_nick_nc_LDADD = \ 341test_namestore_api_lookup_nick_flat_LDADD = \
342 $(top_builddir)/src/testing/libgnunettesting.la \
343 $(top_builddir)/src/util/libgnunetutil.la \
344 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
345 $(top_builddir)/src/namecache/libgnunetnamecache.la \
346 libgnunetnamestore.la
347
348test_namestore_api_lookup_private_flat_SOURCES = \
349 test_namestore_api_lookup_private.c
350test_namestore_api_lookup_private_flat_LDADD = \
351 $(top_builddir)/src/testing/libgnunettesting.la \
352 $(top_builddir)/src/util/libgnunetutil.la \
353 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
354 $(top_builddir)/src/namecache/libgnunetnamecache.la \
355 libgnunetnamestore.la
356
357test_namestore_api_lookup_private_sqlite_SOURCES = \
358 test_namestore_api_lookup_private.c
359test_namestore_api_lookup_private_sqlite_LDADD = \
266 $(top_builddir)/src/testing/libgnunettesting.la \ 360 $(top_builddir)/src/testing/libgnunettesting.la \
267 $(top_builddir)/src/util/libgnunetutil.la \ 361 $(top_builddir)/src/util/libgnunetutil.la \
268 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \ 362 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
269 $(top_builddir)/src/namecache/libgnunetnamecache.la \ 363 $(top_builddir)/src/namecache/libgnunetnamecache.la \
270 libgnunetnamestore.la 364 libgnunetnamestore.la
271 365
272test_namestore_api_lookup_private_nc_SOURCES = \ 366test_namestore_api_lookup_private_postgres_SOURCES = \
273 test_namestore_api_lookup_private.c 367 test_namestore_api_lookup_private.c
274test_namestore_api_lookup_private_nc_LDADD = \ 368test_namestore_api_lookup_private_postgres_LDADD = \
369 $(top_builddir)/src/testing/libgnunettesting.la \
370 $(top_builddir)/src/util/libgnunetutil.la \
371 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
372 $(top_builddir)/src/namecache/libgnunetnamecache.la \
373 libgnunetnamestore.la
374
375test_namestore_api_lookup_shadow_flat_SOURCES = \
376 test_namestore_api_lookup_shadow.c
377test_namestore_api_lookup_shadow_flat_LDADD = \
378 $(top_builddir)/src/testing/libgnunettesting.la \
379 $(top_builddir)/src/util/libgnunetutil.la \
380 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
381 $(top_builddir)/src/namecache/libgnunetnamecache.la \
382 libgnunetnamestore.la
383
384test_namestore_api_lookup_shadow_sqlite_SOURCES = \
385 test_namestore_api_lookup_shadow.c
386test_namestore_api_lookup_shadow_sqlite_LDADD = \
275 $(top_builddir)/src/testing/libgnunettesting.la \ 387 $(top_builddir)/src/testing/libgnunettesting.la \
276 $(top_builddir)/src/util/libgnunetutil.la \ 388 $(top_builddir)/src/util/libgnunetutil.la \
277 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \ 389 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
278 $(top_builddir)/src/namecache/libgnunetnamecache.la \ 390 $(top_builddir)/src/namecache/libgnunetnamecache.la \
279 libgnunetnamestore.la 391 libgnunetnamestore.la
280 392
281test_namestore_api_lookup_shadow_nc_SOURCES = \ 393test_namestore_api_lookup_shadow_postgres_SOURCES = \
282 test_namestore_api_lookup_shadow.c 394 test_namestore_api_lookup_shadow.c
283test_namestore_api_lookup_shadow_nc_LDADD = \ 395test_namestore_api_lookup_shadow_postgres_LDADD = \
396 $(top_builddir)/src/testing/libgnunettesting.la \
397 $(top_builddir)/src/util/libgnunetutil.la \
398 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
399 $(top_builddir)/src/namecache/libgnunetnamecache.la \
400 libgnunetnamestore.la
401
402test_namestore_api_lookup_shadow_filter_flat_SOURCES = \
403 test_namestore_api_lookup_shadow_filter.c
404test_namestore_api_lookup_shadow_filter_flat_LDADD = \
284 $(top_builddir)/src/testing/libgnunettesting.la \ 405 $(top_builddir)/src/testing/libgnunettesting.la \
285 $(top_builddir)/src/util/libgnunetutil.la \ 406 $(top_builddir)/src/util/libgnunetutil.la \
286 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \ 407 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
287 $(top_builddir)/src/namecache/libgnunetnamecache.la \ 408 $(top_builddir)/src/namecache/libgnunetnamecache.la \
288 libgnunetnamestore.la 409 libgnunetnamestore.la
289 410
290test_namestore_api_lookup_shadow_filter_nc_SOURCES = \ 411test_namestore_api_lookup_shadow_filter_sqlite_SOURCES = \
412 test_namestore_api_lookup_shadow_filter.c
413test_namestore_api_lookup_shadow_filter_sqlite_LDADD = \
414 $(top_builddir)/src/testing/libgnunettesting.la \
415 $(top_builddir)/src/util/libgnunetutil.la \
416 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
417 $(top_builddir)/src/namecache/libgnunetnamecache.la \
418 libgnunetnamestore.la
419test_namestore_api_lookup_shadow_filter_postgres_SOURCES = \
291 test_namestore_api_lookup_shadow_filter.c 420 test_namestore_api_lookup_shadow_filter.c
292test_namestore_api_lookup_shadow_filter_nc_LDADD = \ 421test_namestore_api_lookup_shadow_filter_postgres_LDADD = \
293 $(top_builddir)/src/testing/libgnunettesting.la \ 422 $(top_builddir)/src/testing/libgnunettesting.la \
294 $(top_builddir)/src/util/libgnunetutil.la \ 423 $(top_builddir)/src/util/libgnunetutil.la \
295 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \ 424 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
296 $(top_builddir)/src/namecache/libgnunetnamecache.la \ 425 $(top_builddir)/src/namecache/libgnunetnamecache.la \
297 libgnunetnamestore.la 426 libgnunetnamestore.la
298 427
299test_namestore_api_remove_nc_SOURCES = \ 428test_namestore_api_remove_sqlite_SOURCES = \
429 test_namestore_api_remove.c
430test_namestore_api_remove_sqlite_LDADD = \
431 $(top_builddir)/src/testing/libgnunettesting.la \
432 $(top_builddir)/src/util/libgnunetutil.la \
433 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
434 libgnunetnamestore.la
435
436test_namestore_api_remove_postgres_SOURCES = \
300 test_namestore_api_remove.c 437 test_namestore_api_remove.c
301test_namestore_api_remove_nc_LDADD = \ 438test_namestore_api_remove_postgres_LDADD = \
302 $(top_builddir)/src/testing/libgnunettesting.la \ 439 $(top_builddir)/src/testing/libgnunettesting.la \
303 $(top_builddir)/src/util/libgnunetutil.la \ 440 $(top_builddir)/src/util/libgnunetutil.la \
304 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \ 441 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
305 libgnunetnamestore.la 442 libgnunetnamestore.la
306 443
307test_namestore_api_remove_not_existing_record_nc_SOURCES = \ 444test_namestore_api_remove_flat_SOURCES = \
445 test_namestore_api_remove.c
446test_namestore_api_remove_flat_LDADD = \
447 $(top_builddir)/src/testing/libgnunettesting.la \
448 $(top_builddir)/src/util/libgnunetutil.la \
449 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
450 libgnunetnamestore.la
451
452test_namestore_api_remove_not_existing_record_flat_SOURCES = \
453 test_namestore_api_remove_not_existing_record.c
454test_namestore_api_remove_not_existing_record_flat_LDADD = \
455 $(top_builddir)/src/testing/libgnunettesting.la \
456 $(top_builddir)/src/util/libgnunetutil.la \
457 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
458 libgnunetnamestore.la
459
460test_namestore_api_remove_not_existing_record_sqlite_SOURCES = \
461 test_namestore_api_remove_not_existing_record.c
462test_namestore_api_remove_not_existing_record_sqlite_LDADD = \
463 $(top_builddir)/src/testing/libgnunettesting.la \
464 $(top_builddir)/src/util/libgnunetutil.la \
465 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
466 libgnunetnamestore.la
467
468test_namestore_api_remove_not_existing_record_postgres_SOURCES = \
308 test_namestore_api_remove_not_existing_record.c 469 test_namestore_api_remove_not_existing_record.c
309test_namestore_api_remove_not_existing_record_nc_LDADD = \ 470test_namestore_api_remove_not_existing_record_postgres_LDADD = \
310 $(top_builddir)/src/testing/libgnunettesting.la \ 471 $(top_builddir)/src/testing/libgnunettesting.la \
311 $(top_builddir)/src/util/libgnunetutil.la \ 472 $(top_builddir)/src/util/libgnunetutil.la \
312 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \ 473 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
313 libgnunetnamestore.la 474 libgnunetnamestore.la
314 475
315test_namestore_api_zone_to_name_SOURCES = \ 476test_namestore_api_zone_to_name_flat_SOURCES = \
477 test_namestore_api_zone_to_name.c
478test_namestore_api_zone_to_name_flat_LDADD = \
479 $(top_builddir)/src/testing/libgnunettesting.la \
480 $(top_builddir)/src/util/libgnunetutil.la \
481 libgnunetnamestore.la
482
483test_namestore_api_zone_to_name_sqlite_SOURCES = \
484 test_namestore_api_zone_to_name.c
485test_namestore_api_zone_to_name_sqlite_LDADD = \
486 $(top_builddir)/src/testing/libgnunettesting.la \
487 $(top_builddir)/src/util/libgnunetutil.la \
488 libgnunetnamestore.la
489
490test_namestore_api_zone_to_name_postgres_SOURCES = \
316 test_namestore_api_zone_to_name.c 491 test_namestore_api_zone_to_name.c
317test_namestore_api_zone_to_name_LDADD = \ 492test_namestore_api_zone_to_name_postgres_LDADD = \
318 $(top_builddir)/src/testing/libgnunettesting.la \ 493 $(top_builddir)/src/testing/libgnunettesting.la \
319 $(top_builddir)/src/util/libgnunetutil.la \ 494 $(top_builddir)/src/util/libgnunetutil.la \
320 libgnunetnamestore.la 495 libgnunetnamestore.la
321 496
322test_namestore_api_monitoring_nc_SOURCES = \ 497test_namestore_api_monitoring_flat_SOURCES = \
498 test_namestore_api_monitoring.c
499test_namestore_api_monitoring_flat_LDADD = \
500 $(top_builddir)/src/testing/libgnunettesting.la \
501 libgnunetnamestore.la \
502 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
503 $(top_builddir)/src/util/libgnunetutil.la
504
505test_namestore_api_monitoring_sqlite_SOURCES = \
506 test_namestore_api_monitoring.c
507test_namestore_api_monitoring_sqlite_LDADD = \
508 $(top_builddir)/src/testing/libgnunettesting.la \
509 libgnunetnamestore.la \
510 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
511 $(top_builddir)/src/util/libgnunetutil.la
512
513test_namestore_api_monitoring_postgres_SOURCES = \
323 test_namestore_api_monitoring.c 514 test_namestore_api_monitoring.c
324test_namestore_api_monitoring_nc_LDADD = \ 515test_namestore_api_monitoring_postgres_LDADD = \
516 $(top_builddir)/src/testing/libgnunettesting.la \
517 libgnunetnamestore.la \
518 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
519 $(top_builddir)/src/util/libgnunetutil.la
520
521test_namestore_api_monitoring_existing_flat_SOURCES = \
522 test_namestore_api_monitoring_existing.c
523test_namestore_api_monitoring_existing_flat_LDADD = \
524 $(top_builddir)/src/testing/libgnunettesting.la \
525 libgnunetnamestore.la \
526 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
527 $(top_builddir)/src/util/libgnunetutil.la
528
529test_namestore_api_monitoring_existing_sqlite_SOURCES = \
530 test_namestore_api_monitoring_existing.c
531test_namestore_api_monitoring_existing_sqlite_LDADD = \
325 $(top_builddir)/src/testing/libgnunettesting.la \ 532 $(top_builddir)/src/testing/libgnunettesting.la \
326 libgnunetnamestore.la \ 533 libgnunetnamestore.la \
327 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \ 534 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
328 $(top_builddir)/src/util/libgnunetutil.la 535 $(top_builddir)/src/util/libgnunetutil.la
329 536
330test_namestore_api_monitoring_existing_nc_SOURCES = \ 537test_namestore_api_monitoring_existing_postgres_SOURCES = \
331 test_namestore_api_monitoring_existing.c 538 test_namestore_api_monitoring_existing.c
332test_namestore_api_monitoring_existing_nc_LDADD = \ 539test_namestore_api_monitoring_existing_postgres_LDADD = \
333 $(top_builddir)/src/testing/libgnunettesting.la \ 540 $(top_builddir)/src/testing/libgnunettesting.la \
334 libgnunetnamestore.la \ 541 libgnunetnamestore.la \
335 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \ 542 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
336 $(top_builddir)/src/util/libgnunetutil.la 543 $(top_builddir)/src/util/libgnunetutil.la
337 544
338test_namestore_api_zone_iteration_nc_SOURCES = \ 545test_namestore_api_zone_iteration_flat_SOURCES = \
339 test_namestore_api_zone_iteration.c 546 test_namestore_api_zone_iteration.c
340test_namestore_api_zone_iteration_nc_LDADD = \ 547test_namestore_api_zone_iteration_flat_LDADD = \
341 $(top_builddir)/src/testing/libgnunettesting.la \ 548 $(top_builddir)/src/testing/libgnunettesting.la \
342 $(top_builddir)/src/util/libgnunetutil.la \ 549 $(top_builddir)/src/util/libgnunetutil.la \
343 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \ 550 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
344 libgnunetnamestore.la 551 libgnunetnamestore.la
345 552
346test_namestore_api_zone_iteration_nick_nc_SOURCES = \ 553test_namestore_api_zone_iteration_sqlite_SOURCES = \
554 test_namestore_api_zone_iteration.c
555test_namestore_api_zone_iteration_sqlite_LDADD = \
556 $(top_builddir)/src/testing/libgnunettesting.la \
557 $(top_builddir)/src/util/libgnunetutil.la \
558 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
559 libgnunetnamestore.la
560
561test_namestore_api_zone_iteration_postgres_SOURCES = \
562 test_namestore_api_zone_iteration.c
563test_namestore_api_zone_iteration_postgres_LDADD = \
564 $(top_builddir)/src/testing/libgnunettesting.la \
565 $(top_builddir)/src/util/libgnunetutil.la \
566 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
567 libgnunetnamestore.la
568
569perf_namestore_api_zone_iteration_postgres_SOURCES = \
570 perf_namestore_api_zone_iteration.c
571perf_namestore_api_zone_iteration_postgres_LDADD = \
572 $(top_builddir)/src/testing/libgnunettesting.la \
573 $(top_builddir)/src/util/libgnunetutil.la \
574 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
575 libgnunetnamestore.la
576
577perf_namestore_api_zone_iteration_sqlite_SOURCES = \
578 perf_namestore_api_zone_iteration.c
579perf_namestore_api_zone_iteration_sqlite_LDADD = \
580 $(top_builddir)/src/testing/libgnunettesting.la \
581 $(top_builddir)/src/util/libgnunetutil.la \
582 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
583 libgnunetnamestore.la
584
585perf_namestore_api_zone_iteration_flat_SOURCES = \
586 perf_namestore_api_zone_iteration.c
587perf_namestore_api_zone_iteration_flat_LDADD = \
588 $(top_builddir)/src/testing/libgnunettesting.la \
589 $(top_builddir)/src/util/libgnunetutil.la \
590 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
591 libgnunetnamestore.la
592
593test_namestore_api_zone_iteration_nick_flat_SOURCES = \
347 test_namestore_api_zone_iteration_nick.c 594 test_namestore_api_zone_iteration_nick.c
348test_namestore_api_zone_iteration_nick_nc_LDADD = \ 595test_namestore_api_zone_iteration_nick_flat_LDADD = \
349 $(top_builddir)/src/testing/libgnunettesting.la \ 596 $(top_builddir)/src/testing/libgnunettesting.la \
350 $(top_builddir)/src/util/libgnunetutil.la \ 597 $(top_builddir)/src/util/libgnunetutil.la \
351 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \ 598 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
352 libgnunetnamestore.la 599 libgnunetnamestore.la
353 600
601test_namestore_api_zone_iteration_nick_sqlite_SOURCES = \
602 test_namestore_api_zone_iteration_nick.c
603test_namestore_api_zone_iteration_nick_sqlite_LDADD = \
604 $(top_builddir)/src/testing/libgnunettesting.la \
605 $(top_builddir)/src/util/libgnunetutil.la \
606 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
607 libgnunetnamestore.la
354 608
355test_namestore_api_zone_iteration_specific_zone_nc_SOURCES = \ 609test_namestore_api_zone_iteration_nick_postgres_SOURCES = \
610 test_namestore_api_zone_iteration_nick.c
611test_namestore_api_zone_iteration_nick_postgres_LDADD = \
612 $(top_builddir)/src/testing/libgnunettesting.la \
613 $(top_builddir)/src/util/libgnunetutil.la \
614 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
615 libgnunetnamestore.la
616
617test_namestore_api_zone_iteration_specific_zone_flat_SOURCES = \
618 test_namestore_api_zone_iteration_specific_zone.c
619test_namestore_api_zone_iteration_specific_zone_flat_LDADD = \
620 $(top_builddir)/src/testing/libgnunettesting.la \
621 $(top_builddir)/src/util/libgnunetutil.la \
622 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
623 libgnunetnamestore.la
624
625test_namestore_api_zone_iteration_specific_zone_sqlite_SOURCES = \
356 test_namestore_api_zone_iteration_specific_zone.c 626 test_namestore_api_zone_iteration_specific_zone.c
357test_namestore_api_zone_iteration_specific_zone_nc_LDADD = \ 627test_namestore_api_zone_iteration_specific_zone_sqlite_LDADD = \
628 $(top_builddir)/src/testing/libgnunettesting.la \
629 $(top_builddir)/src/util/libgnunetutil.la \
630 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
631 libgnunetnamestore.la
632
633test_namestore_api_zone_iteration_specific_zone_postgres_SOURCES = \
634 test_namestore_api_zone_iteration_specific_zone.c
635test_namestore_api_zone_iteration_specific_zone_postgres_LDADD = \
636 $(top_builddir)/src/testing/libgnunettesting.la \
637 $(top_builddir)/src/util/libgnunetutil.la \
638 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
639 libgnunetnamestore.la
640
641test_namestore_api_zone_iteration_stop_flat_SOURCES = \
642 test_namestore_api_zone_iteration_stop.c
643test_namestore_api_zone_iteration_stop_flat_LDADD = \
644 $(top_builddir)/src/testing/libgnunettesting.la \
645 $(top_builddir)/src/util/libgnunetutil.la \
646 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
647 libgnunetnamestore.la
648
649test_namestore_api_zone_iteration_stop_sqlite_SOURCES = \
650 test_namestore_api_zone_iteration_stop.c
651test_namestore_api_zone_iteration_stop_sqlite_LDADD = \
358 $(top_builddir)/src/testing/libgnunettesting.la \ 652 $(top_builddir)/src/testing/libgnunettesting.la \
359 $(top_builddir)/src/util/libgnunetutil.la \ 653 $(top_builddir)/src/util/libgnunetutil.la \
360 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \ 654 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
361 libgnunetnamestore.la 655 libgnunetnamestore.la
362 656
363test_namestore_api_zone_iteration_stop_nc_SOURCES = \ 657test_namestore_api_zone_iteration_stop_postgres_SOURCES = \
364 test_namestore_api_zone_iteration_stop.c 658 test_namestore_api_zone_iteration_stop.c
365test_namestore_api_zone_iteration_stop_nc_LDADD = \ 659test_namestore_api_zone_iteration_stop_postgres_LDADD = \
366 $(top_builddir)/src/testing/libgnunettesting.la \ 660 $(top_builddir)/src/testing/libgnunettesting.la \
367 $(top_builddir)/src/util/libgnunetutil.la \ 661 $(top_builddir)/src/util/libgnunetutil.la \
368 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \ 662 $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
@@ -393,6 +687,12 @@ check_SCRIPTS = \
393 687
394EXTRA_DIST = \ 688EXTRA_DIST = \
395 test_namestore_api.conf \ 689 test_namestore_api.conf \
690 test_namestore_api_postgres.conf \
691 test_namestore_api_sqlite.conf \
692 test_namestore_api_flat.conf \
693 perf_namestore_api_postgres.conf \
694 perf_namestore_api_sqlite.conf \
695 perf_namestore_api_flat.conf \
396 test_plugin_namestore_sqlite.conf \ 696 test_plugin_namestore_sqlite.conf \
397 test_plugin_namestore_postgres.conf \ 697 test_plugin_namestore_postgres.conf \
398 test_plugin_namestore_flat.conf \ 698 test_plugin_namestore_flat.conf \
diff --git a/src/namestore/gnunet-namestore-fcfsd.c b/src/namestore/gnunet-namestore-fcfsd.c
index ddd609918..903253b1a 100644
--- a/src/namestore/gnunet-namestore-fcfsd.c
+++ b/src/namestore/gnunet-namestore-fcfsd.c
@@ -324,15 +324,18 @@ iterate_cb (void *cls,
324 char* pkey; 324 char* pkey;
325 char* new_buf; 325 char* new_buf;
326 326
327 (void) zone_key;
327 if (1 != rd_len) 328 if (1 != rd_len)
328 { 329 {
329 GNUNET_NAMESTORE_zone_iterator_next (zr->list_it); 330 GNUNET_NAMESTORE_zone_iterator_next (zr->list_it,
331 1);
330 return; 332 return;
331 } 333 }
332 334
333 if (GNUNET_GNSRECORD_TYPE_PKEY != rd->record_type) 335 if (GNUNET_GNSRECORD_TYPE_PKEY != rd->record_type)
334 { 336 {
335 GNUNET_NAMESTORE_zone_iterator_next (zr->list_it); 337 GNUNET_NAMESTORE_zone_iterator_next (zr->list_it,
338 1);
336 return; 339 return;
337 } 340 }
338 341
@@ -343,7 +346,8 @@ iterate_cb (void *cls,
343 if (NULL == pkey) 346 if (NULL == pkey)
344 { 347 {
345 GNUNET_break (0); 348 GNUNET_break (0);
346 GNUNET_NAMESTORE_zone_iterator_next (zr->list_it); 349 GNUNET_NAMESTORE_zone_iterator_next (zr->list_it,
350 1);
347 return; 351 return;
348 } 352 }
349 if (bytes_free < (strlen (name) + strlen (pkey) + 40)) 353 if (bytes_free < (strlen (name) + strlen (pkey) + 40))
@@ -359,7 +363,8 @@ iterate_cb (void *cls,
359 name, 363 name,
360 pkey); 364 pkey);
361 zr->write_offset = strlen (zr->zoneinfo); 365 zr->write_offset = strlen (zr->zoneinfo);
362 GNUNET_NAMESTORE_zone_iterator_next (zr->list_it); 366 GNUNET_NAMESTORE_zone_iterator_next (zr->list_it,
367 1);
363 GNUNET_free (pkey); 368 GNUNET_free (pkey);
364} 369}
365 370
@@ -435,6 +440,7 @@ fill_s_reply (const char *info,
435 char *reply; 440 char *reply;
436 struct MHD_Response *response; 441 struct MHD_Response *response;
437 442
443 (void) request;
438 GNUNET_asprintf (&reply, 444 GNUNET_asprintf (&reply,
439 SUBMIT_PAGE, 445 SUBMIT_PAGE,
440 info, 446 info,
@@ -583,6 +589,8 @@ zone_to_name_cb (void *cls,
583 struct Request *request = cls; 589 struct Request *request = cls;
584 struct GNUNET_GNSRECORD_Data r; 590 struct GNUNET_GNSRECORD_Data r;
585 591
592 (void) rd;
593 (void) zone_key;
586 request->qe = NULL; 594 request->qe = NULL;
587 if (0 != rd_count) 595 if (0 != rd_count)
588 { 596 {
@@ -640,6 +648,9 @@ lookup_block_processor (void *cls,
640{ 648{
641 struct Request *request = cls; 649 struct Request *request = cls;
642 650
651 (void) label;
652 (void) rd;
653 (void) zone;
643 request->qe = NULL; 654 request->qe = NULL;
644 if (0 == rd_count) 655 if (0 == rd_count)
645 { 656 {
@@ -711,6 +722,8 @@ create_response (void *cls,
711 struct GNUNET_CRYPTO_EcdsaPublicKey pub; 722 struct GNUNET_CRYPTO_EcdsaPublicKey pub;
712 int ret; 723 int ret;
713 724
725 (void) cls;
726 (void) version;
714 if ( (0 == strcmp (method, MHD_HTTP_METHOD_GET)) || 727 if ( (0 == strcmp (method, MHD_HTTP_METHOD_GET)) ||
715 (0 == strcmp (method, MHD_HTTP_METHOD_HEAD)) ) 728 (0 == strcmp (method, MHD_HTTP_METHOD_HEAD)) )
716 { 729 {
diff --git a/src/namestore/gnunet-namestore.c b/src/namestore/gnunet-namestore.c
index 9a1805af4..d329dcb3b 100644
--- a/src/namestore/gnunet-namestore.c
+++ b/src/namestore/gnunet-namestore.c
@@ -398,7 +398,8 @@ display_record (void *cls,
398 if ( (NULL != name) && 398 if ( (NULL != name) &&
399 (0 != strcmp (name, rname)) ) 399 (0 != strcmp (name, rname)) )
400 { 400 {
401 GNUNET_NAMESTORE_zone_iterator_next (list_it); 401 GNUNET_NAMESTORE_zone_iterator_next (list_it,
402 1);
402 return; 403 return;
403 } 404 }
404 FPRINTF (stdout, 405 FPRINTF (stdout,
@@ -408,7 +409,7 @@ display_record (void *cls,
408 { 409 {
409 if ( (GNUNET_GNSRECORD_TYPE_NICK == rd[i].record_type) && 410 if ( (GNUNET_GNSRECORD_TYPE_NICK == rd[i].record_type) &&
410 (0 != strcmp (rname, 411 (0 != strcmp (rname,
411 GNUNET_GNS_MASTERZONE_STR)) ) 412 GNUNET_GNS_EMPTY_LABEL_AT)) )
412 continue; 413 continue;
413 typestring = GNUNET_GNSRECORD_number_to_typename (rd[i].record_type); 414 typestring = GNUNET_GNSRECORD_number_to_typename (rd[i].record_type);
414 s = GNUNET_GNSRECORD_value_to_string (rd[i].record_type, 415 s = GNUNET_GNSRECORD_value_to_string (rd[i].record_type,
@@ -441,7 +442,8 @@ display_record (void *cls,
441 GNUNET_free (s); 442 GNUNET_free (s);
442 } 443 }
443 FPRINTF (stdout, "%s", "\n"); 444 FPRINTF (stdout, "%s", "\n");
444 GNUNET_NAMESTORE_zone_iterator_next (list_it); 445 GNUNET_NAMESTORE_zone_iterator_next (list_it,
446 1);
445} 447}
446 448
447 449
@@ -659,6 +661,8 @@ handle_reverse_lookup (void *cls,
659{ 661{
660 (void) cls; 662 (void) cls;
661 (void) zone; 663 (void) zone;
664 (void) rd_count;
665 (void) rd;
662 reverse_qe = NULL; 666 reverse_qe = NULL;
663 if (NULL == label) 667 if (NULL == label)
664 FPRINTF (stdout, 668 FPRINTF (stdout,
@@ -710,6 +714,7 @@ del_monitor (void *cls,
710 char *vs; 714 char *vs;
711 715
712 (void) cls; 716 (void) cls;
717 (void) zone;
713 del_qe = NULL; 718 del_qe = NULL;
714 if (0 == rd_count) 719 if (0 == rd_count)
715 { 720 {
@@ -1090,6 +1095,9 @@ id_connect_cb (void *cls,
1090{ 1095{
1091 const struct GNUNET_CONFIGURATION_Handle *cfg = cls; 1096 const struct GNUNET_CONFIGURATION_Handle *cfg = cls;
1092 1097
1098 (void) cls;
1099 (void) ctx;
1100 (void) name;
1093 if (NULL == ego) 1101 if (NULL == ego)
1094 { 1102 {
1095 get_default = GNUNET_IDENTITY_get (idh, 1103 get_default = GNUNET_IDENTITY_get (idh,
diff --git a/src/namestore/gnunet-service-namestore.c b/src/namestore/gnunet-service-namestore.c
index 3a3291c07..fa189dbc3 100644
--- a/src/namestore/gnunet-service-namestore.c
+++ b/src/namestore/gnunet-service-namestore.c
@@ -31,6 +31,7 @@
31#include "gnunet_namecache_service.h" 31#include "gnunet_namecache_service.h"
32#include "gnunet_namestore_service.h" 32#include "gnunet_namestore_service.h"
33#include "gnunet_namestore_plugin.h" 33#include "gnunet_namestore_plugin.h"
34#include "gnunet_statistics_service.h"
34#include "gnunet_signatures.h" 35#include "gnunet_signatures.h"
35#include "namestore.h" 36#include "namestore.h"
36 37
@@ -74,6 +75,15 @@ struct ZoneIteration
74 struct GNUNET_CRYPTO_EcdsaPrivateKey zone; 75 struct GNUNET_CRYPTO_EcdsaPrivateKey zone;
75 76
76 /** 77 /**
78 * Last sequence number in the zone iteration used to address next
79 * result of the zone iteration in the store
80 *
81 * Initialy set to 0.
82 * Updated in #zone_iterate_proc()
83 */
84 uint64_t seq;
85
86 /**
77 * The operation id fot the zone iteration in the response for the client 87 * The operation id fot the zone iteration in the response for the client
78 */ 88 */
79 uint32_t request_id; 89 uint32_t request_id;
@@ -82,8 +92,8 @@ struct ZoneIteration
82 * Offset of the zone iteration used to address next result of the zone 92 * Offset of the zone iteration used to address next result of the zone
83 * iteration in the store 93 * iteration in the store
84 * 94 *
85 * Initialy set to 0 in handle_iteration_start 95 * Initialy set to 0 in #handle_iteration_start
86 * Incremented with by every call to handle_iteration_next 96 * Incremented with by every call to #handle_iteration_next
87 */ 97 */
88 uint32_t offset; 98 uint32_t offset;
89 99
@@ -151,13 +161,13 @@ struct ZoneMonitor
151 struct GNUNET_SCHEDULER_Task *task; 161 struct GNUNET_SCHEDULER_Task *task;
152 162
153 /** 163 /**
154 * Offset of the zone iteration used to address next result of the zone 164 * Last sequence number in the zone iteration used to address next
155 * iteration in the store 165 * result of the zone iteration in the store
156 * 166 *
157 * Initialy set to 0. 167 * Initialy set to 0.
158 * Incremented with by every call to #handle_iteration_next 168 * Updated in #monitor_iterate_cb()
159 */ 169 */
160 uint32_t offset; 170 uint64_t seq;
161 171
162}; 172};
163 173
@@ -206,6 +216,11 @@ static const struct GNUNET_CRYPTO_EcdsaPrivateKey zero;
206static const struct GNUNET_CONFIGURATION_Handle *GSN_cfg; 216static const struct GNUNET_CONFIGURATION_Handle *GSN_cfg;
207 217
208/** 218/**
219 * Handle to the statistics service
220 */
221static struct GNUNET_STATISTICS_Handle *statistics;
222
223/**
209 * Namecache handle. 224 * Namecache handle.
210 */ 225 */
211static struct GNUNET_NAMECACHE_Handle *namecache; 226static struct GNUNET_NAMECACHE_Handle *namecache;
@@ -245,6 +260,18 @@ static struct ZoneMonitor *monitor_tail;
245 */ 260 */
246static struct GNUNET_NotificationContext *monitor_nc; 261static struct GNUNET_NotificationContext *monitor_nc;
247 262
263/**
264 * Optimize block insertion by caching map of private keys to
265 * public keys in memory?
266 */
267static int cache_keys;
268
269/**
270 * Use the namecache? Doing so creates additional cryptographic
271 * operations whenever we touch a record.
272 */
273static int disable_namecache;
274
248 275
249/** 276/**
250 * Task run during shutdown. 277 * Task run during shutdown.
@@ -269,8 +296,11 @@ cleanup_task (void *cls)
269 cop); 296 cop);
270 GNUNET_free (cop); 297 GNUNET_free (cop);
271 } 298 }
272 GNUNET_NAMECACHE_disconnect (namecache); 299 if (NULL != namecache)
273 namecache = NULL; 300 {
301 GNUNET_NAMECACHE_disconnect (namecache);
302 namecache = NULL;
303 }
274 GNUNET_break (NULL == GNUNET_PLUGIN_unload (db_lib_name, 304 GNUNET_break (NULL == GNUNET_PLUGIN_unload (db_lib_name,
275 GSN_database)); 305 GSN_database));
276 GNUNET_free (db_lib_name); 306 GNUNET_free (db_lib_name);
@@ -280,6 +310,12 @@ cleanup_task (void *cls)
280 GNUNET_notification_context_destroy (monitor_nc); 310 GNUNET_notification_context_destroy (monitor_nc);
281 monitor_nc = NULL; 311 monitor_nc = NULL;
282 } 312 }
313 if (NULL != statistics)
314 {
315 GNUNET_STATISTICS_destroy (statistics,
316 GNUNET_NO);
317 statistics = NULL;
318 }
283} 319}
284 320
285 321
@@ -362,18 +398,20 @@ client_connect_cb (void *cls,
362 398
363 399
364/** 400/**
365 * Function called with the records for the #GNUNET_GNS_MASTERZONE_STR 401 * Function called with the records for the #GNUNET_GNS_EMPTY_LABEL_AT
366 * label in the zone. Used to locate the #GNUNET_GNSRECORD_TYPE_NICK 402 * label in the zone. Used to locate the #GNUNET_GNSRECORD_TYPE_NICK
367 * record, which (if found) is then copied to @a cls for future use. 403 * record, which (if found) is then copied to @a cls for future use.
368 * 404 *
369 * @param cls a `struct GNUNET_GNSRECORD_Data **` for storing the nick (if found) 405 * @param cls a `struct GNUNET_GNSRECORD_Data **` for storing the nick (if found)
406 * @param seq sequence number of the record
370 * @param private_key the private key of the zone (unused) 407 * @param private_key the private key of the zone (unused)
371 * @param label should be #GNUNET_GNS_MASTERZONE_STR 408 * @param label should be #GNUNET_GNS_EMPTY_LABEL_AT
372 * @param rd_count number of records in @a rd 409 * @param rd_count number of records in @a rd
373 * @param rd records stored under @a label in the zone 410 * @param rd records stored under @a label in the zone
374 */ 411 */
375static void 412static void
376lookup_nick_it (void *cls, 413lookup_nick_it (void *cls,
414 uint64_t seq,
377 const struct GNUNET_CRYPTO_EcdsaPrivateKey *private_key, 415 const struct GNUNET_CRYPTO_EcdsaPrivateKey *private_key,
378 const char *label, 416 const char *label,
379 unsigned int rd_count, 417 unsigned int rd_count,
@@ -382,7 +420,8 @@ lookup_nick_it (void *cls,
382 struct GNUNET_GNSRECORD_Data **res = cls; 420 struct GNUNET_GNSRECORD_Data **res = cls;
383 421
384 (void) private_key; 422 (void) private_key;
385 if (0 != strcmp (label, GNUNET_GNS_MASTERZONE_STR)) 423 (void) seq;
424 if (0 != strcmp (label, GNUNET_GNS_EMPTY_LABEL_AT))
386 { 425 {
387 GNUNET_break (0); 426 GNUNET_break (0);
388 return; 427 return;
@@ -423,14 +462,14 @@ get_nick_record (const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone)
423 nick = NULL; 462 nick = NULL;
424 res = GSN_database->lookup_records (GSN_database->cls, 463 res = GSN_database->lookup_records (GSN_database->cls,
425 zone, 464 zone,
426 GNUNET_GNS_MASTERZONE_STR, 465 GNUNET_GNS_EMPTY_LABEL_AT,
427 &lookup_nick_it, 466 &lookup_nick_it,
428 &nick); 467 &nick);
429 if ( (GNUNET_OK != res) || 468 if ( (GNUNET_OK != res) ||
430 (NULL == nick) ) 469 (NULL == nick) )
431 { 470 {
432 GNUNET_CRYPTO_ecdsa_key_get_public (zone, &pub); 471 GNUNET_CRYPTO_ecdsa_key_get_public (zone, &pub);
433 GNUNET_log (GNUNET_ERROR_TYPE_INFO | GNUNET_ERROR_TYPE_BULK, 472 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG | GNUNET_ERROR_TYPE_BULK,
434 "No nick name set for zone `%s'\n", 473 "No nick name set for zone `%s'\n",
435 GNUNET_GNSRECORD_z2s (&pub)); 474 GNUNET_GNSRECORD_z2s (&pub));
436 return NULL; 475 return NULL;
@@ -439,9 +478,24 @@ get_nick_record (const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone)
439} 478}
440 479
441 480
481/**
482 * Merge the nick record @a nick_rd with the rest of the
483 * record set given in @a rd2. Store the result in @a rdc_res
484 * and @a rd_res. The @a nick_rd's expiration time is set to
485 * the maximum expiration time of all of the records in @a rd2.
486 *
487 * @param nick_rd the nick record to integrate
488 * @param rd2_length length of the @a rd2 array
489 * @param rd2 array of records
490 * @param rdc_res[out] length of the resulting @a rd_res array
491 * @param rd_res[out] set to an array of records,
492 * including @a nick_rd and @a rd2;
493 * all of the variable-size 'data' fields in @a rd2 are
494 * allocated in the same chunk of memory!
495 */
442static void 496static void
443merge_with_nick_records (const struct GNUNET_GNSRECORD_Data *nick_rd, 497merge_with_nick_records (const struct GNUNET_GNSRECORD_Data *nick_rd,
444 unsigned int rdc2, 498 unsigned int rd2_length,
445 const struct GNUNET_GNSRECORD_Data *rd2, 499 const struct GNUNET_GNSRECORD_Data *rd2,
446 unsigned int *rdc_res, 500 unsigned int *rdc_res,
447 struct GNUNET_GNSRECORD_Data **rd_res) 501 struct GNUNET_GNSRECORD_Data **rd_res)
@@ -452,24 +506,22 @@ merge_with_nick_records (const struct GNUNET_GNSRECORD_Data *nick_rd,
452 int record_offset; 506 int record_offset;
453 size_t data_offset; 507 size_t data_offset;
454 508
455 (*rdc_res) = 1 + rdc2; 509 (*rdc_res) = 1 + rd2_length;
456 if (0 == 1 + rdc2) 510 if (0 == 1 + rd2_length)
457 { 511 {
458 (*rd_res) = NULL; 512 (*rd_res) = NULL;
459 return; 513 return;
460 } 514 }
461
462 req = 0; 515 req = 0;
463 for (unsigned int c=0; c< 1; c++) 516 for (unsigned int c=0; c< 1; c++)
464 req += sizeof (struct GNUNET_GNSRECORD_Data) + nick_rd[c].data_size; 517 req += sizeof (struct GNUNET_GNSRECORD_Data) + nick_rd[c].data_size;
465 for (unsigned int c=0; c< rdc2; c++) 518 for (unsigned int c=0; c< rd2_length; c++)
466 req += sizeof (struct GNUNET_GNSRECORD_Data) + rd2[c].data_size; 519 req += sizeof (struct GNUNET_GNSRECORD_Data) + rd2[c].data_size;
467 (*rd_res) = GNUNET_malloc (req); 520 (*rd_res) = GNUNET_malloc (req);
468 data = (char *) &(*rd_res)[1 + rdc2]; 521 data = (char *) &(*rd_res)[1 + rd2_length];
469 data_offset = 0; 522 data_offset = 0;
470 latest_expiration = 0; 523 latest_expiration = 0;
471 524 for (unsigned int c=0; c< rd2_length; c++)
472 for (unsigned int c=0; c< rdc2; c++)
473 { 525 {
474 if (0 != (rd2[c].flags & GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION)) 526 if (0 != (rd2[c].flags & GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION))
475 { 527 {
@@ -486,17 +538,15 @@ merge_with_nick_records (const struct GNUNET_GNSRECORD_Data *nick_rd,
486 rd2[c].data_size); 538 rd2[c].data_size);
487 data_offset += (*rd_res)[c].data_size; 539 data_offset += (*rd_res)[c].data_size;
488 } 540 }
489 record_offset = rdc2; 541 /* append nick */
490 for (unsigned int c=0; c< 1; c++) 542 record_offset = rd2_length;
491 { 543 (*rd_res)[record_offset] = *nick_rd;
492 (*rd_res)[c+record_offset] = nick_rd[c]; 544 (*rd_res)[record_offset].expiration_time = latest_expiration;
493 (*rd_res)[c+record_offset].expiration_time = latest_expiration; 545 (*rd_res)[record_offset].data = (void *) &data[data_offset];
494 (*rd_res)[c+record_offset].data = (void *) &data[data_offset]; 546 GNUNET_memcpy ((void *) (*rd_res)[record_offset].data,
495 GNUNET_memcpy ((void *) (*rd_res)[c+record_offset].data, 547 nick_rd->data,
496 nick_rd[c].data, 548 nick_rd->data_size);
497 nick_rd[c].data_size); 549 data_offset += (*rd_res)[record_offset].data_size;
498 data_offset += (*rd_res)[c+record_offset].data_size;
499 }
500 GNUNET_assert (req == (sizeof (struct GNUNET_GNSRECORD_Data)) * (*rdc_res) + data_offset); 550 GNUNET_assert (req == (sizeof (struct GNUNET_GNSRECORD_Data)) * (*rdc_res) + data_offset);
501} 551}
502 552
@@ -531,7 +581,9 @@ send_lookup_response (struct NamestoreClient *nc,
531 char *rd_ser; 581 char *rd_ser;
532 582
533 nick = get_nick_record (zone_key); 583 nick = get_nick_record (zone_key);
534 if ((NULL != nick) && (0 != strcmp(name, GNUNET_GNS_MASTERZONE_STR))) 584 if ( (NULL != nick) &&
585 (0 != strcmp (name,
586 GNUNET_GNS_EMPTY_LABEL_AT)))
535 { 587 {
536 nick->flags = (nick->flags | GNUNET_GNSRECORD_RF_PRIVATE) ^ GNUNET_GNSRECORD_RF_PRIVATE; 588 nick->flags = (nick->flags | GNUNET_GNSRECORD_RF_PRIVATE) ^ GNUNET_GNSRECORD_RF_PRIVATE;
537 merge_with_nick_records (nick, 589 merge_with_nick_records (nick,
@@ -569,6 +621,10 @@ send_lookup_response (struct NamestoreClient *nc,
569 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 621 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
570 "Sending RECORD_RESULT message with %u records\n", 622 "Sending RECORD_RESULT message with %u records\n",
571 res_count); 623 res_count);
624 GNUNET_STATISTICS_update (statistics,
625 "Record sets sent to clients",
626 1,
627 GNUNET_NO);
572 GNUNET_MQ_send (nc->mq, 628 GNUNET_MQ_send (nc->mq,
573 env); 629 env);
574 if (rd != res) 630 if (rd != res)
@@ -659,6 +715,7 @@ refresh_block (struct NamestoreClient *nc,
659 struct GNUNET_GNSRECORD_Data *nick; 715 struct GNUNET_GNSRECORD_Data *nick;
660 struct GNUNET_GNSRECORD_Data *res; 716 struct GNUNET_GNSRECORD_Data *res;
661 unsigned int res_count; 717 unsigned int res_count;
718 struct GNUNET_TIME_Absolute exp_time;
662 719
663 nick = get_nick_record (zone_key); 720 nick = get_nick_record (zone_key);
664 res_count = rd_count; 721 res_count = rd_count;
@@ -672,26 +729,51 @@ refresh_block (struct NamestoreClient *nc,
672 &res); 729 &res);
673 GNUNET_free (nick); 730 GNUNET_free (nick);
674 } 731 }
675
676 if (0 == res_count) 732 if (0 == res_count)
677 block = GNUNET_GNSRECORD_block_create (zone_key, 733 {
678 GNUNET_TIME_UNIT_ZERO_ABS, 734 send_store_response (nc,
679 name, 735 GNUNET_OK,
680 res, rd_count); 736 rid);
737 return; /* no data, no need to update cache */
738 }
739 if (GNUNET_YES == disable_namecache)
740 {
741 GNUNET_STATISTICS_update (statistics,
742 "Namecache updates skipped (NC disabled)",
743 1,
744 GNUNET_NO);
745 send_store_response (nc,
746 GNUNET_OK,
747 rid);
748 return;
749 }
750 exp_time = GNUNET_GNSRECORD_record_get_expiration_time (res_count,
751 res);
752 if (cache_keys)
753 block = GNUNET_GNSRECORD_block_create2 (zone_key,
754 exp_time,
755 name,
756 res,
757 res_count);
681 else 758 else
682 block = GNUNET_GNSRECORD_block_create (zone_key, 759 block = GNUNET_GNSRECORD_block_create (zone_key,
683 GNUNET_GNSRECORD_record_get_expiration_time (res_count, 760 exp_time,
684 res),
685 name, 761 name,
686 res, res_count); 762 res,
763 res_count);
687 GNUNET_assert (NULL != block); 764 GNUNET_assert (NULL != block);
688 GNUNET_CRYPTO_ecdsa_key_get_public (zone_key, 765 GNUNET_CRYPTO_ecdsa_key_get_public (zone_key,
689 &pkey); 766 &pkey);
690 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 767 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
691 "Caching block for label `%s' with %u records in zone `%s' in namecache\n", 768 "Caching block for label `%s' with %u records and expiration %s in zone `%s' in namecache\n",
692 name, 769 name,
693 res_count, 770 res_count,
771 GNUNET_STRINGS_absolute_time_to_string (exp_time),
694 GNUNET_GNSRECORD_z2s (&pkey)); 772 GNUNET_GNSRECORD_z2s (&pkey));
773 GNUNET_STATISTICS_update (statistics,
774 "Namecache updates pushed",
775 1,
776 GNUNET_NO);
695 cop = GNUNET_new (struct CacheOperation); 777 cop = GNUNET_new (struct CacheOperation);
696 cop->nc = nc; 778 cop->nc = nc;
697 cop->rid = rid; 779 cop->rid = rid;
@@ -746,9 +828,11 @@ struct RecordLookupContext
746 828
747/** 829/**
748 * FIXME. 830 * FIXME.
831 * @param seq sequence number of the record
749 */ 832 */
750static void 833static void
751lookup_it (void *cls, 834lookup_it (void *cls,
835 uint64_t seq,
752 const struct GNUNET_CRYPTO_EcdsaPrivateKey *private_key, 836 const struct GNUNET_CRYPTO_EcdsaPrivateKey *private_key,
753 const char *label, 837 const char *label,
754 unsigned int rd_count, 838 unsigned int rd_count,
@@ -759,23 +843,26 @@ lookup_it (void *cls,
759 unsigned int rdc_res; 843 unsigned int rdc_res;
760 844
761 (void) private_key; 845 (void) private_key;
762 if (0 == strcmp (label, rlc->label)) 846 (void) seq;
847 if (0 == strcmp (label,
848 rlc->label))
763 { 849 {
764 rlc->found = GNUNET_YES; 850 rlc->found = GNUNET_YES;
765 if (0 != rd_count) 851 if (0 != rd_count)
766 { 852 {
767 if ( (NULL != rlc->nick) && 853 if ( (NULL != rlc->nick) &&
768 (0 != strcmp (label, 854 (0 != strcmp (label,
769 GNUNET_GNS_MASTERZONE_STR)) ) 855 GNUNET_GNS_EMPTY_LABEL_AT)) )
770 { 856 {
771 /* Merge */ 857 /* Merge */
772 rd_res = NULL; 858 rd_res = NULL;
773 rdc_res = 0; 859 rdc_res = 0;
774 rlc->nick->flags = (rlc->nick->flags | GNUNET_GNSRECORD_RF_PRIVATE) ^ GNUNET_GNSRECORD_RF_PRIVATE; 860 rlc->nick->flags = (rlc->nick->flags | GNUNET_GNSRECORD_RF_PRIVATE) ^ GNUNET_GNSRECORD_RF_PRIVATE;
775 merge_with_nick_records (rlc->nick, 861 merge_with_nick_records (rlc->nick,
776 rd_count, rd, 862 rd_count,
777 &rdc_res, &rd_res); 863 rd,
778 864 &rdc_res,
865 &rd_res);
779 rlc->rd_ser_len = GNUNET_GNSRECORD_records_get_size (rdc_res, 866 rlc->rd_ser_len = GNUNET_GNSRECORD_records_get_size (rdc_res,
780 rd_res); 867 rd_res);
781 rlc->res_rd_count = rdc_res; 868 rlc->res_rd_count = rdc_res;
@@ -945,7 +1032,8 @@ check_record_store (void *cls,
945 GNUNET_break (0); 1032 GNUNET_break (0);
946 return GNUNET_SYSERR; 1033 return GNUNET_SYSERR;
947 } 1034 }
948 if ((0 == name_len) || (name_len > MAX_NAME_LEN)) 1035 if ( (0 == name_len) ||
1036 (name_len > MAX_NAME_LEN) )
949 { 1037 {
950 GNUNET_break (0); 1038 GNUNET_break (0);
951 return GNUNET_SYSERR; 1039 return GNUNET_SYSERR;
@@ -979,7 +1067,6 @@ handle_record_store (void *cls,
979 const char *rd_ser; 1067 const char *rd_ser;
980 unsigned int rd_count; 1068 unsigned int rd_count;
981 int res; 1069 int res;
982 struct GNUNET_CRYPTO_EcdsaPublicKey pubkey;
983 struct ZoneMonitor *zm; 1070 struct ZoneMonitor *zm;
984 1071
985 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 1072 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
@@ -1006,8 +1093,6 @@ handle_record_store (void *cls,
1006 } 1093 }
1007 1094
1008 /* Extracting and converting private key */ 1095 /* Extracting and converting private key */
1009 GNUNET_CRYPTO_ecdsa_key_get_public (&rp_msg->private_key,
1010 &pubkey);
1011 conv_name = GNUNET_GNSRECORD_string_to_lowercase (name_tmp); 1096 conv_name = GNUNET_GNSRECORD_string_to_lowercase (name_tmp);
1012 if (NULL == conv_name) 1097 if (NULL == conv_name)
1013 { 1098 {
@@ -1018,18 +1103,16 @@ handle_record_store (void *cls,
1018 return; 1103 return;
1019 } 1104 }
1020 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 1105 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1021 "Creating %u records for name `%s' in zone `%s'\n", 1106 "Creating %u records for name `%s'\n",
1022 (unsigned int) rd_count, 1107 (unsigned int) rd_count,
1023 conv_name, 1108 conv_name);
1024 GNUNET_GNSRECORD_z2s (&pubkey));
1025
1026 if ( (0 == rd_count) && 1109 if ( (0 == rd_count) &&
1027 (GNUNET_NO == 1110 (GNUNET_NO ==
1028 GSN_database->iterate_records (GSN_database->cls, 1111 GSN_database->lookup_records (GSN_database->cls,
1029 &rp_msg->private_key, 1112 &rp_msg->private_key,
1030 0, 1113 conv_name,
1031 NULL, 1114 NULL,
1032 0)) ) 1115 0)) )
1033 { 1116 {
1034 /* This name does not exist, so cannot be removed */ 1117 /* This name does not exist, so cannot be removed */
1035 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 1118 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
@@ -1043,12 +1126,12 @@ handle_record_store (void *cls,
1043 unsigned int rd_clean_off; 1126 unsigned int rd_clean_off;
1044 1127
1045 /* remove "NICK" records, unless this is for the 1128 /* remove "NICK" records, unless this is for the
1046 #GNUNET_GNS_MASTERZONE_STR label */ 1129 #GNUNET_GNS_EMPTY_LABEL_AT label */
1047 rd_clean_off = 0; 1130 rd_clean_off = 0;
1048 for (unsigned int i=0;i<rd_count;i++) 1131 for (unsigned int i=0;i<rd_count;i++)
1049 { 1132 {
1050 rd_clean[rd_clean_off] = rd[i]; 1133 rd_clean[rd_clean_off] = rd[i];
1051 if ( (0 == strcmp (GNUNET_GNS_MASTERZONE_STR, 1134 if ( (0 == strcmp (GNUNET_GNS_EMPTY_LABEL_AT,
1052 conv_name)) || 1135 conv_name)) ||
1053 (GNUNET_GNSRECORD_TYPE_NICK != rd[i].record_type) ) 1136 (GNUNET_GNSRECORD_TYPE_NICK != rd[i].record_type) )
1054 rd_clean_off++; 1137 rd_clean_off++;
@@ -1062,7 +1145,8 @@ handle_record_store (void *cls,
1062 { 1145 {
1063 for (zm = monitor_head; NULL != zm; zm = zm->next) 1146 for (zm = monitor_head; NULL != zm; zm = zm->next)
1064 { 1147 {
1065 if ( (0 == memcmp (&rp_msg->private_key, &zm->zone, 1148 if ( (0 == memcmp (&rp_msg->private_key,
1149 &zm->zone,
1066 sizeof (struct GNUNET_CRYPTO_EcdsaPrivateKey))) || 1150 sizeof (struct GNUNET_CRYPTO_EcdsaPrivateKey))) ||
1067 (0 == memcmp (&zm->zone, 1151 (0 == memcmp (&zm->zone,
1068 &zero, 1152 &zero,
@@ -1142,6 +1226,7 @@ struct ZoneToNameCtx
1142 * Zone to name iterator 1226 * Zone to name iterator
1143 * 1227 *
1144 * @param cls struct ZoneToNameCtx * 1228 * @param cls struct ZoneToNameCtx *
1229 * @param seq sequence number of the record
1145 * @param zone_key the zone key 1230 * @param zone_key the zone key
1146 * @param name name 1231 * @param name name
1147 * @param rd_count number of records in @a rd 1232 * @param rd_count number of records in @a rd
@@ -1149,6 +1234,7 @@ struct ZoneToNameCtx
1149 */ 1234 */
1150static void 1235static void
1151handle_zone_to_name_it (void *cls, 1236handle_zone_to_name_it (void *cls,
1237 uint64_t seq,
1152 const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone_key, 1238 const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone_key,
1153 const char *name, 1239 const char *name,
1154 unsigned int rd_count, 1240 unsigned int rd_count,
@@ -1164,6 +1250,7 @@ handle_zone_to_name_it (void *cls,
1164 char *name_tmp; 1250 char *name_tmp;
1165 char *rd_tmp; 1251 char *rd_tmp;
1166 1252
1253 (void) seq;
1167 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 1254 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1168 "Found result for zone-to-name lookup: `%s'\n", 1255 "Found result for zone-to-name lookup: `%s'\n",
1169 name); 1256 name);
@@ -1252,29 +1339,6 @@ handle_zone_to_name (void *cls,
1252 1339
1253 1340
1254/** 1341/**
1255 * Zone iteration processor result
1256 */
1257enum ZoneIterationResult
1258{
1259 /**
1260 * Iteration start.
1261 */
1262 IT_START = 0,
1263
1264 /**
1265 * Found records,
1266 * Continue to iterate with next iteration_next call
1267 */
1268 IT_SUCCESS_MORE_AVAILABLE = 1,
1269
1270 /**
1271 * Iteration complete
1272 */
1273 IT_SUCCESS_NOT_MORE_RESULTS_AVAILABLE = 2
1274};
1275
1276
1277/**
1278 * Context for record remove operations passed from 1342 * Context for record remove operations passed from
1279 * #run_zone_iteration_round to #zone_iterate_proc as closure 1343 * #run_zone_iteration_round to #zone_iterate_proc as closure
1280 */ 1344 */
@@ -1286,13 +1350,9 @@ struct ZoneIterationProcResult
1286 struct ZoneIteration *zi; 1350 struct ZoneIteration *zi;
1287 1351
1288 /** 1352 /**
1289 * Iteration result: iteration done? 1353 * Number of results left to be returned in this iteration.
1290 * #IT_SUCCESS_MORE_AVAILABLE: if there may be more results overall but
1291 * we got one for now and have sent it to the client
1292 * #IT_SUCCESS_NOT_MORE_RESULTS_AVAILABLE: if there are no further results,
1293 * #IT_START: if we are still trying to find a result.
1294 */ 1354 */
1295 int res_iteration_finished; 1355 uint64_t limit;
1296 1356
1297}; 1357};
1298 1358
@@ -1300,7 +1360,8 @@ struct ZoneIterationProcResult
1300/** 1360/**
1301 * Process results for zone iteration from database 1361 * Process results for zone iteration from database
1302 * 1362 *
1303 * @param cls struct ZoneIterationProcResult *proc 1363 * @param cls struct ZoneIterationProcResult
1364 * @param seq sequence number of the record
1304 * @param zone_key the zone key 1365 * @param zone_key the zone key
1305 * @param name name 1366 * @param name name
1306 * @param rd_count number of records for this name 1367 * @param rd_count number of records for this name
@@ -1308,6 +1369,7 @@ struct ZoneIterationProcResult
1308 */ 1369 */
1309static void 1370static void
1310zone_iterate_proc (void *cls, 1371zone_iterate_proc (void *cls,
1372 uint64_t seq,
1311 const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone_key, 1373 const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone_key,
1312 const char *name, 1374 const char *name,
1313 unsigned int rd_count, 1375 unsigned int rd_count,
@@ -1316,21 +1378,28 @@ zone_iterate_proc (void *cls,
1316 struct ZoneIterationProcResult *proc = cls; 1378 struct ZoneIterationProcResult *proc = cls;
1317 int do_refresh_block; 1379 int do_refresh_block;
1318 1380
1319 if ((NULL == zone_key) && (NULL == name)) 1381 if ( (NULL == zone_key) &&
1382 (NULL == name) )
1320 { 1383 {
1321 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 1384 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1322 "Iteration done\n"); 1385 "Iteration done\n");
1323 proc->res_iteration_finished = IT_SUCCESS_NOT_MORE_RESULTS_AVAILABLE;
1324 return; 1386 return;
1325 } 1387 }
1326 if ((NULL == zone_key) || (NULL == name)) 1388 if ( (NULL == zone_key) ||
1389 (NULL == name) )
1327 { 1390 {
1328 /* what is this!? should never happen */ 1391 /* what is this!? should never happen */
1329 proc->res_iteration_finished = IT_START;
1330 GNUNET_break (0); 1392 GNUNET_break (0);
1331 return; 1393 return;
1332 } 1394 }
1333 proc->res_iteration_finished = IT_SUCCESS_MORE_AVAILABLE; 1395 if (0 == proc->limit)
1396 {
1397 /* what is this!? should never happen */
1398 GNUNET_break (0);
1399 return;
1400 }
1401 proc->limit--;
1402 proc->zi->seq = seq;
1334 send_lookup_response (proc->zi->nc, 1403 send_lookup_response (proc->zi->nc,
1335 proc->zi->request_id, 1404 proc->zi->request_id,
1336 zone_key, 1405 zone_key,
@@ -1351,7 +1420,6 @@ zone_iterate_proc (void *cls,
1351 name, 1420 name,
1352 rd_count, 1421 rd_count,
1353 rd); 1422 rd);
1354
1355} 1423}
1356 1424
1357 1425
@@ -1359,41 +1427,58 @@ zone_iterate_proc (void *cls,
1359 * Perform the next round of the zone iteration. 1427 * Perform the next round of the zone iteration.
1360 * 1428 *
1361 * @param zi zone iterator to process 1429 * @param zi zone iterator to process
1430 * @param limit number of results to return in one pass
1362 */ 1431 */
1363static void 1432static void
1364run_zone_iteration_round (struct ZoneIteration *zi) 1433run_zone_iteration_round (struct ZoneIteration *zi,
1434 uint64_t limit)
1365{ 1435{
1366 struct ZoneIterationProcResult proc; 1436 struct ZoneIterationProcResult proc;
1367 struct GNUNET_MQ_Envelope *env; 1437 struct GNUNET_MQ_Envelope *env;
1368 struct RecordResultMessage *rrm; 1438 struct RecordResultMessage *rrm;
1369 int ret; 1439 struct GNUNET_TIME_Absolute start;
1440 struct GNUNET_TIME_Relative duration;
1370 1441
1371 memset (&proc, 0, sizeof (proc)); 1442 memset (&proc,
1443 0,
1444 sizeof (proc));
1445 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1446 "Asked to return up to %llu records at position %llu\n",
1447 (unsigned long long) limit,
1448 (unsigned long long) zi->seq);
1372 proc.zi = zi; 1449 proc.zi = zi;
1373 proc.res_iteration_finished = IT_START; 1450 proc.limit = limit;
1374 while (IT_START == proc.res_iteration_finished) 1451 start = GNUNET_TIME_absolute_get ();
1375 { 1452 GNUNET_break (GNUNET_SYSERR !=
1376 if (GNUNET_SYSERR == 1453 GSN_database->iterate_records (GSN_database->cls,
1377 (ret = GSN_database->iterate_records (GSN_database->cls, 1454 (0 == memcmp (&zi->zone,
1378 (0 == memcmp (&zi->zone, &zero, sizeof (zero))) 1455 &zero,
1379 ? NULL 1456 sizeof (zero)))
1380 : &zi->zone, 1457 ? NULL
1381 zi->offset, 1458 : &zi->zone,
1382 &zone_iterate_proc, &proc))) 1459 zi->seq,
1383 { 1460 limit,
1384 GNUNET_break (0); 1461 &zone_iterate_proc,
1385 break; 1462 &proc));
1386 } 1463 duration = GNUNET_TIME_absolute_get_duration (start);
1387 if (GNUNET_NO == ret) 1464 duration = GNUNET_TIME_relative_divide (duration,
1388 proc.res_iteration_finished = IT_SUCCESS_NOT_MORE_RESULTS_AVAILABLE; 1465 limit - proc.limit);
1389 zi->offset++; 1466 GNUNET_STATISTICS_set (statistics,
1390 } 1467 "NAMESTORE iteration delay (μs/record)",
1391 if (IT_SUCCESS_MORE_AVAILABLE == proc.res_iteration_finished) 1468 duration.rel_value_us,
1469 GNUNET_NO);
1470 if (0 == proc.limit)
1392 { 1471 {
1393 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 1472 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1394 "More results available\n"); 1473 "Returned %llu results, more results available\n",
1395 return; /* more results later */ 1474 (unsigned long long) limit);
1475 return; /* more results later after we get the
1476 #GNUNET_MESSAGE_TYPE_NAMESTORE_ZONE_ITERATION_NEXT message */
1396 } 1477 }
1478 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1479 "Completed iteration after %llu/%llu results\n",
1480 (unsigned long long) (limit - proc.limit),
1481 (unsigned long long) limit);
1397 /* send empty response to indicate end of list */ 1482 /* send empty response to indicate end of list */
1398 env = GNUNET_MQ_msg (rrm, 1483 env = GNUNET_MQ_msg (rrm,
1399 GNUNET_MESSAGE_TYPE_NAMESTORE_RECORD_RESULT); 1484 GNUNET_MESSAGE_TYPE_NAMESTORE_RECORD_RESULT);
@@ -1431,7 +1516,8 @@ handle_iteration_start (void *cls,
1431 GNUNET_CONTAINER_DLL_insert (nc->op_head, 1516 GNUNET_CONTAINER_DLL_insert (nc->op_head,
1432 nc->op_tail, 1517 nc->op_tail,
1433 zi); 1518 zi);
1434 run_zone_iteration_round (zi); 1519 run_zone_iteration_round (zi,
1520 1);
1435 GNUNET_SERVICE_client_continue (nc->client); 1521 GNUNET_SERVICE_client_continue (nc->client);
1436} 1522}
1437 1523
@@ -1451,8 +1537,7 @@ handle_iteration_stop (void *cls,
1451 uint32_t rid; 1537 uint32_t rid;
1452 1538
1453 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 1539 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1454 "Received `%s' message\n", 1540 "Received ZONE_ITERATION_STOP message\n");
1455 "ZONE_ITERATION_STOP");
1456 rid = ntohl (zis_msg->gns_header.r_id); 1541 rid = ntohl (zis_msg->gns_header.r_id);
1457 for (zi = nc->op_head; NULL != zi; zi = zi->next) 1542 for (zi = nc->op_head; NULL != zi; zi = zi->next)
1458 if (zi->request_id == rid) 1543 if (zi->request_id == rid)
@@ -1484,10 +1569,16 @@ handle_iteration_next (void *cls,
1484 struct NamestoreClient *nc = cls; 1569 struct NamestoreClient *nc = cls;
1485 struct ZoneIteration *zi; 1570 struct ZoneIteration *zi;
1486 uint32_t rid; 1571 uint32_t rid;
1572 uint64_t limit;
1487 1573
1488 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 1574 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1489 "Received ZONE_ITERATION_NEXT message\n"); 1575 "Received ZONE_ITERATION_NEXT message\n");
1576 GNUNET_STATISTICS_update (statistics,
1577 "Iteration NEXT messages received",
1578 1,
1579 GNUNET_NO);
1490 rid = ntohl (zis_msg->gns_header.r_id); 1580 rid = ntohl (zis_msg->gns_header.r_id);
1581 limit = GNUNET_ntohll (zis_msg->limit);
1491 for (zi = nc->op_head; NULL != zi; zi = zi->next) 1582 for (zi = nc->op_head; NULL != zi; zi = zi->next)
1492 if (zi->request_id == rid) 1583 if (zi->request_id == rid)
1493 break; 1584 break;
@@ -1497,7 +1588,8 @@ handle_iteration_next (void *cls,
1497 GNUNET_SERVICE_client_drop (nc->client); 1588 GNUNET_SERVICE_client_drop (nc->client);
1498 return; 1589 return;
1499 } 1590 }
1500 run_zone_iteration_round (zi); 1591 run_zone_iteration_round (zi,
1592 limit);
1501 GNUNET_SERVICE_client_continue (nc->client); 1593 GNUNET_SERVICE_client_continue (nc->client);
1502} 1594}
1503 1595
@@ -1533,6 +1625,7 @@ monitor_next (void *cls);
1533 * A #GNUNET_NAMESTORE_RecordIterator for monitors. 1625 * A #GNUNET_NAMESTORE_RecordIterator for monitors.
1534 * 1626 *
1535 * @param cls a 'struct ZoneMonitor *' with information about the monitor 1627 * @param cls a 'struct ZoneMonitor *' with information about the monitor
1628 * @param seq sequence number of the record
1536 * @param zone_key zone key of the zone 1629 * @param zone_key zone key of the zone
1537 * @param name name 1630 * @param name name
1538 * @param rd_count number of records in @a rd 1631 * @param rd_count number of records in @a rd
@@ -1540,6 +1633,7 @@ monitor_next (void *cls);
1540 */ 1633 */
1541static void 1634static void
1542monitor_iterate_cb (void *cls, 1635monitor_iterate_cb (void *cls,
1636 uint64_t seq,
1543 const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone_key, 1637 const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone_key,
1544 const char *name, 1638 const char *name,
1545 unsigned int rd_count, 1639 unsigned int rd_count,
@@ -1547,12 +1641,17 @@ monitor_iterate_cb (void *cls,
1547{ 1641{
1548 struct ZoneMonitor *zm = cls; 1642 struct ZoneMonitor *zm = cls;
1549 1643
1644 zm->seq = seq;
1550 if (NULL == name) 1645 if (NULL == name)
1551 { 1646 {
1552 /* finished with iteration */ 1647 /* finished with iteration */
1553 monitor_sync (zm); 1648 monitor_sync (zm);
1554 return; 1649 return;
1555 } 1650 }
1651 GNUNET_STATISTICS_update (statistics,
1652 "Monitor notifications sent",
1653 1,
1654 GNUNET_NO);
1556 send_lookup_response (zm->nc, 1655 send_lookup_response (zm->nc,
1557 0, 1656 0,
1558 zone_key, 1657 zone_key,
@@ -1598,7 +1697,7 @@ handle_monitor_start (void *cls,
1598 1697
1599 1698
1600/** 1699/**
1601 * Obtain the next datum during the zone monitor's zone intiial iteration. 1700 * Obtain the next datum during the zone monitor's zone initial iteration.
1602 * 1701 *
1603 * @param cls zone monitor that does its initial iteration 1702 * @param cls zone monitor that does its initial iteration
1604 */ 1703 */
@@ -1615,7 +1714,8 @@ monitor_next (void *cls)
1615 sizeof (zero))) 1714 sizeof (zero)))
1616 ? NULL 1715 ? NULL
1617 : &zm->zone, 1716 : &zm->zone,
1618 zm->offset++, 1717 zm->seq,
1718 1,
1619 &monitor_iterate_cb, 1719 &monitor_iterate_cb,
1620 zm); 1720 zm);
1621 if (GNUNET_SYSERR == ret) 1721 if (GNUNET_SYSERR == ret)
@@ -1650,9 +1750,19 @@ run (void *cls,
1650 (void) service; 1750 (void) service;
1651 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 1751 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1652 "Starting namestore service\n"); 1752 "Starting namestore service\n");
1753 cache_keys = GNUNET_CONFIGURATION_get_value_yesno (cfg,
1754 "namestore",
1755 "CACHE_KEYS");
1756 disable_namecache = GNUNET_CONFIGURATION_get_value_yesno (cfg,
1757 "namecache",
1758 "DISABLE");
1653 GSN_cfg = cfg; 1759 GSN_cfg = cfg;
1654 monitor_nc = GNUNET_notification_context_create (1); 1760 monitor_nc = GNUNET_notification_context_create (1);
1655 namecache = GNUNET_NAMECACHE_connect (cfg); 1761 if (GNUNET_YES != disable_namecache)
1762 {
1763 namecache = GNUNET_NAMECACHE_connect (cfg);
1764 GNUNET_assert (NULL != namecache);
1765 }
1656 /* Loading database plugin */ 1766 /* Loading database plugin */
1657 if (GNUNET_OK != 1767 if (GNUNET_OK !=
1658 GNUNET_CONFIGURATION_get_value_string (cfg, 1768 GNUNET_CONFIGURATION_get_value_string (cfg,
@@ -1668,6 +1778,8 @@ run (void *cls,
1668 GSN_database = GNUNET_PLUGIN_load (db_lib_name, 1778 GSN_database = GNUNET_PLUGIN_load (db_lib_name,
1669 (void *) GSN_cfg); 1779 (void *) GSN_cfg);
1670 GNUNET_free (database); 1780 GNUNET_free (database);
1781 statistics = GNUNET_STATISTICS_create ("namestore",
1782 cfg);
1671 GNUNET_SCHEDULER_add_shutdown (&cleanup_task, 1783 GNUNET_SCHEDULER_add_shutdown (&cleanup_task,
1672 NULL); 1784 NULL);
1673 if (NULL == GSN_database) 1785 if (NULL == GSN_database)
diff --git a/src/namestore/gnunet-zoneimport.c b/src/namestore/gnunet-zoneimport.c
index 4f4151c94..0fd0a4ab8 100644
--- a/src/namestore/gnunet-zoneimport.c
+++ b/src/namestore/gnunet-zoneimport.c
@@ -21,9 +21,6 @@
21 * @file src/namestore/gnunet-zoneimport.c 21 * @file src/namestore/gnunet-zoneimport.c
22 * @brief import a DNS zone for publication in GNS, incremental 22 * @brief import a DNS zone for publication in GNS, incremental
23 * @author Christian Grothoff 23 * @author Christian Grothoff
24 *
25 * TODO:
26 * - set NICKname for zone's records
27 */ 24 */
28#include "platform.h" 25#include "platform.h"
29#include <gnunet_util_lib.h> 26#include <gnunet_util_lib.h>
@@ -31,13 +28,14 @@
31#include <gnunet_dnsparser_lib.h> 28#include <gnunet_dnsparser_lib.h>
32#include <gnunet_gnsrecord_lib.h> 29#include <gnunet_gnsrecord_lib.h>
33#include <gnunet_namestore_service.h> 30#include <gnunet_namestore_service.h>
31#include <gnunet_statistics_service.h>
34#include <gnunet_identity_service.h> 32#include <gnunet_identity_service.h>
35 33
36 34
37/** 35/**
38 * Maximum number of queries pending at the same time. 36 * Maximum number of queries pending at the same time.
39 */ 37 */
40#define THRESH 20 38#define THRESH 100
41 39
42/** 40/**
43 * TIME_THRESH is in usecs. How quickly do we submit fresh queries. 41 * TIME_THRESH is in usecs. How quickly do we submit fresh queries.
@@ -51,10 +49,20 @@
51#define MAX_RETRIES 5 49#define MAX_RETRIES 5
52 50
53/** 51/**
54 * After how many lookups should we always sync to disk? 52 * How many DNS requests do we at most issue in rapid series?
53 */
54#define MAX_SERIES 10
55
56/**
57 * How long do we wait at least between series of requests?
55 */ 58 */
56#define TRANSACTION_SYNC_FREQ 100 59#define SERIES_DELAY GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MICROSECONDS, 10)
57 60
61/**
62 * How many requests do we request from NAMESTORE in one batch
63 * during our initial iteration?
64 */
65#define NS_BATCH_SIZE 1024
58 66
59/** 67/**
60 * Some zones may include authoritative records for other 68 * Some zones may include authoritative records for other
@@ -112,7 +120,9 @@ struct Record
112 120
113 121
114/** 122/**
115 * Request we should make. 123 * Request we should make. We keep this struct in memory per request,
124 * thus optimizing it is crucial for the overall memory consumption of
125 * the zone importer.
116 */ 126 */
117struct Request 127struct Request
118{ 128{
@@ -149,21 +159,13 @@ struct Request
149 struct GNUNET_DNSSTUB_RequestSocket *rs; 159 struct GNUNET_DNSSTUB_RequestSocket *rs;
150 160
151 /** 161 /**
152 * Raw DNS query. 162 * Hostname we are resolving, allocated at the end of
153 */ 163 * this struct (optimizing memory consumption by reducing
154 void *raw; 164 * total number of allocations).
155
156 /**
157 * Hostname we are resolving.
158 */ 165 */
159 char *hostname; 166 char *hostname;
160 167
161 /** 168 /**
162 * Label (without TLD) which we are resolving.
163 */
164 char *label;
165
166 /**
167 * Namestore operation pending for this record. 169 * Namestore operation pending for this record.
168 */ 170 */
169 struct GNUNET_NAMESTORE_QueueEntry *qe; 171 struct GNUNET_NAMESTORE_QueueEntry *qe;
@@ -174,12 +176,6 @@ struct Request
174 const struct Zone *zone; 176 const struct Zone *zone;
175 177
176 /** 178 /**
177 * Answer we got back and are currently parsing, or NULL
178 * if not active.
179 */
180 struct GNUNET_DNSPARSER_Packet *p;
181
182 /**
183 * At what time does the (earliest) of the returned records 179 * At what time does the (earliest) of the returned records
184 * for this name expire? At this point, we need to re-fetch 180 * for this name expire? At this point, we need to re-fetch
185 * the record. 181 * the record.
@@ -187,20 +183,18 @@ struct Request
187 struct GNUNET_TIME_Absolute expires; 183 struct GNUNET_TIME_Absolute expires;
188 184
189 /** 185 /**
190 * Number of bytes in @e raw. 186 * While we are fetching the record, the value is set to the
187 * starting time of the DNS operation. While doing a
188 * NAMESTORE store, again set to the start time of the
189 * NAMESTORE operation.
191 */ 190 */
192 size_t raw_len; 191 struct GNUNET_TIME_Absolute op_start_time;
193
194 /**
195 * When did we last issue this request?
196 */
197 time_t time;
198 192
199 /** 193 /**
200 * How often did we issue this query? (And failed, reset 194 * How often did we issue this query? (And failed, reset
201 * to zero once we were successful.) 195 * to zero once we were successful.)
202 */ 196 */
203 int issue_num; 197 unsigned int issue_num;
204 198
205 /** 199 /**
206 * random 16-bit DNS query identifier. 200 * random 16-bit DNS query identifier.
@@ -210,6 +204,14 @@ struct Request
210 204
211 205
212/** 206/**
207 * Command-line argument specifying desired size of the hash map with
208 * all of our pending names. Usually, we use an automatically growing
209 * map, but this is only OK up to about a million entries. Above that
210 * number, the user must explicitly specify the size at startup.
211 */
212static unsigned int map_size = 1024;
213
214/**
213 * Handle to the identity service. 215 * Handle to the identity service.
214 */ 216 */
215static struct GNUNET_IDENTITY_Handle *id; 217static struct GNUNET_IDENTITY_Handle *id;
@@ -220,21 +222,36 @@ static struct GNUNET_IDENTITY_Handle *id;
220static struct GNUNET_NAMESTORE_Handle *ns; 222static struct GNUNET_NAMESTORE_Handle *ns;
221 223
222/** 224/**
225 * Handle to the statistics service.
226 */
227static struct GNUNET_STATISTICS_Handle *stats;
228
229/**
223 * Context for DNS resolution. 230 * Context for DNS resolution.
224 */ 231 */
225static struct GNUNET_DNSSTUB_Context *ctx; 232static struct GNUNET_DNSSTUB_Context *ctx;
226 233
227/** 234/**
228 * The number of queries that are outstanding 235 * The number of DNS queries that are outstanding
229 */ 236 */
230static unsigned int pending; 237static unsigned int pending;
231 238
232/** 239/**
240 * The number of NAMESTORE record store operations that are outstanding
241 */
242static unsigned int pending_rs;
243
244/**
233 * Number of lookups we performed overall. 245 * Number of lookups we performed overall.
234 */ 246 */
235static unsigned int lookups; 247static unsigned int lookups;
236 248
237/** 249/**
250 * Number of records we had cached.
251 */
252static unsigned int cached;
253
254/**
238 * How many hostnames did we reject (malformed). 255 * How many hostnames did we reject (malformed).
239 */ 256 */
240static unsigned int rejects; 257static unsigned int rejects;
@@ -250,6 +267,11 @@ static unsigned int failures;
250static unsigned int records; 267static unsigned int records;
251 268
252/** 269/**
270 * Number of record sets given to namestore.
271 */
272static unsigned int record_sets;
273
274/**
253 * Heap of all requests to perform, sorted by 275 * Heap of all requests to perform, sorted by
254 * the time we should next do the request (i.e. by expires). 276 * the time we should next do the request (i.e. by expires).
255 */ 277 */
@@ -271,9 +293,16 @@ static struct Request *req_tail;
271static struct GNUNET_SCHEDULER_Task *t; 293static struct GNUNET_SCHEDULER_Task *t;
272 294
273/** 295/**
274 * Which DNS server do we use for queries? 296 * Hash map of requests for which we may still get a response from
297 * the namestore. Set to NULL once the initial namestore iteration
298 * is done.
299 */
300static struct GNUNET_CONTAINER_MultiHashMap *ns_pending;
301
302/**
303 * Current zone iteration handle.
275 */ 304 */
276static char *dns_server; 305static struct GNUNET_NAMESTORE_ZoneIterator *zone_it;
277 306
278/** 307/**
279 * Head of list of zones we are managing. 308 * Head of list of zones we are managing.
@@ -285,6 +314,47 @@ static struct Zone *zone_head;
285 */ 314 */
286static struct Zone *zone_tail; 315static struct Zone *zone_tail;
287 316
317/**
318 * After how many more results must #ns_lookup_result_cb() ask
319 * the namestore for more?
320 */
321static uint64_t ns_iterator_trigger_next;
322
323/**
324 * Number of DNS requests counted in latency total.
325 */
326static uint64_t total_dns_latency_cnt;
327
328/**
329 * Sum of DNS latencies observed.
330 */
331static struct GNUNET_TIME_Relative total_dns_latency;
332
333/**
334 * Number of records processed (DNS lookup, no NAMESTORE) in total.
335 */
336static uint64_t total_reg_proc_dns;
337
338/**
339 * Number of records processed (DNS lookup, with NAMESTORE) in total.
340 */
341static uint64_t total_reg_proc_dns_ns;
342
343/**
344 * Start time of the regular processing.
345 */
346static struct GNUNET_TIME_Absolute start_time_reg_proc;
347
348/**
349 * Last time we worked before going idle.
350 */
351static struct GNUNET_TIME_Absolute sleep_time_reg_proc;
352
353/**
354 * Time we slept just waiting for work.
355 */
356static struct GNUNET_TIME_Relative idle_time;
357
288 358
289/** 359/**
290 * Callback for #for_all_records 360 * Callback for #for_all_records
@@ -335,15 +405,107 @@ for_all_records (const struct GNUNET_DNSPARSER_Packet *p,
335 405
336 406
337/** 407/**
338 * Free @a req and data structures reachable from it. 408 * Return just the label of the hostname in @a req.
339 * 409 *
340 * @param req request to free 410 * @param req request to process hostname of
411 * @return statically allocated pointer to the label,
412 * overwritten upon the next request!
413 */
414static const char *
415get_label (struct Request *req)
416{
417 static char label[64];
418 const char *dot;
419
420 dot = strchr (req->hostname,
421 (unsigned char) '.');
422 if (NULL == dot)
423 {
424 GNUNET_break (0);
425 return NULL;
426 }
427 if (((size_t) (dot - req->hostname)) >= sizeof (label))
428 {
429 GNUNET_break (0);
430 return NULL;
431 }
432 memcpy (label,
433 req->hostname,
434 dot - req->hostname);
435 label[dot - req->hostname] = '\0';
436 return label;
437}
438
439
440/**
441 * Build DNS query for @a hostname.
442 *
443 * @param hostname host to build query for
444 * @param raw_size[out] number of bytes in the query
445 * @return NULL on error, otherwise pointer to statically (!)
446 * allocated query buffer
447 */
448static void *
449build_dns_query (struct Request *req,
450 size_t *raw_size)
451{
452 static char raw[512];
453 char *rawp;
454 struct GNUNET_DNSPARSER_Packet p;
455 struct GNUNET_DNSPARSER_Query q;
456
457 q.name = (char *) req->hostname;
458 q.type = GNUNET_DNSPARSER_TYPE_NS;
459 q.dns_traffic_class = GNUNET_TUN_DNS_CLASS_INTERNET;
460
461 memset (&p,
462 0,
463 sizeof (p));
464 p.num_queries = 1;
465 p.queries = &q;
466 p.id = req->id;
467 if (GNUNET_OK !=
468 GNUNET_DNSPARSER_pack (&p,
469 UINT16_MAX,
470 &rawp,
471 raw_size))
472 {
473 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
474 "Failed to pack query for hostname `%s'\n",
475 req->hostname);
476 rejects++;
477 return NULL;
478 }
479 if (*raw_size > sizeof (raw))
480 {
481 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
482 "Failed to pack query for hostname `%s'\n",
483 req->hostname);
484 rejects++;
485 GNUNET_break (0);
486 GNUNET_free (rawp);
487 return NULL;
488 }
489 memcpy (raw,
490 rawp,
491 *raw_size);
492 GNUNET_free (rawp);
493 return raw;
494}
495
496
497
498/**
499 * Free records associated with @a req.
500 *
501 * @param req request to free records of
341 */ 502 */
342static void 503static void
343free_request (struct Request *req) 504free_records (struct Request *req)
344{ 505{
345 struct Record *rec; 506 struct Record *rec;
346 507
508 /* Free records */
347 while (NULL != (rec = req->rec_head)) 509 while (NULL != (rec = req->rec_head))
348 { 510 {
349 GNUNET_CONTAINER_DLL_remove (req->rec_head, 511 GNUNET_CONTAINER_DLL_remove (req->rec_head,
@@ -351,9 +513,18 @@ free_request (struct Request *req)
351 rec); 513 rec);
352 GNUNET_free (rec); 514 GNUNET_free (rec);
353 } 515 }
354 GNUNET_free (req->hostname); 516}
355 GNUNET_free (req->label); 517
356 GNUNET_free (req->raw); 518
519/**
520 * Free @a req and data structures reachable from it.
521 *
522 * @param req request to free
523 */
524static void
525free_request (struct Request *req)
526{
527 free_records (req);
357 GNUNET_free (req); 528 GNUNET_free (req);
358} 529}
359 530
@@ -382,6 +553,7 @@ insert_sorted (struct Request *req)
382 { 553 {
383 if (NULL != t) 554 if (NULL != t)
384 GNUNET_SCHEDULER_cancel (t); 555 GNUNET_SCHEDULER_cancel (t);
556 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
385 t = GNUNET_SCHEDULER_add_at (req->expires, 557 t = GNUNET_SCHEDULER_add_at (req->expires,
386 &process_queue, 558 &process_queue,
387 NULL); 559 NULL);
@@ -564,16 +736,35 @@ check_for_glue (void *cls,
564 736
565 737
566/** 738/**
739 * Closure for #process_record().
740 */
741struct ProcessRecordContext
742{
743 /**
744 * Answer we got back and are currently parsing, or NULL
745 * if not active.
746 */
747 struct GNUNET_DNSPARSER_Packet *p;
748
749 /**
750 * Request we are processing.
751 */
752 struct Request *req;
753};
754
755
756/**
567 * We received @a rec for @a req. Remember the answer. 757 * We received @a rec for @a req. Remember the answer.
568 * 758 *
569 * @param cls a `struct Request` 759 * @param cls a `struct ProcessRecordContext`
570 * @param rec response 760 * @param rec response
571 */ 761 */
572static void 762static void
573process_record (void *cls, 763process_record (void *cls,
574 const struct GNUNET_DNSPARSER_Record *rec) 764 const struct GNUNET_DNSPARSER_Record *rec)
575{ 765{
576 struct Request *req = cls; 766 struct ProcessRecordContext *prc = cls;
767 struct Request *req = prc->req;
577 char dst[65536]; 768 char dst[65536];
578 size_t dst_len; 769 size_t dst_len;
579 size_t off; 770 size_t off;
@@ -585,7 +776,7 @@ process_record (void *cls,
585 req->hostname)) 776 req->hostname))
586 { 777 {
587 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 778 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
588 "DNS returned record for `%s' of type %u while resolving `%s'\n", 779 "DNS returned record from zone `%s' of type %u while resolving `%s'\n",
589 rec->name, 780 rec->name,
590 (unsigned int) rec->type, 781 (unsigned int) rec->type,
591 req->hostname); 782 req->hostname);
@@ -600,6 +791,10 @@ process_record (void *cls,
600 req->hostname); 791 req->hostname);
601 return; /* record expired */ 792 return; /* record expired */
602 } 793 }
794 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
795 "DNS returned record that expires at %s for `%s'\n",
796 GNUNET_STRINGS_absolute_time_to_string (rec->expiration_time),
797 req->hostname);
603 switch (rec->type) 798 switch (rec->type)
604 { 799 {
605 case GNUNET_DNSPARSER_TYPE_NS: 800 case GNUNET_DNSPARSER_TYPE_NS:
@@ -610,7 +805,7 @@ process_record (void *cls,
610 gc.req = req; 805 gc.req = req;
611 gc.ns = rec->data.hostname; 806 gc.ns = rec->data.hostname;
612 gc.found = GNUNET_NO; 807 gc.found = GNUNET_NO;
613 for_all_records (req->p, 808 for_all_records (prc->p,
614 &check_for_glue, 809 &check_for_glue,
615 &gc); 810 &gc);
616 if ( (GNUNET_NO == gc.found) && 811 if ( (GNUNET_NO == gc.found) &&
@@ -665,10 +860,10 @@ process_record (void *cls,
665 break; 860 break;
666 case GNUNET_DNSPARSER_TYPE_DNAME: 861 case GNUNET_DNSPARSER_TYPE_DNAME:
667 /* No support for DNAME in GNS yet! FIXME: support later! */ 862 /* No support for DNAME in GNS yet! FIXME: support later! */
668 fprintf (stdout, 863 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
669 "FIXME: not supported: %s DNAME %s\n", 864 "FIXME: not supported: %s DNAME %s\n",
670 rec->name, 865 rec->name,
671 rec->data.hostname); 866 rec->data.hostname);
672 break; 867 break;
673 case GNUNET_DNSPARSER_TYPE_MX: 868 case GNUNET_DNSPARSER_TYPE_MX:
674 if (GNUNET_OK == 869 if (GNUNET_OK ==
@@ -794,10 +989,10 @@ store_completed_cb (void *cls,
794 int32_t success, 989 int32_t success,
795 const char *emsg) 990 const char *emsg)
796{ 991{
992 static struct GNUNET_TIME_Absolute last;
797 struct Request *req = cls; 993 struct Request *req = cls;
798 994
799 req->qe = NULL; 995 req->qe = NULL;
800 pending--;
801 if (GNUNET_SYSERR == success) 996 if (GNUNET_SYSERR == success)
802 { 997 {
803 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 998 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
@@ -807,9 +1002,75 @@ store_completed_cb (void *cls,
807 } 1002 }
808 else 1003 else
809 { 1004 {
810 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1005 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
811 "Stored records under `%s'\n", 1006 "Stored records under `%s' (%d)\n",
812 req->label); 1007 req->hostname,
1008 success);
1009 }
1010 total_reg_proc_dns_ns++; /* finished regular processing */
1011 pending_rs--;
1012 free_records (req);
1013 /* compute NAMESTORE statistics */
1014 {
1015 static uint64_t total_ns_latency_cnt;
1016 static struct GNUNET_TIME_Relative total_ns_latency;
1017 struct GNUNET_TIME_Relative ns_latency;
1018
1019 ns_latency = GNUNET_TIME_absolute_get_duration (req->op_start_time);
1020 total_ns_latency = GNUNET_TIME_relative_add (total_ns_latency,
1021 ns_latency);
1022 if (0 == total_ns_latency_cnt)
1023 last = GNUNET_TIME_absolute_get ();
1024 total_ns_latency_cnt++;
1025 if (0 == (total_ns_latency_cnt % 1000))
1026 {
1027 struct GNUNET_TIME_Relative delta;
1028
1029 delta = GNUNET_TIME_absolute_get_duration (last);
1030 last = GNUNET_TIME_absolute_get ();
1031 fprintf (stderr,
1032 "Processed 1000 records in %s\n",
1033 GNUNET_STRINGS_relative_time_to_string (delta,
1034 GNUNET_YES));
1035 GNUNET_STATISTICS_set (stats,
1036 "# average NAMESTORE PUT latency (μs)",
1037 total_ns_latency.rel_value_us / total_ns_latency_cnt,
1038 GNUNET_NO);
1039 }
1040 }
1041 /* compute and publish overall velocity */
1042 if (0 == (total_reg_proc_dns_ns % 100) )
1043 {
1044 struct GNUNET_TIME_Relative runtime;
1045
1046 runtime = GNUNET_TIME_absolute_get_duration (start_time_reg_proc);
1047 runtime = GNUNET_TIME_relative_subtract (runtime,
1048 idle_time);
1049 runtime = GNUNET_TIME_relative_divide (runtime,
1050 total_reg_proc_dns + total_reg_proc_dns_ns);
1051 GNUNET_STATISTICS_set (stats,
1052 "# Regular processing completed without NAMESTORE",
1053 total_reg_proc_dns,
1054 GNUNET_NO);
1055 GNUNET_STATISTICS_set (stats,
1056 "# Regular processing completed with NAMESTORE PUT",
1057 total_reg_proc_dns_ns,
1058 GNUNET_NO);
1059 GNUNET_STATISTICS_set (stats,
1060 "# average request processing latency (μs)",
1061 runtime.rel_value_us,
1062 GNUNET_NO);
1063 GNUNET_STATISTICS_set (stats,
1064 "# total time spent idle (μs)",
1065 idle_time.rel_value_us,
1066 GNUNET_NO);
1067 }
1068
1069 if (NULL == t)
1070 {
1071 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1072 t = GNUNET_SCHEDULER_add_now (&process_queue,
1073 NULL);
813 } 1074 }
814} 1075}
815 1076
@@ -818,13 +1079,11 @@ store_completed_cb (void *cls,
818 * Function called with the result of a DNS resolution. 1079 * Function called with the result of a DNS resolution.
819 * 1080 *
820 * @param cls closure with the `struct Request` 1081 * @param cls closure with the `struct Request`
821 * @param rs socket that received the response
822 * @param dns dns response, never NULL 1082 * @param dns dns response, never NULL
823 * @param dns_len number of bytes in @a dns 1083 * @param dns_len number of bytes in @a dns
824 */ 1084 */
825static void 1085static void
826process_result (void *cls, 1086process_result (void *cls,
827 struct GNUNET_DNSSTUB_RequestSocket *rs,
828 const struct GNUNET_TUN_DnsHeader *dns, 1087 const struct GNUNET_TUN_DnsHeader *dns,
829 size_t dns_len) 1088 size_t dns_len)
830{ 1089{
@@ -833,7 +1092,6 @@ process_result (void *cls,
833 struct GNUNET_DNSPARSER_Packet *p; 1092 struct GNUNET_DNSPARSER_Packet *p;
834 unsigned int rd_count; 1093 unsigned int rd_count;
835 1094
836 (void) rs;
837 GNUNET_assert (NULL == req->hn); 1095 GNUNET_assert (NULL == req->hn);
838 if (NULL == dns) 1096 if (NULL == dns)
839 { 1097 {
@@ -842,26 +1100,50 @@ process_result (void *cls,
842 req_tail, 1100 req_tail,
843 req); 1101 req);
844 pending--; 1102 pending--;
1103 if (NULL == t)
1104 {
1105 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1106 t = GNUNET_SCHEDULER_add_now (&process_queue,
1107 NULL);
1108 }
845 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 1109 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
846 "Stub gave up on DNS reply for `%s'\n", 1110 "Stub gave up on DNS reply for `%s'\n",
847 req->hostname); 1111 req->hostname);
1112 GNUNET_STATISTICS_update (stats,
1113 "# DNS lookups timed out",
1114 1,
1115 GNUNET_NO);
848 if (req->issue_num > MAX_RETRIES) 1116 if (req->issue_num > MAX_RETRIES)
849 { 1117 {
850 failures++; 1118 failures++;
851 free_request (req); 1119 free_request (req);
1120 GNUNET_STATISTICS_update (stats,
1121 "# requests given up on",
1122 1,
1123 GNUNET_NO);
852 return; 1124 return;
853 } 1125 }
1126 total_reg_proc_dns++;
854 req->rs = NULL; 1127 req->rs = NULL;
855 insert_sorted (req); 1128 insert_sorted (req);
856 return; 1129 return;
857 } 1130 }
858 if (req->id != dns->id) 1131 if (req->id != dns->id)
1132 {
1133 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1134 "DNS ID did not match request, ignoring reply\n");
1135 GNUNET_STATISTICS_update (stats,
1136 "# DNS ID missmatches",
1137 1,
1138 GNUNET_NO);
859 return; 1139 return;
1140 }
860 GNUNET_CONTAINER_DLL_remove (req_head, 1141 GNUNET_CONTAINER_DLL_remove (req_head,
861 req_tail, 1142 req_tail,
862 req); 1143 req);
863 GNUNET_DNSSTUB_resolve_cancel (req->rs); 1144 GNUNET_DNSSTUB_resolve_cancel (req->rs);
864 req->rs = NULL; 1145 req->rs = NULL;
1146 pending--;
865 p = GNUNET_DNSPARSER_parse ((const char *) dns, 1147 p = GNUNET_DNSPARSER_parse ((const char *) dns,
866 dns_len); 1148 dns_len);
867 if (NULL == p) 1149 if (NULL == p)
@@ -869,35 +1151,59 @@ process_result (void *cls,
869 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 1151 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
870 "Failed to parse DNS reply for `%s'\n", 1152 "Failed to parse DNS reply for `%s'\n",
871 req->hostname); 1153 req->hostname);
1154 GNUNET_STATISTICS_update (stats,
1155 "# DNS parser errors",
1156 1,
1157 GNUNET_NO);
1158 if (NULL == t)
1159 {
1160 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1161 t = GNUNET_SCHEDULER_add_now (&process_queue,
1162 NULL);
1163 }
872 if (req->issue_num > MAX_RETRIES) 1164 if (req->issue_num > MAX_RETRIES)
873 { 1165 {
874 failures++; 1166 failures++;
875 insert_sorted (req); 1167 free_request (req);
876 pending--; 1168 GNUNET_STATISTICS_update (stats,
1169 "# requests given up on",
1170 1,
1171 GNUNET_NO);
877 return; 1172 return;
878 } 1173 }
879 insert_sorted (req); 1174 insert_sorted (req);
880 pending--;
881 return; 1175 return;
882 } 1176 }
883 /* Free old/legacy records */
884 while (NULL != (rec = req->rec_head))
885 {
886 GNUNET_CONTAINER_DLL_remove (req->rec_head,
887 req->rec_tail,
888 rec);
889 GNUNET_free (rec);
890 }
891 /* import new records */ 1177 /* import new records */
892 req->issue_num = 0; /* success, reset counter! */ 1178 req->issue_num = 0; /* success, reset counter! */
893 req->p = p; 1179 {
894 for_all_records (p, 1180 struct ProcessRecordContext prc = {
895 &process_record, 1181 .req = req,
896 req); 1182 .p = p
897 req->p = NULL; 1183 };
1184
1185 for_all_records (p,
1186 &process_record,
1187 &prc);
1188 }
898 GNUNET_DNSPARSER_free_packet (p); 1189 GNUNET_DNSPARSER_free_packet (p);
899 /* count records found, determine minimum expiration time */ 1190 /* count records found, determine minimum expiration time */
900 req->expires = GNUNET_TIME_UNIT_FOREVER_ABS; 1191 req->expires = GNUNET_TIME_UNIT_FOREVER_ABS;
1192 {
1193 struct GNUNET_TIME_Relative dns_latency;
1194
1195 dns_latency = GNUNET_TIME_absolute_get_duration (req->op_start_time);
1196 total_dns_latency = GNUNET_TIME_relative_add (total_dns_latency,
1197 dns_latency);
1198 total_dns_latency_cnt++;
1199 if (0 == (total_dns_latency_cnt % 1000))
1200 {
1201 GNUNET_STATISTICS_set (stats,
1202 "# average DNS lookup latency (μs)",
1203 total_dns_latency.rel_value_us / total_dns_latency_cnt,
1204 GNUNET_NO);
1205 }
1206 }
901 rd_count = 0; 1207 rd_count = 0;
902 for (rec = req->rec_head; NULL != rec; rec = rec->next) 1208 for (rec = req->rec_head; NULL != rec; rec = rec->next)
903 { 1209 {
@@ -915,8 +1221,18 @@ process_result (void *cls,
915 /* Instead of going for SOA, simplified for now to look each 1221 /* Instead of going for SOA, simplified for now to look each
916 day in case we got an empty response */ 1222 day in case we got an empty response */
917 if (0 == rd_count) 1223 if (0 == rd_count)
1224 {
918 req->expires 1225 req->expires
919 = GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_DAYS); 1226 = GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_DAYS);
1227 GNUNET_STATISTICS_update (stats,
1228 "# empty DNS replies (usually NXDOMAIN)",
1229 1,
1230 GNUNET_NO);
1231 }
1232 else
1233 {
1234 record_sets++;
1235 }
920 /* convert records to namestore import format */ 1236 /* convert records to namestore import format */
921 { 1237 {
922 struct GNUNET_GNSRECORD_Data rd[GNUNET_NZL(rd_count)]; 1238 struct GNUNET_GNSRECORD_Data rd[GNUNET_NZL(rd_count)];
@@ -925,67 +1241,22 @@ process_result (void *cls,
925 /* convert linked list into array */ 1241 /* convert linked list into array */
926 for (rec = req->rec_head; NULL != rec; rec =rec->next) 1242 for (rec = req->rec_head; NULL != rec; rec =rec->next)
927 rd[off++] = rec->grd; 1243 rd[off++] = rec->grd;
1244 pending_rs++;
1245 req->op_start_time = GNUNET_TIME_absolute_get ();
928 req->qe = GNUNET_NAMESTORE_records_store (ns, 1246 req->qe = GNUNET_NAMESTORE_records_store (ns,
929 &req->zone->key, 1247 &req->zone->key,
930 req->label, 1248 get_label (req),
931 rd_count, 1249 rd_count,
932 rd, 1250 rd,
933 &store_completed_cb, 1251 &store_completed_cb,
934 req); 1252 req);
1253 GNUNET_assert (NULL != req->qe);
935 } 1254 }
936 insert_sorted (req); 1255 insert_sorted (req);
937} 1256}
938 1257
939 1258
940/** 1259/**
941 * Submit a request to DNS unless we need to slow down because
942 * we are at the rate limit.
943 *
944 * @param req request to submit
945 * @return #GNUNET_OK if request was submitted
946 * #GNUNET_NO if request was already submitted
947 * #GNUNET_SYSERR if we are at the rate limit
948 */
949static int
950submit_req (struct Request *req)
951{
952 static struct GNUNET_TIME_Absolute last_request;
953 struct GNUNET_TIME_Absolute now;
954
955 if (NULL != req->qe)
956 return GNUNET_NO; /* namestore op still pending */
957 if (NULL != req->rs)
958 {
959 GNUNET_break (0);
960 return GNUNET_NO; /* already submitted */
961 }
962 now = GNUNET_TIME_absolute_get ();
963 if ( (now.abs_value_us - last_request.abs_value_us < TIME_THRESH) ||
964 (pending >= THRESH) )
965 return GNUNET_SYSERR;
966 GNUNET_CONTAINER_DLL_insert (req_head,
967 req_tail,
968 req);
969 GNUNET_assert (NULL == req->rs);
970 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
971 "Requesting resolution for `%s'\n",
972 req->hostname);
973 req->rs = GNUNET_DNSSTUB_resolve2 (ctx,
974 req->raw,
975 req->raw_len,
976 &process_result,
977 req);
978 GNUNET_assert (NULL != req->rs);
979 req->issue_num++;
980 last_request = now;
981 lookups++;
982 pending++;
983 req->time = time (NULL);
984 return GNUNET_OK;
985}
986
987
988/**
989 * Process as many requests as possible from the queue. 1260 * Process as many requests as possible from the queue.
990 * 1261 *
991 * @param cls NULL 1262 * @param cls NULL
@@ -994,26 +1265,79 @@ static void
994process_queue (void *cls) 1265process_queue (void *cls)
995{ 1266{
996 struct Request *req; 1267 struct Request *req;
1268 unsigned int series;
1269 void *raw;
1270 size_t raw_size;
1271 struct GNUNET_TIME_Relative delay;
997 1272
998 (void) cls; 1273 (void) cls;
1274 delay = GNUNET_TIME_absolute_get_duration (sleep_time_reg_proc);
1275 idle_time = GNUNET_TIME_relative_add (idle_time,
1276 delay);
1277 series = 0;
999 t = NULL; 1278 t = NULL;
1000 while (1) 1279 while (pending + pending_rs < THRESH)
1001 { 1280 {
1002 req = GNUNET_CONTAINER_heap_peek (req_heap); 1281 req = GNUNET_CONTAINER_heap_peek (req_heap);
1003 if (NULL == req) 1282 if (NULL == req)
1004 break; 1283 break;
1284 if (NULL != req->qe)
1285 return; /* namestore op still pending */
1286 if (NULL != req->rs)
1287 {
1288 GNUNET_break (0);
1289 return; /* already submitted */
1290 }
1005 if (GNUNET_TIME_absolute_get_remaining (req->expires).rel_value_us > 0) 1291 if (GNUNET_TIME_absolute_get_remaining (req->expires).rel_value_us > 0)
1006 break; 1292 break;
1007 if (GNUNET_OK != submit_req (req))
1008 break;
1009 GNUNET_assert (req == 1293 GNUNET_assert (req ==
1010 GNUNET_CONTAINER_heap_remove_root (req_heap)); 1294 GNUNET_CONTAINER_heap_remove_root (req_heap));
1011 req->hn = NULL; 1295 req->hn = NULL;
1296 GNUNET_CONTAINER_DLL_insert (req_head,
1297 req_tail,
1298 req);
1299 GNUNET_assert (NULL == req->rs);
1300 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1301 "Requesting resolution for `%s'\n",
1302 req->hostname);
1303 raw = build_dns_query (req,
1304 &raw_size);
1305 if (NULL == raw)
1306 {
1307 GNUNET_break (0);
1308 free_request (req);
1309 continue;
1310 }
1311 req->op_start_time = GNUNET_TIME_absolute_get ();
1312 req->rs = GNUNET_DNSSTUB_resolve (ctx,
1313 raw,
1314 raw_size,
1315 &process_result,
1316 req);
1317 GNUNET_assert (NULL != req->rs);
1318 req->issue_num++;
1319 lookups++;
1320 pending++;
1321 series++;
1322 if (series > MAX_SERIES)
1323 break;
1324 }
1325 if (pending + pending_rs >= THRESH)
1326 {
1327 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1328 "Stopped processing queue (%u+%u/%u)]\n",
1329 pending,
1330 pending_rs,
1331 THRESH);
1332 return; /* wait for replies */
1012 } 1333 }
1013
1014 req = GNUNET_CONTAINER_heap_peek (req_heap); 1334 req = GNUNET_CONTAINER_heap_peek (req_heap);
1015 if (NULL == req) 1335 if (NULL == req)
1336 {
1337 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1338 "Stopped processing queue: empty queue\n");
1016 return; 1339 return;
1340 }
1017 if (GNUNET_TIME_absolute_get_remaining (req->expires).rel_value_us > 0) 1341 if (GNUNET_TIME_absolute_get_remaining (req->expires).rel_value_us > 0)
1018 { 1342 {
1019 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1343 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
@@ -1022,20 +1346,43 @@ process_queue (void *cls)
1022 req->hostname); 1346 req->hostname);
1023 if (NULL != t) 1347 if (NULL != t)
1024 GNUNET_SCHEDULER_cancel (t); 1348 GNUNET_SCHEDULER_cancel (t);
1349 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1025 t = GNUNET_SCHEDULER_add_at (req->expires, 1350 t = GNUNET_SCHEDULER_add_at (req->expires,
1026 &process_queue, 1351 &process_queue,
1027 NULL); 1352 NULL);
1353 return;
1028 } 1354 }
1029 else 1355 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1030 { 1356 "Throttling\n");
1031 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 1357 if (NULL != t)
1032 "Throttling for 1ms\n"); 1358 GNUNET_SCHEDULER_cancel (t);
1033 if (NULL != t) 1359 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1034 GNUNET_SCHEDULER_cancel (t); 1360 t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY,
1035 t = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MILLISECONDS, 1361 &process_queue,
1036 &process_queue, 1362 NULL);
1037 NULL); 1363}
1038 } 1364
1365
1366/**
1367 * Iterator called during #do_shutdown() to free requests in
1368 * the #ns_pending map.
1369 *
1370 * @param cls NULL
1371 * @param key unused
1372 * @param value the `struct Request` to free
1373 * @return #GNUNET_OK
1374 */
1375static int
1376free_request_it (void *cls,
1377 const struct GNUNET_HashCode *key,
1378 void *value)
1379{
1380 struct Request *req = value;
1381
1382 (void) cls;
1383 (void) key;
1384 free_request (req);
1385 return GNUNET_OK;
1039} 1386}
1040 1387
1041 1388
@@ -1061,33 +1408,50 @@ do_shutdown (void *cls)
1061 GNUNET_SCHEDULER_cancel (t); 1408 GNUNET_SCHEDULER_cancel (t);
1062 t = NULL; 1409 t = NULL;
1063 } 1410 }
1064 if (NULL != ns)
1065 {
1066 GNUNET_NAMESTORE_disconnect (ns);
1067 ns = NULL;
1068 }
1069 if (NULL != ctx)
1070 {
1071 GNUNET_DNSSTUB_stop (ctx);
1072 ctx = NULL;
1073 }
1074 while (NULL != (req = req_head)) 1411 while (NULL != (req = req_head))
1075 { 1412 {
1076 GNUNET_CONTAINER_DLL_remove (req_head, 1413 GNUNET_CONTAINER_DLL_remove (req_head,
1077 req_tail, 1414 req_tail,
1078 req); 1415 req);
1416 if (NULL != req->qe)
1417 GNUNET_NAMESTORE_cancel (req->qe);
1079 free_request (req); 1418 free_request (req);
1080 } 1419 }
1081 while (NULL != (req = GNUNET_CONTAINER_heap_remove_root (req_heap))) 1420 while (NULL != (req = GNUNET_CONTAINER_heap_remove_root (req_heap)))
1082 { 1421 {
1083 req->hn = NULL; 1422 req->hn = NULL;
1423 if (NULL != req->qe)
1424 GNUNET_NAMESTORE_cancel (req->qe);
1084 free_request (req); 1425 free_request (req);
1085 } 1426 }
1427 if (NULL != zone_it)
1428 {
1429 GNUNET_NAMESTORE_zone_iteration_stop (zone_it);
1430 zone_it = NULL;
1431 }
1432 if (NULL != ns)
1433 {
1434 GNUNET_NAMESTORE_disconnect (ns);
1435 ns = NULL;
1436 }
1437 if (NULL != ctx)
1438 {
1439 GNUNET_DNSSTUB_stop (ctx);
1440 ctx = NULL;
1441 }
1086 if (NULL != req_heap) 1442 if (NULL != req_heap)
1087 { 1443 {
1088 GNUNET_CONTAINER_heap_destroy (req_heap); 1444 GNUNET_CONTAINER_heap_destroy (req_heap);
1089 req_heap = NULL; 1445 req_heap = NULL;
1090 } 1446 }
1447 if (NULL != ns_pending)
1448 {
1449 GNUNET_CONTAINER_multihashmap_iterate (ns_pending,
1450 &free_request_it,
1451 NULL);
1452 GNUNET_CONTAINER_multihashmap_destroy (ns_pending);
1453 ns_pending = NULL;
1454 }
1091 while (NULL != (zone = zone_head)) 1455 while (NULL != (zone = zone_head))
1092 { 1456 {
1093 GNUNET_CONTAINER_DLL_remove (zone_head, 1457 GNUNET_CONTAINER_DLL_remove (zone_head,
@@ -1096,52 +1460,93 @@ do_shutdown (void *cls)
1096 GNUNET_free (zone->domain); 1460 GNUNET_free (zone->domain);
1097 GNUNET_free (zone); 1461 GNUNET_free (zone);
1098 } 1462 }
1463 if (NULL != stats)
1464 {
1465 GNUNET_STATISTICS_destroy (stats, GNUNET_NO);
1466 stats = NULL;
1467 }
1099} 1468}
1100 1469
1101 1470
1102/** 1471/**
1103 * Function called if #GNUNET_NAMESTORE_records_lookup() failed. 1472 * Function called if #GNUNET_NAMESTORE_records_lookup() failed.
1104 * Continues resolution based on assumption namestore has no data. 1473 * Just logs an error.
1105 * 1474 *
1106 * @param cls a `struct Request` 1475 * @param cls a `struct Zone`
1107 */ 1476 */
1108static void 1477static void
1109ns_lookup_error_cb (void *cls) 1478ns_lookup_error_cb (void *cls)
1110{ 1479{
1111 struct Request *req = cls; 1480 struct Zone *zone = cls;
1112 1481
1113 req->qe = NULL;
1114 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1482 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1115 "Failed to load data from namestore for `%s'\n", 1483 "Failed to load data from namestore for zone `%s'\n",
1116 req->label); 1484 zone->domain);
1117 insert_sorted (req);
1118} 1485}
1119 1486
1120 1487
1121/** 1488/**
1122 * Process a record that was stored in the namestore. 1489 * Process a record that was stored in the namestore.
1123 * 1490 *
1124 * @param cls a `struct Request *` 1491 * @param cls a `struct Zone *`
1125 * @param zone private key of the zone 1492 * @param key private key of the zone
1126 * @param label label of the records 1493 * @param label label of the records
1127 * @param rd_count number of entries in @a rd array, 0 if label was deleted 1494 * @param rd_count number of entries in @a rd array, 0 if label was deleted
1128 * @param rd array of records with data to store 1495 * @param rd array of records with data to store
1129 */ 1496 */
1130static void 1497static void
1131ns_lookup_result_cb (void *cls, 1498ns_lookup_result_cb (void *cls,
1132 const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone, 1499 const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
1133 const char *label, 1500 const char *label,
1134 unsigned int rd_count, 1501 unsigned int rd_count,
1135 const struct GNUNET_GNSRECORD_Data *rd) 1502 const struct GNUNET_GNSRECORD_Data *rd)
1136{ 1503{
1137 struct Request *req = cls; 1504 struct Zone *zone = cls;
1505 struct Request *req;
1506 struct GNUNET_HashCode hc;
1507 char *fqdn;
1138 1508
1139 req->qe = NULL; 1509 ns_iterator_trigger_next--;
1140 GNUNET_break (0 == memcmp (zone, 1510 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1511 "Obtained NAMESTORE reply, %llu left in round\n",
1512 (unsigned long long) ns_iterator_trigger_next);
1513 if (0 == ns_iterator_trigger_next)
1514 {
1515 ns_iterator_trigger_next = NS_BATCH_SIZE;
1516 GNUNET_STATISTICS_update (stats,
1517 "# NAMESTORE records requested from cache",
1518 ns_iterator_trigger_next,
1519 GNUNET_NO);
1520 GNUNET_NAMESTORE_zone_iterator_next (zone_it,
1521 ns_iterator_trigger_next);
1522 }
1523 GNUNET_asprintf (&fqdn,
1524 "%s.%s",
1525 label,
1526 zone->domain);
1527 GNUNET_CRYPTO_hash (fqdn,
1528 strlen (fqdn) + 1,
1529 &hc);
1530 GNUNET_free (fqdn);
1531 req = GNUNET_CONTAINER_multihashmap_get (ns_pending,
1532 &hc);
1533 if (NULL == req)
1534 {
1535 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1536 "Ignoring record `%s' in zone `%s': not on my list!\n",
1537 label,
1538 zone->domain);
1539 return;
1540 }
1541 GNUNET_assert (GNUNET_OK ==
1542 GNUNET_CONTAINER_multihashmap_remove (ns_pending,
1543 &hc,
1544 req));
1545 GNUNET_break (0 == memcmp (key,
1141 &req->zone->key, 1546 &req->zone->key,
1142 sizeof (*zone))); 1547 sizeof (*key)));
1143 GNUNET_break (0 == strcasecmp (label, 1548 GNUNET_break (0 == strcasecmp (label,
1144 req->label)); 1549 get_label (req)));
1145 for (unsigned int i=0;i<rd_count;i++) 1550 for (unsigned int i=0;i<rd_count;i++)
1146 { 1551 {
1147 struct GNUNET_TIME_Absolute at; 1552 struct GNUNET_TIME_Absolute at;
@@ -1157,12 +1562,13 @@ ns_lookup_result_cb (void *cls,
1157 { 1562 {
1158 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1563 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1159 "Empty record set in namestore for `%s'\n", 1564 "Empty record set in namestore for `%s'\n",
1160 req->label); 1565 req->hostname);
1161 } 1566 }
1162 else 1567 else
1163 { 1568 {
1164 unsigned int pos = 0; 1569 unsigned int pos = 0;
1165 1570
1571 cached++;
1166 req->expires = GNUNET_TIME_UNIT_FOREVER_ABS; 1572 req->expires = GNUNET_TIME_UNIT_FOREVER_ABS;
1167 for (struct Record *rec = req->rec_head; 1573 for (struct Record *rec = req->rec_head;
1168 NULL != rec; 1574 NULL != rec;
@@ -1180,8 +1586,10 @@ ns_lookup_result_cb (void *cls,
1180 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1586 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1181 "Hot-start with %u existing records for `%s'\n", 1587 "Hot-start with %u existing records for `%s'\n",
1182 pos, 1588 pos,
1183 req->label); 1589 req->hostname);
1184 } 1590 }
1591 free_records (req);
1592
1185 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 1593 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1186 "Adding `%s' to worklist to start at %s\n", 1594 "Adding `%s' to worklist to start at %s\n",
1187 req->hostname, 1595 req->hostname,
@@ -1198,13 +1606,11 @@ ns_lookup_result_cb (void *cls,
1198static void 1606static void
1199queue (const char *hostname) 1607queue (const char *hostname)
1200{ 1608{
1201 struct GNUNET_DNSPARSER_Packet p;
1202 struct GNUNET_DNSPARSER_Query q;
1203 struct Request *req; 1609 struct Request *req;
1204 char *raw;
1205 size_t raw_size;
1206 const char *dot; 1610 const char *dot;
1207 struct Zone *zone; 1611 struct Zone *zone;
1612 size_t hlen;
1613 struct GNUNET_HashCode hc;
1208 1614
1209 if (GNUNET_OK != 1615 if (GNUNET_OK !=
1210 GNUNET_DNSPARSER_check_name (hostname)) 1616 GNUNET_DNSPARSER_check_name (hostname))
@@ -1237,46 +1643,131 @@ queue (const char *hostname)
1237 dot + 1); 1643 dot + 1);
1238 return; 1644 return;
1239 } 1645 }
1240 q.name = (char *) hostname;
1241 q.type = GNUNET_DNSPARSER_TYPE_NS;
1242 q.dns_traffic_class = GNUNET_TUN_DNS_CLASS_INTERNET;
1243
1244 memset (&p,
1245 0,
1246 sizeof (p));
1247 p.num_queries = 1;
1248 p.queries = &q;
1249 p.id = (uint16_t) GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE,
1250 UINT16_MAX);
1251 1646
1647 hlen = strlen (hostname) + 1;
1648 req = GNUNET_malloc (sizeof (struct Request) + hlen);
1649 req->zone = zone;
1650 req->hostname = (char *) &req[1];
1651 memcpy (req->hostname,
1652 hostname,
1653 hlen);
1654 req->id = (uint16_t) GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE,
1655 UINT16_MAX);
1656 GNUNET_CRYPTO_hash (req->hostname,
1657 hlen,
1658 &hc);
1252 if (GNUNET_OK != 1659 if (GNUNET_OK !=
1253 GNUNET_DNSPARSER_pack (&p, 1660 GNUNET_CONTAINER_multihashmap_put (ns_pending,
1254 UINT16_MAX, 1661 &hc,
1255 &raw, 1662 req,
1256 &raw_size)) 1663 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))
1257 { 1664 {
1258 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 1665 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1259 "Failed to pack query for hostname `%s'\n", 1666 "Duplicate hostname `%s' ignored\n",
1260 hostname); 1667 hostname);
1261 rejects++; 1668 GNUNET_free (req);
1262 return; 1669 return;
1263 } 1670 }
1671}
1672
1673
1674/**
1675 * We have completed the initial iteration over the namestore's database.
1676 * This function is called on each of the remaining records in
1677 * #move_to_queue to #queue() them, as we will simply not find existing
1678 * records for them any longer.
1679 *
1680 * @param cls NULL
1681 * @param key unused
1682 * @param value a `struct Request`
1683 * @return #GNUNET_OK (continue to iterate)
1684 */
1685static int
1686move_to_queue (void *cls,
1687 const struct GNUNET_HashCode *key,
1688 void *value)
1689{
1690 struct Request *req = value;
1691
1692 (void) cls;
1693 (void) key;
1694 insert_sorted (req);
1695 return GNUNET_OK;
1696}
1697
1698
1699/**
1700 * Iterate over all of the zones we care about and see which records
1701 * we may need to re-fetch when.
1702 *
1703 * @param cls NULL
1704 */
1705static void
1706iterate_zones (void *cls)
1707{
1708 static struct Zone *last;
1709
1710 (void) cls;
1711 if (NULL != zone_it)
1712 {
1713 zone_it = NULL;
1714 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1715 "Finished iteration over zone `%s'!\n",
1716 last->domain);
1717 /* subtract left-overs from previous iteration */
1718 GNUNET_STATISTICS_update (stats,
1719 "# NAMESTORE records requested from cache",
1720 (long long) (- ns_iterator_trigger_next),
1721 GNUNET_NO);
1722 ns_iterator_trigger_next = 0;
1723 }
1724 GNUNET_assert (NULL != zone_tail);
1725 if (zone_tail == last)
1726 {
1727 /* Done iterating over relevant zones in NAMESTORE, move
1728 rest of hash map to work queue as well. */
1729 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1730 "Finished all NAMESTORE iterations!\n");
1731 GNUNET_STATISTICS_set (stats,
1732 "# Domain names without cached reply",
1733 GNUNET_CONTAINER_multihashmap_size (ns_pending),
1734 GNUNET_NO);
1735 GNUNET_CONTAINER_multihashmap_iterate (ns_pending,
1736 &move_to_queue,
1737 NULL);
1738 GNUNET_CONTAINER_multihashmap_destroy (ns_pending);
1739 ns_pending = NULL;
1740 start_time_reg_proc = GNUNET_TIME_absolute_get ();
1741 total_reg_proc_dns = 0;
1742 total_reg_proc_dns_ns = 0;
1743 return;
1744 }
1745 if (NULL == last)
1746 last = zone_head;
1747 else
1748 last = last->next;
1749 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1750 "Starting iteration over zone `%s'!\n",
1751 last->domain);
1752 /* subtract left-overs from previous iteration */
1753 GNUNET_STATISTICS_update (stats,
1754 "# NAMESTORE records requested from cache",
1755 1,
1756 GNUNET_NO);
1757 ns_iterator_trigger_next = 1;
1758 GNUNET_STATISTICS_update (stats,
1759 "# zones iterated",
1760 1,
1761 GNUNET_NO);
1762 zone_it = GNUNET_NAMESTORE_zone_iteration_start (ns,
1763 &last->key,
1764 &ns_lookup_error_cb,
1765 NULL,
1766 &ns_lookup_result_cb,
1767 last,
1768 &iterate_zones,
1769 NULL);
1264 1770
1265 req = GNUNET_new (struct Request);
1266 req->zone = zone;
1267 req->hostname = GNUNET_strdup (hostname);
1268 req->raw = raw;
1269 req->raw_len = raw_size;
1270 req->id = p.id;
1271 req->label = GNUNET_strndup (hostname,
1272 dot - hostname);
1273 req->qe = GNUNET_NAMESTORE_records_lookup (ns,
1274 &req->zone->key,
1275 req->label,
1276 &ns_lookup_error_cb,
1277 req,
1278 &ns_lookup_result_cb,
1279 req);
1280} 1771}
1281 1772
1282 1773
@@ -1288,12 +1779,17 @@ queue (const char *hostname)
1288static void 1779static void
1289process_stdin (void *cls) 1780process_stdin (void *cls)
1290{ 1781{
1782 static struct GNUNET_TIME_Absolute last;
1783 static uint64_t idot;
1291 char hn[256]; 1784 char hn[256];
1292 1785
1293 (void) cls; 1786 (void) cls;
1294 t = NULL; 1787 t = NULL;
1295 GNUNET_IDENTITY_disconnect (id); 1788 if (NULL != id)
1296 id = NULL; 1789 {
1790 GNUNET_IDENTITY_disconnect (id);
1791 id = NULL;
1792 }
1297 while (NULL != 1793 while (NULL !=
1298 fgets (hn, 1794 fgets (hn,
1299 sizeof (hn), 1795 sizeof (hn),
@@ -1301,8 +1797,34 @@ process_stdin (void *cls)
1301 { 1797 {
1302 if (strlen(hn) > 0) 1798 if (strlen(hn) > 0)
1303 hn[strlen(hn)-1] = '\0'; /* eat newline */ 1799 hn[strlen(hn)-1] = '\0'; /* eat newline */
1800 if (0 == idot)
1801 last = GNUNET_TIME_absolute_get ();
1802 idot++;
1803 if (0 == idot % 10000)
1804 {
1805 struct GNUNET_TIME_Relative delta;
1806
1807 delta = GNUNET_TIME_absolute_get_duration (last);
1808 last = GNUNET_TIME_absolute_get ();
1809 fprintf (stderr,
1810 "Imported 10000 records in %s\n",
1811 GNUNET_STRINGS_relative_time_to_string (delta,
1812 GNUNET_YES));
1813 GNUNET_STATISTICS_set (stats,
1814 "# domain names provided",
1815 idot,
1816 GNUNET_NO);
1817 }
1304 queue (hn); 1818 queue (hn);
1305 } 1819 }
1820 fprintf (stderr,
1821 "Done reading %llu domain names\n",
1822 (unsigned long long) idot);
1823 GNUNET_STATISTICS_set (stats,
1824 "# domain names provided",
1825 idot,
1826 GNUNET_NO);
1827 iterate_zones (NULL);
1306} 1828}
1307 1829
1308 1830
@@ -1358,7 +1880,7 @@ identity_cb (void *cls,
1358 else 1880 else
1359 { 1881 {
1360 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 1882 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1361 "Specified zone not found\n"); 1883 "No zone found\n");
1362 GNUNET_SCHEDULER_shutdown (); 1884 GNUNET_SCHEDULER_shutdown ();
1363 return; 1885 return;
1364 } 1886 }
@@ -1395,14 +1917,44 @@ run (void *cls,
1395 (void) cls; 1917 (void) cls;
1396 (void) args; 1918 (void) args;
1397 (void) cfgfile; 1919 (void) cfgfile;
1920 stats = GNUNET_STATISTICS_create ("zoneimport",
1921 cfg);
1398 req_heap = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN); 1922 req_heap = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
1399 ctx = GNUNET_DNSSTUB_start (dns_server); 1923 ns_pending = GNUNET_CONTAINER_multihashmap_create (map_size,
1924 GNUNET_NO);
1925 if (NULL == ns_pending)
1926 {
1927 fprintf (stderr,
1928 "Failed to allocate memory for main hash map\n");
1929 return;
1930 }
1931 ctx = GNUNET_DNSSTUB_start (256);
1400 if (NULL == ctx) 1932 if (NULL == ctx)
1401 { 1933 {
1402 fprintf (stderr, 1934 fprintf (stderr,
1403 "Failed to initialize GNUnet DNS STUB\n"); 1935 "Failed to initialize GNUnet DNS STUB\n");
1404 return; 1936 return;
1405 } 1937 }
1938 if (NULL == args[0])
1939 {
1940 fprintf (stderr,
1941 "You must provide a list of DNS resolvers on the command line\n");
1942 return;
1943 }
1944 for (unsigned int i=0;NULL != args[i];i++)
1945 {
1946 if (GNUNET_OK !=
1947 GNUNET_DNSSTUB_add_dns_ip (ctx,
1948 args[i]))
1949 {
1950 fprintf (stderr,
1951 "Failed to use `%s' for DNS resolver\n",
1952 args[i]);
1953 return;
1954 }
1955 }
1956
1957
1406 GNUNET_SCHEDULER_add_shutdown (&do_shutdown, 1958 GNUNET_SCHEDULER_add_shutdown (&do_shutdown,
1407 NULL); 1959 NULL);
1408 ns = GNUNET_NAMESTORE_connect (cfg); 1960 ns = GNUNET_NAMESTORE_connect (cfg);
@@ -1429,12 +1981,11 @@ main (int argc,
1429 char *const*argv) 1981 char *const*argv)
1430{ 1982{
1431 struct GNUNET_GETOPT_CommandLineOption options[] = { 1983 struct GNUNET_GETOPT_CommandLineOption options[] = {
1432 GNUNET_GETOPT_option_mandatory 1984 GNUNET_GETOPT_option_uint ('s',
1433 (GNUNET_GETOPT_option_string ('s', 1985 "size",
1434 "server", 1986 "MAPSIZE",
1435 "IP", 1987 gettext_noop ("size to use for the main hash map"),
1436 "which DNS server should be used", 1988 &map_size),
1437 &dns_server)),
1438 GNUNET_GETOPT_OPTION_END 1989 GNUNET_GETOPT_OPTION_END
1439 }; 1990 };
1440 1991
@@ -1451,12 +2002,16 @@ main (int argc,
1451 NULL); 2002 NULL);
1452 GNUNET_free ((void*) argv); 2003 GNUNET_free ((void*) argv);
1453 fprintf (stderr, 2004 fprintf (stderr,
1454 "Rejected %u names, did %u lookups, found %u records, %u lookups failed, %u pending on shutdown\n", 2005 "Rejected %u names, had %u cached, did %u lookups, stored %u record sets\n"
2006 "Found %u records, %u lookups failed, %u/%u pending on shutdown\n",
1455 rejects, 2007 rejects,
2008 cached,
1456 lookups, 2009 lookups,
2010 record_sets,
1457 records, 2011 records,
1458 failures, 2012 failures,
1459 pending); 2013 pending,
2014 pending_rs);
1460 return 0; 2015 return 0;
1461} 2016}
1462 2017
diff --git a/src/namestore/namestore.conf.in b/src/namestore/namestore.conf.in
index f19ac5643..8b5e440b8 100644
--- a/src/namestore/namestore.conf.in
+++ b/src/namestore/namestore.conf.in
@@ -9,8 +9,17 @@ HOSTNAME = localhost
9BINARY = gnunet-service-namestore 9BINARY = gnunet-service-namestore
10ACCEPT_FROM = 127.0.0.1; 10ACCEPT_FROM = 127.0.0.1;
11ACCEPT_FROM6 = ::1; 11ACCEPT_FROM6 = ::1;
12
13# Which database should we use?
12DATABASE = sqlite 14DATABASE = sqlite
13 15
16# Should we optimize publishing record by caching the mapping
17# from zone private keys to zone public keys in memory?
18# (Set to NO if totally paranoid about keeping private keys
19# in RAM longer than necessary.)
20CACHE_KEYS = YES
21
22
14[namestore-sqlite] 23[namestore-sqlite]
15FILENAME = $GNUNET_DATA_HOME/namestore/sqlite.db 24FILENAME = $GNUNET_DATA_HOME/namestore/sqlite.db
16 25
@@ -38,5 +47,3 @@ UNIXPATH = $GNUNET_RUNTIME_DIR/gnunet-service-fcfsd.sock
38 47
39# On what port does the FCFS daemon listen for HTTP clients? 48# On what port does the FCFS daemon listen for HTTP clients?
40HTTPPORT = 18080 49HTTPPORT = 18080
41
42
diff --git a/src/namestore/namestore.h b/src/namestore/namestore.h
index ec7f28704..207b35662 100644
--- a/src/namestore/namestore.h
+++ b/src/namestore/namestore.h
@@ -161,7 +161,7 @@ struct LabelLookupResponseMessage
161 * Length of serialized record data 161 * Length of serialized record data
162 */ 162 */
163 uint16_t rd_len GNUNET_PACKED; 163 uint16_t rd_len GNUNET_PACKED;
164 164
165 /** 165 /**
166 * Number of records contained 166 * Number of records contained
167 */ 167 */
@@ -169,7 +169,7 @@ struct LabelLookupResponseMessage
169 169
170 /** 170 /**
171 * Was the label found in the database?? 171 * Was the label found in the database??
172 * GNUNET_YES or GNUNET_NO 172 * #GNUNET_YES or #GNUNET_NO
173 */ 173 */
174 uint16_t found GNUNET_PACKED; 174 uint16_t found GNUNET_PACKED;
175 175
@@ -345,6 +345,14 @@ struct ZoneIterationNextMessage
345 * Type will be #GNUNET_MESSAGE_TYPE_NAMESTORE_ZONE_ITERATION_NEXT 345 * Type will be #GNUNET_MESSAGE_TYPE_NAMESTORE_ZONE_ITERATION_NEXT
346 */ 346 */
347 struct GNUNET_NAMESTORE_Header gns_header; 347 struct GNUNET_NAMESTORE_Header gns_header;
348
349 /**
350 * Number of records to return to the iterator in one shot
351 * (before #GNUNET_MESSAGE_TYPE_NAMESTORE_ZONE_ITERATION_NEXT
352 * should be send again). In NBO.
353 */
354 uint64_t limit;
355
348}; 356};
349 357
350 358
diff --git a/src/namestore/namestore_api.c b/src/namestore/namestore_api.c
index a187bd250..ab356838b 100644
--- a/src/namestore/namestore_api.c
+++ b/src/namestore/namestore_api.c
@@ -1087,7 +1087,7 @@ GNUNET_NAMESTORE_set_nick (struct GNUNET_NAMESTORE_Handle *h,
1087 rd.flags |= GNUNET_GNSRECORD_RF_PRIVATE; 1087 rd.flags |= GNUNET_GNSRECORD_RF_PRIVATE;
1088 return GNUNET_NAMESTORE_records_store (h, 1088 return GNUNET_NAMESTORE_records_store (h,
1089 pkey, 1089 pkey,
1090 GNUNET_GNS_MASTERZONE_STR, 1090 GNUNET_GNS_EMPTY_LABEL_AT,
1091 1, 1091 1,
1092 &rd, 1092 &rd,
1093 cont, 1093 cont,
@@ -1281,19 +1281,24 @@ GNUNET_NAMESTORE_zone_iteration_start (struct GNUNET_NAMESTORE_Handle *h,
1281 * for the next record. 1281 * for the next record.
1282 * 1282 *
1283 * @param it the iterator 1283 * @param it the iterator
1284 * @param limit number of records to return to the iterator in one shot
1285 * (before #GNUNET_NAMESTORE_zone_iterator_next is to be called again)
1284 */ 1286 */
1285void 1287void
1286GNUNET_NAMESTORE_zone_iterator_next (struct GNUNET_NAMESTORE_ZoneIterator *it) 1288GNUNET_NAMESTORE_zone_iterator_next (struct GNUNET_NAMESTORE_ZoneIterator *it,
1289 uint64_t limit)
1287{ 1290{
1288 struct GNUNET_NAMESTORE_Handle *h = it->h; 1291 struct GNUNET_NAMESTORE_Handle *h = it->h;
1289 struct ZoneIterationNextMessage *msg; 1292 struct ZoneIterationNextMessage *msg;
1290 struct GNUNET_MQ_Envelope *env; 1293 struct GNUNET_MQ_Envelope *env;
1291 1294
1292 LOG (GNUNET_ERROR_TYPE_DEBUG, 1295 LOG (GNUNET_ERROR_TYPE_DEBUG,
1293 "Sending ZONE_ITERATION_NEXT message\n"); 1296 "Sending ZONE_ITERATION_NEXT message with limit %llu\n",
1297 (unsigned long long) limit);
1294 env = GNUNET_MQ_msg (msg, 1298 env = GNUNET_MQ_msg (msg,
1295 GNUNET_MESSAGE_TYPE_NAMESTORE_ZONE_ITERATION_NEXT); 1299 GNUNET_MESSAGE_TYPE_NAMESTORE_ZONE_ITERATION_NEXT);
1296 msg->gns_header.r_id = htonl (it->op_id); 1300 msg->gns_header.r_id = htonl (it->op_id);
1301 msg->limit = GNUNET_htonll (limit);
1297 GNUNET_MQ_send (h->mq, 1302 GNUNET_MQ_send (h->mq,
1298 env); 1303 env);
1299} 1304}
diff --git a/src/namestore/perf_namestore_api_flat.conf b/src/namestore/perf_namestore_api_flat.conf
new file mode 100644
index 000000000..26e2f2c51
--- /dev/null
+++ b/src/namestore/perf_namestore_api_flat.conf
@@ -0,0 +1,7 @@
1@INLINE@ test_namestore_api.conf
2
3[namestore]
4DATABASE = flat
5
6[namecache]
7DISABLE = YES
diff --git a/src/namestore/perf_namestore_api_postgres.conf b/src/namestore/perf_namestore_api_postgres.conf
new file mode 100644
index 000000000..259ce35e7
--- /dev/null
+++ b/src/namestore/perf_namestore_api_postgres.conf
@@ -0,0 +1,7 @@
1@INLINE@ test_namestore_api.conf
2
3[namestore]
4DATABASE = postgres
5
6[namecache]
7DISABLE = YES
diff --git a/src/namestore/perf_namestore_api_sqlite.conf b/src/namestore/perf_namestore_api_sqlite.conf
new file mode 100644
index 000000000..72b609226
--- /dev/null
+++ b/src/namestore/perf_namestore_api_sqlite.conf
@@ -0,0 +1,4 @@
1@INLINE@ test_namestore_api.conf
2
3[namecache]
4DISABLE = YES
diff --git a/src/namestore/perf_namestore_api_zone_iteration.c b/src/namestore/perf_namestore_api_zone_iteration.c
new file mode 100644
index 000000000..4ef8d3407
--- /dev/null
+++ b/src/namestore/perf_namestore_api_zone_iteration.c
@@ -0,0 +1,388 @@
1/*
2 This file is part of GNUnet.
3 Copyright (C) 2013, 2018 GNUnet e.V.
4
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 3, or (at your
8 option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
14
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
18 Boston, MA 02110-1301, USA.
19*/
20/**
21 * @file namestore/perf_namestore_api_zone_iteration.c
22 * @brief testcase for zone iteration functionality: iterate all zones
23 * @author Christian Grothoff
24 */
25#include "platform.h"
26#include "gnunet_namestore_service.h"
27#include "gnunet_testing_lib.h"
28#include "namestore.h"
29
30/**
31 * A #BENCHMARK_SIZE of 1000 takes less than a minute on a reasonably
32 * modern system, so 30 minutes should be OK even for very, very
33 * slow systems.
34 */
35#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 30)
36
37/**
38 * The runtime of the benchmark is expected to be linear
39 * for the iteration phase with a *good* database. The FLAT
40 * database uses a quadratic retrieval algorithm,
41 * hence it should be quadratic in the size.
42 */
43#define BENCHMARK_SIZE 1000
44
45/**
46 * Maximum record size
47 */
48#define MAX_REC_SIZE 500
49
50/**
51 * How big are the blocks we fetch? Note that the first block is
52 * always just 1 record set per current API. Smaller block
53 * sizes will make quadratic iteration-by-offset penalties
54 * more pronounced.
55 */
56#define BLOCK_SIZE 100
57
58static struct GNUNET_NAMESTORE_Handle *nsh;
59
60static struct GNUNET_SCHEDULER_Task *timeout_task;
61
62static struct GNUNET_SCHEDULER_Task *t;
63
64static struct GNUNET_CRYPTO_EcdsaPrivateKey *privkey;
65
66static struct GNUNET_NAMESTORE_ZoneIterator *zi;
67
68static struct GNUNET_NAMESTORE_QueueEntry *qe;
69
70static int res;
71
72static char *directory;
73
74static unsigned int off;
75
76static unsigned int left_until_next;
77
78static uint8_t seen[1 + BENCHMARK_SIZE / 8];
79
80static struct GNUNET_TIME_Absolute start;
81
82
83/**
84 * Terminate everything
85 *
86 * @param cls NULL
87 */
88static void
89end (void *cls)
90{
91 (void) cls;
92 if (NULL != qe)
93 {
94 GNUNET_NAMESTORE_cancel (qe);
95 qe = NULL;
96 }
97 if (NULL != zi)
98 {
99 GNUNET_NAMESTORE_zone_iteration_stop (zi);
100 zi = NULL;
101 }
102 if (NULL != nsh)
103 {
104 GNUNET_NAMESTORE_disconnect (nsh);
105 nsh = NULL;
106 }
107 if (NULL != t)
108 {
109 GNUNET_SCHEDULER_cancel (t);
110 t = NULL;
111 }
112 if (NULL != timeout_task)
113 {
114 GNUNET_SCHEDULER_cancel (timeout_task);
115 timeout_task = NULL;
116 }
117 if (NULL != privkey)
118 {
119 GNUNET_free (privkey);
120 privkey = NULL;
121 }
122}
123
124
125/**
126 * End with timeout. As this is a benchmark, we do not
127 * fail hard but return "skipped".
128 */
129static void
130timeout (void *cls)
131{
132 (void) cls;
133 timeout_task = NULL;
134 GNUNET_SCHEDULER_shutdown ();
135 res = 77;
136}
137
138
139static struct GNUNET_GNSRECORD_Data *
140create_record (unsigned int count)
141{
142 struct GNUNET_GNSRECORD_Data *rd;
143
144 rd = GNUNET_malloc (count + sizeof (struct GNUNET_GNSRECORD_Data));
145 rd->expiration_time = GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_HOURS).abs_value_us;
146 rd->record_type = count;
147 rd->data_size = count;
148 rd->data = (void *) &rd[1];
149 rd->flags = 0;
150 memset (&rd[1],
151 'a',
152 count);
153 return rd;
154}
155
156
157static void
158zone_end (void *cls)
159{
160 struct GNUNET_TIME_Relative delay;
161
162 zi = NULL;
163 delay = GNUNET_TIME_absolute_get_duration (start);
164 fprintf (stdout,
165 "Iterating over %u records took %s\n",
166 off,
167 GNUNET_STRINGS_relative_time_to_string (delay,
168 GNUNET_YES));
169 if (BENCHMARK_SIZE == off)
170 {
171 res = 0;
172 }
173 else
174 {
175 GNUNET_break (0);
176 res = 1;
177 }
178 GNUNET_SCHEDULER_shutdown ();
179}
180
181
182static void
183fail_cb (void *cls)
184{
185 zi = NULL;
186 res = 2;
187 GNUNET_break (0);
188 GNUNET_SCHEDULER_shutdown ();
189}
190
191
192static void
193zone_proc (void *cls,
194 const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone,
195 const char *label,
196 unsigned int rd_count,
197 const struct GNUNET_GNSRECORD_Data *rd)
198{
199 struct GNUNET_GNSRECORD_Data *wrd;
200 unsigned int xoff;
201
202 GNUNET_assert (NULL != zone);
203 if (1 != sscanf (label,
204 "l%u",
205 &xoff))
206 {
207 res = 3;
208 GNUNET_break (0);
209 GNUNET_SCHEDULER_shutdown ();
210 return;
211 }
212 if ( (xoff > BENCHMARK_SIZE) ||
213 (0 != (seen[xoff / 8] & (1U << (xoff % 8)))) )
214 {
215 res = 3;
216 GNUNET_break (0);
217 GNUNET_SCHEDULER_shutdown ();
218 return;
219 }
220 seen[xoff / 8] |= (1U << (xoff % 8));
221 wrd = create_record (xoff % MAX_REC_SIZE);
222 if ( (rd->record_type != wrd->record_type) ||
223 (rd->data_size != wrd->data_size) ||
224 (rd->flags != wrd->flags) )
225 {
226 res = 4;
227 GNUNET_break (0);
228 GNUNET_SCHEDULER_shutdown ();
229 GNUNET_free (wrd);
230 return;
231 }
232 if (0 != memcmp (rd->data,
233 wrd->data,
234 wrd->data_size))
235 {
236 res = 4;
237 GNUNET_break (0);
238 GNUNET_SCHEDULER_shutdown ();
239 GNUNET_free (wrd);
240 return;
241 }
242 GNUNET_free (wrd);
243 if (0 != memcmp (zone,
244 privkey,
245 sizeof (struct GNUNET_CRYPTO_EcdsaPrivateKey)))
246 {
247 res = 5;
248 GNUNET_break (0);
249 GNUNET_SCHEDULER_shutdown ();
250 return;
251 }
252 off++;
253 left_until_next--;
254 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
255 "Obtained record %u, expecting %u more until asking for mor explicitly\n",
256 off,
257 left_until_next);
258 if (0 == left_until_next)
259 {
260 left_until_next = BLOCK_SIZE;
261 GNUNET_NAMESTORE_zone_iterator_next (zi,
262 left_until_next);
263 }
264}
265
266
267static void
268publish_record (void *cls);
269
270
271static void
272put_cont (void *cls,
273 int32_t success,
274 const char *emsg)
275{
276 (void) cls;
277 qe = NULL;
278 GNUNET_assert (GNUNET_OK == success);
279 t = GNUNET_SCHEDULER_add_now (&publish_record,
280 NULL);
281}
282
283
284static void
285publish_record (void *cls)
286{
287 struct GNUNET_GNSRECORD_Data *rd;
288 char *label;
289
290 (void) cls;
291 t = NULL;
292 if (BENCHMARK_SIZE == off)
293 {
294 struct GNUNET_TIME_Relative delay;
295
296 delay = GNUNET_TIME_absolute_get_duration (start);
297 fprintf (stdout,
298 "Inserting %u records took %s\n",
299 off,
300 GNUNET_STRINGS_relative_time_to_string (delay,
301 GNUNET_YES));
302 start = GNUNET_TIME_absolute_get ();
303 off = 0;
304 left_until_next = 1;
305 zi = GNUNET_NAMESTORE_zone_iteration_start (nsh,
306 NULL,
307 &fail_cb,
308 NULL,
309 &zone_proc,
310 NULL,
311 &zone_end,
312 NULL);
313 GNUNET_assert (NULL != zi);
314 return;
315 }
316 rd = create_record ((++off) % MAX_REC_SIZE);
317 GNUNET_asprintf (&label,
318 "l%u",
319 off);
320 qe = GNUNET_NAMESTORE_records_store (nsh,
321 privkey,
322 label,
323 1, rd,
324 &put_cont,
325 NULL);
326 GNUNET_free (label);
327 GNUNET_free (rd);
328}
329
330
331static void
332run (void *cls,
333 const struct GNUNET_CONFIGURATION_Handle *cfg,
334 struct GNUNET_TESTING_Peer *peer)
335{
336 directory = NULL;
337 GNUNET_assert (GNUNET_OK ==
338 GNUNET_CONFIGURATION_get_value_string(cfg,
339 "PATHS",
340 "GNUNET_TEST_HOME",
341 &directory));
342 GNUNET_DISK_directory_remove (directory);
343 GNUNET_SCHEDULER_add_shutdown (&end,
344 NULL);
345 timeout_task = GNUNET_SCHEDULER_add_delayed (TIMEOUT,
346 &timeout,
347 NULL);
348 nsh = GNUNET_NAMESTORE_connect (cfg);
349 GNUNET_assert (NULL != nsh);
350 privkey = GNUNET_CRYPTO_ecdsa_key_create ();
351 GNUNET_assert (NULL != privkey);
352 start = GNUNET_TIME_absolute_get ();
353 t = GNUNET_SCHEDULER_add_now (&publish_record,
354 NULL);
355}
356
357
358int
359main (int argc,
360 char *argv[])
361{
362 const char *plugin_name;
363 char *cfg_name;
364
365 plugin_name = GNUNET_TESTING_get_testname_from_underscore (argv[0]);
366 GNUNET_asprintf (&cfg_name,
367 "perf_namestore_api_%s.conf",
368 plugin_name);
369 res = 1;
370 if (0 !=
371 GNUNET_TESTING_peer_run ("perf-namestore-api-zone-iteration",
372 cfg_name,
373 &run,
374 NULL))
375 {
376 res = 1;
377 }
378 GNUNET_free (cfg_name);
379 if (NULL != directory)
380 {
381 GNUNET_DISK_directory_remove (directory);
382 GNUNET_free (directory);
383 }
384 return res;
385}
386
387
388/* end of perf_namestore_api_zone_iteration.c */
diff --git a/src/namestore/plugin_namestore_flat.c b/src/namestore/plugin_namestore_flat.c
index 305fe7ba1..88b3ce9b4 100644
--- a/src/namestore/plugin_namestore_flat.c
+++ b/src/namestore/plugin_namestore_flat.c
@@ -86,6 +86,7 @@ struct Plugin
86 86
87}; 87};
88 88
89
89struct FlatFileEntry 90struct FlatFileEntry
90{ 91{
91 /** 92 /**
@@ -435,45 +436,51 @@ namestore_flat_store_records (void *cls,
435 UINT64_MAX); 436 UINT64_MAX);
436 key_len = strlen (label) + sizeof (struct GNUNET_CRYPTO_EcdsaPrivateKey); 437 key_len = strlen (label) + sizeof (struct GNUNET_CRYPTO_EcdsaPrivateKey);
437 key = GNUNET_malloc (key_len); 438 key = GNUNET_malloc (key_len);
438 GNUNET_memcpy (key, label, strlen (label)); 439 GNUNET_memcpy (key,
439 GNUNET_memcpy (key+strlen(label), 440 label,
440 zone_key, 441 strlen (label));
441 sizeof (struct GNUNET_CRYPTO_EcdsaPrivateKey)); 442 GNUNET_memcpy (key + strlen(label),
443 zone_key,
444 sizeof (struct GNUNET_CRYPTO_EcdsaPrivateKey));
442 GNUNET_CRYPTO_hash (key, 445 GNUNET_CRYPTO_hash (key,
443 key_len, 446 key_len,
444 &hkey); 447 &hkey);
445 448 GNUNET_CONTAINER_multihashmap_remove_all (plugin->hm,
446 GNUNET_CONTAINER_multihashmap_remove_all (plugin->hm, &hkey); 449 &hkey);
447 450 if (0 == rd_count)
448 if (0 < rd_count)
449 { 451 {
450 entry = GNUNET_new (struct FlatFileEntry); 452 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
451 entry->private_key = GNUNET_new (struct GNUNET_CRYPTO_EcdsaPrivateKey); 453 "sqlite",
452 GNUNET_asprintf (&entry->label, 454 "Record deleted\n");
453 label, 455 return GNUNET_OK;
454 strlen (label));
455 GNUNET_memcpy (entry->private_key,
456 zone_key,
457 sizeof (struct GNUNET_CRYPTO_EcdsaPrivateKey));
458 entry->rvalue = rvalue;
459 entry->record_count = rd_count;
460 entry->record_data = GNUNET_new_array (rd_count,
461 struct GNUNET_GNSRECORD_Data);
462 for (unsigned int i = 0; i < rd_count; i++)
463 {
464 entry->record_data[i].expiration_time = rd[i].expiration_time;
465 entry->record_data[i].record_type = rd[i].record_type;
466 entry->record_data[i].flags = rd[i].flags;
467 entry->record_data[i].data_size = rd[i].data_size;
468 entry->record_data[i].data = GNUNET_malloc (rd[i].data_size);
469 GNUNET_memcpy ((char*)entry->record_data[i].data, rd[i].data, rd[i].data_size);
470 }
471 return GNUNET_CONTAINER_multihashmap_put (plugin->hm,
472 &hkey,
473 entry,
474 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY);
475 } 456 }
476 return GNUNET_NO; 457 entry = GNUNET_new (struct FlatFileEntry);
458 entry->private_key = GNUNET_new (struct GNUNET_CRYPTO_EcdsaPrivateKey);
459 GNUNET_asprintf (&entry->label,
460 label,
461 strlen (label));
462 GNUNET_memcpy (entry->private_key,
463 zone_key,
464 sizeof (struct GNUNET_CRYPTO_EcdsaPrivateKey));
465 entry->rvalue = rvalue;
466 entry->record_count = rd_count;
467 entry->record_data = GNUNET_new_array (rd_count,
468 struct GNUNET_GNSRECORD_Data);
469 for (unsigned int i = 0; i < rd_count; i++)
470 {
471 entry->record_data[i].expiration_time = rd[i].expiration_time;
472 entry->record_data[i].record_type = rd[i].record_type;
473 entry->record_data[i].flags = rd[i].flags;
474 entry->record_data[i].data_size = rd[i].data_size;
475 entry->record_data[i].data = GNUNET_malloc (rd[i].data_size);
476 GNUNET_memcpy ((char*)entry->record_data[i].data,
477 rd[i].data,
478 rd[i].data_size);
479 }
480 return GNUNET_CONTAINER_multihashmap_put (plugin->hm,
481 &hkey,
482 entry,
483 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY);
477} 484}
478 485
479 486
@@ -485,7 +492,7 @@ namestore_flat_store_records (void *cls,
485 * @param label name of the record in the zone 492 * @param label name of the record in the zone
486 * @param iter function to call with the result 493 * @param iter function to call with the result
487 * @param iter_cls closure for @a iter 494 * @param iter_cls closure for @a iter
488 * @return #GNUNET_OK on success, else #GNUNET_SYSERR 495 * @return #GNUNET_OK on success, #GNUNET_NO for no results, else #GNUNET_SYSERR
489 */ 496 */
490static int 497static int
491namestore_flat_lookup_records (void *cls, 498namestore_flat_lookup_records (void *cls,
@@ -502,6 +509,7 @@ namestore_flat_lookup_records (void *cls,
502 509
503 if (NULL == zone) 510 if (NULL == zone)
504 { 511 {
512 GNUNET_break (0);
505 return GNUNET_SYSERR; 513 return GNUNET_SYSERR;
506 } 514 }
507 key_len = strlen (label) + sizeof (struct GNUNET_CRYPTO_EcdsaPrivateKey); 515 key_len = strlen (label) + sizeof (struct GNUNET_CRYPTO_EcdsaPrivateKey);
@@ -524,6 +532,7 @@ namestore_flat_lookup_records (void *cls,
524 return GNUNET_NO; 532 return GNUNET_NO;
525 if (NULL != iter) 533 if (NULL != iter)
526 iter (iter_cls, 534 iter (iter_cls,
535 0,
527 entry->private_key, 536 entry->private_key,
528 entry->label, 537 entry->label,
529 entry->record_count, 538 entry->record_count,
@@ -532,30 +541,85 @@ namestore_flat_lookup_records (void *cls,
532} 541}
533 542
534 543
544/**
545 * Closure for #iterate_zones.
546 */
547struct IterateContext
548{
549 /**
550 * How many more records should we skip before returning results?
551 */
552 uint64_t offset;
553
554 /**
555 * How many more records should we return?
556 */
557 uint64_t limit;
558
559 /**
560 * What is the position of the current entry, counting
561 * starts from 1.
562 */
563 uint64_t pos;
564
565 /**
566 * Target zone.
567 */
568 const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone;
569
570 /**
571 * Function to call on each record.
572 */
573 GNUNET_NAMESTORE_RecordIterator iter;
574
575 /**
576 * Closure for @e iter.
577 */
578 void *iter_cls;
579
580};
581
582
583/**
584 * Helper function for #namestore_flat_iterate_records().
585 *
586 * @param cls a `struct IterateContext`
587 * @param key unused
588 * @param value a `struct FlatFileEntry`
589 * @return #GNUNET_YES to continue the iteration
590 */
535static int 591static int
536iterate_zones (void *cls, 592iterate_zones (void *cls,
537 const struct GNUNET_HashCode *key, 593 const struct GNUNET_HashCode *key,
538 void *value) 594 void *value)
539{ 595{
540 struct Plugin *plugin = cls; 596 struct IterateContext *ic = cls;
541 struct FlatFileEntry *entry = value; 597 struct FlatFileEntry *entry = value;
542 598
543 (void) key; 599 (void) key;
544 if ((plugin->target_offset > plugin->offset) || 600 ic->pos++;
545 ( (NULL != plugin->iter_zone) && 601 if (0 == ic->limit)
546 (0 != memcmp (entry->private_key, 602 return GNUNET_NO;
547 plugin->iter_zone, 603 if ( (NULL != ic->zone) &&
548 sizeof (struct GNUNET_CRYPTO_EcdsaPrivateKey))))) { 604 (0 != memcmp (entry->private_key,
549 plugin->offset++; 605 ic->zone,
606 sizeof (struct GNUNET_CRYPTO_EcdsaPrivateKey))) )
607 return GNUNET_YES;
608 if (ic->offset > 0)
609 {
610 ic->offset--;
550 return GNUNET_YES; 611 return GNUNET_YES;
551 } 612 }
552 plugin->iter (plugin->iter_cls, 613 ic->iter (ic->iter_cls,
553 entry->private_key, 614 ic->pos,
554 entry->label, 615 entry->private_key,
555 entry->record_count, 616 entry->label,
556 entry->record_data); 617 entry->record_count,
557 plugin->iter_result_found = GNUNET_YES; 618 entry->record_data);
558 return GNUNET_NO; 619 ic->limit--;
620 if (0 == ic->limit)
621 return GNUNET_NO;
622 return GNUNET_YES;
559} 623}
560 624
561 625
@@ -565,32 +629,33 @@ iterate_zones (void *cls,
565 * 629 *
566 * @param cls closure (internal context for the plugin) 630 * @param cls closure (internal context for the plugin)
567 * @param zone hash of public key of the zone, NULL to iterate over all zones 631 * @param zone hash of public key of the zone, NULL to iterate over all zones
568 * @param offset offset in the list of all matching records 632 * @param serial serial number to exclude in the list of all matching records
633 * @param limit maximum number of results to return to @a iter
569 * @param iter function to call with the result 634 * @param iter function to call with the result
570 * @param iter_cls closure for @a iter 635 * @param iter_cls closure for @a iter
571 * @return #GNUNET_OK on success, #GNUNET_NO if there were no results, #GNUNET_SYSERR on error 636 * @return #GNUNET_OK on success, #GNUNET_NO if there were no more results, #GNUNET_SYSERR on error
572 */ 637 */
573static int 638static int
574namestore_flat_iterate_records (void *cls, 639namestore_flat_iterate_records (void *cls,
575 const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone, 640 const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone,
576 uint64_t offset, 641 uint64_t serial,
642 uint64_t limit,
577 GNUNET_NAMESTORE_RecordIterator iter, 643 GNUNET_NAMESTORE_RecordIterator iter,
578 void *iter_cls) 644 void *iter_cls)
579{ 645{
580 struct Plugin *plugin = cls; 646 struct Plugin *plugin = cls;
581 647 struct IterateContext ic;
582 /* FIXME: maybe use separate closure to better handle 648
583 recursive calls? */ 649 ic.offset = serial;
584 plugin->target_offset = offset; 650 ic.pos = 0;
585 plugin->offset = 0; 651 ic.limit = limit;
586 plugin->iter = iter; 652 ic.iter = iter;
587 plugin->iter_cls = iter_cls; 653 ic.iter_cls = iter_cls;
588 plugin->iter_zone = zone; 654 ic.zone = zone;
589 plugin->iter_result_found = GNUNET_NO;
590 GNUNET_CONTAINER_multihashmap_iterate (plugin->hm, 655 GNUNET_CONTAINER_multihashmap_iterate (plugin->hm,
591 &iterate_zones, 656 &iterate_zones,
592 plugin); 657 &ic);
593 return plugin->iter_result_found; 658 return (0 == ic.limit) ? GNUNET_OK : GNUNET_NO;
594} 659}
595 660
596 661
@@ -617,6 +682,7 @@ zone_to_name (void *cls,
617 sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey))) 682 sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey)))
618 { 683 {
619 plugin->iter (plugin->iter_cls, 684 plugin->iter (plugin->iter_cls,
685 0,
620 entry->private_key, 686 entry->private_key,
621 entry->label, 687 entry->label,
622 entry->record_count, 688 entry->record_count,
diff --git a/src/namestore/plugin_namestore_postgres.c b/src/namestore/plugin_namestore_postgres.c
index a9c19d517..d7907b1a6 100644
--- a/src/namestore/plugin_namestore_postgres.c
+++ b/src/namestore/plugin_namestore_postgres.c
@@ -65,23 +65,27 @@ static int
65database_setup (struct Plugin *plugin) 65database_setup (struct Plugin *plugin)
66{ 66{
67 struct GNUNET_PQ_ExecuteStatement es_temporary = 67 struct GNUNET_PQ_ExecuteStatement es_temporary =
68 GNUNET_PQ_make_execute ("CREATE TEMPORARY TABLE IF NOT EXISTS ns097records (" 68 GNUNET_PQ_make_execute ("CREATE TEMPORARY TABLE IF NOT EXISTS ns098records ("
69 " seq BIGSERIAL PRIMARY KEY,"
69 " zone_private_key BYTEA NOT NULL DEFAULT ''," 70 " zone_private_key BYTEA NOT NULL DEFAULT '',"
70 " pkey BYTEA DEFAULT ''," 71 " pkey BYTEA DEFAULT '',"
71 " rvalue BYTEA NOT NULL DEFAULT ''," 72 " rvalue BYTEA NOT NULL DEFAULT '',"
72 " record_count INTEGER NOT NULL DEFAULT 0," 73 " record_count INTEGER NOT NULL DEFAULT 0,"
73 " record_data BYTEA NOT NULL DEFAULT ''," 74 " record_data BYTEA NOT NULL DEFAULT '',"
74 " label TEXT NOT NULL DEFAULT ''" 75 " label TEXT NOT NULL DEFAULT '',"
76 " CONSTRAINT zl UNIQUE (zone_private_key,label)"
75 ")" 77 ")"
76 "WITH OIDS"); 78 "WITH OIDS");
77 struct GNUNET_PQ_ExecuteStatement es_default = 79 struct GNUNET_PQ_ExecuteStatement es_default =
78 GNUNET_PQ_make_execute ("CREATE TABLE IF NOT EXISTS ns097records (" 80 GNUNET_PQ_make_execute ("CREATE TABLE IF NOT EXISTS ns098records ("
81 " seq BIGSERIAL PRIMARY KEY,"
79 " zone_private_key BYTEA NOT NULL DEFAULT ''," 82 " zone_private_key BYTEA NOT NULL DEFAULT '',"
80 " pkey BYTEA DEFAULT ''," 83 " pkey BYTEA DEFAULT '',"
81 " rvalue BYTEA NOT NULL DEFAULT ''," 84 " rvalue BYTEA NOT NULL DEFAULT '',"
82 " record_count INTEGER NOT NULL DEFAULT 0," 85 " record_count INTEGER NOT NULL DEFAULT 0,"
83 " record_data BYTEA NOT NULL DEFAULT ''," 86 " record_data BYTEA NOT NULL DEFAULT '',"
84 " label TEXT NOT NULL DEFAULT ''" 87 " label TEXT NOT NULL DEFAULT '',"
88 " CONSTRAINT zl UNIQUE (zone_private_key,label)"
85 ")" 89 ")"
86 "WITH OIDS"); 90 "WITH OIDS");
87 const struct GNUNET_PQ_ExecuteStatement *cr; 91 const struct GNUNET_PQ_ExecuteStatement *cr;
@@ -125,13 +129,13 @@ database_setup (struct Plugin *plugin)
125 struct GNUNET_PQ_ExecuteStatement es[] = { 129 struct GNUNET_PQ_ExecuteStatement es[] = {
126 *cr, 130 *cr,
127 GNUNET_PQ_make_try_execute ("CREATE INDEX IF NOT EXISTS ir_pkey_reverse " 131 GNUNET_PQ_make_try_execute ("CREATE INDEX IF NOT EXISTS ir_pkey_reverse "
128 "ON ns097records (zone_private_key,pkey)"), 132 "ON ns098records (zone_private_key,pkey)"),
129 GNUNET_PQ_make_try_execute ("CREATE INDEX IF NOT EXISTS ir_pkey_iter " 133 GNUNET_PQ_make_try_execute ("CREATE INDEX IF NOT EXISTS ir_pkey_iter "
130 "ON ns097records (zone_private_key,rvalue)"), 134 "ON ns098records (zone_private_key,seq)"),
131 GNUNET_PQ_make_try_execute ("CREATE INDEX IF NOT EXISTS it_iter "
132 "ON ns097records (rvalue)"),
133 GNUNET_PQ_make_try_execute ("CREATE INDEX IF NOT EXISTS ir_label " 135 GNUNET_PQ_make_try_execute ("CREATE INDEX IF NOT EXISTS ir_label "
134 "ON ns097records (label)"), 136 "ON ns098records (label)"),
137 GNUNET_PQ_make_try_execute ("CREATE INDEX IF NOT EXISTS zone_label "
138 "ON ns098records (zone_private_key,label)"),
135 GNUNET_PQ_EXECUTE_STATEMENT_END 139 GNUNET_PQ_EXECUTE_STATEMENT_END
136 }; 140 };
137 141
@@ -148,23 +152,35 @@ database_setup (struct Plugin *plugin)
148 { 152 {
149 struct GNUNET_PQ_PreparedStatement ps[] = { 153 struct GNUNET_PQ_PreparedStatement ps[] = {
150 GNUNET_PQ_make_prepare ("store_records", 154 GNUNET_PQ_make_prepare ("store_records",
151 "INSERT INTO ns097records (zone_private_key, pkey, rvalue, record_count, record_data, label) VALUES " 155 "INSERT INTO ns098records"
152 "($1, $2, $3, $4, $5, $6)", 6), 156 " (zone_private_key, pkey, rvalue, record_count, record_data, label)"
157 " VALUES ($1, $2, $3, $4, $5, $6)"
158 " ON CONFLICT ON CONSTRAINT zl"
159 " DO UPDATE"
160 " SET pkey=$2,rvalue=$3,record_count=$4,record_data=$5"
161 " WHERE ns098records.zone_private_key = $1"
162 " AND ns098records.label = $6",
163 6),
153 GNUNET_PQ_make_prepare ("delete_records", 164 GNUNET_PQ_make_prepare ("delete_records",
154 "DELETE FROM ns097records " 165 "DELETE FROM ns098records "
155 "WHERE zone_private_key=$1 AND label=$2", 2), 166 "WHERE zone_private_key=$1 AND label=$2",
167 2),
156 GNUNET_PQ_make_prepare ("zone_to_name", 168 GNUNET_PQ_make_prepare ("zone_to_name",
157 "SELECT record_count,record_data,label FROM ns097records" 169 "SELECT seq,record_count,record_data,label FROM ns098records"
158 " WHERE zone_private_key=$1 AND pkey=$2", 2), 170 " WHERE zone_private_key=$1 AND pkey=$2",
171 2),
159 GNUNET_PQ_make_prepare ("iterate_zone", 172 GNUNET_PQ_make_prepare ("iterate_zone",
160 "SELECT record_count,record_data,label FROM ns097records " 173 "SELECT seq,record_count,record_data,label FROM ns098records "
161 "WHERE zone_private_key=$1 ORDER BY rvalue LIMIT 1 OFFSET $2", 2), 174 "WHERE zone_private_key=$1 AND seq > $2 ORDER BY seq ASC LIMIT $3",
175 3),
162 GNUNET_PQ_make_prepare ("iterate_all_zones", 176 GNUNET_PQ_make_prepare ("iterate_all_zones",
163 "SELECT record_count,record_data,label,zone_private_key" 177 "SELECT seq,record_count,record_data,label,zone_private_key"
164 " FROM ns097records ORDER BY rvalue LIMIT 1 OFFSET $1", 1), 178 " FROM ns098records WHERE seq > $1 ORDER BY seq ASC LIMIT $2",
179 2),
165 GNUNET_PQ_make_prepare ("lookup_label", 180 GNUNET_PQ_make_prepare ("lookup_label",
166 "SELECT record_count,record_data,label " 181 "SELECT seq,record_count,record_data,label "
167 "FROM ns097records WHERE zone_private_key=$1 AND label=$2", 2), 182 "FROM ns098records WHERE zone_private_key=$1 AND label=$2",
183 2),
168 GNUNET_PQ_PREPARED_STATEMENT_END 184 GNUNET_PQ_PREPARED_STATEMENT_END
169 }; 185 };
170 186
@@ -206,7 +222,9 @@ namestore_postgres_store_records (void *cls,
206 uint32_t rd_count32 = (uint32_t) rd_count; 222 uint32_t rd_count32 = (uint32_t) rd_count;
207 size_t data_size; 223 size_t data_size;
208 224
209 memset (&pkey, 0, sizeof (pkey)); 225 memset (&pkey,
226 0,
227 sizeof (pkey));
210 for (unsigned int i=0;i<rd_count;i++) 228 for (unsigned int i=0;i<rd_count;i++)
211 if (GNUNET_GNSRECORD_TYPE_PKEY == rd[i].record_type) 229 if (GNUNET_GNSRECORD_TYPE_PKEY == rd[i].record_type)
212 { 230 {
@@ -218,12 +236,38 @@ namestore_postgres_store_records (void *cls,
218 } 236 }
219 rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, 237 rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
220 UINT64_MAX); 238 UINT64_MAX);
221 data_size = GNUNET_GNSRECORD_records_get_size (rd_count, rd); 239 data_size = GNUNET_GNSRECORD_records_get_size (rd_count,
240 rd);
222 if (data_size > 64 * 65536) 241 if (data_size > 64 * 65536)
223 { 242 {
224 GNUNET_break (0); 243 GNUNET_break (0);
225 return GNUNET_SYSERR; 244 return GNUNET_SYSERR;
226 } 245 }
246 /* if record set is empty, delete existing records */
247 if (0 == rd_count)
248 {
249 struct GNUNET_PQ_QueryParam params[] = {
250 GNUNET_PQ_query_param_auto_from_type (zone_key),
251 GNUNET_PQ_query_param_string (label),
252 GNUNET_PQ_query_param_end
253 };
254 enum GNUNET_DB_QueryStatus res;
255
256 res = GNUNET_PQ_eval_prepared_non_select (plugin->dbh,
257 "delete_records",
258 params);
259 if ( (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT != res) &&
260 (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS != res) )
261 {
262 GNUNET_break (0);
263 return GNUNET_SYSERR;
264 }
265 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
266 "postgres",
267 "Record deleted\n");
268 return GNUNET_OK;
269 }
270 /* otherwise, UPSERT (i.e. UPDATE if exists, otherwise INSERT) */
227 { 271 {
228 char data[data_size]; 272 char data[data_size];
229 struct GNUNET_PQ_QueryParam params[] = { 273 struct GNUNET_PQ_QueryParam params[] = {
@@ -278,6 +322,12 @@ struct ParserContext
278 * Zone key, NULL if part of record. 322 * Zone key, NULL if part of record.
279 */ 323 */
280 const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone_key; 324 const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone_key;
325
326 /**
327 * Number of results still to return (counted down by
328 * number of results given to iterator).
329 */
330 uint64_t limit;
281}; 331};
282 332
283 333
@@ -296,14 +346,18 @@ parse_result_call_iterator (void *cls,
296{ 346{
297 struct ParserContext *pc = cls; 347 struct ParserContext *pc = cls;
298 348
349 if (NULL == pc->iter)
350 return; /* no need to do more work */
299 for (unsigned int i=0;i<num_results;i++) 351 for (unsigned int i=0;i<num_results;i++)
300 { 352 {
353 uint64_t serial;
301 void *data; 354 void *data;
302 size_t data_size; 355 size_t data_size;
303 uint32_t record_count; 356 uint32_t record_count;
304 char *label; 357 char *label;
305 struct GNUNET_CRYPTO_EcdsaPrivateKey zk; 358 struct GNUNET_CRYPTO_EcdsaPrivateKey zk;
306 struct GNUNET_PQ_ResultSpec rs_with_zone[] = { 359 struct GNUNET_PQ_ResultSpec rs_with_zone[] = {
360 GNUNET_PQ_result_spec_uint64 ("seq", &serial),
307 GNUNET_PQ_result_spec_uint32 ("record_count", &record_count), 361 GNUNET_PQ_result_spec_uint32 ("record_count", &record_count),
308 GNUNET_PQ_result_spec_variable_size ("record_data", &data, &data_size), 362 GNUNET_PQ_result_spec_variable_size ("record_data", &data, &data_size),
309 GNUNET_PQ_result_spec_string ("label", &label), 363 GNUNET_PQ_result_spec_string ("label", &label),
@@ -311,6 +365,7 @@ parse_result_call_iterator (void *cls,
311 GNUNET_PQ_result_spec_end 365 GNUNET_PQ_result_spec_end
312 }; 366 };
313 struct GNUNET_PQ_ResultSpec rs_without_zone[] = { 367 struct GNUNET_PQ_ResultSpec rs_without_zone[] = {
368 GNUNET_PQ_result_spec_uint64 ("seq", &serial),
314 GNUNET_PQ_result_spec_uint32 ("record_count", &record_count), 369 GNUNET_PQ_result_spec_uint32 ("record_count", &record_count),
315 GNUNET_PQ_result_spec_variable_size ("record_data", &data, &data_size), 370 GNUNET_PQ_result_spec_variable_size ("record_data", &data, &data_size),
316 GNUNET_PQ_result_spec_string ("label", &label), 371 GNUNET_PQ_result_spec_string ("label", &label),
@@ -351,6 +406,7 @@ parse_result_call_iterator (void *cls,
351 return; 406 return;
352 } 407 }
353 pc->iter (pc->iter_cls, 408 pc->iter (pc->iter_cls,
409 serial,
354 (NULL == pc->zone_key) ? &zk : pc->zone_key, 410 (NULL == pc->zone_key) ? &zk : pc->zone_key,
355 label, 411 label,
356 record_count, 412 record_count,
@@ -358,6 +414,7 @@ parse_result_call_iterator (void *cls,
358 } 414 }
359 GNUNET_PQ_cleanup_result (rs); 415 GNUNET_PQ_cleanup_result (rs);
360 } 416 }
417 pc->limit -= num_results;
361} 418}
362 419
363 420
@@ -369,7 +426,7 @@ parse_result_call_iterator (void *cls,
369 * @param label name of the record in the zone 426 * @param label name of the record in the zone
370 * @param iter function to call with the result 427 * @param iter function to call with the result
371 * @param iter_cls closure for @a iter 428 * @param iter_cls closure for @a iter
372 * @return #GNUNET_OK on success, else #GNUNET_SYSERR 429 * @return #GNUNET_OK on success, #GNUNET_NO for no results, else #GNUNET_SYSERR
373 */ 430 */
374static int 431static int
375namestore_postgres_lookup_records (void *cls, 432namestore_postgres_lookup_records (void *cls,
@@ -387,6 +444,11 @@ namestore_postgres_lookup_records (void *cls,
387 struct ParserContext pc; 444 struct ParserContext pc;
388 enum GNUNET_DB_QueryStatus res; 445 enum GNUNET_DB_QueryStatus res;
389 446
447 if (NULL == zone)
448 {
449 GNUNET_break (0);
450 return GNUNET_SYSERR;
451 }
390 pc.iter = iter; 452 pc.iter = iter;
391 pc.iter_cls = iter_cls; 453 pc.iter_cls = iter_cls;
392 pc.zone_key = zone; 454 pc.zone_key = zone;
@@ -395,8 +457,10 @@ namestore_postgres_lookup_records (void *cls,
395 params, 457 params,
396 &parse_result_call_iterator, 458 &parse_result_call_iterator,
397 &pc); 459 &pc);
398 if (res <= 0) 460 if (res < 0)
399 return GNUNET_SYSERR; 461 return GNUNET_SYSERR;
462 if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == res)
463 return GNUNET_NO;
400 return GNUNET_OK; 464 return GNUNET_OK;
401} 465}
402 466
@@ -407,15 +471,17 @@ namestore_postgres_lookup_records (void *cls,
407 * 471 *
408 * @param cls closure (internal context for the plugin) 472 * @param cls closure (internal context for the plugin)
409 * @param zone hash of public key of the zone, NULL to iterate over all zones 473 * @param zone hash of public key of the zone, NULL to iterate over all zones
410 * @param offset offset in the list of all matching records 474 * @param serial serial number to exclude in the list of all matching records
475 * @param limit maximum number of results to fetch
411 * @param iter function to call with the result 476 * @param iter function to call with the result
412 * @param iter_cls closure for @a iter 477 * @param iter_cls closure for @a iter
413 * @return #GNUNET_OK on success, #GNUNET_NO if there were no results, #GNUNET_SYSERR on error 478 * @return #GNUNET_OK on success, #GNUNET_NO if there were no more results, #GNUNET_SYSERR on error
414 */ 479 */
415static int 480static int
416namestore_postgres_iterate_records (void *cls, 481namestore_postgres_iterate_records (void *cls,
417 const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone, 482 const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone,
418 uint64_t offset, 483 uint64_t serial,
484 uint64_t limit,
419 GNUNET_NAMESTORE_RecordIterator iter, 485 GNUNET_NAMESTORE_RecordIterator iter,
420 void *iter_cls) 486 void *iter_cls)
421{ 487{
@@ -426,10 +492,12 @@ namestore_postgres_iterate_records (void *cls,
426 pc.iter = iter; 492 pc.iter = iter;
427 pc.iter_cls = iter_cls; 493 pc.iter_cls = iter_cls;
428 pc.zone_key = zone; 494 pc.zone_key = zone;
495 pc.limit = limit;
429 if (NULL == zone) 496 if (NULL == zone)
430 { 497 {
431 struct GNUNET_PQ_QueryParam params_without_zone[] = { 498 struct GNUNET_PQ_QueryParam params_without_zone[] = {
432 GNUNET_PQ_query_param_uint64 (&offset), 499 GNUNET_PQ_query_param_uint64 (&serial),
500 GNUNET_PQ_query_param_uint64 (&limit),
433 GNUNET_PQ_query_param_end 501 GNUNET_PQ_query_param_end
434 }; 502 };
435 503
@@ -443,7 +511,8 @@ namestore_postgres_iterate_records (void *cls,
443 { 511 {
444 struct GNUNET_PQ_QueryParam params_with_zone[] = { 512 struct GNUNET_PQ_QueryParam params_with_zone[] = {
445 GNUNET_PQ_query_param_auto_from_type (zone), 513 GNUNET_PQ_query_param_auto_from_type (zone),
446 GNUNET_PQ_query_param_uint64 (&offset), 514 GNUNET_PQ_query_param_uint64 (&serial),
515 GNUNET_PQ_query_param_uint64 (&limit),
447 GNUNET_PQ_query_param_end 516 GNUNET_PQ_query_param_end
448 }; 517 };
449 518
@@ -456,9 +525,9 @@ namestore_postgres_iterate_records (void *cls,
456 if (res < 0) 525 if (res < 0)
457 return GNUNET_SYSERR; 526 return GNUNET_SYSERR;
458 527
459 if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == res) 528 if ( (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == res) ||
529 (pc.limit > 0) )
460 return GNUNET_NO; 530 return GNUNET_NO;
461
462 return GNUNET_OK; 531 return GNUNET_OK;
463} 532}
464 533
diff --git a/src/namestore/plugin_namestore_sqlite.c b/src/namestore/plugin_namestore_sqlite.c
index 5ad84688c..f62be1e18 100644
--- a/src/namestore/plugin_namestore_sqlite.c
+++ b/src/namestore/plugin_namestore_sqlite.c
@@ -147,15 +147,13 @@ create_indices (sqlite3 * dbh)
147 /* create indices */ 147 /* create indices */
148 if ( (SQLITE_OK != 148 if ( (SQLITE_OK !=
149 sqlite3_exec (dbh, 149 sqlite3_exec (dbh,
150 "CREATE INDEX IF NOT EXISTS ir_pkey_reverse ON ns097records (zone_private_key,pkey)", 150 "CREATE INDEX IF NOT EXISTS ir_pkey_reverse "
151 "ON ns098records (zone_private_key,pkey)",
151 NULL, NULL, NULL)) || 152 NULL, NULL, NULL)) ||
152 (SQLITE_OK != 153 (SQLITE_OK !=
153 sqlite3_exec (dbh, 154 sqlite3_exec (dbh,
154 "CREATE INDEX IF NOT EXISTS ir_pkey_iter ON ns097records (zone_private_key,rvalue)", 155 "CREATE INDEX IF NOT EXISTS ir_pkey_iter "
155 NULL, NULL, NULL)) || 156 "ON ns098records (zone_private_key,uid)",
156 (SQLITE_OK !=
157 sqlite3_exec (dbh,
158 "CREATE INDEX IF NOT EXISTS it_iter ON ns097records (rvalue)",
159 NULL, NULL, NULL)) ) 157 NULL, NULL, NULL)) )
160 LOG (GNUNET_ERROR_TYPE_ERROR, 158 LOG (GNUNET_ERROR_TYPE_ERROR,
161 "Failed to create indices: %s\n", 159 "Failed to create indices: %s\n",
@@ -260,22 +258,24 @@ database_setup (struct Plugin *plugin)
260 /* Create table */ 258 /* Create table */
261 CHECK (SQLITE_OK == 259 CHECK (SQLITE_OK ==
262 sq_prepare (plugin->dbh, 260 sq_prepare (plugin->dbh,
263 "SELECT 1 FROM sqlite_master WHERE tbl_name = 'ns097records'", 261 "SELECT 1 FROM sqlite_master WHERE tbl_name = 'ns098records'",
264 &stmt)); 262 &stmt));
265 if ((sqlite3_step (stmt) == SQLITE_DONE) && 263 if ( (sqlite3_step (stmt) == SQLITE_DONE) &&
266 (sqlite3_exec 264 (SQLITE_OK !=
267 (plugin->dbh, 265 sqlite3_exec (plugin->dbh,
268 "CREATE TABLE ns097records (" 266 "CREATE TABLE ns098records ("
269 " zone_private_key BLOB NOT NULL," 267 " uid INTEGER PRIMARY KEY,"
270 " pkey BLOB," 268 " zone_private_key BLOB NOT NULL,"
271 " rvalue INT8 NOT NULL," 269 " pkey BLOB,"
272 " record_count INT NOT NULL," 270 " rvalue INT8 NOT NULL,"
273 " record_data BLOB NOT NULL," 271 " record_count INT NOT NULL,"
274 " label TEXT NOT NULL" 272 " record_data BLOB NOT NULL,"
275 ")", 273 " label TEXT NOT NULL"
276 NULL, NULL, NULL) != SQLITE_OK)) 274 ")",
275 NULL, NULL, NULL)) )
277 { 276 {
278 LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, 277 LOG_SQLITE (plugin,
278 GNUNET_ERROR_TYPE_ERROR,
279 "sqlite3_exec"); 279 "sqlite3_exec");
280 sqlite3_finalize (stmt); 280 sqlite3_finalize (stmt);
281 return GNUNET_SYSERR; 281 return GNUNET_SYSERR;
@@ -286,33 +286,40 @@ database_setup (struct Plugin *plugin)
286 286
287 if ( (SQLITE_OK != 287 if ( (SQLITE_OK !=
288 sq_prepare (plugin->dbh, 288 sq_prepare (plugin->dbh,
289 "INSERT INTO ns097records (zone_private_key, pkey, rvalue, record_count, record_data, label)" 289 "INSERT INTO ns098records (zone_private_key, pkey, rvalue, record_count, record_data, label)"
290 " VALUES (?, ?, ?, ?, ?, ?)", 290 " VALUES (?, ?, ?, ?, ?, ?)",
291 &plugin->store_records)) || 291 &plugin->store_records)) ||
292 (SQLITE_OK != 292 (SQLITE_OK !=
293 sq_prepare (plugin->dbh, 293 sq_prepare (plugin->dbh,
294 "DELETE FROM ns097records WHERE zone_private_key=? AND label=?", 294 "DELETE FROM ns098records WHERE zone_private_key=? AND label=?",
295 &plugin->delete_records)) || 295 &plugin->delete_records)) ||
296 (SQLITE_OK != 296 (SQLITE_OK !=
297 sq_prepare (plugin->dbh, 297 sq_prepare (plugin->dbh,
298 "SELECT record_count,record_data,label" 298 "SELECT uid,record_count,record_data,label"
299 " FROM ns097records WHERE zone_private_key=? AND pkey=?", 299 " FROM ns098records"
300 " WHERE zone_private_key=? AND pkey=?",
300 &plugin->zone_to_name)) || 301 &plugin->zone_to_name)) ||
301 (SQLITE_OK != 302 (SQLITE_OK !=
302 sq_prepare (plugin->dbh, 303 sq_prepare (plugin->dbh,
303 "SELECT record_count,record_data,label" 304 "SELECT uid,record_count,record_data,label"
304 " FROM ns097records WHERE zone_private_key=?" 305 " FROM ns098records"
305 " ORDER BY rvalue LIMIT 1 OFFSET ?", 306 " WHERE zone_private_key=? AND _rowid_ >= ?"
307 " ORDER BY _rowid_ ASC"
308 " LIMIT ?",
306 &plugin->iterate_zone)) || 309 &plugin->iterate_zone)) ||
307 (SQLITE_OK != 310 (SQLITE_OK !=
308 sq_prepare (plugin->dbh, 311 sq_prepare (plugin->dbh,
309 "SELECT record_count,record_data,label,zone_private_key" 312 "SELECT uid,record_count,record_data,label,zone_private_key"
310 " FROM ns097records ORDER BY rvalue LIMIT 1 OFFSET ?", 313 " FROM ns098records"
314 " WHERE _rowid_ >= ?"
315 " ORDER BY _rowid_ ASC"
316 " LIMIT ?",
311 &plugin->iterate_all_zones)) || 317 &plugin->iterate_all_zones)) ||
312 (SQLITE_OK != 318 (SQLITE_OK !=
313 sq_prepare (plugin->dbh, 319 sq_prepare (plugin->dbh,
314 "SELECT record_count,record_data,label,zone_private_key" 320 "SELECT uid,record_count,record_data,label,zone_private_key"
315 " FROM ns097records WHERE zone_private_key=? AND label=?", 321 " FROM ns098records"
322 " WHERE zone_private_key=? AND label=?",
316 &plugin->lookup_label)) 323 &plugin->lookup_label))
317 ) 324 )
318 { 325 {
@@ -405,10 +412,11 @@ namestore_sqlite_store_records (void *cls,
405 struct GNUNET_CRYPTO_EcdsaPublicKey pkey; 412 struct GNUNET_CRYPTO_EcdsaPublicKey pkey;
406 uint64_t rvalue; 413 uint64_t rvalue;
407 size_t data_size; 414 size_t data_size;
408 unsigned int i;
409 415
410 memset (&pkey, 0, sizeof (pkey)); 416 memset (&pkey,
411 for (i=0;i<rd_count;i++) 417 0,
418 sizeof (pkey));
419 for (unsigned int i=0;i<rd_count;i++)
412 if (GNUNET_GNSRECORD_TYPE_PKEY == rd[i].record_type) 420 if (GNUNET_GNSRECORD_TYPE_PKEY == rd[i].record_type)
413 { 421 {
414 GNUNET_break (sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey) == 422 GNUNET_break (sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey) ==
@@ -526,93 +534,115 @@ namestore_sqlite_store_records (void *cls,
526 * @param plugin plugin context 534 * @param plugin plugin context
527 * @param stmt to run (and then clean up) 535 * @param stmt to run (and then clean up)
528 * @param zone_key private key of the zone 536 * @param zone_key private key of the zone
537 * @param limit maximum number of results to fetch
529 * @param iter iterator to call with the result 538 * @param iter iterator to call with the result
530 * @param iter_cls closure for @a iter 539 * @param iter_cls closure for @a iter
531 * @return #GNUNET_OK on success, #GNUNET_NO if there were no results, #GNUNET_SYSERR on error 540 * @return #GNUNET_OK on success, #GNUNET_NO if there were no results, #GNUNET_SYSERR on error
532 */ 541 */
533static int 542static int
534get_record_and_call_iterator (struct Plugin *plugin, 543get_records_and_call_iterator (struct Plugin *plugin,
535 sqlite3_stmt *stmt, 544 sqlite3_stmt *stmt,
536 const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone_key, 545 const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone_key,
537 GNUNET_NAMESTORE_RecordIterator iter, 546 uint64_t limit,
538 void *iter_cls) 547 GNUNET_NAMESTORE_RecordIterator iter,
548 void *iter_cls)
539{ 549{
540 uint32_t record_count;
541 size_t data_size;
542 void *data;
543 char *label;
544 struct GNUNET_CRYPTO_EcdsaPrivateKey zk;
545 int ret; 550 int ret;
546 int sret; 551 int sret;
547 552
548 ret = GNUNET_NO; 553 ret = GNUNET_OK;
549 if (SQLITE_ROW == (sret = sqlite3_step (stmt))) 554 for (uint64_t i = 0;i<limit;i++)
550 { 555 {
551 struct GNUNET_SQ_ResultSpec rs[] = { 556 sret = sqlite3_step (stmt);
552 GNUNET_SQ_result_spec_uint32 (&record_count),
553 GNUNET_SQ_result_spec_variable_size (&data, &data_size),
554 GNUNET_SQ_result_spec_string (&label),
555 GNUNET_SQ_result_spec_end
556 };
557 struct GNUNET_SQ_ResultSpec rsx[] = {
558 GNUNET_SQ_result_spec_uint32 (&record_count),
559 GNUNET_SQ_result_spec_variable_size (&data, &data_size),
560 GNUNET_SQ_result_spec_string (&label),
561 GNUNET_SQ_result_spec_auto_from_type (&zk),
562 GNUNET_SQ_result_spec_end
563 };
564 557
565 if (NULL == zone_key) 558 if (SQLITE_DONE == sret)
566 { 559 {
567 zone_key = &zk; 560 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
568 ret = GNUNET_SQ_extract_result (stmt, 561 "Iteration done (no results)\n");
569 rsx); 562 ret = GNUNET_NO;
570 } 563 break;
571 else
572 {
573 ret = GNUNET_SQ_extract_result (stmt,
574 rs);
575 } 564 }
576 if ( (GNUNET_OK != ret) || 565 if (SQLITE_ROW != sret)
577 (record_count > 64 * 1024) )
578 { 566 {
579 /* sanity check, don't stack allocate far too much just 567 LOG_SQLITE (plugin,
580 because database might contain a large value here */ 568 GNUNET_ERROR_TYPE_ERROR,
581 GNUNET_break (0); 569 "sqlite_step");
582 ret = GNUNET_SYSERR; 570 ret = GNUNET_SYSERR;
571 break;
583 } 572 }
584 else 573
585 { 574 {
586 struct GNUNET_GNSRECORD_Data rd[record_count]; 575 uint64_t seq;
576 uint32_t record_count;
577 size_t data_size;
578 void *data;
579 char *label;
580 struct GNUNET_CRYPTO_EcdsaPrivateKey zk;
581 struct GNUNET_SQ_ResultSpec rs[] = {
582 GNUNET_SQ_result_spec_uint64 (&seq),
583 GNUNET_SQ_result_spec_uint32 (&record_count),
584 GNUNET_SQ_result_spec_variable_size (&data,
585 &data_size),
586 GNUNET_SQ_result_spec_string (&label),
587 GNUNET_SQ_result_spec_end
588 };
589 struct GNUNET_SQ_ResultSpec rsx[] = {
590 GNUNET_SQ_result_spec_uint64 (&seq),
591 GNUNET_SQ_result_spec_uint32 (&record_count),
592 GNUNET_SQ_result_spec_variable_size (&data,
593 &data_size),
594 GNUNET_SQ_result_spec_string (&label),
595 GNUNET_SQ_result_spec_auto_from_type (&zk),
596 GNUNET_SQ_result_spec_end
597 };
587 598
588 if (GNUNET_OK != 599 if (NULL == zone_key)
589 GNUNET_GNSRECORD_records_deserialize (data_size,
590 data,
591 record_count,
592 rd))
593 { 600 {
594 GNUNET_break (0); 601 zone_key = &zk;
595 ret = GNUNET_SYSERR; 602 ret = GNUNET_SQ_extract_result (stmt,
603 rsx);
596 } 604 }
597 else 605 else
598 { 606 {
599 if (NULL != iter) 607 ret = GNUNET_SQ_extract_result (stmt,
600 iter (iter_cls, 608 rs);
601 zone_key, 609 }
602 label, 610 if ( (GNUNET_OK != ret) ||
603 record_count, 611 (record_count > 64 * 1024) )
604 rd); 612 {
605 ret = GNUNET_YES; 613 /* sanity check, don't stack allocate far too much just
614 because database might contain a large value here */
615 GNUNET_break (0);
616 ret = GNUNET_SYSERR;
617 break;
606 } 618 }
619 else
620 {
621 struct GNUNET_GNSRECORD_Data rd[record_count];
622
623 if (GNUNET_OK !=
624 GNUNET_GNSRECORD_records_deserialize (data_size,
625 data,
626 record_count,
627 rd))
628 {
629 GNUNET_break (0);
630 ret = GNUNET_SYSERR;
631 break;
632 }
633 else
634 {
635 if (NULL != iter)
636 iter (iter_cls,
637 seq + 1,
638 zone_key,
639 label,
640 record_count,
641 rd);
642 }
643 }
644 GNUNET_SQ_cleanup_result (rs);
607 } 645 }
608 GNUNET_SQ_cleanup_result (rs);
609 }
610 else
611 {
612 if (SQLITE_DONE != sret)
613 LOG_SQLITE (plugin,
614 GNUNET_ERROR_TYPE_ERROR,
615 "sqlite_step");
616 } 646 }
617 GNUNET_SQ_reset (plugin->dbh, 647 GNUNET_SQ_reset (plugin->dbh,
618 stmt); 648 stmt);
@@ -628,7 +658,7 @@ get_record_and_call_iterator (struct Plugin *plugin,
628 * @param label name of the record in the zone 658 * @param label name of the record in the zone
629 * @param iter function to call with the result 659 * @param iter function to call with the result
630 * @param iter_cls closure for @a iter 660 * @param iter_cls closure for @a iter
631 * @return #GNUNET_OK on success, else #GNUNET_SYSERR 661 * @return #GNUNET_OK on success, #GNUNET_NO for no results, else #GNUNET_SYSERR
632 */ 662 */
633static int 663static int
634namestore_sqlite_lookup_records (void *cls, 664namestore_sqlite_lookup_records (void *cls,
@@ -645,7 +675,10 @@ namestore_sqlite_lookup_records (void *cls,
645 }; 675 };
646 676
647 if (NULL == zone) 677 if (NULL == zone)
678 {
679 GNUNET_break (0);
648 return GNUNET_SYSERR; 680 return GNUNET_SYSERR;
681 }
649 if (GNUNET_OK != 682 if (GNUNET_OK !=
650 GNUNET_SQ_bind (plugin->lookup_label, 683 GNUNET_SQ_bind (plugin->lookup_label,
651 params)) 684 params))
@@ -656,11 +689,12 @@ namestore_sqlite_lookup_records (void *cls,
656 plugin->lookup_label); 689 plugin->lookup_label);
657 return GNUNET_SYSERR; 690 return GNUNET_SYSERR;
658 } 691 }
659 return get_record_and_call_iterator (plugin, 692 return get_records_and_call_iterator (plugin,
660 plugin->lookup_label, 693 plugin->lookup_label,
661 zone, 694 zone,
662 iter, 695 1,
663 iter_cls); 696 iter,
697 iter_cls);
664} 698}
665 699
666 700
@@ -670,15 +704,17 @@ namestore_sqlite_lookup_records (void *cls,
670 * 704 *
671 * @param cls closure (internal context for the plugin) 705 * @param cls closure (internal context for the plugin)
672 * @param zone hash of public key of the zone, NULL to iterate over all zones 706 * @param zone hash of public key of the zone, NULL to iterate over all zones
673 * @param offset offset in the list of all matching records 707 * @param serial serial number to exclude in the list of all matching records
708 * @param limit maximum number of results to return
674 * @param iter function to call with the result 709 * @param iter function to call with the result
675 * @param iter_cls closure for @a iter 710 * @param iter_cls closure for @a iter
676 * @return #GNUNET_OK on success, #GNUNET_NO if there were no results, #GNUNET_SYSERR on error 711 * @return #GNUNET_OK on success, #GNUNET_NO if there were no more results, #GNUNET_SYSERR on error
677 */ 712 */
678static int 713static int
679namestore_sqlite_iterate_records (void *cls, 714namestore_sqlite_iterate_records (void *cls,
680 const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone, 715 const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone,
681 uint64_t offset, 716 uint64_t serial,
717 uint64_t limit,
682 GNUNET_NAMESTORE_RecordIterator iter, 718 GNUNET_NAMESTORE_RecordIterator iter,
683 void *iter_cls) 719 void *iter_cls)
684{ 720{
@@ -689,7 +725,8 @@ namestore_sqlite_iterate_records (void *cls,
689 if (NULL == zone) 725 if (NULL == zone)
690 { 726 {
691 struct GNUNET_SQ_QueryParam params[] = { 727 struct GNUNET_SQ_QueryParam params[] = {
692 GNUNET_SQ_query_param_uint64 (&offset), 728 GNUNET_SQ_query_param_uint64 (&serial),
729 GNUNET_SQ_query_param_uint64 (&limit),
693 GNUNET_SQ_query_param_end 730 GNUNET_SQ_query_param_end
694 }; 731 };
695 732
@@ -701,7 +738,8 @@ namestore_sqlite_iterate_records (void *cls,
701 { 738 {
702 struct GNUNET_SQ_QueryParam params[] = { 739 struct GNUNET_SQ_QueryParam params[] = {
703 GNUNET_SQ_query_param_auto_from_type (zone), 740 GNUNET_SQ_query_param_auto_from_type (zone),
704 GNUNET_SQ_query_param_uint64 (&offset), 741 GNUNET_SQ_query_param_uint64 (&serial),
742 GNUNET_SQ_query_param_uint64 (&limit),
705 GNUNET_SQ_query_param_end 743 GNUNET_SQ_query_param_end
706 }; 744 };
707 745
@@ -718,11 +756,12 @@ namestore_sqlite_iterate_records (void *cls,
718 stmt); 756 stmt);
719 return GNUNET_SYSERR; 757 return GNUNET_SYSERR;
720 } 758 }
721 return get_record_and_call_iterator (plugin, 759 return get_records_and_call_iterator (plugin,
722 stmt, 760 stmt,
723 zone, 761 zone,
724 iter, 762 limit,
725 iter_cls); 763 iter,
764 iter_cls);
726} 765}
727 766
728 767
@@ -741,7 +780,8 @@ static int
741namestore_sqlite_zone_to_name (void *cls, 780namestore_sqlite_zone_to_name (void *cls,
742 const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone, 781 const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone,
743 const struct GNUNET_CRYPTO_EcdsaPublicKey *value_zone, 782 const struct GNUNET_CRYPTO_EcdsaPublicKey *value_zone,
744 GNUNET_NAMESTORE_RecordIterator iter, void *iter_cls) 783 GNUNET_NAMESTORE_RecordIterator iter,
784 void *iter_cls)
745{ 785{
746 struct Plugin *plugin = cls; 786 struct Plugin *plugin = cls;
747 struct GNUNET_SQ_QueryParam params[] = { 787 struct GNUNET_SQ_QueryParam params[] = {
@@ -764,11 +804,12 @@ namestore_sqlite_zone_to_name (void *cls,
764 LOG (GNUNET_ERROR_TYPE_DEBUG, 804 LOG (GNUNET_ERROR_TYPE_DEBUG,
765 "Performing reverse lookup for `%s'\n", 805 "Performing reverse lookup for `%s'\n",
766 GNUNET_GNSRECORD_z2s (value_zone)); 806 GNUNET_GNSRECORD_z2s (value_zone));
767 return get_record_and_call_iterator (plugin, 807 return get_records_and_call_iterator (plugin,
768 plugin->zone_to_name, 808 plugin->zone_to_name,
769 zone, 809 zone,
770 iter, 810 1,
771 iter_cls); 811 iter,
812 iter_cls);
772} 813}
773 814
774 815
@@ -787,7 +828,9 @@ libgnunet_plugin_namestore_sqlite_init (void *cls)
787 828
788 if (NULL != plugin.cfg) 829 if (NULL != plugin.cfg)
789 return NULL; /* can only initialize once! */ 830 return NULL; /* can only initialize once! */
790 memset (&plugin, 0, sizeof (struct Plugin)); 831 memset (&plugin,
832 0,
833 sizeof (struct Plugin));
791 plugin.cfg = cfg; 834 plugin.cfg = cfg;
792 if (GNUNET_OK != database_setup (&plugin)) 835 if (GNUNET_OK != database_setup (&plugin))
793 { 836 {
diff --git a/src/namestore/plugin_rest_namestore.c b/src/namestore/plugin_rest_namestore.c
index 4602106da..fd1528a1d 100644
--- a/src/namestore/plugin_rest_namestore.c
+++ b/src/namestore/plugin_rest_namestore.c
@@ -452,7 +452,8 @@ namestore_list_response (void *cls,
452 "%s does not match %s\n", 452 "%s does not match %s\n",
453 rname, 453 rname,
454 handle->name); 454 handle->name);
455 GNUNET_NAMESTORE_zone_iterator_next (handle->list_it); 455 GNUNET_NAMESTORE_zone_iterator_next (handle->list_it,
456 1);
456 return; 457 return;
457 } 458 }
458 459
@@ -460,7 +461,7 @@ namestore_list_response (void *cls,
460 for (unsigned int i=0; i<rd_len; i++) 461 for (unsigned int i=0; i<rd_len; i++)
461 { 462 {
462 if ( (GNUNET_GNSRECORD_TYPE_NICK == rd[i].record_type) && 463 if ( (GNUNET_GNSRECORD_TYPE_NICK == rd[i].record_type) &&
463 (0 != strcmp (rname, GNUNET_GNS_MASTERZONE_STR)) ) 464 (0 != strcmp (rname, GNUNET_GNS_EMPTY_LABEL_AT)) )
464 continue; 465 continue;
465 466
466 if ( (rd[i].record_type != handle->type) && 467 if ( (rd[i].record_type != handle->type) &&
@@ -483,7 +484,8 @@ namestore_list_response (void *cls,
483 } 484 }
484 485
485 json_decref (result_array); 486 json_decref (result_array);
486 GNUNET_NAMESTORE_zone_iterator_next (handle->list_it); 487 GNUNET_NAMESTORE_zone_iterator_next (handle->list_it,
488 1);
487} 489}
488 490
489 491
diff --git a/src/namestore/test_namestore_api.conf b/src/namestore/test_namestore_api.conf
index a1a674d89..29ff90b5d 100644
--- a/src/namestore/test_namestore_api.conf
+++ b/src/namestore/test_namestore_api.conf
@@ -1,8 +1,19 @@
1@INLINE@ ../../contrib/no_forcestart.conf
2@INLINE@ ../../contrib/no_autostart_above_core.conf
3
1[PATHS] 4[PATHS]
2GNUNET_TEST_HOME = /tmp/test-gnunet-namestore/ 5GNUNET_TEST_HOME = /tmp/test-gnunet-namestore/
3 6
4[namestore] 7[namestore]
5DATABASE = sqlite 8DATABASE = sqlite
9AUTOSTART = YES
10
11[namecache]
12DATABASE = sqlite
13AUTOSTART = YES
14
15[identity]
16AUTOSTART = YES
6 17
7[namestore-sqlite] 18[namestore-sqlite]
8FILENAME = $GNUNET_TEST_HOME/namestore/sqlite_test.db 19FILENAME = $GNUNET_TEST_HOME/namestore/sqlite_test.db
diff --git a/src/namestore/test_namestore_api_flat.conf b/src/namestore/test_namestore_api_flat.conf
new file mode 100644
index 000000000..49fe25468
--- /dev/null
+++ b/src/namestore/test_namestore_api_flat.conf
@@ -0,0 +1,4 @@
1@INLINE@ test_namestore_api.conf
2
3[namestore]
4DATABASE = flat
diff --git a/src/namestore/test_namestore_api_lookup_nick.c b/src/namestore/test_namestore_api_lookup_nick.c
index de958cee2..50d1fd9a9 100644
--- a/src/namestore/test_namestore_api_lookup_nick.c
+++ b/src/namestore/test_namestore_api_lookup_nick.c
@@ -18,8 +18,8 @@
18 Boston, MA 02110-1301, USA. 18 Boston, MA 02110-1301, USA.
19*/ 19*/
20/** 20/**
21 * @file namestore/test_namestore_api_store.c 21 * @file namestore/test_namestore_api_lookup_nick.c
22 * @brief testcase for namestore_api.c: store a record 22 * @brief testcase for namestore_api.c: NICK records
23 */ 23 */
24#include "platform.h" 24#include "platform.h"
25#include "gnunet_namestore_service.h" 25#include "gnunet_namestore_service.h"
@@ -317,15 +317,23 @@ run (void *cls,
317int 317int
318main (int argc, char *argv[]) 318main (int argc, char *argv[])
319{ 319{
320 const char *plugin_name;
321 char *cfg_name;
322
323 plugin_name = GNUNET_TESTING_get_testname_from_underscore (argv[0]);
324 GNUNET_asprintf (&cfg_name,
325 "test_namestore_api_%s.conf",
326 plugin_name);
320 res = 1; 327 res = 1;
321 if (0 != 328 if (0 !=
322 GNUNET_TESTING_peer_run ("test-namestore-api", 329 GNUNET_TESTING_peer_run ("test-namestore-api-lookup-nick",
323 "test_namestore_api.conf", 330 cfg_name,
324 &run, 331 &run,
325 NULL)) 332 NULL))
326 { 333 {
327 res = 1; 334 res = 1;
328 } 335 }
336 GNUNET_free (cfg_name);
329 if (NULL != directory) 337 if (NULL != directory)
330 { 338 {
331 GNUNET_DISK_directory_remove (directory); 339 GNUNET_DISK_directory_remove (directory);
diff --git a/src/namestore/test_namestore_api_lookup_private.c b/src/namestore/test_namestore_api_lookup_private.c
index 57505c48b..7866749f1 100644
--- a/src/namestore/test_namestore_api_lookup_private.c
+++ b/src/namestore/test_namestore_api_lookup_private.c
@@ -230,15 +230,23 @@ run (void *cls,
230int 230int
231main (int argc, char *argv[]) 231main (int argc, char *argv[])
232{ 232{
233 const char *plugin_name;
234 char *cfg_name;
235
236 plugin_name = GNUNET_TESTING_get_testname_from_underscore (argv[0]);
237 GNUNET_asprintf (&cfg_name,
238 "test_namestore_api_%s.conf",
239 plugin_name);
233 res = 1; 240 res = 1;
234 if (0 != 241 if (0 !=
235 GNUNET_TESTING_peer_run ("test-namestore-api", 242 GNUNET_TESTING_peer_run ("test-namestore-api-lookup-private",
236 "test_namestore_api.conf", 243 cfg_name,
237 &run, 244 &run,
238 NULL)) 245 NULL))
239 { 246 {
240 res = 1; 247 res = 1;
241 } 248 }
249 GNUNET_free (cfg_name);
242 if (NULL != directory) 250 if (NULL != directory)
243 { 251 {
244 GNUNET_DISK_directory_remove (directory); 252 GNUNET_DISK_directory_remove (directory);
@@ -248,4 +256,4 @@ main (int argc, char *argv[])
248} 256}
249 257
250 258
251/* end of test_namestore_api_store.c */ 259/* end of test_namestore_api_lookup_private.c */
diff --git a/src/namestore/test_namestore_api_lookup_public.c b/src/namestore/test_namestore_api_lookup_public.c
index 09d6b302d..02ca16042 100644
--- a/src/namestore/test_namestore_api_lookup_public.c
+++ b/src/namestore/test_namestore_api_lookup_public.c
@@ -237,15 +237,23 @@ run (void *cls,
237int 237int
238main (int argc, char *argv[]) 238main (int argc, char *argv[])
239{ 239{
240 const char *plugin_name;
241 char *cfg_name;
242
243 plugin_name = GNUNET_TESTING_get_testname_from_underscore (argv[0]);
244 GNUNET_asprintf (&cfg_name,
245 "test_namestore_api_%s.conf",
246 plugin_name);
240 res = 1; 247 res = 1;
241 if (0 != 248 if (0 !=
242 GNUNET_TESTING_peer_run ("test-namestore-api", 249 GNUNET_TESTING_peer_run ("test-namestore-api",
243 "test_namestore_api.conf", 250 cfg_name,
244 &run, 251 &run,
245 NULL)) 252 NULL))
246 { 253 {
247 res = 1; 254 res = 1;
248 } 255 }
256 GNUNET_free (cfg_name);
249 if (NULL != directory) 257 if (NULL != directory)
250 { 258 {
251 GNUNET_DISK_directory_remove (directory); 259 GNUNET_DISK_directory_remove (directory);
diff --git a/src/namestore/test_namestore_api_lookup_shadow.c b/src/namestore/test_namestore_api_lookup_shadow.c
index 7f9a90704..e80335796 100644
--- a/src/namestore/test_namestore_api_lookup_shadow.c
+++ b/src/namestore/test_namestore_api_lookup_shadow.c
@@ -18,7 +18,7 @@
18 Boston, MA 02110-1301, USA. 18 Boston, MA 02110-1301, USA.
19*/ 19*/
20/** 20/**
21 * @file namestore/test_namestore_api_lookup_shadow_filter.c 21 * @file namestore/test_namestore_api_lookup_shadow.c
22 * @brief testcase for namestore_api.c: store a shadow record and perform a lookup 22 * @brief testcase for namestore_api.c: store a shadow record and perform a lookup
23 * test passes if test returns the record but without the shadow flag since no 23 * test passes if test returns the record but without the shadow flag since no
24 * other valid record is available 24 * other valid record is available
@@ -267,15 +267,23 @@ run (void *cls,
267int 267int
268main (int argc, char *argv[]) 268main (int argc, char *argv[])
269{ 269{
270 const char *plugin_name;
271 char *cfg_name;
272
273 plugin_name = GNUNET_TESTING_get_testname_from_underscore (argv[0]);
274 GNUNET_asprintf (&cfg_name,
275 "test_namestore_api_%s.conf",
276 plugin_name);
270 res = 1; 277 res = 1;
271 if (0 != 278 if (0 !=
272 GNUNET_TESTING_peer_run ("test-namestore-api", 279 GNUNET_TESTING_peer_run ("test-namestore-api-lookup-shadow",
273 "test_namestore_api.conf", 280 cfg_name,
274 &run, 281 &run,
275 NULL)) 282 NULL))
276 { 283 {
277 res = 1; 284 res = 1;
278 } 285 }
286 GNUNET_free (cfg_name);
279 if (NULL != directory) 287 if (NULL != directory)
280 { 288 {
281 GNUNET_DISK_directory_remove (directory); 289 GNUNET_DISK_directory_remove (directory);
@@ -285,4 +293,4 @@ main (int argc, char *argv[])
285} 293}
286 294
287 295
288/* end of test_namestore_api_lookup_shadow_filter.c */ 296/* end of test_namestore_api_lookup_shadow.c */
diff --git a/src/namestore/test_namestore_api_lookup_shadow_filter.c b/src/namestore/test_namestore_api_lookup_shadow_filter.c
index a22baa17b..5b8811a23 100644
--- a/src/namestore/test_namestore_api_lookup_shadow_filter.c
+++ b/src/namestore/test_namestore_api_lookup_shadow_filter.c
@@ -345,15 +345,23 @@ run (void *cls,
345int 345int
346main (int argc, char *argv[]) 346main (int argc, char *argv[])
347{ 347{
348 const char *plugin_name;
349 char *cfg_name;
350
351 plugin_name = GNUNET_TESTING_get_testname_from_underscore (argv[0]);
352 GNUNET_asprintf (&cfg_name,
353 "test_namestore_api_%s.conf",
354 plugin_name);
348 res = 1; 355 res = 1;
349 if (0 != 356 if (0 !=
350 GNUNET_TESTING_peer_run ("test-namestore-api", 357 GNUNET_TESTING_peer_run ("test-namestore-api-lookup-shadow-filter",
351 "test_namestore_api.conf", 358 cfg_name,
352 &run, 359 &run,
353 NULL)) 360 NULL))
354 { 361 {
355 res = 1; 362 res = 1;
356 } 363 }
364 GNUNET_free (cfg_name);
357 if (NULL != directory) 365 if (NULL != directory)
358 { 366 {
359 GNUNET_DISK_directory_remove (directory); 367 GNUNET_DISK_directory_remove (directory);
diff --git a/src/namestore/test_namestore_api_monitoring.c b/src/namestore/test_namestore_api_monitoring.c
index efbd6badf..f6d4fe226 100644
--- a/src/namestore/test_namestore_api_monitoring.c
+++ b/src/namestore/test_namestore_api_monitoring.c
@@ -209,7 +209,9 @@ zone_proc (void *cls,
209 209
210 210
211static void 211static void
212put_cont (void *cls, int32_t success, const char *emsg) 212put_cont (void *cls,
213 int32_t success,
214 const char *emsg)
213{ 215{
214 static int c = 0; 216 static int c = 0;
215 char *label = cls; 217 char *label = cls;
@@ -232,10 +234,12 @@ put_cont (void *cls, int32_t success, const char *emsg)
232 else 234 else
233 { 235 {
234 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 236 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
235 "Failed to created records\n"); 237 "Failed to create record `%s'\n",
238 label);
236 GNUNET_break (0); 239 GNUNET_break (0);
237 GNUNET_SCHEDULER_cancel (endbadly_task); 240 GNUNET_SCHEDULER_cancel (endbadly_task);
238 endbadly_task = GNUNET_SCHEDULER_add_now (&endbadly, NULL); 241 endbadly_task = GNUNET_SCHEDULER_add_now (&endbadly,
242 NULL);
239 } 243 }
240} 244}
241 245
@@ -341,10 +345,16 @@ run (void *cls,
341 /* name in different zone */ 345 /* name in different zone */
342 GNUNET_asprintf(&s_name_3, "dummy3"); 346 GNUNET_asprintf(&s_name_3, "dummy3");
343 s_rd_3 = create_record(1); 347 s_rd_3 = create_record(1);
344 GNUNET_assert (NULL != (ns_ops[2] = GNUNET_NAMESTORE_records_store (nsh, privkey2, s_name_3, 348 GNUNET_assert (NULL != (ns_ops[2] =
345 1, s_rd_3, &put_cont, s_name_3))); 349 GNUNET_NAMESTORE_records_store (nsh,
346 350 privkey2,
347 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Created record 1\n"); 351 s_name_3,
352 1,
353 s_rd_3,
354 &put_cont,
355 s_name_3)));
356 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
357 "Created record 1\n");
348 GNUNET_asprintf(&s_name_1, "dummy1"); 358 GNUNET_asprintf(&s_name_1, "dummy1");
349 s_rd_1 = create_record(1); 359 s_rd_1 = create_record(1);
350 GNUNET_assert (NULL != (ns_ops[0] = GNUNET_NAMESTORE_records_store(nsh, privkey, s_name_1, 360 GNUNET_assert (NULL != (ns_ops[0] = GNUNET_NAMESTORE_records_store(nsh, privkey, s_name_1,
@@ -364,15 +374,23 @@ run (void *cls,
364int 374int
365main (int argc, char *argv[]) 375main (int argc, char *argv[])
366{ 376{
377 const char *plugin_name;
378 char *cfg_name;
379
380 plugin_name = GNUNET_TESTING_get_testname_from_underscore (argv[0]);
381 GNUNET_asprintf (&cfg_name,
382 "test_namestore_api_%s.conf",
383 plugin_name);
367 res = 1; 384 res = 1;
368 if (0 != 385 if (0 !=
369 GNUNET_TESTING_peer_run ("test-namestore-api-monitoring", 386 GNUNET_TESTING_peer_run ("test-namestore-api-monitoring",
370 "test_namestore_api.conf", 387 cfg_name,
371 &run, 388 &run,
372 NULL)) 389 NULL))
373 { 390 {
374 res = 1; 391 res = 1;
375 } 392 }
393 GNUNET_free (cfg_name);
376 if (NULL != directory) 394 if (NULL != directory)
377 { 395 {
378 GNUNET_DISK_directory_remove (directory); 396 GNUNET_DISK_directory_remove (directory);
diff --git a/src/namestore/test_namestore_api_monitoring_existing.c b/src/namestore/test_namestore_api_monitoring_existing.c
index cd1838b5c..17f726eb1 100644
--- a/src/namestore/test_namestore_api_monitoring_existing.c
+++ b/src/namestore/test_namestore_api_monitoring_existing.c
@@ -71,18 +71,18 @@ do_shutdown ()
71 71
72 if (NULL != ns_ops[0]) 72 if (NULL != ns_ops[0])
73 { 73 {
74 GNUNET_NAMESTORE_cancel(ns_ops[0]); 74 GNUNET_NAMESTORE_cancel(ns_ops[0]);
75 ns_ops[0] = NULL; 75 ns_ops[0] = NULL;
76 } 76 }
77 if (NULL != ns_ops[1]) 77 if (NULL != ns_ops[1])
78 { 78 {
79 GNUNET_NAMESTORE_cancel(ns_ops[1]); 79 GNUNET_NAMESTORE_cancel(ns_ops[1]);
80 ns_ops[1] = NULL; 80 ns_ops[1] = NULL;
81 } 81 }
82 if (NULL != ns_ops[2]) 82 if (NULL != ns_ops[2])
83 { 83 {
84 GNUNET_NAMESTORE_cancel(ns_ops[2]); 84 GNUNET_NAMESTORE_cancel(ns_ops[2]);
85 ns_ops[2] = NULL; 85 ns_ops[2] = NULL;
86 } 86 }
87 87
88 if (NULL != nsh) 88 if (NULL != nsh)
@@ -274,18 +274,20 @@ put_cont (void *cls, int32_t success, const char *emsg)
274static struct GNUNET_GNSRECORD_Data * 274static struct GNUNET_GNSRECORD_Data *
275create_record (unsigned int count) 275create_record (unsigned int count)
276{ 276{
277 unsigned int c;
278 struct GNUNET_GNSRECORD_Data * rd; 277 struct GNUNET_GNSRECORD_Data * rd;
279 278
280 rd = GNUNET_malloc (count * sizeof (struct GNUNET_GNSRECORD_Data)); 279 rd = GNUNET_new_array (count,
281 for (c = 0; c < count; c++) 280 struct GNUNET_GNSRECORD_Data);
281 for (unsigned int c = 0; c < count; c++)
282 { 282 {
283 rd[c].expiration_time = GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_HOURS).abs_value_us; 283 rd[c].expiration_time = GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_HOURS).abs_value_us;
284 rd[c].record_type = 1111; 284 rd[c].record_type = 1111;
285 rd[c].data_size = 50; 285 rd[c].data_size = 50;
286 rd[c].data = GNUNET_malloc(50); 286 rd[c].data = GNUNET_malloc(50);
287 rd[c].flags = 0; 287 rd[c].flags = 0;
288 memset ((char *) rd[c].data, 'a', 50); 288 memset ((char *) rd[c].data,
289 'a',
290 50);
289 } 291 }
290 return rd; 292 return rd;
291} 293}
@@ -300,7 +302,10 @@ run (void *cls,
300 302
301 directory = NULL; 303 directory = NULL;
302 GNUNET_assert (GNUNET_OK == 304 GNUNET_assert (GNUNET_OK ==
303 GNUNET_CONFIGURATION_get_value_string(mycfg, "PATHS", "GNUNET_TEST_HOME", &directory)); 305 GNUNET_CONFIGURATION_get_value_string (mycfg,
306 "PATHS",
307 "GNUNET_TEST_HOME",
308 &directory));
304 GNUNET_DISK_directory_remove (directory); 309 GNUNET_DISK_directory_remove (directory);
305 310
306 res = 1; 311 res = 1;
@@ -363,15 +368,23 @@ run (void *cls,
363int 368int
364main (int argc, char *argv[]) 369main (int argc, char *argv[])
365{ 370{
371 const char *plugin_name;
372 char *cfg_name;
373
374 plugin_name = GNUNET_TESTING_get_testname_from_underscore (argv[0]);
375 GNUNET_asprintf (&cfg_name,
376 "test_namestore_api_%s.conf",
377 plugin_name);
366 res = 1; 378 res = 1;
367 if (0 != 379 if (0 !=
368 GNUNET_TESTING_peer_run ("test-namestore-api-monitoring", 380 GNUNET_TESTING_peer_run ("test-namestore-api-monitoring-existing",
369 "test_namestore_api.conf", 381 cfg_name,
370 &run, 382 &run,
371 NULL)) 383 NULL))
372 { 384 {
373 res = 1; 385 res = 1;
374 } 386 }
387 GNUNET_free (cfg_name);
375 if (NULL != directory) 388 if (NULL != directory)
376 { 389 {
377 GNUNET_DISK_directory_remove (directory); 390 GNUNET_DISK_directory_remove (directory);
@@ -381,4 +394,4 @@ main (int argc, char *argv[])
381} 394}
382 395
383 396
384/* end of test_namestore_api_monitoring.c */ 397/* end of test_namestore_api_monitoring_existing.c */
diff --git a/src/namestore/test_namestore_api_postgres.conf b/src/namestore/test_namestore_api_postgres.conf
new file mode 100644
index 000000000..397cb4b2f
--- /dev/null
+++ b/src/namestore/test_namestore_api_postgres.conf
@@ -0,0 +1,4 @@
1@INLINE@ test_namestore_api.conf
2
3[namestore]
4DATABASE = postgres
diff --git a/src/namestore/test_namestore_api_remove.c b/src/namestore/test_namestore_api_remove.c
index 2d670c1ee..532a751da 100644
--- a/src/namestore/test_namestore_api_remove.c
+++ b/src/namestore/test_namestore_api_remove.c
@@ -120,7 +120,8 @@ remove_cont (void *cls,
120 120
121 121
122static void 122static void
123put_cont (void *cls, int32_t success, 123put_cont (void *cls,
124 int32_t success,
124 const char *emsg) 125 const char *emsg)
125{ 126{
126 const char *name = cls; 127 const char *name = cls;
@@ -161,20 +162,27 @@ run (void *cls,
161 162
162 directory = NULL; 163 directory = NULL;
163 GNUNET_assert (GNUNET_OK == 164 GNUNET_assert (GNUNET_OK ==
164 GNUNET_CONFIGURATION_get_value_string(cfg, "PATHS", "GNUNET_TEST_HOME", &directory)); 165 GNUNET_CONFIGURATION_get_value_string(cfg,
166 "PATHS",
167 "GNUNET_TEST_HOME",
168 &directory));
165 GNUNET_DISK_directory_remove (directory); 169 GNUNET_DISK_directory_remove (directory);
166 170
167 endbadly_task = GNUNET_SCHEDULER_add_delayed (TIMEOUT, 171 endbadly_task = GNUNET_SCHEDULER_add_delayed (TIMEOUT,
168 &endbadly, NULL); 172 &endbadly,
173 NULL);
169 GNUNET_asprintf (&hostkey_file, 174 GNUNET_asprintf (&hostkey_file,
170 "zonefiles%s%s", 175 "zonefiles%s%s",
171 DIR_SEPARATOR_STR, 176 DIR_SEPARATOR_STR,
172 "N0UJMP015AFUNR2BTNM3FKPBLG38913BL8IDMCO2H0A1LIB81960.zkey"); 177 "N0UJMP015AFUNR2BTNM3FKPBLG38913BL8IDMCO2H0A1LIB81960.zkey");
173 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Using zonekey file `%s' \n", hostkey_file); 178 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
179 "Using zonekey file `%s' \n",
180 hostkey_file);
174 privkey = GNUNET_CRYPTO_ecdsa_key_create_from_file (hostkey_file); 181 privkey = GNUNET_CRYPTO_ecdsa_key_create_from_file (hostkey_file);
175 GNUNET_free (hostkey_file); 182 GNUNET_free (hostkey_file);
176 GNUNET_assert (privkey != NULL); 183 GNUNET_assert (privkey != NULL);
177 GNUNET_CRYPTO_ecdsa_key_get_public (privkey, &pubkey); 184 GNUNET_CRYPTO_ecdsa_key_get_public (privkey,
185 &pubkey);
178 186
179 removed = GNUNET_NO; 187 removed = GNUNET_NO;
180 188
@@ -201,15 +209,23 @@ run (void *cls,
201int 209int
202main (int argc, char *argv[]) 210main (int argc, char *argv[])
203{ 211{
212 const char *plugin_name;
213 char *cfg_name;
214
215 plugin_name = GNUNET_TESTING_get_testname_from_underscore (argv[0]);
216 GNUNET_asprintf (&cfg_name,
217 "test_namestore_api_%s.conf",
218 plugin_name);
204 res = 1; 219 res = 1;
205 if (0 != 220 if (0 !=
206 GNUNET_TESTING_peer_run ("test-namestore-api", 221 GNUNET_TESTING_peer_run ("test-namestore-api-remove",
207 "test_namestore_api.conf", 222 cfg_name,
208 &run, 223 &run,
209 NULL)) 224 NULL))
210 { 225 {
211 res = 1; 226 res = 1;
212 } 227 }
228 GNUNET_free (cfg_name);
213 if (NULL != directory) 229 if (NULL != directory)
214 { 230 {
215 GNUNET_DISK_directory_remove (directory); 231 GNUNET_DISK_directory_remove (directory);
diff --git a/src/namestore/test_namestore_api_remove_not_existing_record.c b/src/namestore/test_namestore_api_remove_not_existing_record.c
index ef199cdf6..2f20c3636 100644
--- a/src/namestore/test_namestore_api_remove_not_existing_record.c
+++ b/src/namestore/test_namestore_api_remove_not_existing_record.c
@@ -167,15 +167,23 @@ run (void *cls,
167int 167int
168main (int argc, char *argv[]) 168main (int argc, char *argv[])
169{ 169{
170 const char *plugin_name;
171 char *cfg_name;
172
173 plugin_name = GNUNET_TESTING_get_testname_from_underscore (argv[0]);
174 GNUNET_asprintf (&cfg_name,
175 "test_namestore_api_%s.conf",
176 plugin_name);
170 res = 1; 177 res = 1;
171 if (0 != 178 if (0 !=
172 GNUNET_TESTING_peer_run ("test-namestore-api", 179 GNUNET_TESTING_peer_run ("test-namestore-api-remove-non-existing-record",
173 "test_namestore_api.conf", 180 cfg_name,
174 &run, 181 &run,
175 NULL)) 182 NULL))
176 { 183 {
177 res = 1; 184 res = 1;
178 } 185 }
186 GNUNET_free (cfg_name);
179 if (NULL != directory) 187 if (NULL != directory)
180 { 188 {
181 GNUNET_DISK_directory_remove (directory); 189 GNUNET_DISK_directory_remove (directory);
diff --git a/src/namestore/test_namestore_api_sqlite.conf b/src/namestore/test_namestore_api_sqlite.conf
new file mode 100644
index 000000000..5416daec3
--- /dev/null
+++ b/src/namestore/test_namestore_api_sqlite.conf
@@ -0,0 +1,4 @@
1@INLINE@ test_namestore_api.conf
2
3[namestore]
4DATABASE = sqlite
diff --git a/src/namestore/test_namestore_api_store.c b/src/namestore/test_namestore_api_store.c
index d80676770..4e51678a1 100644
--- a/src/namestore/test_namestore_api_store.c
+++ b/src/namestore/test_namestore_api_store.c
@@ -159,15 +159,23 @@ run (void *cls,
159int 159int
160main (int argc, char *argv[]) 160main (int argc, char *argv[])
161{ 161{
162 const char *plugin_name;
163 char *cfg_name;
164
165 plugin_name = GNUNET_TESTING_get_testname_from_underscore (argv[0]);
166 GNUNET_asprintf (&cfg_name,
167 "test_namestore_api_%s.conf",
168 plugin_name);
162 res = 1; 169 res = 1;
163 if (0 != 170 if (0 !=
164 GNUNET_TESTING_peer_run ("test-namestore-api", 171 GNUNET_TESTING_peer_run ("test-namestore-api",
165 "test_namestore_api.conf", 172 cfg_name,
166 &run, 173 &run,
167 NULL)) 174 NULL))
168 { 175 {
169 res = 1; 176 res = 1;
170 } 177 }
178 GNUNET_free (cfg_name);
171 if (NULL != directory) 179 if (NULL != directory)
172 { 180 {
173 GNUNET_DISK_directory_remove (directory); 181 GNUNET_DISK_directory_remove (directory);
diff --git a/src/namestore/test_namestore_api_store_update.c b/src/namestore/test_namestore_api_store_update.c
index ed5a399bd..0a4551f21 100644
--- a/src/namestore/test_namestore_api_store_update.c
+++ b/src/namestore/test_namestore_api_store_update.c
@@ -274,15 +274,23 @@ run (void *cls,
274int 274int
275main (int argc, char *argv[]) 275main (int argc, char *argv[])
276{ 276{
277 const char *plugin_name;
278 char *cfg_name;
279
280 plugin_name = GNUNET_TESTING_get_testname_from_underscore (argv[0]);
281 GNUNET_asprintf (&cfg_name,
282 "test_namestore_api_%s.conf",
283 plugin_name);
277 res = 1; 284 res = 1;
278 if (0 != 285 if (0 !=
279 GNUNET_TESTING_peer_run ("test-namestore-api-store-update", 286 GNUNET_TESTING_peer_run ("test-namestore-api-store-update",
280 "test_namestore_api.conf", 287 cfg_name,
281 &run, 288 &run,
282 NULL)) 289 NULL))
283 { 290 {
284 res = 1; 291 res = 1;
285 } 292 }
293 GNUNET_free (cfg_name);
286 if (NULL != directory) 294 if (NULL != directory)
287 { 295 {
288 GNUNET_DISK_directory_remove (directory); 296 GNUNET_DISK_directory_remove (directory);
diff --git a/src/namestore/test_namestore_api_zone_iteration.c b/src/namestore/test_namestore_api_zone_iteration.c
index 8960be55d..806605d94 100644
--- a/src/namestore/test_namestore_api_zone_iteration.c
+++ b/src/namestore/test_namestore_api_zone_iteration.c
@@ -276,7 +276,8 @@ zone_proc (void *cls,
276 returned_records ++; 276 returned_records ++;
277 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 277 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
278 "Telling namestore to send the next result\n"); 278 "Telling namestore to send the next result\n");
279 GNUNET_NAMESTORE_zone_iterator_next (zi); 279 GNUNET_NAMESTORE_zone_iterator_next (zi,
280 1);
280 } 281 }
281 else 282 else
282 { 283 {
@@ -486,15 +487,23 @@ run (void *cls,
486int 487int
487main (int argc, char *argv[]) 488main (int argc, char *argv[])
488{ 489{
490 const char *plugin_name;
491 char *cfg_name;
492
493 plugin_name = GNUNET_TESTING_get_testname_from_underscore (argv[0]);
494 GNUNET_asprintf (&cfg_name,
495 "test_namestore_api_%s.conf",
496 plugin_name);
489 res = 1; 497 res = 1;
490 if (0 != 498 if (0 !=
491 GNUNET_TESTING_peer_run ("test-namestore-api-zone-iteration", 499 GNUNET_TESTING_peer_run ("test-namestore-api-zone-iteration",
492 "test_namestore_api.conf", 500 cfg_name,
493 &run, 501 &run,
494 NULL)) 502 NULL))
495 { 503 {
496 res = 1; 504 res = 1;
497 } 505 }
506 GNUNET_free (cfg_name);
498 if (NULL != directory) 507 if (NULL != directory)
499 { 508 {
500 GNUNET_DISK_directory_remove (directory); 509 GNUNET_DISK_directory_remove (directory);
diff --git a/src/namestore/test_namestore_api_zone_iteration_nick.c b/src/namestore/test_namestore_api_zone_iteration_nick.c
index 791702f97..a88646864 100644
--- a/src/namestore/test_namestore_api_zone_iteration_nick.c
+++ b/src/namestore/test_namestore_api_zone_iteration_nick.c
@@ -245,7 +245,8 @@ zone_proc (void *cls,
245 returned_records ++; 245 returned_records ++;
246 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 246 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
247 "Telling namestore to send the next result\n"); 247 "Telling namestore to send the next result\n");
248 GNUNET_NAMESTORE_zone_iterator_next (zi); 248 GNUNET_NAMESTORE_zone_iterator_next (zi,
249 1);
249 } 250 }
250 else 251 else
251 { 252 {
@@ -489,15 +490,23 @@ run (void *cls,
489int 490int
490main (int argc, char *argv[]) 491main (int argc, char *argv[])
491{ 492{
493 const char *plugin_name;
494 char *cfg_name;
495
496 plugin_name = GNUNET_TESTING_get_testname_from_underscore (argv[0]);
497 GNUNET_asprintf (&cfg_name,
498 "test_namestore_api_%s.conf",
499 plugin_name);
492 res = 1; 500 res = 1;
493 if (0 != 501 if (0 !=
494 GNUNET_TESTING_peer_run ("test-namestore-api-zone-iteration", 502 GNUNET_TESTING_peer_run ("test-namestore-api-zone-iteration-nick",
495 "test_namestore_api.conf", 503 cfg_name,
496 &run, 504 &run,
497 NULL)) 505 NULL))
498 { 506 {
499 res = 1; 507 res = 1;
500 } 508 }
509 GNUNET_free (cfg_name);
501 if (NULL != directory) 510 if (NULL != directory)
502 { 511 {
503 GNUNET_DISK_directory_remove (directory); 512 GNUNET_DISK_directory_remove (directory);
diff --git a/src/namestore/test_namestore_api_zone_iteration_specific_zone.c b/src/namestore/test_namestore_api_zone_iteration_specific_zone.c
index c5ae927b0..a4fb320e9 100644
--- a/src/namestore/test_namestore_api_zone_iteration_specific_zone.c
+++ b/src/namestore/test_namestore_api_zone_iteration_specific_zone.c
@@ -238,7 +238,8 @@ zone_proc (void *cls,
238 returned_records ++; 238 returned_records ++;
239 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 239 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
240 "Telling namestore to send the next result\n"); 240 "Telling namestore to send the next result\n");
241 GNUNET_NAMESTORE_zone_iterator_next (zi); 241 GNUNET_NAMESTORE_zone_iterator_next (zi,
242 1);
242 } 243 }
243 else 244 else
244 { 245 {
@@ -484,15 +485,23 @@ run (void *cls,
484int 485int
485main (int argc, char *argv[]) 486main (int argc, char *argv[])
486{ 487{
488 const char *plugin_name;
489 char *cfg_name;
490
491 plugin_name = GNUNET_TESTING_get_testname_from_underscore (argv[0]);
492 GNUNET_asprintf (&cfg_name,
493 "test_namestore_api_%s.conf",
494 plugin_name);
487 res = 1; 495 res = 1;
488 if (0 != 496 if (0 !=
489 GNUNET_TESTING_peer_run ("test-namestore-api-zone-iteration", 497 GNUNET_TESTING_peer_run ("test-namestore-api-zone-iteration-specific-zone",
490 "test_namestore_api.conf", 498 cfg_name,
491 &run, 499 &run,
492 NULL)) 500 NULL))
493 { 501 {
494 res = 1; 502 res = 1;
495 } 503 }
504 GNUNET_free (cfg_name);
496 if (NULL != directory) 505 if (NULL != directory)
497 { 506 {
498 GNUNET_DISK_directory_remove (directory); 507 GNUNET_DISK_directory_remove (directory);
diff --git a/src/namestore/test_namestore_api_zone_iteration_stop.c b/src/namestore/test_namestore_api_zone_iteration_stop.c
index a5f040150..c7be5fead 100644
--- a/src/namestore/test_namestore_api_zone_iteration_stop.c
+++ b/src/namestore/test_namestore_api_zone_iteration_stop.c
@@ -271,7 +271,8 @@ zone_proc (void *cls,
271 returned_records ++; 271 returned_records ++;
272 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 272 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
273 "Telling namestore to send the next result\n"); 273 "Telling namestore to send the next result\n");
274 GNUNET_NAMESTORE_zone_iterator_next (zi); 274 GNUNET_NAMESTORE_zone_iterator_next (zi,
275 1);
275 } 276 }
276 else 277 else
277 { 278 {
@@ -507,15 +508,23 @@ run (void *cls,
507int 508int
508main (int argc, char *argv[]) 509main (int argc, char *argv[])
509{ 510{
511 const char *plugin_name;
512 char *cfg_name;
513
514 plugin_name = GNUNET_TESTING_get_testname_from_underscore (argv[0]);
515 GNUNET_asprintf (&cfg_name,
516 "test_namestore_api_%s.conf",
517 plugin_name);
510 res = 1; 518 res = 1;
511 if (0 != 519 if (0 !=
512 GNUNET_TESTING_peer_run ("test-namestore-api-zone-iteration-stop", 520 GNUNET_TESTING_peer_run ("test-namestore-api-zone-iteration-stop",
513 "test_namestore_api.conf", 521 cfg_name,
514 &run, 522 &run,
515 NULL)) 523 NULL))
516 { 524 {
517 res = 1; 525 res = 1;
518 } 526 }
527 GNUNET_free (cfg_name);
519 if (NULL != directory) 528 if (NULL != directory)
520 { 529 {
521 GNUNET_DISK_directory_remove (directory); 530 GNUNET_DISK_directory_remove (directory);
diff --git a/src/namestore/test_namestore_api_zone_to_name.c b/src/namestore/test_namestore_api_zone_to_name.c
index 2bc7d34a3..5b088d90b 100644
--- a/src/namestore/test_namestore_api_zone_to_name.c
+++ b/src/namestore/test_namestore_api_zone_to_name.c
@@ -221,7 +221,7 @@ run (void *cls,
221 /* load privat key */ 221 /* load privat key */
222 { 222 {
223 char *zonekey_file; 223 char *zonekey_file;
224 224
225 GNUNET_asprintf (&zonekey_file, 225 GNUNET_asprintf (&zonekey_file,
226 "zonefiles%s%s", 226 "zonefiles%s%s",
227 DIR_SEPARATOR_STR, 227 DIR_SEPARATOR_STR,
@@ -242,13 +242,13 @@ run (void *cls,
242 sizeof (s_zone_value)); 242 sizeof (s_zone_value));
243 { 243 {
244 struct GNUNET_GNSRECORD_Data rd; 244 struct GNUNET_GNSRECORD_Data rd;
245 245
246 rd.expiration_time = GNUNET_TIME_absolute_get().abs_value_us; 246 rd.expiration_time = GNUNET_TIME_absolute_get().abs_value_us;
247 rd.record_type = GNUNET_GNSRECORD_TYPE_PKEY; 247 rd.record_type = GNUNET_GNSRECORD_TYPE_PKEY;
248 rd.data_size = sizeof (s_zone_value); 248 rd.data_size = sizeof (s_zone_value);
249 rd.data = &s_zone_value; 249 rd.data = &s_zone_value;
250 rd.flags = 0; 250 rd.flags = 0;
251 251
252 nsh = GNUNET_NAMESTORE_connect (cfg); 252 nsh = GNUNET_NAMESTORE_connect (cfg);
253 GNUNET_break (NULL != nsh); 253 GNUNET_break (NULL != nsh);
254 GNUNET_NAMESTORE_records_store (nsh, 254 GNUNET_NAMESTORE_records_store (nsh,
@@ -266,17 +266,24 @@ int
266main (int argc, 266main (int argc,
267 char *argv[]) 267 char *argv[])
268{ 268{
269 const char *plugin_name;
270 char *cfg_name;
271
269 (void) argc; 272 (void) argc;
270 (void) argv; 273 plugin_name = GNUNET_TESTING_get_testname_from_underscore (argv[0]);
274 GNUNET_asprintf (&cfg_name,
275 "test_namestore_api_%s.conf",
276 plugin_name);
271 res = 1; 277 res = 1;
272 if (0 != 278 if (0 !=
273 GNUNET_TESTING_peer_run ("test-namestore-api-zone-to-name", 279 GNUNET_TESTING_peer_run ("test-namestore-api-zone-to-name",
274 "test_namestore_api.conf", 280 cfg_name,
275 &run, 281 &run,
276 NULL)) 282 NULL))
277 { 283 {
278 res = 1; 284 res = 1;
279 } 285 }
286 GNUNET_free (cfg_name);
280 if (NULL != directory) 287 if (NULL != directory)
281 { 288 {
282 GNUNET_DISK_directory_remove (directory); 289 GNUNET_DISK_directory_remove (directory);
diff --git a/src/namestore/test_plugin_namestore.c b/src/namestore/test_plugin_namestore.c
index aa5d10809..6bccd1706 100644
--- a/src/namestore/test_plugin_namestore.c
+++ b/src/namestore/test_plugin_namestore.c
@@ -47,8 +47,12 @@ unload_plugin (struct GNUNET_NAMESTORE_PluginFunctions *api)
47{ 47{
48 char *libname; 48 char *libname;
49 49
50 GNUNET_asprintf (&libname, "libgnunet_plugin_namestore_%s", plugin_name); 50 GNUNET_asprintf (&libname,
51 GNUNET_break (NULL == GNUNET_PLUGIN_unload (libname, api)); 51 "libgnunet_plugin_namestore_%s",
52 plugin_name);
53 GNUNET_break (NULL ==
54 GNUNET_PLUGIN_unload (libname,
55 api));
52 GNUNET_free (libname); 56 GNUNET_free (libname);
53} 57}
54 58
@@ -65,12 +69,17 @@ load_plugin (const struct GNUNET_CONFIGURATION_Handle *cfg)
65 struct GNUNET_NAMESTORE_PluginFunctions *ret; 69 struct GNUNET_NAMESTORE_PluginFunctions *ret;
66 char *libname; 70 char *libname;
67 71
68 GNUNET_log (GNUNET_ERROR_TYPE_INFO, _("Loading `%s' namestore plugin\n"), 72 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
73 _("Loading `%s' namestore plugin\n"),
69 plugin_name); 74 plugin_name);
70 GNUNET_asprintf (&libname, "libgnunet_plugin_namestore_%s", plugin_name); 75 GNUNET_asprintf (&libname,
76 "libgnunet_plugin_namestore_%s",
77 plugin_name);
71 if (NULL == (ret = GNUNET_PLUGIN_load (libname, (void*) cfg))) 78 if (NULL == (ret = GNUNET_PLUGIN_load (libname, (void*) cfg)))
72 { 79 {
73 FPRINTF (stderr, "Failed to load plugin `%s'!\n", plugin_name); 80 FPRINTF (stderr,
81 "Failed to load plugin `%s'!\n",
82 plugin_name);
74 GNUNET_free (libname); 83 GNUNET_free (libname);
75 return NULL; 84 return NULL;
76 } 85 }
@@ -81,54 +90,66 @@ load_plugin (const struct GNUNET_CONFIGURATION_Handle *cfg)
81 90
82static void 91static void
83test_record (void *cls, 92test_record (void *cls,
84 const struct GNUNET_CRYPTO_EcdsaPrivateKey *private_key, 93 uint64_t seq,
85 const char *label, 94 const struct GNUNET_CRYPTO_EcdsaPrivateKey *private_key,
86 unsigned int rd_count, 95 const char *label,
87 const struct GNUNET_GNSRECORD_Data *rd) 96 unsigned int rd_count,
97 const struct GNUNET_GNSRECORD_Data *rd)
88{ 98{
89 int *idp = cls; 99 int *idp = cls;
90 int id = *idp; 100 int id = *idp;
91 struct GNUNET_CRYPTO_EcdsaPrivateKey tzone_private_key; 101 struct GNUNET_CRYPTO_EcdsaPrivateKey tzone_private_key;
92 char tname[64]; 102 char tname[64];
93 unsigned int trd_count = 1 + (id % 1024); 103 unsigned int trd_count = 1 + (id % 1024);
94 unsigned int i;
95 104
96 GNUNET_snprintf (tname, sizeof (tname), 105 GNUNET_snprintf (tname,
97 "a%u", (unsigned int ) id); 106 sizeof (tname),
98 for (i=0;i<trd_count;i++) 107 "a%u",
108 (unsigned int ) id);
109 for (unsigned int i=0;i<trd_count;i++)
99 { 110 {
100 GNUNET_assert (rd[i].data_size == id % 10); 111 GNUNET_assert (rd[i].data_size == id % 10);
101 GNUNET_assert (0 == memcmp ("Hello World", rd[i].data, id % 10)); 112 GNUNET_assert (0 == memcmp ("Hello World", rd[i].data, id % 10));
102 GNUNET_assert (rd[i].record_type == 1 + (id % 13)); 113 GNUNET_assert (rd[i].record_type == 1 + (id % 13));
103 GNUNET_assert (rd[i].flags == 0); 114 GNUNET_assert (rd[i].flags == 0);
104 } 115 }
105 memset (&tzone_private_key, (id % 241), sizeof (tzone_private_key)); 116 memset (&tzone_private_key,
117 (id % 241),
118 sizeof (tzone_private_key));
106 GNUNET_assert (0 == strcmp (label, tname)); 119 GNUNET_assert (0 == strcmp (label, tname));
107 GNUNET_assert (0 == memcmp (&tzone_private_key, private_key, sizeof (struct GNUNET_CRYPTO_EcdsaPrivateKey))); 120 GNUNET_assert (0 == memcmp (&tzone_private_key,
121 private_key,
122 sizeof (struct GNUNET_CRYPTO_EcdsaPrivateKey)));
108} 123}
109 124
110 125
111static void 126static void
112get_record (struct GNUNET_NAMESTORE_PluginFunctions *nsp, int id) 127get_record (struct GNUNET_NAMESTORE_PluginFunctions *nsp,
128 int id)
113{ 129{
114 GNUNET_assert (GNUNET_OK == nsp->iterate_records (nsp->cls, 130 GNUNET_assert (GNUNET_OK ==
115 NULL, 0, &test_record, &id)); 131 nsp->iterate_records (nsp->cls,
132 NULL,
133 0,
134 1,
135 &test_record,
136 &id));
116} 137}
117 138
118 139
119static void 140static void
120put_record (struct GNUNET_NAMESTORE_PluginFunctions *nsp, int id) 141put_record (struct GNUNET_NAMESTORE_PluginFunctions *nsp,
142 int id)
121{ 143{
122 struct GNUNET_CRYPTO_EcdsaPrivateKey zone_private_key; 144 struct GNUNET_CRYPTO_EcdsaPrivateKey zone_private_key;
123 char label[64]; 145 char label[64];
124 unsigned int rd_count = 1 + (id % 1024); 146 unsigned int rd_count = 1 + (id % 1024);
125 struct GNUNET_GNSRECORD_Data rd[rd_count]; 147 struct GNUNET_GNSRECORD_Data rd[rd_count];
126 struct GNUNET_CRYPTO_EcdsaSignature signature; 148 struct GNUNET_CRYPTO_EcdsaSignature signature;
127 unsigned int i;
128 149
129 GNUNET_snprintf (label, sizeof (label), 150 GNUNET_snprintf (label, sizeof (label),
130 "a%u", (unsigned int ) id); 151 "a%u", (unsigned int ) id);
131 for (i=0;i<rd_count;i++) 152 for (unsigned int i=0;i<rd_count;i++)
132 { 153 {
133 rd[i].data = "Hello World"; 154 rd[i].data = "Hello World";
134 rd[i].data_size = id % 10; 155 rd[i].data_size = id % 10;
@@ -138,16 +159,19 @@ put_record (struct GNUNET_NAMESTORE_PluginFunctions *nsp, int id)
138 } 159 }
139 memset (&zone_private_key, (id % 241), sizeof (zone_private_key)); 160 memset (&zone_private_key, (id % 241), sizeof (zone_private_key));
140 memset (&signature, (id % 243), sizeof (signature)); 161 memset (&signature, (id % 243), sizeof (signature));
141 GNUNET_assert (GNUNET_OK == nsp->store_records (nsp->cls, 162 GNUNET_assert (GNUNET_OK ==
142 &zone_private_key, 163 nsp->store_records (nsp->cls,
143 label, 164 &zone_private_key,
144 rd_count, 165 label,
145 rd)); 166 rd_count,
167 rd));
146} 168}
147 169
148 170
149static void 171static void
150run (void *cls, char *const *args, const char *cfgfile, 172run (void *cls,
173 char *const *args,
174 const char *cfgfile,
151 const struct GNUNET_CONFIGURATION_Handle *cfg) 175 const struct GNUNET_CONFIGURATION_Handle *cfg)
152{ 176{
153 struct GNUNET_NAMESTORE_PluginFunctions *nsp; 177 struct GNUNET_NAMESTORE_PluginFunctions *nsp;
@@ -169,7 +193,8 @@ run (void *cls, char *const *args, const char *cfgfile,
169 193
170 194
171int 195int
172main (int argc, char *argv[]) 196main (int argc,
197 char *argv[])
173{ 198{
174 char cfg_name[128]; 199 char cfg_name[128];
175 char *const xargv[] = { 200 char *const xargv[] = {
@@ -182,18 +207,25 @@ main (int argc, char *argv[])
182 GNUNET_GETOPT_OPTION_END 207 GNUNET_GETOPT_OPTION_END
183 }; 208 };
184 209
185 //GNUNET_DISK_directory_remove ("/tmp/gnunet-test-plugin-namestore-sqlite");
186 GNUNET_log_setup ("test-plugin-namestore", 210 GNUNET_log_setup ("test-plugin-namestore",
187 "WARNING", 211 "WARNING",
188 NULL); 212 NULL);
189 plugin_name = GNUNET_TESTING_get_testname_from_underscore (argv[0]); 213 plugin_name = GNUNET_TESTING_get_testname_from_underscore (argv[0]);
190 GNUNET_snprintf (cfg_name, sizeof (cfg_name), "test_plugin_namestore_%s.conf", 214 GNUNET_snprintf (cfg_name,
215 sizeof (cfg_name),
216 "test_plugin_namestore_%s.conf",
191 plugin_name); 217 plugin_name);
192 GNUNET_PROGRAM_run ((sizeof (xargv) / sizeof (char *)) - 1, xargv, 218 GNUNET_PROGRAM_run ((sizeof (xargv) / sizeof (char *)) - 1,
193 "test-plugin-namestore", "nohelp", options, &run, NULL); 219 xargv,
220 "test-plugin-namestore",
221 "nohelp",
222 options,
223 &run,
224 NULL);
194 if (ok != 0) 225 if (ok != 0)
195 FPRINTF (stderr, "Missed some testcases: %d\n", ok); 226 FPRINTF (stderr,
196 //GNUNET_DISK_directory_remove ("/tmp/gnunet-test-plugin-namestore-sqlite"); 227 "Missed some testcases: %d\n",
228 ok);
197 return ok; 229 return ok;
198} 230}
199 231
diff --git a/src/util/container_multihashmap.c b/src/util/container_multihashmap.c
index ffeb4a71f..6001fc1df 100644
--- a/src/util/container_multihashmap.c
+++ b/src/util/container_multihashmap.c
@@ -175,14 +175,36 @@ struct GNUNET_CONTAINER_MultiHashMap *
175GNUNET_CONTAINER_multihashmap_create (unsigned int len, 175GNUNET_CONTAINER_multihashmap_create (unsigned int len,
176 int do_not_copy_keys) 176 int do_not_copy_keys)
177{ 177{
178 struct GNUNET_CONTAINER_MultiHashMap *map; 178 struct GNUNET_CONTAINER_MultiHashMap *hm;
179 179
180 GNUNET_assert (len > 0); 180 GNUNET_assert (len > 0);
181 map = GNUNET_new (struct GNUNET_CONTAINER_MultiHashMap); 181 hm = GNUNET_new (struct GNUNET_CONTAINER_MultiHashMap);
182 map->map = GNUNET_malloc (len * sizeof (union MapEntry)); 182 if (len * sizeof (union MapEntry) > GNUNET_MAX_MALLOC_CHECKED)
183 map->map_length = len; 183 {
184 map->use_small_entries = do_not_copy_keys; 184 size_t s;
185 return map; 185 /* application *explicitly* requested very large map, hopefully
186 it checks the return value... */
187 s = len * sizeof (union MapEntry);
188 if ( (s / sizeof (union MapEntry)) != len)
189 return NULL; /* integer overflow on multiplication */
190 if (NULL == (hm->map = GNUNET_malloc_large (s)))
191 {
192 /* out of memory */
193 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
194 "Out of memory allocating large hash map (%u entries)\n",
195 len);
196 GNUNET_free (hm);
197 return NULL;
198 }
199 }
200 else
201 {
202 hm->map = GNUNET_new_array (len,
203 union MapEntry);
204 }
205 hm->map_length = len;
206 hm->use_small_entries = do_not_copy_keys;
207 return hm;
186} 208}
187 209
188 210
@@ -196,11 +218,10 @@ void
196GNUNET_CONTAINER_multihashmap_destroy (struct GNUNET_CONTAINER_MultiHashMap 218GNUNET_CONTAINER_multihashmap_destroy (struct GNUNET_CONTAINER_MultiHashMap
197 *map) 219 *map)
198{ 220{
199 unsigned int i; 221 for (unsigned int i = 0; i < map->map_length; i++)
200 union MapEntry me;
201
202 for (i = 0; i < map->map_length; i++)
203 { 222 {
223 union MapEntry me;
224
204 me = map->map[i]; 225 me = map->map[i];
205 if (map->use_small_entries) 226 if (map->use_small_entries)
206 { 227 {
@@ -257,8 +278,7 @@ idx_of (const struct GNUNET_CONTAINER_MultiHashMap *map,
257 * @return the number of key value pairs 278 * @return the number of key value pairs
258 */ 279 */
259unsigned int 280unsigned int
260GNUNET_CONTAINER_multihashmap_size (const struct GNUNET_CONTAINER_MultiHashMap 281GNUNET_CONTAINER_multihashmap_size (const struct GNUNET_CONTAINER_MultiHashMap *map)
261 *map)
262{ 282{
263 return map->size; 283 return map->size;
264} 284}
@@ -656,17 +676,22 @@ grow (struct GNUNET_CONTAINER_MultiHashMap *map)
656 unsigned int old_len; 676 unsigned int old_len;
657 unsigned int new_len; 677 unsigned int new_len;
658 unsigned int idx; 678 unsigned int idx;
659 unsigned int i;
660 679
661 map->modification_counter++; 680 map->modification_counter++;
662 681
663 old_map = map->map; 682 old_map = map->map;
664 old_len = map->map_length; 683 old_len = map->map_length;
665 new_len = old_len * 2; 684 new_len = old_len * 2;
666 new_map = GNUNET_malloc (sizeof (union MapEntry) * new_len); 685 /* if we would exceed heap size limit for the _first_ time,
686 try staying just below the limit */
687 if ( (new_len * sizeof (union MapEntry) > GNUNET_MAX_MALLOC_CHECKED) &&
688 ((old_len+1) * sizeof (union MapEntry) < GNUNET_MAX_MALLOC_CHECKED) )
689 new_len = GNUNET_MAX_MALLOC_CHECKED / sizeof (union MapEntry);
690 new_map = GNUNET_new_array (new_len,
691 union MapEntry);
667 map->map_length = new_len; 692 map->map_length = new_len;
668 map->map = new_map; 693 map->map = new_map;
669 for (i = 0; i < old_len; i++) 694 for (unsigned int i = 0; i < old_len; i++)
670 { 695 {
671 if (map->use_small_entries) 696 if (map->use_small_entries)
672 { 697 {
diff --git a/src/util/mq.c b/src/util/mq.c
index af700836c..dbcce704d 100644
--- a/src/util/mq.c
+++ b/src/util/mq.c
@@ -578,11 +578,9 @@ void
578GNUNET_MQ_set_handlers_closure (struct GNUNET_MQ_Handle *mq, 578GNUNET_MQ_set_handlers_closure (struct GNUNET_MQ_Handle *mq,
579 void *handlers_cls) 579 void *handlers_cls)
580{ 580{
581 unsigned int i;
582
583 if (NULL == mq->handlers) 581 if (NULL == mq->handlers)
584 return; 582 return;
585 for (i=0;NULL != mq->handlers[i].cb; i++) 583 for (unsigned int i=0;NULL != mq->handlers[i].cb; i++)
586 mq->handlers[i].cls = handlers_cls; 584 mq->handlers[i].cls = handlers_cls;
587} 585}
588 586
@@ -782,7 +780,9 @@ GNUNET_MQ_notify_sent (struct GNUNET_MQ_Envelope *ev,
782 GNUNET_SCHEDULER_TaskCallback cb, 780 GNUNET_SCHEDULER_TaskCallback cb,
783 void *cb_cls) 781 void *cb_cls)
784{ 782{
785 GNUNET_assert (NULL == ev->sent_cb); 783 /* allow setting *OR* clearing callback */
784 GNUNET_assert ( (NULL == ev->sent_cb) ||
785 (NULL == cb) );
786 ev->sent_cb = cb; 786 ev->sent_cb = cb;
787 ev->sent_cls = cb_cls; 787 ev->sent_cls = cb_cls;
788} 788}
diff --git a/src/util/resolver_api.c b/src/util/resolver_api.c
index b92fd6a02..68f02587a 100644
--- a/src/util/resolver_api.c
+++ b/src/util/resolver_api.c
@@ -478,7 +478,15 @@ handle_response (void *cls,
478 char *nret; 478 char *nret;
479 479
480 (void) cls; 480 (void) cls;
481 GNUNET_assert (NULL != rh); 481 if (NULL == rh)
482 {
483 /* Resolver service sent extra replies to query (after terminator)? Bad! */
484 GNUNET_break (0);
485 GNUNET_MQ_destroy (mq);
486 mq = NULL;
487 reconnect ();
488 return;
489 }
482 size = ntohs (msg->size); 490 size = ntohs (msg->size);
483 if (size == sizeof (struct GNUNET_MessageHeader)) 491 if (size == sizeof (struct GNUNET_MessageHeader))
484 { 492 {
diff --git a/src/zonemaster/gnunet-service-zonemaster.c b/src/zonemaster/gnunet-service-zonemaster.c
index b3a2cb8b3..7129cf44f 100644
--- a/src/zonemaster/gnunet-service-zonemaster.c
+++ b/src/zonemaster/gnunet-service-zonemaster.c
@@ -36,20 +36,38 @@
36#define LOG_STRERROR_FILE(kind,syscall,filename) GNUNET_log_from_strerror_file (kind, "util", syscall, filename) 36#define LOG_STRERROR_FILE(kind,syscall,filename) GNUNET_log_from_strerror_file (kind, "util", syscall, filename)
37 37
38 38
39/**
40 * How often should we (re)publish each record before
41 * it expires?
42 */
43#define PUBLISH_OPS_PER_EXPIRATION 4
39 44
40/** 45/**
41 * The initial interval in milliseconds btween puts in 46 * How often do we measure the delta between desired zone
42 * a zone iteration 47 * iteration speed and actual speed, and tell statistics
48 * service about it?
43 */ 49 */
44#define INITIAL_PUT_INTERVAL GNUNET_TIME_UNIT_MILLISECONDS 50#define DELTA_INTERVAL 100
45 51
46/** 52/**
47 * The lower bound for the zone iteration interval 53 * How many records do we fetch in one shot from the namestore?
48 */ 54 */
49#define MINIMUM_ZONE_ITERATION_INTERVAL GNUNET_TIME_UNIT_SECONDS 55#define NS_BLOCK_SIZE 1000
56
57/**
58 * How many pending DHT operations do we allow at most?
59 */
60#define DHT_QUEUE_LIMIT 2000
61
62/**
63 * The initial interval in milliseconds btween puts in
64 * a zone iteration
65 */
66#define INITIAL_ZONE_ITERATION_INTERVAL GNUNET_TIME_UNIT_MILLISECONDS
50 67
51/** 68/**
52 * The upper bound for the zone iteration interval 69 * The upper bound for the zone iteration interval
70 * (per record).
53 */ 71 */
54#define MAXIMUM_ZONE_ITERATION_INTERVAL GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15) 72#define MAXIMUM_ZONE_ITERATION_INTERVAL GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15)
55 73
@@ -79,22 +97,27 @@
79/** 97/**
80 * Handle for DHT PUT activity triggered from the namestore monitor. 98 * Handle for DHT PUT activity triggered from the namestore monitor.
81 */ 99 */
82struct MonitorActivity 100struct DhtPutActivity
83{ 101{
84 /** 102 /**
85 * Kept in a DLL. 103 * Kept in a DLL.
86 */ 104 */
87 struct MonitorActivity *next; 105 struct DhtPutActivity *next;
88 106
89 /** 107 /**
90 * Kept in a DLL. 108 * Kept in a DLL.
91 */ 109 */
92 struct MonitorActivity *prev; 110 struct DhtPutActivity *prev;
93 111
94 /** 112 /**
95 * Handle for the DHT PUT operation. 113 * Handle for the DHT PUT operation.
96 */ 114 */
97 struct GNUNET_DHT_PutHandle *ph; 115 struct GNUNET_DHT_PutHandle *ph;
116
117 /**
118 * When was this PUT initiated?
119 */
120 struct GNUNET_TIME_Absolute start_date;
98}; 121};
99 122
100 123
@@ -131,12 +154,32 @@ static struct GNUNET_NAMESTORE_ZoneMonitor *zmon;
131/** 154/**
132 * Head of monitor activities; kept in a DLL. 155 * Head of monitor activities; kept in a DLL.
133 */ 156 */
134static struct MonitorActivity *ma_head; 157static struct DhtPutActivity *ma_head;
135 158
136/** 159/**
137 * Tail of monitor activities; kept in a DLL. 160 * Tail of monitor activities; kept in a DLL.
138 */ 161 */
139static struct MonitorActivity *ma_tail; 162static struct DhtPutActivity *ma_tail;
163
164/**
165 * Head of iteration put activities; kept in a DLL.
166 */
167static struct DhtPutActivity *it_head;
168
169/**
170 * Tail of iteration put activities; kept in a DLL.
171 */
172static struct DhtPutActivity *it_tail;
173
174/**
175 * Number of entries in the DHT queue #it_head.
176 */
177static unsigned int dht_queue_length;
178
179/**
180 * Number of entries in the DHT queue #ma_head.
181 */
182static unsigned int ma_queue_length;
140 183
141/** 184/**
142 * Useful for zone update for DHT put 185 * Useful for zone update for DHT put
@@ -149,15 +192,31 @@ static unsigned long long num_public_records;
149static unsigned long long last_num_public_records; 192static unsigned long long last_num_public_records;
150 193
151/** 194/**
195 * Number of successful put operations performed in the current
196 * measurement cycle (as measured in #check_zone_namestore_next()).
197 */
198static unsigned long long put_cnt;
199
200/**
201 * What is the frequency at which we currently would like
202 * to perform DHT puts (per record)? Calculated in
203 * update_velocity() from the #zone_publish_time_window()
204 * and the total number of record sets we have (so far)
205 * observed in the zone.
206 */
207static struct GNUNET_TIME_Relative target_iteration_velocity_per_record;
208
209/**
152 * Minimum relative expiration time of records seem during the current 210 * Minimum relative expiration time of records seem during the current
153 * zone iteration. 211 * zone iteration.
154 */ 212 */
155static struct GNUNET_TIME_Relative min_relative_record_time; 213static struct GNUNET_TIME_Relative min_relative_record_time;
156 214
157/** 215/**
158 * Zone iteration PUT interval. 216 * Minimum relative expiration time of records seem during the last
217 * zone iteration.
159 */ 218 */
160static struct GNUNET_TIME_Relative put_interval; 219static struct GNUNET_TIME_Relative last_min_relative_record_time;
161 220
162/** 221/**
163 * Default time window for zone iteration 222 * Default time window for zone iteration
@@ -171,16 +230,43 @@ static struct GNUNET_TIME_Relative zone_publish_time_window_default;
171static struct GNUNET_TIME_Relative zone_publish_time_window; 230static struct GNUNET_TIME_Relative zone_publish_time_window;
172 231
173/** 232/**
233 * When did we last start measuring the #DELTA_INTERVAL successful
234 * DHT puts? Used for velocity calculations.
235 */
236static struct GNUNET_TIME_Absolute last_put_100;
237
238/**
239 * By how much should we try to increase our per-record iteration speed
240 * (over the desired speed calculated directly from the #put_interval)?
241 * Basically this value corresponds to the per-record CPU time overhead
242 * we have.
243 */
244static struct GNUNET_TIME_Relative sub_delta;
245
246/**
174 * zone publish task 247 * zone publish task
175 */ 248 */
176static struct GNUNET_SCHEDULER_Task *zone_publish_task; 249static struct GNUNET_SCHEDULER_Task *zone_publish_task;
177 250
178/** 251/**
252 * How many more values are left for the current query before we need
253 * to explicitly ask the namestore for more?
254 */
255static unsigned int ns_iteration_left;
256
257/**
179 * #GNUNET_YES if zone has never been published before 258 * #GNUNET_YES if zone has never been published before
180 */ 259 */
181static int first_zone_iteration; 260static int first_zone_iteration;
182 261
183/** 262/**
263 * Optimize block insertion by caching map of private keys to
264 * public keys in memory?
265 */
266static int cache_keys;
267
268
269/**
184 * Task run during shutdown. 270 * Task run during shutdown.
185 * 271 *
186 * @param cls unused 272 * @param cls unused
@@ -189,8 +275,9 @@ static int first_zone_iteration;
189static void 275static void
190shutdown_task (void *cls) 276shutdown_task (void *cls)
191{ 277{
192 struct MonitorActivity *ma; 278 struct DhtPutActivity *ma;
193 279
280 (void) cls;
194 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 281 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
195 "Shutting down!\n"); 282 "Shutting down!\n");
196 while (NULL != (ma = ma_head)) 283 while (NULL != (ma = ma_head))
@@ -201,6 +288,15 @@ shutdown_task (void *cls)
201 ma); 288 ma);
202 GNUNET_free (ma); 289 GNUNET_free (ma);
203 } 290 }
291 while (NULL != (ma = it_head))
292 {
293 GNUNET_DHT_put_cancel (ma->ph);
294 GNUNET_CONTAINER_DLL_remove (it_head,
295 it_tail,
296 ma);
297 dht_queue_length--;
298 GNUNET_free (ma);
299 }
204 if (NULL != statistics) 300 if (NULL != statistics)
205 { 301 {
206 GNUNET_STATISTICS_destroy (statistics, 302 GNUNET_STATISTICS_destroy (statistics,
@@ -243,14 +339,18 @@ shutdown_task (void *cls)
243/** 339/**
244 * Method called periodically that triggers iteration over authoritative records 340 * Method called periodically that triggers iteration over authoritative records
245 * 341 *
246 * @param cls closure 342 * @param cls NULL
247 */ 343 */
248static void 344static void
249publish_zone_dht_next (void *cls) 345publish_zone_namestore_next (void *cls)
250{ 346{
347 (void) cls;
251 zone_publish_task = NULL; 348 zone_publish_task = NULL;
252 GNUNET_assert (NULL != namestore_iter); 349 GNUNET_assert (NULL != namestore_iter);
253 GNUNET_NAMESTORE_zone_iterator_next (namestore_iter); 350 GNUNET_assert (0 == ns_iteration_left);
351 ns_iteration_left = NS_BLOCK_SIZE;
352 GNUNET_NAMESTORE_zone_iterator_next (namestore_iter,
353 NS_BLOCK_SIZE);
254} 354}
255 355
256 356
@@ -264,58 +364,224 @@ publish_zone_dht_start (void *cls);
264 364
265 365
266/** 366/**
267 * Continuation called from DHT once the PUT operation is done. 367 * Continuation called from DHT once the PUT operation triggered
368 * by a monitor is done.
268 * 369 *
269 * @param cls closure, NULL if called from regular iteration, 370 * @param cls a `struct DhtPutActivity`
270 * `struct MonitorActivity` if called from #handle_monitor_event.
271 * @param success #GNUNET_OK on success
272 */ 371 */
273static void 372static void
274dht_put_continuation (void *cls, 373dht_put_monitor_continuation (void *cls)
275 int success)
276{ 374{
277 struct MonitorActivity *ma = cls; 375 struct DhtPutActivity *ma = cls;
278 struct GNUNET_TIME_Relative next_put_interval;
279 376
280 num_public_records++; 377 ma_queue_length--;
281 if (NULL == ma) 378 GNUNET_CONTAINER_DLL_remove (ma_head,
379 ma_tail,
380 ma);
381 GNUNET_free (ma);
382}
383
384
385/**
386 * Calculate #target_iteration_velocity_per_record.
387 */
388static void
389calculate_put_interval ()
390{
391 if (0 == num_public_records)
282 { 392 {
283 active_put = NULL; 393 /**
284 if ( (num_public_records > last_num_public_records) && 394 * If no records are known (startup) or none present
285 (GNUNET_NO == first_zone_iteration) ) 395 * we can safely set the interval to the value for a single
396 * record
397 */
398 target_iteration_velocity_per_record = zone_publish_time_window;
399 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG | GNUNET_ERROR_TYPE_BULK,
400 "No records in namestore database.\n");
401 }
402 else
403 {
404 last_min_relative_record_time
405 = GNUNET_TIME_relative_min (last_min_relative_record_time,
406 min_relative_record_time);
407 zone_publish_time_window
408 = GNUNET_TIME_relative_min (GNUNET_TIME_relative_divide (last_min_relative_record_time,
409 PUBLISH_OPS_PER_EXPIRATION),
410 zone_publish_time_window_default);
411 target_iteration_velocity_per_record
412 = GNUNET_TIME_relative_divide (zone_publish_time_window,
413 last_num_public_records);
414 }
415 target_iteration_velocity_per_record
416 = GNUNET_TIME_relative_min (target_iteration_velocity_per_record,
417 MAXIMUM_ZONE_ITERATION_INTERVAL);
418 GNUNET_STATISTICS_set (statistics,
419 "Minimum relative record expiration (in μs)",
420 last_min_relative_record_time.rel_value_us,
421 GNUNET_NO);
422 GNUNET_STATISTICS_set (statistics,
423 "Zone publication time window (in μs)",
424 zone_publish_time_window.rel_value_us,
425 GNUNET_NO);
426 GNUNET_STATISTICS_set (statistics,
427 "Target zone iteration velocity (μs)",
428 target_iteration_velocity_per_record.rel_value_us,
429 GNUNET_NO);
430}
431
432
433/**
434 * Re-calculate our velocity and the desired velocity.
435 * We have succeeded in making #DELTA_INTERVAL puts, so
436 * now calculate the new desired delay between puts.
437 *
438 * @param cnt how many records were processed since the last call?
439 */
440static void
441update_velocity (unsigned int cnt)
442{
443 struct GNUNET_TIME_Relative delta;
444 unsigned long long pct = 0;
445
446 if (0 == cnt)
447 return;
448 /* How fast were we really? */
449 delta = GNUNET_TIME_absolute_get_duration (last_put_100);
450 delta.rel_value_us /= cnt;
451 last_put_100 = GNUNET_TIME_absolute_get ();
452
453 /* calculate expected frequency */
454 if ( (num_public_records > last_num_public_records) &&
455 (GNUNET_NO == first_zone_iteration) )
456 {
457 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
458 "Last record count was lower than current record count. Reducing interval.\n");
459 last_num_public_records = num_public_records * LATE_ITERATION_SPEEDUP_FACTOR;
460 calculate_put_interval ();
461 }
462 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
463 "Desired global zone iteration interval is %s/record!\n",
464 GNUNET_STRINGS_relative_time_to_string (target_iteration_velocity_per_record,
465 GNUNET_YES));
466
467 /* Tell statistics actual vs. desired speed */
468 GNUNET_STATISTICS_set (statistics,
469 "Current zone iteration velocity (μs/record)",
470 delta.rel_value_us,
471 GNUNET_NO);
472 /* update "sub_delta" based on difference, taking
473 previous sub_delta into account! */
474 if (target_iteration_velocity_per_record.rel_value_us > delta.rel_value_us)
475 {
476 /* We were too fast, reduce sub_delta! */
477 struct GNUNET_TIME_Relative corr;
478
479 corr = GNUNET_TIME_relative_subtract (target_iteration_velocity_per_record,
480 delta);
481 if (sub_delta.rel_value_us > delta.rel_value_us)
286 { 482 {
287 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 483 /* Reduce sub_delta by corr */
288 "Last record count was lower than current record count. Reducing interval.\n"); 484 sub_delta = GNUNET_TIME_relative_subtract (sub_delta,
289 put_interval = GNUNET_TIME_relative_divide (zone_publish_time_window, 485 corr);
290 num_public_records);
291 next_put_interval = GNUNET_TIME_relative_divide (put_interval,
292 LATE_ITERATION_SPEEDUP_FACTOR);
293 } 486 }
294 else 487 else
295 next_put_interval = put_interval; 488 {
296 next_put_interval = GNUNET_TIME_relative_min (next_put_interval, 489 /* We're doing fine with waiting the full time, this
297 MAXIMUM_ZONE_ITERATION_INTERVAL); 490 should theoretically only happen if we run at
298 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 491 infinite speed. */
299 "PUT complete, next PUT in %s!\n", 492 sub_delta = GNUNET_TIME_UNIT_ZERO;
300 GNUNET_STRINGS_relative_time_to_string (next_put_interval, 493 }
301 GNUNET_YES));
302
303 GNUNET_STATISTICS_set (statistics,
304 "Current zone iteration interval (ms)",
305 next_put_interval.rel_value_us / 1000LL,
306 GNUNET_NO);
307 GNUNET_assert (NULL == zone_publish_task);
308 zone_publish_task = GNUNET_SCHEDULER_add_delayed (next_put_interval,
309 &publish_zone_dht_next,
310 NULL);
311 } 494 }
312 else 495 else if (target_iteration_velocity_per_record.rel_value_us < delta.rel_value_us)
313 { 496 {
314 GNUNET_CONTAINER_DLL_remove (ma_head, 497 /* We were too slow, increase sub_delta! */
315 ma_tail, 498 struct GNUNET_TIME_Relative corr;
316 ma); 499
317 GNUNET_free (ma); 500 corr = GNUNET_TIME_relative_subtract (delta,
501 target_iteration_velocity_per_record);
502 sub_delta = GNUNET_TIME_relative_add (sub_delta,
503 corr);
504 if (sub_delta.rel_value_us > target_iteration_velocity_per_record.rel_value_us)
505 {
506 /* CPU overload detected, we cannot go at desired speed,
507 as this would mean using a negative delay. */
508 /* compute how much faster we would want to be for
509 the desired velocity */
510 if (0 == target_iteration_velocity_per_record.rel_value_us)
511 pct = UINT64_MAX; /* desired speed is infinity ... */
512 else
513 pct = (sub_delta.rel_value_us -
514 target_iteration_velocity_per_record.rel_value_us) * 100LLU
515 / target_iteration_velocity_per_record.rel_value_us;
516 sub_delta = target_iteration_velocity_per_record;
517 }
318 } 518 }
519 GNUNET_STATISTICS_set (statistics,
520 "# size of the DHT queue (it)",
521 dht_queue_length,
522 GNUNET_NO);
523 GNUNET_STATISTICS_set (statistics,
524 "# size of the DHT queue (mon)",
525 ma_queue_length,
526 GNUNET_NO);
527 GNUNET_STATISTICS_set (statistics,
528 "% speed increase needed for target velocity",
529 pct,
530 GNUNET_NO);
531 GNUNET_STATISTICS_set (statistics,
532 "# records processed in current iteration",
533 num_public_records,
534 GNUNET_NO);
535}
536
537
538/**
539 * Check if the current zone iteration needs to be continued
540 * by calling #publish_zone_namestore_next(), and if so with what delay.
541 */
542static void
543check_zone_namestore_next ()
544{
545 struct GNUNET_TIME_Relative delay;
546
547 if (0 != ns_iteration_left)
548 return; /* current NAMESTORE iteration not yet done */
549 update_velocity (put_cnt);
550 put_cnt = 0;
551 delay = GNUNET_TIME_relative_subtract (target_iteration_velocity_per_record,
552 sub_delta);
553 /* We delay *once* per #NS_BLOCK_SIZE, so we need to multiply the
554 per-record delay calculated so far with the #NS_BLOCK_SIZE */
555 GNUNET_STATISTICS_set (statistics,
556 "Current artificial NAMESTORE delay (μs/record)",
557 delay.rel_value_us,
558 GNUNET_NO);
559 delay = GNUNET_TIME_relative_multiply (delay,
560 NS_BLOCK_SIZE);
561 GNUNET_assert (NULL == zone_publish_task);
562 zone_publish_task = GNUNET_SCHEDULER_add_delayed (delay,
563 &publish_zone_namestore_next,
564 NULL);
565}
566
567
568/**
569 * Continuation called from DHT once the PUT operation is done.
570 *
571 * @param cls a `struct DhtPutActivity`
572 */
573static void
574dht_put_continuation (void *cls)
575{
576 struct DhtPutActivity *ma = cls;
577
578 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
579 "PUT complete\n");
580 dht_queue_length--;
581 GNUNET_CONTAINER_DLL_remove (it_head,
582 it_tail,
583 ma);
584 GNUNET_free (ma);
319} 585}
320 586
321 587
@@ -336,11 +602,10 @@ convert_records_for_export (const struct GNUNET_GNSRECORD_Data *rd,
336{ 602{
337 struct GNUNET_TIME_Absolute now; 603 struct GNUNET_TIME_Absolute now;
338 unsigned int rd_public_count; 604 unsigned int rd_public_count;
339 unsigned int i;
340 605
341 rd_public_count = 0; 606 rd_public_count = 0;
342 now = GNUNET_TIME_absolute_get (); 607 now = GNUNET_TIME_absolute_get ();
343 for (i=0;i<rd_count;i++) 608 for (unsigned int i=0;i<rd_count;i++)
344 if (0 == (rd[i].flags & GNUNET_GNSRECORD_RF_PRIVATE)) 609 if (0 == (rd[i].flags & GNUNET_GNSRECORD_RF_PRIVATE))
345 { 610 {
346 rd_public[rd_public_count] = rd[i]; 611 rd_public[rd_public_count] = rd[i];
@@ -370,7 +635,8 @@ convert_records_for_export (const struct GNUNET_GNSRECORD_Data *rd,
370 * @param label label to store under 635 * @param label label to store under
371 * @param rd_public public record data 636 * @param rd_public public record data
372 * @param rd_public_count number of records in @a rd_public 637 * @param rd_public_count number of records in @a rd_public
373 * @param pc_arg closure argument to pass to the #dht_put_continuation 638 * @param cont function to call with PUT result
639 * @param cont_cls closure for @a cont
374 * @return DHT PUT handle, NULL on error 640 * @return DHT PUT handle, NULL on error
375 */ 641 */
376static struct GNUNET_DHT_PutHandle * 642static struct GNUNET_DHT_PutHandle *
@@ -378,7 +644,8 @@ perform_dht_put (const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
378 const char *label, 644 const char *label,
379 const struct GNUNET_GNSRECORD_Data *rd_public, 645 const struct GNUNET_GNSRECORD_Data *rd_public,
380 unsigned int rd_public_count, 646 unsigned int rd_public_count,
381 void *pc_arg) 647 GNUNET_SCHEDULER_TaskCallback cont,
648 void *cont_cls)
382{ 649{
383 struct GNUNET_GNSRECORD_Block *block; 650 struct GNUNET_GNSRECORD_Block *block;
384 struct GNUNET_HashCode query; 651 struct GNUNET_HashCode query;
@@ -388,25 +655,40 @@ perform_dht_put (const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
388 655
389 expire = GNUNET_GNSRECORD_record_get_expiration_time (rd_public_count, 656 expire = GNUNET_GNSRECORD_record_get_expiration_time (rd_public_count,
390 rd_public); 657 rd_public);
391 block = GNUNET_GNSRECORD_block_create (key, 658 if (cache_keys)
392 expire, 659 block = GNUNET_GNSRECORD_block_create2 (key,
393 label, 660 expire,
394 rd_public, 661 label,
395 rd_public_count); 662 rd_public,
663 rd_public_count);
664 else
665 block = GNUNET_GNSRECORD_block_create (key,
666 expire,
667 label,
668 rd_public,
669 rd_public_count);
396 if (NULL == block) 670 if (NULL == block)
671 {
672 GNUNET_break (0);
397 return NULL; /* whoops */ 673 return NULL; /* whoops */
674 }
398 block_size = ntohl (block->purpose.size) 675 block_size = ntohl (block->purpose.size)
399 + sizeof (struct GNUNET_CRYPTO_EcdsaSignature) 676 + sizeof (struct GNUNET_CRYPTO_EcdsaSignature)
400 + sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey); 677 + sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey);
401 GNUNET_GNSRECORD_query_from_private_key (key, 678 GNUNET_GNSRECORD_query_from_private_key (key,
402 label, 679 label,
403 &query); 680 &query);
681 GNUNET_STATISTICS_update (statistics,
682 "DHT put operations initiated",
683 1,
684 GNUNET_NO);
404 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 685 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
405 "Storing %u record(s) for label `%s' in DHT with expiration `%s' under key %s\n", 686 "Storing %u record(s) for label `%s' in DHT with expiration `%s' under key %s\n",
406 rd_public_count, 687 rd_public_count,
407 label, 688 label,
408 GNUNET_STRINGS_absolute_time_to_string (expire), 689 GNUNET_STRINGS_absolute_time_to_string (expire),
409 GNUNET_h2s (&query)); 690 GNUNET_h2s (&query));
691 num_public_records++;
410 ret = GNUNET_DHT_put (dht_handle, 692 ret = GNUNET_DHT_put (dht_handle,
411 &query, 693 &query,
412 DHT_GNS_REPLICATION_LEVEL, 694 DHT_GNS_REPLICATION_LEVEL,
@@ -415,8 +697,8 @@ perform_dht_put (const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
415 block_size, 697 block_size,
416 block, 698 block,
417 expire, 699 expire,
418 &dht_put_continuation, 700 cont,
419 pc_arg); 701 cont_cls);
420 GNUNET_free (block); 702 GNUNET_free (block);
421 return ret; 703 return ret;
422} 704}
@@ -424,10 +706,13 @@ perform_dht_put (const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
424 706
425/** 707/**
426 * We encountered an error in our zone iteration. 708 * We encountered an error in our zone iteration.
709 *
710 * @param cls NULL
427 */ 711 */
428static void 712static void
429zone_iteration_error (void *cls) 713zone_iteration_error (void *cls)
430{ 714{
715 (void) cls;
431 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 716 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
432 "Got disconnected from namestore database, retrying.\n"); 717 "Got disconnected from namestore database, retrying.\n");
433 namestore_iter = NULL; 718 namestore_iter = NULL;
@@ -451,66 +736,47 @@ zone_iteration_error (void *cls)
451 736
452/** 737/**
453 * Zone iteration is completed. 738 * Zone iteration is completed.
739 *
740 * @param cls NULL
454 */ 741 */
455static void 742static void
456zone_iteration_finished (void *cls) 743zone_iteration_finished (void *cls)
457{ 744{
745 (void) cls;
458 /* we're done with one iteration, calculate when to do the next one */ 746 /* we're done with one iteration, calculate when to do the next one */
459 namestore_iter = NULL; 747 namestore_iter = NULL;
460 last_num_public_records = num_public_records; 748 last_num_public_records = num_public_records;
461 first_zone_iteration = GNUNET_NO; 749 first_zone_iteration = GNUNET_NO;
462 if (0 == num_public_records) 750 last_min_relative_record_time = min_relative_record_time;
463 { 751 calculate_put_interval ();
464 /**
465 * If no records are known (startup) or none present
466 * we can safely set the interval to the value for a single
467 * record
468 */
469 put_interval = zone_publish_time_window;
470 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG | GNUNET_ERROR_TYPE_BULK,
471 "No records in namestore database.\n");
472 }
473 else
474 {
475 /* If records are present, next publication is based on the minimum
476 * relative expiration time of the records published divided by 4
477 */
478 zone_publish_time_window
479 = GNUNET_TIME_relative_min (GNUNET_TIME_relative_divide (min_relative_record_time, 4),
480 zone_publish_time_window_default);
481 put_interval = GNUNET_TIME_relative_divide (zone_publish_time_window,
482 num_public_records);
483 }
484 /* reset for next iteration */ 752 /* reset for next iteration */
485 min_relative_record_time = GNUNET_TIME_UNIT_FOREVER_REL; 753 min_relative_record_time
486 put_interval = GNUNET_TIME_relative_max (MINIMUM_ZONE_ITERATION_INTERVAL, 754 = GNUNET_TIME_relative_multiply (GNUNET_DHT_DEFAULT_REPUBLISH_FREQUENCY,
487 put_interval); 755 PUBLISH_OPS_PER_EXPIRATION);
488 put_interval = GNUNET_TIME_relative_min (put_interval,
489 MAXIMUM_ZONE_ITERATION_INTERVAL);
490 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 756 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
491 "Zone iteration finished. Adjusted zone iteration interval to %s\n", 757 "Zone iteration finished. Adjusted zone iteration interval to %s\n",
492 GNUNET_STRINGS_relative_time_to_string (put_interval, 758 GNUNET_STRINGS_relative_time_to_string (target_iteration_velocity_per_record,
493 GNUNET_YES)); 759 GNUNET_YES));
494 GNUNET_STATISTICS_set (statistics, 760 GNUNET_STATISTICS_set (statistics,
495 "Current zone iteration interval (in ms)", 761 "Target zone iteration velocity s)",
496 put_interval.rel_value_us / 1000LL, 762 target_iteration_velocity_per_record.rel_value_us,
497 GNUNET_NO); 763 GNUNET_NO);
498 GNUNET_STATISTICS_update (statistics,
499 "Number of zone iterations",
500 1,
501 GNUNET_NO);
502 GNUNET_STATISTICS_set (statistics, 764 GNUNET_STATISTICS_set (statistics,
503 "Number of public records in DHT", 765 "Number of public records in DHT",
504 last_num_public_records, 766 last_num_public_records,
505 GNUNET_NO); 767 GNUNET_NO);
506 GNUNET_assert (NULL == zone_publish_task); 768 GNUNET_assert (NULL == zone_publish_task);
507 if (0 == num_public_records) 769 if (0 == last_num_public_records)
508 zone_publish_task = GNUNET_SCHEDULER_add_delayed (put_interval, 770 {
771 zone_publish_task = GNUNET_SCHEDULER_add_delayed (target_iteration_velocity_per_record,
509 &publish_zone_dht_start, 772 &publish_zone_dht_start,
510 NULL); 773 NULL);
774 }
511 else 775 else
776 {
512 zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start, 777 zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start,
513 NULL); 778 NULL);
779 }
514} 780}
515 781
516 782
@@ -532,32 +798,59 @@ put_gns_record (void *cls,
532{ 798{
533 struct GNUNET_GNSRECORD_Data rd_public[rd_count]; 799 struct GNUNET_GNSRECORD_Data rd_public[rd_count];
534 unsigned int rd_public_count; 800 unsigned int rd_public_count;
801 struct DhtPutActivity *ma;
535 802
803 (void) cls;
804 ns_iteration_left--;
536 rd_public_count = convert_records_for_export (rd, 805 rd_public_count = convert_records_for_export (rd,
537 rd_count, 806 rd_count,
538 rd_public); 807 rd_public);
539
540 if (0 == rd_public_count) 808 if (0 == rd_public_count)
541 { 809 {
542 GNUNET_assert (NULL == zone_publish_task);
543 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 810 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
544 "Record set empty, moving to next record set\n"); 811 "Record set empty, moving to next record set\n");
545 zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_next, 812 check_zone_namestore_next ();
546 NULL);
547 return; 813 return;
548 } 814 }
549 /* We got a set of records to publish */ 815 /* We got a set of records to publish */
550 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 816 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
551 "Starting DHT PUT\n"); 817 "Starting DHT PUT\n");
552 active_put = perform_dht_put (key, 818 ma = GNUNET_new (struct DhtPutActivity);
553 label, 819 ma->start_date = GNUNET_TIME_absolute_get ();
554 rd_public, 820 ma->ph = perform_dht_put (key,
555 rd_public_count, 821 label,
556 NULL); 822 rd_public,
557 if (NULL == active_put) 823 rd_public_count,
824 &dht_put_continuation,
825 ma);
826 put_cnt++;
827 if (0 == put_cnt % DELTA_INTERVAL)
828 update_velocity (DELTA_INTERVAL);
829 check_zone_namestore_next ();
830 if (NULL == ma->ph)
558 { 831 {
559 GNUNET_break (0); 832 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
560 dht_put_continuation (NULL, GNUNET_NO); 833 "Could not perform DHT PUT, is the DHT running?\n");
834 GNUNET_free (ma);
835 return;
836 }
837 dht_queue_length++;
838 GNUNET_CONTAINER_DLL_insert_tail (it_head,
839 it_tail,
840 ma);
841 if (dht_queue_length > DHT_QUEUE_LIMIT)
842 {
843 ma = it_head;
844 GNUNET_CONTAINER_DLL_remove (it_head,
845 it_tail,
846 ma);
847 GNUNET_DHT_put_cancel (ma->ph);
848 dht_queue_length--;
849 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
850 "DHT PUT unconfirmed after %s, aborting PUT\n",
851 GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_duration (ma->start_date),
852 GNUNET_YES));
853 GNUNET_free (ma);
561 } 854 }
562} 855}
563 856
@@ -570,13 +863,18 @@ put_gns_record (void *cls,
570static void 863static void
571publish_zone_dht_start (void *cls) 864publish_zone_dht_start (void *cls)
572{ 865{
866 (void) cls;
573 zone_publish_task = NULL; 867 zone_publish_task = NULL;
574 868 GNUNET_STATISTICS_update (statistics,
869 "Full zone iterations launched",
870 1,
871 GNUNET_NO);
575 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 872 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
576 "Starting DHT zone update!\n"); 873 "Starting DHT zone update!\n");
577 /* start counting again */ 874 /* start counting again */
578 num_public_records = 0; 875 num_public_records = 0;
579 GNUNET_assert (NULL == namestore_iter); 876 GNUNET_assert (NULL == namestore_iter);
877 ns_iteration_left = 1;
580 namestore_iter 878 namestore_iter
581 = GNUNET_NAMESTORE_zone_iteration_start (namestore_handle, 879 = GNUNET_NAMESTORE_zone_iteration_start (namestore_handle,
582 NULL, /* All zones */ 880 NULL, /* All zones */
@@ -586,6 +884,7 @@ publish_zone_dht_start (void *cls)
586 NULL, 884 NULL,
587 &zone_iteration_finished, 885 &zone_iteration_finished,
588 NULL); 886 NULL);
887 GNUNET_assert (NULL != namestore_iter);
589} 888}
590 889
591 890
@@ -608,8 +907,13 @@ handle_monitor_event (void *cls,
608{ 907{
609 struct GNUNET_GNSRECORD_Data rd_public[rd_count]; 908 struct GNUNET_GNSRECORD_Data rd_public[rd_count];
610 unsigned int rd_public_count; 909 unsigned int rd_public_count;
611 struct MonitorActivity *ma; 910 struct DhtPutActivity *ma;
612 911
912 (void) cls;
913 GNUNET_STATISTICS_update (statistics,
914 "Namestore monitor events received",
915 1,
916 GNUNET_NO);
613 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 917 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
614 "Received %u records for label `%s' via namestore monitor\n", 918 "Received %u records for label `%s' via namestore monitor\n",
615 rd_count, 919 rd_count,
@@ -621,11 +925,14 @@ handle_monitor_event (void *cls,
621 rd_public); 925 rd_public);
622 if (0 == rd_public_count) 926 if (0 == rd_public_count)
623 return; /* nothing to do */ 927 return; /* nothing to do */
624 ma = GNUNET_new (struct MonitorActivity); 928 num_public_records++;
929 ma = GNUNET_new (struct DhtPutActivity);
930 ma->start_date = GNUNET_TIME_absolute_get ();
625 ma->ph = perform_dht_put (zone, 931 ma->ph = perform_dht_put (zone,
626 label, 932 label,
627 rd, 933 rd,
628 rd_count, 934 rd_count,
935 &dht_put_monitor_continuation,
629 ma); 936 ma);
630 if (NULL == ma->ph) 937 if (NULL == ma->ph)
631 { 938 {
@@ -633,9 +940,24 @@ handle_monitor_event (void *cls,
633 GNUNET_free (ma); 940 GNUNET_free (ma);
634 return; 941 return;
635 } 942 }
636 GNUNET_CONTAINER_DLL_insert (ma_head, 943 GNUNET_CONTAINER_DLL_insert_tail (ma_head,
637 ma_tail, 944 ma_tail,
638 ma); 945 ma);
946 ma_queue_length++;
947 if (ma_queue_length > DHT_QUEUE_LIMIT)
948 {
949 ma = ma_head;
950 GNUNET_CONTAINER_DLL_remove (ma_head,
951 ma_tail,
952 ma);
953 GNUNET_DHT_put_cancel (ma->ph);
954 ma_queue_length--;
955 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
956 "DHT PUT unconfirmed after %s, aborting PUT\n",
957 GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_duration (ma->start_date),
958 GNUNET_YES));
959 GNUNET_free (ma);
960 }
639} 961}
640 962
641 963
@@ -648,22 +970,28 @@ handle_monitor_event (void *cls,
648static void 970static void
649monitor_sync_event (void *cls) 971monitor_sync_event (void *cls)
650{ 972{
973 (void) cls;
651 if ( (NULL == zone_publish_task) && 974 if ( (NULL == zone_publish_task) &&
652 (NULL == namestore_iter) ) 975 (NULL == namestore_iter) )
653 zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start, 976 zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start,
654 NULL); 977 NULL);
655} 978}
656 979
657 980
658/** 981/**
659 * The zone monitor is now in SYNC with the current state of the 982 * The zone monitor encountered an IPC error trying to to get in
660 * name store. Start to perform periodic iterations. 983 * sync. Restart from the beginning.
661 * 984 *
662 * @param cls NULL 985 * @param cls NULL
663 */ 986 */
664static void 987static void
665handle_monitor_error (void *cls) 988handle_monitor_error (void *cls)
666{ 989{
990 (void) cls;
991 GNUNET_STATISTICS_update (statistics,
992 "Namestore monitor errors encountered",
993 1,
994 GNUNET_NO);
667 if (NULL != zone_publish_task) 995 if (NULL != zone_publish_task)
668 { 996 {
669 GNUNET_SCHEDULER_cancel (zone_publish_task); 997 GNUNET_SCHEDULER_cancel (zone_publish_task);
@@ -698,7 +1026,13 @@ run (void *cls,
698{ 1026{
699 unsigned long long max_parallel_bg_queries = 128; 1027 unsigned long long max_parallel_bg_queries = 128;
700 1028
701 min_relative_record_time = GNUNET_TIME_UNIT_FOREVER_REL; 1029 (void) cls;
1030 (void) service;
1031 last_put_100 = GNUNET_TIME_absolute_get (); /* first time! */
1032 min_relative_record_time
1033 = GNUNET_TIME_relative_multiply (GNUNET_DHT_DEFAULT_REPUBLISH_FREQUENCY,
1034 PUBLISH_OPS_PER_EXPIRATION);
1035 target_iteration_velocity_per_record = INITIAL_ZONE_ITERATION_INTERVAL;
702 namestore_handle = GNUNET_NAMESTORE_connect (c); 1036 namestore_handle = GNUNET_NAMESTORE_connect (c);
703 if (NULL == namestore_handle) 1037 if (NULL == namestore_handle)
704 { 1038 {
@@ -707,8 +1041,9 @@ run (void *cls,
707 GNUNET_SCHEDULER_shutdown (); 1041 GNUNET_SCHEDULER_shutdown ();
708 return; 1042 return;
709 } 1043 }
710 1044 cache_keys = GNUNET_CONFIGURATION_get_value_yesno (c,
711 put_interval = INITIAL_PUT_INTERVAL; 1045 "namestore",
1046 "CACHE_KEYS");
712 zone_publish_time_window_default = DEFAULT_ZONE_PUBLISH_TIME_WINDOW; 1047 zone_publish_time_window_default = DEFAULT_ZONE_PUBLISH_TIME_WINDOW;
713 if (GNUNET_OK == 1048 if (GNUNET_OK ==
714 GNUNET_CONFIGURATION_get_value_time (c, 1049 GNUNET_CONFIGURATION_get_value_time (c,
@@ -733,20 +1068,26 @@ run (void *cls,
733 max_parallel_bg_queries); 1068 max_parallel_bg_queries);
734 } 1069 }
735 if (0 == max_parallel_bg_queries) 1070 if (0 == max_parallel_bg_queries)
736 max_parallel_bg_queries = 1; 1071 max_parallel_bg_queries = 1;
737 dht_handle = GNUNET_DHT_connect (c, 1072 dht_handle = GNUNET_DHT_connect (c,
738 (unsigned int) max_parallel_bg_queries); 1073 (unsigned int) max_parallel_bg_queries);
739 if (NULL == dht_handle) 1074 if (NULL == dht_handle)
740 { 1075 {
741 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 1076 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
742 _("Could not connect to DHT!\n")); 1077 _("Could not connect to DHT!\n"));
743 GNUNET_SCHEDULER_add_now (&shutdown_task, NULL); 1078 GNUNET_SCHEDULER_add_now (&shutdown_task,
1079 NULL);
744 return; 1080 return;
745 } 1081 }
746 1082
747 /* Schedule periodic put for our records. */ 1083 /* Schedule periodic put for our records. */
748 first_zone_iteration = GNUNET_YES;\ 1084 first_zone_iteration = GNUNET_YES;
749 statistics = GNUNET_STATISTICS_create ("zonemaster", c); 1085 statistics = GNUNET_STATISTICS_create ("zonemaster",
1086 c);
1087 GNUNET_STATISTICS_set (statistics,
1088 "Target zone iteration velocity (μs)",
1089 target_iteration_velocity_per_record.rel_value_us,
1090 GNUNET_NO);
750 zmon = GNUNET_NAMESTORE_zone_monitor_start (c, 1091 zmon = GNUNET_NAMESTORE_zone_monitor_start (c,
751 NULL, 1092 NULL,
752 GNUNET_NO, 1093 GNUNET_NO,
@@ -757,7 +1098,8 @@ run (void *cls,
757 &monitor_sync_event, 1098 &monitor_sync_event,
758 NULL); 1099 NULL);
759 GNUNET_break (NULL != zmon); 1100 GNUNET_break (NULL != zmon);
760 GNUNET_SCHEDULER_add_shutdown (&shutdown_task, NULL); 1101 GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
1102 NULL);
761} 1103}
762 1104
763 1105