move to @ for empty label, fix empty-label resolution bug in GNS API, install gnunet-zoneimport and add man page

14 files changed, 91 insertions, 72 deletions

diff --git a/doc/man/Makefile.am b/doc/man/Makefile.am
index c40363b59..a6a116dca 100644
--- a/doc/man/Makefile.am
+++ b/doc/man/Makefile.am
@@ -17,7 +17,6 @@ man_MANS = \
   gnunet-ecc.1 \
   gnunet-fs.1 \
   gnunet-gns.1 \
-  gnunet-gns-import.1 \
   gnunet-gns-proxy.1 \
   gnunet-identity.1 \
   gnunet-cadet.1 \
@@ -42,6 +41,7 @@ man_MANS = \
   gnunet-transport-certificate-creation.1 \
   gnunet-unindex.1 \
   gnunet-uri.1 \
-  gnunet-vpn.1
+  gnunet-vpn.1 \
+  gnunet-zoneimport.1
 
 EXTRA_DIST = ${man_MANS}
diff --git a/doc/man/gnunet-fs.1 b/doc/man/gnunet-fs.1
index 53a900b46..65f104d61 100644
--- a/doc/man/gnunet-fs.1
+++ b/doc/man/gnunet-fs.1
@@ -29,10 +29,9 @@ print the version number
 \fB\-V\fR, \fB\-\-verbose\fR
 be verbose
 
-.SH NOTES
 
-
-.SH "REPORTING BUGS"
+.SH BUGS
 Report bugs by using mantis <https://gnunet.org/bugs/> or by sending electronic mail to <gnunet\-developers@gnu.org>
-.SH "SEE ALSO"
+
+.SH SEE ALSO
 \fBgnunet\-publish\fP(1)
diff --git a/doc/man/gnunet-gns-import.1 b/doc/man/gnunet-gns-import.1
deleted file mode 100644
index f90f7f375..000000000
--- a/doc/man/gnunet-gns-import.1
+++ /dev/null
@@ -1,17 +0,0 @@
-.TH GNUNET\-GNS\-IMPORT 1 "Jan 31, 2014" "GNUnet"
-
-.SH NAME
-gnunet\-gns\-import \- import some GNS authorities into your GNS namestore
-
-.SH SYNOPSIS
-.B gnunet\-gns\-import
-.br
-
-.SH DESCRIPTION
-\fBgnunet\-gns\-import\fP calls gnunet-namestore to fetch some GNS authorities into your GNS namestore.
-
-.SH BUGS
-Report bugs by using Mantis <https://gnunet.org/bugs/> or by sending electronic mail to <gnunet\-developers@gnu.org>
-
-.SH SEE ALSO
-gnunet\-gns(1), gnunet\-namestore(1)
diff --git a/doc/man/gnunet-zoneimport.1 b/doc/man/gnunet-zoneimport.1
new file mode 100644
index 000000000..d268bf9ff
--- /dev/null
+++ b/doc/man/gnunet-zoneimport.1
@@ -0,0 +1,45 @@
+.TH GNUNET\-ZONEIMPORT 1 "Apr 23, 2018" "GNUnet"
+
+.SH NAME
+gnunet\-zoneimport \- import DNS zone into GNS zone
+
+.SH SYNOPSIS
+.B gnunet\-zoneimport -s IP
+.br
+
+.SH DESCRIPTION
+\fBgnunet\-zoneimport\fP reads a list of domain names (FQDN) from stdin and issues DNS queries for each of the domain names given.  It then checks if a local ego with a name matching the domain exists. Specifically, if the domain name is "example.fr", it will check if an ego "fr" exists, while for a domain "example.com.fr" it will look for an ego called "com.fr"). If so, it will convert the DNS records into GNS records (in particular converting NS records and glue records to GNS2DNS records) and add them to the namestore under the label ("example" in the examples above).
+
+gnunet\-zoneimport will usually never terminate: it will check when DNS records expire, and re-issue requests when the old DNS records have expired so that GNS always has the latest data.
+
+gnunet\-zoneimport will issue many DNS queries in parallel, but is rate-limited in various ways, so most DNS servers should easily handle the load.  gnunet\-zoneimport will perform a limited number of retries if queries fail.
+
+gnunet\-zoneimport operates incrementally. It will check if the namestore already has (non-expired) records stored for a given name in the respective zone and not issue those requests again.  Thus, it is fine to restart gnunet\-zoneimport whenever the list of domain names changes.
+
+Finally, gnunet\-zoneimport keeps information for each domain name in memory.  This consumes about 200 bytes per label, or 2 GB for 10 million labels.
+
+.SH OPTIONS
+.B
+.IP "\-c FILENAME,  \-\-config=FILENAME"
+Use the configuration file FILENAME.
+.B
+.IP "\-s IP, \-\-server IP"
+IP address of the DNS server to query. Should be the authoritative server for the domain (lookup the IP address of the NS server!) and not some recursive resolver (which would be both slow and is more likely to return only partial information).
+.B
+.IP "\-h, \-\-help"
+Print short help on options.
+
+.SH NOTES
+
+Typical invocaton would be:
+
+$ gnunet\-zoneimport -s 1.2.3.4 < names.txt
+
+
+
+
+.SH BUGS
+Report bugs by using Mantis <https://gnunet.org/bugs/> or by sending electronic mail to <gnunet\-developers@gnu.org>
+
+.SH SEE ALSO
+gnunet\-gns(1), gnunet\-namestore(1)
diff --git a/src/gns/gns_tld_api.c b/src/gns/gns_tld_api.c
index 293e37140..8a4d03b11 100644
--- a/src/gns/gns_tld_api.c
+++ b/src/gns/gns_tld_api.c
@@ -94,7 +94,7 @@ struct GNUNET_GNS_LookupWithTldRequest
  * @return the part of @a name after the last ".",
  *         or @a name if @a name does not contain a "."
  */
-static const char *
+static char *
 get_tld (const char *name)
 {
   const char *tld;
@@ -105,14 +105,14 @@ get_tld (const char *name)
     tld = name;
   else
     tld++; /* skip the '.' */
-  return tld;
+  return GNUNET_strdup (tld);
 }
 
 
 /**
  * Eat the TLD of the given @a name.
  *
- * @param name a name
+ * @param[in,out] name a name
  */
 static void
 eat_tld (char *name)
@@ -124,7 +124,7 @@ eat_tld (char *name)
                  (unsigned char) '.');
   if (NULL == tld)
     strcpy (name,
-            GNUNET_GNS_MASTERZONE_STR);
+            GNUNET_GNS_EMPTY_LABEL_AT);
   else
     *tld = '\0';
 }
@@ -229,7 +229,7 @@ GNUNET_GNS_lookup_with_tld (struct GNUNET_GNS_Handle *handle,
                             void *proc_cls)
 {
   struct GNUNET_GNS_LookupWithTldRequest *ltr;
-  const char *tld;
+  char *tld;
   char *dot_tld;
   char *zonestr;
   struct GNUNET_CRYPTO_EcdsaPublicKey pkey;
@@ -251,6 +251,7 @@ GNUNET_GNS_lookup_with_tld (struct GNUNET_GNS_Handle *handle,
     eat_tld (ltr->name);
     lookup_with_public_key (ltr,
                             &pkey);
+    GNUNET_free (tld);
     return ltr;
   }
 
@@ -277,6 +278,7 @@ GNUNET_GNS_lookup_with_tld (struct GNUNET_GNS_Handle *handle,
       GNUNET_free (dot_tld);
       GNUNET_free (ltr->name);
       GNUNET_free (ltr);
+      GNUNET_free (tld);
       return NULL;
     }
     GNUNET_free (dot_tld);
@@ -284,6 +286,7 @@ GNUNET_GNS_lookup_with_tld (struct GNUNET_GNS_Handle *handle,
     eat_tld (ltr->name);
     lookup_with_public_key (ltr,
                             &pkey);
+    GNUNET_free (tld);
     return ltr;
   }
   GNUNET_free (dot_tld);
@@ -301,6 +304,7 @@ GNUNET_GNS_lookup_with_tld (struct GNUNET_GNS_Handle *handle,
                                            tld,
                                            &identity_zone_cb,
                                            ltr);
+  GNUNET_free (tld);
   if (NULL == ltr->id_op)
   {
     GNUNET_free (ltr->name);
diff --git a/src/gns/gnunet-service-gns_resolver.c b/src/gns/gnunet-service-gns_resolver.c
index 74eb47f29..745a2f3bd 100644
--- a/src/gns/gnunet-service-gns_resolver.c
+++ b/src/gns/gnunet-service-gns_resolver.c
@@ -2096,14 +2096,14 @@ handle_gns_resolution_result (void *cls,
           rd_off++;
           if (GNUNET_GNSRECORD_TYPE_PKEY != rh->record_type)
           {
-            /* try to resolve "+" */
+            /* try to resolve "@" */
             struct AuthorityChain *ac;
 
             ac = GNUNET_new (struct AuthorityChain);
             ac->rh = rh;
             ac->gns_authority = GNUNET_YES;
             ac->authority_info.gns_authority = pub;
-            ac->label = GNUNET_strdup (GNUNET_GNS_MASTERZONE_STR);
+            ac->label = GNUNET_strdup (GNUNET_GNS_EMPTY_LABEL_AT);
             ac->suggested_shortening_label = NULL;
             ac->shortening_started = GNUNET_NO;
             GNUNET_CONTAINER_DLL_insert_tail (rh->ac_head,
@@ -2629,8 +2629,8 @@ start_resolver_lookup (void *cls)
   ac->suggested_shortening_label = NULL;
   if (NULL == ac->label)
     /* name was just the "TLD", so we default to label
-       #GNUNET_GNS_MASTERZONE_STR */
-    ac->label = GNUNET_strdup (GNUNET_GNS_MASTERZONE_STR);
+       #GNUNET_GNS_EMPTY_LABEL_AT */
+    ac->label = GNUNET_strdup (GNUNET_GNS_EMPTY_LABEL_AT);
   ac->gns_authority = GNUNET_YES;
   ac->authority_info.gns_authority = rh->authority_zone;
   GNUNET_CONTAINER_DLL_insert_tail (rh->ac_head,
diff --git a/src/include/gnunet_gns_service.h b/src/include/gnunet_gns_service.h
index 8c1f64783..ff3110406 100644
--- a/src/include/gnunet_gns_service.h
+++ b/src/include/gnunet_gns_service.h
@@ -48,13 +48,10 @@ extern "C"
 
 
 /**
- * String we use to indicate the local master zone or a
- * root entry in the current zone.
- *
- * FIXME: probably should be changed to "@" and renamed
- * (this name is confusing!)
+ * String we use to indicate an empty label (top-level
+ * entry in the zone).  DNS uses "@", so do we.
  */
-#define GNUNET_GNS_MASTERZONE_STR "+"
+#define GNUNET_GNS_EMPTY_LABEL_AT "@"
 
 /**
  * Connection to the GNS service.
diff --git a/src/namecache/plugin_namecache_sqlite.c b/src/namecache/plugin_namecache_sqlite.c
index 37d6d3b62..e8a61a232 100644
--- a/src/namecache/plugin_namecache_sqlite.c
+++ b/src/namecache/plugin_namecache_sqlite.c
@@ -411,10 +411,10 @@ namecache_sqlite_cache_block (void *cls,
   GNUNET_CRYPTO_hash (&block->derived_key,
                       sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey),
                       &query);
-  fprintf (stderr,
-           "Caching new version of block %s (expires %llu)\n",
-           GNUNET_h2s (&query),
-           (unsigned long long) expiration.abs_value_us);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Caching new version of block %s (expires %llu)\n",
+              GNUNET_h2s (&query),
+              (unsigned long long) expiration.abs_value_us);
   expiration = GNUNET_TIME_absolute_ntoh (block->expiration_time);
   if (block_size > 64 * 65536)
   {
diff --git a/src/namestore/Makefile.am b/src/namestore/Makefile.am
index fd8f8054f..5f18506a7 100644
--- a/src/namestore/Makefile.am
+++ b/src/namestore/Makefile.am
@@ -124,9 +124,7 @@ libexec_PROGRAMS = \
  gnunet-service-namestore
 
 bin_PROGRAMS = \
- gnunet-namestore
-
-noinst_PROGRAMS = \
+ gnunet-namestore \
  gnunet-zoneimport
 
 if HAVE_MHD
diff --git a/src/namestore/gnunet-namestore.c b/src/namestore/gnunet-namestore.c
index 9a1805af4..660737595 100644
--- a/src/namestore/gnunet-namestore.c
+++ b/src/namestore/gnunet-namestore.c
@@ -408,7 +408,7 @@ display_record (void *cls,
   {
     if ( (GNUNET_GNSRECORD_TYPE_NICK == rd[i].record_type) &&
          (0 != strcmp (rname,
-                       GNUNET_GNS_MASTERZONE_STR)) )
+                       GNUNET_GNS_EMPTY_LABEL_AT)) )
       continue;
     typestring = GNUNET_GNSRECORD_number_to_typename (rd[i].record_type);
     s = GNUNET_GNSRECORD_value_to_string (rd[i].record_type,
diff --git a/src/namestore/gnunet-service-namestore.c b/src/namestore/gnunet-service-namestore.c
index 3a3291c07..3685c93af 100644
--- a/src/namestore/gnunet-service-namestore.c
+++ b/src/namestore/gnunet-service-namestore.c
@@ -362,13 +362,13 @@ client_connect_cb (void *cls,
 
 
 /**
- * Function called with the records for the #GNUNET_GNS_MASTERZONE_STR
+ * Function called with the records for the #GNUNET_GNS_EMPTY_LABEL_AT
  * label in the zone.  Used to locate the #GNUNET_GNSRECORD_TYPE_NICK
  * record, which (if found) is then copied to @a cls for future use.
  *
  * @param cls a `struct GNUNET_GNSRECORD_Data **` for storing the nick (if found)
  * @param private_key the private key of the zone (unused)
- * @param label should be #GNUNET_GNS_MASTERZONE_STR
+ * @param label should be #GNUNET_GNS_EMPTY_LABEL_AT
  * @param rd_count number of records in @a rd
  * @param rd records stored under @a label in the zone
  */
@@ -382,7 +382,7 @@ lookup_nick_it (void *cls,
   struct GNUNET_GNSRECORD_Data **res = cls;
 
   (void) private_key;
-  if (0 != strcmp (label, GNUNET_GNS_MASTERZONE_STR))
+  if (0 != strcmp (label, GNUNET_GNS_EMPTY_LABEL_AT))
   {
     GNUNET_break (0);
     return;
@@ -423,7 +423,7 @@ get_nick_record (const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone)
   nick = NULL;
   res = GSN_database->lookup_records (GSN_database->cls,
                                       zone,
-                                      GNUNET_GNS_MASTERZONE_STR,
+                                      GNUNET_GNS_EMPTY_LABEL_AT,
                                       &lookup_nick_it,
                                       &nick);
   if ( (GNUNET_OK != res) ||
@@ -531,7 +531,7 @@ send_lookup_response (struct NamestoreClient *nc,
   char *rd_ser;
 
   nick = get_nick_record (zone_key);
-  if ((NULL != nick) && (0 != strcmp(name, GNUNET_GNS_MASTERZONE_STR)))
+  if ((NULL != nick) && (0 != strcmp(name, GNUNET_GNS_EMPTY_LABEL_AT)))
   {
     nick->flags = (nick->flags | GNUNET_GNSRECORD_RF_PRIVATE) ^ GNUNET_GNSRECORD_RF_PRIVATE;
     merge_with_nick_records (nick,
@@ -766,7 +766,7 @@ lookup_it (void *cls,
     {
       if ( (NULL != rlc->nick) &&
            (0 != strcmp (label,
-                         GNUNET_GNS_MASTERZONE_STR)) )
+                         GNUNET_GNS_EMPTY_LABEL_AT)) )
       {
         /* Merge */
         rd_res = NULL;
@@ -1043,12 +1043,12 @@ handle_record_store (void *cls,
       unsigned int rd_clean_off;
 
       /* remove "NICK" records, unless this is for the
-         #GNUNET_GNS_MASTERZONE_STR label */
+         #GNUNET_GNS_EMPTY_LABEL_AT label */
       rd_clean_off = 0;
       for (unsigned int i=0;i<rd_count;i++)
       {
         rd_clean[rd_clean_off] = rd[i];
-        if ( (0 == strcmp (GNUNET_GNS_MASTERZONE_STR,
+        if ( (0 == strcmp (GNUNET_GNS_EMPTY_LABEL_AT,
                            conv_name)) ||
              (GNUNET_GNSRECORD_TYPE_NICK != rd[i].record_type) )
           rd_clean_off++;
diff --git a/src/namestore/gnunet-zoneimport.c b/src/namestore/gnunet-zoneimport.c
index 279bfddea..89afeeeb1 100644
--- a/src/namestore/gnunet-zoneimport.c
+++ b/src/namestore/gnunet-zoneimport.c
@@ -21,9 +21,6 @@
  * @file src/namestore/gnunet-zoneimport.c
  * @brief import a DNS zone for publication in GNS, incremental
  * @author Christian Grothoff
- *
- * TODO:
- * - set NICKname for zone's records
  */
 #include "platform.h"
 #include <gnunet_util_lib.h>
@@ -50,11 +47,6 @@
  */
 #define MAX_RETRIES 5
 
-/**
- * After how many lookups should we always sync to disk?
- */
-#define TRANSACTION_SYNC_FREQ 100
-
 
 /**
  * Some zones may include authoritative records for other
@@ -795,6 +787,7 @@ store_completed_cb (void *cls,
                     const char *emsg)
 {
   struct Request *req = cls;
+  struct Record *rec;
 
   req->qe = NULL;
   pending--;
@@ -811,6 +804,14 @@ store_completed_cb (void *cls,
                 "Stored records under `%s'\n",
                 req->label);
   }
+  /* Free records */
+  while (NULL != (rec = req->rec_head))
+  {
+    GNUNET_CONTAINER_DLL_remove (req->rec_head,
+                                 req->rec_tail,
+                                 rec);
+    GNUNET_free (rec);
+  }
 }
 
 
@@ -877,14 +878,6 @@ process_result (void *cls,
     pending--;
     return;
   }
-  /* Free old/legacy records */
-  while (NULL != (rec = req->rec_head))
-  {
-    GNUNET_CONTAINER_DLL_remove (req->rec_head,
-                                 req->rec_tail,
-                                 rec);
-    GNUNET_free (rec);
-  }
   /* import new records */
   req->issue_num = 0; /* success, reset counter! */
   req->p = p;
diff --git a/src/namestore/namestore_api.c b/src/namestore/namestore_api.c
index a187bd250..05bdf17da 100644
--- a/src/namestore/namestore_api.c
+++ b/src/namestore/namestore_api.c
@@ -1087,7 +1087,7 @@ GNUNET_NAMESTORE_set_nick (struct GNUNET_NAMESTORE_Handle *h,
   rd.flags |= GNUNET_GNSRECORD_RF_PRIVATE;
   return GNUNET_NAMESTORE_records_store (h,
                                          pkey,
-                                         GNUNET_GNS_MASTERZONE_STR,
+                                         GNUNET_GNS_EMPTY_LABEL_AT,
                                          1,
                                          &rd,
                                          cont,
diff --git a/src/namestore/plugin_rest_namestore.c b/src/namestore/plugin_rest_namestore.c
index 4602106da..aedb39159 100644
--- a/src/namestore/plugin_rest_namestore.c
+++ b/src/namestore/plugin_rest_namestore.c
@@ -460,7 +460,7 @@ namestore_list_response (void *cls,
   for (unsigned int i=0; i<rd_len; i++)
   {
     if ( (GNUNET_GNSRECORD_TYPE_NICK == rd[i].record_type) &&
-         (0 != strcmp (rname, GNUNET_GNS_MASTERZONE_STR)) )
+         (0 != strcmp (rname, GNUNET_GNS_EMPTY_LABEL_AT)) )
       continue;
 
     if ( (rd[i].record_type != handle->type) &&