/* This file is part of GNUnet Copyright (C) 2018 GNUnet e.V. GNUnet is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 3, or (at your option) any later version. GNUnet is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with GNUnet; see the file COPYING. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ /** * @file src/namestore/gnunet-zoneimport.c * @brief import a DNS zone for publication in GNS, incremental * @author Christian Grothoff */ #include "platform.h" #include #include #include #include #include #include /** * Maximum number of queries pending at the same time. */ #define THRESH 100 /** * TIME_THRESH is in usecs. How quickly do we submit fresh queries. * Used as an additional throttle. */ #define TIME_THRESH 10 /** * How often do we retry a query before giving up for good? */ #define MAX_RETRIES 5 /** * How many DNS requests do we at most issue in rapid series? */ #define MAX_SERIES 10 /** * How long do we wait at least between series of requests? */ #define SERIES_DELAY GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MICROSECONDS, 10) /** * How many requests do we request from NAMESTORE in one batch * during our initial iteration? */ #define NS_BATCH_SIZE 1024 /** * Some zones may include authoritative records for other * zones, such as foo.com.uk or bar.com.fr. As for GNS * each dot represents a zone cut, we then need to create a * zone on-the-fly to capture those records properly. */ struct Zone { /** * Kept in a DLL. */ struct Zone *next; /** * Kept in a DLL. */ struct Zone *prev; /** * Domain of the zone (i.e. "fr" or "com.fr") */ char *domain; /** * Private key of the zone. */ struct GNUNET_CRYPTO_EcdsaPrivateKey key; }; /** * Record for the request to be stored by GNS. */ struct Record { /** * Kept in a DLL. */ struct Record *next; /** * Kept in a DLL. */ struct Record *prev; /** * GNS record. */ struct GNUNET_GNSRECORD_Data grd; }; /** * Request we should make. We keep this struct in memory per request, * thus optimizing it is crucial for the overall memory consumption of * the zone importer. */ struct Request { /** * Requests are kept in a heap while waiting to be resolved. */ struct GNUNET_CONTAINER_HeapNode *hn; /** * Active requests are kept in a DLL. */ struct Request *next; /** * Active requests are kept in a DLL. */ struct Request *prev; /** * Head of records that should be published in GNS for * this hostname. */ struct Record *rec_head; /** * Tail of records that should be published in GNS for * this hostname. */ struct Record *rec_tail; /** * Socket used to make the request, NULL if not active. */ struct GNUNET_DNSSTUB_RequestSocket *rs; /** * Hostname we are resolving, allocated at the end of * this struct (optimizing memory consumption by reducing * total number of allocations). */ char *hostname; /** * Namestore operation pending for this record. */ struct GNUNET_NAMESTORE_QueueEntry *qe; /** * Zone responsible for this request. */ const struct Zone *zone; /** * At what time does the (earliest) of the returned records * for this name expire? At this point, we need to re-fetch * the record. */ struct GNUNET_TIME_Absolute expires; /** * How often did we issue this query? (And failed, reset * to zero once we were successful.) */ unsigned int issue_num; /** * random 16-bit DNS query identifier. */ uint16_t id; }; /** * Handle to the identity service. */ static struct GNUNET_IDENTITY_Handle *id; /** * Namestore handle. */ static struct GNUNET_NAMESTORE_Handle *ns; /** * Context for DNS resolution. */ static struct GNUNET_DNSSTUB_Context *ctx; /** * The number of DNS queries that are outstanding */ static unsigned int pending; /** * The number of NAMESTORE record store operations that are outstanding */ static unsigned int pending_rs; /** * Number of lookups we performed overall. */ static unsigned int lookups; /** * How many hostnames did we reject (malformed). */ static unsigned int rejects; /** * Number of lookups that failed. */ static unsigned int failures; /** * Number of records we found. */ static unsigned int records; /** * Heap of all requests to perform, sorted by * the time we should next do the request (i.e. by expires). */ static struct GNUNET_CONTAINER_Heap *req_heap; /** * Active requests are kept in a DLL. */ static struct Request *req_head; /** * Active requests are kept in a DLL. */ static struct Request *req_tail; /** * Main task. */ static struct GNUNET_SCHEDULER_Task *t; /** * Hash map of requests for which we may still get a response from * the namestore. Set to NULL once the initial namestore iteration * is done. */ static struct GNUNET_CONTAINER_MultiHashMap *ns_pending; /** * Current zone iteration handle. */ static struct GNUNET_NAMESTORE_ZoneIterator *zone_it; /** * Head of list of zones we are managing. */ static struct Zone *zone_head; /** * Tail of list of zones we are managing. */ static struct Zone *zone_tail; /** * After how many more results must #ns_lookup_result_cb() ask * the namestore for more? */ static uint64_t ns_iterator_trigger_next; /** * Callback for #for_all_records * * @param cls closure * @param rec a DNS record */ typedef void (*RecordProcessor) (void *cls, const struct GNUNET_DNSPARSER_Record *rec); /** * Call @a rp for each record in @a p, regardless of * what response section it is in. * * @param p packet from DNS * @param rp function to call * @param rp_cls closure for @a rp */ static void for_all_records (const struct GNUNET_DNSPARSER_Packet *p, RecordProcessor rp, void *rp_cls) { for (unsigned int i=0;inum_answers;i++) { struct GNUNET_DNSPARSER_Record *rs = &p->answers[i]; rp (rp_cls, rs); } for (unsigned int i=0;inum_authority_records;i++) { struct GNUNET_DNSPARSER_Record *rs = &p->authority_records[i]; rp (rp_cls, rs); } for (unsigned int i=0;inum_additional_records;i++) { struct GNUNET_DNSPARSER_Record *rs = &p->additional_records[i]; rp (rp_cls, rs); } } /** * Return just the label of the hostname in @a req. * * @param req request to process hostname of * @return statically allocated pointer to the label, * overwritten upon the next request! */ static const char * get_label (struct Request *req) { static char label[64]; const char *dot; dot = strchr (req->hostname, (unsigned char) '.'); if (NULL == dot) { GNUNET_break (0); return NULL; } if (((size_t) (dot - req->hostname)) >= sizeof (label)) { GNUNET_break (0); return NULL; } memcpy (label, req->hostname, dot - req->hostname); label[dot - req->hostname] = '\0'; return label; } /** * Build DNS query for @a hostname. * * @param hostname host to build query for * @param raw_size[out] number of bytes in the query * @return NULL on error, otherwise pointer to statically (!) * allocated query buffer */ static void * build_dns_query (struct Request *req, size_t *raw_size) { static char raw[512]; char *rawp; struct GNUNET_DNSPARSER_Packet p; struct GNUNET_DNSPARSER_Query q; q.name = (char *) req->hostname; q.type = GNUNET_DNSPARSER_TYPE_NS; q.dns_traffic_class = GNUNET_TUN_DNS_CLASS_INTERNET; memset (&p, 0, sizeof (p)); p.num_queries = 1; p.queries = &q; p.id = req->id; if (GNUNET_OK != GNUNET_DNSPARSER_pack (&p, UINT16_MAX, &rawp, raw_size)) { GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Failed to pack query for hostname `%s'\n", req->hostname); rejects++; return NULL; } if (*raw_size > sizeof (raw)) { GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Failed to pack query for hostname `%s'\n", req->hostname); rejects++; GNUNET_break (0); GNUNET_free (rawp); return NULL; } memcpy (raw, rawp, *raw_size); GNUNET_free (rawp); return raw; } /** * Free records associated with @a req. * * @param req request to free records of */ static void free_records (struct Request *req) { struct Record *rec; /* Free records */ while (NULL != (rec = req->rec_head)) { GNUNET_CONTAINER_DLL_remove (req->rec_head, req->rec_tail, rec); GNUNET_free (rec); } } /** * Free @a req and data structures reachable from it. * * @param req request to free */ static void free_request (struct Request *req) { free_records (req); GNUNET_free (req); } /** * Process as many requests as possible from the queue. * * @param cls NULL */ static void process_queue (void *cls); /** * Insert @a req into DLL sorted by next fetch time. * * @param req request to insert into #req_heap */ static void insert_sorted (struct Request *req) { req->hn = GNUNET_CONTAINER_heap_insert (req_heap, req, req->expires.abs_value_us); if (req == GNUNET_CONTAINER_heap_peek (req_heap)) { if (NULL != t) GNUNET_SCHEDULER_cancel (t); t = GNUNET_SCHEDULER_add_at (req->expires, &process_queue, NULL); } } /** * Add record to the GNS record set for @a req. * * @param req the request to expand GNS record set for * @param type type to use * @param expiration_time when should @a rec expire * @param data raw data to store * @param data_len number of bytes in @a data */ static void add_record (struct Request *req, uint32_t type, struct GNUNET_TIME_Absolute expiration_time, const void *data, size_t data_len) { struct Record *rec; rec = GNUNET_malloc (sizeof (struct Record) + data_len); rec->grd.data = &rec[1]; rec->grd.expiration_time = expiration_time.abs_value_us; rec->grd.data_size = data_len; rec->grd.record_type = type; rec->grd.flags = GNUNET_GNSRECORD_RF_NONE; GNUNET_memcpy (&rec[1], data, data_len); GNUNET_CONTAINER_DLL_insert (req->rec_head, req->rec_tail, rec); } /** * Closure for #check_for_glue. */ struct GlueClosure { /** * Overall request we are processing. */ struct Request *req; /** * NS name we are looking for glue for. */ const char *ns; /** * Set to #GNUNET_YES if glue was found. */ int found; }; /** * Try to find glue records for a given NS record. * * @param cls a `struct GlueClosure *` * @param rec record that may contain glue information */ static void check_for_glue (void *cls, const struct GNUNET_DNSPARSER_Record *rec) { struct GlueClosure *gc = cls; char dst[65536]; size_t dst_len; size_t off; char ip[INET6_ADDRSTRLEN+1]; socklen_t ip_size = (socklen_t) sizeof (ip); if (0 != strcasecmp (rec->name, gc->ns)) return; dst_len = sizeof (dst); off = 0; switch (rec->type) { case GNUNET_DNSPARSER_TYPE_A: if (sizeof (struct in_addr) != rec->data.raw.data_len) { GNUNET_break (0); return; } if (NULL == inet_ntop (AF_INET, rec->data.raw.data, ip, ip_size)) { GNUNET_break (0); return; } if ( (GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst, dst_len, &off, gc->req->hostname)) && (GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst, dst_len, &off, ip)) ) { add_record (gc->req, GNUNET_GNSRECORD_TYPE_GNS2DNS, rec->expiration_time, dst, off); gc->found = GNUNET_YES; } break; case GNUNET_DNSPARSER_TYPE_AAAA: if (sizeof (struct in6_addr) != rec->data.raw.data_len) { GNUNET_break (0); return; } if (NULL == inet_ntop (AF_INET6, rec->data.raw.data, ip, ip_size)) { GNUNET_break (0); return; } if ( (GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst, dst_len, &off, gc->req->hostname)) && (GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst, dst_len, &off, ip)) ) { add_record (gc->req, GNUNET_GNSRECORD_TYPE_GNS2DNS, rec->expiration_time, dst, off); gc->found = GNUNET_YES; } break; case GNUNET_DNSPARSER_TYPE_CNAME: if ( (GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst, dst_len, &off, gc->req->hostname)) && (GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst, dst_len, &off, rec->data.hostname)) ) { add_record (gc->req, GNUNET_GNSRECORD_TYPE_GNS2DNS, rec->expiration_time, dst, off); gc->found = GNUNET_YES; } break; default: /* useless, do nothing */ break; } } /** * Closure for #process_record(). */ struct ProcessRecordContext { /** * Answer we got back and are currently parsing, or NULL * if not active. */ struct GNUNET_DNSPARSER_Packet *p; /** * Request we are processing. */ struct Request *req; }; /** * We received @a rec for @a req. Remember the answer. * * @param cls a `struct ProcessRecordContext` * @param rec response */ static void process_record (void *cls, const struct GNUNET_DNSPARSER_Record *rec) { struct ProcessRecordContext *prc = cls; struct Request *req = prc->req; char dst[65536]; size_t dst_len; size_t off; dst_len = sizeof (dst); off = 0; records++; if (0 != strcasecmp (rec->name, req->hostname)) { GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "DNS returned record from zone `%s' of type %u while resolving `%s'\n", rec->name, (unsigned int) rec->type, req->hostname); return; /* does not match hostname, might be glue, but not useful for this pass! */ } if (0 == GNUNET_TIME_absolute_get_remaining (rec->expiration_time).rel_value_us) { GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "DNS returned expired record for `%s'\n", req->hostname); return; /* record expired */ } GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "DNS returned record that expires at %s for `%s'\n", GNUNET_STRINGS_absolute_time_to_string (rec->expiration_time), req->hostname); switch (rec->type) { case GNUNET_DNSPARSER_TYPE_NS: { struct GlueClosure gc; /* check for glue */ gc.req = req; gc.ns = rec->data.hostname; gc.found = GNUNET_NO; for_all_records (prc->p, &check_for_glue, &gc); if ( (GNUNET_NO == gc.found) && (GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst, dst_len, &off, req->hostname)) && (GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst, dst_len, &off, rec->data.hostname)) ) { /* FIXME: actually check if this is out-of-bailiwick, and if not request explicit resolution... */ GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Converted OOB (`%s') NS record for `%s'\n", rec->data.hostname, rec->name); add_record (req, GNUNET_GNSRECORD_TYPE_GNS2DNS, rec->expiration_time, dst, off); } else { GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Converted NS record for `%s' using glue\n", rec->name); } break; } case GNUNET_DNSPARSER_TYPE_CNAME: if (GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst, dst_len, &off, rec->data.hostname)) { GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Converting CNAME (`%s') record for `%s'\n", rec->data.hostname, rec->name); add_record (req, rec->type, rec->expiration_time, dst, off); } break; case GNUNET_DNSPARSER_TYPE_DNAME: /* No support for DNAME in GNS yet! FIXME: support later! */ GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "FIXME: not supported: %s DNAME %s\n", rec->name, rec->data.hostname); break; case GNUNET_DNSPARSER_TYPE_MX: if (GNUNET_OK == GNUNET_DNSPARSER_builder_add_mx (dst, dst_len, &off, rec->data.mx)) { GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Converting MX (`%s') record for `%s'\n", rec->data.mx->mxhost, rec->name); add_record (req, rec->type, rec->expiration_time, dst, off); } break; case GNUNET_DNSPARSER_TYPE_SOA: if (GNUNET_OK == GNUNET_DNSPARSER_builder_add_soa (dst, dst_len, &off, rec->data.soa)) { /* NOTE: GNS does not really use SOAs */ GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Converting SOA record for `%s'\n", rec->name); add_record (req, rec->type, rec->expiration_time, dst, off); } break; case GNUNET_DNSPARSER_TYPE_SRV: if (GNUNET_OK == GNUNET_DNSPARSER_builder_add_srv (dst, dst_len, &off, rec->data.srv)) { GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Converting SRV record for `%s'\n", rec->name); add_record (req, rec->type, rec->expiration_time, dst, off); } break; case GNUNET_DNSPARSER_TYPE_PTR: if (GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst, dst_len, &off, rec->data.hostname)) { /* !?: what does a PTR record do in a regular TLD??? */ GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Converting PTR record for `%s' (weird)\n", rec->name); add_record (req, rec->type, rec->expiration_time, dst, off); } break; case GNUNET_DNSPARSER_TYPE_CERT: if (GNUNET_OK == GNUNET_DNSPARSER_builder_add_cert (dst, dst_len, &off, rec->data.cert)) { GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Converting CERT record for `%s'\n", rec->name); add_record (req, rec->type, rec->expiration_time, dst, off); } break; /* Rest is 'raw' encoded and just needs to be copied IF the hostname matches the requested name; otherwise we simply cannot use it. */ case GNUNET_DNSPARSER_TYPE_A: case GNUNET_DNSPARSER_TYPE_AAAA: case GNUNET_DNSPARSER_TYPE_TXT: default: GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Converting record of type %u for `%s'\n", (unsigned int) rec->type, rec->name); add_record (req, rec->type, rec->expiration_time, rec->data.raw.data, rec->data.raw.data_len); break; } } /** * Continuation called to notify client about result of the * operation. * * @param cls closure with our `struct Request` * @param success #GNUNET_SYSERR on failure (including timeout/queue drop/failure to validate) * #GNUNET_NO if content was already there or not found * #GNUNET_YES (or other positive value) on success * @param emsg NULL on success, otherwise an error message */ static void store_completed_cb (void *cls, int32_t success, const char *emsg) { static struct GNUNET_TIME_Absolute last; static unsigned int pdot; struct Request *req = cls; req->qe = NULL; pending_rs--; if (NULL == t) t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY, &process_queue, NULL); if (GNUNET_SYSERR == success) { GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Failed to store zone data for `%s': %s\n", req->hostname, emsg); } else { GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Stored records under `%s'\n", req->hostname); if (0 == pdot) last = GNUNET_TIME_absolute_get (); pdot++; if (0 == pdot % 1000) { struct GNUNET_TIME_Relative delta; delta = GNUNET_TIME_absolute_get_duration (last); last = GNUNET_TIME_absolute_get (); fprintf (stderr, "Processed 1000 records in %s\n", GNUNET_STRINGS_relative_time_to_string (delta, GNUNET_YES)); } } free_records (req); } /** * Function called with the result of a DNS resolution. * * @param cls closure with the `struct Request` * @param dns dns response, never NULL * @param dns_len number of bytes in @a dns */ static void process_result (void *cls, const struct GNUNET_TUN_DnsHeader *dns, size_t dns_len) { struct Request *req = cls; struct Record *rec; struct GNUNET_DNSPARSER_Packet *p; unsigned int rd_count; GNUNET_assert (NULL == req->hn); if (NULL == dns) { /* stub gave up */ GNUNET_CONTAINER_DLL_remove (req_head, req_tail, req); pending--; if (NULL == t) t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY, &process_queue, NULL); GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Stub gave up on DNS reply for `%s'\n", req->hostname); if (req->issue_num > MAX_RETRIES) { failures++; free_request (req); return; } req->rs = NULL; insert_sorted (req); return; } if (req->id != dns->id) { GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "DNS ID did not match request, ignoring reply\n"); return; } GNUNET_CONTAINER_DLL_remove (req_head, req_tail, req); GNUNET_DNSSTUB_resolve_cancel (req->rs); req->rs = NULL; pending--; p = GNUNET_DNSPARSER_parse ((const char *) dns, dns_len); if (NULL == p) { GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Failed to parse DNS reply for `%s'\n", req->hostname); if (req->issue_num > MAX_RETRIES) { failures++; insert_sorted (req); if (NULL == t) t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY, &process_queue, NULL); return; } insert_sorted (req); if (NULL == t) t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY, &process_queue, NULL); return; } /* import new records */ req->issue_num = 0; /* success, reset counter! */ { struct ProcessRecordContext prc = { .req = req, .p = p }; for_all_records (p, &process_record, &prc); } GNUNET_DNSPARSER_free_packet (p); /* count records found, determine minimum expiration time */ req->expires = GNUNET_TIME_UNIT_FOREVER_ABS; rd_count = 0; for (rec = req->rec_head; NULL != rec; rec = rec->next) { struct GNUNET_TIME_Absolute at; at.abs_value_us = rec->grd.expiration_time; req->expires = GNUNET_TIME_absolute_min (req->expires, at); rd_count++; } GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Obtained %u records for `%s'\n", rd_count, req->hostname); /* Instead of going for SOA, simplified for now to look each day in case we got an empty response */ if (0 == rd_count) req->expires = GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_DAYS); /* convert records to namestore import format */ { struct GNUNET_GNSRECORD_Data rd[GNUNET_NZL(rd_count)]; unsigned int off = 0; /* convert linked list into array */ for (rec = req->rec_head; NULL != rec; rec =rec->next) rd[off++] = rec->grd; pending_rs++; req->qe = GNUNET_NAMESTORE_records_store (ns, &req->zone->key, get_label (req), rd_count, rd, &store_completed_cb, req); GNUNET_assert (NULL != req->qe); } insert_sorted (req); } /** * Process as many requests as possible from the queue. * * @param cls NULL */ static void process_queue (void *cls) { struct Request *req; unsigned int series; void *raw; size_t raw_size; (void) cls; series = 0; t = NULL; while (pending + pending_rs < THRESH) { req = GNUNET_CONTAINER_heap_peek (req_heap); if (NULL == req) break; if (NULL != req->qe) return; /* namestore op still pending */ if (NULL != req->rs) { GNUNET_break (0); return; /* already submitted */ } if (GNUNET_TIME_absolute_get_remaining (req->expires).rel_value_us > 0) break; GNUNET_assert (req == GNUNET_CONTAINER_heap_remove_root (req_heap)); req->hn = NULL; GNUNET_CONTAINER_DLL_insert (req_head, req_tail, req); GNUNET_assert (NULL == req->rs); GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Requesting resolution for `%s'\n", req->hostname); raw = build_dns_query (req, &raw_size); if (NULL == raw) { GNUNET_break (0); free_request (req); continue; } req->rs = GNUNET_DNSSTUB_resolve (ctx, raw, raw_size, &process_result, req); GNUNET_assert (NULL != req->rs); req->issue_num++; lookups++; pending++; series++; if (series > MAX_SERIES) break; } if (pending + pending_rs >= THRESH) return; /* wait for replies */ req = GNUNET_CONTAINER_heap_peek (req_heap); if (NULL == req) return; if (GNUNET_TIME_absolute_get_remaining (req->expires).rel_value_us > 0) { GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Waiting until %s for next record (`%s') to expire\n", GNUNET_STRINGS_absolute_time_to_string (req->expires), req->hostname); if (NULL != t) GNUNET_SCHEDULER_cancel (t); t = GNUNET_SCHEDULER_add_at (req->expires, &process_queue, NULL); return; } GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Throttling\n"); if (NULL != t) GNUNET_SCHEDULER_cancel (t); t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY, &process_queue, NULL); } /** * Iterator called during #do_shutdown() to free requests in * the #ns_pending map. * * @param cls NULL * @param key unused * @param value the `struct Request` to free * @return #GNUNET_OK */ static int free_request_it (void *cls, const struct GNUNET_HashCode *key, void *value) { struct Request *req = value; (void) cls; (void) key; free_request (req); return GNUNET_OK; } /** * Clean up and terminate the process. * * @param cls NULL */ static void do_shutdown (void *cls) { struct Request *req; struct Zone *zone; (void) cls; if (NULL != id) { GNUNET_IDENTITY_disconnect (id); id = NULL; } if (NULL != t) { GNUNET_SCHEDULER_cancel (t); t = NULL; } while (NULL != (req = req_head)) { GNUNET_CONTAINER_DLL_remove (req_head, req_tail, req); if (NULL != req->qe) GNUNET_NAMESTORE_cancel (req->qe); free_request (req); } while (NULL != (req = GNUNET_CONTAINER_heap_remove_root (req_heap))) { req->hn = NULL; if (NULL != req->qe) GNUNET_NAMESTORE_cancel (req->qe); free_request (req); } if (NULL != zone_it) { GNUNET_NAMESTORE_zone_iteration_stop (zone_it); zone_it = NULL; } if (NULL != ns) { GNUNET_NAMESTORE_disconnect (ns); ns = NULL; } if (NULL != ctx) { GNUNET_DNSSTUB_stop (ctx); ctx = NULL; } if (NULL != req_heap) { GNUNET_CONTAINER_heap_destroy (req_heap); req_heap = NULL; } if (NULL != ns_pending) { GNUNET_CONTAINER_multihashmap_iterate (ns_pending, &free_request_it, NULL); GNUNET_CONTAINER_multihashmap_destroy (ns_pending); ns_pending = NULL; } while (NULL != (zone = zone_head)) { GNUNET_CONTAINER_DLL_remove (zone_head, zone_tail, zone); GNUNET_free (zone->domain); GNUNET_free (zone); } } /** * Function called if #GNUNET_NAMESTORE_records_lookup() failed. * Just logs an error. * * @param cls a `struct Zone` */ static void ns_lookup_error_cb (void *cls) { struct Zone *zone = cls; GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Failed to load data from namestore for zone `%s'\n", zone->domain); } /** * Process a record that was stored in the namestore. * * @param cls a `struct Zone *` * @param key private key of the zone * @param label label of the records * @param rd_count number of entries in @a rd array, 0 if label was deleted * @param rd array of records with data to store */ static void ns_lookup_result_cb (void *cls, const struct GNUNET_CRYPTO_EcdsaPrivateKey *key, const char *label, unsigned int rd_count, const struct GNUNET_GNSRECORD_Data *rd) { struct Zone *zone = cls; struct Request *req; struct GNUNET_HashCode hc; char *fqdn; ns_iterator_trigger_next--; if (0 == ns_iterator_trigger_next) { ns_iterator_trigger_next = NS_BATCH_SIZE; GNUNET_NAMESTORE_zone_iterator_next (zone_it, ns_iterator_trigger_next); } GNUNET_asprintf (&fqdn, "%s.%s", label, zone->domain); GNUNET_CRYPTO_hash (fqdn, strlen (fqdn) + 1, &hc); GNUNET_free (fqdn); req = GNUNET_CONTAINER_multihashmap_get (ns_pending, &hc); if (NULL == req) { GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Ignoring record `%s' in zone `%s': not on my list!\n", label, zone->domain); return; } GNUNET_assert (GNUNET_OK == GNUNET_CONTAINER_multihashmap_remove (ns_pending, &hc, req)); GNUNET_break (0 == memcmp (key, &req->zone->key, sizeof (*key))); GNUNET_break (0 == strcasecmp (label, get_label (req))); for (unsigned int i=0;iexpiration_time; add_record (req, rd->record_type, at, rd->data, rd->data_size); } if (0 == rd_count) { GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Empty record set in namestore for `%s'\n", req->hostname); } else { unsigned int pos = 0; req->expires = GNUNET_TIME_UNIT_FOREVER_ABS; for (struct Record *rec = req->rec_head; NULL != rec; rec = rec->next) { struct GNUNET_TIME_Absolute at; at.abs_value_us = rec->grd.expiration_time; req->expires = GNUNET_TIME_absolute_min (req->expires, at); pos++; } if (0 == pos) req->expires = GNUNET_TIME_UNIT_ZERO_ABS; GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Hot-start with %u existing records for `%s'\n", pos, req->hostname); } free_records (req); GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Adding `%s' to worklist to start at %s\n", req->hostname, GNUNET_STRINGS_absolute_time_to_string (req->expires)); insert_sorted (req); } /** * Add @a hostname to the list of requests to be made. * * @param hostname name to resolve */ static void queue (const char *hostname) { struct Request *req; const char *dot; struct Zone *zone; size_t hlen; struct GNUNET_HashCode hc; if (GNUNET_OK != GNUNET_DNSPARSER_check_name (hostname)) { GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Refusing invalid hostname `%s'\n", hostname); rejects++; return; } dot = strchr (hostname, (unsigned char) '.'); if (NULL == dot) { GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Refusing invalid hostname `%s' (lacks '.')\n", hostname); rejects++; return; } for (zone = zone_head; NULL != zone; zone = zone->next) if (0 == strcmp (zone->domain, dot + 1)) break; if (NULL == zone) { rejects++; GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Domain name `%s' not in ego list!\n", dot + 1); return; } hlen = strlen (hostname) + 1; req = GNUNET_malloc (sizeof (struct Request) + hlen); req->zone = zone; req->hostname = (char *) &req[1]; memcpy (req->hostname, hostname, hlen); req->id = (uint16_t) GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE, UINT16_MAX); GNUNET_CRYPTO_hash (req->hostname, hlen, &hc); if (GNUNET_OK != GNUNET_CONTAINER_multihashmap_put (ns_pending, &hc, req, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)) { GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Duplicate hostname `%s' ignored\n", hostname); GNUNET_free (req); return; } } /** * We have completed the initial iteration over the namestore's database. * This function is called on each of the remaining records in * #move_to_queue to #queue() them, as we will simply not find existing * records for them any longer. * * @param cls NULL * @param key unused * @param value a `struct Request` * @return #GNUNET_OK (continue to iterate) */ static int move_to_queue (void *cls, const struct GNUNET_HashCode *key, void *value) { struct Request *req = value; (void) cls; (void) key; insert_sorted (req); return GNUNET_OK; } /** * Iterate over all of the zones we care about and see which records * we may need to re-fetch when. * * @param cls NULL */ static void iterate_zones (void *cls) { static struct Zone *last; (void) cls; if (NULL != zone_it) { zone_it = NULL; GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Finished iteration over zone `%s'!\n", last->domain); } GNUNET_assert (NULL != zone_tail); if (zone_tail == last) { /* Done iterating over relevant zones in NAMESTORE, move rest of hash map to work queue as well. */ GNUNET_CONTAINER_multihashmap_iterate (ns_pending, &move_to_queue, NULL); GNUNET_CONTAINER_multihashmap_destroy (ns_pending); ns_pending = NULL; return; } if (NULL == last) last = zone_head; else last = last->next; GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Starting iteration over zone `%s'!\n", last->domain); ns_iterator_trigger_next = 1; zone_it = GNUNET_NAMESTORE_zone_iteration_start (ns, &last->key, &ns_lookup_error_cb, NULL, &ns_lookup_result_cb, last, &iterate_zones, NULL); } /** * Begin processing hostnames from stdin. * * @param cls NULL */ static void process_stdin (void *cls) { static struct GNUNET_TIME_Absolute last; static unsigned int pdot; char hn[256]; (void) cls; t = NULL; if (NULL != id) { GNUNET_IDENTITY_disconnect (id); id = NULL; } while (NULL != fgets (hn, sizeof (hn), stdin)) { if (strlen(hn) > 0) hn[strlen(hn)-1] = '\0'; /* eat newline */ if (0 == pdot) last = GNUNET_TIME_absolute_get (); pdot++; if (0 == pdot % 1000) { struct GNUNET_TIME_Relative delta; delta = GNUNET_TIME_absolute_get_duration (last); last = GNUNET_TIME_absolute_get (); fprintf (stderr, "Imported 1000 records in %s\n", GNUNET_STRINGS_relative_time_to_string (delta, GNUNET_YES)); } queue (hn); } fprintf (stderr, "\n"); iterate_zones (NULL); } /** * Method called to inform about the egos of this peer. * * When used with #GNUNET_IDENTITY_connect, this function is * initially called for all egos and then again whenever a * ego's name changes or if it is deleted. At the end of * the initial pass over all egos, the function is once called * with 'NULL' for @a ego. That does NOT mean that the callback won't * be invoked in the future or that there was an error. * * When used with #GNUNET_IDENTITY_create or #GNUNET_IDENTITY_get, * this function is only called ONCE, and 'NULL' being passed in * @a ego does indicate an error (i.e. name is taken or no default * value is known). If @a ego is non-NULL and if '*ctx' * is set in those callbacks, the value WILL be passed to a subsequent * call to the identity callback of #GNUNET_IDENTITY_connect (if * that one was not NULL). * * When an identity is renamed, this function is called with the * (known) @a ego but the NEW @a name. * * When an identity is deleted, this function is called with the * (known) ego and "NULL" for the @a name. In this case, * the @a ego is henceforth invalid (and the @a ctx should also be * cleaned up). * * @param cls closure * @param ego ego handle * @param ctx context for application to store data for this ego * (during the lifetime of this process, initially NULL) * @param name name assigned by the user for this ego, * NULL if the user just deleted the ego and it * must thus no longer be used */ static void identity_cb (void *cls, struct GNUNET_IDENTITY_Ego *ego, void **ctx, const char *name) { (void) cls; (void) ctx; if (NULL == ego) { if (NULL != zone_head) { t = GNUNET_SCHEDULER_add_now (&process_stdin, NULL); } else { GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "No zone found\n"); GNUNET_SCHEDULER_shutdown (); return; } } if (NULL != name) { struct Zone *zone; zone = GNUNET_new (struct Zone); zone->key = *GNUNET_IDENTITY_ego_get_private_key (ego); zone->domain = GNUNET_strdup (name); GNUNET_CONTAINER_DLL_insert (zone_head, zone_tail, zone); } } /** * Process requests from the queue, then if the queue is * not empty, try again. * * @param cls NULL * @param args remaining command-line arguments * @param cfgfile name of the configuration file used (for saving, can be NULL!) * @param cfg configuration */ static void run (void *cls, char *const *args, const char *cfgfile, const struct GNUNET_CONFIGURATION_Handle *cfg) { (void) cls; (void) args; (void) cfgfile; req_heap = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN); ns_pending = GNUNET_CONTAINER_multihashmap_create (1024, GNUNET_NO); ctx = GNUNET_DNSSTUB_start (256); if (NULL == ctx) { fprintf (stderr, "Failed to initialize GNUnet DNS STUB\n"); return; } if (NULL == args[0]) { fprintf (stderr, "You must provide a list of DNS resolvers on the command line\n"); return; } for (unsigned int i=0;NULL != args[i];i++) { if (GNUNET_OK != GNUNET_DNSSTUB_add_dns_ip (ctx, args[i])) { fprintf (stderr, "Failed to use `%s' for DNS resolver\n", args[i]); return; } } GNUNET_SCHEDULER_add_shutdown (&do_shutdown, NULL); ns = GNUNET_NAMESTORE_connect (cfg); if (NULL == ns) { GNUNET_SCHEDULER_shutdown (); return; } id = GNUNET_IDENTITY_connect (cfg, &identity_cb, NULL); } /** * Call with IP address of resolver to query. * * @param argc should be 2 * @param argv[1] should contain IP address * @return 0 on success */ int main (int argc, char *const*argv) { struct GNUNET_GETOPT_CommandLineOption options[] = { GNUNET_GETOPT_OPTION_END }; if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv)) return 2; GNUNET_PROGRAM_run (argc, argv, "gnunet-zoneimport", "import DNS zone into namestore", options, &run, NULL); GNUNET_free ((void*) argv); fprintf (stderr, "Rejected %u names, did %u lookups, found %u records, %u lookups failed, %u/%u pending on shutdown\n", rejects, lookups, records, failures, pending, pending_rs); return 0; } /* end of gnunet-zoneimport.c */