aboutsummaryrefslogtreecommitdiff
path: root/src/cli/namestore/gnunet-zoneimport.c
diff options
context:
space:
mode:
Diffstat (limited to 'src/cli/namestore/gnunet-zoneimport.c')
-rw-r--r--src/cli/namestore/gnunet-zoneimport.c1872
1 files changed, 1872 insertions, 0 deletions
diff --git a/src/cli/namestore/gnunet-zoneimport.c b/src/cli/namestore/gnunet-zoneimport.c
new file mode 100644
index 000000000..c7e0cf65f
--- /dev/null
+++ b/src/cli/namestore/gnunet-zoneimport.c
@@ -0,0 +1,1872 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2018 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20/**
21 * @file src/namestore/gnunet-zoneimport.c
22 * @brief import a DNS zone for publication in GNS, incremental
23 * @author Christian Grothoff
24 */
25#include "platform.h"
26#include <gnunet_util_lib.h>
27#include <gnunet_gnsrecord_lib.h>
28#include <gnunet_namestore_service.h>
29#include <gnunet_statistics_service.h>
30#include <gnunet_identity_service.h>
31
32
33/**
34 * Maximum number of queries pending at the same time.
35 */
36#define THRESH 100
37
38/**
39 * TIME_THRESH is in usecs. How quickly do we submit fresh queries.
40 * Used as an additional throttle.
41 */
42#define TIME_THRESH 10
43
44/**
45 * How often do we retry a query before giving up for good?
46 */
47#define MAX_RETRIES 5
48
49/**
50 * How many DNS requests do we at most issue in rapid series?
51 */
52#define MAX_SERIES 10
53
54/**
55 * How long do we wait at least between series of requests?
56 */
57#define SERIES_DELAY \
58 GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MICROSECONDS, 10)
59
60/**
61 * How long do DNS records have to last at least after being imported?
62 */
63static struct GNUNET_TIME_Relative minimum_expiration_time;
64
65/**
66 * How many requests do we request from NAMESTORE in one batch
67 * during our initial iteration?
68 */
69#define NS_BATCH_SIZE 1024
70
71/**
72 * Some zones may include authoritative records for other
73 * zones, such as foo.com.uk or bar.com.fr. As for GNS
74 * each dot represents a zone cut, we then need to create a
75 * zone on-the-fly to capture those records properly.
76 */
77struct Zone
78{
79 /**
80 * Kept in a DLL.
81 */
82 struct Zone *next;
83
84 /**
85 * Kept in a DLL.
86 */
87 struct Zone *prev;
88
89 /**
90 * Domain of the zone (i.e. "fr" or "com.fr")
91 */
92 char *domain;
93
94 /**
95 * Private key of the zone.
96 */
97 struct GNUNET_CRYPTO_PrivateKey key;
98};
99
100
101/**
102 * Record for the request to be stored by GNS.
103 */
104struct Record
105{
106 /**
107 * Kept in a DLL.
108 */
109 struct Record *next;
110
111 /**
112 * Kept in a DLL.
113 */
114 struct Record *prev;
115
116 /**
117 * GNS record.
118 */
119 struct GNUNET_GNSRECORD_Data grd;
120};
121
122
123/**
124 * Request we should make. We keep this struct in memory per request,
125 * thus optimizing it is crucial for the overall memory consumption of
126 * the zone importer.
127 */
128struct Request
129{
130 /**
131 * Requests are kept in a heap while waiting to be resolved.
132 */
133 struct GNUNET_CONTAINER_HeapNode *hn;
134
135 /**
136 * Active requests are kept in a DLL.
137 */
138 struct Request *next;
139
140 /**
141 * Active requests are kept in a DLL.
142 */
143 struct Request *prev;
144
145 /**
146 * Head of records that should be published in GNS for
147 * this hostname.
148 */
149 struct Record *rec_head;
150
151 /**
152 * Tail of records that should be published in GNS for
153 * this hostname.
154 */
155 struct Record *rec_tail;
156
157 /**
158 * Socket used to make the request, NULL if not active.
159 */
160 struct GNUNET_DNSSTUB_RequestSocket *rs;
161
162 /**
163 * Hostname we are resolving, allocated at the end of
164 * this struct (optimizing memory consumption by reducing
165 * total number of allocations).
166 */
167 char *hostname;
168
169 /**
170 * Namestore operation pending for this record.
171 */
172 struct GNUNET_NAMESTORE_QueueEntry *qe;
173
174 /**
175 * Zone responsible for this request.
176 */
177 const struct Zone *zone;
178
179 /**
180 * At what time does the (earliest) of the returned records
181 * for this name expire? At this point, we need to re-fetch
182 * the record.
183 */
184 struct GNUNET_TIME_Absolute expires;
185
186 /**
187 * While we are fetching the record, the value is set to the
188 * starting time of the DNS operation. While doing a
189 * NAMESTORE store, again set to the start time of the
190 * NAMESTORE operation.
191 */
192 struct GNUNET_TIME_Absolute op_start_time;
193
194 /**
195 * How often did we issue this query? (And failed, reset
196 * to zero once we were successful.)
197 */
198 unsigned int issue_num;
199
200 /**
201 * random 16-bit DNS query identifier.
202 */
203 uint16_t id;
204};
205
206
207/**
208 * Command-line argument specifying desired size of the hash map with
209 * all of our pending names. Usually, we use an automatically growing
210 * map, but this is only OK up to about a million entries. Above that
211 * number, the user must explicitly specify the size at startup.
212 */
213static unsigned int map_size = 1024;
214
215/**
216 * Handle to the identity service.
217 */
218static struct GNUNET_IDENTITY_Handle *id;
219
220/**
221 * Namestore handle.
222 */
223static struct GNUNET_NAMESTORE_Handle *ns;
224
225/**
226 * Handle to the statistics service.
227 */
228static struct GNUNET_STATISTICS_Handle *stats;
229
230/**
231 * Context for DNS resolution.
232 */
233static struct GNUNET_DNSSTUB_Context *ctx;
234
235/**
236 * The number of DNS queries that are outstanding
237 */
238static unsigned int pending;
239
240/**
241 * The number of NAMESTORE record store operations that are outstanding
242 */
243static unsigned int pending_rs;
244
245/**
246 * Number of lookups we performed overall.
247 */
248static unsigned int lookups;
249
250/**
251 * Number of records we had cached.
252 */
253static unsigned int cached;
254
255/**
256 * How many hostnames did we reject (malformed).
257 */
258static unsigned int rejects;
259
260/**
261 * Number of lookups that failed.
262 */
263static unsigned int failures;
264
265/**
266 * Number of records we found.
267 */
268static unsigned int records;
269
270/**
271 * Number of record sets given to namestore.
272 */
273static unsigned int record_sets;
274
275/**
276 * Heap of all requests to perform, sorted by
277 * the time we should next do the request (i.e. by expires).
278 */
279static struct GNUNET_CONTAINER_Heap *req_heap;
280
281/**
282 * Active requests are kept in a DLL.
283 */
284static struct Request *req_head;
285
286/**
287 * Active requests are kept in a DLL.
288 */
289static struct Request *req_tail;
290
291/**
292 * Main task.
293 */
294static struct GNUNET_SCHEDULER_Task *t;
295
296/**
297 * Hash map of requests for which we may still get a response from
298 * the namestore. Set to NULL once the initial namestore iteration
299 * is done.
300 */
301static struct GNUNET_CONTAINER_MultiHashMap *ns_pending;
302
303/**
304 * Current zone iteration handle.
305 */
306static struct GNUNET_NAMESTORE_ZoneIterator *zone_it;
307
308/**
309 * Head of list of zones we are managing.
310 */
311static struct Zone *zone_head;
312
313/**
314 * Tail of list of zones we are managing.
315 */
316static struct Zone *zone_tail;
317
318/**
319 * After how many more results must #ns_lookup_result_cb() ask
320 * the namestore for more?
321 */
322static uint64_t ns_iterator_trigger_next;
323
324/**
325 * Number of DNS requests counted in latency total.
326 */
327static uint64_t total_dns_latency_cnt;
328
329/**
330 * Sum of DNS latencies observed.
331 */
332static struct GNUNET_TIME_Relative total_dns_latency;
333
334/**
335 * Number of records processed (DNS lookup, no NAMESTORE) in total.
336 */
337static uint64_t total_reg_proc_dns;
338
339/**
340 * Number of records processed (DNS lookup, with NAMESTORE) in total.
341 */
342static uint64_t total_reg_proc_dns_ns;
343
344/**
345 * Start time of the regular processing.
346 */
347static struct GNUNET_TIME_Absolute start_time_reg_proc;
348
349/**
350 * Last time we worked before going idle.
351 */
352static struct GNUNET_TIME_Absolute sleep_time_reg_proc;
353
354/**
355 * Time we slept just waiting for work.
356 */
357static struct GNUNET_TIME_Relative idle_time;
358
359
360/**
361 * Callback for #for_all_records
362 *
363 * @param cls closure
364 * @param rec a DNS record
365 */
366typedef void (*RecordProcessor) (void *cls,
367 const struct GNUNET_DNSPARSER_Record *rec);
368
369
370/**
371 * Call @a rp for each record in @a p, regardless of
372 * what response section it is in.
373 *
374 * @param p packet from DNS
375 * @param rp function to call
376 * @param rp_cls closure for @a rp
377 */
378static void
379for_all_records (const struct GNUNET_DNSPARSER_Packet *p,
380 RecordProcessor rp,
381 void *rp_cls)
382{
383 for (unsigned int i = 0; i < p->num_answers; i++)
384 {
385 struct GNUNET_DNSPARSER_Record *rs = &p->answers[i];
386
387 rp (rp_cls, rs);
388 }
389 for (unsigned int i = 0; i < p->num_authority_records; i++)
390 {
391 struct GNUNET_DNSPARSER_Record *rs = &p->authority_records[i];
392
393 rp (rp_cls, rs);
394 }
395 for (unsigned int i = 0; i < p->num_additional_records; i++)
396 {
397 struct GNUNET_DNSPARSER_Record *rs = &p->additional_records[i];
398
399 rp (rp_cls, rs);
400 }
401}
402
403
404/**
405 * Return just the label of the hostname in @a req.
406 *
407 * @param req request to process hostname of
408 * @return statically allocated pointer to the label,
409 * overwritten upon the next request!
410 */
411static const char *
412get_label (struct Request *req)
413{
414 static char label[64];
415 const char *dot;
416
417 dot = strchr (req->hostname, (unsigned char) '.');
418 if (NULL == dot)
419 {
420 GNUNET_break (0);
421 return NULL;
422 }
423 if (((size_t) (dot - req->hostname)) >= sizeof(label))
424 {
425 GNUNET_break (0);
426 return NULL;
427 }
428 GNUNET_memcpy (label, req->hostname, dot - req->hostname);
429 label[dot - req->hostname] = '\0';
430 return label;
431}
432
433
434/**
435 * Build DNS query for @a hostname.
436 *
437 * @param hostname host to build query for
438 * @param[out] raw_size number of bytes in the query
439 * @return NULL on error, otherwise pointer to statically (!)
440 * allocated query buffer
441 */
442static void *
443build_dns_query (struct Request *req, size_t *raw_size)
444{
445 static char raw[512];
446 char *rawp;
447 struct GNUNET_DNSPARSER_Packet p;
448 struct GNUNET_DNSPARSER_Query q;
449 int ret;
450
451 q.name = (char *) req->hostname;
452 q.type = GNUNET_DNSPARSER_TYPE_NS;
453 q.dns_traffic_class = GNUNET_TUN_DNS_CLASS_INTERNET;
454
455 memset (&p, 0, sizeof(p));
456 p.num_queries = 1;
457 p.queries = &q;
458 p.id = req->id;
459 ret = GNUNET_DNSPARSER_pack (&p, UINT16_MAX, &rawp, raw_size);
460 if (GNUNET_OK != ret)
461 {
462 if (GNUNET_NO == ret)
463 GNUNET_free (rawp);
464 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
465 "Failed to pack query for hostname `%s'\n",
466 req->hostname);
467 rejects++;
468 return NULL;
469 }
470 if (*raw_size > sizeof(raw))
471 {
472 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
473 "Failed to pack query for hostname `%s'\n",
474 req->hostname);
475 rejects++;
476 GNUNET_break (0);
477 GNUNET_free (rawp);
478 return NULL;
479 }
480 GNUNET_memcpy (raw, rawp, *raw_size);
481 GNUNET_free (rawp);
482 return raw;
483}
484
485
486/**
487 * Free records associated with @a req.
488 *
489 * @param req request to free records of
490 */
491static void
492free_records (struct Request *req)
493{
494 struct Record *rec;
495
496 /* Free records */
497 while (NULL != (rec = req->rec_head))
498 {
499 GNUNET_CONTAINER_DLL_remove (req->rec_head, req->rec_tail, rec);
500 GNUNET_free (rec);
501 }
502}
503
504
505/**
506 * Free @a req and data structures reachable from it.
507 *
508 * @param req request to free
509 */
510static void
511free_request (struct Request *req)
512{
513 free_records (req);
514 GNUNET_free (req);
515}
516
517
518/**
519 * Process as many requests as possible from the queue.
520 *
521 * @param cls NULL
522 */
523static void
524process_queue (void *cls);
525
526
527/**
528 * Insert @a req into DLL sorted by next fetch time.
529 *
530 * @param req request to insert into #req_heap
531 */
532static void
533insert_sorted (struct Request *req)
534{
535 req->hn =
536 GNUNET_CONTAINER_heap_insert (req_heap, req, req->expires.abs_value_us);
537 if (req == GNUNET_CONTAINER_heap_peek (req_heap))
538 {
539 if (NULL != t)
540 GNUNET_SCHEDULER_cancel (t);
541 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
542 t = GNUNET_SCHEDULER_add_at (req->expires, &process_queue, NULL);
543 }
544}
545
546
547/**
548 * Add record to the GNS record set for @a req.
549 *
550 * @param req the request to expand GNS record set for
551 * @param type type to use
552 * @param expiration_time when should @a rec expire
553 * @param data raw data to store
554 * @param data_len number of bytes in @a data
555 */
556static void
557add_record (struct Request *req,
558 uint32_t type,
559 struct GNUNET_TIME_Absolute expiration_time,
560 const void *data,
561 size_t data_len)
562{
563 struct Record *rec;
564
565 rec = GNUNET_malloc (sizeof(struct Record) + data_len);
566 rec->grd.data = &rec[1];
567 rec->grd.expiration_time = expiration_time.abs_value_us;
568 rec->grd.data_size = data_len;
569 rec->grd.record_type = type;
570 rec->grd.flags = GNUNET_GNSRECORD_RF_NONE;
571 GNUNET_memcpy (&rec[1], data, data_len);
572 GNUNET_CONTAINER_DLL_insert (req->rec_head, req->rec_tail, rec);
573}
574
575
576/**
577 * Closure for #check_for_glue.
578 */
579struct GlueClosure
580{
581 /**
582 * Overall request we are processing.
583 */
584 struct Request *req;
585
586 /**
587 * NS name we are looking for glue for.
588 */
589 const char *ns;
590
591 /**
592 * Set to #GNUNET_YES if glue was found.
593 */
594 int found;
595};
596
597
598/**
599 * Try to find glue records for a given NS record.
600 *
601 * @param cls a `struct GlueClosure *`
602 * @param rec record that may contain glue information
603 */
604static void
605check_for_glue (void *cls, const struct GNUNET_DNSPARSER_Record *rec)
606{
607 struct GlueClosure *gc = cls;
608 char dst[65536];
609 size_t dst_len;
610 size_t off;
611 char ip[INET6_ADDRSTRLEN + 1];
612 socklen_t ip_size = (socklen_t) sizeof(ip);
613 struct GNUNET_TIME_Absolute expiration_time;
614 struct GNUNET_TIME_Relative left;
615
616 if (0 != strcasecmp (rec->name, gc->ns))
617 return;
618 expiration_time = rec->expiration_time;
619 left = GNUNET_TIME_absolute_get_remaining (expiration_time);
620 if (0 == left.rel_value_us)
621 return; /* ignore expired glue records */
622 /* if expiration window is too short, bump it to configured minimum */
623 if (left.rel_value_us < minimum_expiration_time.rel_value_us)
624 expiration_time =
625 GNUNET_TIME_relative_to_absolute (minimum_expiration_time);
626 dst_len = sizeof(dst);
627 off = 0;
628 switch (rec->type)
629 {
630 case GNUNET_DNSPARSER_TYPE_A:
631 if (sizeof(struct in_addr) != rec->data.raw.data_len)
632 {
633 GNUNET_break (0);
634 return;
635 }
636 if (NULL == inet_ntop (AF_INET, rec->data.raw.data, ip, ip_size))
637 {
638 GNUNET_break (0);
639 return;
640 }
641 if ((GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst,
642 dst_len,
643 &off,
644 gc->req->hostname)) &&
645 (GNUNET_OK ==
646 GNUNET_DNSPARSER_builder_add_name (dst, dst_len, &off, ip)))
647 {
648 add_record (gc->req,
649 GNUNET_GNSRECORD_TYPE_GNS2DNS,
650 expiration_time,
651 dst,
652 off);
653 gc->found = GNUNET_YES;
654 }
655 break;
656
657 case GNUNET_DNSPARSER_TYPE_AAAA:
658 if (sizeof(struct in6_addr) != rec->data.raw.data_len)
659 {
660 GNUNET_break (0);
661 return;
662 }
663 if (NULL == inet_ntop (AF_INET6, rec->data.raw.data, ip, ip_size))
664 {
665 GNUNET_break (0);
666 return;
667 }
668 if ((GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst,
669 dst_len,
670 &off,
671 gc->req->hostname)) &&
672 (GNUNET_OK ==
673 GNUNET_DNSPARSER_builder_add_name (dst, dst_len, &off, ip)))
674 {
675 add_record (gc->req,
676 GNUNET_GNSRECORD_TYPE_GNS2DNS,
677 expiration_time,
678 dst,
679 off);
680 gc->found = GNUNET_YES;
681 }
682 break;
683
684 case GNUNET_DNSPARSER_TYPE_CNAME:
685 if ((GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst,
686 dst_len,
687 &off,
688 gc->req->hostname)) &&
689 (GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst,
690 dst_len,
691 &off,
692 rec->data.hostname)))
693 {
694 add_record (gc->req,
695 GNUNET_GNSRECORD_TYPE_GNS2DNS,
696 expiration_time,
697 dst,
698 off);
699 gc->found = GNUNET_YES;
700 }
701 break;
702
703 default:
704 /* useless, do nothing */
705 break;
706 }
707}
708
709
710/**
711 * Closure for #process_record().
712 */
713struct ProcessRecordContext
714{
715 /**
716 * Answer we got back and are currently parsing, or NULL
717 * if not active.
718 */
719 struct GNUNET_DNSPARSER_Packet *p;
720
721 /**
722 * Request we are processing.
723 */
724 struct Request *req;
725};
726
727
728/**
729 * We received @a rec for @a req. Remember the answer.
730 *
731 * @param cls a `struct ProcessRecordContext`
732 * @param rec response
733 */
734static void
735process_record (void *cls, const struct GNUNET_DNSPARSER_Record *rec)
736{
737 struct ProcessRecordContext *prc = cls;
738 struct Request *req = prc->req;
739 char dst[65536];
740 size_t dst_len;
741 size_t off;
742 struct GNUNET_TIME_Absolute expiration_time;
743 struct GNUNET_TIME_Relative left;
744
745 dst_len = sizeof(dst);
746 off = 0;
747 records++;
748 if (0 != strcasecmp (rec->name, req->hostname))
749 {
750 GNUNET_log (
751 GNUNET_ERROR_TYPE_DEBUG,
752 "DNS returned record from zone `%s' of type %u while resolving `%s'\n",
753 rec->name,
754 (unsigned int) rec->type,
755 req->hostname);
756 return; /* does not match hostname, might be glue, but
757 not useful for this pass! */
758 }
759 expiration_time = rec->expiration_time;
760 left = GNUNET_TIME_absolute_get_remaining (expiration_time);
761 if (0 == left.rel_value_us)
762 {
763 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
764 "DNS returned expired record for `%s'\n",
765 req->hostname);
766 GNUNET_STATISTICS_update (stats,
767 "# expired records obtained from DNS",
768 1,
769 GNUNET_NO);
770 return; /* record expired */
771 }
772
773 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
774 "DNS returned record that expires at %s for `%s'\n",
775 GNUNET_STRINGS_absolute_time_to_string (expiration_time),
776 req->hostname);
777 /* if expiration window is too short, bump it to configured minimum */
778 if (left.rel_value_us < minimum_expiration_time.rel_value_us)
779 expiration_time =
780 GNUNET_TIME_relative_to_absolute (minimum_expiration_time);
781 switch (rec->type)
782 {
783 case GNUNET_DNSPARSER_TYPE_NS: {
784 struct GlueClosure gc;
785
786 /* check for glue */
787 gc.req = req;
788 gc.ns = rec->data.hostname;
789 gc.found = GNUNET_NO;
790 for_all_records (prc->p, &check_for_glue, &gc);
791 if ((GNUNET_NO == gc.found) &&
792 (GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst,
793 dst_len,
794 &off,
795 req->hostname)) &&
796 (GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst,
797 dst_len,
798 &off,
799 rec->data.hostname)))
800 {
801 /* FIXME: actually check if this is out-of-bailiwick,
802 and if not request explicit resolution... */
803 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
804 "Converted OOB (`%s') NS record for `%s'\n",
805 rec->data.hostname,
806 rec->name);
807 add_record (req,
808 GNUNET_GNSRECORD_TYPE_GNS2DNS,
809 expiration_time,
810 dst,
811 off);
812 }
813 else
814 {
815 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
816 "Converted NS record for `%s' using glue\n",
817 rec->name);
818 }
819 break;
820 }
821
822 case GNUNET_DNSPARSER_TYPE_CNAME:
823 if (GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst,
824 dst_len,
825 &off,
826 rec->data.hostname))
827 {
828 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
829 "Converting CNAME (`%s') record for `%s'\n",
830 rec->data.hostname,
831 rec->name);
832 add_record (req, rec->type, expiration_time, dst, off);
833 }
834 break;
835
836 case GNUNET_DNSPARSER_TYPE_DNAME:
837 /* No support for DNAME in GNS yet! FIXME: support later! */
838 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
839 "FIXME: not supported: %s DNAME %s\n",
840 rec->name,
841 rec->data.hostname);
842 break;
843
844 case GNUNET_DNSPARSER_TYPE_MX:
845 if (GNUNET_OK ==
846 GNUNET_DNSPARSER_builder_add_mx (dst, dst_len, &off, rec->data.mx))
847 {
848 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
849 "Converting MX (`%s') record for `%s'\n",
850 rec->data.mx->mxhost,
851 rec->name);
852 add_record (req, rec->type, expiration_time, dst, off);
853 }
854 break;
855
856 case GNUNET_DNSPARSER_TYPE_SOA:
857 if (GNUNET_OK ==
858 GNUNET_DNSPARSER_builder_add_soa (dst, dst_len, &off, rec->data.soa))
859 {
860 /* NOTE: GNS does not really use SOAs */
861 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
862 "Converting SOA record for `%s'\n",
863 rec->name);
864 add_record (req, rec->type, expiration_time, dst, off);
865 }
866 break;
867
868 case GNUNET_DNSPARSER_TYPE_SRV:
869 if (GNUNET_OK ==
870 GNUNET_DNSPARSER_builder_add_srv (dst, dst_len, &off, rec->data.srv))
871 {
872 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
873 "Converting SRV record for `%s'\n",
874 rec->name);
875 add_record (req, rec->type, expiration_time, dst, off);
876 }
877 break;
878
879 case GNUNET_DNSPARSER_TYPE_PTR:
880 if (GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst,
881 dst_len,
882 &off,
883 rec->data.hostname))
884 {
885 /* !?: what does a PTR record do in a regular TLD??? */
886 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
887 "Converting PTR record for `%s' (weird)\n",
888 rec->name);
889 add_record (req, rec->type, expiration_time, dst, off);
890 }
891 break;
892
893 case GNUNET_DNSPARSER_TYPE_CERT:
894 if (GNUNET_OK ==
895 GNUNET_DNSPARSER_builder_add_cert (dst, dst_len, &off, rec->data.cert))
896 {
897 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
898 "Converting CERT record for `%s'\n",
899 rec->name);
900 add_record (req, rec->type, expiration_time, dst, off);
901 }
902 break;
903
904 /* Rest is 'raw' encoded and just needs to be copied IF
905 the hostname matches the requested name; otherwise we
906 simply cannot use it. */
907 case GNUNET_DNSPARSER_TYPE_A:
908 case GNUNET_DNSPARSER_TYPE_AAAA:
909 case GNUNET_DNSPARSER_TYPE_TXT:
910 default:
911 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
912 "Converting record of type %u for `%s'\n",
913 (unsigned int) rec->type,
914 rec->name);
915 add_record (req,
916 rec->type,
917 expiration_time,
918 rec->data.raw.data,
919 rec->data.raw.data_len);
920 break;
921 }
922}
923
924
925static void
926store_completed_cb (void *cls, enum GNUNET_ErrorCode ec)
927{
928 static struct GNUNET_TIME_Absolute last;
929 struct Request *req = cls;
930
931 req->qe = NULL;
932 if (GNUNET_EC_NONE != ec)
933 {
934 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
935 "Failed to store zone data for `%s': %s\n",
936 req->hostname,
937 GNUNET_ErrorCode_get_hint (ec));
938 }
939 else
940 {
941 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
942 "Stored records under `%s' (%d)\n",
943 req->hostname,
944 ec);
945 }
946 total_reg_proc_dns_ns++; /* finished regular processing */
947 pending_rs--;
948 free_records (req);
949 /* compute NAMESTORE statistics */
950 {
951 static uint64_t total_ns_latency_cnt;
952 static struct GNUNET_TIME_Relative total_ns_latency;
953 struct GNUNET_TIME_Relative ns_latency;
954
955 ns_latency = GNUNET_TIME_absolute_get_duration (req->op_start_time);
956 total_ns_latency = GNUNET_TIME_relative_add (total_ns_latency, ns_latency);
957 if (0 == total_ns_latency_cnt)
958 last = GNUNET_TIME_absolute_get ();
959 total_ns_latency_cnt++;
960 if (0 == (total_ns_latency_cnt % 1000))
961 {
962 struct GNUNET_TIME_Relative delta;
963
964 delta = GNUNET_TIME_absolute_get_duration (last);
965 last = GNUNET_TIME_absolute_get ();
966 fprintf (stderr,
967 "Processed 1000 records in %s\n",
968 GNUNET_STRINGS_relative_time_to_string (delta, GNUNET_YES));
969 GNUNET_STATISTICS_set (stats,
970 "# average NAMESTORE PUT latency (μs)",
971 total_ns_latency.rel_value_us
972 / total_ns_latency_cnt,
973 GNUNET_NO);
974 }
975 }
976 /* compute and publish overall velocity */
977 if (0 == (total_reg_proc_dns_ns % 100))
978 {
979 struct GNUNET_TIME_Relative runtime;
980
981 runtime = GNUNET_TIME_absolute_get_duration (start_time_reg_proc);
982 runtime = GNUNET_TIME_relative_subtract (runtime, idle_time);
983 runtime =
984 GNUNET_TIME_relative_divide (runtime,
985 total_reg_proc_dns + total_reg_proc_dns_ns);
986 GNUNET_STATISTICS_set (stats,
987 "# Regular processing completed without NAMESTORE",
988 total_reg_proc_dns,
989 GNUNET_NO);
990 GNUNET_STATISTICS_set (stats,
991 "# Regular processing completed with NAMESTORE PUT",
992 total_reg_proc_dns_ns,
993 GNUNET_NO);
994 GNUNET_STATISTICS_set (stats,
995 "# average request processing latency (μs)",
996 runtime.rel_value_us,
997 GNUNET_NO);
998 GNUNET_STATISTICS_set (stats,
999 "# total time spent idle (μs)",
1000 idle_time.rel_value_us,
1001 GNUNET_NO);
1002 }
1003
1004 if (NULL == t)
1005 {
1006 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1007 t = GNUNET_SCHEDULER_add_now (&process_queue, NULL);
1008 }
1009}
1010
1011
1012/**
1013 * Function called with the result of a DNS resolution.
1014 *
1015 * @param cls closure with the `struct Request`
1016 * @param dns dns response, never NULL
1017 * @param dns_len number of bytes in @a dns
1018 */
1019static void
1020process_result (void *cls,
1021 const struct GNUNET_TUN_DnsHeader *dns,
1022 size_t dns_len)
1023{
1024 struct Request *req = cls;
1025 struct Record *rec;
1026 struct GNUNET_DNSPARSER_Packet *p;
1027 unsigned int rd_count;
1028
1029 GNUNET_assert (NULL == req->hn);
1030 if (NULL == dns)
1031 {
1032 /* stub gave up */
1033 GNUNET_CONTAINER_DLL_remove (req_head, req_tail, req);
1034 pending--;
1035 if (NULL == t)
1036 {
1037 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1038 t = GNUNET_SCHEDULER_add_now (&process_queue, NULL);
1039 }
1040 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1041 "Stub gave up on DNS reply for `%s'\n",
1042 req->hostname);
1043 GNUNET_STATISTICS_update (stats, "# DNS lookups timed out", 1, GNUNET_NO);
1044 if (req->issue_num > MAX_RETRIES)
1045 {
1046 failures++;
1047 free_request (req);
1048 GNUNET_STATISTICS_update (stats, "# requests given up on", 1, GNUNET_NO);
1049 return;
1050 }
1051 total_reg_proc_dns++;
1052 req->rs = NULL;
1053 insert_sorted (req);
1054 return;
1055 }
1056 if (req->id != dns->id)
1057 {
1058 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1059 "DNS ID did not match request, ignoring reply\n");
1060 GNUNET_STATISTICS_update (stats, "# DNS ID mismatches", 1, GNUNET_NO);
1061 return;
1062 }
1063 GNUNET_CONTAINER_DLL_remove (req_head, req_tail, req);
1064 GNUNET_DNSSTUB_resolve_cancel (req->rs);
1065 req->rs = NULL;
1066 pending--;
1067 p = GNUNET_DNSPARSER_parse ((const char *) dns, dns_len);
1068 if (NULL == p)
1069 {
1070 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1071 "Failed to parse DNS reply for `%s'\n",
1072 req->hostname);
1073 GNUNET_STATISTICS_update (stats, "# DNS parser errors", 1, GNUNET_NO);
1074 if (NULL == t)
1075 {
1076 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1077 t = GNUNET_SCHEDULER_add_now (&process_queue, NULL);
1078 }
1079 if (req->issue_num > MAX_RETRIES)
1080 {
1081 failures++;
1082 free_request (req);
1083 GNUNET_STATISTICS_update (stats, "# requests given up on", 1, GNUNET_NO);
1084 return;
1085 }
1086 insert_sorted (req);
1087 return;
1088 }
1089 /* import new records */
1090 req->issue_num = 0; /* success, reset counter! */
1091 {
1092 struct ProcessRecordContext prc = { .req = req, .p = p };
1093
1094 for_all_records (p, &process_record, &prc);
1095 }
1096 GNUNET_DNSPARSER_free_packet (p);
1097 /* count records found, determine minimum expiration time */
1098 req->expires = GNUNET_TIME_UNIT_FOREVER_ABS;
1099 {
1100 struct GNUNET_TIME_Relative dns_latency;
1101
1102 dns_latency = GNUNET_TIME_absolute_get_duration (req->op_start_time);
1103 total_dns_latency =
1104 GNUNET_TIME_relative_add (total_dns_latency, dns_latency);
1105 total_dns_latency_cnt++;
1106 if (0 == (total_dns_latency_cnt % 1000))
1107 {
1108 GNUNET_STATISTICS_set (stats,
1109 "# average DNS lookup latency (μs)",
1110 total_dns_latency.rel_value_us
1111 / total_dns_latency_cnt,
1112 GNUNET_NO);
1113 }
1114 }
1115 rd_count = 0;
1116 for (rec = req->rec_head; NULL != rec; rec = rec->next)
1117 {
1118 struct GNUNET_TIME_Absolute at;
1119
1120 at.abs_value_us = rec->grd.expiration_time;
1121 req->expires = GNUNET_TIME_absolute_min (req->expires, at);
1122 rd_count++;
1123 }
1124 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1125 "Obtained %u records for `%s'\n",
1126 rd_count,
1127 req->hostname);
1128 /* Instead of going for SOA, simplified for now to look each
1129 day in case we got an empty response */
1130 if (0 == rd_count)
1131 {
1132 req->expires = GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_DAYS);
1133 GNUNET_STATISTICS_update (stats,
1134 "# empty DNS replies (usually NXDOMAIN)",
1135 1,
1136 GNUNET_NO);
1137 }
1138 else
1139 {
1140 record_sets++;
1141 }
1142 /* convert records to namestore import format */
1143 {
1144 struct GNUNET_GNSRECORD_Data rd[GNUNET_NZL (rd_count)];
1145 unsigned int off = 0;
1146
1147 /* convert linked list into array */
1148 for (rec = req->rec_head; NULL != rec; rec = rec->next)
1149 rd[off++] = rec->grd;
1150 pending_rs++;
1151 req->op_start_time = GNUNET_TIME_absolute_get ();
1152 req->qe = GNUNET_NAMESTORE_records_store (ns,
1153 &req->zone->key,
1154 get_label (req),
1155 rd_count,
1156 rd,
1157 &store_completed_cb,
1158 req);
1159 GNUNET_assert (NULL != req->qe);
1160 }
1161 insert_sorted (req);
1162}
1163
1164
1165/**
1166 * Process as many requests as possible from the queue.
1167 *
1168 * @param cls NULL
1169 */
1170static void
1171process_queue (void *cls)
1172{
1173 struct Request *req;
1174 unsigned int series;
1175 void *raw;
1176 size_t raw_size;
1177 struct GNUNET_TIME_Relative delay;
1178
1179 (void) cls;
1180 delay = GNUNET_TIME_absolute_get_duration (sleep_time_reg_proc);
1181 idle_time = GNUNET_TIME_relative_add (idle_time, delay);
1182 series = 0;
1183 t = NULL;
1184 while (pending + pending_rs < THRESH)
1185 {
1186 req = GNUNET_CONTAINER_heap_peek (req_heap);
1187 if (NULL == req)
1188 break;
1189 if (NULL != req->qe)
1190 return; /* namestore op still pending */
1191 if (NULL != req->rs)
1192 {
1193 GNUNET_break (0);
1194 return; /* already submitted */
1195 }
1196 if (GNUNET_TIME_absolute_get_remaining (req->expires).rel_value_us > 0)
1197 break;
1198 GNUNET_assert (req == GNUNET_CONTAINER_heap_remove_root (req_heap));
1199 req->hn = NULL;
1200 GNUNET_CONTAINER_DLL_insert (req_head, req_tail, req);
1201 GNUNET_assert (NULL == req->rs);
1202 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1203 "Requesting resolution for `%s'\n",
1204 req->hostname);
1205 raw = build_dns_query (req, &raw_size);
1206 if (NULL == raw)
1207 {
1208 GNUNET_break (0);
1209 free_request (req);
1210 continue;
1211 }
1212 req->op_start_time = GNUNET_TIME_absolute_get ();
1213 req->rs = GNUNET_DNSSTUB_resolve (ctx, raw, raw_size, &process_result, req);
1214 GNUNET_assert (NULL != req->rs);
1215 req->issue_num++;
1216 lookups++;
1217 pending++;
1218 series++;
1219 if (series > MAX_SERIES)
1220 break;
1221 }
1222 if (pending + pending_rs >= THRESH)
1223 {
1224 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1225 "Stopped processing queue (%u+%u/%u)]\n",
1226 pending,
1227 pending_rs,
1228 THRESH);
1229 return; /* wait for replies */
1230 }
1231 req = GNUNET_CONTAINER_heap_peek (req_heap);
1232 if (NULL == req)
1233 {
1234 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1235 "Stopped processing queue: empty queue\n");
1236 return;
1237 }
1238 if (GNUNET_TIME_absolute_get_remaining (req->expires).rel_value_us > 0)
1239 {
1240 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1241 "Waiting until %s for next record (`%s') to expire\n",
1242 GNUNET_STRINGS_absolute_time_to_string (req->expires),
1243 req->hostname);
1244 if (NULL != t)
1245 GNUNET_SCHEDULER_cancel (t);
1246 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1247 t = GNUNET_SCHEDULER_add_at (req->expires, &process_queue, NULL);
1248 return;
1249 }
1250 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Throttling\n");
1251 if (NULL != t)
1252 GNUNET_SCHEDULER_cancel (t);
1253 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1254 t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY, &process_queue, NULL);
1255}
1256
1257
1258/**
1259 * Iterator called during #do_shutdown() to free requests in
1260 * the #ns_pending map.
1261 *
1262 * @param cls NULL
1263 * @param key unused
1264 * @param value the `struct Request` to free
1265 * @return #GNUNET_OK
1266 */
1267static int
1268free_request_it (void *cls, const struct GNUNET_HashCode *key, void *value)
1269{
1270 struct Request *req = value;
1271
1272 (void) cls;
1273 (void) key;
1274 free_request (req);
1275 return GNUNET_OK;
1276}
1277
1278
1279/**
1280 * Clean up and terminate the process.
1281 *
1282 * @param cls NULL
1283 */
1284static void
1285do_shutdown (void *cls)
1286{
1287 struct Request *req;
1288 struct Zone *zone;
1289
1290 (void) cls;
1291 if (NULL != id)
1292 {
1293 GNUNET_IDENTITY_disconnect (id);
1294 id = NULL;
1295 }
1296 if (NULL != t)
1297 {
1298 GNUNET_SCHEDULER_cancel (t);
1299 t = NULL;
1300 }
1301 while (NULL != (req = req_head))
1302 {
1303 GNUNET_CONTAINER_DLL_remove (req_head, req_tail, req);
1304 if (NULL != req->qe)
1305 GNUNET_NAMESTORE_cancel (req->qe);
1306 free_request (req);
1307 }
1308 while (NULL != (req = GNUNET_CONTAINER_heap_remove_root (req_heap)))
1309 {
1310 req->hn = NULL;
1311 if (NULL != req->qe)
1312 GNUNET_NAMESTORE_cancel (req->qe);
1313 free_request (req);
1314 }
1315 if (NULL != zone_it)
1316 {
1317 GNUNET_NAMESTORE_zone_iteration_stop (zone_it);
1318 zone_it = NULL;
1319 }
1320 if (NULL != ns)
1321 {
1322 GNUNET_NAMESTORE_disconnect (ns);
1323 ns = NULL;
1324 }
1325 if (NULL != ctx)
1326 {
1327 GNUNET_DNSSTUB_stop (ctx);
1328 ctx = NULL;
1329 }
1330 if (NULL != req_heap)
1331 {
1332 GNUNET_CONTAINER_heap_destroy (req_heap);
1333 req_heap = NULL;
1334 }
1335 if (NULL != ns_pending)
1336 {
1337 GNUNET_CONTAINER_multihashmap_iterate (ns_pending, &free_request_it, NULL);
1338 GNUNET_CONTAINER_multihashmap_destroy (ns_pending);
1339 ns_pending = NULL;
1340 }
1341 while (NULL != (zone = zone_head))
1342 {
1343 GNUNET_CONTAINER_DLL_remove (zone_head, zone_tail, zone);
1344 GNUNET_free (zone->domain);
1345 GNUNET_free (zone);
1346 }
1347 if (NULL != stats)
1348 {
1349 GNUNET_STATISTICS_destroy (stats, GNUNET_NO);
1350 stats = NULL;
1351 }
1352}
1353
1354
1355/**
1356 * Iterate over all of the zones we care about and see which records
1357 * we may need to re-fetch when.
1358 *
1359 * @param cls NULL
1360 */
1361static void
1362iterate_zones (void *cls);
1363
1364
1365/**
1366 * Function called if #GNUNET_NAMESTORE_records_lookup() failed.
1367 * Just logs an error.
1368 *
1369 * @param cls a `struct Zone`
1370 */
1371static void
1372ns_lookup_error_cb (void *cls)
1373{
1374 struct Zone *zone = cls;
1375
1376 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1377 "Failed to load data from namestore for zone `%s'\n",
1378 zone->domain);
1379 zone_it = NULL;
1380 ns_iterator_trigger_next = 0;
1381 iterate_zones (NULL);
1382}
1383
1384
1385/**
1386 * Process a record that was stored in the namestore.
1387 *
1388 * @param cls a `struct Zone *`
1389 * @param key private key of the zone
1390 * @param label label of the records
1391 * @param rd_count number of entries in @a rd array, 0 if label was deleted
1392 * @param rd array of records with data to store
1393 */
1394static void
1395ns_lookup_result_cb (void *cls,
1396 const struct GNUNET_CRYPTO_PrivateKey *key,
1397 const char *label,
1398 unsigned int rd_count,
1399 const struct GNUNET_GNSRECORD_Data *rd)
1400{
1401 struct Zone *zone = cls;
1402 struct Request *req;
1403 struct GNUNET_HashCode hc;
1404 char *fqdn;
1405
1406 ns_iterator_trigger_next--;
1407 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1408 "Obtained NAMESTORE reply, %llu left in round\n",
1409 (unsigned long long) ns_iterator_trigger_next);
1410 if (0 == ns_iterator_trigger_next)
1411 {
1412 ns_iterator_trigger_next = NS_BATCH_SIZE;
1413 GNUNET_STATISTICS_update (stats,
1414 "# NAMESTORE records requested from cache",
1415 ns_iterator_trigger_next,
1416 GNUNET_NO);
1417 GNUNET_NAMESTORE_zone_iterator_next (zone_it, ns_iterator_trigger_next);
1418 }
1419 GNUNET_asprintf (&fqdn, "%s.%s", label, zone->domain);
1420 GNUNET_CRYPTO_hash (fqdn, strlen (fqdn) + 1, &hc);
1421 GNUNET_free (fqdn);
1422 req = GNUNET_CONTAINER_multihashmap_get (ns_pending, &hc);
1423 if (NULL == req)
1424 {
1425 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1426 "Ignoring record `%s' in zone `%s': not on my list!\n",
1427 label,
1428 zone->domain);
1429 return;
1430 }
1431 GNUNET_assert (GNUNET_OK ==
1432 GNUNET_CONTAINER_multihashmap_remove (ns_pending, &hc, req));
1433 GNUNET_break (0 == GNUNET_memcmp (key, &req->zone->key));
1434 GNUNET_break (0 == strcasecmp (label, get_label (req)));
1435 for (unsigned int i = 0; i < rd_count; i++)
1436 {
1437 struct GNUNET_TIME_Absolute at;
1438
1439 if (0 != (rd->flags & GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION))
1440 {
1441 struct GNUNET_TIME_Relative rel;
1442
1443 rel.rel_value_us = rd->expiration_time;
1444 at = GNUNET_TIME_relative_to_absolute (rel);
1445 }
1446 else
1447 {
1448 at.abs_value_us = rd->expiration_time;
1449 }
1450 add_record (req, rd->record_type, at, rd->data, rd->data_size);
1451 }
1452 if (0 == rd_count)
1453 {
1454 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1455 "Empty record set in namestore for `%s'\n",
1456 req->hostname);
1457 }
1458 else
1459 {
1460 unsigned int pos = 0;
1461
1462 cached++;
1463 req->expires = GNUNET_TIME_UNIT_FOREVER_ABS;
1464 for (struct Record *rec = req->rec_head; NULL != rec; rec = rec->next)
1465 {
1466 struct GNUNET_TIME_Absolute at;
1467
1468 at.abs_value_us = rec->grd.expiration_time;
1469 req->expires = GNUNET_TIME_absolute_min (req->expires, at);
1470 pos++;
1471 }
1472 if (0 == pos)
1473 req->expires = GNUNET_TIME_UNIT_ZERO_ABS;
1474 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1475 "Hot-start with %u existing records for `%s'\n",
1476 pos,
1477 req->hostname);
1478 }
1479 free_records (req);
1480
1481 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1482 "Adding `%s' to worklist to start at %s\n",
1483 req->hostname,
1484 GNUNET_STRINGS_absolute_time_to_string (req->expires));
1485 insert_sorted (req);
1486}
1487
1488
1489/**
1490 * Add @a hostname to the list of requests to be made.
1491 *
1492 * @param hostname name to resolve
1493 */
1494static void
1495queue (const char *hostname)
1496{
1497 struct Request *req;
1498 const char *dot;
1499 struct Zone *zone;
1500 size_t hlen;
1501 struct GNUNET_HashCode hc;
1502
1503 if (GNUNET_OK != GNUNET_DNSPARSER_check_name (hostname))
1504 {
1505 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1506 "Refusing invalid hostname `%s'\n",
1507 hostname);
1508 rejects++;
1509 return;
1510 }
1511 dot = strchr (hostname, (unsigned char) '.');
1512 if (NULL == dot)
1513 {
1514 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1515 "Refusing invalid hostname `%s' (lacks '.')\n",
1516 hostname);
1517 rejects++;
1518 return;
1519 }
1520 for (zone = zone_head; NULL != zone; zone = zone->next)
1521 if (0 == strcmp (zone->domain, dot + 1))
1522 break;
1523 if (NULL == zone)
1524 {
1525 rejects++;
1526 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1527 "Domain name `%s' not in ego list!\n",
1528 dot + 1);
1529 return;
1530 }
1531
1532 hlen = strlen (hostname) + 1;
1533 req = GNUNET_malloc (sizeof(struct Request) + hlen);
1534 req->zone = zone;
1535 req->hostname = (char *) &req[1];
1536 GNUNET_memcpy (req->hostname, hostname, hlen);
1537 req->id = (uint16_t) GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE,
1538 UINT16_MAX);
1539 GNUNET_CRYPTO_hash (req->hostname, hlen, &hc);
1540 if (GNUNET_OK != GNUNET_CONTAINER_multihashmap_put (
1541 ns_pending,
1542 &hc,
1543 req,
1544 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))
1545 {
1546 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1547 "Duplicate hostname `%s' ignored\n",
1548 hostname);
1549 GNUNET_free (req);
1550 return;
1551 }
1552}
1553
1554
1555/**
1556 * We have completed the initial iteration over the namestore's database.
1557 * This function is called on each of the remaining records in
1558 * #move_to_queue to #queue() them, as we will simply not find existing
1559 * records for them any longer.
1560 *
1561 * @param cls NULL
1562 * @param key unused
1563 * @param value a `struct Request`
1564 * @return #GNUNET_OK (continue to iterate)
1565 */
1566static int
1567move_to_queue (void *cls, const struct GNUNET_HashCode *key, void *value)
1568{
1569 struct Request *req = value;
1570
1571 (void) cls;
1572 (void) key;
1573 insert_sorted (req);
1574 return GNUNET_OK;
1575}
1576
1577
1578/**
1579 * Iterate over all of the zones we care about and see which records
1580 * we may need to re-fetch when.
1581 *
1582 * @param cls NULL
1583 */
1584static void
1585iterate_zones (void *cls)
1586{
1587 static struct Zone *last;
1588
1589 (void) cls;
1590 if (NULL != zone_it)
1591 {
1592 zone_it = NULL;
1593 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1594 "Finished iteration over zone `%s'!\n",
1595 last->domain);
1596 /* subtract left-overs from previous iteration */
1597 GNUNET_STATISTICS_update (stats,
1598 "# NAMESTORE records requested from cache",
1599 (long long) (-ns_iterator_trigger_next),
1600 GNUNET_NO);
1601 ns_iterator_trigger_next = 0;
1602 }
1603 GNUNET_assert (NULL != zone_tail);
1604 if (zone_tail == last)
1605 {
1606 /* Done iterating over relevant zones in NAMESTORE, move
1607 rest of hash map to work queue as well. */
1608 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1609 "Finished all NAMESTORE iterations!\n");
1610 GNUNET_STATISTICS_set (stats,
1611 "# Domain names without cached reply",
1612 GNUNET_CONTAINER_multihashmap_size (ns_pending),
1613 GNUNET_NO);
1614 GNUNET_CONTAINER_multihashmap_iterate (ns_pending, &move_to_queue, NULL);
1615 GNUNET_CONTAINER_multihashmap_destroy (ns_pending);
1616 ns_pending = NULL;
1617 start_time_reg_proc = GNUNET_TIME_absolute_get ();
1618 total_reg_proc_dns = 0;
1619 total_reg_proc_dns_ns = 0;
1620 return;
1621 }
1622 if (NULL == last)
1623 last = zone_head;
1624 else
1625 last = last->next;
1626 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1627 "Starting iteration over zone `%s'!\n",
1628 last->domain);
1629 /* subtract left-overs from previous iteration */
1630 GNUNET_STATISTICS_update (stats,
1631 "# NAMESTORE records requested from cache",
1632 1,
1633 GNUNET_NO);
1634 ns_iterator_trigger_next = 1;
1635 GNUNET_STATISTICS_update (stats, "# zones iterated", 1, GNUNET_NO);
1636 zone_it = GNUNET_NAMESTORE_zone_iteration_start (ns,
1637 &last->key,
1638 &ns_lookup_error_cb,
1639 NULL,
1640 &ns_lookup_result_cb,
1641 last,
1642 &iterate_zones,
1643 NULL);
1644}
1645
1646
1647/**
1648 * Begin processing hostnames from stdin.
1649 *
1650 * @param cls NULL
1651 */
1652static void
1653process_stdin (void *cls)
1654{
1655 static struct GNUNET_TIME_Absolute last;
1656 static uint64_t idot;
1657 char hn[256];
1658
1659 (void) cls;
1660 t = NULL;
1661 if (NULL != id)
1662 {
1663 GNUNET_IDENTITY_disconnect (id);
1664 id = NULL;
1665 }
1666 while (NULL != fgets (hn, sizeof(hn), stdin))
1667 {
1668 if (strlen (hn) > 0)
1669 hn[strlen (hn) - 1] = '\0'; /* eat newline */
1670 if (0 == idot)
1671 last = GNUNET_TIME_absolute_get ();
1672 idot++;
1673 if (0 == idot % 100000)
1674 {
1675 struct GNUNET_TIME_Relative delta;
1676
1677 delta = GNUNET_TIME_absolute_get_duration (last);
1678 last = GNUNET_TIME_absolute_get ();
1679 fprintf (stderr,
1680 "Read 100000 domain names in %s\n",
1681 GNUNET_STRINGS_relative_time_to_string (delta, GNUNET_YES));
1682 GNUNET_STATISTICS_set (stats, "# domain names provided", idot, GNUNET_NO);
1683 }
1684 queue (hn);
1685 }
1686 fprintf (stderr,
1687 "Done reading %llu domain names\n",
1688 (unsigned long long) idot);
1689 GNUNET_STATISTICS_set (stats, "# domain names provided", idot, GNUNET_NO);
1690 iterate_zones (NULL);
1691}
1692
1693
1694/**
1695 * Method called to inform about the egos of this peer.
1696 *
1697 * When used with #GNUNET_IDENTITY_connect, this function is
1698 * initially called for all egos and then again whenever a
1699 * ego's name changes or if it is deleted. At the end of
1700 * the initial pass over all egos, the function is once called
1701 * with 'NULL' for @a ego. That does NOT mean that the callback won't
1702 * be invoked in the future or that there was an error.
1703 *
1704 * When used with #GNUNET_IDENTITY_create or #GNUNET_IDENTITY_get, this
1705 * function is only called ONCE, and 'NULL' being passed in @a ego does
1706 * indicate an error (for example because name is taken or no default value is
1707 * known). If @a ego is non-NULL and if '*ctx' is set in those callbacks, the
1708 * value WILL be passed to a subsequent call to the identity callback of
1709 * #GNUNET_IDENTITY_connect (if that one was not NULL).
1710 *
1711 * When an identity is renamed, this function is called with the
1712 * (known) @a ego but the NEW @a name.
1713 *
1714 * When an identity is deleted, this function is called with the
1715 * (known) ego and "NULL" for the @a name. In this case,
1716 * the @a ego is henceforth invalid (and the @a ctx should also be
1717 * cleaned up).
1718 *
1719 * @param cls closure
1720 * @param ego ego handle, NULL for end of list
1721 * @param ctx context for application to store data for this ego
1722 * (during the lifetime of this process, initially NULL)
1723 * @param name name assigned by the user for this ego,
1724 * NULL if the user just deleted the ego and it
1725 * must thus no longer be used
1726 */
1727static void
1728identity_cb (void *cls,
1729 struct GNUNET_IDENTITY_Ego *ego,
1730 void **ctx,
1731 const char *name)
1732{
1733 (void) cls;
1734 (void) ctx;
1735
1736 if (NULL == ego)
1737 {
1738 /* end of iteration */
1739 if (NULL == zone_head)
1740 {
1741 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "No zone found\n");
1742 GNUNET_SCHEDULER_shutdown ();
1743 return;
1744 }
1745 /* zone_head non-null, process hostnames from stdin */
1746 t = GNUNET_SCHEDULER_add_now (&process_stdin, NULL);
1747 return;
1748 }
1749 if (NULL != name)
1750 {
1751 struct Zone *zone;
1752
1753 zone = GNUNET_new (struct Zone);
1754 zone->key = *GNUNET_IDENTITY_ego_get_private_key (ego);
1755 zone->domain = GNUNET_strdup (name);
1756 GNUNET_CONTAINER_DLL_insert (zone_head, zone_tail, zone);
1757 }
1758}
1759
1760
1761/**
1762 * Process requests from the queue, then if the queue is
1763 * not empty, try again.
1764 *
1765 * @param cls NULL
1766 * @param args remaining command-line arguments
1767 * @param cfgfile name of the configuration file used (for saving, can be NULL!)
1768 * @param cfg configuration
1769 */
1770static void
1771run (void *cls,
1772 char *const *args,
1773 const char *cfgfile,
1774 const struct GNUNET_CONFIGURATION_Handle *cfg)
1775{
1776 (void) cls;
1777 (void) args;
1778 (void) cfgfile;
1779 stats = GNUNET_STATISTICS_create ("zoneimport", cfg);
1780 req_heap = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
1781 ns_pending = GNUNET_CONTAINER_multihashmap_create (map_size, GNUNET_NO);
1782 if (NULL == ns_pending)
1783 {
1784 fprintf (stderr, "Failed to allocate memory for main hash map\n");
1785 return;
1786 }
1787 ctx = GNUNET_DNSSTUB_start (256);
1788 if (NULL == ctx)
1789 {
1790 fprintf (stderr, "Failed to initialize GNUnet DNS STUB\n");
1791 return;
1792 }
1793 if (NULL == args[0])
1794 {
1795 fprintf (stderr,
1796 "You must provide a list of DNS resolvers on the command line\n");
1797 return;
1798 }
1799 for (unsigned int i = 0; NULL != args[i]; i++)
1800 {
1801 if (GNUNET_OK != GNUNET_DNSSTUB_add_dns_ip (ctx, args[i]))
1802 {
1803 fprintf (stderr, "Failed to use `%s' for DNS resolver\n", args[i]);
1804 return;
1805 }
1806 }
1807
1808
1809 GNUNET_SCHEDULER_add_shutdown (&do_shutdown, NULL);
1810 ns = GNUNET_NAMESTORE_connect (cfg);
1811 if (NULL == ns)
1812 {
1813 GNUNET_SCHEDULER_shutdown ();
1814 return;
1815 }
1816 id = GNUNET_IDENTITY_connect (cfg, &identity_cb, NULL);
1817}
1818
1819
1820/**
1821 * Call with IP address of resolver to query.
1822 *
1823 * @param argc should be 2
1824 * @param argv[1] should contain IP address
1825 * @return 0 on success
1826 */
1827int
1828main (int argc, char *const *argv)
1829{
1830 struct GNUNET_GETOPT_CommandLineOption options[] =
1831 { GNUNET_GETOPT_option_uint ('s',
1832 "size",
1833 "MAPSIZE",
1834 gettext_noop (
1835 "size to use for the main hash map"),
1836 &map_size),
1837 GNUNET_GETOPT_option_relative_time (
1838 'm',
1839 "minimum-expiration",
1840 "RELATIVETIME",
1841 gettext_noop ("minimum expiration time we assume for imported records"),
1842 &minimum_expiration_time),
1843 GNUNET_GETOPT_OPTION_END };
1844 int ret;
1845
1846 if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
1847 return 2;
1848 if (GNUNET_OK != (ret = GNUNET_PROGRAM_run (argc,
1849 argv,
1850 "gnunet-zoneimport",
1851 "import DNS zone into namestore",
1852 options,
1853 &run,
1854 NULL)))
1855 return ret;
1856 GNUNET_free_nz ((void *) argv);
1857 fprintf (stderr,
1858 "Rejected %u names, had %u cached, did %u lookups, stored %u record sets\n"
1859 "Found %u records, %u lookups failed, %u/%u pending on shutdown\n",
1860 rejects,
1861 cached,
1862 lookups,
1863 record_sets,
1864 records,
1865 failures,
1866 pending,
1867 pending_rs);
1868 return 0;
1869}
1870
1871
1872/* end of gnunet-zoneimport.c */
generated by cgit v1.2.3 (git 2.39.1) at 2024-05-21 22:22:42 +0000