aboutsummaryrefslogtreecommitdiff
path: root/src/cli/namestore/gnunet-zoneimport.c
diff options
context:
space:
mode:
Diffstat (limited to 'src/cli/namestore/gnunet-zoneimport.c')
-rw-r--r--src/cli/namestore/gnunet-zoneimport.c1882
1 files changed, 1882 insertions, 0 deletions
diff --git a/src/cli/namestore/gnunet-zoneimport.c b/src/cli/namestore/gnunet-zoneimport.c
new file mode 100644
index 000000000..aaed808dd
--- /dev/null
+++ b/src/cli/namestore/gnunet-zoneimport.c
@@ -0,0 +1,1882 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2018 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20/**
21 * @file src/namestore/gnunet-zoneimport.c
22 * @brief import a DNS zone for publication in GNS, incremental
23 * @author Christian Grothoff
24 */
25#include <gnunet_util_lib.h>
26#include <gnunet_gnsrecord_lib.h>
27#include <gnunet_namestore_service.h>
28#include <gnunet_statistics_service.h>
29#include <gnunet_identity_service.h>
30
31
32/**
33 * Maximum number of queries pending at the same time.
34 */
35#define THRESH 100
36
37/**
38 * TIME_THRESH is in usecs. How quickly do we submit fresh queries.
39 * Used as an additional throttle.
40 */
41#define TIME_THRESH 10
42
43/**
44 * How often do we retry a query before giving up for good?
45 */
46#define MAX_RETRIES 5
47
48/**
49 * How many DNS requests do we at most issue in rapid series?
50 */
51#define MAX_SERIES 10
52
53/**
54 * How long do we wait at least between series of requests?
55 */
56#define SERIES_DELAY \
57 GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MICROSECONDS, 10)
58
59/**
60 * How long do DNS records have to last at least after being imported?
61 */
62static struct GNUNET_TIME_Relative minimum_expiration_time;
63
64/**
65 * How many requests do we request from NAMESTORE in one batch
66 * during our initial iteration?
67 */
68#define NS_BATCH_SIZE 1024
69
70/**
71 * Some zones may include authoritative records for other
72 * zones, such as foo.com.uk or bar.com.fr. As for GNS
73 * each dot represents a zone cut, we then need to create a
74 * zone on-the-fly to capture those records properly.
75 */
76struct Zone
77{
78 /**
79 * Kept in a DLL.
80 */
81 struct Zone *next;
82
83 /**
84 * Kept in a DLL.
85 */
86 struct Zone *prev;
87
88 /**
89 * Domain of the zone (i.e. "fr" or "com.fr")
90 */
91 char *domain;
92
93 /**
94 * Private key of the zone.
95 */
96 struct GNUNET_CRYPTO_PrivateKey key;
97};
98
99
100/**
101 * Record for the request to be stored by GNS.
102 */
103struct Record
104{
105 /**
106 * Kept in a DLL.
107 */
108 struct Record *next;
109
110 /**
111 * Kept in a DLL.
112 */
113 struct Record *prev;
114
115 /**
116 * GNS record.
117 */
118 struct GNUNET_GNSRECORD_Data grd;
119};
120
121
122/**
123 * Request we should make. We keep this struct in memory per request,
124 * thus optimizing it is crucial for the overall memory consumption of
125 * the zone importer.
126 */
127struct Request
128{
129 /**
130 * Requests are kept in a heap while waiting to be resolved.
131 */
132 struct GNUNET_CONTAINER_HeapNode *hn;
133
134 /**
135 * Active requests are kept in a DLL.
136 */
137 struct Request *next;
138
139 /**
140 * Active requests are kept in a DLL.
141 */
142 struct Request *prev;
143
144 /**
145 * Head of records that should be published in GNS for
146 * this hostname.
147 */
148 struct Record *rec_head;
149
150 /**
151 * Tail of records that should be published in GNS for
152 * this hostname.
153 */
154 struct Record *rec_tail;
155
156 /**
157 * Socket used to make the request, NULL if not active.
158 */
159 struct GNUNET_DNSSTUB_RequestSocket *rs;
160
161 /**
162 * Hostname we are resolving, allocated at the end of
163 * this struct (optimizing memory consumption by reducing
164 * total number of allocations).
165 */
166 char *hostname;
167
168 /**
169 * Namestore operation pending for this record.
170 */
171 struct GNUNET_NAMESTORE_QueueEntry *qe;
172
173 /**
174 * Zone responsible for this request.
175 */
176 const struct Zone *zone;
177
178 /**
179 * At what time does the (earliest) of the returned records
180 * for this name expire? At this point, we need to re-fetch
181 * the record.
182 */
183 struct GNUNET_TIME_Absolute expires;
184
185 /**
186 * While we are fetching the record, the value is set to the
187 * starting time of the DNS operation. While doing a
188 * NAMESTORE store, again set to the start time of the
189 * NAMESTORE operation.
190 */
191 struct GNUNET_TIME_Absolute op_start_time;
192
193 /**
194 * How often did we issue this query? (And failed, reset
195 * to zero once we were successful.)
196 */
197 unsigned int issue_num;
198
199 /**
200 * random 16-bit DNS query identifier.
201 */
202 uint16_t id;
203};
204
205
206/**
207 * Command-line argument specifying desired size of the hash map with
208 * all of our pending names. Usually, we use an automatically growing
209 * map, but this is only OK up to about a million entries. Above that
210 * number, the user must explicitly specify the size at startup.
211 */
212static unsigned int map_size = 1024;
213
214/**
215 * Handle to the identity service.
216 */
217static struct GNUNET_IDENTITY_Handle *id;
218
219/**
220 * Namestore handle.
221 */
222static struct GNUNET_NAMESTORE_Handle *ns;
223
224/**
225 * Handle to the statistics service.
226 */
227static struct GNUNET_STATISTICS_Handle *stats;
228
229/**
230 * Context for DNS resolution.
231 */
232static struct GNUNET_DNSSTUB_Context *ctx;
233
234/**
235 * The number of DNS queries that are outstanding
236 */
237static unsigned int pending;
238
239/**
240 * The number of NAMESTORE record store operations that are outstanding
241 */
242static unsigned int pending_rs;
243
244/**
245 * Number of lookups we performed overall.
246 */
247static unsigned int lookups;
248
249/**
250 * Number of records we had cached.
251 */
252static unsigned int cached;
253
254/**
255 * How many hostnames did we reject (malformed).
256 */
257static unsigned int rejects;
258
259/**
260 * Number of lookups that failed.
261 */
262static unsigned int failures;
263
264/**
265 * Number of records we found.
266 */
267static unsigned int records;
268
269/**
270 * Number of record sets given to namestore.
271 */
272static unsigned int record_sets;
273
274/**
275 * Heap of all requests to perform, sorted by
276 * the time we should next do the request (i.e. by expires).
277 */
278static struct GNUNET_CONTAINER_Heap *req_heap;
279
280/**
281 * Active requests are kept in a DLL.
282 */
283static struct Request *req_head;
284
285/**
286 * Active requests are kept in a DLL.
287 */
288static struct Request *req_tail;
289
290/**
291 * Main task.
292 */
293static struct GNUNET_SCHEDULER_Task *t;
294
295/**
296 * Hash map of requests for which we may still get a response from
297 * the namestore. Set to NULL once the initial namestore iteration
298 * is done.
299 */
300static struct GNUNET_CONTAINER_MultiHashMap *ns_pending;
301
302/**
303 * Current zone iteration handle.
304 */
305static struct GNUNET_NAMESTORE_ZoneIterator *zone_it;
306
307/**
308 * Head of list of zones we are managing.
309 */
310static struct Zone *zone_head;
311
312/**
313 * Tail of list of zones we are managing.
314 */
315static struct Zone *zone_tail;
316
317/**
318 * After how many more results must #ns_lookup_result_cb() ask
319 * the namestore for more?
320 */
321static uint64_t ns_iterator_trigger_next;
322
323/**
324 * Number of DNS requests counted in latency total.
325 */
326static uint64_t total_dns_latency_cnt;
327
328/**
329 * Sum of DNS latencies observed.
330 */
331static struct GNUNET_TIME_Relative total_dns_latency;
332
333/**
334 * Number of records processed (DNS lookup, no NAMESTORE) in total.
335 */
336static uint64_t total_reg_proc_dns;
337
338/**
339 * Number of records processed (DNS lookup, with NAMESTORE) in total.
340 */
341static uint64_t total_reg_proc_dns_ns;
342
343/**
344 * Start time of the regular processing.
345 */
346static struct GNUNET_TIME_Absolute start_time_reg_proc;
347
348/**
349 * Last time we worked before going idle.
350 */
351static struct GNUNET_TIME_Absolute sleep_time_reg_proc;
352
353/**
354 * Time we slept just waiting for work.
355 */
356static struct GNUNET_TIME_Relative idle_time;
357
358
359/**
360 * Callback for #for_all_records
361 *
362 * @param cls closure
363 * @param rec a DNS record
364 */
365typedef void (*RecordProcessor) (void *cls,
366 const struct GNUNET_DNSPARSER_Record *rec);
367
368
369/**
370 * Call @a rp for each record in @a p, regardless of
371 * what response section it is in.
372 *
373 * @param p packet from DNS
374 * @param rp function to call
375 * @param rp_cls closure for @a rp
376 */
377static void
378for_all_records (const struct GNUNET_DNSPARSER_Packet *p,
379 RecordProcessor rp,
380 void *rp_cls)
381{
382 for (unsigned int i = 0; i < p->num_answers; i++)
383 {
384 struct GNUNET_DNSPARSER_Record *rs = &p->answers[i];
385
386 rp (rp_cls, rs);
387 }
388 for (unsigned int i = 0; i < p->num_authority_records; i++)
389 {
390 struct GNUNET_DNSPARSER_Record *rs = &p->authority_records[i];
391
392 rp (rp_cls, rs);
393 }
394 for (unsigned int i = 0; i < p->num_additional_records; i++)
395 {
396 struct GNUNET_DNSPARSER_Record *rs = &p->additional_records[i];
397
398 rp (rp_cls, rs);
399 }
400}
401
402
403/**
404 * Return just the label of the hostname in @a req.
405 *
406 * @param req request to process hostname of
407 * @return statically allocated pointer to the label,
408 * overwritten upon the next request!
409 */
410static const char *
411get_label (struct Request *req)
412{
413 static char label[64];
414 const char *dot;
415
416 dot = strchr (req->hostname, (unsigned char) '.');
417 if (NULL == dot)
418 {
419 GNUNET_break (0);
420 return NULL;
421 }
422 if (((size_t) (dot - req->hostname)) >= sizeof(label))
423 {
424 GNUNET_break (0);
425 return NULL;
426 }
427 GNUNET_memcpy (label, req->hostname, dot - req->hostname);
428 label[dot - req->hostname] = '\0';
429 return label;
430}
431
432
433/**
434 * Build DNS query for @a hostname.
435 *
436 * @param hostname host to build query for
437 * @param[out] raw_size number of bytes in the query
438 * @return NULL on error, otherwise pointer to statically (!)
439 * allocated query buffer
440 */
441static void *
442build_dns_query (struct Request *req, size_t *raw_size)
443{
444 static char raw[512];
445 char *rawp;
446 struct GNUNET_DNSPARSER_Packet p;
447 struct GNUNET_DNSPARSER_Query q;
448 int ret;
449
450 q.name = (char *) req->hostname;
451 q.type = GNUNET_DNSPARSER_TYPE_NS;
452 q.dns_traffic_class = GNUNET_TUN_DNS_CLASS_INTERNET;
453
454 memset (&p, 0, sizeof(p));
455 p.num_queries = 1;
456 p.queries = &q;
457 p.id = req->id;
458 ret = GNUNET_DNSPARSER_pack (&p, UINT16_MAX, &rawp, raw_size);
459 if (GNUNET_OK != ret)
460 {
461 if (GNUNET_NO == ret)
462 GNUNET_free (rawp);
463 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
464 "Failed to pack query for hostname `%s'\n",
465 req->hostname);
466 rejects++;
467 return NULL;
468 }
469 if (*raw_size > sizeof(raw))
470 {
471 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
472 "Failed to pack query for hostname `%s'\n",
473 req->hostname);
474 rejects++;
475 GNUNET_break (0);
476 GNUNET_free (rawp);
477 return NULL;
478 }
479 GNUNET_memcpy (raw, rawp, *raw_size);
480 GNUNET_free (rawp);
481 return raw;
482}
483
484
485/**
486 * Free records associated with @a req.
487 *
488 * @param req request to free records of
489 */
490static void
491free_records (struct Request *req)
492{
493 struct Record *rec;
494
495 /* Free records */
496 while (NULL != (rec = req->rec_head))
497 {
498 GNUNET_CONTAINER_DLL_remove (req->rec_head, req->rec_tail, rec);
499 GNUNET_free (rec);
500 }
501}
502
503
504/**
505 * Free @a req and data structures reachable from it.
506 *
507 * @param req request to free
508 */
509static void
510free_request (struct Request *req)
511{
512 free_records (req);
513 GNUNET_free (req);
514}
515
516
517/**
518 * Process as many requests as possible from the queue.
519 *
520 * @param cls NULL
521 */
522static void
523process_queue (void *cls);
524
525
526/**
527 * Insert @a req into DLL sorted by next fetch time.
528 *
529 * @param req request to insert into #req_heap
530 */
531static void
532insert_sorted (struct Request *req)
533{
534 req->hn =
535 GNUNET_CONTAINER_heap_insert (req_heap, req, req->expires.abs_value_us);
536 if (req == GNUNET_CONTAINER_heap_peek (req_heap))
537 {
538 if (NULL != t)
539 GNUNET_SCHEDULER_cancel (t);
540 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
541 t = GNUNET_SCHEDULER_add_at (req->expires, &process_queue, NULL);
542 }
543}
544
545
546/**
547 * Add record to the GNS record set for @a req.
548 *
549 * @param req the request to expand GNS record set for
550 * @param type type to use
551 * @param expiration_time when should @a rec expire
552 * @param data raw data to store
553 * @param data_len number of bytes in @a data
554 */
555static void
556add_record (struct Request *req,
557 uint32_t type,
558 struct GNUNET_TIME_Absolute expiration_time,
559 const void *data,
560 size_t data_len)
561{
562 struct Record *rec;
563
564 rec = GNUNET_malloc (sizeof(struct Record) + data_len);
565 rec->grd.data = &rec[1];
566 rec->grd.expiration_time = expiration_time.abs_value_us;
567 rec->grd.data_size = data_len;
568 rec->grd.record_type = type;
569 rec->grd.flags = GNUNET_GNSRECORD_RF_NONE;
570 GNUNET_memcpy (&rec[1], data, data_len);
571 GNUNET_CONTAINER_DLL_insert (req->rec_head, req->rec_tail, rec);
572}
573
574
575/**
576 * Closure for #check_for_glue.
577 */
578struct GlueClosure
579{
580 /**
581 * Overall request we are processing.
582 */
583 struct Request *req;
584
585 /**
586 * NS name we are looking for glue for.
587 */
588 const char *ns;
589
590 /**
591 * Set to #GNUNET_YES if glue was found.
592 */
593 int found;
594};
595
596
597/**
598 * Try to find glue records for a given NS record.
599 *
600 * @param cls a `struct GlueClosure *`
601 * @param rec record that may contain glue information
602 */
603static void
604check_for_glue (void *cls, const struct GNUNET_DNSPARSER_Record *rec)
605{
606 struct GlueClosure *gc = cls;
607 char dst[65536];
608 size_t dst_len;
609 size_t off;
610 char ip[INET6_ADDRSTRLEN + 1];
611 socklen_t ip_size = (socklen_t) sizeof(ip);
612 struct GNUNET_TIME_Absolute expiration_time;
613 struct GNUNET_TIME_Relative left;
614
615 if (0 != strcasecmp (rec->name, gc->ns))
616 return;
617 expiration_time = rec->expiration_time;
618 left = GNUNET_TIME_absolute_get_remaining (expiration_time);
619 if (0 == left.rel_value_us)
620 return; /* ignore expired glue records */
621 /* if expiration window is too short, bump it to configured minimum */
622 if (left.rel_value_us < minimum_expiration_time.rel_value_us)
623 expiration_time =
624 GNUNET_TIME_relative_to_absolute (minimum_expiration_time);
625 dst_len = sizeof(dst);
626 off = 0;
627 switch (rec->type)
628 {
629 case GNUNET_DNSPARSER_TYPE_A:
630 if (sizeof(struct in_addr) != rec->data.raw.data_len)
631 {
632 GNUNET_break (0);
633 return;
634 }
635 if (NULL == inet_ntop (AF_INET, rec->data.raw.data, ip, ip_size))
636 {
637 GNUNET_break (0);
638 return;
639 }
640 if ((GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst,
641 dst_len,
642 &off,
643 gc->req->hostname)) &&
644 (GNUNET_OK ==
645 GNUNET_DNSPARSER_builder_add_name (dst, dst_len, &off, ip)))
646 {
647 add_record (gc->req,
648 GNUNET_GNSRECORD_TYPE_GNS2DNS,
649 expiration_time,
650 dst,
651 off);
652 gc->found = GNUNET_YES;
653 }
654 break;
655
656 case GNUNET_DNSPARSER_TYPE_AAAA:
657 if (sizeof(struct in6_addr) != rec->data.raw.data_len)
658 {
659 GNUNET_break (0);
660 return;
661 }
662 if (NULL == inet_ntop (AF_INET6, rec->data.raw.data, ip, ip_size))
663 {
664 GNUNET_break (0);
665 return;
666 }
667 if ((GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst,
668 dst_len,
669 &off,
670 gc->req->hostname)) &&
671 (GNUNET_OK ==
672 GNUNET_DNSPARSER_builder_add_name (dst, dst_len, &off, ip)))
673 {
674 add_record (gc->req,
675 GNUNET_GNSRECORD_TYPE_GNS2DNS,
676 expiration_time,
677 dst,
678 off);
679 gc->found = GNUNET_YES;
680 }
681 break;
682
683 case GNUNET_DNSPARSER_TYPE_CNAME:
684 if ((GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst,
685 dst_len,
686 &off,
687 gc->req->hostname)) &&
688 (GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst,
689 dst_len,
690 &off,
691 rec->data.hostname)))
692 {
693 add_record (gc->req,
694 GNUNET_GNSRECORD_TYPE_GNS2DNS,
695 expiration_time,
696 dst,
697 off);
698 gc->found = GNUNET_YES;
699 }
700 break;
701
702 default:
703 /* useless, do nothing */
704 break;
705 }
706}
707
708
709/**
710 * Closure for #process_record().
711 */
712struct ProcessRecordContext
713{
714 /**
715 * Answer we got back and are currently parsing, or NULL
716 * if not active.
717 */
718 struct GNUNET_DNSPARSER_Packet *p;
719
720 /**
721 * Request we are processing.
722 */
723 struct Request *req;
724};
725
726
727/**
728 * We received @a rec for @a req. Remember the answer.
729 *
730 * @param cls a `struct ProcessRecordContext`
731 * @param rec response
732 */
733static void
734process_record (void *cls, const struct GNUNET_DNSPARSER_Record *rec)
735{
736 struct ProcessRecordContext *prc = cls;
737 struct Request *req = prc->req;
738 char dst[65536];
739 size_t dst_len;
740 size_t off;
741 struct GNUNET_TIME_Absolute expiration_time;
742 struct GNUNET_TIME_Relative left;
743
744 dst_len = sizeof(dst);
745 off = 0;
746 records++;
747 if (0 != strcasecmp (rec->name, req->hostname))
748 {
749 GNUNET_log (
750 GNUNET_ERROR_TYPE_DEBUG,
751 "DNS returned record from zone `%s' of type %u while resolving `%s'\n",
752 rec->name,
753 (unsigned int) rec->type,
754 req->hostname);
755 return; /* does not match hostname, might be glue, but
756 not useful for this pass! */
757 }
758 expiration_time = rec->expiration_time;
759 left = GNUNET_TIME_absolute_get_remaining (expiration_time);
760 if (0 == left.rel_value_us)
761 {
762 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
763 "DNS returned expired record for `%s'\n",
764 req->hostname);
765 GNUNET_STATISTICS_update (stats,
766 "# expired records obtained from DNS",
767 1,
768 GNUNET_NO);
769 return; /* record expired */
770 }
771
772 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
773 "DNS returned record that expires at %s for `%s'\n",
774 GNUNET_STRINGS_absolute_time_to_string (expiration_time),
775 req->hostname);
776 /* if expiration window is too short, bump it to configured minimum */
777 if (left.rel_value_us < minimum_expiration_time.rel_value_us)
778 expiration_time =
779 GNUNET_TIME_relative_to_absolute (minimum_expiration_time);
780 switch (rec->type)
781 {
782 case GNUNET_DNSPARSER_TYPE_NS: {
783 struct GlueClosure gc;
784
785 /* check for glue */
786 gc.req = req;
787 gc.ns = rec->data.hostname;
788 gc.found = GNUNET_NO;
789 for_all_records (prc->p, &check_for_glue, &gc);
790 if ((GNUNET_NO == gc.found) &&
791 (GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst,
792 dst_len,
793 &off,
794 req->hostname)) &&
795 (GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst,
796 dst_len,
797 &off,
798 rec->data.hostname)))
799 {
800 /* FIXME: actually check if this is out-of-bailiwick,
801 and if not request explicit resolution... */
802 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
803 "Converted OOB (`%s') NS record for `%s'\n",
804 rec->data.hostname,
805 rec->name);
806 add_record (req,
807 GNUNET_GNSRECORD_TYPE_GNS2DNS,
808 expiration_time,
809 dst,
810 off);
811 }
812 else
813 {
814 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
815 "Converted NS record for `%s' using glue\n",
816 rec->name);
817 }
818 break;
819 }
820
821 case GNUNET_DNSPARSER_TYPE_CNAME:
822 if (GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst,
823 dst_len,
824 &off,
825 rec->data.hostname))
826 {
827 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
828 "Converting CNAME (`%s') record for `%s'\n",
829 rec->data.hostname,
830 rec->name);
831 add_record (req, rec->type, expiration_time, dst, off);
832 }
833 break;
834
835 case GNUNET_DNSPARSER_TYPE_DNAME:
836 /* No support for DNAME in GNS yet! FIXME: support later! */
837 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
838 "FIXME: not supported: %s DNAME %s\n",
839 rec->name,
840 rec->data.hostname);
841 break;
842
843 case GNUNET_DNSPARSER_TYPE_MX:
844 if (GNUNET_OK ==
845 GNUNET_DNSPARSER_builder_add_mx (dst, dst_len, &off, rec->data.mx))
846 {
847 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
848 "Converting MX (`%s') record for `%s'\n",
849 rec->data.mx->mxhost,
850 rec->name);
851 add_record (req, rec->type, expiration_time, dst, off);
852 }
853 break;
854
855 case GNUNET_DNSPARSER_TYPE_SOA:
856 if (GNUNET_OK ==
857 GNUNET_DNSPARSER_builder_add_soa (dst, dst_len, &off, rec->data.soa))
858 {
859 /* NOTE: GNS does not really use SOAs */
860 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
861 "Converting SOA record for `%s'\n",
862 rec->name);
863 add_record (req, rec->type, expiration_time, dst, off);
864 }
865 break;
866
867 case GNUNET_DNSPARSER_TYPE_SRV:
868 if (GNUNET_OK ==
869 GNUNET_DNSPARSER_builder_add_srv (dst, dst_len, &off, rec->data.srv))
870 {
871 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
872 "Converting SRV record for `%s'\n",
873 rec->name);
874 add_record (req, rec->type, expiration_time, dst, off);
875 }
876 break;
877
878 case GNUNET_DNSPARSER_TYPE_URI:
879 if (GNUNET_OK ==
880 GNUNET_DNSPARSER_builder_add_uri (dst, dst_len, &off, rec->data.uri))
881 {
882 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
883 "Converting URI record for `%s'\n",
884 rec->name);
885 add_record (req, rec->type, expiration_time, dst, off);
886 }
887 break;
888
889 case GNUNET_DNSPARSER_TYPE_PTR:
890 if (GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst,
891 dst_len,
892 &off,
893 rec->data.hostname))
894 {
895 /* !?: what does a PTR record do in a regular TLD??? */
896 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
897 "Converting PTR record for `%s' (weird)\n",
898 rec->name);
899 add_record (req, rec->type, expiration_time, dst, off);
900 }
901 break;
902
903 case GNUNET_DNSPARSER_TYPE_CERT:
904 if (GNUNET_OK ==
905 GNUNET_DNSPARSER_builder_add_cert (dst, dst_len, &off, rec->data.cert))
906 {
907 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
908 "Converting CERT record for `%s'\n",
909 rec->name);
910 add_record (req, rec->type, expiration_time, dst, off);
911 }
912 break;
913
914 /* Rest is 'raw' encoded and just needs to be copied IF
915 the hostname matches the requested name; otherwise we
916 simply cannot use it. */
917 case GNUNET_DNSPARSER_TYPE_A:
918 case GNUNET_DNSPARSER_TYPE_AAAA:
919 case GNUNET_DNSPARSER_TYPE_TXT:
920 default:
921 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
922 "Converting record of type %u for `%s'\n",
923 (unsigned int) rec->type,
924 rec->name);
925 add_record (req,
926 rec->type,
927 expiration_time,
928 rec->data.raw.data,
929 rec->data.raw.data_len);
930 break;
931 }
932}
933
934
935static void
936store_completed_cb (void *cls, enum GNUNET_ErrorCode ec)
937{
938 static struct GNUNET_TIME_Absolute last;
939 struct Request *req = cls;
940
941 req->qe = NULL;
942 if (GNUNET_EC_NONE != ec)
943 {
944 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
945 "Failed to store zone data for `%s': %s\n",
946 req->hostname,
947 GNUNET_ErrorCode_get_hint (ec));
948 }
949 else
950 {
951 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
952 "Stored records under `%s' (%d)\n",
953 req->hostname,
954 ec);
955 }
956 total_reg_proc_dns_ns++; /* finished regular processing */
957 pending_rs--;
958 free_records (req);
959 /* compute NAMESTORE statistics */
960 {
961 static uint64_t total_ns_latency_cnt;
962 static struct GNUNET_TIME_Relative total_ns_latency;
963 struct GNUNET_TIME_Relative ns_latency;
964
965 ns_latency = GNUNET_TIME_absolute_get_duration (req->op_start_time);
966 total_ns_latency = GNUNET_TIME_relative_add (total_ns_latency, ns_latency);
967 if (0 == total_ns_latency_cnt)
968 last = GNUNET_TIME_absolute_get ();
969 total_ns_latency_cnt++;
970 if (0 == (total_ns_latency_cnt % 1000))
971 {
972 struct GNUNET_TIME_Relative delta;
973
974 delta = GNUNET_TIME_absolute_get_duration (last);
975 last = GNUNET_TIME_absolute_get ();
976 fprintf (stderr,
977 "Processed 1000 records in %s\n",
978 GNUNET_STRINGS_relative_time_to_string (delta, GNUNET_YES));
979 GNUNET_STATISTICS_set (stats,
980 "# average NAMESTORE PUT latency (μs)",
981 total_ns_latency.rel_value_us
982 / total_ns_latency_cnt,
983 GNUNET_NO);
984 }
985 }
986 /* compute and publish overall velocity */
987 if (0 == (total_reg_proc_dns_ns % 100))
988 {
989 struct GNUNET_TIME_Relative runtime;
990
991 runtime = GNUNET_TIME_absolute_get_duration (start_time_reg_proc);
992 runtime = GNUNET_TIME_relative_subtract (runtime, idle_time);
993 runtime =
994 GNUNET_TIME_relative_divide (runtime,
995 total_reg_proc_dns + total_reg_proc_dns_ns);
996 GNUNET_STATISTICS_set (stats,
997 "# Regular processing completed without NAMESTORE",
998 total_reg_proc_dns,
999 GNUNET_NO);
1000 GNUNET_STATISTICS_set (stats,
1001 "# Regular processing completed with NAMESTORE PUT",
1002 total_reg_proc_dns_ns,
1003 GNUNET_NO);
1004 GNUNET_STATISTICS_set (stats,
1005 "# average request processing latency (μs)",
1006 runtime.rel_value_us,
1007 GNUNET_NO);
1008 GNUNET_STATISTICS_set (stats,
1009 "# total time spent idle (μs)",
1010 idle_time.rel_value_us,
1011 GNUNET_NO);
1012 }
1013
1014 if (NULL == t)
1015 {
1016 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1017 t = GNUNET_SCHEDULER_add_now (&process_queue, NULL);
1018 }
1019}
1020
1021
1022/**
1023 * Function called with the result of a DNS resolution.
1024 *
1025 * @param cls closure with the `struct Request`
1026 * @param dns dns response, never NULL
1027 * @param dns_len number of bytes in @a dns
1028 */
1029static void
1030process_result (void *cls,
1031 const struct GNUNET_TUN_DnsHeader *dns,
1032 size_t dns_len)
1033{
1034 struct Request *req = cls;
1035 struct Record *rec;
1036 struct GNUNET_DNSPARSER_Packet *p;
1037 unsigned int rd_count;
1038
1039 GNUNET_assert (NULL == req->hn);
1040 if (NULL == dns)
1041 {
1042 /* stub gave up */
1043 GNUNET_CONTAINER_DLL_remove (req_head, req_tail, req);
1044 pending--;
1045 if (NULL == t)
1046 {
1047 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1048 t = GNUNET_SCHEDULER_add_now (&process_queue, NULL);
1049 }
1050 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1051 "Stub gave up on DNS reply for `%s'\n",
1052 req->hostname);
1053 GNUNET_STATISTICS_update (stats, "# DNS lookups timed out", 1, GNUNET_NO);
1054 if (req->issue_num > MAX_RETRIES)
1055 {
1056 failures++;
1057 free_request (req);
1058 GNUNET_STATISTICS_update (stats, "# requests given up on", 1, GNUNET_NO);
1059 return;
1060 }
1061 total_reg_proc_dns++;
1062 req->rs = NULL;
1063 insert_sorted (req);
1064 return;
1065 }
1066 if (req->id != dns->id)
1067 {
1068 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1069 "DNS ID did not match request, ignoring reply\n");
1070 GNUNET_STATISTICS_update (stats, "# DNS ID mismatches", 1, GNUNET_NO);
1071 return;
1072 }
1073 GNUNET_CONTAINER_DLL_remove (req_head, req_tail, req);
1074 GNUNET_DNSSTUB_resolve_cancel (req->rs);
1075 req->rs = NULL;
1076 pending--;
1077 p = GNUNET_DNSPARSER_parse ((const char *) dns, dns_len);
1078 if (NULL == p)
1079 {
1080 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1081 "Failed to parse DNS reply for `%s'\n",
1082 req->hostname);
1083 GNUNET_STATISTICS_update (stats, "# DNS parser errors", 1, GNUNET_NO);
1084 if (NULL == t)
1085 {
1086 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1087 t = GNUNET_SCHEDULER_add_now (&process_queue, NULL);
1088 }
1089 if (req->issue_num > MAX_RETRIES)
1090 {
1091 failures++;
1092 free_request (req);
1093 GNUNET_STATISTICS_update (stats, "# requests given up on", 1, GNUNET_NO);
1094 return;
1095 }
1096 insert_sorted (req);
1097 return;
1098 }
1099 /* import new records */
1100 req->issue_num = 0; /* success, reset counter! */
1101 {
1102 struct ProcessRecordContext prc = { .req = req, .p = p };
1103
1104 for_all_records (p, &process_record, &prc);
1105 }
1106 GNUNET_DNSPARSER_free_packet (p);
1107 /* count records found, determine minimum expiration time */
1108 req->expires = GNUNET_TIME_UNIT_FOREVER_ABS;
1109 {
1110 struct GNUNET_TIME_Relative dns_latency;
1111
1112 dns_latency = GNUNET_TIME_absolute_get_duration (req->op_start_time);
1113 total_dns_latency =
1114 GNUNET_TIME_relative_add (total_dns_latency, dns_latency);
1115 total_dns_latency_cnt++;
1116 if (0 == (total_dns_latency_cnt % 1000))
1117 {
1118 GNUNET_STATISTICS_set (stats,
1119 "# average DNS lookup latency (μs)",
1120 total_dns_latency.rel_value_us
1121 / total_dns_latency_cnt,
1122 GNUNET_NO);
1123 }
1124 }
1125 rd_count = 0;
1126 for (rec = req->rec_head; NULL != rec; rec = rec->next)
1127 {
1128 struct GNUNET_TIME_Absolute at;
1129
1130 at.abs_value_us = rec->grd.expiration_time;
1131 req->expires = GNUNET_TIME_absolute_min (req->expires, at);
1132 rd_count++;
1133 }
1134 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1135 "Obtained %u records for `%s'\n",
1136 rd_count,
1137 req->hostname);
1138 /* Instead of going for SOA, simplified for now to look each
1139 day in case we got an empty response */
1140 if (0 == rd_count)
1141 {
1142 req->expires = GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_DAYS);
1143 GNUNET_STATISTICS_update (stats,
1144 "# empty DNS replies (usually NXDOMAIN)",
1145 1,
1146 GNUNET_NO);
1147 }
1148 else
1149 {
1150 record_sets++;
1151 }
1152 /* convert records to namestore import format */
1153 {
1154 struct GNUNET_GNSRECORD_Data rd[GNUNET_NZL (rd_count)];
1155 unsigned int off = 0;
1156
1157 /* convert linked list into array */
1158 for (rec = req->rec_head; NULL != rec; rec = rec->next)
1159 rd[off++] = rec->grd;
1160 pending_rs++;
1161 req->op_start_time = GNUNET_TIME_absolute_get ();
1162 req->qe = GNUNET_NAMESTORE_record_set_store (ns,
1163 &req->zone->key,
1164 get_label (req),
1165 rd_count,
1166 rd,
1167 &store_completed_cb,
1168 req);
1169 GNUNET_assert (NULL != req->qe);
1170 }
1171 insert_sorted (req);
1172}
1173
1174
1175/**
1176 * Process as many requests as possible from the queue.
1177 *
1178 * @param cls NULL
1179 */
1180static void
1181process_queue (void *cls)
1182{
1183 struct Request *req;
1184 unsigned int series;
1185 void *raw;
1186 size_t raw_size;
1187 struct GNUNET_TIME_Relative delay;
1188
1189 (void) cls;
1190 delay = GNUNET_TIME_absolute_get_duration (sleep_time_reg_proc);
1191 idle_time = GNUNET_TIME_relative_add (idle_time, delay);
1192 series = 0;
1193 t = NULL;
1194 while (pending + pending_rs < THRESH)
1195 {
1196 req = GNUNET_CONTAINER_heap_peek (req_heap);
1197 if (NULL == req)
1198 break;
1199 if (NULL != req->qe)
1200 return; /* namestore op still pending */
1201 if (NULL != req->rs)
1202 {
1203 GNUNET_break (0);
1204 return; /* already submitted */
1205 }
1206 if (GNUNET_TIME_absolute_get_remaining (req->expires).rel_value_us > 0)
1207 break;
1208 GNUNET_assert (req == GNUNET_CONTAINER_heap_remove_root (req_heap));
1209 req->hn = NULL;
1210 GNUNET_CONTAINER_DLL_insert (req_head, req_tail, req);
1211 GNUNET_assert (NULL == req->rs);
1212 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1213 "Requesting resolution for `%s'\n",
1214 req->hostname);
1215 raw = build_dns_query (req, &raw_size);
1216 if (NULL == raw)
1217 {
1218 GNUNET_break (0);
1219 free_request (req);
1220 continue;
1221 }
1222 req->op_start_time = GNUNET_TIME_absolute_get ();
1223 req->rs = GNUNET_DNSSTUB_resolve (ctx, raw, raw_size, &process_result, req);
1224 GNUNET_assert (NULL != req->rs);
1225 req->issue_num++;
1226 lookups++;
1227 pending++;
1228 series++;
1229 if (series > MAX_SERIES)
1230 break;
1231 }
1232 if (pending + pending_rs >= THRESH)
1233 {
1234 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1235 "Stopped processing queue (%u+%u/%u)]\n",
1236 pending,
1237 pending_rs,
1238 THRESH);
1239 return; /* wait for replies */
1240 }
1241 req = GNUNET_CONTAINER_heap_peek (req_heap);
1242 if (NULL == req)
1243 {
1244 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1245 "Stopped processing queue: empty queue\n");
1246 return;
1247 }
1248 if (GNUNET_TIME_absolute_get_remaining (req->expires).rel_value_us > 0)
1249 {
1250 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1251 "Waiting until %s for next record (`%s') to expire\n",
1252 GNUNET_STRINGS_absolute_time_to_string (req->expires),
1253 req->hostname);
1254 if (NULL != t)
1255 GNUNET_SCHEDULER_cancel (t);
1256 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1257 t = GNUNET_SCHEDULER_add_at (req->expires, &process_queue, NULL);
1258 return;
1259 }
1260 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Throttling\n");
1261 if (NULL != t)
1262 GNUNET_SCHEDULER_cancel (t);
1263 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1264 t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY, &process_queue, NULL);
1265}
1266
1267
1268/**
1269 * Iterator called during #do_shutdown() to free requests in
1270 * the #ns_pending map.
1271 *
1272 * @param cls NULL
1273 * @param key unused
1274 * @param value the `struct Request` to free
1275 * @return #GNUNET_OK
1276 */
1277static int
1278free_request_it (void *cls, const struct GNUNET_HashCode *key, void *value)
1279{
1280 struct Request *req = value;
1281
1282 (void) cls;
1283 (void) key;
1284 free_request (req);
1285 return GNUNET_OK;
1286}
1287
1288
1289/**
1290 * Clean up and terminate the process.
1291 *
1292 * @param cls NULL
1293 */
1294static void
1295do_shutdown (void *cls)
1296{
1297 struct Request *req;
1298 struct Zone *zone;
1299
1300 (void) cls;
1301 if (NULL != id)
1302 {
1303 GNUNET_IDENTITY_disconnect (id);
1304 id = NULL;
1305 }
1306 if (NULL != t)
1307 {
1308 GNUNET_SCHEDULER_cancel (t);
1309 t = NULL;
1310 }
1311 while (NULL != (req = req_head))
1312 {
1313 GNUNET_CONTAINER_DLL_remove (req_head, req_tail, req);
1314 if (NULL != req->qe)
1315 GNUNET_NAMESTORE_cancel (req->qe);
1316 free_request (req);
1317 }
1318 while (NULL != (req = GNUNET_CONTAINER_heap_remove_root (req_heap)))
1319 {
1320 req->hn = NULL;
1321 if (NULL != req->qe)
1322 GNUNET_NAMESTORE_cancel (req->qe);
1323 free_request (req);
1324 }
1325 if (NULL != zone_it)
1326 {
1327 GNUNET_NAMESTORE_zone_iteration_stop (zone_it);
1328 zone_it = NULL;
1329 }
1330 if (NULL != ns)
1331 {
1332 GNUNET_NAMESTORE_disconnect (ns);
1333 ns = NULL;
1334 }
1335 if (NULL != ctx)
1336 {
1337 GNUNET_DNSSTUB_stop (ctx);
1338 ctx = NULL;
1339 }
1340 if (NULL != req_heap)
1341 {
1342 GNUNET_CONTAINER_heap_destroy (req_heap);
1343 req_heap = NULL;
1344 }
1345 if (NULL != ns_pending)
1346 {
1347 GNUNET_CONTAINER_multihashmap_iterate (ns_pending, &free_request_it, NULL);
1348 GNUNET_CONTAINER_multihashmap_destroy (ns_pending);
1349 ns_pending = NULL;
1350 }
1351 while (NULL != (zone = zone_head))
1352 {
1353 GNUNET_CONTAINER_DLL_remove (zone_head, zone_tail, zone);
1354 GNUNET_free (zone->domain);
1355 GNUNET_free (zone);
1356 }
1357 if (NULL != stats)
1358 {
1359 GNUNET_STATISTICS_destroy (stats, GNUNET_NO);
1360 stats = NULL;
1361 }
1362}
1363
1364
1365/**
1366 * Iterate over all of the zones we care about and see which records
1367 * we may need to re-fetch when.
1368 *
1369 * @param cls NULL
1370 */
1371static void
1372iterate_zones (void *cls);
1373
1374
1375/**
1376 * Function called if #GNUNET_NAMESTORE_records_lookup() failed.
1377 * Just logs an error.
1378 *
1379 * @param cls a `struct Zone`
1380 */
1381static void
1382ns_lookup_error_cb (void *cls)
1383{
1384 struct Zone *zone = cls;
1385
1386 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1387 "Failed to load data from namestore for zone `%s'\n",
1388 zone->domain);
1389 zone_it = NULL;
1390 ns_iterator_trigger_next = 0;
1391 iterate_zones (NULL);
1392}
1393
1394
1395/**
1396 * Process a record that was stored in the namestore.
1397 *
1398 * @param cls a `struct Zone *`
1399 * @param key private key of the zone
1400 * @param label label of the records
1401 * @param rd_count number of entries in @a rd array, 0 if label was deleted
1402 * @param rd array of records with data to store
1403 */
1404static void
1405ns_lookup_result_cb (void *cls,
1406 const struct GNUNET_CRYPTO_PrivateKey *key,
1407 const char *label,
1408 unsigned int rd_count,
1409 const struct GNUNET_GNSRECORD_Data *rd)
1410{
1411 struct Zone *zone = cls;
1412 struct Request *req;
1413 struct GNUNET_HashCode hc;
1414 char *fqdn;
1415
1416 ns_iterator_trigger_next--;
1417 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1418 "Obtained NAMESTORE reply, %llu left in round\n",
1419 (unsigned long long) ns_iterator_trigger_next);
1420 if (0 == ns_iterator_trigger_next)
1421 {
1422 ns_iterator_trigger_next = NS_BATCH_SIZE;
1423 GNUNET_STATISTICS_update (stats,
1424 "# NAMESTORE records requested from cache",
1425 ns_iterator_trigger_next,
1426 GNUNET_NO);
1427 GNUNET_NAMESTORE_zone_iterator_next (zone_it, ns_iterator_trigger_next);
1428 }
1429 GNUNET_asprintf (&fqdn, "%s.%s", label, zone->domain);
1430 GNUNET_CRYPTO_hash (fqdn, strlen (fqdn) + 1, &hc);
1431 GNUNET_free (fqdn);
1432 req = GNUNET_CONTAINER_multihashmap_get (ns_pending, &hc);
1433 if (NULL == req)
1434 {
1435 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1436 "Ignoring record `%s' in zone `%s': not on my list!\n",
1437 label,
1438 zone->domain);
1439 return;
1440 }
1441 GNUNET_assert (GNUNET_OK ==
1442 GNUNET_CONTAINER_multihashmap_remove (ns_pending, &hc, req));
1443 GNUNET_break (0 == GNUNET_memcmp (key, &req->zone->key));
1444 GNUNET_break (0 == strcasecmp (label, get_label (req)));
1445 for (unsigned int i = 0; i < rd_count; i++)
1446 {
1447 struct GNUNET_TIME_Absolute at;
1448
1449 if (0 != (rd->flags & GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION))
1450 {
1451 struct GNUNET_TIME_Relative rel;
1452
1453 rel.rel_value_us = rd->expiration_time;
1454 at = GNUNET_TIME_relative_to_absolute (rel);
1455 }
1456 else
1457 {
1458 at.abs_value_us = rd->expiration_time;
1459 }
1460 add_record (req, rd->record_type, at, rd->data, rd->data_size);
1461 }
1462 if (0 == rd_count)
1463 {
1464 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1465 "Empty record set in namestore for `%s'\n",
1466 req->hostname);
1467 }
1468 else
1469 {
1470 unsigned int pos = 0;
1471
1472 cached++;
1473 req->expires = GNUNET_TIME_UNIT_FOREVER_ABS;
1474 for (struct Record *rec = req->rec_head; NULL != rec; rec = rec->next)
1475 {
1476 struct GNUNET_TIME_Absolute at;
1477
1478 at.abs_value_us = rec->grd.expiration_time;
1479 req->expires = GNUNET_TIME_absolute_min (req->expires, at);
1480 pos++;
1481 }
1482 if (0 == pos)
1483 req->expires = GNUNET_TIME_UNIT_ZERO_ABS;
1484 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1485 "Hot-start with %u existing records for `%s'\n",
1486 pos,
1487 req->hostname);
1488 }
1489 free_records (req);
1490
1491 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1492 "Adding `%s' to worklist to start at %s\n",
1493 req->hostname,
1494 GNUNET_STRINGS_absolute_time_to_string (req->expires));
1495 insert_sorted (req);
1496}
1497
1498
1499/**
1500 * Add @a hostname to the list of requests to be made.
1501 *
1502 * @param hostname name to resolve
1503 */
1504static void
1505queue (const char *hostname)
1506{
1507 struct Request *req;
1508 const char *dot;
1509 struct Zone *zone;
1510 size_t hlen;
1511 struct GNUNET_HashCode hc;
1512
1513 if (GNUNET_OK != GNUNET_DNSPARSER_check_name (hostname))
1514 {
1515 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1516 "Refusing invalid hostname `%s'\n",
1517 hostname);
1518 rejects++;
1519 return;
1520 }
1521 dot = strchr (hostname, (unsigned char) '.');
1522 if (NULL == dot)
1523 {
1524 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1525 "Refusing invalid hostname `%s' (lacks '.')\n",
1526 hostname);
1527 rejects++;
1528 return;
1529 }
1530 for (zone = zone_head; NULL != zone; zone = zone->next)
1531 if (0 == strcmp (zone->domain, dot + 1))
1532 break;
1533 if (NULL == zone)
1534 {
1535 rejects++;
1536 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1537 "Domain name `%s' not in ego list!\n",
1538 dot + 1);
1539 return;
1540 }
1541
1542 hlen = strlen (hostname) + 1;
1543 req = GNUNET_malloc (sizeof(struct Request) + hlen);
1544 req->zone = zone;
1545 req->hostname = (char *) &req[1];
1546 GNUNET_memcpy (req->hostname, hostname, hlen);
1547 req->id = (uint16_t) GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE,
1548 UINT16_MAX);
1549 GNUNET_CRYPTO_hash (req->hostname, hlen, &hc);
1550 if (GNUNET_OK != GNUNET_CONTAINER_multihashmap_put (
1551 ns_pending,
1552 &hc,
1553 req,
1554 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))
1555 {
1556 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1557 "Duplicate hostname `%s' ignored\n",
1558 hostname);
1559 GNUNET_free (req);
1560 return;
1561 }
1562}
1563
1564
1565/**
1566 * We have completed the initial iteration over the namestore's database.
1567 * This function is called on each of the remaining records in
1568 * #move_to_queue to #queue() them, as we will simply not find existing
1569 * records for them any longer.
1570 *
1571 * @param cls NULL
1572 * @param key unused
1573 * @param value a `struct Request`
1574 * @return #GNUNET_OK (continue to iterate)
1575 */
1576static int
1577move_to_queue (void *cls, const struct GNUNET_HashCode *key, void *value)
1578{
1579 struct Request *req = value;
1580
1581 (void) cls;
1582 (void) key;
1583 insert_sorted (req);
1584 return GNUNET_OK;
1585}
1586
1587
1588/**
1589 * Iterate over all of the zones we care about and see which records
1590 * we may need to re-fetch when.
1591 *
1592 * @param cls NULL
1593 */
1594static void
1595iterate_zones (void *cls)
1596{
1597 static struct Zone *last;
1598
1599 (void) cls;
1600 if (NULL != zone_it)
1601 {
1602 zone_it = NULL;
1603 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1604 "Finished iteration over zone `%s'!\n",
1605 last->domain);
1606 /* subtract left-overs from previous iteration */
1607 GNUNET_STATISTICS_update (stats,
1608 "# NAMESTORE records requested from cache",
1609 (long long) (-ns_iterator_trigger_next),
1610 GNUNET_NO);
1611 ns_iterator_trigger_next = 0;
1612 }
1613 GNUNET_assert (NULL != zone_tail);
1614 if (zone_tail == last)
1615 {
1616 /* Done iterating over relevant zones in NAMESTORE, move
1617 rest of hash map to work queue as well. */
1618 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1619 "Finished all NAMESTORE iterations!\n");
1620 GNUNET_STATISTICS_set (stats,
1621 "# Domain names without cached reply",
1622 GNUNET_CONTAINER_multihashmap_size (ns_pending),
1623 GNUNET_NO);
1624 GNUNET_CONTAINER_multihashmap_iterate (ns_pending, &move_to_queue, NULL);
1625 GNUNET_CONTAINER_multihashmap_destroy (ns_pending);
1626 ns_pending = NULL;
1627 start_time_reg_proc = GNUNET_TIME_absolute_get ();
1628 total_reg_proc_dns = 0;
1629 total_reg_proc_dns_ns = 0;
1630 return;
1631 }
1632 if (NULL == last)
1633 last = zone_head;
1634 else
1635 last = last->next;
1636 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1637 "Starting iteration over zone `%s'!\n",
1638 last->domain);
1639 /* subtract left-overs from previous iteration */
1640 GNUNET_STATISTICS_update (stats,
1641 "# NAMESTORE records requested from cache",
1642 1,
1643 GNUNET_NO);
1644 ns_iterator_trigger_next = 1;
1645 GNUNET_STATISTICS_update (stats, "# zones iterated", 1, GNUNET_NO);
1646 zone_it = GNUNET_NAMESTORE_zone_iteration_start (ns,
1647 &last->key,
1648 &ns_lookup_error_cb,
1649 NULL,
1650 &ns_lookup_result_cb,
1651 last,
1652 &iterate_zones,
1653 NULL);
1654}
1655
1656
1657/**
1658 * Begin processing hostnames from stdin.
1659 *
1660 * @param cls NULL
1661 */
1662static void
1663process_stdin (void *cls)
1664{
1665 static struct GNUNET_TIME_Absolute last;
1666 static uint64_t idot;
1667 char hn[256];
1668
1669 (void) cls;
1670 t = NULL;
1671 if (NULL != id)
1672 {
1673 GNUNET_IDENTITY_disconnect (id);
1674 id = NULL;
1675 }
1676 while (NULL != fgets (hn, sizeof(hn), stdin))
1677 {
1678 if (strlen (hn) > 0)
1679 hn[strlen (hn) - 1] = '\0'; /* eat newline */
1680 if (0 == idot)
1681 last = GNUNET_TIME_absolute_get ();
1682 idot++;
1683 if (0 == idot % 100000)
1684 {
1685 struct GNUNET_TIME_Relative delta;
1686
1687 delta = GNUNET_TIME_absolute_get_duration (last);
1688 last = GNUNET_TIME_absolute_get ();
1689 fprintf (stderr,
1690 "Read 100000 domain names in %s\n",
1691 GNUNET_STRINGS_relative_time_to_string (delta, GNUNET_YES));
1692 GNUNET_STATISTICS_set (stats, "# domain names provided", idot, GNUNET_NO);
1693 }
1694 queue (hn);
1695 }
1696 fprintf (stderr,
1697 "Done reading %llu domain names\n",
1698 (unsigned long long) idot);
1699 GNUNET_STATISTICS_set (stats, "# domain names provided", idot, GNUNET_NO);
1700 iterate_zones (NULL);
1701}
1702
1703
1704/**
1705 * Method called to inform about the egos of this peer.
1706 *
1707 * When used with #GNUNET_IDENTITY_connect, this function is
1708 * initially called for all egos and then again whenever a
1709 * ego's name changes or if it is deleted. At the end of
1710 * the initial pass over all egos, the function is once called
1711 * with 'NULL' for @a ego. That does NOT mean that the callback won't
1712 * be invoked in the future or that there was an error.
1713 *
1714 * When used with #GNUNET_IDENTITY_create or #GNUNET_IDENTITY_get, this
1715 * function is only called ONCE, and 'NULL' being passed in @a ego does
1716 * indicate an error (for example because name is taken or no default value is
1717 * known). If @a ego is non-NULL and if '*ctx' is set in those callbacks, the
1718 * value WILL be passed to a subsequent call to the identity callback of
1719 * #GNUNET_IDENTITY_connect (if that one was not NULL).
1720 *
1721 * When an identity is renamed, this function is called with the
1722 * (known) @a ego but the NEW @a name.
1723 *
1724 * When an identity is deleted, this function is called with the
1725 * (known) ego and "NULL" for the @a name. In this case,
1726 * the @a ego is henceforth invalid (and the @a ctx should also be
1727 * cleaned up).
1728 *
1729 * @param cls closure
1730 * @param ego ego handle, NULL for end of list
1731 * @param ctx context for application to store data for this ego
1732 * (during the lifetime of this process, initially NULL)
1733 * @param name name assigned by the user for this ego,
1734 * NULL if the user just deleted the ego and it
1735 * must thus no longer be used
1736 */
1737static void
1738identity_cb (void *cls,
1739 struct GNUNET_IDENTITY_Ego *ego,
1740 void **ctx,
1741 const char *name)
1742{
1743 (void) cls;
1744 (void) ctx;
1745
1746 if (NULL == ego)
1747 {
1748 /* end of iteration */
1749 if (NULL == zone_head)
1750 {
1751 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "No zone found\n");
1752 GNUNET_SCHEDULER_shutdown ();
1753 return;
1754 }
1755 /* zone_head non-null, process hostnames from stdin */
1756 t = GNUNET_SCHEDULER_add_now (&process_stdin, NULL);
1757 return;
1758 }
1759 if (NULL != name)
1760 {
1761 struct Zone *zone;
1762
1763 zone = GNUNET_new (struct Zone);
1764 zone->key = *GNUNET_IDENTITY_ego_get_private_key (ego);
1765 zone->domain = GNUNET_strdup (name);
1766 GNUNET_CONTAINER_DLL_insert (zone_head, zone_tail, zone);
1767 }
1768}
1769
1770
1771/**
1772 * Process requests from the queue, then if the queue is
1773 * not empty, try again.
1774 *
1775 * @param cls NULL
1776 * @param args remaining command-line arguments
1777 * @param cfgfile name of the configuration file used (for saving, can be NULL!)
1778 * @param cfg configuration
1779 */
1780static void
1781run (void *cls,
1782 char *const *args,
1783 const char *cfgfile,
1784 const struct GNUNET_CONFIGURATION_Handle *cfg)
1785{
1786 (void) cls;
1787 (void) args;
1788 (void) cfgfile;
1789 stats = GNUNET_STATISTICS_create ("zoneimport", cfg);
1790 req_heap = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
1791 ns_pending = GNUNET_CONTAINER_multihashmap_create (map_size, GNUNET_NO);
1792 if (NULL == ns_pending)
1793 {
1794 fprintf (stderr, "Failed to allocate memory for main hash map\n");
1795 return;
1796 }
1797 ctx = GNUNET_DNSSTUB_start (256);
1798 if (NULL == ctx)
1799 {
1800 fprintf (stderr, "Failed to initialize GNUnet DNS STUB\n");
1801 return;
1802 }
1803 if (NULL == args[0])
1804 {
1805 fprintf (stderr,
1806 "You must provide a list of DNS resolvers on the command line\n");
1807 return;
1808 }
1809 for (unsigned int i = 0; NULL != args[i]; i++)
1810 {
1811 if (GNUNET_OK != GNUNET_DNSSTUB_add_dns_ip (ctx, args[i]))
1812 {
1813 fprintf (stderr, "Failed to use `%s' for DNS resolver\n", args[i]);
1814 return;
1815 }
1816 }
1817
1818
1819 GNUNET_SCHEDULER_add_shutdown (&do_shutdown, NULL);
1820 ns = GNUNET_NAMESTORE_connect (cfg);
1821 if (NULL == ns)
1822 {
1823 GNUNET_SCHEDULER_shutdown ();
1824 return;
1825 }
1826 id = GNUNET_IDENTITY_connect (cfg, &identity_cb, NULL);
1827}
1828
1829
1830/**
1831 * Call with IP address of resolver to query.
1832 *
1833 * @param argc should be 2
1834 * @param argv[1] should contain IP address
1835 * @return 0 on success
1836 */
1837int
1838main (int argc, char *const *argv)
1839{
1840 struct GNUNET_GETOPT_CommandLineOption options[] =
1841 { GNUNET_GETOPT_option_uint ('s',
1842 "size",
1843 "MAPSIZE",
1844 gettext_noop (
1845 "size to use for the main hash map"),
1846 &map_size),
1847 GNUNET_GETOPT_option_relative_time (
1848 'm',
1849 "minimum-expiration",
1850 "RELATIVETIME",
1851 gettext_noop ("minimum expiration time we assume for imported records"),
1852 &minimum_expiration_time),
1853 GNUNET_GETOPT_OPTION_END };
1854 int ret;
1855
1856 if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
1857 return 2;
1858 if (GNUNET_OK != (ret = GNUNET_PROGRAM_run (argc,
1859 argv,
1860 "gnunet-zoneimport",
1861 "import DNS zone into namestore",
1862 options,
1863 &run,
1864 NULL)))
1865 return ret;
1866 GNUNET_free_nz ((void *) argv);
1867 fprintf (stderr,
1868 "Rejected %u names, had %u cached, did %u lookups, stored %u record sets\n"
1869 "Found %u records, %u lookups failed, %u/%u pending on shutdown\n",
1870 rejects,
1871 cached,
1872 lookups,
1873 record_sets,
1874 records,
1875 failures,
1876 pending,
1877 pending_rs);
1878 return 0;
1879}
1880
1881
1882/* end of gnunet-zoneimport.c */
generated by cgit v1.2.3 (git 2.39.1) at 2024-05-17 14:36:32 +0000