aboutsummaryrefslogtreecommitdiff
path: root/src/namestore/gnunet-zoneimport.c
diff options
context:
space:
mode:
Diffstat (limited to 'src/namestore/gnunet-zoneimport.c')
-rw-r--r--src/namestore/gnunet-zoneimport.c1884
1 files changed, 0 insertions, 1884 deletions
diff --git a/src/namestore/gnunet-zoneimport.c b/src/namestore/gnunet-zoneimport.c
deleted file mode 100644
index 68b43a016..000000000
--- a/src/namestore/gnunet-zoneimport.c
+++ /dev/null
@@ -1,1884 +0,0 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2018 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20/**
21 * @file src/namestore/gnunet-zoneimport.c
22 * @brief import a DNS zone for publication in GNS, incremental
23 * @author Christian Grothoff
24 */
25#include "platform.h"
26#include <gnunet_util_lib.h>
27#include <gnunet_dnsstub_lib.h>
28#include <gnunet_dnsparser_lib.h>
29#include <gnunet_gnsrecord_lib.h>
30#include <gnunet_namestore_service.h>
31#include <gnunet_statistics_service.h>
32#include <gnunet_identity_service.h>
33
34
35/**
36 * Maximum number of queries pending at the same time.
37 */
38#define THRESH 100
39
40/**
41 * TIME_THRESH is in usecs. How quickly do we submit fresh queries.
42 * Used as an additional throttle.
43 */
44#define TIME_THRESH 10
45
46/**
47 * How often do we retry a query before giving up for good?
48 */
49#define MAX_RETRIES 5
50
51/**
52 * How many DNS requests do we at most issue in rapid series?
53 */
54#define MAX_SERIES 10
55
56/**
57 * How long do we wait at least between series of requests?
58 */
59#define SERIES_DELAY \
60 GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MICROSECONDS, 10)
61
62/**
63 * How long do DNS records have to last at least after being imported?
64 */
65static struct GNUNET_TIME_Relative minimum_expiration_time;
66
67/**
68 * How many requests do we request from NAMESTORE in one batch
69 * during our initial iteration?
70 */
71#define NS_BATCH_SIZE 1024
72
73/**
74 * Some zones may include authoritative records for other
75 * zones, such as foo.com.uk or bar.com.fr. As for GNS
76 * each dot represents a zone cut, we then need to create a
77 * zone on-the-fly to capture those records properly.
78 */
79struct Zone
80{
81 /**
82 * Kept in a DLL.
83 */
84 struct Zone *next;
85
86 /**
87 * Kept in a DLL.
88 */
89 struct Zone *prev;
90
91 /**
92 * Domain of the zone (i.e. "fr" or "com.fr")
93 */
94 char *domain;
95
96 /**
97 * Private key of the zone.
98 */
99 struct GNUNET_IDENTITY_PrivateKey key;
100};
101
102
103/**
104 * Record for the request to be stored by GNS.
105 */
106struct Record
107{
108 /**
109 * Kept in a DLL.
110 */
111 struct Record *next;
112
113 /**
114 * Kept in a DLL.
115 */
116 struct Record *prev;
117
118 /**
119 * GNS record.
120 */
121 struct GNUNET_GNSRECORD_Data grd;
122};
123
124
125/**
126 * Request we should make. We keep this struct in memory per request,
127 * thus optimizing it is crucial for the overall memory consumption of
128 * the zone importer.
129 */
130struct Request
131{
132 /**
133 * Requests are kept in a heap while waiting to be resolved.
134 */
135 struct GNUNET_CONTAINER_HeapNode *hn;
136
137 /**
138 * Active requests are kept in a DLL.
139 */
140 struct Request *next;
141
142 /**
143 * Active requests are kept in a DLL.
144 */
145 struct Request *prev;
146
147 /**
148 * Head of records that should be published in GNS for
149 * this hostname.
150 */
151 struct Record *rec_head;
152
153 /**
154 * Tail of records that should be published in GNS for
155 * this hostname.
156 */
157 struct Record *rec_tail;
158
159 /**
160 * Socket used to make the request, NULL if not active.
161 */
162 struct GNUNET_DNSSTUB_RequestSocket *rs;
163
164 /**
165 * Hostname we are resolving, allocated at the end of
166 * this struct (optimizing memory consumption by reducing
167 * total number of allocations).
168 */
169 char *hostname;
170
171 /**
172 * Namestore operation pending for this record.
173 */
174 struct GNUNET_NAMESTORE_QueueEntry *qe;
175
176 /**
177 * Zone responsible for this request.
178 */
179 const struct Zone *zone;
180
181 /**
182 * At what time does the (earliest) of the returned records
183 * for this name expire? At this point, we need to re-fetch
184 * the record.
185 */
186 struct GNUNET_TIME_Absolute expires;
187
188 /**
189 * While we are fetching the record, the value is set to the
190 * starting time of the DNS operation. While doing a
191 * NAMESTORE store, again set to the start time of the
192 * NAMESTORE operation.
193 */
194 struct GNUNET_TIME_Absolute op_start_time;
195
196 /**
197 * How often did we issue this query? (And failed, reset
198 * to zero once we were successful.)
199 */
200 unsigned int issue_num;
201
202 /**
203 * random 16-bit DNS query identifier.
204 */
205 uint16_t id;
206};
207
208
209/**
210 * Command-line argument specifying desired size of the hash map with
211 * all of our pending names. Usually, we use an automatically growing
212 * map, but this is only OK up to about a million entries. Above that
213 * number, the user must explicitly specify the size at startup.
214 */
215static unsigned int map_size = 1024;
216
217/**
218 * Handle to the identity service.
219 */
220static struct GNUNET_IDENTITY_Handle *id;
221
222/**
223 * Namestore handle.
224 */
225static struct GNUNET_NAMESTORE_Handle *ns;
226
227/**
228 * Handle to the statistics service.
229 */
230static struct GNUNET_STATISTICS_Handle *stats;
231
232/**
233 * Context for DNS resolution.
234 */
235static struct GNUNET_DNSSTUB_Context *ctx;
236
237/**
238 * The number of DNS queries that are outstanding
239 */
240static unsigned int pending;
241
242/**
243 * The number of NAMESTORE record store operations that are outstanding
244 */
245static unsigned int pending_rs;
246
247/**
248 * Number of lookups we performed overall.
249 */
250static unsigned int lookups;
251
252/**
253 * Number of records we had cached.
254 */
255static unsigned int cached;
256
257/**
258 * How many hostnames did we reject (malformed).
259 */
260static unsigned int rejects;
261
262/**
263 * Number of lookups that failed.
264 */
265static unsigned int failures;
266
267/**
268 * Number of records we found.
269 */
270static unsigned int records;
271
272/**
273 * Number of record sets given to namestore.
274 */
275static unsigned int record_sets;
276
277/**
278 * Heap of all requests to perform, sorted by
279 * the time we should next do the request (i.e. by expires).
280 */
281static struct GNUNET_CONTAINER_Heap *req_heap;
282
283/**
284 * Active requests are kept in a DLL.
285 */
286static struct Request *req_head;
287
288/**
289 * Active requests are kept in a DLL.
290 */
291static struct Request *req_tail;
292
293/**
294 * Main task.
295 */
296static struct GNUNET_SCHEDULER_Task *t;
297
298/**
299 * Hash map of requests for which we may still get a response from
300 * the namestore. Set to NULL once the initial namestore iteration
301 * is done.
302 */
303static struct GNUNET_CONTAINER_MultiHashMap *ns_pending;
304
305/**
306 * Current zone iteration handle.
307 */
308static struct GNUNET_NAMESTORE_ZoneIterator *zone_it;
309
310/**
311 * Head of list of zones we are managing.
312 */
313static struct Zone *zone_head;
314
315/**
316 * Tail of list of zones we are managing.
317 */
318static struct Zone *zone_tail;
319
320/**
321 * After how many more results must #ns_lookup_result_cb() ask
322 * the namestore for more?
323 */
324static uint64_t ns_iterator_trigger_next;
325
326/**
327 * Number of DNS requests counted in latency total.
328 */
329static uint64_t total_dns_latency_cnt;
330
331/**
332 * Sum of DNS latencies observed.
333 */
334static struct GNUNET_TIME_Relative total_dns_latency;
335
336/**
337 * Number of records processed (DNS lookup, no NAMESTORE) in total.
338 */
339static uint64_t total_reg_proc_dns;
340
341/**
342 * Number of records processed (DNS lookup, with NAMESTORE) in total.
343 */
344static uint64_t total_reg_proc_dns_ns;
345
346/**
347 * Start time of the regular processing.
348 */
349static struct GNUNET_TIME_Absolute start_time_reg_proc;
350
351/**
352 * Last time we worked before going idle.
353 */
354static struct GNUNET_TIME_Absolute sleep_time_reg_proc;
355
356/**
357 * Time we slept just waiting for work.
358 */
359static struct GNUNET_TIME_Relative idle_time;
360
361
362/**
363 * Callback for #for_all_records
364 *
365 * @param cls closure
366 * @param rec a DNS record
367 */
368typedef void (*RecordProcessor) (void *cls,
369 const struct GNUNET_DNSPARSER_Record *rec);
370
371
372/**
373 * Call @a rp for each record in @a p, regardless of
374 * what response section it is in.
375 *
376 * @param p packet from DNS
377 * @param rp function to call
378 * @param rp_cls closure for @a rp
379 */
380static void
381for_all_records (const struct GNUNET_DNSPARSER_Packet *p,
382 RecordProcessor rp,
383 void *rp_cls)
384{
385 for (unsigned int i = 0; i < p->num_answers; i++)
386 {
387 struct GNUNET_DNSPARSER_Record *rs = &p->answers[i];
388
389 rp (rp_cls, rs);
390 }
391 for (unsigned int i = 0; i < p->num_authority_records; i++)
392 {
393 struct GNUNET_DNSPARSER_Record *rs = &p->authority_records[i];
394
395 rp (rp_cls, rs);
396 }
397 for (unsigned int i = 0; i < p->num_additional_records; i++)
398 {
399 struct GNUNET_DNSPARSER_Record *rs = &p->additional_records[i];
400
401 rp (rp_cls, rs);
402 }
403}
404
405
406/**
407 * Return just the label of the hostname in @a req.
408 *
409 * @param req request to process hostname of
410 * @return statically allocated pointer to the label,
411 * overwritten upon the next request!
412 */
413static const char *
414get_label (struct Request *req)
415{
416 static char label[64];
417 const char *dot;
418
419 dot = strchr (req->hostname, (unsigned char) '.');
420 if (NULL == dot)
421 {
422 GNUNET_break (0);
423 return NULL;
424 }
425 if (((size_t) (dot - req->hostname)) >= sizeof(label))
426 {
427 GNUNET_break (0);
428 return NULL;
429 }
430 GNUNET_memcpy (label, req->hostname, dot - req->hostname);
431 label[dot - req->hostname] = '\0';
432 return label;
433}
434
435
436/**
437 * Build DNS query for @a hostname.
438 *
439 * @param hostname host to build query for
440 * @param raw_size[out] number of bytes in the query
441 * @return NULL on error, otherwise pointer to statically (!)
442 * allocated query buffer
443 */
444static void *
445build_dns_query (struct Request *req, size_t *raw_size)
446{
447 static char raw[512];
448 char *rawp;
449 struct GNUNET_DNSPARSER_Packet p;
450 struct GNUNET_DNSPARSER_Query q;
451 int ret;
452
453 q.name = (char *) req->hostname;
454 q.type = GNUNET_DNSPARSER_TYPE_NS;
455 q.dns_traffic_class = GNUNET_TUN_DNS_CLASS_INTERNET;
456
457 memset (&p, 0, sizeof(p));
458 p.num_queries = 1;
459 p.queries = &q;
460 p.id = req->id;
461 ret = GNUNET_DNSPARSER_pack (&p, UINT16_MAX, &rawp, raw_size);
462 if (GNUNET_OK != ret)
463 {
464 if (GNUNET_NO == ret)
465 GNUNET_free (rawp);
466 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
467 "Failed to pack query for hostname `%s'\n",
468 req->hostname);
469 rejects++;
470 return NULL;
471 }
472 if (*raw_size > sizeof(raw))
473 {
474 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
475 "Failed to pack query for hostname `%s'\n",
476 req->hostname);
477 rejects++;
478 GNUNET_break (0);
479 GNUNET_free (rawp);
480 return NULL;
481 }
482 GNUNET_memcpy (raw, rawp, *raw_size);
483 GNUNET_free (rawp);
484 return raw;
485}
486
487
488/**
489 * Free records associated with @a req.
490 *
491 * @param req request to free records of
492 */
493static void
494free_records (struct Request *req)
495{
496 struct Record *rec;
497
498 /* Free records */
499 while (NULL != (rec = req->rec_head))
500 {
501 GNUNET_CONTAINER_DLL_remove (req->rec_head, req->rec_tail, rec);
502 GNUNET_free (rec);
503 }
504}
505
506
507/**
508 * Free @a req and data structures reachable from it.
509 *
510 * @param req request to free
511 */
512static void
513free_request (struct Request *req)
514{
515 free_records (req);
516 GNUNET_free (req);
517}
518
519
520/**
521 * Process as many requests as possible from the queue.
522 *
523 * @param cls NULL
524 */
525static void
526process_queue (void *cls);
527
528
529/**
530 * Insert @a req into DLL sorted by next fetch time.
531 *
532 * @param req request to insert into #req_heap
533 */
534static void
535insert_sorted (struct Request *req)
536{
537 req->hn =
538 GNUNET_CONTAINER_heap_insert (req_heap, req, req->expires.abs_value_us);
539 if (req == GNUNET_CONTAINER_heap_peek (req_heap))
540 {
541 if (NULL != t)
542 GNUNET_SCHEDULER_cancel (t);
543 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
544 t = GNUNET_SCHEDULER_add_at (req->expires, &process_queue, NULL);
545 }
546}
547
548
549/**
550 * Add record to the GNS record set for @a req.
551 *
552 * @param req the request to expand GNS record set for
553 * @param type type to use
554 * @param expiration_time when should @a rec expire
555 * @param data raw data to store
556 * @param data_len number of bytes in @a data
557 */
558static void
559add_record (struct Request *req,
560 uint32_t type,
561 struct GNUNET_TIME_Absolute expiration_time,
562 const void *data,
563 size_t data_len)
564{
565 struct Record *rec;
566
567 rec = GNUNET_malloc (sizeof(struct Record) + data_len);
568 rec->grd.data = &rec[1];
569 rec->grd.expiration_time = expiration_time.abs_value_us;
570 rec->grd.data_size = data_len;
571 rec->grd.record_type = type;
572 rec->grd.flags = GNUNET_GNSRECORD_RF_NONE;
573 GNUNET_memcpy (&rec[1], data, data_len);
574 GNUNET_CONTAINER_DLL_insert (req->rec_head, req->rec_tail, rec);
575}
576
577
578/**
579 * Closure for #check_for_glue.
580 */
581struct GlueClosure
582{
583 /**
584 * Overall request we are processing.
585 */
586 struct Request *req;
587
588 /**
589 * NS name we are looking for glue for.
590 */
591 const char *ns;
592
593 /**
594 * Set to #GNUNET_YES if glue was found.
595 */
596 int found;
597};
598
599
600/**
601 * Try to find glue records for a given NS record.
602 *
603 * @param cls a `struct GlueClosure *`
604 * @param rec record that may contain glue information
605 */
606static void
607check_for_glue (void *cls, const struct GNUNET_DNSPARSER_Record *rec)
608{
609 struct GlueClosure *gc = cls;
610 char dst[65536];
611 size_t dst_len;
612 size_t off;
613 char ip[INET6_ADDRSTRLEN + 1];
614 socklen_t ip_size = (socklen_t) sizeof(ip);
615 struct GNUNET_TIME_Absolute expiration_time;
616 struct GNUNET_TIME_Relative left;
617
618 if (0 != strcasecmp (rec->name, gc->ns))
619 return;
620 expiration_time = rec->expiration_time;
621 left = GNUNET_TIME_absolute_get_remaining (expiration_time);
622 if (0 == left.rel_value_us)
623 return; /* ignore expired glue records */
624 /* if expiration window is too short, bump it to configured minimum */
625 if (left.rel_value_us < minimum_expiration_time.rel_value_us)
626 expiration_time =
627 GNUNET_TIME_relative_to_absolute (minimum_expiration_time);
628 dst_len = sizeof(dst);
629 off = 0;
630 switch (rec->type)
631 {
632 case GNUNET_DNSPARSER_TYPE_A:
633 if (sizeof(struct in_addr) != rec->data.raw.data_len)
634 {
635 GNUNET_break (0);
636 return;
637 }
638 if (NULL == inet_ntop (AF_INET, rec->data.raw.data, ip, ip_size))
639 {
640 GNUNET_break (0);
641 return;
642 }
643 if ((GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst,
644 dst_len,
645 &off,
646 gc->req->hostname)) &&
647 (GNUNET_OK ==
648 GNUNET_DNSPARSER_builder_add_name (dst, dst_len, &off, ip)))
649 {
650 add_record (gc->req,
651 GNUNET_GNSRECORD_TYPE_GNS2DNS,
652 expiration_time,
653 dst,
654 off);
655 gc->found = GNUNET_YES;
656 }
657 break;
658
659 case GNUNET_DNSPARSER_TYPE_AAAA:
660 if (sizeof(struct in6_addr) != rec->data.raw.data_len)
661 {
662 GNUNET_break (0);
663 return;
664 }
665 if (NULL == inet_ntop (AF_INET6, rec->data.raw.data, ip, ip_size))
666 {
667 GNUNET_break (0);
668 return;
669 }
670 if ((GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst,
671 dst_len,
672 &off,
673 gc->req->hostname)) &&
674 (GNUNET_OK ==
675 GNUNET_DNSPARSER_builder_add_name (dst, dst_len, &off, ip)))
676 {
677 add_record (gc->req,
678 GNUNET_GNSRECORD_TYPE_GNS2DNS,
679 expiration_time,
680 dst,
681 off);
682 gc->found = GNUNET_YES;
683 }
684 break;
685
686 case GNUNET_DNSPARSER_TYPE_CNAME:
687 if ((GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst,
688 dst_len,
689 &off,
690 gc->req->hostname)) &&
691 (GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst,
692 dst_len,
693 &off,
694 rec->data.hostname)))
695 {
696 add_record (gc->req,
697 GNUNET_GNSRECORD_TYPE_GNS2DNS,
698 expiration_time,
699 dst,
700 off);
701 gc->found = GNUNET_YES;
702 }
703 break;
704
705 default:
706 /* useless, do nothing */
707 break;
708 }
709}
710
711
712/**
713 * Closure for #process_record().
714 */
715struct ProcessRecordContext
716{
717 /**
718 * Answer we got back and are currently parsing, or NULL
719 * if not active.
720 */
721 struct GNUNET_DNSPARSER_Packet *p;
722
723 /**
724 * Request we are processing.
725 */
726 struct Request *req;
727};
728
729
730/**
731 * We received @a rec for @a req. Remember the answer.
732 *
733 * @param cls a `struct ProcessRecordContext`
734 * @param rec response
735 */
736static void
737process_record (void *cls, const struct GNUNET_DNSPARSER_Record *rec)
738{
739 struct ProcessRecordContext *prc = cls;
740 struct Request *req = prc->req;
741 char dst[65536];
742 size_t dst_len;
743 size_t off;
744 struct GNUNET_TIME_Absolute expiration_time;
745 struct GNUNET_TIME_Relative left;
746
747 dst_len = sizeof(dst);
748 off = 0;
749 records++;
750 if (0 != strcasecmp (rec->name, req->hostname))
751 {
752 GNUNET_log (
753 GNUNET_ERROR_TYPE_DEBUG,
754 "DNS returned record from zone `%s' of type %u while resolving `%s'\n",
755 rec->name,
756 (unsigned int) rec->type,
757 req->hostname);
758 return; /* does not match hostname, might be glue, but
759 not useful for this pass! */
760 }
761 expiration_time = rec->expiration_time;
762 left = GNUNET_TIME_absolute_get_remaining (expiration_time);
763 if (0 == left.rel_value_us)
764 {
765 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
766 "DNS returned expired record for `%s'\n",
767 req->hostname);
768 GNUNET_STATISTICS_update (stats,
769 "# expired records obtained from DNS",
770 1,
771 GNUNET_NO);
772 return; /* record expired */
773 }
774
775 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
776 "DNS returned record that expires at %s for `%s'\n",
777 GNUNET_STRINGS_absolute_time_to_string (expiration_time),
778 req->hostname);
779 /* if expiration window is too short, bump it to configured minimum */
780 if (left.rel_value_us < minimum_expiration_time.rel_value_us)
781 expiration_time =
782 GNUNET_TIME_relative_to_absolute (minimum_expiration_time);
783 switch (rec->type)
784 {
785 case GNUNET_DNSPARSER_TYPE_NS: {
786 struct GlueClosure gc;
787
788 /* check for glue */
789 gc.req = req;
790 gc.ns = rec->data.hostname;
791 gc.found = GNUNET_NO;
792 for_all_records (prc->p, &check_for_glue, &gc);
793 if ((GNUNET_NO == gc.found) &&
794 (GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst,
795 dst_len,
796 &off,
797 req->hostname)) &&
798 (GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst,
799 dst_len,
800 &off,
801 rec->data.hostname)))
802 {
803 /* FIXME: actually check if this is out-of-bailiwick,
804 and if not request explicit resolution... */
805 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
806 "Converted OOB (`%s') NS record for `%s'\n",
807 rec->data.hostname,
808 rec->name);
809 add_record (req,
810 GNUNET_GNSRECORD_TYPE_GNS2DNS,
811 expiration_time,
812 dst,
813 off);
814 }
815 else
816 {
817 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
818 "Converted NS record for `%s' using glue\n",
819 rec->name);
820 }
821 break;
822 }
823
824 case GNUNET_DNSPARSER_TYPE_CNAME:
825 if (GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst,
826 dst_len,
827 &off,
828 rec->data.hostname))
829 {
830 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
831 "Converting CNAME (`%s') record for `%s'\n",
832 rec->data.hostname,
833 rec->name);
834 add_record (req, rec->type, expiration_time, dst, off);
835 }
836 break;
837
838 case GNUNET_DNSPARSER_TYPE_DNAME:
839 /* No support for DNAME in GNS yet! FIXME: support later! */
840 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
841 "FIXME: not supported: %s DNAME %s\n",
842 rec->name,
843 rec->data.hostname);
844 break;
845
846 case GNUNET_DNSPARSER_TYPE_MX:
847 if (GNUNET_OK ==
848 GNUNET_DNSPARSER_builder_add_mx (dst, dst_len, &off, rec->data.mx))
849 {
850 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
851 "Converting MX (`%s') record for `%s'\n",
852 rec->data.mx->mxhost,
853 rec->name);
854 add_record (req, rec->type, expiration_time, dst, off);
855 }
856 break;
857
858 case GNUNET_DNSPARSER_TYPE_SOA:
859 if (GNUNET_OK ==
860 GNUNET_DNSPARSER_builder_add_soa (dst, dst_len, &off, rec->data.soa))
861 {
862 /* NOTE: GNS does not really use SOAs */
863 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
864 "Converting SOA record for `%s'\n",
865 rec->name);
866 add_record (req, rec->type, expiration_time, dst, off);
867 }
868 break;
869
870 case GNUNET_DNSPARSER_TYPE_SRV:
871 if (GNUNET_OK ==
872 GNUNET_DNSPARSER_builder_add_srv (dst, dst_len, &off, rec->data.srv))
873 {
874 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
875 "Converting SRV record for `%s'\n",
876 rec->name);
877 add_record (req, rec->type, expiration_time, dst, off);
878 }
879 break;
880
881 case GNUNET_DNSPARSER_TYPE_PTR:
882 if (GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst,
883 dst_len,
884 &off,
885 rec->data.hostname))
886 {
887 /* !?: what does a PTR record do in a regular TLD??? */
888 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
889 "Converting PTR record for `%s' (weird)\n",
890 rec->name);
891 add_record (req, rec->type, expiration_time, dst, off);
892 }
893 break;
894
895 case GNUNET_DNSPARSER_TYPE_CERT:
896 if (GNUNET_OK ==
897 GNUNET_DNSPARSER_builder_add_cert (dst, dst_len, &off, rec->data.cert))
898 {
899 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
900 "Converting CERT record for `%s'\n",
901 rec->name);
902 add_record (req, rec->type, expiration_time, dst, off);
903 }
904 break;
905
906 /* Rest is 'raw' encoded and just needs to be copied IF
907 the hostname matches the requested name; otherwise we
908 simply cannot use it. */
909 case GNUNET_DNSPARSER_TYPE_A:
910 case GNUNET_DNSPARSER_TYPE_AAAA:
911 case GNUNET_DNSPARSER_TYPE_TXT:
912 default:
913 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
914 "Converting record of type %u for `%s'\n",
915 (unsigned int) rec->type,
916 rec->name);
917 add_record (req,
918 rec->type,
919 expiration_time,
920 rec->data.raw.data,
921 rec->data.raw.data_len);
922 break;
923 }
924}
925
926
927/**
928 * Continuation called to notify client about result of the
929 * operation.
930 *
931 * @param cls closure with our `struct Request`
932 * @param success #GNUNET_SYSERR on failure (including timeout/queue drop/failure to validate)
933 * #GNUNET_NO if content was already there or not found
934 * #GNUNET_YES (or other positive value) on success
935 * @param emsg NULL on success, otherwise an error message
936 */
937static void
938store_completed_cb (void *cls, int32_t success, const char *emsg)
939{
940 static struct GNUNET_TIME_Absolute last;
941 struct Request *req = cls;
942
943 req->qe = NULL;
944 if (GNUNET_SYSERR == success)
945 {
946 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
947 "Failed to store zone data for `%s': %s\n",
948 req->hostname,
949 emsg);
950 }
951 else
952 {
953 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
954 "Stored records under `%s' (%d)\n",
955 req->hostname,
956 success);
957 }
958 total_reg_proc_dns_ns++; /* finished regular processing */
959 pending_rs--;
960 free_records (req);
961 /* compute NAMESTORE statistics */
962 {
963 static uint64_t total_ns_latency_cnt;
964 static struct GNUNET_TIME_Relative total_ns_latency;
965 struct GNUNET_TIME_Relative ns_latency;
966
967 ns_latency = GNUNET_TIME_absolute_get_duration (req->op_start_time);
968 total_ns_latency = GNUNET_TIME_relative_add (total_ns_latency, ns_latency);
969 if (0 == total_ns_latency_cnt)
970 last = GNUNET_TIME_absolute_get ();
971 total_ns_latency_cnt++;
972 if (0 == (total_ns_latency_cnt % 1000))
973 {
974 struct GNUNET_TIME_Relative delta;
975
976 delta = GNUNET_TIME_absolute_get_duration (last);
977 last = GNUNET_TIME_absolute_get ();
978 fprintf (stderr,
979 "Processed 1000 records in %s\n",
980 GNUNET_STRINGS_relative_time_to_string (delta, GNUNET_YES));
981 GNUNET_STATISTICS_set (stats,
982 "# average NAMESTORE PUT latency (μs)",
983 total_ns_latency.rel_value_us
984 / total_ns_latency_cnt,
985 GNUNET_NO);
986 }
987 }
988 /* compute and publish overall velocity */
989 if (0 == (total_reg_proc_dns_ns % 100))
990 {
991 struct GNUNET_TIME_Relative runtime;
992
993 runtime = GNUNET_TIME_absolute_get_duration (start_time_reg_proc);
994 runtime = GNUNET_TIME_relative_subtract (runtime, idle_time);
995 runtime =
996 GNUNET_TIME_relative_divide (runtime,
997 total_reg_proc_dns + total_reg_proc_dns_ns);
998 GNUNET_STATISTICS_set (stats,
999 "# Regular processing completed without NAMESTORE",
1000 total_reg_proc_dns,
1001 GNUNET_NO);
1002 GNUNET_STATISTICS_set (stats,
1003 "# Regular processing completed with NAMESTORE PUT",
1004 total_reg_proc_dns_ns,
1005 GNUNET_NO);
1006 GNUNET_STATISTICS_set (stats,
1007 "# average request processing latency (μs)",
1008 runtime.rel_value_us,
1009 GNUNET_NO);
1010 GNUNET_STATISTICS_set (stats,
1011 "# total time spent idle (μs)",
1012 idle_time.rel_value_us,
1013 GNUNET_NO);
1014 }
1015
1016 if (NULL == t)
1017 {
1018 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1019 t = GNUNET_SCHEDULER_add_now (&process_queue, NULL);
1020 }
1021}
1022
1023
1024/**
1025 * Function called with the result of a DNS resolution.
1026 *
1027 * @param cls closure with the `struct Request`
1028 * @param dns dns response, never NULL
1029 * @param dns_len number of bytes in @a dns
1030 */
1031static void
1032process_result (void *cls,
1033 const struct GNUNET_TUN_DnsHeader *dns,
1034 size_t dns_len)
1035{
1036 struct Request *req = cls;
1037 struct Record *rec;
1038 struct GNUNET_DNSPARSER_Packet *p;
1039 unsigned int rd_count;
1040
1041 GNUNET_assert (NULL == req->hn);
1042 if (NULL == dns)
1043 {
1044 /* stub gave up */
1045 GNUNET_CONTAINER_DLL_remove (req_head, req_tail, req);
1046 pending--;
1047 if (NULL == t)
1048 {
1049 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1050 t = GNUNET_SCHEDULER_add_now (&process_queue, NULL);
1051 }
1052 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1053 "Stub gave up on DNS reply for `%s'\n",
1054 req->hostname);
1055 GNUNET_STATISTICS_update (stats, "# DNS lookups timed out", 1, GNUNET_NO);
1056 if (req->issue_num > MAX_RETRIES)
1057 {
1058 failures++;
1059 free_request (req);
1060 GNUNET_STATISTICS_update (stats, "# requests given up on", 1, GNUNET_NO);
1061 return;
1062 }
1063 total_reg_proc_dns++;
1064 req->rs = NULL;
1065 insert_sorted (req);
1066 return;
1067 }
1068 if (req->id != dns->id)
1069 {
1070 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1071 "DNS ID did not match request, ignoring reply\n");
1072 GNUNET_STATISTICS_update (stats, "# DNS ID mismatches", 1, GNUNET_NO);
1073 return;
1074 }
1075 GNUNET_CONTAINER_DLL_remove (req_head, req_tail, req);
1076 GNUNET_DNSSTUB_resolve_cancel (req->rs);
1077 req->rs = NULL;
1078 pending--;
1079 p = GNUNET_DNSPARSER_parse ((const char *) dns, dns_len);
1080 if (NULL == p)
1081 {
1082 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1083 "Failed to parse DNS reply for `%s'\n",
1084 req->hostname);
1085 GNUNET_STATISTICS_update (stats, "# DNS parser errors", 1, GNUNET_NO);
1086 if (NULL == t)
1087 {
1088 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1089 t = GNUNET_SCHEDULER_add_now (&process_queue, NULL);
1090 }
1091 if (req->issue_num > MAX_RETRIES)
1092 {
1093 failures++;
1094 free_request (req);
1095 GNUNET_STATISTICS_update (stats, "# requests given up on", 1, GNUNET_NO);
1096 return;
1097 }
1098 insert_sorted (req);
1099 return;
1100 }
1101 /* import new records */
1102 req->issue_num = 0; /* success, reset counter! */
1103 {
1104 struct ProcessRecordContext prc = { .req = req, .p = p };
1105
1106 for_all_records (p, &process_record, &prc);
1107 }
1108 GNUNET_DNSPARSER_free_packet (p);
1109 /* count records found, determine minimum expiration time */
1110 req->expires = GNUNET_TIME_UNIT_FOREVER_ABS;
1111 {
1112 struct GNUNET_TIME_Relative dns_latency;
1113
1114 dns_latency = GNUNET_TIME_absolute_get_duration (req->op_start_time);
1115 total_dns_latency =
1116 GNUNET_TIME_relative_add (total_dns_latency, dns_latency);
1117 total_dns_latency_cnt++;
1118 if (0 == (total_dns_latency_cnt % 1000))
1119 {
1120 GNUNET_STATISTICS_set (stats,
1121 "# average DNS lookup latency (μs)",
1122 total_dns_latency.rel_value_us
1123 / total_dns_latency_cnt,
1124 GNUNET_NO);
1125 }
1126 }
1127 rd_count = 0;
1128 for (rec = req->rec_head; NULL != rec; rec = rec->next)
1129 {
1130 struct GNUNET_TIME_Absolute at;
1131
1132 at.abs_value_us = rec->grd.expiration_time;
1133 req->expires = GNUNET_TIME_absolute_min (req->expires, at);
1134 rd_count++;
1135 }
1136 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1137 "Obtained %u records for `%s'\n",
1138 rd_count,
1139 req->hostname);
1140 /* Instead of going for SOA, simplified for now to look each
1141 day in case we got an empty response */
1142 if (0 == rd_count)
1143 {
1144 req->expires = GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_DAYS);
1145 GNUNET_STATISTICS_update (stats,
1146 "# empty DNS replies (usually NXDOMAIN)",
1147 1,
1148 GNUNET_NO);
1149 }
1150 else
1151 {
1152 record_sets++;
1153 }
1154 /* convert records to namestore import format */
1155 {
1156 struct GNUNET_GNSRECORD_Data rd[GNUNET_NZL (rd_count)];
1157 unsigned int off = 0;
1158
1159 /* convert linked list into array */
1160 for (rec = req->rec_head; NULL != rec; rec = rec->next)
1161 rd[off++] = rec->grd;
1162 pending_rs++;
1163 req->op_start_time = GNUNET_TIME_absolute_get ();
1164 req->qe = GNUNET_NAMESTORE_records_store (ns,
1165 &req->zone->key,
1166 get_label (req),
1167 rd_count,
1168 rd,
1169 &store_completed_cb,
1170 req);
1171 GNUNET_assert (NULL != req->qe);
1172 }
1173 insert_sorted (req);
1174}
1175
1176
1177/**
1178 * Process as many requests as possible from the queue.
1179 *
1180 * @param cls NULL
1181 */
1182static void
1183process_queue (void *cls)
1184{
1185 struct Request *req;
1186 unsigned int series;
1187 void *raw;
1188 size_t raw_size;
1189 struct GNUNET_TIME_Relative delay;
1190
1191 (void) cls;
1192 delay = GNUNET_TIME_absolute_get_duration (sleep_time_reg_proc);
1193 idle_time = GNUNET_TIME_relative_add (idle_time, delay);
1194 series = 0;
1195 t = NULL;
1196 while (pending + pending_rs < THRESH)
1197 {
1198 req = GNUNET_CONTAINER_heap_peek (req_heap);
1199 if (NULL == req)
1200 break;
1201 if (NULL != req->qe)
1202 return; /* namestore op still pending */
1203 if (NULL != req->rs)
1204 {
1205 GNUNET_break (0);
1206 return; /* already submitted */
1207 }
1208 if (GNUNET_TIME_absolute_get_remaining (req->expires).rel_value_us > 0)
1209 break;
1210 GNUNET_assert (req == GNUNET_CONTAINER_heap_remove_root (req_heap));
1211 req->hn = NULL;
1212 GNUNET_CONTAINER_DLL_insert (req_head, req_tail, req);
1213 GNUNET_assert (NULL == req->rs);
1214 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1215 "Requesting resolution for `%s'\n",
1216 req->hostname);
1217 raw = build_dns_query (req, &raw_size);
1218 if (NULL == raw)
1219 {
1220 GNUNET_break (0);
1221 free_request (req);
1222 continue;
1223 }
1224 req->op_start_time = GNUNET_TIME_absolute_get ();
1225 req->rs = GNUNET_DNSSTUB_resolve (ctx, raw, raw_size, &process_result, req);
1226 GNUNET_assert (NULL != req->rs);
1227 req->issue_num++;
1228 lookups++;
1229 pending++;
1230 series++;
1231 if (series > MAX_SERIES)
1232 break;
1233 }
1234 if (pending + pending_rs >= THRESH)
1235 {
1236 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1237 "Stopped processing queue (%u+%u/%u)]\n",
1238 pending,
1239 pending_rs,
1240 THRESH);
1241 return; /* wait for replies */
1242 }
1243 req = GNUNET_CONTAINER_heap_peek (req_heap);
1244 if (NULL == req)
1245 {
1246 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1247 "Stopped processing queue: empty queue\n");
1248 return;
1249 }
1250 if (GNUNET_TIME_absolute_get_remaining (req->expires).rel_value_us > 0)
1251 {
1252 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1253 "Waiting until %s for next record (`%s') to expire\n",
1254 GNUNET_STRINGS_absolute_time_to_string (req->expires),
1255 req->hostname);
1256 if (NULL != t)
1257 GNUNET_SCHEDULER_cancel (t);
1258 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1259 t = GNUNET_SCHEDULER_add_at (req->expires, &process_queue, NULL);
1260 return;
1261 }
1262 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Throttling\n");
1263 if (NULL != t)
1264 GNUNET_SCHEDULER_cancel (t);
1265 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1266 t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY, &process_queue, NULL);
1267}
1268
1269
1270/**
1271 * Iterator called during #do_shutdown() to free requests in
1272 * the #ns_pending map.
1273 *
1274 * @param cls NULL
1275 * @param key unused
1276 * @param value the `struct Request` to free
1277 * @return #GNUNET_OK
1278 */
1279static int
1280free_request_it (void *cls, const struct GNUNET_HashCode *key, void *value)
1281{
1282 struct Request *req = value;
1283
1284 (void) cls;
1285 (void) key;
1286 free_request (req);
1287 return GNUNET_OK;
1288}
1289
1290
1291/**
1292 * Clean up and terminate the process.
1293 *
1294 * @param cls NULL
1295 */
1296static void
1297do_shutdown (void *cls)
1298{
1299 struct Request *req;
1300 struct Zone *zone;
1301
1302 (void) cls;
1303 if (NULL != id)
1304 {
1305 GNUNET_IDENTITY_disconnect (id);
1306 id = NULL;
1307 }
1308 if (NULL != t)
1309 {
1310 GNUNET_SCHEDULER_cancel (t);
1311 t = NULL;
1312 }
1313 while (NULL != (req = req_head))
1314 {
1315 GNUNET_CONTAINER_DLL_remove (req_head, req_tail, req);
1316 if (NULL != req->qe)
1317 GNUNET_NAMESTORE_cancel (req->qe);
1318 free_request (req);
1319 }
1320 while (NULL != (req = GNUNET_CONTAINER_heap_remove_root (req_heap)))
1321 {
1322 req->hn = NULL;
1323 if (NULL != req->qe)
1324 GNUNET_NAMESTORE_cancel (req->qe);
1325 free_request (req);
1326 }
1327 if (NULL != zone_it)
1328 {
1329 GNUNET_NAMESTORE_zone_iteration_stop (zone_it);
1330 zone_it = NULL;
1331 }
1332 if (NULL != ns)
1333 {
1334 GNUNET_NAMESTORE_disconnect (ns);
1335 ns = NULL;
1336 }
1337 if (NULL != ctx)
1338 {
1339 GNUNET_DNSSTUB_stop (ctx);
1340 ctx = NULL;
1341 }
1342 if (NULL != req_heap)
1343 {
1344 GNUNET_CONTAINER_heap_destroy (req_heap);
1345 req_heap = NULL;
1346 }
1347 if (NULL != ns_pending)
1348 {
1349 GNUNET_CONTAINER_multihashmap_iterate (ns_pending, &free_request_it, NULL);
1350 GNUNET_CONTAINER_multihashmap_destroy (ns_pending);
1351 ns_pending = NULL;
1352 }
1353 while (NULL != (zone = zone_head))
1354 {
1355 GNUNET_CONTAINER_DLL_remove (zone_head, zone_tail, zone);
1356 GNUNET_free (zone->domain);
1357 GNUNET_free (zone);
1358 }
1359 if (NULL != stats)
1360 {
1361 GNUNET_STATISTICS_destroy (stats, GNUNET_NO);
1362 stats = NULL;
1363 }
1364}
1365
1366
1367/**
1368 * Iterate over all of the zones we care about and see which records
1369 * we may need to re-fetch when.
1370 *
1371 * @param cls NULL
1372 */
1373static void
1374iterate_zones (void *cls);
1375
1376
1377/**
1378 * Function called if #GNUNET_NAMESTORE_records_lookup() failed.
1379 * Just logs an error.
1380 *
1381 * @param cls a `struct Zone`
1382 */
1383static void
1384ns_lookup_error_cb (void *cls)
1385{
1386 struct Zone *zone = cls;
1387
1388 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1389 "Failed to load data from namestore for zone `%s'\n",
1390 zone->domain);
1391 zone_it = NULL;
1392 ns_iterator_trigger_next = 0;
1393 iterate_zones (NULL);
1394}
1395
1396
1397/**
1398 * Process a record that was stored in the namestore.
1399 *
1400 * @param cls a `struct Zone *`
1401 * @param key private key of the zone
1402 * @param label label of the records
1403 * @param rd_count number of entries in @a rd array, 0 if label was deleted
1404 * @param rd array of records with data to store
1405 */
1406static void
1407ns_lookup_result_cb (void *cls,
1408 const struct GNUNET_IDENTITY_PrivateKey *key,
1409 const char *label,
1410 unsigned int rd_count,
1411 const struct GNUNET_GNSRECORD_Data *rd)
1412{
1413 struct Zone *zone = cls;
1414 struct Request *req;
1415 struct GNUNET_HashCode hc;
1416 char *fqdn;
1417
1418 ns_iterator_trigger_next--;
1419 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1420 "Obtained NAMESTORE reply, %llu left in round\n",
1421 (unsigned long long) ns_iterator_trigger_next);
1422 if (0 == ns_iterator_trigger_next)
1423 {
1424 ns_iterator_trigger_next = NS_BATCH_SIZE;
1425 GNUNET_STATISTICS_update (stats,
1426 "# NAMESTORE records requested from cache",
1427 ns_iterator_trigger_next,
1428 GNUNET_NO);
1429 GNUNET_NAMESTORE_zone_iterator_next (zone_it, ns_iterator_trigger_next);
1430 }
1431 GNUNET_asprintf (&fqdn, "%s.%s", label, zone->domain);
1432 GNUNET_CRYPTO_hash (fqdn, strlen (fqdn) + 1, &hc);
1433 GNUNET_free (fqdn);
1434 req = GNUNET_CONTAINER_multihashmap_get (ns_pending, &hc);
1435 if (NULL == req)
1436 {
1437 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1438 "Ignoring record `%s' in zone `%s': not on my list!\n",
1439 label,
1440 zone->domain);
1441 return;
1442 }
1443 GNUNET_assert (GNUNET_OK ==
1444 GNUNET_CONTAINER_multihashmap_remove (ns_pending, &hc, req));
1445 GNUNET_break (0 == GNUNET_memcmp (key, &req->zone->key));
1446 GNUNET_break (0 == strcasecmp (label, get_label (req)));
1447 for (unsigned int i = 0; i < rd_count; i++)
1448 {
1449 struct GNUNET_TIME_Absolute at;
1450
1451 if (0 != (rd->flags & GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION))
1452 {
1453 struct GNUNET_TIME_Relative rel;
1454
1455 rel.rel_value_us = rd->expiration_time;
1456 at = GNUNET_TIME_relative_to_absolute (rel);
1457 }
1458 else
1459 {
1460 at.abs_value_us = rd->expiration_time;
1461 }
1462 add_record (req, rd->record_type, at, rd->data, rd->data_size);
1463 }
1464 if (0 == rd_count)
1465 {
1466 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1467 "Empty record set in namestore for `%s'\n",
1468 req->hostname);
1469 }
1470 else
1471 {
1472 unsigned int pos = 0;
1473
1474 cached++;
1475 req->expires = GNUNET_TIME_UNIT_FOREVER_ABS;
1476 for (struct Record *rec = req->rec_head; NULL != rec; rec = rec->next)
1477 {
1478 struct GNUNET_TIME_Absolute at;
1479
1480 at.abs_value_us = rec->grd.expiration_time;
1481 req->expires = GNUNET_TIME_absolute_min (req->expires, at);
1482 pos++;
1483 }
1484 if (0 == pos)
1485 req->expires = GNUNET_TIME_UNIT_ZERO_ABS;
1486 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1487 "Hot-start with %u existing records for `%s'\n",
1488 pos,
1489 req->hostname);
1490 }
1491 free_records (req);
1492
1493 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1494 "Adding `%s' to worklist to start at %s\n",
1495 req->hostname,
1496 GNUNET_STRINGS_absolute_time_to_string (req->expires));
1497 insert_sorted (req);
1498}
1499
1500
1501/**
1502 * Add @a hostname to the list of requests to be made.
1503 *
1504 * @param hostname name to resolve
1505 */
1506static void
1507queue (const char *hostname)
1508{
1509 struct Request *req;
1510 const char *dot;
1511 struct Zone *zone;
1512 size_t hlen;
1513 struct GNUNET_HashCode hc;
1514
1515 if (GNUNET_OK != GNUNET_DNSPARSER_check_name (hostname))
1516 {
1517 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1518 "Refusing invalid hostname `%s'\n",
1519 hostname);
1520 rejects++;
1521 return;
1522 }
1523 dot = strchr (hostname, (unsigned char) '.');
1524 if (NULL == dot)
1525 {
1526 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1527 "Refusing invalid hostname `%s' (lacks '.')\n",
1528 hostname);
1529 rejects++;
1530 return;
1531 }
1532 for (zone = zone_head; NULL != zone; zone = zone->next)
1533 if (0 == strcmp (zone->domain, dot + 1))
1534 break;
1535 if (NULL == zone)
1536 {
1537 rejects++;
1538 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1539 "Domain name `%s' not in ego list!\n",
1540 dot + 1);
1541 return;
1542 }
1543
1544 hlen = strlen (hostname) + 1;
1545 req = GNUNET_malloc (sizeof(struct Request) + hlen);
1546 req->zone = zone;
1547 req->hostname = (char *) &req[1];
1548 GNUNET_memcpy (req->hostname, hostname, hlen);
1549 req->id = (uint16_t) GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE,
1550 UINT16_MAX);
1551 GNUNET_CRYPTO_hash (req->hostname, hlen, &hc);
1552 if (GNUNET_OK != GNUNET_CONTAINER_multihashmap_put (
1553 ns_pending,
1554 &hc,
1555 req,
1556 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))
1557 {
1558 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1559 "Duplicate hostname `%s' ignored\n",
1560 hostname);
1561 GNUNET_free (req);
1562 return;
1563 }
1564}
1565
1566
1567/**
1568 * We have completed the initial iteration over the namestore's database.
1569 * This function is called on each of the remaining records in
1570 * #move_to_queue to #queue() them, as we will simply not find existing
1571 * records for them any longer.
1572 *
1573 * @param cls NULL
1574 * @param key unused
1575 * @param value a `struct Request`
1576 * @return #GNUNET_OK (continue to iterate)
1577 */
1578static int
1579move_to_queue (void *cls, const struct GNUNET_HashCode *key, void *value)
1580{
1581 struct Request *req = value;
1582
1583 (void) cls;
1584 (void) key;
1585 insert_sorted (req);
1586 return GNUNET_OK;
1587}
1588
1589
1590/**
1591 * Iterate over all of the zones we care about and see which records
1592 * we may need to re-fetch when.
1593 *
1594 * @param cls NULL
1595 */
1596static void
1597iterate_zones (void *cls)
1598{
1599 static struct Zone *last;
1600
1601 (void) cls;
1602 if (NULL != zone_it)
1603 {
1604 zone_it = NULL;
1605 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1606 "Finished iteration over zone `%s'!\n",
1607 last->domain);
1608 /* subtract left-overs from previous iteration */
1609 GNUNET_STATISTICS_update (stats,
1610 "# NAMESTORE records requested from cache",
1611 (long long) (-ns_iterator_trigger_next),
1612 GNUNET_NO);
1613 ns_iterator_trigger_next = 0;
1614 }
1615 GNUNET_assert (NULL != zone_tail);
1616 if (zone_tail == last)
1617 {
1618 /* Done iterating over relevant zones in NAMESTORE, move
1619 rest of hash map to work queue as well. */
1620 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1621 "Finished all NAMESTORE iterations!\n");
1622 GNUNET_STATISTICS_set (stats,
1623 "# Domain names without cached reply",
1624 GNUNET_CONTAINER_multihashmap_size (ns_pending),
1625 GNUNET_NO);
1626 GNUNET_CONTAINER_multihashmap_iterate (ns_pending, &move_to_queue, NULL);
1627 GNUNET_CONTAINER_multihashmap_destroy (ns_pending);
1628 ns_pending = NULL;
1629 start_time_reg_proc = GNUNET_TIME_absolute_get ();
1630 total_reg_proc_dns = 0;
1631 total_reg_proc_dns_ns = 0;
1632 return;
1633 }
1634 if (NULL == last)
1635 last = zone_head;
1636 else
1637 last = last->next;
1638 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1639 "Starting iteration over zone `%s'!\n",
1640 last->domain);
1641 /* subtract left-overs from previous iteration */
1642 GNUNET_STATISTICS_update (stats,
1643 "# NAMESTORE records requested from cache",
1644 1,
1645 GNUNET_NO);
1646 ns_iterator_trigger_next = 1;
1647 GNUNET_STATISTICS_update (stats, "# zones iterated", 1, GNUNET_NO);
1648 zone_it = GNUNET_NAMESTORE_zone_iteration_start (ns,
1649 &last->key,
1650 &ns_lookup_error_cb,
1651 NULL,
1652 &ns_lookup_result_cb,
1653 last,
1654 &iterate_zones,
1655 NULL);
1656}
1657
1658
1659/**
1660 * Begin processing hostnames from stdin.
1661 *
1662 * @param cls NULL
1663 */
1664static void
1665process_stdin (void *cls)
1666{
1667 static struct GNUNET_TIME_Absolute last;
1668 static uint64_t idot;
1669 char hn[256];
1670
1671 (void) cls;
1672 t = NULL;
1673 if (NULL != id)
1674 {
1675 GNUNET_IDENTITY_disconnect (id);
1676 id = NULL;
1677 }
1678 while (NULL != fgets (hn, sizeof(hn), stdin))
1679 {
1680 if (strlen (hn) > 0)
1681 hn[strlen (hn) - 1] = '\0'; /* eat newline */
1682 if (0 == idot)
1683 last = GNUNET_TIME_absolute_get ();
1684 idot++;
1685 if (0 == idot % 100000)
1686 {
1687 struct GNUNET_TIME_Relative delta;
1688
1689 delta = GNUNET_TIME_absolute_get_duration (last);
1690 last = GNUNET_TIME_absolute_get ();
1691 fprintf (stderr,
1692 "Read 100000 domain names in %s\n",
1693 GNUNET_STRINGS_relative_time_to_string (delta, GNUNET_YES));
1694 GNUNET_STATISTICS_set (stats, "# domain names provided", idot, GNUNET_NO);
1695 }
1696 queue (hn);
1697 }
1698 fprintf (stderr,
1699 "Done reading %llu domain names\n",
1700 (unsigned long long) idot);
1701 GNUNET_STATISTICS_set (stats, "# domain names provided", idot, GNUNET_NO);
1702 iterate_zones (NULL);
1703}
1704
1705
1706/**
1707 * Method called to inform about the egos of this peer.
1708 *
1709 * When used with #GNUNET_IDENTITY_connect, this function is
1710 * initially called for all egos and then again whenever a
1711 * ego's name changes or if it is deleted. At the end of
1712 * the initial pass over all egos, the function is once called
1713 * with 'NULL' for @a ego. That does NOT mean that the callback won't
1714 * be invoked in the future or that there was an error.
1715 *
1716 * When used with #GNUNET_IDENTITY_create or #GNUNET_IDENTITY_get, this
1717 * function is only called ONCE, and 'NULL' being passed in @a ego does
1718 * indicate an error (for example because name is taken or no default value is
1719 * known). If @a ego is non-NULL and if '*ctx' is set in those callbacks, the
1720 * value WILL be passed to a subsequent call to the identity callback of
1721 * #GNUNET_IDENTITY_connect (if that one was not NULL).
1722 *
1723 * When an identity is renamed, this function is called with the
1724 * (known) @a ego but the NEW @a name.
1725 *
1726 * When an identity is deleted, this function is called with the
1727 * (known) ego and "NULL" for the @a name. In this case,
1728 * the @a ego is henceforth invalid (and the @a ctx should also be
1729 * cleaned up).
1730 *
1731 * @param cls closure
1732 * @param ego ego handle, NULL for end of list
1733 * @param ctx context for application to store data for this ego
1734 * (during the lifetime of this process, initially NULL)
1735 * @param name name assigned by the user for this ego,
1736 * NULL if the user just deleted the ego and it
1737 * must thus no longer be used
1738 */
1739static void
1740identity_cb (void *cls,
1741 struct GNUNET_IDENTITY_Ego *ego,
1742 void **ctx,
1743 const char *name)
1744{
1745 (void) cls;
1746 (void) ctx;
1747
1748 if (NULL == ego)
1749 {
1750 /* end of iteration */
1751 if (NULL == zone_head)
1752 {
1753 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "No zone found\n");
1754 GNUNET_SCHEDULER_shutdown ();
1755 return;
1756 }
1757 /* zone_head non-null, process hostnames from stdin */
1758 t = GNUNET_SCHEDULER_add_now (&process_stdin, NULL);
1759 return;
1760 }
1761 if (NULL != name)
1762 {
1763 struct Zone *zone;
1764
1765 zone = GNUNET_new (struct Zone);
1766 zone->key = *GNUNET_IDENTITY_ego_get_private_key (ego);
1767 zone->domain = GNUNET_strdup (name);
1768 GNUNET_CONTAINER_DLL_insert (zone_head, zone_tail, zone);
1769 }
1770}
1771
1772
1773/**
1774 * Process requests from the queue, then if the queue is
1775 * not empty, try again.
1776 *
1777 * @param cls NULL
1778 * @param args remaining command-line arguments
1779 * @param cfgfile name of the configuration file used (for saving, can be NULL!)
1780 * @param cfg configuration
1781 */
1782static void
1783run (void *cls,
1784 char *const *args,
1785 const char *cfgfile,
1786 const struct GNUNET_CONFIGURATION_Handle *cfg)
1787{
1788 (void) cls;
1789 (void) args;
1790 (void) cfgfile;
1791 stats = GNUNET_STATISTICS_create ("zoneimport", cfg);
1792 req_heap = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
1793 ns_pending = GNUNET_CONTAINER_multihashmap_create (map_size, GNUNET_NO);
1794 if (NULL == ns_pending)
1795 {
1796 fprintf (stderr, "Failed to allocate memory for main hash map\n");
1797 return;
1798 }
1799 ctx = GNUNET_DNSSTUB_start (256);
1800 if (NULL == ctx)
1801 {
1802 fprintf (stderr, "Failed to initialize GNUnet DNS STUB\n");
1803 return;
1804 }
1805 if (NULL == args[0])
1806 {
1807 fprintf (stderr,
1808 "You must provide a list of DNS resolvers on the command line\n");
1809 return;
1810 }
1811 for (unsigned int i = 0; NULL != args[i]; i++)
1812 {
1813 if (GNUNET_OK != GNUNET_DNSSTUB_add_dns_ip (ctx, args[i]))
1814 {
1815 fprintf (stderr, "Failed to use `%s' for DNS resolver\n", args[i]);
1816 return;
1817 }
1818 }
1819
1820
1821 GNUNET_SCHEDULER_add_shutdown (&do_shutdown, NULL);
1822 ns = GNUNET_NAMESTORE_connect (cfg);
1823 if (NULL == ns)
1824 {
1825 GNUNET_SCHEDULER_shutdown ();
1826 return;
1827 }
1828 id = GNUNET_IDENTITY_connect (cfg, &identity_cb, NULL);
1829}
1830
1831
1832/**
1833 * Call with IP address of resolver to query.
1834 *
1835 * @param argc should be 2
1836 * @param argv[1] should contain IP address
1837 * @return 0 on success
1838 */
1839int
1840main (int argc, char *const *argv)
1841{
1842 struct GNUNET_GETOPT_CommandLineOption options[] =
1843 { GNUNET_GETOPT_option_uint ('s',
1844 "size",
1845 "MAPSIZE",
1846 gettext_noop (
1847 "size to use for the main hash map"),
1848 &map_size),
1849 GNUNET_GETOPT_option_relative_time (
1850 'm',
1851 "minimum-expiration",
1852 "RELATIVETIME",
1853 gettext_noop ("minimum expiration time we assume for imported records"),
1854 &minimum_expiration_time),
1855 GNUNET_GETOPT_OPTION_END };
1856 int ret;
1857
1858 if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
1859 return 2;
1860 if (GNUNET_OK != (ret = GNUNET_PROGRAM_run (argc,
1861 argv,
1862 "gnunet-zoneimport",
1863 "import DNS zone into namestore",
1864 options,
1865 &run,
1866 NULL)))
1867 return ret;
1868 GNUNET_free_nz ((void *) argv);
1869 fprintf (stderr,
1870 "Rejected %u names, had %u cached, did %u lookups, stored %u record sets\n"
1871 "Found %u records, %u lookups failed, %u/%u pending on shutdown\n",
1872 rejects,
1873 cached,
1874 lookups,
1875 record_sets,
1876 records,
1877 failures,
1878 pending,
1879 pending_rs);
1880 return 0;
1881}
1882
1883
1884/* end of gnunet-zoneimport.c */
generated by cgit v1.2.3 (git 2.39.1) at 2024-04-19 04:45:29 +0000