diff options
-rw-r--r-- | doc/man/gnunet-zoneimport.1 | 5 | ||||
-rw-r--r-- | src/namestore/gnunet-zoneimport.c | 202 | ||||
-rw-r--r-- | src/util/container_multihashmap.c | 55 | ||||
-rw-r--r-- | src/zonemaster/gnunet-service-zonemaster.c | 66 |
4 files changed, 220 insertions, 108 deletions
diff --git a/doc/man/gnunet-zoneimport.1 b/doc/man/gnunet-zoneimport.1 index 687e6f900..06b3a6bcf 100644 --- a/doc/man/gnunet-zoneimport.1 +++ b/doc/man/gnunet-zoneimport.1 | |||
@@ -27,12 +27,15 @@ Use the configuration file FILENAME. | |||
27 | .B | 27 | .B |
28 | .IP "\-h, \-\-help" | 28 | .IP "\-h, \-\-help" |
29 | Print short help on options. | 29 | Print short help on options. |
30 | .B | ||
31 | .IP "\-s MAPSIZE, \-\-size=MAPSIZE" | ||
32 | Specifies the size (in number of entries) to use for the main hash map. The value provided should be at least twice the number of domain names that will be given to the tool. This option is required for very large zones where the number of records encountered is too large for the automatic growth mechanism to work (that one is limited to at most 16 MB allocations for security reasons). Do not worry about this unless you are importing millions of domain names from a zone. | ||
30 | 33 | ||
31 | .SH NOTES | 34 | .SH NOTES |
32 | 35 | ||
33 | Typical invocaton would be: | 36 | Typical invocaton would be: |
34 | 37 | ||
35 | $ gnunet\-zoneimport -s 1.2.3.4 < names.txt | 38 | $ gnunet\-zoneimport 1.2.3.4 < names.txt |
36 | 39 | ||
37 | 40 | ||
38 | 41 | ||
diff --git a/src/namestore/gnunet-zoneimport.c b/src/namestore/gnunet-zoneimport.c index e24cb26dd..0fd0a4ab8 100644 --- a/src/namestore/gnunet-zoneimport.c +++ b/src/namestore/gnunet-zoneimport.c | |||
@@ -204,6 +204,14 @@ struct Request | |||
204 | 204 | ||
205 | 205 | ||
206 | /** | 206 | /** |
207 | * Command-line argument specifying desired size of the hash map with | ||
208 | * all of our pending names. Usually, we use an automatically growing | ||
209 | * map, but this is only OK up to about a million entries. Above that | ||
210 | * number, the user must explicitly specify the size at startup. | ||
211 | */ | ||
212 | static unsigned int map_size = 1024; | ||
213 | |||
214 | /** | ||
207 | * Handle to the identity service. | 215 | * Handle to the identity service. |
208 | */ | 216 | */ |
209 | static struct GNUNET_IDENTITY_Handle *id; | 217 | static struct GNUNET_IDENTITY_Handle *id; |
@@ -323,14 +331,29 @@ static uint64_t total_dns_latency_cnt; | |||
323 | static struct GNUNET_TIME_Relative total_dns_latency; | 331 | static struct GNUNET_TIME_Relative total_dns_latency; |
324 | 332 | ||
325 | /** | 333 | /** |
326 | * Number of NAMESTORE requests counted in latency total. | 334 | * Number of records processed (DNS lookup, no NAMESTORE) in total. |
335 | */ | ||
336 | static uint64_t total_reg_proc_dns; | ||
337 | |||
338 | /** | ||
339 | * Number of records processed (DNS lookup, with NAMESTORE) in total. | ||
340 | */ | ||
341 | static uint64_t total_reg_proc_dns_ns; | ||
342 | |||
343 | /** | ||
344 | * Start time of the regular processing. | ||
345 | */ | ||
346 | static struct GNUNET_TIME_Absolute start_time_reg_proc; | ||
347 | |||
348 | /** | ||
349 | * Last time we worked before going idle. | ||
327 | */ | 350 | */ |
328 | static uint64_t total_ns_latency_cnt; | 351 | static struct GNUNET_TIME_Absolute sleep_time_reg_proc; |
329 | 352 | ||
330 | /** | 353 | /** |
331 | * Sum of NAMESTORE latencies observed. | 354 | * Time we slept just waiting for work. |
332 | */ | 355 | */ |
333 | static struct GNUNET_TIME_Relative total_ns_latency; | 356 | static struct GNUNET_TIME_Relative idle_time; |
334 | 357 | ||
335 | 358 | ||
336 | /** | 359 | /** |
@@ -530,6 +553,7 @@ insert_sorted (struct Request *req) | |||
530 | { | 553 | { |
531 | if (NULL != t) | 554 | if (NULL != t) |
532 | GNUNET_SCHEDULER_cancel (t); | 555 | GNUNET_SCHEDULER_cancel (t); |
556 | sleep_time_reg_proc = GNUNET_TIME_absolute_get (); | ||
533 | t = GNUNET_SCHEDULER_add_at (req->expires, | 557 | t = GNUNET_SCHEDULER_add_at (req->expires, |
534 | &process_queue, | 558 | &process_queue, |
535 | NULL); | 559 | NULL); |
@@ -966,37 +990,9 @@ store_completed_cb (void *cls, | |||
966 | const char *emsg) | 990 | const char *emsg) |
967 | { | 991 | { |
968 | static struct GNUNET_TIME_Absolute last; | 992 | static struct GNUNET_TIME_Absolute last; |
969 | static unsigned int pdot; | ||
970 | struct Request *req = cls; | 993 | struct Request *req = cls; |
971 | 994 | ||
972 | req->qe = NULL; | 995 | req->qe = NULL; |
973 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
974 | "Stored record set in database (%d)\n", | ||
975 | success); | ||
976 | pending_rs--; | ||
977 | { | ||
978 | struct GNUNET_TIME_Relative ns_latency; | ||
979 | |||
980 | ns_latency = GNUNET_TIME_absolute_get_duration (req->op_start_time); | ||
981 | total_ns_latency = GNUNET_TIME_relative_add (total_ns_latency, | ||
982 | ns_latency); | ||
983 | total_ns_latency_cnt++; | ||
984 | if (0 == (total_ns_latency_cnt % 1000)) | ||
985 | { | ||
986 | GNUNET_STATISTICS_update (stats, | ||
987 | "# average NAMESTORE PUT latency (μs)", | ||
988 | total_ns_latency.rel_value_us / total_ns_latency_cnt, | ||
989 | GNUNET_NO); | ||
990 | GNUNET_STATISTICS_update (stats, | ||
991 | "# NAMESTORE PUTs", | ||
992 | total_ns_latency_cnt, | ||
993 | GNUNET_NO); | ||
994 | } | ||
995 | } | ||
996 | |||
997 | if (NULL == t) | ||
998 | t = GNUNET_SCHEDULER_add_now (&process_queue, | ||
999 | NULL); | ||
1000 | if (GNUNET_SYSERR == success) | 996 | if (GNUNET_SYSERR == success) |
1001 | { | 997 | { |
1002 | GNUNET_log (GNUNET_ERROR_TYPE_ERROR, | 998 | GNUNET_log (GNUNET_ERROR_TYPE_ERROR, |
@@ -1007,12 +1003,26 @@ store_completed_cb (void *cls, | |||
1007 | else | 1003 | else |
1008 | { | 1004 | { |
1009 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 1005 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, |
1010 | "Stored records under `%s'\n", | 1006 | "Stored records under `%s' (%d)\n", |
1011 | req->hostname); | 1007 | req->hostname, |
1012 | if (0 == pdot) | 1008 | success); |
1009 | } | ||
1010 | total_reg_proc_dns_ns++; /* finished regular processing */ | ||
1011 | pending_rs--; | ||
1012 | free_records (req); | ||
1013 | /* compute NAMESTORE statistics */ | ||
1014 | { | ||
1015 | static uint64_t total_ns_latency_cnt; | ||
1016 | static struct GNUNET_TIME_Relative total_ns_latency; | ||
1017 | struct GNUNET_TIME_Relative ns_latency; | ||
1018 | |||
1019 | ns_latency = GNUNET_TIME_absolute_get_duration (req->op_start_time); | ||
1020 | total_ns_latency = GNUNET_TIME_relative_add (total_ns_latency, | ||
1021 | ns_latency); | ||
1022 | if (0 == total_ns_latency_cnt) | ||
1013 | last = GNUNET_TIME_absolute_get (); | 1023 | last = GNUNET_TIME_absolute_get (); |
1014 | pdot++; | 1024 | total_ns_latency_cnt++; |
1015 | if (0 == pdot % 1000) | 1025 | if (0 == (total_ns_latency_cnt % 1000)) |
1016 | { | 1026 | { |
1017 | struct GNUNET_TIME_Relative delta; | 1027 | struct GNUNET_TIME_Relative delta; |
1018 | 1028 | ||
@@ -1022,9 +1032,46 @@ store_completed_cb (void *cls, | |||
1022 | "Processed 1000 records in %s\n", | 1032 | "Processed 1000 records in %s\n", |
1023 | GNUNET_STRINGS_relative_time_to_string (delta, | 1033 | GNUNET_STRINGS_relative_time_to_string (delta, |
1024 | GNUNET_YES)); | 1034 | GNUNET_YES)); |
1035 | GNUNET_STATISTICS_set (stats, | ||
1036 | "# average NAMESTORE PUT latency (μs)", | ||
1037 | total_ns_latency.rel_value_us / total_ns_latency_cnt, | ||
1038 | GNUNET_NO); | ||
1025 | } | 1039 | } |
1026 | } | 1040 | } |
1027 | free_records (req); | 1041 | /* compute and publish overall velocity */ |
1042 | if (0 == (total_reg_proc_dns_ns % 100) ) | ||
1043 | { | ||
1044 | struct GNUNET_TIME_Relative runtime; | ||
1045 | |||
1046 | runtime = GNUNET_TIME_absolute_get_duration (start_time_reg_proc); | ||
1047 | runtime = GNUNET_TIME_relative_subtract (runtime, | ||
1048 | idle_time); | ||
1049 | runtime = GNUNET_TIME_relative_divide (runtime, | ||
1050 | total_reg_proc_dns + total_reg_proc_dns_ns); | ||
1051 | GNUNET_STATISTICS_set (stats, | ||
1052 | "# Regular processing completed without NAMESTORE", | ||
1053 | total_reg_proc_dns, | ||
1054 | GNUNET_NO); | ||
1055 | GNUNET_STATISTICS_set (stats, | ||
1056 | "# Regular processing completed with NAMESTORE PUT", | ||
1057 | total_reg_proc_dns_ns, | ||
1058 | GNUNET_NO); | ||
1059 | GNUNET_STATISTICS_set (stats, | ||
1060 | "# average request processing latency (μs)", | ||
1061 | runtime.rel_value_us, | ||
1062 | GNUNET_NO); | ||
1063 | GNUNET_STATISTICS_set (stats, | ||
1064 | "# total time spent idle (μs)", | ||
1065 | idle_time.rel_value_us, | ||
1066 | GNUNET_NO); | ||
1067 | } | ||
1068 | |||
1069 | if (NULL == t) | ||
1070 | { | ||
1071 | sleep_time_reg_proc = GNUNET_TIME_absolute_get (); | ||
1072 | t = GNUNET_SCHEDULER_add_now (&process_queue, | ||
1073 | NULL); | ||
1074 | } | ||
1028 | } | 1075 | } |
1029 | 1076 | ||
1030 | 1077 | ||
@@ -1054,8 +1101,11 @@ process_result (void *cls, | |||
1054 | req); | 1101 | req); |
1055 | pending--; | 1102 | pending--; |
1056 | if (NULL == t) | 1103 | if (NULL == t) |
1104 | { | ||
1105 | sleep_time_reg_proc = GNUNET_TIME_absolute_get (); | ||
1057 | t = GNUNET_SCHEDULER_add_now (&process_queue, | 1106 | t = GNUNET_SCHEDULER_add_now (&process_queue, |
1058 | NULL); | 1107 | NULL); |
1108 | } | ||
1059 | GNUNET_log (GNUNET_ERROR_TYPE_ERROR, | 1109 | GNUNET_log (GNUNET_ERROR_TYPE_ERROR, |
1060 | "Stub gave up on DNS reply for `%s'\n", | 1110 | "Stub gave up on DNS reply for `%s'\n", |
1061 | req->hostname); | 1111 | req->hostname); |
@@ -1073,6 +1123,7 @@ process_result (void *cls, | |||
1073 | GNUNET_NO); | 1123 | GNUNET_NO); |
1074 | return; | 1124 | return; |
1075 | } | 1125 | } |
1126 | total_reg_proc_dns++; | ||
1076 | req->rs = NULL; | 1127 | req->rs = NULL; |
1077 | insert_sorted (req); | 1128 | insert_sorted (req); |
1078 | return; | 1129 | return; |
@@ -1105,8 +1156,11 @@ process_result (void *cls, | |||
1105 | 1, | 1156 | 1, |
1106 | GNUNET_NO); | 1157 | GNUNET_NO); |
1107 | if (NULL == t) | 1158 | if (NULL == t) |
1159 | { | ||
1160 | sleep_time_reg_proc = GNUNET_TIME_absolute_get (); | ||
1108 | t = GNUNET_SCHEDULER_add_now (&process_queue, | 1161 | t = GNUNET_SCHEDULER_add_now (&process_queue, |
1109 | NULL); | 1162 | NULL); |
1163 | } | ||
1110 | if (req->issue_num > MAX_RETRIES) | 1164 | if (req->issue_num > MAX_RETRIES) |
1111 | { | 1165 | { |
1112 | failures++; | 1166 | failures++; |
@@ -1144,14 +1198,10 @@ process_result (void *cls, | |||
1144 | total_dns_latency_cnt++; | 1198 | total_dns_latency_cnt++; |
1145 | if (0 == (total_dns_latency_cnt % 1000)) | 1199 | if (0 == (total_dns_latency_cnt % 1000)) |
1146 | { | 1200 | { |
1147 | GNUNET_STATISTICS_update (stats, | 1201 | GNUNET_STATISTICS_set (stats, |
1148 | "# average DNS latency (μs)", | 1202 | "# average DNS lookup latency (μs)", |
1149 | total_dns_latency.rel_value_us / total_dns_latency_cnt, | 1203 | total_dns_latency.rel_value_us / total_dns_latency_cnt, |
1150 | GNUNET_NO); | 1204 | GNUNET_NO); |
1151 | GNUNET_STATISTICS_update (stats, | ||
1152 | "# DNS replies", | ||
1153 | total_dns_latency_cnt, | ||
1154 | GNUNET_NO); | ||
1155 | } | 1205 | } |
1156 | } | 1206 | } |
1157 | rd_count = 0; | 1207 | rd_count = 0; |
@@ -1171,10 +1221,18 @@ process_result (void *cls, | |||
1171 | /* Instead of going for SOA, simplified for now to look each | 1221 | /* Instead of going for SOA, simplified for now to look each |
1172 | day in case we got an empty response */ | 1222 | day in case we got an empty response */ |
1173 | if (0 == rd_count) | 1223 | if (0 == rd_count) |
1224 | { | ||
1174 | req->expires | 1225 | req->expires |
1175 | = GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_DAYS); | 1226 | = GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_DAYS); |
1227 | GNUNET_STATISTICS_update (stats, | ||
1228 | "# empty DNS replies (usually NXDOMAIN)", | ||
1229 | 1, | ||
1230 | GNUNET_NO); | ||
1231 | } | ||
1176 | else | 1232 | else |
1233 | { | ||
1177 | record_sets++; | 1234 | record_sets++; |
1235 | } | ||
1178 | /* convert records to namestore import format */ | 1236 | /* convert records to namestore import format */ |
1179 | { | 1237 | { |
1180 | struct GNUNET_GNSRECORD_Data rd[GNUNET_NZL(rd_count)]; | 1238 | struct GNUNET_GNSRECORD_Data rd[GNUNET_NZL(rd_count)]; |
@@ -1210,8 +1268,12 @@ process_queue (void *cls) | |||
1210 | unsigned int series; | 1268 | unsigned int series; |
1211 | void *raw; | 1269 | void *raw; |
1212 | size_t raw_size; | 1270 | size_t raw_size; |
1271 | struct GNUNET_TIME_Relative delay; | ||
1213 | 1272 | ||
1214 | (void) cls; | 1273 | (void) cls; |
1274 | delay = GNUNET_TIME_absolute_get_duration (sleep_time_reg_proc); | ||
1275 | idle_time = GNUNET_TIME_relative_add (idle_time, | ||
1276 | delay); | ||
1215 | series = 0; | 1277 | series = 0; |
1216 | t = NULL; | 1278 | t = NULL; |
1217 | while (pending + pending_rs < THRESH) | 1279 | while (pending + pending_rs < THRESH) |
@@ -1284,6 +1346,7 @@ process_queue (void *cls) | |||
1284 | req->hostname); | 1346 | req->hostname); |
1285 | if (NULL != t) | 1347 | if (NULL != t) |
1286 | GNUNET_SCHEDULER_cancel (t); | 1348 | GNUNET_SCHEDULER_cancel (t); |
1349 | sleep_time_reg_proc = GNUNET_TIME_absolute_get (); | ||
1287 | t = GNUNET_SCHEDULER_add_at (req->expires, | 1350 | t = GNUNET_SCHEDULER_add_at (req->expires, |
1288 | &process_queue, | 1351 | &process_queue, |
1289 | NULL); | 1352 | NULL); |
@@ -1293,6 +1356,7 @@ process_queue (void *cls) | |||
1293 | "Throttling\n"); | 1356 | "Throttling\n"); |
1294 | if (NULL != t) | 1357 | if (NULL != t) |
1295 | GNUNET_SCHEDULER_cancel (t); | 1358 | GNUNET_SCHEDULER_cancel (t); |
1359 | sleep_time_reg_proc = GNUNET_TIME_absolute_get (); | ||
1296 | t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY, | 1360 | t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY, |
1297 | &process_queue, | 1361 | &process_queue, |
1298 | NULL); | 1362 | NULL); |
@@ -1450,9 +1514,9 @@ ns_lookup_result_cb (void *cls, | |||
1450 | { | 1514 | { |
1451 | ns_iterator_trigger_next = NS_BATCH_SIZE; | 1515 | ns_iterator_trigger_next = NS_BATCH_SIZE; |
1452 | GNUNET_STATISTICS_update (stats, | 1516 | GNUNET_STATISTICS_update (stats, |
1453 | "# NAMESTORE records requested", | 1517 | "# NAMESTORE records requested from cache", |
1454 | ns_iterator_trigger_next, | 1518 | ns_iterator_trigger_next, |
1455 | GNUNET_NO); | 1519 | GNUNET_NO); |
1456 | GNUNET_NAMESTORE_zone_iterator_next (zone_it, | 1520 | GNUNET_NAMESTORE_zone_iterator_next (zone_it, |
1457 | ns_iterator_trigger_next); | 1521 | ns_iterator_trigger_next); |
1458 | } | 1522 | } |
@@ -1652,7 +1716,7 @@ iterate_zones (void *cls) | |||
1652 | last->domain); | 1716 | last->domain); |
1653 | /* subtract left-overs from previous iteration */ | 1717 | /* subtract left-overs from previous iteration */ |
1654 | GNUNET_STATISTICS_update (stats, | 1718 | GNUNET_STATISTICS_update (stats, |
1655 | "# NAMESTORE records requested", | 1719 | "# NAMESTORE records requested from cache", |
1656 | (long long) (- ns_iterator_trigger_next), | 1720 | (long long) (- ns_iterator_trigger_next), |
1657 | GNUNET_NO); | 1721 | GNUNET_NO); |
1658 | ns_iterator_trigger_next = 0; | 1722 | ns_iterator_trigger_next = 0; |
@@ -1665,7 +1729,7 @@ iterate_zones (void *cls) | |||
1665 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 1729 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, |
1666 | "Finished all NAMESTORE iterations!\n"); | 1730 | "Finished all NAMESTORE iterations!\n"); |
1667 | GNUNET_STATISTICS_set (stats, | 1731 | GNUNET_STATISTICS_set (stats, |
1668 | "# NAMESTORE lookups without reply", | 1732 | "# Domain names without cached reply", |
1669 | GNUNET_CONTAINER_multihashmap_size (ns_pending), | 1733 | GNUNET_CONTAINER_multihashmap_size (ns_pending), |
1670 | GNUNET_NO); | 1734 | GNUNET_NO); |
1671 | GNUNET_CONTAINER_multihashmap_iterate (ns_pending, | 1735 | GNUNET_CONTAINER_multihashmap_iterate (ns_pending, |
@@ -1673,6 +1737,9 @@ iterate_zones (void *cls) | |||
1673 | NULL); | 1737 | NULL); |
1674 | GNUNET_CONTAINER_multihashmap_destroy (ns_pending); | 1738 | GNUNET_CONTAINER_multihashmap_destroy (ns_pending); |
1675 | ns_pending = NULL; | 1739 | ns_pending = NULL; |
1740 | start_time_reg_proc = GNUNET_TIME_absolute_get (); | ||
1741 | total_reg_proc_dns = 0; | ||
1742 | total_reg_proc_dns_ns = 0; | ||
1676 | return; | 1743 | return; |
1677 | } | 1744 | } |
1678 | if (NULL == last) | 1745 | if (NULL == last) |
@@ -1684,7 +1751,7 @@ iterate_zones (void *cls) | |||
1684 | last->domain); | 1751 | last->domain); |
1685 | /* subtract left-overs from previous iteration */ | 1752 | /* subtract left-overs from previous iteration */ |
1686 | GNUNET_STATISTICS_update (stats, | 1753 | GNUNET_STATISTICS_update (stats, |
1687 | "# NAMESTORE records requested", | 1754 | "# NAMESTORE records requested from cache", |
1688 | 1, | 1755 | 1, |
1689 | GNUNET_NO); | 1756 | GNUNET_NO); |
1690 | ns_iterator_trigger_next = 1; | 1757 | ns_iterator_trigger_next = 1; |
@@ -1713,7 +1780,7 @@ static void | |||
1713 | process_stdin (void *cls) | 1780 | process_stdin (void *cls) |
1714 | { | 1781 | { |
1715 | static struct GNUNET_TIME_Absolute last; | 1782 | static struct GNUNET_TIME_Absolute last; |
1716 | static unsigned int pdot; | 1783 | static uint64_t idot; |
1717 | char hn[256]; | 1784 | char hn[256]; |
1718 | 1785 | ||
1719 | (void) cls; | 1786 | (void) cls; |
@@ -1730,30 +1797,32 @@ process_stdin (void *cls) | |||
1730 | { | 1797 | { |
1731 | if (strlen(hn) > 0) | 1798 | if (strlen(hn) > 0) |
1732 | hn[strlen(hn)-1] = '\0'; /* eat newline */ | 1799 | hn[strlen(hn)-1] = '\0'; /* eat newline */ |
1733 | if (0 == pdot) | 1800 | if (0 == idot) |
1734 | last = GNUNET_TIME_absolute_get (); | 1801 | last = GNUNET_TIME_absolute_get (); |
1735 | pdot++; | 1802 | idot++; |
1736 | if (0 == pdot % 1000) | 1803 | if (0 == idot % 10000) |
1737 | { | 1804 | { |
1738 | struct GNUNET_TIME_Relative delta; | 1805 | struct GNUNET_TIME_Relative delta; |
1739 | 1806 | ||
1740 | delta = GNUNET_TIME_absolute_get_duration (last); | 1807 | delta = GNUNET_TIME_absolute_get_duration (last); |
1741 | last = GNUNET_TIME_absolute_get (); | 1808 | last = GNUNET_TIME_absolute_get (); |
1742 | fprintf (stderr, | 1809 | fprintf (stderr, |
1743 | "Imported 1000 records in %s\n", | 1810 | "Imported 10000 records in %s\n", |
1744 | GNUNET_STRINGS_relative_time_to_string (delta, | 1811 | GNUNET_STRINGS_relative_time_to_string (delta, |
1745 | GNUNET_YES)); | 1812 | GNUNET_YES)); |
1746 | GNUNET_STATISTICS_set (stats, | 1813 | GNUNET_STATISTICS_set (stats, |
1747 | "# domain names provided", | 1814 | "# domain names provided", |
1748 | pdot, | 1815 | idot, |
1749 | GNUNET_NO); | 1816 | GNUNET_NO); |
1750 | } | 1817 | } |
1751 | queue (hn); | 1818 | queue (hn); |
1752 | } | 1819 | } |
1753 | fprintf (stderr, "\n"); | 1820 | fprintf (stderr, |
1821 | "Done reading %llu domain names\n", | ||
1822 | (unsigned long long) idot); | ||
1754 | GNUNET_STATISTICS_set (stats, | 1823 | GNUNET_STATISTICS_set (stats, |
1755 | "# domain names provided", | 1824 | "# domain names provided", |
1756 | pdot, | 1825 | idot, |
1757 | GNUNET_NO); | 1826 | GNUNET_NO); |
1758 | iterate_zones (NULL); | 1827 | iterate_zones (NULL); |
1759 | } | 1828 | } |
@@ -1851,8 +1920,14 @@ run (void *cls, | |||
1851 | stats = GNUNET_STATISTICS_create ("zoneimport", | 1920 | stats = GNUNET_STATISTICS_create ("zoneimport", |
1852 | cfg); | 1921 | cfg); |
1853 | req_heap = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN); | 1922 | req_heap = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN); |
1854 | ns_pending = GNUNET_CONTAINER_multihashmap_create (1024, | 1923 | ns_pending = GNUNET_CONTAINER_multihashmap_create (map_size, |
1855 | GNUNET_NO); | 1924 | GNUNET_NO); |
1925 | if (NULL == ns_pending) | ||
1926 | { | ||
1927 | fprintf (stderr, | ||
1928 | "Failed to allocate memory for main hash map\n"); | ||
1929 | return; | ||
1930 | } | ||
1856 | ctx = GNUNET_DNSSTUB_start (256); | 1931 | ctx = GNUNET_DNSSTUB_start (256); |
1857 | if (NULL == ctx) | 1932 | if (NULL == ctx) |
1858 | { | 1933 | { |
@@ -1906,6 +1981,11 @@ main (int argc, | |||
1906 | char *const*argv) | 1981 | char *const*argv) |
1907 | { | 1982 | { |
1908 | struct GNUNET_GETOPT_CommandLineOption options[] = { | 1983 | struct GNUNET_GETOPT_CommandLineOption options[] = { |
1984 | GNUNET_GETOPT_option_uint ('s', | ||
1985 | "size", | ||
1986 | "MAPSIZE", | ||
1987 | gettext_noop ("size to use for the main hash map"), | ||
1988 | &map_size), | ||
1909 | GNUNET_GETOPT_OPTION_END | 1989 | GNUNET_GETOPT_OPTION_END |
1910 | }; | 1990 | }; |
1911 | 1991 | ||
diff --git a/src/util/container_multihashmap.c b/src/util/container_multihashmap.c index ffeb4a71f..6001fc1df 100644 --- a/src/util/container_multihashmap.c +++ b/src/util/container_multihashmap.c | |||
@@ -175,14 +175,36 @@ struct GNUNET_CONTAINER_MultiHashMap * | |||
175 | GNUNET_CONTAINER_multihashmap_create (unsigned int len, | 175 | GNUNET_CONTAINER_multihashmap_create (unsigned int len, |
176 | int do_not_copy_keys) | 176 | int do_not_copy_keys) |
177 | { | 177 | { |
178 | struct GNUNET_CONTAINER_MultiHashMap *map; | 178 | struct GNUNET_CONTAINER_MultiHashMap *hm; |
179 | 179 | ||
180 | GNUNET_assert (len > 0); | 180 | GNUNET_assert (len > 0); |
181 | map = GNUNET_new (struct GNUNET_CONTAINER_MultiHashMap); | 181 | hm = GNUNET_new (struct GNUNET_CONTAINER_MultiHashMap); |
182 | map->map = GNUNET_malloc (len * sizeof (union MapEntry)); | 182 | if (len * sizeof (union MapEntry) > GNUNET_MAX_MALLOC_CHECKED) |
183 | map->map_length = len; | 183 | { |
184 | map->use_small_entries = do_not_copy_keys; | 184 | size_t s; |
185 | return map; | 185 | /* application *explicitly* requested very large map, hopefully |
186 | it checks the return value... */ | ||
187 | s = len * sizeof (union MapEntry); | ||
188 | if ( (s / sizeof (union MapEntry)) != len) | ||
189 | return NULL; /* integer overflow on multiplication */ | ||
190 | if (NULL == (hm->map = GNUNET_malloc_large (s))) | ||
191 | { | ||
192 | /* out of memory */ | ||
193 | GNUNET_log (GNUNET_ERROR_TYPE_WARNING, | ||
194 | "Out of memory allocating large hash map (%u entries)\n", | ||
195 | len); | ||
196 | GNUNET_free (hm); | ||
197 | return NULL; | ||
198 | } | ||
199 | } | ||
200 | else | ||
201 | { | ||
202 | hm->map = GNUNET_new_array (len, | ||
203 | union MapEntry); | ||
204 | } | ||
205 | hm->map_length = len; | ||
206 | hm->use_small_entries = do_not_copy_keys; | ||
207 | return hm; | ||
186 | } | 208 | } |
187 | 209 | ||
188 | 210 | ||
@@ -196,11 +218,10 @@ void | |||
196 | GNUNET_CONTAINER_multihashmap_destroy (struct GNUNET_CONTAINER_MultiHashMap | 218 | GNUNET_CONTAINER_multihashmap_destroy (struct GNUNET_CONTAINER_MultiHashMap |
197 | *map) | 219 | *map) |
198 | { | 220 | { |
199 | unsigned int i; | 221 | for (unsigned int i = 0; i < map->map_length; i++) |
200 | union MapEntry me; | ||
201 | |||
202 | for (i = 0; i < map->map_length; i++) | ||
203 | { | 222 | { |
223 | union MapEntry me; | ||
224 | |||
204 | me = map->map[i]; | 225 | me = map->map[i]; |
205 | if (map->use_small_entries) | 226 | if (map->use_small_entries) |
206 | { | 227 | { |
@@ -257,8 +278,7 @@ idx_of (const struct GNUNET_CONTAINER_MultiHashMap *map, | |||
257 | * @return the number of key value pairs | 278 | * @return the number of key value pairs |
258 | */ | 279 | */ |
259 | unsigned int | 280 | unsigned int |
260 | GNUNET_CONTAINER_multihashmap_size (const struct GNUNET_CONTAINER_MultiHashMap | 281 | GNUNET_CONTAINER_multihashmap_size (const struct GNUNET_CONTAINER_MultiHashMap *map) |
261 | *map) | ||
262 | { | 282 | { |
263 | return map->size; | 283 | return map->size; |
264 | } | 284 | } |
@@ -656,17 +676,22 @@ grow (struct GNUNET_CONTAINER_MultiHashMap *map) | |||
656 | unsigned int old_len; | 676 | unsigned int old_len; |
657 | unsigned int new_len; | 677 | unsigned int new_len; |
658 | unsigned int idx; | 678 | unsigned int idx; |
659 | unsigned int i; | ||
660 | 679 | ||
661 | map->modification_counter++; | 680 | map->modification_counter++; |
662 | 681 | ||
663 | old_map = map->map; | 682 | old_map = map->map; |
664 | old_len = map->map_length; | 683 | old_len = map->map_length; |
665 | new_len = old_len * 2; | 684 | new_len = old_len * 2; |
666 | new_map = GNUNET_malloc (sizeof (union MapEntry) * new_len); | 685 | /* if we would exceed heap size limit for the _first_ time, |
686 | try staying just below the limit */ | ||
687 | if ( (new_len * sizeof (union MapEntry) > GNUNET_MAX_MALLOC_CHECKED) && | ||
688 | ((old_len+1) * sizeof (union MapEntry) < GNUNET_MAX_MALLOC_CHECKED) ) | ||
689 | new_len = GNUNET_MAX_MALLOC_CHECKED / sizeof (union MapEntry); | ||
690 | new_map = GNUNET_new_array (new_len, | ||
691 | union MapEntry); | ||
667 | map->map_length = new_len; | 692 | map->map_length = new_len; |
668 | map->map = new_map; | 693 | map->map = new_map; |
669 | for (i = 0; i < old_len; i++) | 694 | for (unsigned int i = 0; i < old_len; i++) |
670 | { | 695 | { |
671 | if (map->use_small_entries) | 696 | if (map->use_small_entries) |
672 | { | 697 | { |
diff --git a/src/zonemaster/gnunet-service-zonemaster.c b/src/zonemaster/gnunet-service-zonemaster.c index 55e1a0eee..322a1093b 100644 --- a/src/zonemaster/gnunet-service-zonemaster.c +++ b/src/zonemaster/gnunet-service-zonemaster.c | |||
@@ -63,7 +63,7 @@ | |||
63 | * The initial interval in milliseconds btween puts in | 63 | * The initial interval in milliseconds btween puts in |
64 | * a zone iteration | 64 | * a zone iteration |
65 | */ | 65 | */ |
66 | #define INITIAL_PUT_INTERVAL GNUNET_TIME_UNIT_MILLISECONDS | 66 | #define INITIAL_ZONE_ITERATION_INTERVAL GNUNET_TIME_UNIT_MILLISECONDS |
67 | 67 | ||
68 | /** | 68 | /** |
69 | * The upper bound for the zone iteration interval | 69 | * The upper bound for the zone iteration interval |
@@ -204,7 +204,7 @@ static unsigned long long put_cnt; | |||
204 | * and the total number of record sets we have (so far) | 204 | * and the total number of record sets we have (so far) |
205 | * observed in the zone. | 205 | * observed in the zone. |
206 | */ | 206 | */ |
207 | static struct GNUNET_TIME_Relative next_put_interval; | 207 | static struct GNUNET_TIME_Relative target_iteration_velocity_per_record; |
208 | 208 | ||
209 | /** | 209 | /** |
210 | * Minimum relative expiration time of records seem during the current | 210 | * Minimum relative expiration time of records seem during the current |
@@ -383,7 +383,7 @@ dht_put_monitor_continuation (void *cls) | |||
383 | 383 | ||
384 | 384 | ||
385 | /** | 385 | /** |
386 | * Calculate #next_put_interval. | 386 | * Calculate #target_iteration_velocity_per_record. |
387 | */ | 387 | */ |
388 | static void | 388 | static void |
389 | calculate_put_interval () | 389 | calculate_put_interval () |
@@ -395,7 +395,7 @@ calculate_put_interval () | |||
395 | * we can safely set the interval to the value for a single | 395 | * we can safely set the interval to the value for a single |
396 | * record | 396 | * record |
397 | */ | 397 | */ |
398 | next_put_interval = zone_publish_time_window; | 398 | target_iteration_velocity_per_record = zone_publish_time_window; |
399 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG | GNUNET_ERROR_TYPE_BULK, | 399 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG | GNUNET_ERROR_TYPE_BULK, |
400 | "No records in namestore database.\n"); | 400 | "No records in namestore database.\n"); |
401 | } | 401 | } |
@@ -408,24 +408,24 @@ calculate_put_interval () | |||
408 | = GNUNET_TIME_relative_min (GNUNET_TIME_relative_divide (last_min_relative_record_time, | 408 | = GNUNET_TIME_relative_min (GNUNET_TIME_relative_divide (last_min_relative_record_time, |
409 | PUBLISH_OPS_PER_EXPIRATION), | 409 | PUBLISH_OPS_PER_EXPIRATION), |
410 | zone_publish_time_window_default); | 410 | zone_publish_time_window_default); |
411 | next_put_interval | 411 | target_iteration_velocity_per_record |
412 | = GNUNET_TIME_relative_divide (zone_publish_time_window, | 412 | = GNUNET_TIME_relative_divide (zone_publish_time_window, |
413 | last_num_public_records); | 413 | last_num_public_records); |
414 | } | 414 | } |
415 | next_put_interval | 415 | target_iteration_velocity_per_record |
416 | = GNUNET_TIME_relative_min (next_put_interval, | 416 | = GNUNET_TIME_relative_min (target_iteration_velocity_per_record, |
417 | MAXIMUM_ZONE_ITERATION_INTERVAL); | 417 | MAXIMUM_ZONE_ITERATION_INTERVAL); |
418 | GNUNET_STATISTICS_set (statistics, | 418 | GNUNET_STATISTICS_set (statistics, |
419 | "Minimum relative record expiration (in ms)", | 419 | "Minimum relative record expiration (in μs)", |
420 | last_min_relative_record_time.rel_value_us / 1000LL, | 420 | last_min_relative_record_time.rel_value_us, |
421 | GNUNET_NO); | 421 | GNUNET_NO); |
422 | GNUNET_STATISTICS_set (statistics, | 422 | GNUNET_STATISTICS_set (statistics, |
423 | "Zone publication time window (in ms)", | 423 | "Zone publication time window (in μs)", |
424 | zone_publish_time_window.rel_value_us / 1000LL, | 424 | zone_publish_time_window.rel_value_us, |
425 | GNUNET_NO); | 425 | GNUNET_NO); |
426 | GNUNET_STATISTICS_set (statistics, | 426 | GNUNET_STATISTICS_set (statistics, |
427 | "Target zone iteration velocity (μs)", | 427 | "Target zone iteration velocity (μs)", |
428 | next_put_interval.rel_value_us, | 428 | target_iteration_velocity_per_record.rel_value_us, |
429 | GNUNET_NO); | 429 | GNUNET_NO); |
430 | } | 430 | } |
431 | 431 | ||
@@ -461,7 +461,7 @@ update_velocity (unsigned int cnt) | |||
461 | } | 461 | } |
462 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 462 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, |
463 | "Desired global zone iteration interval is %s/record!\n", | 463 | "Desired global zone iteration interval is %s/record!\n", |
464 | GNUNET_STRINGS_relative_time_to_string (next_put_interval, | 464 | GNUNET_STRINGS_relative_time_to_string (target_iteration_velocity_per_record, |
465 | GNUNET_YES)); | 465 | GNUNET_YES)); |
466 | 466 | ||
467 | /* Tell statistics actual vs. desired speed */ | 467 | /* Tell statistics actual vs. desired speed */ |
@@ -471,12 +471,12 @@ update_velocity (unsigned int cnt) | |||
471 | GNUNET_NO); | 471 | GNUNET_NO); |
472 | /* update "sub_delta" based on difference, taking | 472 | /* update "sub_delta" based on difference, taking |
473 | previous sub_delta into account! */ | 473 | previous sub_delta into account! */ |
474 | if (next_put_interval.rel_value_us > delta.rel_value_us) | 474 | if (target_iteration_velocity_per_record.rel_value_us > delta.rel_value_us) |
475 | { | 475 | { |
476 | /* We were too fast, reduce sub_delta! */ | 476 | /* We were too fast, reduce sub_delta! */ |
477 | struct GNUNET_TIME_Relative corr; | 477 | struct GNUNET_TIME_Relative corr; |
478 | 478 | ||
479 | corr = GNUNET_TIME_relative_subtract (next_put_interval, | 479 | corr = GNUNET_TIME_relative_subtract (target_iteration_velocity_per_record, |
480 | delta); | 480 | delta); |
481 | if (sub_delta.rel_value_us > delta.rel_value_us) | 481 | if (sub_delta.rel_value_us > delta.rel_value_us) |
482 | { | 482 | { |
@@ -492,28 +492,28 @@ update_velocity (unsigned int cnt) | |||
492 | sub_delta = GNUNET_TIME_UNIT_ZERO; | 492 | sub_delta = GNUNET_TIME_UNIT_ZERO; |
493 | } | 493 | } |
494 | } | 494 | } |
495 | else if (next_put_interval.rel_value_us < delta.rel_value_us) | 495 | else if (target_iteration_velocity_per_record.rel_value_us < delta.rel_value_us) |
496 | { | 496 | { |
497 | /* We were too slow, increase sub_delta! */ | 497 | /* We were too slow, increase sub_delta! */ |
498 | struct GNUNET_TIME_Relative corr; | 498 | struct GNUNET_TIME_Relative corr; |
499 | 499 | ||
500 | corr = GNUNET_TIME_relative_subtract (delta, | 500 | corr = GNUNET_TIME_relative_subtract (delta, |
501 | next_put_interval); | 501 | target_iteration_velocity_per_record); |
502 | sub_delta = GNUNET_TIME_relative_add (sub_delta, | 502 | sub_delta = GNUNET_TIME_relative_add (sub_delta, |
503 | corr); | 503 | corr); |
504 | if (sub_delta.rel_value_us > next_put_interval.rel_value_us) | 504 | if (sub_delta.rel_value_us > target_iteration_velocity_per_record.rel_value_us) |
505 | { | 505 | { |
506 | /* CPU overload detected, we cannot go at desired speed, | 506 | /* CPU overload detected, we cannot go at desired speed, |
507 | as this would mean using a negative delay. */ | 507 | as this would mean using a negative delay. */ |
508 | /* compute how much faster we would want to be for | 508 | /* compute how much faster we would want to be for |
509 | the desired velocity */ | 509 | the desired velocity */ |
510 | if (0 == next_put_interval.rel_value_us) | 510 | if (0 == target_iteration_velocity_per_record.rel_value_us) |
511 | pct = UINT64_MAX; /* desired speed is infinity ... */ | 511 | pct = UINT64_MAX; /* desired speed is infinity ... */ |
512 | else | 512 | else |
513 | pct = (sub_delta.rel_value_us - | 513 | pct = (sub_delta.rel_value_us - |
514 | next_put_interval.rel_value_us) * 100LLU | 514 | target_iteration_velocity_per_record.rel_value_us) * 100LLU |
515 | / next_put_interval.rel_value_us; | 515 | / target_iteration_velocity_per_record.rel_value_us; |
516 | sub_delta = next_put_interval; | 516 | sub_delta = target_iteration_velocity_per_record; |
517 | } | 517 | } |
518 | } | 518 | } |
519 | GNUNET_STATISTICS_set (statistics, | 519 | GNUNET_STATISTICS_set (statistics, |
@@ -548,7 +548,7 @@ check_zone_namestore_next () | |||
548 | return; /* current NAMESTORE iteration not yet done */ | 548 | return; /* current NAMESTORE iteration not yet done */ |
549 | update_velocity (put_cnt); | 549 | update_velocity (put_cnt); |
550 | put_cnt = 0; | 550 | put_cnt = 0; |
551 | delay = GNUNET_TIME_relative_subtract (next_put_interval, | 551 | delay = GNUNET_TIME_relative_subtract (target_iteration_velocity_per_record, |
552 | sub_delta); | 552 | sub_delta); |
553 | /* We delay *once* per #NS_BLOCK_SIZE, so we need to multiply the | 553 | /* We delay *once* per #NS_BLOCK_SIZE, so we need to multiply the |
554 | per-record delay calculated so far with the #NS_BLOCK_SIZE */ | 554 | per-record delay calculated so far with the #NS_BLOCK_SIZE */ |
@@ -755,11 +755,11 @@ zone_iteration_finished (void *cls) | |||
755 | PUBLISH_OPS_PER_EXPIRATION); | 755 | PUBLISH_OPS_PER_EXPIRATION); |
756 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 756 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, |
757 | "Zone iteration finished. Adjusted zone iteration interval to %s\n", | 757 | "Zone iteration finished. Adjusted zone iteration interval to %s\n", |
758 | GNUNET_STRINGS_relative_time_to_string (next_put_interval, | 758 | GNUNET_STRINGS_relative_time_to_string (target_iteration_velocity_per_record, |
759 | GNUNET_YES)); | 759 | GNUNET_YES)); |
760 | GNUNET_STATISTICS_set (statistics, | 760 | GNUNET_STATISTICS_set (statistics, |
761 | "Current zone iteration interval (in ms)", | 761 | "Target zone iteration velocity (μs)", |
762 | next_put_interval.rel_value_us / 1000LL, | 762 | target_iteration_velocity_per_record.rel_value_us, |
763 | GNUNET_NO); | 763 | GNUNET_NO); |
764 | GNUNET_STATISTICS_set (statistics, | 764 | GNUNET_STATISTICS_set (statistics, |
765 | "Number of public records in DHT", | 765 | "Number of public records in DHT", |
@@ -767,12 +767,16 @@ zone_iteration_finished (void *cls) | |||
767 | GNUNET_NO); | 767 | GNUNET_NO); |
768 | GNUNET_assert (NULL == zone_publish_task); | 768 | GNUNET_assert (NULL == zone_publish_task); |
769 | if (0 == last_num_public_records) | 769 | if (0 == last_num_public_records) |
770 | zone_publish_task = GNUNET_SCHEDULER_add_delayed (next_put_interval, | 770 | { |
771 | zone_publish_task = GNUNET_SCHEDULER_add_delayed (target_iteration_velocity_per_record, | ||
771 | &publish_zone_dht_start, | 772 | &publish_zone_dht_start, |
772 | NULL); | 773 | NULL); |
774 | } | ||
773 | else | 775 | else |
776 | { | ||
774 | zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start, | 777 | zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start, |
775 | NULL); | 778 | NULL); |
779 | } | ||
776 | } | 780 | } |
777 | 781 | ||
778 | 782 | ||
@@ -1028,7 +1032,7 @@ run (void *cls, | |||
1028 | min_relative_record_time | 1032 | min_relative_record_time |
1029 | = GNUNET_TIME_relative_multiply (GNUNET_DHT_DEFAULT_REPUBLISH_FREQUENCY, | 1033 | = GNUNET_TIME_relative_multiply (GNUNET_DHT_DEFAULT_REPUBLISH_FREQUENCY, |
1030 | PUBLISH_OPS_PER_EXPIRATION); | 1034 | PUBLISH_OPS_PER_EXPIRATION); |
1031 | next_put_interval = INITIAL_PUT_INTERVAL; | 1035 | target_iteration_velocity_per_record = INITIAL_ZONE_ITERATION_INTERVAL; |
1032 | namestore_handle = GNUNET_NAMESTORE_connect (c); | 1036 | namestore_handle = GNUNET_NAMESTORE_connect (c); |
1033 | if (NULL == namestore_handle) | 1037 | if (NULL == namestore_handle) |
1034 | { | 1038 | { |
@@ -1077,12 +1081,12 @@ run (void *cls, | |||
1077 | } | 1081 | } |
1078 | 1082 | ||
1079 | /* Schedule periodic put for our records. */ | 1083 | /* Schedule periodic put for our records. */ |
1080 | first_zone_iteration = GNUNET_YES;\ | 1084 | first_zone_iteration = GNUNET_YES; |
1081 | statistics = GNUNET_STATISTICS_create ("zonemaster", | 1085 | statistics = GNUNET_STATISTICS_create ("zonemaster", |
1082 | c); | 1086 | c); |
1083 | GNUNET_STATISTICS_set (statistics, | 1087 | GNUNET_STATISTICS_set (statistics, |
1084 | "Target zone iteration velocity (μs)", | 1088 | "Target zone iteration velocity (μs)", |
1085 | next_put_interval.rel_value_us, | 1089 | target_iteration_velocity_per_record.rel_value_us, |
1086 | GNUNET_NO); | 1090 | GNUNET_NO); |
1087 | zmon = GNUNET_NAMESTORE_zone_monitor_start (c, | 1091 | zmon = GNUNET_NAMESTORE_zone_monitor_start (c, |
1088 | NULL, | 1092 | NULL, |