aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--doc/man/gnunet-zoneimport.15
-rw-r--r--src/namestore/gnunet-zoneimport.c202
-rw-r--r--src/util/container_multihashmap.c55
-rw-r--r--src/zonemaster/gnunet-service-zonemaster.c66
4 files changed, 220 insertions, 108 deletions
diff --git a/doc/man/gnunet-zoneimport.1 b/doc/man/gnunet-zoneimport.1
index 687e6f900..06b3a6bcf 100644
--- a/doc/man/gnunet-zoneimport.1
+++ b/doc/man/gnunet-zoneimport.1
@@ -27,12 +27,15 @@ Use the configuration file FILENAME.
27.B 27.B
28.IP "\-h, \-\-help" 28.IP "\-h, \-\-help"
29Print short help on options. 29Print short help on options.
30.B
31.IP "\-s MAPSIZE, \-\-size=MAPSIZE"
32Specifies the size (in number of entries) to use for the main hash map. The value provided should be at least twice the number of domain names that will be given to the tool. This option is required for very large zones where the number of records encountered is too large for the automatic growth mechanism to work (that one is limited to at most 16 MB allocations for security reasons). Do not worry about this unless you are importing millions of domain names from a zone.
30 33
31.SH NOTES 34.SH NOTES
32 35
33Typical invocaton would be: 36Typical invocaton would be:
34 37
35$ gnunet\-zoneimport -s 1.2.3.4 < names.txt 38$ gnunet\-zoneimport 1.2.3.4 < names.txt
36 39
37 40
38 41
diff --git a/src/namestore/gnunet-zoneimport.c b/src/namestore/gnunet-zoneimport.c
index e24cb26dd..0fd0a4ab8 100644
--- a/src/namestore/gnunet-zoneimport.c
+++ b/src/namestore/gnunet-zoneimport.c
@@ -204,6 +204,14 @@ struct Request
204 204
205 205
206/** 206/**
207 * Command-line argument specifying desired size of the hash map with
208 * all of our pending names. Usually, we use an automatically growing
209 * map, but this is only OK up to about a million entries. Above that
210 * number, the user must explicitly specify the size at startup.
211 */
212static unsigned int map_size = 1024;
213
214/**
207 * Handle to the identity service. 215 * Handle to the identity service.
208 */ 216 */
209static struct GNUNET_IDENTITY_Handle *id; 217static struct GNUNET_IDENTITY_Handle *id;
@@ -323,14 +331,29 @@ static uint64_t total_dns_latency_cnt;
323static struct GNUNET_TIME_Relative total_dns_latency; 331static struct GNUNET_TIME_Relative total_dns_latency;
324 332
325/** 333/**
326 * Number of NAMESTORE requests counted in latency total. 334 * Number of records processed (DNS lookup, no NAMESTORE) in total.
335 */
336static uint64_t total_reg_proc_dns;
337
338/**
339 * Number of records processed (DNS lookup, with NAMESTORE) in total.
340 */
341static uint64_t total_reg_proc_dns_ns;
342
343/**
344 * Start time of the regular processing.
345 */
346static struct GNUNET_TIME_Absolute start_time_reg_proc;
347
348/**
349 * Last time we worked before going idle.
327 */ 350 */
328static uint64_t total_ns_latency_cnt; 351static struct GNUNET_TIME_Absolute sleep_time_reg_proc;
329 352
330/** 353/**
331 * Sum of NAMESTORE latencies observed. 354 * Time we slept just waiting for work.
332 */ 355 */
333static struct GNUNET_TIME_Relative total_ns_latency; 356static struct GNUNET_TIME_Relative idle_time;
334 357
335 358
336/** 359/**
@@ -530,6 +553,7 @@ insert_sorted (struct Request *req)
530 { 553 {
531 if (NULL != t) 554 if (NULL != t)
532 GNUNET_SCHEDULER_cancel (t); 555 GNUNET_SCHEDULER_cancel (t);
556 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
533 t = GNUNET_SCHEDULER_add_at (req->expires, 557 t = GNUNET_SCHEDULER_add_at (req->expires,
534 &process_queue, 558 &process_queue,
535 NULL); 559 NULL);
@@ -966,37 +990,9 @@ store_completed_cb (void *cls,
966 const char *emsg) 990 const char *emsg)
967{ 991{
968 static struct GNUNET_TIME_Absolute last; 992 static struct GNUNET_TIME_Absolute last;
969 static unsigned int pdot;
970 struct Request *req = cls; 993 struct Request *req = cls;
971 994
972 req->qe = NULL; 995 req->qe = NULL;
973 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
974 "Stored record set in database (%d)\n",
975 success);
976 pending_rs--;
977 {
978 struct GNUNET_TIME_Relative ns_latency;
979
980 ns_latency = GNUNET_TIME_absolute_get_duration (req->op_start_time);
981 total_ns_latency = GNUNET_TIME_relative_add (total_ns_latency,
982 ns_latency);
983 total_ns_latency_cnt++;
984 if (0 == (total_ns_latency_cnt % 1000))
985 {
986 GNUNET_STATISTICS_update (stats,
987 "# average NAMESTORE PUT latency (μs)",
988 total_ns_latency.rel_value_us / total_ns_latency_cnt,
989 GNUNET_NO);
990 GNUNET_STATISTICS_update (stats,
991 "# NAMESTORE PUTs",
992 total_ns_latency_cnt,
993 GNUNET_NO);
994 }
995 }
996
997 if (NULL == t)
998 t = GNUNET_SCHEDULER_add_now (&process_queue,
999 NULL);
1000 if (GNUNET_SYSERR == success) 996 if (GNUNET_SYSERR == success)
1001 { 997 {
1002 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 998 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
@@ -1007,12 +1003,26 @@ store_completed_cb (void *cls,
1007 else 1003 else
1008 { 1004 {
1009 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 1005 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1010 "Stored records under `%s'\n", 1006 "Stored records under `%s' (%d)\n",
1011 req->hostname); 1007 req->hostname,
1012 if (0 == pdot) 1008 success);
1009 }
1010 total_reg_proc_dns_ns++; /* finished regular processing */
1011 pending_rs--;
1012 free_records (req);
1013 /* compute NAMESTORE statistics */
1014 {
1015 static uint64_t total_ns_latency_cnt;
1016 static struct GNUNET_TIME_Relative total_ns_latency;
1017 struct GNUNET_TIME_Relative ns_latency;
1018
1019 ns_latency = GNUNET_TIME_absolute_get_duration (req->op_start_time);
1020 total_ns_latency = GNUNET_TIME_relative_add (total_ns_latency,
1021 ns_latency);
1022 if (0 == total_ns_latency_cnt)
1013 last = GNUNET_TIME_absolute_get (); 1023 last = GNUNET_TIME_absolute_get ();
1014 pdot++; 1024 total_ns_latency_cnt++;
1015 if (0 == pdot % 1000) 1025 if (0 == (total_ns_latency_cnt % 1000))
1016 { 1026 {
1017 struct GNUNET_TIME_Relative delta; 1027 struct GNUNET_TIME_Relative delta;
1018 1028
@@ -1022,9 +1032,46 @@ store_completed_cb (void *cls,
1022 "Processed 1000 records in %s\n", 1032 "Processed 1000 records in %s\n",
1023 GNUNET_STRINGS_relative_time_to_string (delta, 1033 GNUNET_STRINGS_relative_time_to_string (delta,
1024 GNUNET_YES)); 1034 GNUNET_YES));
1035 GNUNET_STATISTICS_set (stats,
1036 "# average NAMESTORE PUT latency (μs)",
1037 total_ns_latency.rel_value_us / total_ns_latency_cnt,
1038 GNUNET_NO);
1025 } 1039 }
1026 } 1040 }
1027 free_records (req); 1041 /* compute and publish overall velocity */
1042 if (0 == (total_reg_proc_dns_ns % 100) )
1043 {
1044 struct GNUNET_TIME_Relative runtime;
1045
1046 runtime = GNUNET_TIME_absolute_get_duration (start_time_reg_proc);
1047 runtime = GNUNET_TIME_relative_subtract (runtime,
1048 idle_time);
1049 runtime = GNUNET_TIME_relative_divide (runtime,
1050 total_reg_proc_dns + total_reg_proc_dns_ns);
1051 GNUNET_STATISTICS_set (stats,
1052 "# Regular processing completed without NAMESTORE",
1053 total_reg_proc_dns,
1054 GNUNET_NO);
1055 GNUNET_STATISTICS_set (stats,
1056 "# Regular processing completed with NAMESTORE PUT",
1057 total_reg_proc_dns_ns,
1058 GNUNET_NO);
1059 GNUNET_STATISTICS_set (stats,
1060 "# average request processing latency (μs)",
1061 runtime.rel_value_us,
1062 GNUNET_NO);
1063 GNUNET_STATISTICS_set (stats,
1064 "# total time spent idle (μs)",
1065 idle_time.rel_value_us,
1066 GNUNET_NO);
1067 }
1068
1069 if (NULL == t)
1070 {
1071 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1072 t = GNUNET_SCHEDULER_add_now (&process_queue,
1073 NULL);
1074 }
1028} 1075}
1029 1076
1030 1077
@@ -1054,8 +1101,11 @@ process_result (void *cls,
1054 req); 1101 req);
1055 pending--; 1102 pending--;
1056 if (NULL == t) 1103 if (NULL == t)
1104 {
1105 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1057 t = GNUNET_SCHEDULER_add_now (&process_queue, 1106 t = GNUNET_SCHEDULER_add_now (&process_queue,
1058 NULL); 1107 NULL);
1108 }
1059 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 1109 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1060 "Stub gave up on DNS reply for `%s'\n", 1110 "Stub gave up on DNS reply for `%s'\n",
1061 req->hostname); 1111 req->hostname);
@@ -1073,6 +1123,7 @@ process_result (void *cls,
1073 GNUNET_NO); 1123 GNUNET_NO);
1074 return; 1124 return;
1075 } 1125 }
1126 total_reg_proc_dns++;
1076 req->rs = NULL; 1127 req->rs = NULL;
1077 insert_sorted (req); 1128 insert_sorted (req);
1078 return; 1129 return;
@@ -1105,8 +1156,11 @@ process_result (void *cls,
1105 1, 1156 1,
1106 GNUNET_NO); 1157 GNUNET_NO);
1107 if (NULL == t) 1158 if (NULL == t)
1159 {
1160 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1108 t = GNUNET_SCHEDULER_add_now (&process_queue, 1161 t = GNUNET_SCHEDULER_add_now (&process_queue,
1109 NULL); 1162 NULL);
1163 }
1110 if (req->issue_num > MAX_RETRIES) 1164 if (req->issue_num > MAX_RETRIES)
1111 { 1165 {
1112 failures++; 1166 failures++;
@@ -1144,14 +1198,10 @@ process_result (void *cls,
1144 total_dns_latency_cnt++; 1198 total_dns_latency_cnt++;
1145 if (0 == (total_dns_latency_cnt % 1000)) 1199 if (0 == (total_dns_latency_cnt % 1000))
1146 { 1200 {
1147 GNUNET_STATISTICS_update (stats, 1201 GNUNET_STATISTICS_set (stats,
1148 "# average DNS latency (μs)", 1202 "# average DNS lookup latency (μs)",
1149 total_dns_latency.rel_value_us / total_dns_latency_cnt, 1203 total_dns_latency.rel_value_us / total_dns_latency_cnt,
1150 GNUNET_NO); 1204 GNUNET_NO);
1151 GNUNET_STATISTICS_update (stats,
1152 "# DNS replies",
1153 total_dns_latency_cnt,
1154 GNUNET_NO);
1155 } 1205 }
1156 } 1206 }
1157 rd_count = 0; 1207 rd_count = 0;
@@ -1171,10 +1221,18 @@ process_result (void *cls,
1171 /* Instead of going for SOA, simplified for now to look each 1221 /* Instead of going for SOA, simplified for now to look each
1172 day in case we got an empty response */ 1222 day in case we got an empty response */
1173 if (0 == rd_count) 1223 if (0 == rd_count)
1224 {
1174 req->expires 1225 req->expires
1175 = GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_DAYS); 1226 = GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_DAYS);
1227 GNUNET_STATISTICS_update (stats,
1228 "# empty DNS replies (usually NXDOMAIN)",
1229 1,
1230 GNUNET_NO);
1231 }
1176 else 1232 else
1233 {
1177 record_sets++; 1234 record_sets++;
1235 }
1178 /* convert records to namestore import format */ 1236 /* convert records to namestore import format */
1179 { 1237 {
1180 struct GNUNET_GNSRECORD_Data rd[GNUNET_NZL(rd_count)]; 1238 struct GNUNET_GNSRECORD_Data rd[GNUNET_NZL(rd_count)];
@@ -1210,8 +1268,12 @@ process_queue (void *cls)
1210 unsigned int series; 1268 unsigned int series;
1211 void *raw; 1269 void *raw;
1212 size_t raw_size; 1270 size_t raw_size;
1271 struct GNUNET_TIME_Relative delay;
1213 1272
1214 (void) cls; 1273 (void) cls;
1274 delay = GNUNET_TIME_absolute_get_duration (sleep_time_reg_proc);
1275 idle_time = GNUNET_TIME_relative_add (idle_time,
1276 delay);
1215 series = 0; 1277 series = 0;
1216 t = NULL; 1278 t = NULL;
1217 while (pending + pending_rs < THRESH) 1279 while (pending + pending_rs < THRESH)
@@ -1284,6 +1346,7 @@ process_queue (void *cls)
1284 req->hostname); 1346 req->hostname);
1285 if (NULL != t) 1347 if (NULL != t)
1286 GNUNET_SCHEDULER_cancel (t); 1348 GNUNET_SCHEDULER_cancel (t);
1349 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1287 t = GNUNET_SCHEDULER_add_at (req->expires, 1350 t = GNUNET_SCHEDULER_add_at (req->expires,
1288 &process_queue, 1351 &process_queue,
1289 NULL); 1352 NULL);
@@ -1293,6 +1356,7 @@ process_queue (void *cls)
1293 "Throttling\n"); 1356 "Throttling\n");
1294 if (NULL != t) 1357 if (NULL != t)
1295 GNUNET_SCHEDULER_cancel (t); 1358 GNUNET_SCHEDULER_cancel (t);
1359 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1296 t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY, 1360 t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY,
1297 &process_queue, 1361 &process_queue,
1298 NULL); 1362 NULL);
@@ -1450,9 +1514,9 @@ ns_lookup_result_cb (void *cls,
1450 { 1514 {
1451 ns_iterator_trigger_next = NS_BATCH_SIZE; 1515 ns_iterator_trigger_next = NS_BATCH_SIZE;
1452 GNUNET_STATISTICS_update (stats, 1516 GNUNET_STATISTICS_update (stats,
1453 "# NAMESTORE records requested", 1517 "# NAMESTORE records requested from cache",
1454 ns_iterator_trigger_next, 1518 ns_iterator_trigger_next,
1455 GNUNET_NO); 1519 GNUNET_NO);
1456 GNUNET_NAMESTORE_zone_iterator_next (zone_it, 1520 GNUNET_NAMESTORE_zone_iterator_next (zone_it,
1457 ns_iterator_trigger_next); 1521 ns_iterator_trigger_next);
1458 } 1522 }
@@ -1652,7 +1716,7 @@ iterate_zones (void *cls)
1652 last->domain); 1716 last->domain);
1653 /* subtract left-overs from previous iteration */ 1717 /* subtract left-overs from previous iteration */
1654 GNUNET_STATISTICS_update (stats, 1718 GNUNET_STATISTICS_update (stats,
1655 "# NAMESTORE records requested", 1719 "# NAMESTORE records requested from cache",
1656 (long long) (- ns_iterator_trigger_next), 1720 (long long) (- ns_iterator_trigger_next),
1657 GNUNET_NO); 1721 GNUNET_NO);
1658 ns_iterator_trigger_next = 0; 1722 ns_iterator_trigger_next = 0;
@@ -1665,7 +1729,7 @@ iterate_zones (void *cls)
1665 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 1729 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1666 "Finished all NAMESTORE iterations!\n"); 1730 "Finished all NAMESTORE iterations!\n");
1667 GNUNET_STATISTICS_set (stats, 1731 GNUNET_STATISTICS_set (stats,
1668 "# NAMESTORE lookups without reply", 1732 "# Domain names without cached reply",
1669 GNUNET_CONTAINER_multihashmap_size (ns_pending), 1733 GNUNET_CONTAINER_multihashmap_size (ns_pending),
1670 GNUNET_NO); 1734 GNUNET_NO);
1671 GNUNET_CONTAINER_multihashmap_iterate (ns_pending, 1735 GNUNET_CONTAINER_multihashmap_iterate (ns_pending,
@@ -1673,6 +1737,9 @@ iterate_zones (void *cls)
1673 NULL); 1737 NULL);
1674 GNUNET_CONTAINER_multihashmap_destroy (ns_pending); 1738 GNUNET_CONTAINER_multihashmap_destroy (ns_pending);
1675 ns_pending = NULL; 1739 ns_pending = NULL;
1740 start_time_reg_proc = GNUNET_TIME_absolute_get ();
1741 total_reg_proc_dns = 0;
1742 total_reg_proc_dns_ns = 0;
1676 return; 1743 return;
1677 } 1744 }
1678 if (NULL == last) 1745 if (NULL == last)
@@ -1684,7 +1751,7 @@ iterate_zones (void *cls)
1684 last->domain); 1751 last->domain);
1685 /* subtract left-overs from previous iteration */ 1752 /* subtract left-overs from previous iteration */
1686 GNUNET_STATISTICS_update (stats, 1753 GNUNET_STATISTICS_update (stats,
1687 "# NAMESTORE records requested", 1754 "# NAMESTORE records requested from cache",
1688 1, 1755 1,
1689 GNUNET_NO); 1756 GNUNET_NO);
1690 ns_iterator_trigger_next = 1; 1757 ns_iterator_trigger_next = 1;
@@ -1713,7 +1780,7 @@ static void
1713process_stdin (void *cls) 1780process_stdin (void *cls)
1714{ 1781{
1715 static struct GNUNET_TIME_Absolute last; 1782 static struct GNUNET_TIME_Absolute last;
1716 static unsigned int pdot; 1783 static uint64_t idot;
1717 char hn[256]; 1784 char hn[256];
1718 1785
1719 (void) cls; 1786 (void) cls;
@@ -1730,30 +1797,32 @@ process_stdin (void *cls)
1730 { 1797 {
1731 if (strlen(hn) > 0) 1798 if (strlen(hn) > 0)
1732 hn[strlen(hn)-1] = '\0'; /* eat newline */ 1799 hn[strlen(hn)-1] = '\0'; /* eat newline */
1733 if (0 == pdot) 1800 if (0 == idot)
1734 last = GNUNET_TIME_absolute_get (); 1801 last = GNUNET_TIME_absolute_get ();
1735 pdot++; 1802 idot++;
1736 if (0 == pdot % 1000) 1803 if (0 == idot % 10000)
1737 { 1804 {
1738 struct GNUNET_TIME_Relative delta; 1805 struct GNUNET_TIME_Relative delta;
1739 1806
1740 delta = GNUNET_TIME_absolute_get_duration (last); 1807 delta = GNUNET_TIME_absolute_get_duration (last);
1741 last = GNUNET_TIME_absolute_get (); 1808 last = GNUNET_TIME_absolute_get ();
1742 fprintf (stderr, 1809 fprintf (stderr,
1743 "Imported 1000 records in %s\n", 1810 "Imported 10000 records in %s\n",
1744 GNUNET_STRINGS_relative_time_to_string (delta, 1811 GNUNET_STRINGS_relative_time_to_string (delta,
1745 GNUNET_YES)); 1812 GNUNET_YES));
1746 GNUNET_STATISTICS_set (stats, 1813 GNUNET_STATISTICS_set (stats,
1747 "# domain names provided", 1814 "# domain names provided",
1748 pdot, 1815 idot,
1749 GNUNET_NO); 1816 GNUNET_NO);
1750 } 1817 }
1751 queue (hn); 1818 queue (hn);
1752 } 1819 }
1753 fprintf (stderr, "\n"); 1820 fprintf (stderr,
1821 "Done reading %llu domain names\n",
1822 (unsigned long long) idot);
1754 GNUNET_STATISTICS_set (stats, 1823 GNUNET_STATISTICS_set (stats,
1755 "# domain names provided", 1824 "# domain names provided",
1756 pdot, 1825 idot,
1757 GNUNET_NO); 1826 GNUNET_NO);
1758 iterate_zones (NULL); 1827 iterate_zones (NULL);
1759} 1828}
@@ -1851,8 +1920,14 @@ run (void *cls,
1851 stats = GNUNET_STATISTICS_create ("zoneimport", 1920 stats = GNUNET_STATISTICS_create ("zoneimport",
1852 cfg); 1921 cfg);
1853 req_heap = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN); 1922 req_heap = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
1854 ns_pending = GNUNET_CONTAINER_multihashmap_create (1024, 1923 ns_pending = GNUNET_CONTAINER_multihashmap_create (map_size,
1855 GNUNET_NO); 1924 GNUNET_NO);
1925 if (NULL == ns_pending)
1926 {
1927 fprintf (stderr,
1928 "Failed to allocate memory for main hash map\n");
1929 return;
1930 }
1856 ctx = GNUNET_DNSSTUB_start (256); 1931 ctx = GNUNET_DNSSTUB_start (256);
1857 if (NULL == ctx) 1932 if (NULL == ctx)
1858 { 1933 {
@@ -1906,6 +1981,11 @@ main (int argc,
1906 char *const*argv) 1981 char *const*argv)
1907{ 1982{
1908 struct GNUNET_GETOPT_CommandLineOption options[] = { 1983 struct GNUNET_GETOPT_CommandLineOption options[] = {
1984 GNUNET_GETOPT_option_uint ('s',
1985 "size",
1986 "MAPSIZE",
1987 gettext_noop ("size to use for the main hash map"),
1988 &map_size),
1909 GNUNET_GETOPT_OPTION_END 1989 GNUNET_GETOPT_OPTION_END
1910 }; 1990 };
1911 1991
diff --git a/src/util/container_multihashmap.c b/src/util/container_multihashmap.c
index ffeb4a71f..6001fc1df 100644
--- a/src/util/container_multihashmap.c
+++ b/src/util/container_multihashmap.c
@@ -175,14 +175,36 @@ struct GNUNET_CONTAINER_MultiHashMap *
175GNUNET_CONTAINER_multihashmap_create (unsigned int len, 175GNUNET_CONTAINER_multihashmap_create (unsigned int len,
176 int do_not_copy_keys) 176 int do_not_copy_keys)
177{ 177{
178 struct GNUNET_CONTAINER_MultiHashMap *map; 178 struct GNUNET_CONTAINER_MultiHashMap *hm;
179 179
180 GNUNET_assert (len > 0); 180 GNUNET_assert (len > 0);
181 map = GNUNET_new (struct GNUNET_CONTAINER_MultiHashMap); 181 hm = GNUNET_new (struct GNUNET_CONTAINER_MultiHashMap);
182 map->map = GNUNET_malloc (len * sizeof (union MapEntry)); 182 if (len * sizeof (union MapEntry) > GNUNET_MAX_MALLOC_CHECKED)
183 map->map_length = len; 183 {
184 map->use_small_entries = do_not_copy_keys; 184 size_t s;
185 return map; 185 /* application *explicitly* requested very large map, hopefully
186 it checks the return value... */
187 s = len * sizeof (union MapEntry);
188 if ( (s / sizeof (union MapEntry)) != len)
189 return NULL; /* integer overflow on multiplication */
190 if (NULL == (hm->map = GNUNET_malloc_large (s)))
191 {
192 /* out of memory */
193 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
194 "Out of memory allocating large hash map (%u entries)\n",
195 len);
196 GNUNET_free (hm);
197 return NULL;
198 }
199 }
200 else
201 {
202 hm->map = GNUNET_new_array (len,
203 union MapEntry);
204 }
205 hm->map_length = len;
206 hm->use_small_entries = do_not_copy_keys;
207 return hm;
186} 208}
187 209
188 210
@@ -196,11 +218,10 @@ void
196GNUNET_CONTAINER_multihashmap_destroy (struct GNUNET_CONTAINER_MultiHashMap 218GNUNET_CONTAINER_multihashmap_destroy (struct GNUNET_CONTAINER_MultiHashMap
197 *map) 219 *map)
198{ 220{
199 unsigned int i; 221 for (unsigned int i = 0; i < map->map_length; i++)
200 union MapEntry me;
201
202 for (i = 0; i < map->map_length; i++)
203 { 222 {
223 union MapEntry me;
224
204 me = map->map[i]; 225 me = map->map[i];
205 if (map->use_small_entries) 226 if (map->use_small_entries)
206 { 227 {
@@ -257,8 +278,7 @@ idx_of (const struct GNUNET_CONTAINER_MultiHashMap *map,
257 * @return the number of key value pairs 278 * @return the number of key value pairs
258 */ 279 */
259unsigned int 280unsigned int
260GNUNET_CONTAINER_multihashmap_size (const struct GNUNET_CONTAINER_MultiHashMap 281GNUNET_CONTAINER_multihashmap_size (const struct GNUNET_CONTAINER_MultiHashMap *map)
261 *map)
262{ 282{
263 return map->size; 283 return map->size;
264} 284}
@@ -656,17 +676,22 @@ grow (struct GNUNET_CONTAINER_MultiHashMap *map)
656 unsigned int old_len; 676 unsigned int old_len;
657 unsigned int new_len; 677 unsigned int new_len;
658 unsigned int idx; 678 unsigned int idx;
659 unsigned int i;
660 679
661 map->modification_counter++; 680 map->modification_counter++;
662 681
663 old_map = map->map; 682 old_map = map->map;
664 old_len = map->map_length; 683 old_len = map->map_length;
665 new_len = old_len * 2; 684 new_len = old_len * 2;
666 new_map = GNUNET_malloc (sizeof (union MapEntry) * new_len); 685 /* if we would exceed heap size limit for the _first_ time,
686 try staying just below the limit */
687 if ( (new_len * sizeof (union MapEntry) > GNUNET_MAX_MALLOC_CHECKED) &&
688 ((old_len+1) * sizeof (union MapEntry) < GNUNET_MAX_MALLOC_CHECKED) )
689 new_len = GNUNET_MAX_MALLOC_CHECKED / sizeof (union MapEntry);
690 new_map = GNUNET_new_array (new_len,
691 union MapEntry);
667 map->map_length = new_len; 692 map->map_length = new_len;
668 map->map = new_map; 693 map->map = new_map;
669 for (i = 0; i < old_len; i++) 694 for (unsigned int i = 0; i < old_len; i++)
670 { 695 {
671 if (map->use_small_entries) 696 if (map->use_small_entries)
672 { 697 {
diff --git a/src/zonemaster/gnunet-service-zonemaster.c b/src/zonemaster/gnunet-service-zonemaster.c
index 55e1a0eee..322a1093b 100644
--- a/src/zonemaster/gnunet-service-zonemaster.c
+++ b/src/zonemaster/gnunet-service-zonemaster.c
@@ -63,7 +63,7 @@
63 * The initial interval in milliseconds btween puts in 63 * The initial interval in milliseconds btween puts in
64 * a zone iteration 64 * a zone iteration
65 */ 65 */
66#define INITIAL_PUT_INTERVAL GNUNET_TIME_UNIT_MILLISECONDS 66#define INITIAL_ZONE_ITERATION_INTERVAL GNUNET_TIME_UNIT_MILLISECONDS
67 67
68/** 68/**
69 * The upper bound for the zone iteration interval 69 * The upper bound for the zone iteration interval
@@ -204,7 +204,7 @@ static unsigned long long put_cnt;
204 * and the total number of record sets we have (so far) 204 * and the total number of record sets we have (so far)
205 * observed in the zone. 205 * observed in the zone.
206 */ 206 */
207static struct GNUNET_TIME_Relative next_put_interval; 207static struct GNUNET_TIME_Relative target_iteration_velocity_per_record;
208 208
209/** 209/**
210 * Minimum relative expiration time of records seem during the current 210 * Minimum relative expiration time of records seem during the current
@@ -383,7 +383,7 @@ dht_put_monitor_continuation (void *cls)
383 383
384 384
385/** 385/**
386 * Calculate #next_put_interval. 386 * Calculate #target_iteration_velocity_per_record.
387 */ 387 */
388static void 388static void
389calculate_put_interval () 389calculate_put_interval ()
@@ -395,7 +395,7 @@ calculate_put_interval ()
395 * we can safely set the interval to the value for a single 395 * we can safely set the interval to the value for a single
396 * record 396 * record
397 */ 397 */
398 next_put_interval = zone_publish_time_window; 398 target_iteration_velocity_per_record = zone_publish_time_window;
399 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG | GNUNET_ERROR_TYPE_BULK, 399 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG | GNUNET_ERROR_TYPE_BULK,
400 "No records in namestore database.\n"); 400 "No records in namestore database.\n");
401 } 401 }
@@ -408,24 +408,24 @@ calculate_put_interval ()
408 = GNUNET_TIME_relative_min (GNUNET_TIME_relative_divide (last_min_relative_record_time, 408 = GNUNET_TIME_relative_min (GNUNET_TIME_relative_divide (last_min_relative_record_time,
409 PUBLISH_OPS_PER_EXPIRATION), 409 PUBLISH_OPS_PER_EXPIRATION),
410 zone_publish_time_window_default); 410 zone_publish_time_window_default);
411 next_put_interval 411 target_iteration_velocity_per_record
412 = GNUNET_TIME_relative_divide (zone_publish_time_window, 412 = GNUNET_TIME_relative_divide (zone_publish_time_window,
413 last_num_public_records); 413 last_num_public_records);
414 } 414 }
415 next_put_interval 415 target_iteration_velocity_per_record
416 = GNUNET_TIME_relative_min (next_put_interval, 416 = GNUNET_TIME_relative_min (target_iteration_velocity_per_record,
417 MAXIMUM_ZONE_ITERATION_INTERVAL); 417 MAXIMUM_ZONE_ITERATION_INTERVAL);
418 GNUNET_STATISTICS_set (statistics, 418 GNUNET_STATISTICS_set (statistics,
419 "Minimum relative record expiration (in ms)", 419 "Minimum relative record expiration (in μs)",
420 last_min_relative_record_time.rel_value_us / 1000LL, 420 last_min_relative_record_time.rel_value_us,
421 GNUNET_NO); 421 GNUNET_NO);
422 GNUNET_STATISTICS_set (statistics, 422 GNUNET_STATISTICS_set (statistics,
423 "Zone publication time window (in ms)", 423 "Zone publication time window (in μs)",
424 zone_publish_time_window.rel_value_us / 1000LL, 424 zone_publish_time_window.rel_value_us,
425 GNUNET_NO); 425 GNUNET_NO);
426 GNUNET_STATISTICS_set (statistics, 426 GNUNET_STATISTICS_set (statistics,
427 "Target zone iteration velocity (μs)", 427 "Target zone iteration velocity (μs)",
428 next_put_interval.rel_value_us, 428 target_iteration_velocity_per_record.rel_value_us,
429 GNUNET_NO); 429 GNUNET_NO);
430} 430}
431 431
@@ -461,7 +461,7 @@ update_velocity (unsigned int cnt)
461 } 461 }
462 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 462 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
463 "Desired global zone iteration interval is %s/record!\n", 463 "Desired global zone iteration interval is %s/record!\n",
464 GNUNET_STRINGS_relative_time_to_string (next_put_interval, 464 GNUNET_STRINGS_relative_time_to_string (target_iteration_velocity_per_record,
465 GNUNET_YES)); 465 GNUNET_YES));
466 466
467 /* Tell statistics actual vs. desired speed */ 467 /* Tell statistics actual vs. desired speed */
@@ -471,12 +471,12 @@ update_velocity (unsigned int cnt)
471 GNUNET_NO); 471 GNUNET_NO);
472 /* update "sub_delta" based on difference, taking 472 /* update "sub_delta" based on difference, taking
473 previous sub_delta into account! */ 473 previous sub_delta into account! */
474 if (next_put_interval.rel_value_us > delta.rel_value_us) 474 if (target_iteration_velocity_per_record.rel_value_us > delta.rel_value_us)
475 { 475 {
476 /* We were too fast, reduce sub_delta! */ 476 /* We were too fast, reduce sub_delta! */
477 struct GNUNET_TIME_Relative corr; 477 struct GNUNET_TIME_Relative corr;
478 478
479 corr = GNUNET_TIME_relative_subtract (next_put_interval, 479 corr = GNUNET_TIME_relative_subtract (target_iteration_velocity_per_record,
480 delta); 480 delta);
481 if (sub_delta.rel_value_us > delta.rel_value_us) 481 if (sub_delta.rel_value_us > delta.rel_value_us)
482 { 482 {
@@ -492,28 +492,28 @@ update_velocity (unsigned int cnt)
492 sub_delta = GNUNET_TIME_UNIT_ZERO; 492 sub_delta = GNUNET_TIME_UNIT_ZERO;
493 } 493 }
494 } 494 }
495 else if (next_put_interval.rel_value_us < delta.rel_value_us) 495 else if (target_iteration_velocity_per_record.rel_value_us < delta.rel_value_us)
496 { 496 {
497 /* We were too slow, increase sub_delta! */ 497 /* We were too slow, increase sub_delta! */
498 struct GNUNET_TIME_Relative corr; 498 struct GNUNET_TIME_Relative corr;
499 499
500 corr = GNUNET_TIME_relative_subtract (delta, 500 corr = GNUNET_TIME_relative_subtract (delta,
501 next_put_interval); 501 target_iteration_velocity_per_record);
502 sub_delta = GNUNET_TIME_relative_add (sub_delta, 502 sub_delta = GNUNET_TIME_relative_add (sub_delta,
503 corr); 503 corr);
504 if (sub_delta.rel_value_us > next_put_interval.rel_value_us) 504 if (sub_delta.rel_value_us > target_iteration_velocity_per_record.rel_value_us)
505 { 505 {
506 /* CPU overload detected, we cannot go at desired speed, 506 /* CPU overload detected, we cannot go at desired speed,
507 as this would mean using a negative delay. */ 507 as this would mean using a negative delay. */
508 /* compute how much faster we would want to be for 508 /* compute how much faster we would want to be for
509 the desired velocity */ 509 the desired velocity */
510 if (0 == next_put_interval.rel_value_us) 510 if (0 == target_iteration_velocity_per_record.rel_value_us)
511 pct = UINT64_MAX; /* desired speed is infinity ... */ 511 pct = UINT64_MAX; /* desired speed is infinity ... */
512 else 512 else
513 pct = (sub_delta.rel_value_us - 513 pct = (sub_delta.rel_value_us -
514 next_put_interval.rel_value_us) * 100LLU 514 target_iteration_velocity_per_record.rel_value_us) * 100LLU
515 / next_put_interval.rel_value_us; 515 / target_iteration_velocity_per_record.rel_value_us;
516 sub_delta = next_put_interval; 516 sub_delta = target_iteration_velocity_per_record;
517 } 517 }
518 } 518 }
519 GNUNET_STATISTICS_set (statistics, 519 GNUNET_STATISTICS_set (statistics,
@@ -548,7 +548,7 @@ check_zone_namestore_next ()
548 return; /* current NAMESTORE iteration not yet done */ 548 return; /* current NAMESTORE iteration not yet done */
549 update_velocity (put_cnt); 549 update_velocity (put_cnt);
550 put_cnt = 0; 550 put_cnt = 0;
551 delay = GNUNET_TIME_relative_subtract (next_put_interval, 551 delay = GNUNET_TIME_relative_subtract (target_iteration_velocity_per_record,
552 sub_delta); 552 sub_delta);
553 /* We delay *once* per #NS_BLOCK_SIZE, so we need to multiply the 553 /* We delay *once* per #NS_BLOCK_SIZE, so we need to multiply the
554 per-record delay calculated so far with the #NS_BLOCK_SIZE */ 554 per-record delay calculated so far with the #NS_BLOCK_SIZE */
@@ -755,11 +755,11 @@ zone_iteration_finished (void *cls)
755 PUBLISH_OPS_PER_EXPIRATION); 755 PUBLISH_OPS_PER_EXPIRATION);
756 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 756 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
757 "Zone iteration finished. Adjusted zone iteration interval to %s\n", 757 "Zone iteration finished. Adjusted zone iteration interval to %s\n",
758 GNUNET_STRINGS_relative_time_to_string (next_put_interval, 758 GNUNET_STRINGS_relative_time_to_string (target_iteration_velocity_per_record,
759 GNUNET_YES)); 759 GNUNET_YES));
760 GNUNET_STATISTICS_set (statistics, 760 GNUNET_STATISTICS_set (statistics,
761 "Current zone iteration interval (in ms)", 761 "Target zone iteration velocity s)",
762 next_put_interval.rel_value_us / 1000LL, 762 target_iteration_velocity_per_record.rel_value_us,
763 GNUNET_NO); 763 GNUNET_NO);
764 GNUNET_STATISTICS_set (statistics, 764 GNUNET_STATISTICS_set (statistics,
765 "Number of public records in DHT", 765 "Number of public records in DHT",
@@ -767,12 +767,16 @@ zone_iteration_finished (void *cls)
767 GNUNET_NO); 767 GNUNET_NO);
768 GNUNET_assert (NULL == zone_publish_task); 768 GNUNET_assert (NULL == zone_publish_task);
769 if (0 == last_num_public_records) 769 if (0 == last_num_public_records)
770 zone_publish_task = GNUNET_SCHEDULER_add_delayed (next_put_interval, 770 {
771 zone_publish_task = GNUNET_SCHEDULER_add_delayed (target_iteration_velocity_per_record,
771 &publish_zone_dht_start, 772 &publish_zone_dht_start,
772 NULL); 773 NULL);
774 }
773 else 775 else
776 {
774 zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start, 777 zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start,
775 NULL); 778 NULL);
779 }
776} 780}
777 781
778 782
@@ -1028,7 +1032,7 @@ run (void *cls,
1028 min_relative_record_time 1032 min_relative_record_time
1029 = GNUNET_TIME_relative_multiply (GNUNET_DHT_DEFAULT_REPUBLISH_FREQUENCY, 1033 = GNUNET_TIME_relative_multiply (GNUNET_DHT_DEFAULT_REPUBLISH_FREQUENCY,
1030 PUBLISH_OPS_PER_EXPIRATION); 1034 PUBLISH_OPS_PER_EXPIRATION);
1031 next_put_interval = INITIAL_PUT_INTERVAL; 1035 target_iteration_velocity_per_record = INITIAL_ZONE_ITERATION_INTERVAL;
1032 namestore_handle = GNUNET_NAMESTORE_connect (c); 1036 namestore_handle = GNUNET_NAMESTORE_connect (c);
1033 if (NULL == namestore_handle) 1037 if (NULL == namestore_handle)
1034 { 1038 {
@@ -1077,12 +1081,12 @@ run (void *cls,
1077 } 1081 }
1078 1082
1079 /* Schedule periodic put for our records. */ 1083 /* Schedule periodic put for our records. */
1080 first_zone_iteration = GNUNET_YES;\ 1084 first_zone_iteration = GNUNET_YES;
1081 statistics = GNUNET_STATISTICS_create ("zonemaster", 1085 statistics = GNUNET_STATISTICS_create ("zonemaster",
1082 c); 1086 c);
1083 GNUNET_STATISTICS_set (statistics, 1087 GNUNET_STATISTICS_set (statistics,
1084 "Target zone iteration velocity (μs)", 1088 "Target zone iteration velocity (μs)",
1085 next_put_interval.rel_value_us, 1089 target_iteration_velocity_per_record.rel_value_us,
1086 GNUNET_NO); 1090 GNUNET_NO);
1087 zmon = GNUNET_NAMESTORE_zone_monitor_start (c, 1091 zmon = GNUNET_NAMESTORE_zone_monitor_start (c,
1088 NULL, 1092 NULL,