aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorChristian Grothoff <christian@grothoff.org>2018-05-02 01:01:57 +0200
committerChristian Grothoff <christian@grothoff.org>2018-05-02 01:01:57 +0200
commit857f8c009c4fde3d3ec9d6d0b4af48e93684fd84 (patch)
tree079b460ad7555b148720c74b3601429c662c4113 /src
parentffe43cb1e86f49ddff73554df200853248af012d (diff)
downloadgnunet-857f8c009c4fde3d3ec9d6d0b4af48e93684fd84.tar.gz
gnunet-857f8c009c4fde3d3ec9d6d0b4af48e93684fd84.zip
misc improvements to statistics and large-scale behavior of zoneimport/zonemaster
Diffstat (limited to 'src')
-rw-r--r--src/namestore/gnunet-zoneimport.c202
-rw-r--r--src/util/container_multihashmap.c55
-rw-r--r--src/zonemaster/gnunet-service-zonemaster.c66
3 files changed, 216 insertions, 107 deletions
diff --git a/src/namestore/gnunet-zoneimport.c b/src/namestore/gnunet-zoneimport.c
index e24cb26dd..0fd0a4ab8 100644
--- a/src/namestore/gnunet-zoneimport.c
+++ b/src/namestore/gnunet-zoneimport.c
@@ -204,6 +204,14 @@ struct Request
204 204
205 205
206/** 206/**
207 * Command-line argument specifying desired size of the hash map with
208 * all of our pending names. Usually, we use an automatically growing
209 * map, but this is only OK up to about a million entries. Above that
210 * number, the user must explicitly specify the size at startup.
211 */
212static unsigned int map_size = 1024;
213
214/**
207 * Handle to the identity service. 215 * Handle to the identity service.
208 */ 216 */
209static struct GNUNET_IDENTITY_Handle *id; 217static struct GNUNET_IDENTITY_Handle *id;
@@ -323,14 +331,29 @@ static uint64_t total_dns_latency_cnt;
323static struct GNUNET_TIME_Relative total_dns_latency; 331static struct GNUNET_TIME_Relative total_dns_latency;
324 332
325/** 333/**
326 * Number of NAMESTORE requests counted in latency total. 334 * Number of records processed (DNS lookup, no NAMESTORE) in total.
335 */
336static uint64_t total_reg_proc_dns;
337
338/**
339 * Number of records processed (DNS lookup, with NAMESTORE) in total.
340 */
341static uint64_t total_reg_proc_dns_ns;
342
343/**
344 * Start time of the regular processing.
345 */
346static struct GNUNET_TIME_Absolute start_time_reg_proc;
347
348/**
349 * Last time we worked before going idle.
327 */ 350 */
328static uint64_t total_ns_latency_cnt; 351static struct GNUNET_TIME_Absolute sleep_time_reg_proc;
329 352
330/** 353/**
331 * Sum of NAMESTORE latencies observed. 354 * Time we slept just waiting for work.
332 */ 355 */
333static struct GNUNET_TIME_Relative total_ns_latency; 356static struct GNUNET_TIME_Relative idle_time;
334 357
335 358
336/** 359/**
@@ -530,6 +553,7 @@ insert_sorted (struct Request *req)
530 { 553 {
531 if (NULL != t) 554 if (NULL != t)
532 GNUNET_SCHEDULER_cancel (t); 555 GNUNET_SCHEDULER_cancel (t);
556 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
533 t = GNUNET_SCHEDULER_add_at (req->expires, 557 t = GNUNET_SCHEDULER_add_at (req->expires,
534 &process_queue, 558 &process_queue,
535 NULL); 559 NULL);
@@ -966,37 +990,9 @@ store_completed_cb (void *cls,
966 const char *emsg) 990 const char *emsg)
967{ 991{
968 static struct GNUNET_TIME_Absolute last; 992 static struct GNUNET_TIME_Absolute last;
969 static unsigned int pdot;
970 struct Request *req = cls; 993 struct Request *req = cls;
971 994
972 req->qe = NULL; 995 req->qe = NULL;
973 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
974 "Stored record set in database (%d)\n",
975 success);
976 pending_rs--;
977 {
978 struct GNUNET_TIME_Relative ns_latency;
979
980 ns_latency = GNUNET_TIME_absolute_get_duration (req->op_start_time);
981 total_ns_latency = GNUNET_TIME_relative_add (total_ns_latency,
982 ns_latency);
983 total_ns_latency_cnt++;
984 if (0 == (total_ns_latency_cnt % 1000))
985 {
986 GNUNET_STATISTICS_update (stats,
987 "# average NAMESTORE PUT latency (μs)",
988 total_ns_latency.rel_value_us / total_ns_latency_cnt,
989 GNUNET_NO);
990 GNUNET_STATISTICS_update (stats,
991 "# NAMESTORE PUTs",
992 total_ns_latency_cnt,
993 GNUNET_NO);
994 }
995 }
996
997 if (NULL == t)
998 t = GNUNET_SCHEDULER_add_now (&process_queue,
999 NULL);
1000 if (GNUNET_SYSERR == success) 996 if (GNUNET_SYSERR == success)
1001 { 997 {
1002 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 998 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
@@ -1007,12 +1003,26 @@ store_completed_cb (void *cls,
1007 else 1003 else
1008 { 1004 {
1009 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 1005 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1010 "Stored records under `%s'\n", 1006 "Stored records under `%s' (%d)\n",
1011 req->hostname); 1007 req->hostname,
1012 if (0 == pdot) 1008 success);
1009 }
1010 total_reg_proc_dns_ns++; /* finished regular processing */
1011 pending_rs--;
1012 free_records (req);
1013 /* compute NAMESTORE statistics */
1014 {
1015 static uint64_t total_ns_latency_cnt;
1016 static struct GNUNET_TIME_Relative total_ns_latency;
1017 struct GNUNET_TIME_Relative ns_latency;
1018
1019 ns_latency = GNUNET_TIME_absolute_get_duration (req->op_start_time);
1020 total_ns_latency = GNUNET_TIME_relative_add (total_ns_latency,
1021 ns_latency);
1022 if (0 == total_ns_latency_cnt)
1013 last = GNUNET_TIME_absolute_get (); 1023 last = GNUNET_TIME_absolute_get ();
1014 pdot++; 1024 total_ns_latency_cnt++;
1015 if (0 == pdot % 1000) 1025 if (0 == (total_ns_latency_cnt % 1000))
1016 { 1026 {
1017 struct GNUNET_TIME_Relative delta; 1027 struct GNUNET_TIME_Relative delta;
1018 1028
@@ -1022,9 +1032,46 @@ store_completed_cb (void *cls,
1022 "Processed 1000 records in %s\n", 1032 "Processed 1000 records in %s\n",
1023 GNUNET_STRINGS_relative_time_to_string (delta, 1033 GNUNET_STRINGS_relative_time_to_string (delta,
1024 GNUNET_YES)); 1034 GNUNET_YES));
1035 GNUNET_STATISTICS_set (stats,
1036 "# average NAMESTORE PUT latency (μs)",
1037 total_ns_latency.rel_value_us / total_ns_latency_cnt,
1038 GNUNET_NO);
1025 } 1039 }
1026 } 1040 }
1027 free_records (req); 1041 /* compute and publish overall velocity */
1042 if (0 == (total_reg_proc_dns_ns % 100) )
1043 {
1044 struct GNUNET_TIME_Relative runtime;
1045
1046 runtime = GNUNET_TIME_absolute_get_duration (start_time_reg_proc);
1047 runtime = GNUNET_TIME_relative_subtract (runtime,
1048 idle_time);
1049 runtime = GNUNET_TIME_relative_divide (runtime,
1050 total_reg_proc_dns + total_reg_proc_dns_ns);
1051 GNUNET_STATISTICS_set (stats,
1052 "# Regular processing completed without NAMESTORE",
1053 total_reg_proc_dns,
1054 GNUNET_NO);
1055 GNUNET_STATISTICS_set (stats,
1056 "# Regular processing completed with NAMESTORE PUT",
1057 total_reg_proc_dns_ns,
1058 GNUNET_NO);
1059 GNUNET_STATISTICS_set (stats,
1060 "# average request processing latency (μs)",
1061 runtime.rel_value_us,
1062 GNUNET_NO);
1063 GNUNET_STATISTICS_set (stats,
1064 "# total time spent idle (μs)",
1065 idle_time.rel_value_us,
1066 GNUNET_NO);
1067 }
1068
1069 if (NULL == t)
1070 {
1071 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1072 t = GNUNET_SCHEDULER_add_now (&process_queue,
1073 NULL);
1074 }
1028} 1075}
1029 1076
1030 1077
@@ -1054,8 +1101,11 @@ process_result (void *cls,
1054 req); 1101 req);
1055 pending--; 1102 pending--;
1056 if (NULL == t) 1103 if (NULL == t)
1104 {
1105 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1057 t = GNUNET_SCHEDULER_add_now (&process_queue, 1106 t = GNUNET_SCHEDULER_add_now (&process_queue,
1058 NULL); 1107 NULL);
1108 }
1059 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 1109 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1060 "Stub gave up on DNS reply for `%s'\n", 1110 "Stub gave up on DNS reply for `%s'\n",
1061 req->hostname); 1111 req->hostname);
@@ -1073,6 +1123,7 @@ process_result (void *cls,
1073 GNUNET_NO); 1123 GNUNET_NO);
1074 return; 1124 return;
1075 } 1125 }
1126 total_reg_proc_dns++;
1076 req->rs = NULL; 1127 req->rs = NULL;
1077 insert_sorted (req); 1128 insert_sorted (req);
1078 return; 1129 return;
@@ -1105,8 +1156,11 @@ process_result (void *cls,
1105 1, 1156 1,
1106 GNUNET_NO); 1157 GNUNET_NO);
1107 if (NULL == t) 1158 if (NULL == t)
1159 {
1160 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1108 t = GNUNET_SCHEDULER_add_now (&process_queue, 1161 t = GNUNET_SCHEDULER_add_now (&process_queue,
1109 NULL); 1162 NULL);
1163 }
1110 if (req->issue_num > MAX_RETRIES) 1164 if (req->issue_num > MAX_RETRIES)
1111 { 1165 {
1112 failures++; 1166 failures++;
@@ -1144,14 +1198,10 @@ process_result (void *cls,
1144 total_dns_latency_cnt++; 1198 total_dns_latency_cnt++;
1145 if (0 == (total_dns_latency_cnt % 1000)) 1199 if (0 == (total_dns_latency_cnt % 1000))
1146 { 1200 {
1147 GNUNET_STATISTICS_update (stats, 1201 GNUNET_STATISTICS_set (stats,
1148 "# average DNS latency (μs)", 1202 "# average DNS lookup latency (μs)",
1149 total_dns_latency.rel_value_us / total_dns_latency_cnt, 1203 total_dns_latency.rel_value_us / total_dns_latency_cnt,
1150 GNUNET_NO); 1204 GNUNET_NO);
1151 GNUNET_STATISTICS_update (stats,
1152 "# DNS replies",
1153 total_dns_latency_cnt,
1154 GNUNET_NO);
1155 } 1205 }
1156 } 1206 }
1157 rd_count = 0; 1207 rd_count = 0;
@@ -1171,10 +1221,18 @@ process_result (void *cls,
1171 /* Instead of going for SOA, simplified for now to look each 1221 /* Instead of going for SOA, simplified for now to look each
1172 day in case we got an empty response */ 1222 day in case we got an empty response */
1173 if (0 == rd_count) 1223 if (0 == rd_count)
1224 {
1174 req->expires 1225 req->expires
1175 = GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_DAYS); 1226 = GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_DAYS);
1227 GNUNET_STATISTICS_update (stats,
1228 "# empty DNS replies (usually NXDOMAIN)",
1229 1,
1230 GNUNET_NO);
1231 }
1176 else 1232 else
1233 {
1177 record_sets++; 1234 record_sets++;
1235 }
1178 /* convert records to namestore import format */ 1236 /* convert records to namestore import format */
1179 { 1237 {
1180 struct GNUNET_GNSRECORD_Data rd[GNUNET_NZL(rd_count)]; 1238 struct GNUNET_GNSRECORD_Data rd[GNUNET_NZL(rd_count)];
@@ -1210,8 +1268,12 @@ process_queue (void *cls)
1210 unsigned int series; 1268 unsigned int series;
1211 void *raw; 1269 void *raw;
1212 size_t raw_size; 1270 size_t raw_size;
1271 struct GNUNET_TIME_Relative delay;
1213 1272
1214 (void) cls; 1273 (void) cls;
1274 delay = GNUNET_TIME_absolute_get_duration (sleep_time_reg_proc);
1275 idle_time = GNUNET_TIME_relative_add (idle_time,
1276 delay);
1215 series = 0; 1277 series = 0;
1216 t = NULL; 1278 t = NULL;
1217 while (pending + pending_rs < THRESH) 1279 while (pending + pending_rs < THRESH)
@@ -1284,6 +1346,7 @@ process_queue (void *cls)
1284 req->hostname); 1346 req->hostname);
1285 if (NULL != t) 1347 if (NULL != t)
1286 GNUNET_SCHEDULER_cancel (t); 1348 GNUNET_SCHEDULER_cancel (t);
1349 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1287 t = GNUNET_SCHEDULER_add_at (req->expires, 1350 t = GNUNET_SCHEDULER_add_at (req->expires,
1288 &process_queue, 1351 &process_queue,
1289 NULL); 1352 NULL);
@@ -1293,6 +1356,7 @@ process_queue (void *cls)
1293 "Throttling\n"); 1356 "Throttling\n");
1294 if (NULL != t) 1357 if (NULL != t)
1295 GNUNET_SCHEDULER_cancel (t); 1358 GNUNET_SCHEDULER_cancel (t);
1359 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1296 t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY, 1360 t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY,
1297 &process_queue, 1361 &process_queue,
1298 NULL); 1362 NULL);
@@ -1450,9 +1514,9 @@ ns_lookup_result_cb (void *cls,
1450 { 1514 {
1451 ns_iterator_trigger_next = NS_BATCH_SIZE; 1515 ns_iterator_trigger_next = NS_BATCH_SIZE;
1452 GNUNET_STATISTICS_update (stats, 1516 GNUNET_STATISTICS_update (stats,
1453 "# NAMESTORE records requested", 1517 "# NAMESTORE records requested from cache",
1454 ns_iterator_trigger_next, 1518 ns_iterator_trigger_next,
1455 GNUNET_NO); 1519 GNUNET_NO);
1456 GNUNET_NAMESTORE_zone_iterator_next (zone_it, 1520 GNUNET_NAMESTORE_zone_iterator_next (zone_it,
1457 ns_iterator_trigger_next); 1521 ns_iterator_trigger_next);
1458 } 1522 }
@@ -1652,7 +1716,7 @@ iterate_zones (void *cls)
1652 last->domain); 1716 last->domain);
1653 /* subtract left-overs from previous iteration */ 1717 /* subtract left-overs from previous iteration */
1654 GNUNET_STATISTICS_update (stats, 1718 GNUNET_STATISTICS_update (stats,
1655 "# NAMESTORE records requested", 1719 "# NAMESTORE records requested from cache",
1656 (long long) (- ns_iterator_trigger_next), 1720 (long long) (- ns_iterator_trigger_next),
1657 GNUNET_NO); 1721 GNUNET_NO);
1658 ns_iterator_trigger_next = 0; 1722 ns_iterator_trigger_next = 0;
@@ -1665,7 +1729,7 @@ iterate_zones (void *cls)
1665 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 1729 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1666 "Finished all NAMESTORE iterations!\n"); 1730 "Finished all NAMESTORE iterations!\n");
1667 GNUNET_STATISTICS_set (stats, 1731 GNUNET_STATISTICS_set (stats,
1668 "# NAMESTORE lookups without reply", 1732 "# Domain names without cached reply",
1669 GNUNET_CONTAINER_multihashmap_size (ns_pending), 1733 GNUNET_CONTAINER_multihashmap_size (ns_pending),
1670 GNUNET_NO); 1734 GNUNET_NO);
1671 GNUNET_CONTAINER_multihashmap_iterate (ns_pending, 1735 GNUNET_CONTAINER_multihashmap_iterate (ns_pending,
@@ -1673,6 +1737,9 @@ iterate_zones (void *cls)
1673 NULL); 1737 NULL);
1674 GNUNET_CONTAINER_multihashmap_destroy (ns_pending); 1738 GNUNET_CONTAINER_multihashmap_destroy (ns_pending);
1675 ns_pending = NULL; 1739 ns_pending = NULL;
1740 start_time_reg_proc = GNUNET_TIME_absolute_get ();
1741 total_reg_proc_dns = 0;
1742 total_reg_proc_dns_ns = 0;
1676 return; 1743 return;
1677 } 1744 }
1678 if (NULL == last) 1745 if (NULL == last)
@@ -1684,7 +1751,7 @@ iterate_zones (void *cls)
1684 last->domain); 1751 last->domain);
1685 /* subtract left-overs from previous iteration */ 1752 /* subtract left-overs from previous iteration */
1686 GNUNET_STATISTICS_update (stats, 1753 GNUNET_STATISTICS_update (stats,
1687 "# NAMESTORE records requested", 1754 "# NAMESTORE records requested from cache",
1688 1, 1755 1,
1689 GNUNET_NO); 1756 GNUNET_NO);
1690 ns_iterator_trigger_next = 1; 1757 ns_iterator_trigger_next = 1;
@@ -1713,7 +1780,7 @@ static void
1713process_stdin (void *cls) 1780process_stdin (void *cls)
1714{ 1781{
1715 static struct GNUNET_TIME_Absolute last; 1782 static struct GNUNET_TIME_Absolute last;
1716 static unsigned int pdot; 1783 static uint64_t idot;
1717 char hn[256]; 1784 char hn[256];
1718 1785
1719 (void) cls; 1786 (void) cls;
@@ -1730,30 +1797,32 @@ process_stdin (void *cls)
1730 { 1797 {
1731 if (strlen(hn) > 0) 1798 if (strlen(hn) > 0)
1732 hn[strlen(hn)-1] = '\0'; /* eat newline */ 1799 hn[strlen(hn)-1] = '\0'; /* eat newline */
1733 if (0 == pdot) 1800 if (0 == idot)
1734 last = GNUNET_TIME_absolute_get (); 1801 last = GNUNET_TIME_absolute_get ();
1735 pdot++; 1802 idot++;
1736 if (0 == pdot % 1000) 1803 if (0 == idot % 10000)
1737 { 1804 {
1738 struct GNUNET_TIME_Relative delta; 1805 struct GNUNET_TIME_Relative delta;
1739 1806
1740 delta = GNUNET_TIME_absolute_get_duration (last); 1807 delta = GNUNET_TIME_absolute_get_duration (last);
1741 last = GNUNET_TIME_absolute_get (); 1808 last = GNUNET_TIME_absolute_get ();
1742 fprintf (stderr, 1809 fprintf (stderr,
1743 "Imported 1000 records in %s\n", 1810 "Imported 10000 records in %s\n",
1744 GNUNET_STRINGS_relative_time_to_string (delta, 1811 GNUNET_STRINGS_relative_time_to_string (delta,
1745 GNUNET_YES)); 1812 GNUNET_YES));
1746 GNUNET_STATISTICS_set (stats, 1813 GNUNET_STATISTICS_set (stats,
1747 "# domain names provided", 1814 "# domain names provided",
1748 pdot, 1815 idot,
1749 GNUNET_NO); 1816 GNUNET_NO);
1750 } 1817 }
1751 queue (hn); 1818 queue (hn);
1752 } 1819 }
1753 fprintf (stderr, "\n"); 1820 fprintf (stderr,
1821 "Done reading %llu domain names\n",
1822 (unsigned long long) idot);
1754 GNUNET_STATISTICS_set (stats, 1823 GNUNET_STATISTICS_set (stats,
1755 "# domain names provided", 1824 "# domain names provided",
1756 pdot, 1825 idot,
1757 GNUNET_NO); 1826 GNUNET_NO);
1758 iterate_zones (NULL); 1827 iterate_zones (NULL);
1759} 1828}
@@ -1851,8 +1920,14 @@ run (void *cls,
1851 stats = GNUNET_STATISTICS_create ("zoneimport", 1920 stats = GNUNET_STATISTICS_create ("zoneimport",
1852 cfg); 1921 cfg);
1853 req_heap = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN); 1922 req_heap = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
1854 ns_pending = GNUNET_CONTAINER_multihashmap_create (1024, 1923 ns_pending = GNUNET_CONTAINER_multihashmap_create (map_size,
1855 GNUNET_NO); 1924 GNUNET_NO);
1925 if (NULL == ns_pending)
1926 {
1927 fprintf (stderr,
1928 "Failed to allocate memory for main hash map\n");
1929 return;
1930 }
1856 ctx = GNUNET_DNSSTUB_start (256); 1931 ctx = GNUNET_DNSSTUB_start (256);
1857 if (NULL == ctx) 1932 if (NULL == ctx)
1858 { 1933 {
@@ -1906,6 +1981,11 @@ main (int argc,
1906 char *const*argv) 1981 char *const*argv)
1907{ 1982{
1908 struct GNUNET_GETOPT_CommandLineOption options[] = { 1983 struct GNUNET_GETOPT_CommandLineOption options[] = {
1984 GNUNET_GETOPT_option_uint ('s',
1985 "size",
1986 "MAPSIZE",
1987 gettext_noop ("size to use for the main hash map"),
1988 &map_size),
1909 GNUNET_GETOPT_OPTION_END 1989 GNUNET_GETOPT_OPTION_END
1910 }; 1990 };
1911 1991
diff --git a/src/util/container_multihashmap.c b/src/util/container_multihashmap.c
index ffeb4a71f..6001fc1df 100644
--- a/src/util/container_multihashmap.c
+++ b/src/util/container_multihashmap.c
@@ -175,14 +175,36 @@ struct GNUNET_CONTAINER_MultiHashMap *
175GNUNET_CONTAINER_multihashmap_create (unsigned int len, 175GNUNET_CONTAINER_multihashmap_create (unsigned int len,
176 int do_not_copy_keys) 176 int do_not_copy_keys)
177{ 177{
178 struct GNUNET_CONTAINER_MultiHashMap *map; 178 struct GNUNET_CONTAINER_MultiHashMap *hm;
179 179
180 GNUNET_assert (len > 0); 180 GNUNET_assert (len > 0);
181 map = GNUNET_new (struct GNUNET_CONTAINER_MultiHashMap); 181 hm = GNUNET_new (struct GNUNET_CONTAINER_MultiHashMap);
182 map->map = GNUNET_malloc (len * sizeof (union MapEntry)); 182 if (len * sizeof (union MapEntry) > GNUNET_MAX_MALLOC_CHECKED)
183 map->map_length = len; 183 {
184 map->use_small_entries = do_not_copy_keys; 184 size_t s;
185 return map; 185 /* application *explicitly* requested very large map, hopefully
186 it checks the return value... */
187 s = len * sizeof (union MapEntry);
188 if ( (s / sizeof (union MapEntry)) != len)
189 return NULL; /* integer overflow on multiplication */
190 if (NULL == (hm->map = GNUNET_malloc_large (s)))
191 {
192 /* out of memory */
193 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
194 "Out of memory allocating large hash map (%u entries)\n",
195 len);
196 GNUNET_free (hm);
197 return NULL;
198 }
199 }
200 else
201 {
202 hm->map = GNUNET_new_array (len,
203 union MapEntry);
204 }
205 hm->map_length = len;
206 hm->use_small_entries = do_not_copy_keys;
207 return hm;
186} 208}
187 209
188 210
@@ -196,11 +218,10 @@ void
196GNUNET_CONTAINER_multihashmap_destroy (struct GNUNET_CONTAINER_MultiHashMap 218GNUNET_CONTAINER_multihashmap_destroy (struct GNUNET_CONTAINER_MultiHashMap
197 *map) 219 *map)
198{ 220{
199 unsigned int i; 221 for (unsigned int i = 0; i < map->map_length; i++)
200 union MapEntry me;
201
202 for (i = 0; i < map->map_length; i++)
203 { 222 {
223 union MapEntry me;
224
204 me = map->map[i]; 225 me = map->map[i];
205 if (map->use_small_entries) 226 if (map->use_small_entries)
206 { 227 {
@@ -257,8 +278,7 @@ idx_of (const struct GNUNET_CONTAINER_MultiHashMap *map,
257 * @return the number of key value pairs 278 * @return the number of key value pairs
258 */ 279 */
259unsigned int 280unsigned int
260GNUNET_CONTAINER_multihashmap_size (const struct GNUNET_CONTAINER_MultiHashMap 281GNUNET_CONTAINER_multihashmap_size (const struct GNUNET_CONTAINER_MultiHashMap *map)
261 *map)
262{ 282{
263 return map->size; 283 return map->size;
264} 284}
@@ -656,17 +676,22 @@ grow (struct GNUNET_CONTAINER_MultiHashMap *map)
656 unsigned int old_len; 676 unsigned int old_len;
657 unsigned int new_len; 677 unsigned int new_len;
658 unsigned int idx; 678 unsigned int idx;
659 unsigned int i;
660 679
661 map->modification_counter++; 680 map->modification_counter++;
662 681
663 old_map = map->map; 682 old_map = map->map;
664 old_len = map->map_length; 683 old_len = map->map_length;
665 new_len = old_len * 2; 684 new_len = old_len * 2;
666 new_map = GNUNET_malloc (sizeof (union MapEntry) * new_len); 685 /* if we would exceed heap size limit for the _first_ time,
686 try staying just below the limit */
687 if ( (new_len * sizeof (union MapEntry) > GNUNET_MAX_MALLOC_CHECKED) &&
688 ((old_len+1) * sizeof (union MapEntry) < GNUNET_MAX_MALLOC_CHECKED) )
689 new_len = GNUNET_MAX_MALLOC_CHECKED / sizeof (union MapEntry);
690 new_map = GNUNET_new_array (new_len,
691 union MapEntry);
667 map->map_length = new_len; 692 map->map_length = new_len;
668 map->map = new_map; 693 map->map = new_map;
669 for (i = 0; i < old_len; i++) 694 for (unsigned int i = 0; i < old_len; i++)
670 { 695 {
671 if (map->use_small_entries) 696 if (map->use_small_entries)
672 { 697 {
diff --git a/src/zonemaster/gnunet-service-zonemaster.c b/src/zonemaster/gnunet-service-zonemaster.c
index 55e1a0eee..322a1093b 100644
--- a/src/zonemaster/gnunet-service-zonemaster.c
+++ b/src/zonemaster/gnunet-service-zonemaster.c
@@ -63,7 +63,7 @@
63 * The initial interval in milliseconds btween puts in 63 * The initial interval in milliseconds btween puts in
64 * a zone iteration 64 * a zone iteration
65 */ 65 */
66#define INITIAL_PUT_INTERVAL GNUNET_TIME_UNIT_MILLISECONDS 66#define INITIAL_ZONE_ITERATION_INTERVAL GNUNET_TIME_UNIT_MILLISECONDS
67 67
68/** 68/**
69 * The upper bound for the zone iteration interval 69 * The upper bound for the zone iteration interval
@@ -204,7 +204,7 @@ static unsigned long long put_cnt;
204 * and the total number of record sets we have (so far) 204 * and the total number of record sets we have (so far)
205 * observed in the zone. 205 * observed in the zone.
206 */ 206 */
207static struct GNUNET_TIME_Relative next_put_interval; 207static struct GNUNET_TIME_Relative target_iteration_velocity_per_record;
208 208
209/** 209/**
210 * Minimum relative expiration time of records seem during the current 210 * Minimum relative expiration time of records seem during the current
@@ -383,7 +383,7 @@ dht_put_monitor_continuation (void *cls)
383 383
384 384
385/** 385/**
386 * Calculate #next_put_interval. 386 * Calculate #target_iteration_velocity_per_record.
387 */ 387 */
388static void 388static void
389calculate_put_interval () 389calculate_put_interval ()
@@ -395,7 +395,7 @@ calculate_put_interval ()
395 * we can safely set the interval to the value for a single 395 * we can safely set the interval to the value for a single
396 * record 396 * record
397 */ 397 */
398 next_put_interval = zone_publish_time_window; 398 target_iteration_velocity_per_record = zone_publish_time_window;
399 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG | GNUNET_ERROR_TYPE_BULK, 399 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG | GNUNET_ERROR_TYPE_BULK,
400 "No records in namestore database.\n"); 400 "No records in namestore database.\n");
401 } 401 }
@@ -408,24 +408,24 @@ calculate_put_interval ()
408 = GNUNET_TIME_relative_min (GNUNET_TIME_relative_divide (last_min_relative_record_time, 408 = GNUNET_TIME_relative_min (GNUNET_TIME_relative_divide (last_min_relative_record_time,
409 PUBLISH_OPS_PER_EXPIRATION), 409 PUBLISH_OPS_PER_EXPIRATION),
410 zone_publish_time_window_default); 410 zone_publish_time_window_default);
411 next_put_interval 411 target_iteration_velocity_per_record
412 = GNUNET_TIME_relative_divide (zone_publish_time_window, 412 = GNUNET_TIME_relative_divide (zone_publish_time_window,
413 last_num_public_records); 413 last_num_public_records);
414 } 414 }
415 next_put_interval 415 target_iteration_velocity_per_record
416 = GNUNET_TIME_relative_min (next_put_interval, 416 = GNUNET_TIME_relative_min (target_iteration_velocity_per_record,
417 MAXIMUM_ZONE_ITERATION_INTERVAL); 417 MAXIMUM_ZONE_ITERATION_INTERVAL);
418 GNUNET_STATISTICS_set (statistics, 418 GNUNET_STATISTICS_set (statistics,
419 "Minimum relative record expiration (in ms)", 419 "Minimum relative record expiration (in μs)",
420 last_min_relative_record_time.rel_value_us / 1000LL, 420 last_min_relative_record_time.rel_value_us,
421 GNUNET_NO); 421 GNUNET_NO);
422 GNUNET_STATISTICS_set (statistics, 422 GNUNET_STATISTICS_set (statistics,
423 "Zone publication time window (in ms)", 423 "Zone publication time window (in μs)",
424 zone_publish_time_window.rel_value_us / 1000LL, 424 zone_publish_time_window.rel_value_us,
425 GNUNET_NO); 425 GNUNET_NO);
426 GNUNET_STATISTICS_set (statistics, 426 GNUNET_STATISTICS_set (statistics,
427 "Target zone iteration velocity (μs)", 427 "Target zone iteration velocity (μs)",
428 next_put_interval.rel_value_us, 428 target_iteration_velocity_per_record.rel_value_us,
429 GNUNET_NO); 429 GNUNET_NO);
430} 430}
431 431
@@ -461,7 +461,7 @@ update_velocity (unsigned int cnt)
461 } 461 }
462 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 462 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
463 "Desired global zone iteration interval is %s/record!\n", 463 "Desired global zone iteration interval is %s/record!\n",
464 GNUNET_STRINGS_relative_time_to_string (next_put_interval, 464 GNUNET_STRINGS_relative_time_to_string (target_iteration_velocity_per_record,
465 GNUNET_YES)); 465 GNUNET_YES));
466 466
467 /* Tell statistics actual vs. desired speed */ 467 /* Tell statistics actual vs. desired speed */
@@ -471,12 +471,12 @@ update_velocity (unsigned int cnt)
471 GNUNET_NO); 471 GNUNET_NO);
472 /* update "sub_delta" based on difference, taking 472 /* update "sub_delta" based on difference, taking
473 previous sub_delta into account! */ 473 previous sub_delta into account! */
474 if (next_put_interval.rel_value_us > delta.rel_value_us) 474 if (target_iteration_velocity_per_record.rel_value_us > delta.rel_value_us)
475 { 475 {
476 /* We were too fast, reduce sub_delta! */ 476 /* We were too fast, reduce sub_delta! */
477 struct GNUNET_TIME_Relative corr; 477 struct GNUNET_TIME_Relative corr;
478 478
479 corr = GNUNET_TIME_relative_subtract (next_put_interval, 479 corr = GNUNET_TIME_relative_subtract (target_iteration_velocity_per_record,
480 delta); 480 delta);
481 if (sub_delta.rel_value_us > delta.rel_value_us) 481 if (sub_delta.rel_value_us > delta.rel_value_us)
482 { 482 {
@@ -492,28 +492,28 @@ update_velocity (unsigned int cnt)
492 sub_delta = GNUNET_TIME_UNIT_ZERO; 492 sub_delta = GNUNET_TIME_UNIT_ZERO;
493 } 493 }
494 } 494 }
495 else if (next_put_interval.rel_value_us < delta.rel_value_us) 495 else if (target_iteration_velocity_per_record.rel_value_us < delta.rel_value_us)
496 { 496 {
497 /* We were too slow, increase sub_delta! */ 497 /* We were too slow, increase sub_delta! */
498 struct GNUNET_TIME_Relative corr; 498 struct GNUNET_TIME_Relative corr;
499 499
500 corr = GNUNET_TIME_relative_subtract (delta, 500 corr = GNUNET_TIME_relative_subtract (delta,
501 next_put_interval); 501 target_iteration_velocity_per_record);
502 sub_delta = GNUNET_TIME_relative_add (sub_delta, 502 sub_delta = GNUNET_TIME_relative_add (sub_delta,
503 corr); 503 corr);
504 if (sub_delta.rel_value_us > next_put_interval.rel_value_us) 504 if (sub_delta.rel_value_us > target_iteration_velocity_per_record.rel_value_us)
505 { 505 {
506 /* CPU overload detected, we cannot go at desired speed, 506 /* CPU overload detected, we cannot go at desired speed,
507 as this would mean using a negative delay. */ 507 as this would mean using a negative delay. */
508 /* compute how much faster we would want to be for 508 /* compute how much faster we would want to be for
509 the desired velocity */ 509 the desired velocity */
510 if (0 == next_put_interval.rel_value_us) 510 if (0 == target_iteration_velocity_per_record.rel_value_us)
511 pct = UINT64_MAX; /* desired speed is infinity ... */ 511 pct = UINT64_MAX; /* desired speed is infinity ... */
512 else 512 else
513 pct = (sub_delta.rel_value_us - 513 pct = (sub_delta.rel_value_us -
514 next_put_interval.rel_value_us) * 100LLU 514 target_iteration_velocity_per_record.rel_value_us) * 100LLU
515 / next_put_interval.rel_value_us; 515 / target_iteration_velocity_per_record.rel_value_us;
516 sub_delta = next_put_interval; 516 sub_delta = target_iteration_velocity_per_record;
517 } 517 }
518 } 518 }
519 GNUNET_STATISTICS_set (statistics, 519 GNUNET_STATISTICS_set (statistics,
@@ -548,7 +548,7 @@ check_zone_namestore_next ()
548 return; /* current NAMESTORE iteration not yet done */ 548 return; /* current NAMESTORE iteration not yet done */
549 update_velocity (put_cnt); 549 update_velocity (put_cnt);
550 put_cnt = 0; 550 put_cnt = 0;
551 delay = GNUNET_TIME_relative_subtract (next_put_interval, 551 delay = GNUNET_TIME_relative_subtract (target_iteration_velocity_per_record,
552 sub_delta); 552 sub_delta);
553 /* We delay *once* per #NS_BLOCK_SIZE, so we need to multiply the 553 /* We delay *once* per #NS_BLOCK_SIZE, so we need to multiply the
554 per-record delay calculated so far with the #NS_BLOCK_SIZE */ 554 per-record delay calculated so far with the #NS_BLOCK_SIZE */
@@ -755,11 +755,11 @@ zone_iteration_finished (void *cls)
755 PUBLISH_OPS_PER_EXPIRATION); 755 PUBLISH_OPS_PER_EXPIRATION);
756 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 756 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
757 "Zone iteration finished. Adjusted zone iteration interval to %s\n", 757 "Zone iteration finished. Adjusted zone iteration interval to %s\n",
758 GNUNET_STRINGS_relative_time_to_string (next_put_interval, 758 GNUNET_STRINGS_relative_time_to_string (target_iteration_velocity_per_record,
759 GNUNET_YES)); 759 GNUNET_YES));
760 GNUNET_STATISTICS_set (statistics, 760 GNUNET_STATISTICS_set (statistics,
761 "Current zone iteration interval (in ms)", 761 "Target zone iteration velocity s)",
762 next_put_interval.rel_value_us / 1000LL, 762 target_iteration_velocity_per_record.rel_value_us,
763 GNUNET_NO); 763 GNUNET_NO);
764 GNUNET_STATISTICS_set (statistics, 764 GNUNET_STATISTICS_set (statistics,
765 "Number of public records in DHT", 765 "Number of public records in DHT",
@@ -767,12 +767,16 @@ zone_iteration_finished (void *cls)
767 GNUNET_NO); 767 GNUNET_NO);
768 GNUNET_assert (NULL == zone_publish_task); 768 GNUNET_assert (NULL == zone_publish_task);
769 if (0 == last_num_public_records) 769 if (0 == last_num_public_records)
770 zone_publish_task = GNUNET_SCHEDULER_add_delayed (next_put_interval, 770 {
771 zone_publish_task = GNUNET_SCHEDULER_add_delayed (target_iteration_velocity_per_record,
771 &publish_zone_dht_start, 772 &publish_zone_dht_start,
772 NULL); 773 NULL);
774 }
773 else 775 else
776 {
774 zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start, 777 zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start,
775 NULL); 778 NULL);
779 }
776} 780}
777 781
778 782
@@ -1028,7 +1032,7 @@ run (void *cls,
1028 min_relative_record_time 1032 min_relative_record_time
1029 = GNUNET_TIME_relative_multiply (GNUNET_DHT_DEFAULT_REPUBLISH_FREQUENCY, 1033 = GNUNET_TIME_relative_multiply (GNUNET_DHT_DEFAULT_REPUBLISH_FREQUENCY,
1030 PUBLISH_OPS_PER_EXPIRATION); 1034 PUBLISH_OPS_PER_EXPIRATION);
1031 next_put_interval = INITIAL_PUT_INTERVAL; 1035 target_iteration_velocity_per_record = INITIAL_ZONE_ITERATION_INTERVAL;
1032 namestore_handle = GNUNET_NAMESTORE_connect (c); 1036 namestore_handle = GNUNET_NAMESTORE_connect (c);
1033 if (NULL == namestore_handle) 1037 if (NULL == namestore_handle)
1034 { 1038 {
@@ -1077,12 +1081,12 @@ run (void *cls,
1077 } 1081 }
1078 1082
1079 /* Schedule periodic put for our records. */ 1083 /* Schedule periodic put for our records. */
1080 first_zone_iteration = GNUNET_YES;\ 1084 first_zone_iteration = GNUNET_YES;
1081 statistics = GNUNET_STATISTICS_create ("zonemaster", 1085 statistics = GNUNET_STATISTICS_create ("zonemaster",
1082 c); 1086 c);
1083 GNUNET_STATISTICS_set (statistics, 1087 GNUNET_STATISTICS_set (statistics,
1084 "Target zone iteration velocity (μs)", 1088 "Target zone iteration velocity (μs)",
1085 next_put_interval.rel_value_us, 1089 target_iteration_velocity_per_record.rel_value_us,
1086 GNUNET_NO); 1090 GNUNET_NO);
1087 zmon = GNUNET_NAMESTORE_zone_monitor_start (c, 1091 zmon = GNUNET_NAMESTORE_zone_monitor_start (c,
1088 NULL, 1092 NULL,