diff options
author | Christian Grothoff <christian@grothoff.org> | 2018-05-02 01:01:57 +0200 |
---|---|---|
committer | Christian Grothoff <christian@grothoff.org> | 2018-05-02 01:01:57 +0200 |
commit | 857f8c009c4fde3d3ec9d6d0b4af48e93684fd84 (patch) | |
tree | 079b460ad7555b148720c74b3601429c662c4113 /src | |
parent | ffe43cb1e86f49ddff73554df200853248af012d (diff) | |
download | gnunet-857f8c009c4fde3d3ec9d6d0b4af48e93684fd84.tar.gz gnunet-857f8c009c4fde3d3ec9d6d0b4af48e93684fd84.zip |
misc improvements to statistics and large-scale behavior of zoneimport/zonemaster
Diffstat (limited to 'src')
-rw-r--r-- | src/namestore/gnunet-zoneimport.c | 202 | ||||
-rw-r--r-- | src/util/container_multihashmap.c | 55 | ||||
-rw-r--r-- | src/zonemaster/gnunet-service-zonemaster.c | 66 |
3 files changed, 216 insertions, 107 deletions
diff --git a/src/namestore/gnunet-zoneimport.c b/src/namestore/gnunet-zoneimport.c index e24cb26dd..0fd0a4ab8 100644 --- a/src/namestore/gnunet-zoneimport.c +++ b/src/namestore/gnunet-zoneimport.c | |||
@@ -204,6 +204,14 @@ struct Request | |||
204 | 204 | ||
205 | 205 | ||
206 | /** | 206 | /** |
207 | * Command-line argument specifying desired size of the hash map with | ||
208 | * all of our pending names. Usually, we use an automatically growing | ||
209 | * map, but this is only OK up to about a million entries. Above that | ||
210 | * number, the user must explicitly specify the size at startup. | ||
211 | */ | ||
212 | static unsigned int map_size = 1024; | ||
213 | |||
214 | /** | ||
207 | * Handle to the identity service. | 215 | * Handle to the identity service. |
208 | */ | 216 | */ |
209 | static struct GNUNET_IDENTITY_Handle *id; | 217 | static struct GNUNET_IDENTITY_Handle *id; |
@@ -323,14 +331,29 @@ static uint64_t total_dns_latency_cnt; | |||
323 | static struct GNUNET_TIME_Relative total_dns_latency; | 331 | static struct GNUNET_TIME_Relative total_dns_latency; |
324 | 332 | ||
325 | /** | 333 | /** |
326 | * Number of NAMESTORE requests counted in latency total. | 334 | * Number of records processed (DNS lookup, no NAMESTORE) in total. |
335 | */ | ||
336 | static uint64_t total_reg_proc_dns; | ||
337 | |||
338 | /** | ||
339 | * Number of records processed (DNS lookup, with NAMESTORE) in total. | ||
340 | */ | ||
341 | static uint64_t total_reg_proc_dns_ns; | ||
342 | |||
343 | /** | ||
344 | * Start time of the regular processing. | ||
345 | */ | ||
346 | static struct GNUNET_TIME_Absolute start_time_reg_proc; | ||
347 | |||
348 | /** | ||
349 | * Last time we worked before going idle. | ||
327 | */ | 350 | */ |
328 | static uint64_t total_ns_latency_cnt; | 351 | static struct GNUNET_TIME_Absolute sleep_time_reg_proc; |
329 | 352 | ||
330 | /** | 353 | /** |
331 | * Sum of NAMESTORE latencies observed. | 354 | * Time we slept just waiting for work. |
332 | */ | 355 | */ |
333 | static struct GNUNET_TIME_Relative total_ns_latency; | 356 | static struct GNUNET_TIME_Relative idle_time; |
334 | 357 | ||
335 | 358 | ||
336 | /** | 359 | /** |
@@ -530,6 +553,7 @@ insert_sorted (struct Request *req) | |||
530 | { | 553 | { |
531 | if (NULL != t) | 554 | if (NULL != t) |
532 | GNUNET_SCHEDULER_cancel (t); | 555 | GNUNET_SCHEDULER_cancel (t); |
556 | sleep_time_reg_proc = GNUNET_TIME_absolute_get (); | ||
533 | t = GNUNET_SCHEDULER_add_at (req->expires, | 557 | t = GNUNET_SCHEDULER_add_at (req->expires, |
534 | &process_queue, | 558 | &process_queue, |
535 | NULL); | 559 | NULL); |
@@ -966,37 +990,9 @@ store_completed_cb (void *cls, | |||
966 | const char *emsg) | 990 | const char *emsg) |
967 | { | 991 | { |
968 | static struct GNUNET_TIME_Absolute last; | 992 | static struct GNUNET_TIME_Absolute last; |
969 | static unsigned int pdot; | ||
970 | struct Request *req = cls; | 993 | struct Request *req = cls; |
971 | 994 | ||
972 | req->qe = NULL; | 995 | req->qe = NULL; |
973 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
974 | "Stored record set in database (%d)\n", | ||
975 | success); | ||
976 | pending_rs--; | ||
977 | { | ||
978 | struct GNUNET_TIME_Relative ns_latency; | ||
979 | |||
980 | ns_latency = GNUNET_TIME_absolute_get_duration (req->op_start_time); | ||
981 | total_ns_latency = GNUNET_TIME_relative_add (total_ns_latency, | ||
982 | ns_latency); | ||
983 | total_ns_latency_cnt++; | ||
984 | if (0 == (total_ns_latency_cnt % 1000)) | ||
985 | { | ||
986 | GNUNET_STATISTICS_update (stats, | ||
987 | "# average NAMESTORE PUT latency (μs)", | ||
988 | total_ns_latency.rel_value_us / total_ns_latency_cnt, | ||
989 | GNUNET_NO); | ||
990 | GNUNET_STATISTICS_update (stats, | ||
991 | "# NAMESTORE PUTs", | ||
992 | total_ns_latency_cnt, | ||
993 | GNUNET_NO); | ||
994 | } | ||
995 | } | ||
996 | |||
997 | if (NULL == t) | ||
998 | t = GNUNET_SCHEDULER_add_now (&process_queue, | ||
999 | NULL); | ||
1000 | if (GNUNET_SYSERR == success) | 996 | if (GNUNET_SYSERR == success) |
1001 | { | 997 | { |
1002 | GNUNET_log (GNUNET_ERROR_TYPE_ERROR, | 998 | GNUNET_log (GNUNET_ERROR_TYPE_ERROR, |
@@ -1007,12 +1003,26 @@ store_completed_cb (void *cls, | |||
1007 | else | 1003 | else |
1008 | { | 1004 | { |
1009 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 1005 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, |
1010 | "Stored records under `%s'\n", | 1006 | "Stored records under `%s' (%d)\n", |
1011 | req->hostname); | 1007 | req->hostname, |
1012 | if (0 == pdot) | 1008 | success); |
1009 | } | ||
1010 | total_reg_proc_dns_ns++; /* finished regular processing */ | ||
1011 | pending_rs--; | ||
1012 | free_records (req); | ||
1013 | /* compute NAMESTORE statistics */ | ||
1014 | { | ||
1015 | static uint64_t total_ns_latency_cnt; | ||
1016 | static struct GNUNET_TIME_Relative total_ns_latency; | ||
1017 | struct GNUNET_TIME_Relative ns_latency; | ||
1018 | |||
1019 | ns_latency = GNUNET_TIME_absolute_get_duration (req->op_start_time); | ||
1020 | total_ns_latency = GNUNET_TIME_relative_add (total_ns_latency, | ||
1021 | ns_latency); | ||
1022 | if (0 == total_ns_latency_cnt) | ||
1013 | last = GNUNET_TIME_absolute_get (); | 1023 | last = GNUNET_TIME_absolute_get (); |
1014 | pdot++; | 1024 | total_ns_latency_cnt++; |
1015 | if (0 == pdot % 1000) | 1025 | if (0 == (total_ns_latency_cnt % 1000)) |
1016 | { | 1026 | { |
1017 | struct GNUNET_TIME_Relative delta; | 1027 | struct GNUNET_TIME_Relative delta; |
1018 | 1028 | ||
@@ -1022,9 +1032,46 @@ store_completed_cb (void *cls, | |||
1022 | "Processed 1000 records in %s\n", | 1032 | "Processed 1000 records in %s\n", |
1023 | GNUNET_STRINGS_relative_time_to_string (delta, | 1033 | GNUNET_STRINGS_relative_time_to_string (delta, |
1024 | GNUNET_YES)); | 1034 | GNUNET_YES)); |
1035 | GNUNET_STATISTICS_set (stats, | ||
1036 | "# average NAMESTORE PUT latency (μs)", | ||
1037 | total_ns_latency.rel_value_us / total_ns_latency_cnt, | ||
1038 | GNUNET_NO); | ||
1025 | } | 1039 | } |
1026 | } | 1040 | } |
1027 | free_records (req); | 1041 | /* compute and publish overall velocity */ |
1042 | if (0 == (total_reg_proc_dns_ns % 100) ) | ||
1043 | { | ||
1044 | struct GNUNET_TIME_Relative runtime; | ||
1045 | |||
1046 | runtime = GNUNET_TIME_absolute_get_duration (start_time_reg_proc); | ||
1047 | runtime = GNUNET_TIME_relative_subtract (runtime, | ||
1048 | idle_time); | ||
1049 | runtime = GNUNET_TIME_relative_divide (runtime, | ||
1050 | total_reg_proc_dns + total_reg_proc_dns_ns); | ||
1051 | GNUNET_STATISTICS_set (stats, | ||
1052 | "# Regular processing completed without NAMESTORE", | ||
1053 | total_reg_proc_dns, | ||
1054 | GNUNET_NO); | ||
1055 | GNUNET_STATISTICS_set (stats, | ||
1056 | "# Regular processing completed with NAMESTORE PUT", | ||
1057 | total_reg_proc_dns_ns, | ||
1058 | GNUNET_NO); | ||
1059 | GNUNET_STATISTICS_set (stats, | ||
1060 | "# average request processing latency (μs)", | ||
1061 | runtime.rel_value_us, | ||
1062 | GNUNET_NO); | ||
1063 | GNUNET_STATISTICS_set (stats, | ||
1064 | "# total time spent idle (μs)", | ||
1065 | idle_time.rel_value_us, | ||
1066 | GNUNET_NO); | ||
1067 | } | ||
1068 | |||
1069 | if (NULL == t) | ||
1070 | { | ||
1071 | sleep_time_reg_proc = GNUNET_TIME_absolute_get (); | ||
1072 | t = GNUNET_SCHEDULER_add_now (&process_queue, | ||
1073 | NULL); | ||
1074 | } | ||
1028 | } | 1075 | } |
1029 | 1076 | ||
1030 | 1077 | ||
@@ -1054,8 +1101,11 @@ process_result (void *cls, | |||
1054 | req); | 1101 | req); |
1055 | pending--; | 1102 | pending--; |
1056 | if (NULL == t) | 1103 | if (NULL == t) |
1104 | { | ||
1105 | sleep_time_reg_proc = GNUNET_TIME_absolute_get (); | ||
1057 | t = GNUNET_SCHEDULER_add_now (&process_queue, | 1106 | t = GNUNET_SCHEDULER_add_now (&process_queue, |
1058 | NULL); | 1107 | NULL); |
1108 | } | ||
1059 | GNUNET_log (GNUNET_ERROR_TYPE_ERROR, | 1109 | GNUNET_log (GNUNET_ERROR_TYPE_ERROR, |
1060 | "Stub gave up on DNS reply for `%s'\n", | 1110 | "Stub gave up on DNS reply for `%s'\n", |
1061 | req->hostname); | 1111 | req->hostname); |
@@ -1073,6 +1123,7 @@ process_result (void *cls, | |||
1073 | GNUNET_NO); | 1123 | GNUNET_NO); |
1074 | return; | 1124 | return; |
1075 | } | 1125 | } |
1126 | total_reg_proc_dns++; | ||
1076 | req->rs = NULL; | 1127 | req->rs = NULL; |
1077 | insert_sorted (req); | 1128 | insert_sorted (req); |
1078 | return; | 1129 | return; |
@@ -1105,8 +1156,11 @@ process_result (void *cls, | |||
1105 | 1, | 1156 | 1, |
1106 | GNUNET_NO); | 1157 | GNUNET_NO); |
1107 | if (NULL == t) | 1158 | if (NULL == t) |
1159 | { | ||
1160 | sleep_time_reg_proc = GNUNET_TIME_absolute_get (); | ||
1108 | t = GNUNET_SCHEDULER_add_now (&process_queue, | 1161 | t = GNUNET_SCHEDULER_add_now (&process_queue, |
1109 | NULL); | 1162 | NULL); |
1163 | } | ||
1110 | if (req->issue_num > MAX_RETRIES) | 1164 | if (req->issue_num > MAX_RETRIES) |
1111 | { | 1165 | { |
1112 | failures++; | 1166 | failures++; |
@@ -1144,14 +1198,10 @@ process_result (void *cls, | |||
1144 | total_dns_latency_cnt++; | 1198 | total_dns_latency_cnt++; |
1145 | if (0 == (total_dns_latency_cnt % 1000)) | 1199 | if (0 == (total_dns_latency_cnt % 1000)) |
1146 | { | 1200 | { |
1147 | GNUNET_STATISTICS_update (stats, | 1201 | GNUNET_STATISTICS_set (stats, |
1148 | "# average DNS latency (μs)", | 1202 | "# average DNS lookup latency (μs)", |
1149 | total_dns_latency.rel_value_us / total_dns_latency_cnt, | 1203 | total_dns_latency.rel_value_us / total_dns_latency_cnt, |
1150 | GNUNET_NO); | 1204 | GNUNET_NO); |
1151 | GNUNET_STATISTICS_update (stats, | ||
1152 | "# DNS replies", | ||
1153 | total_dns_latency_cnt, | ||
1154 | GNUNET_NO); | ||
1155 | } | 1205 | } |
1156 | } | 1206 | } |
1157 | rd_count = 0; | 1207 | rd_count = 0; |
@@ -1171,10 +1221,18 @@ process_result (void *cls, | |||
1171 | /* Instead of going for SOA, simplified for now to look each | 1221 | /* Instead of going for SOA, simplified for now to look each |
1172 | day in case we got an empty response */ | 1222 | day in case we got an empty response */ |
1173 | if (0 == rd_count) | 1223 | if (0 == rd_count) |
1224 | { | ||
1174 | req->expires | 1225 | req->expires |
1175 | = GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_DAYS); | 1226 | = GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_DAYS); |
1227 | GNUNET_STATISTICS_update (stats, | ||
1228 | "# empty DNS replies (usually NXDOMAIN)", | ||
1229 | 1, | ||
1230 | GNUNET_NO); | ||
1231 | } | ||
1176 | else | 1232 | else |
1233 | { | ||
1177 | record_sets++; | 1234 | record_sets++; |
1235 | } | ||
1178 | /* convert records to namestore import format */ | 1236 | /* convert records to namestore import format */ |
1179 | { | 1237 | { |
1180 | struct GNUNET_GNSRECORD_Data rd[GNUNET_NZL(rd_count)]; | 1238 | struct GNUNET_GNSRECORD_Data rd[GNUNET_NZL(rd_count)]; |
@@ -1210,8 +1268,12 @@ process_queue (void *cls) | |||
1210 | unsigned int series; | 1268 | unsigned int series; |
1211 | void *raw; | 1269 | void *raw; |
1212 | size_t raw_size; | 1270 | size_t raw_size; |
1271 | struct GNUNET_TIME_Relative delay; | ||
1213 | 1272 | ||
1214 | (void) cls; | 1273 | (void) cls; |
1274 | delay = GNUNET_TIME_absolute_get_duration (sleep_time_reg_proc); | ||
1275 | idle_time = GNUNET_TIME_relative_add (idle_time, | ||
1276 | delay); | ||
1215 | series = 0; | 1277 | series = 0; |
1216 | t = NULL; | 1278 | t = NULL; |
1217 | while (pending + pending_rs < THRESH) | 1279 | while (pending + pending_rs < THRESH) |
@@ -1284,6 +1346,7 @@ process_queue (void *cls) | |||
1284 | req->hostname); | 1346 | req->hostname); |
1285 | if (NULL != t) | 1347 | if (NULL != t) |
1286 | GNUNET_SCHEDULER_cancel (t); | 1348 | GNUNET_SCHEDULER_cancel (t); |
1349 | sleep_time_reg_proc = GNUNET_TIME_absolute_get (); | ||
1287 | t = GNUNET_SCHEDULER_add_at (req->expires, | 1350 | t = GNUNET_SCHEDULER_add_at (req->expires, |
1288 | &process_queue, | 1351 | &process_queue, |
1289 | NULL); | 1352 | NULL); |
@@ -1293,6 +1356,7 @@ process_queue (void *cls) | |||
1293 | "Throttling\n"); | 1356 | "Throttling\n"); |
1294 | if (NULL != t) | 1357 | if (NULL != t) |
1295 | GNUNET_SCHEDULER_cancel (t); | 1358 | GNUNET_SCHEDULER_cancel (t); |
1359 | sleep_time_reg_proc = GNUNET_TIME_absolute_get (); | ||
1296 | t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY, | 1360 | t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY, |
1297 | &process_queue, | 1361 | &process_queue, |
1298 | NULL); | 1362 | NULL); |
@@ -1450,9 +1514,9 @@ ns_lookup_result_cb (void *cls, | |||
1450 | { | 1514 | { |
1451 | ns_iterator_trigger_next = NS_BATCH_SIZE; | 1515 | ns_iterator_trigger_next = NS_BATCH_SIZE; |
1452 | GNUNET_STATISTICS_update (stats, | 1516 | GNUNET_STATISTICS_update (stats, |
1453 | "# NAMESTORE records requested", | 1517 | "# NAMESTORE records requested from cache", |
1454 | ns_iterator_trigger_next, | 1518 | ns_iterator_trigger_next, |
1455 | GNUNET_NO); | 1519 | GNUNET_NO); |
1456 | GNUNET_NAMESTORE_zone_iterator_next (zone_it, | 1520 | GNUNET_NAMESTORE_zone_iterator_next (zone_it, |
1457 | ns_iterator_trigger_next); | 1521 | ns_iterator_trigger_next); |
1458 | } | 1522 | } |
@@ -1652,7 +1716,7 @@ iterate_zones (void *cls) | |||
1652 | last->domain); | 1716 | last->domain); |
1653 | /* subtract left-overs from previous iteration */ | 1717 | /* subtract left-overs from previous iteration */ |
1654 | GNUNET_STATISTICS_update (stats, | 1718 | GNUNET_STATISTICS_update (stats, |
1655 | "# NAMESTORE records requested", | 1719 | "# NAMESTORE records requested from cache", |
1656 | (long long) (- ns_iterator_trigger_next), | 1720 | (long long) (- ns_iterator_trigger_next), |
1657 | GNUNET_NO); | 1721 | GNUNET_NO); |
1658 | ns_iterator_trigger_next = 0; | 1722 | ns_iterator_trigger_next = 0; |
@@ -1665,7 +1729,7 @@ iterate_zones (void *cls) | |||
1665 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 1729 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, |
1666 | "Finished all NAMESTORE iterations!\n"); | 1730 | "Finished all NAMESTORE iterations!\n"); |
1667 | GNUNET_STATISTICS_set (stats, | 1731 | GNUNET_STATISTICS_set (stats, |
1668 | "# NAMESTORE lookups without reply", | 1732 | "# Domain names without cached reply", |
1669 | GNUNET_CONTAINER_multihashmap_size (ns_pending), | 1733 | GNUNET_CONTAINER_multihashmap_size (ns_pending), |
1670 | GNUNET_NO); | 1734 | GNUNET_NO); |
1671 | GNUNET_CONTAINER_multihashmap_iterate (ns_pending, | 1735 | GNUNET_CONTAINER_multihashmap_iterate (ns_pending, |
@@ -1673,6 +1737,9 @@ iterate_zones (void *cls) | |||
1673 | NULL); | 1737 | NULL); |
1674 | GNUNET_CONTAINER_multihashmap_destroy (ns_pending); | 1738 | GNUNET_CONTAINER_multihashmap_destroy (ns_pending); |
1675 | ns_pending = NULL; | 1739 | ns_pending = NULL; |
1740 | start_time_reg_proc = GNUNET_TIME_absolute_get (); | ||
1741 | total_reg_proc_dns = 0; | ||
1742 | total_reg_proc_dns_ns = 0; | ||
1676 | return; | 1743 | return; |
1677 | } | 1744 | } |
1678 | if (NULL == last) | 1745 | if (NULL == last) |
@@ -1684,7 +1751,7 @@ iterate_zones (void *cls) | |||
1684 | last->domain); | 1751 | last->domain); |
1685 | /* subtract left-overs from previous iteration */ | 1752 | /* subtract left-overs from previous iteration */ |
1686 | GNUNET_STATISTICS_update (stats, | 1753 | GNUNET_STATISTICS_update (stats, |
1687 | "# NAMESTORE records requested", | 1754 | "# NAMESTORE records requested from cache", |
1688 | 1, | 1755 | 1, |
1689 | GNUNET_NO); | 1756 | GNUNET_NO); |
1690 | ns_iterator_trigger_next = 1; | 1757 | ns_iterator_trigger_next = 1; |
@@ -1713,7 +1780,7 @@ static void | |||
1713 | process_stdin (void *cls) | 1780 | process_stdin (void *cls) |
1714 | { | 1781 | { |
1715 | static struct GNUNET_TIME_Absolute last; | 1782 | static struct GNUNET_TIME_Absolute last; |
1716 | static unsigned int pdot; | 1783 | static uint64_t idot; |
1717 | char hn[256]; | 1784 | char hn[256]; |
1718 | 1785 | ||
1719 | (void) cls; | 1786 | (void) cls; |
@@ -1730,30 +1797,32 @@ process_stdin (void *cls) | |||
1730 | { | 1797 | { |
1731 | if (strlen(hn) > 0) | 1798 | if (strlen(hn) > 0) |
1732 | hn[strlen(hn)-1] = '\0'; /* eat newline */ | 1799 | hn[strlen(hn)-1] = '\0'; /* eat newline */ |
1733 | if (0 == pdot) | 1800 | if (0 == idot) |
1734 | last = GNUNET_TIME_absolute_get (); | 1801 | last = GNUNET_TIME_absolute_get (); |
1735 | pdot++; | 1802 | idot++; |
1736 | if (0 == pdot % 1000) | 1803 | if (0 == idot % 10000) |
1737 | { | 1804 | { |
1738 | struct GNUNET_TIME_Relative delta; | 1805 | struct GNUNET_TIME_Relative delta; |
1739 | 1806 | ||
1740 | delta = GNUNET_TIME_absolute_get_duration (last); | 1807 | delta = GNUNET_TIME_absolute_get_duration (last); |
1741 | last = GNUNET_TIME_absolute_get (); | 1808 | last = GNUNET_TIME_absolute_get (); |
1742 | fprintf (stderr, | 1809 | fprintf (stderr, |
1743 | "Imported 1000 records in %s\n", | 1810 | "Imported 10000 records in %s\n", |
1744 | GNUNET_STRINGS_relative_time_to_string (delta, | 1811 | GNUNET_STRINGS_relative_time_to_string (delta, |
1745 | GNUNET_YES)); | 1812 | GNUNET_YES)); |
1746 | GNUNET_STATISTICS_set (stats, | 1813 | GNUNET_STATISTICS_set (stats, |
1747 | "# domain names provided", | 1814 | "# domain names provided", |
1748 | pdot, | 1815 | idot, |
1749 | GNUNET_NO); | 1816 | GNUNET_NO); |
1750 | } | 1817 | } |
1751 | queue (hn); | 1818 | queue (hn); |
1752 | } | 1819 | } |
1753 | fprintf (stderr, "\n"); | 1820 | fprintf (stderr, |
1821 | "Done reading %llu domain names\n", | ||
1822 | (unsigned long long) idot); | ||
1754 | GNUNET_STATISTICS_set (stats, | 1823 | GNUNET_STATISTICS_set (stats, |
1755 | "# domain names provided", | 1824 | "# domain names provided", |
1756 | pdot, | 1825 | idot, |
1757 | GNUNET_NO); | 1826 | GNUNET_NO); |
1758 | iterate_zones (NULL); | 1827 | iterate_zones (NULL); |
1759 | } | 1828 | } |
@@ -1851,8 +1920,14 @@ run (void *cls, | |||
1851 | stats = GNUNET_STATISTICS_create ("zoneimport", | 1920 | stats = GNUNET_STATISTICS_create ("zoneimport", |
1852 | cfg); | 1921 | cfg); |
1853 | req_heap = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN); | 1922 | req_heap = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN); |
1854 | ns_pending = GNUNET_CONTAINER_multihashmap_create (1024, | 1923 | ns_pending = GNUNET_CONTAINER_multihashmap_create (map_size, |
1855 | GNUNET_NO); | 1924 | GNUNET_NO); |
1925 | if (NULL == ns_pending) | ||
1926 | { | ||
1927 | fprintf (stderr, | ||
1928 | "Failed to allocate memory for main hash map\n"); | ||
1929 | return; | ||
1930 | } | ||
1856 | ctx = GNUNET_DNSSTUB_start (256); | 1931 | ctx = GNUNET_DNSSTUB_start (256); |
1857 | if (NULL == ctx) | 1932 | if (NULL == ctx) |
1858 | { | 1933 | { |
@@ -1906,6 +1981,11 @@ main (int argc, | |||
1906 | char *const*argv) | 1981 | char *const*argv) |
1907 | { | 1982 | { |
1908 | struct GNUNET_GETOPT_CommandLineOption options[] = { | 1983 | struct GNUNET_GETOPT_CommandLineOption options[] = { |
1984 | GNUNET_GETOPT_option_uint ('s', | ||
1985 | "size", | ||
1986 | "MAPSIZE", | ||
1987 | gettext_noop ("size to use for the main hash map"), | ||
1988 | &map_size), | ||
1909 | GNUNET_GETOPT_OPTION_END | 1989 | GNUNET_GETOPT_OPTION_END |
1910 | }; | 1990 | }; |
1911 | 1991 | ||
diff --git a/src/util/container_multihashmap.c b/src/util/container_multihashmap.c index ffeb4a71f..6001fc1df 100644 --- a/src/util/container_multihashmap.c +++ b/src/util/container_multihashmap.c | |||
@@ -175,14 +175,36 @@ struct GNUNET_CONTAINER_MultiHashMap * | |||
175 | GNUNET_CONTAINER_multihashmap_create (unsigned int len, | 175 | GNUNET_CONTAINER_multihashmap_create (unsigned int len, |
176 | int do_not_copy_keys) | 176 | int do_not_copy_keys) |
177 | { | 177 | { |
178 | struct GNUNET_CONTAINER_MultiHashMap *map; | 178 | struct GNUNET_CONTAINER_MultiHashMap *hm; |
179 | 179 | ||
180 | GNUNET_assert (len > 0); | 180 | GNUNET_assert (len > 0); |
181 | map = GNUNET_new (struct GNUNET_CONTAINER_MultiHashMap); | 181 | hm = GNUNET_new (struct GNUNET_CONTAINER_MultiHashMap); |
182 | map->map = GNUNET_malloc (len * sizeof (union MapEntry)); | 182 | if (len * sizeof (union MapEntry) > GNUNET_MAX_MALLOC_CHECKED) |
183 | map->map_length = len; | 183 | { |
184 | map->use_small_entries = do_not_copy_keys; | 184 | size_t s; |
185 | return map; | 185 | /* application *explicitly* requested very large map, hopefully |
186 | it checks the return value... */ | ||
187 | s = len * sizeof (union MapEntry); | ||
188 | if ( (s / sizeof (union MapEntry)) != len) | ||
189 | return NULL; /* integer overflow on multiplication */ | ||
190 | if (NULL == (hm->map = GNUNET_malloc_large (s))) | ||
191 | { | ||
192 | /* out of memory */ | ||
193 | GNUNET_log (GNUNET_ERROR_TYPE_WARNING, | ||
194 | "Out of memory allocating large hash map (%u entries)\n", | ||
195 | len); | ||
196 | GNUNET_free (hm); | ||
197 | return NULL; | ||
198 | } | ||
199 | } | ||
200 | else | ||
201 | { | ||
202 | hm->map = GNUNET_new_array (len, | ||
203 | union MapEntry); | ||
204 | } | ||
205 | hm->map_length = len; | ||
206 | hm->use_small_entries = do_not_copy_keys; | ||
207 | return hm; | ||
186 | } | 208 | } |
187 | 209 | ||
188 | 210 | ||
@@ -196,11 +218,10 @@ void | |||
196 | GNUNET_CONTAINER_multihashmap_destroy (struct GNUNET_CONTAINER_MultiHashMap | 218 | GNUNET_CONTAINER_multihashmap_destroy (struct GNUNET_CONTAINER_MultiHashMap |
197 | *map) | 219 | *map) |
198 | { | 220 | { |
199 | unsigned int i; | 221 | for (unsigned int i = 0; i < map->map_length; i++) |
200 | union MapEntry me; | ||
201 | |||
202 | for (i = 0; i < map->map_length; i++) | ||
203 | { | 222 | { |
223 | union MapEntry me; | ||
224 | |||
204 | me = map->map[i]; | 225 | me = map->map[i]; |
205 | if (map->use_small_entries) | 226 | if (map->use_small_entries) |
206 | { | 227 | { |
@@ -257,8 +278,7 @@ idx_of (const struct GNUNET_CONTAINER_MultiHashMap *map, | |||
257 | * @return the number of key value pairs | 278 | * @return the number of key value pairs |
258 | */ | 279 | */ |
259 | unsigned int | 280 | unsigned int |
260 | GNUNET_CONTAINER_multihashmap_size (const struct GNUNET_CONTAINER_MultiHashMap | 281 | GNUNET_CONTAINER_multihashmap_size (const struct GNUNET_CONTAINER_MultiHashMap *map) |
261 | *map) | ||
262 | { | 282 | { |
263 | return map->size; | 283 | return map->size; |
264 | } | 284 | } |
@@ -656,17 +676,22 @@ grow (struct GNUNET_CONTAINER_MultiHashMap *map) | |||
656 | unsigned int old_len; | 676 | unsigned int old_len; |
657 | unsigned int new_len; | 677 | unsigned int new_len; |
658 | unsigned int idx; | 678 | unsigned int idx; |
659 | unsigned int i; | ||
660 | 679 | ||
661 | map->modification_counter++; | 680 | map->modification_counter++; |
662 | 681 | ||
663 | old_map = map->map; | 682 | old_map = map->map; |
664 | old_len = map->map_length; | 683 | old_len = map->map_length; |
665 | new_len = old_len * 2; | 684 | new_len = old_len * 2; |
666 | new_map = GNUNET_malloc (sizeof (union MapEntry) * new_len); | 685 | /* if we would exceed heap size limit for the _first_ time, |
686 | try staying just below the limit */ | ||
687 | if ( (new_len * sizeof (union MapEntry) > GNUNET_MAX_MALLOC_CHECKED) && | ||
688 | ((old_len+1) * sizeof (union MapEntry) < GNUNET_MAX_MALLOC_CHECKED) ) | ||
689 | new_len = GNUNET_MAX_MALLOC_CHECKED / sizeof (union MapEntry); | ||
690 | new_map = GNUNET_new_array (new_len, | ||
691 | union MapEntry); | ||
667 | map->map_length = new_len; | 692 | map->map_length = new_len; |
668 | map->map = new_map; | 693 | map->map = new_map; |
669 | for (i = 0; i < old_len; i++) | 694 | for (unsigned int i = 0; i < old_len; i++) |
670 | { | 695 | { |
671 | if (map->use_small_entries) | 696 | if (map->use_small_entries) |
672 | { | 697 | { |
diff --git a/src/zonemaster/gnunet-service-zonemaster.c b/src/zonemaster/gnunet-service-zonemaster.c index 55e1a0eee..322a1093b 100644 --- a/src/zonemaster/gnunet-service-zonemaster.c +++ b/src/zonemaster/gnunet-service-zonemaster.c | |||
@@ -63,7 +63,7 @@ | |||
63 | * The initial interval in milliseconds btween puts in | 63 | * The initial interval in milliseconds btween puts in |
64 | * a zone iteration | 64 | * a zone iteration |
65 | */ | 65 | */ |
66 | #define INITIAL_PUT_INTERVAL GNUNET_TIME_UNIT_MILLISECONDS | 66 | #define INITIAL_ZONE_ITERATION_INTERVAL GNUNET_TIME_UNIT_MILLISECONDS |
67 | 67 | ||
68 | /** | 68 | /** |
69 | * The upper bound for the zone iteration interval | 69 | * The upper bound for the zone iteration interval |
@@ -204,7 +204,7 @@ static unsigned long long put_cnt; | |||
204 | * and the total number of record sets we have (so far) | 204 | * and the total number of record sets we have (so far) |
205 | * observed in the zone. | 205 | * observed in the zone. |
206 | */ | 206 | */ |
207 | static struct GNUNET_TIME_Relative next_put_interval; | 207 | static struct GNUNET_TIME_Relative target_iteration_velocity_per_record; |
208 | 208 | ||
209 | /** | 209 | /** |
210 | * Minimum relative expiration time of records seem during the current | 210 | * Minimum relative expiration time of records seem during the current |
@@ -383,7 +383,7 @@ dht_put_monitor_continuation (void *cls) | |||
383 | 383 | ||
384 | 384 | ||
385 | /** | 385 | /** |
386 | * Calculate #next_put_interval. | 386 | * Calculate #target_iteration_velocity_per_record. |
387 | */ | 387 | */ |
388 | static void | 388 | static void |
389 | calculate_put_interval () | 389 | calculate_put_interval () |
@@ -395,7 +395,7 @@ calculate_put_interval () | |||
395 | * we can safely set the interval to the value for a single | 395 | * we can safely set the interval to the value for a single |
396 | * record | 396 | * record |
397 | */ | 397 | */ |
398 | next_put_interval = zone_publish_time_window; | 398 | target_iteration_velocity_per_record = zone_publish_time_window; |
399 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG | GNUNET_ERROR_TYPE_BULK, | 399 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG | GNUNET_ERROR_TYPE_BULK, |
400 | "No records in namestore database.\n"); | 400 | "No records in namestore database.\n"); |
401 | } | 401 | } |
@@ -408,24 +408,24 @@ calculate_put_interval () | |||
408 | = GNUNET_TIME_relative_min (GNUNET_TIME_relative_divide (last_min_relative_record_time, | 408 | = GNUNET_TIME_relative_min (GNUNET_TIME_relative_divide (last_min_relative_record_time, |
409 | PUBLISH_OPS_PER_EXPIRATION), | 409 | PUBLISH_OPS_PER_EXPIRATION), |
410 | zone_publish_time_window_default); | 410 | zone_publish_time_window_default); |
411 | next_put_interval | 411 | target_iteration_velocity_per_record |
412 | = GNUNET_TIME_relative_divide (zone_publish_time_window, | 412 | = GNUNET_TIME_relative_divide (zone_publish_time_window, |
413 | last_num_public_records); | 413 | last_num_public_records); |
414 | } | 414 | } |
415 | next_put_interval | 415 | target_iteration_velocity_per_record |
416 | = GNUNET_TIME_relative_min (next_put_interval, | 416 | = GNUNET_TIME_relative_min (target_iteration_velocity_per_record, |
417 | MAXIMUM_ZONE_ITERATION_INTERVAL); | 417 | MAXIMUM_ZONE_ITERATION_INTERVAL); |
418 | GNUNET_STATISTICS_set (statistics, | 418 | GNUNET_STATISTICS_set (statistics, |
419 | "Minimum relative record expiration (in ms)", | 419 | "Minimum relative record expiration (in μs)", |
420 | last_min_relative_record_time.rel_value_us / 1000LL, | 420 | last_min_relative_record_time.rel_value_us, |
421 | GNUNET_NO); | 421 | GNUNET_NO); |
422 | GNUNET_STATISTICS_set (statistics, | 422 | GNUNET_STATISTICS_set (statistics, |
423 | "Zone publication time window (in ms)", | 423 | "Zone publication time window (in μs)", |
424 | zone_publish_time_window.rel_value_us / 1000LL, | 424 | zone_publish_time_window.rel_value_us, |
425 | GNUNET_NO); | 425 | GNUNET_NO); |
426 | GNUNET_STATISTICS_set (statistics, | 426 | GNUNET_STATISTICS_set (statistics, |
427 | "Target zone iteration velocity (μs)", | 427 | "Target zone iteration velocity (μs)", |
428 | next_put_interval.rel_value_us, | 428 | target_iteration_velocity_per_record.rel_value_us, |
429 | GNUNET_NO); | 429 | GNUNET_NO); |
430 | } | 430 | } |
431 | 431 | ||
@@ -461,7 +461,7 @@ update_velocity (unsigned int cnt) | |||
461 | } | 461 | } |
462 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 462 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, |
463 | "Desired global zone iteration interval is %s/record!\n", | 463 | "Desired global zone iteration interval is %s/record!\n", |
464 | GNUNET_STRINGS_relative_time_to_string (next_put_interval, | 464 | GNUNET_STRINGS_relative_time_to_string (target_iteration_velocity_per_record, |
465 | GNUNET_YES)); | 465 | GNUNET_YES)); |
466 | 466 | ||
467 | /* Tell statistics actual vs. desired speed */ | 467 | /* Tell statistics actual vs. desired speed */ |
@@ -471,12 +471,12 @@ update_velocity (unsigned int cnt) | |||
471 | GNUNET_NO); | 471 | GNUNET_NO); |
472 | /* update "sub_delta" based on difference, taking | 472 | /* update "sub_delta" based on difference, taking |
473 | previous sub_delta into account! */ | 473 | previous sub_delta into account! */ |
474 | if (next_put_interval.rel_value_us > delta.rel_value_us) | 474 | if (target_iteration_velocity_per_record.rel_value_us > delta.rel_value_us) |
475 | { | 475 | { |
476 | /* We were too fast, reduce sub_delta! */ | 476 | /* We were too fast, reduce sub_delta! */ |
477 | struct GNUNET_TIME_Relative corr; | 477 | struct GNUNET_TIME_Relative corr; |
478 | 478 | ||
479 | corr = GNUNET_TIME_relative_subtract (next_put_interval, | 479 | corr = GNUNET_TIME_relative_subtract (target_iteration_velocity_per_record, |
480 | delta); | 480 | delta); |
481 | if (sub_delta.rel_value_us > delta.rel_value_us) | 481 | if (sub_delta.rel_value_us > delta.rel_value_us) |
482 | { | 482 | { |
@@ -492,28 +492,28 @@ update_velocity (unsigned int cnt) | |||
492 | sub_delta = GNUNET_TIME_UNIT_ZERO; | 492 | sub_delta = GNUNET_TIME_UNIT_ZERO; |
493 | } | 493 | } |
494 | } | 494 | } |
495 | else if (next_put_interval.rel_value_us < delta.rel_value_us) | 495 | else if (target_iteration_velocity_per_record.rel_value_us < delta.rel_value_us) |
496 | { | 496 | { |
497 | /* We were too slow, increase sub_delta! */ | 497 | /* We were too slow, increase sub_delta! */ |
498 | struct GNUNET_TIME_Relative corr; | 498 | struct GNUNET_TIME_Relative corr; |
499 | 499 | ||
500 | corr = GNUNET_TIME_relative_subtract (delta, | 500 | corr = GNUNET_TIME_relative_subtract (delta, |
501 | next_put_interval); | 501 | target_iteration_velocity_per_record); |
502 | sub_delta = GNUNET_TIME_relative_add (sub_delta, | 502 | sub_delta = GNUNET_TIME_relative_add (sub_delta, |
503 | corr); | 503 | corr); |
504 | if (sub_delta.rel_value_us > next_put_interval.rel_value_us) | 504 | if (sub_delta.rel_value_us > target_iteration_velocity_per_record.rel_value_us) |
505 | { | 505 | { |
506 | /* CPU overload detected, we cannot go at desired speed, | 506 | /* CPU overload detected, we cannot go at desired speed, |
507 | as this would mean using a negative delay. */ | 507 | as this would mean using a negative delay. */ |
508 | /* compute how much faster we would want to be for | 508 | /* compute how much faster we would want to be for |
509 | the desired velocity */ | 509 | the desired velocity */ |
510 | if (0 == next_put_interval.rel_value_us) | 510 | if (0 == target_iteration_velocity_per_record.rel_value_us) |
511 | pct = UINT64_MAX; /* desired speed is infinity ... */ | 511 | pct = UINT64_MAX; /* desired speed is infinity ... */ |
512 | else | 512 | else |
513 | pct = (sub_delta.rel_value_us - | 513 | pct = (sub_delta.rel_value_us - |
514 | next_put_interval.rel_value_us) * 100LLU | 514 | target_iteration_velocity_per_record.rel_value_us) * 100LLU |
515 | / next_put_interval.rel_value_us; | 515 | / target_iteration_velocity_per_record.rel_value_us; |
516 | sub_delta = next_put_interval; | 516 | sub_delta = target_iteration_velocity_per_record; |
517 | } | 517 | } |
518 | } | 518 | } |
519 | GNUNET_STATISTICS_set (statistics, | 519 | GNUNET_STATISTICS_set (statistics, |
@@ -548,7 +548,7 @@ check_zone_namestore_next () | |||
548 | return; /* current NAMESTORE iteration not yet done */ | 548 | return; /* current NAMESTORE iteration not yet done */ |
549 | update_velocity (put_cnt); | 549 | update_velocity (put_cnt); |
550 | put_cnt = 0; | 550 | put_cnt = 0; |
551 | delay = GNUNET_TIME_relative_subtract (next_put_interval, | 551 | delay = GNUNET_TIME_relative_subtract (target_iteration_velocity_per_record, |
552 | sub_delta); | 552 | sub_delta); |
553 | /* We delay *once* per #NS_BLOCK_SIZE, so we need to multiply the | 553 | /* We delay *once* per #NS_BLOCK_SIZE, so we need to multiply the |
554 | per-record delay calculated so far with the #NS_BLOCK_SIZE */ | 554 | per-record delay calculated so far with the #NS_BLOCK_SIZE */ |
@@ -755,11 +755,11 @@ zone_iteration_finished (void *cls) | |||
755 | PUBLISH_OPS_PER_EXPIRATION); | 755 | PUBLISH_OPS_PER_EXPIRATION); |
756 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 756 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, |
757 | "Zone iteration finished. Adjusted zone iteration interval to %s\n", | 757 | "Zone iteration finished. Adjusted zone iteration interval to %s\n", |
758 | GNUNET_STRINGS_relative_time_to_string (next_put_interval, | 758 | GNUNET_STRINGS_relative_time_to_string (target_iteration_velocity_per_record, |
759 | GNUNET_YES)); | 759 | GNUNET_YES)); |
760 | GNUNET_STATISTICS_set (statistics, | 760 | GNUNET_STATISTICS_set (statistics, |
761 | "Current zone iteration interval (in ms)", | 761 | "Target zone iteration velocity (μs)", |
762 | next_put_interval.rel_value_us / 1000LL, | 762 | target_iteration_velocity_per_record.rel_value_us, |
763 | GNUNET_NO); | 763 | GNUNET_NO); |
764 | GNUNET_STATISTICS_set (statistics, | 764 | GNUNET_STATISTICS_set (statistics, |
765 | "Number of public records in DHT", | 765 | "Number of public records in DHT", |
@@ -767,12 +767,16 @@ zone_iteration_finished (void *cls) | |||
767 | GNUNET_NO); | 767 | GNUNET_NO); |
768 | GNUNET_assert (NULL == zone_publish_task); | 768 | GNUNET_assert (NULL == zone_publish_task); |
769 | if (0 == last_num_public_records) | 769 | if (0 == last_num_public_records) |
770 | zone_publish_task = GNUNET_SCHEDULER_add_delayed (next_put_interval, | 770 | { |
771 | zone_publish_task = GNUNET_SCHEDULER_add_delayed (target_iteration_velocity_per_record, | ||
771 | &publish_zone_dht_start, | 772 | &publish_zone_dht_start, |
772 | NULL); | 773 | NULL); |
774 | } | ||
773 | else | 775 | else |
776 | { | ||
774 | zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start, | 777 | zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start, |
775 | NULL); | 778 | NULL); |
779 | } | ||
776 | } | 780 | } |
777 | 781 | ||
778 | 782 | ||
@@ -1028,7 +1032,7 @@ run (void *cls, | |||
1028 | min_relative_record_time | 1032 | min_relative_record_time |
1029 | = GNUNET_TIME_relative_multiply (GNUNET_DHT_DEFAULT_REPUBLISH_FREQUENCY, | 1033 | = GNUNET_TIME_relative_multiply (GNUNET_DHT_DEFAULT_REPUBLISH_FREQUENCY, |
1030 | PUBLISH_OPS_PER_EXPIRATION); | 1034 | PUBLISH_OPS_PER_EXPIRATION); |
1031 | next_put_interval = INITIAL_PUT_INTERVAL; | 1035 | target_iteration_velocity_per_record = INITIAL_ZONE_ITERATION_INTERVAL; |
1032 | namestore_handle = GNUNET_NAMESTORE_connect (c); | 1036 | namestore_handle = GNUNET_NAMESTORE_connect (c); |
1033 | if (NULL == namestore_handle) | 1037 | if (NULL == namestore_handle) |
1034 | { | 1038 | { |
@@ -1077,12 +1081,12 @@ run (void *cls, | |||
1077 | } | 1081 | } |
1078 | 1082 | ||
1079 | /* Schedule periodic put for our records. */ | 1083 | /* Schedule periodic put for our records. */ |
1080 | first_zone_iteration = GNUNET_YES;\ | 1084 | first_zone_iteration = GNUNET_YES; |
1081 | statistics = GNUNET_STATISTICS_create ("zonemaster", | 1085 | statistics = GNUNET_STATISTICS_create ("zonemaster", |
1082 | c); | 1086 | c); |
1083 | GNUNET_STATISTICS_set (statistics, | 1087 | GNUNET_STATISTICS_set (statistics, |
1084 | "Target zone iteration velocity (μs)", | 1088 | "Target zone iteration velocity (μs)", |
1085 | next_put_interval.rel_value_us, | 1089 | target_iteration_velocity_per_record.rel_value_us, |
1086 | GNUNET_NO); | 1090 | GNUNET_NO); |
1087 | zmon = GNUNET_NAMESTORE_zone_monitor_start (c, | 1091 | zmon = GNUNET_NAMESTORE_zone_monitor_start (c, |
1088 | NULL, | 1092 | NULL, |