diff options
author | Christian Grothoff <christian@grothoff.org> | 2018-05-02 01:01:57 +0200 |
---|---|---|
committer | Christian Grothoff <christian@grothoff.org> | 2018-05-02 01:01:57 +0200 |
commit | 857f8c009c4fde3d3ec9d6d0b4af48e93684fd84 (patch) | |
tree | 079b460ad7555b148720c74b3601429c662c4113 /src/zonemaster/gnunet-service-zonemaster.c | |
parent | ffe43cb1e86f49ddff73554df200853248af012d (diff) | |
download | gnunet-857f8c009c4fde3d3ec9d6d0b4af48e93684fd84.tar.gz gnunet-857f8c009c4fde3d3ec9d6d0b4af48e93684fd84.zip |
misc improvements to statistics and large-scale behavior of zoneimport/zonemaster
Diffstat (limited to 'src/zonemaster/gnunet-service-zonemaster.c')
-rw-r--r-- | src/zonemaster/gnunet-service-zonemaster.c | 66 |
1 files changed, 35 insertions, 31 deletions
diff --git a/src/zonemaster/gnunet-service-zonemaster.c b/src/zonemaster/gnunet-service-zonemaster.c index 55e1a0eee..322a1093b 100644 --- a/src/zonemaster/gnunet-service-zonemaster.c +++ b/src/zonemaster/gnunet-service-zonemaster.c | |||
@@ -63,7 +63,7 @@ | |||
63 | * The initial interval in milliseconds btween puts in | 63 | * The initial interval in milliseconds btween puts in |
64 | * a zone iteration | 64 | * a zone iteration |
65 | */ | 65 | */ |
66 | #define INITIAL_PUT_INTERVAL GNUNET_TIME_UNIT_MILLISECONDS | 66 | #define INITIAL_ZONE_ITERATION_INTERVAL GNUNET_TIME_UNIT_MILLISECONDS |
67 | 67 | ||
68 | /** | 68 | /** |
69 | * The upper bound for the zone iteration interval | 69 | * The upper bound for the zone iteration interval |
@@ -204,7 +204,7 @@ static unsigned long long put_cnt; | |||
204 | * and the total number of record sets we have (so far) | 204 | * and the total number of record sets we have (so far) |
205 | * observed in the zone. | 205 | * observed in the zone. |
206 | */ | 206 | */ |
207 | static struct GNUNET_TIME_Relative next_put_interval; | 207 | static struct GNUNET_TIME_Relative target_iteration_velocity_per_record; |
208 | 208 | ||
209 | /** | 209 | /** |
210 | * Minimum relative expiration time of records seem during the current | 210 | * Minimum relative expiration time of records seem during the current |
@@ -383,7 +383,7 @@ dht_put_monitor_continuation (void *cls) | |||
383 | 383 | ||
384 | 384 | ||
385 | /** | 385 | /** |
386 | * Calculate #next_put_interval. | 386 | * Calculate #target_iteration_velocity_per_record. |
387 | */ | 387 | */ |
388 | static void | 388 | static void |
389 | calculate_put_interval () | 389 | calculate_put_interval () |
@@ -395,7 +395,7 @@ calculate_put_interval () | |||
395 | * we can safely set the interval to the value for a single | 395 | * we can safely set the interval to the value for a single |
396 | * record | 396 | * record |
397 | */ | 397 | */ |
398 | next_put_interval = zone_publish_time_window; | 398 | target_iteration_velocity_per_record = zone_publish_time_window; |
399 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG | GNUNET_ERROR_TYPE_BULK, | 399 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG | GNUNET_ERROR_TYPE_BULK, |
400 | "No records in namestore database.\n"); | 400 | "No records in namestore database.\n"); |
401 | } | 401 | } |
@@ -408,24 +408,24 @@ calculate_put_interval () | |||
408 | = GNUNET_TIME_relative_min (GNUNET_TIME_relative_divide (last_min_relative_record_time, | 408 | = GNUNET_TIME_relative_min (GNUNET_TIME_relative_divide (last_min_relative_record_time, |
409 | PUBLISH_OPS_PER_EXPIRATION), | 409 | PUBLISH_OPS_PER_EXPIRATION), |
410 | zone_publish_time_window_default); | 410 | zone_publish_time_window_default); |
411 | next_put_interval | 411 | target_iteration_velocity_per_record |
412 | = GNUNET_TIME_relative_divide (zone_publish_time_window, | 412 | = GNUNET_TIME_relative_divide (zone_publish_time_window, |
413 | last_num_public_records); | 413 | last_num_public_records); |
414 | } | 414 | } |
415 | next_put_interval | 415 | target_iteration_velocity_per_record |
416 | = GNUNET_TIME_relative_min (next_put_interval, | 416 | = GNUNET_TIME_relative_min (target_iteration_velocity_per_record, |
417 | MAXIMUM_ZONE_ITERATION_INTERVAL); | 417 | MAXIMUM_ZONE_ITERATION_INTERVAL); |
418 | GNUNET_STATISTICS_set (statistics, | 418 | GNUNET_STATISTICS_set (statistics, |
419 | "Minimum relative record expiration (in ms)", | 419 | "Minimum relative record expiration (in μs)", |
420 | last_min_relative_record_time.rel_value_us / 1000LL, | 420 | last_min_relative_record_time.rel_value_us, |
421 | GNUNET_NO); | 421 | GNUNET_NO); |
422 | GNUNET_STATISTICS_set (statistics, | 422 | GNUNET_STATISTICS_set (statistics, |
423 | "Zone publication time window (in ms)", | 423 | "Zone publication time window (in μs)", |
424 | zone_publish_time_window.rel_value_us / 1000LL, | 424 | zone_publish_time_window.rel_value_us, |
425 | GNUNET_NO); | 425 | GNUNET_NO); |
426 | GNUNET_STATISTICS_set (statistics, | 426 | GNUNET_STATISTICS_set (statistics, |
427 | "Target zone iteration velocity (μs)", | 427 | "Target zone iteration velocity (μs)", |
428 | next_put_interval.rel_value_us, | 428 | target_iteration_velocity_per_record.rel_value_us, |
429 | GNUNET_NO); | 429 | GNUNET_NO); |
430 | } | 430 | } |
431 | 431 | ||
@@ -461,7 +461,7 @@ update_velocity (unsigned int cnt) | |||
461 | } | 461 | } |
462 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 462 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, |
463 | "Desired global zone iteration interval is %s/record!\n", | 463 | "Desired global zone iteration interval is %s/record!\n", |
464 | GNUNET_STRINGS_relative_time_to_string (next_put_interval, | 464 | GNUNET_STRINGS_relative_time_to_string (target_iteration_velocity_per_record, |
465 | GNUNET_YES)); | 465 | GNUNET_YES)); |
466 | 466 | ||
467 | /* Tell statistics actual vs. desired speed */ | 467 | /* Tell statistics actual vs. desired speed */ |
@@ -471,12 +471,12 @@ update_velocity (unsigned int cnt) | |||
471 | GNUNET_NO); | 471 | GNUNET_NO); |
472 | /* update "sub_delta" based on difference, taking | 472 | /* update "sub_delta" based on difference, taking |
473 | previous sub_delta into account! */ | 473 | previous sub_delta into account! */ |
474 | if (next_put_interval.rel_value_us > delta.rel_value_us) | 474 | if (target_iteration_velocity_per_record.rel_value_us > delta.rel_value_us) |
475 | { | 475 | { |
476 | /* We were too fast, reduce sub_delta! */ | 476 | /* We were too fast, reduce sub_delta! */ |
477 | struct GNUNET_TIME_Relative corr; | 477 | struct GNUNET_TIME_Relative corr; |
478 | 478 | ||
479 | corr = GNUNET_TIME_relative_subtract (next_put_interval, | 479 | corr = GNUNET_TIME_relative_subtract (target_iteration_velocity_per_record, |
480 | delta); | 480 | delta); |
481 | if (sub_delta.rel_value_us > delta.rel_value_us) | 481 | if (sub_delta.rel_value_us > delta.rel_value_us) |
482 | { | 482 | { |
@@ -492,28 +492,28 @@ update_velocity (unsigned int cnt) | |||
492 | sub_delta = GNUNET_TIME_UNIT_ZERO; | 492 | sub_delta = GNUNET_TIME_UNIT_ZERO; |
493 | } | 493 | } |
494 | } | 494 | } |
495 | else if (next_put_interval.rel_value_us < delta.rel_value_us) | 495 | else if (target_iteration_velocity_per_record.rel_value_us < delta.rel_value_us) |
496 | { | 496 | { |
497 | /* We were too slow, increase sub_delta! */ | 497 | /* We were too slow, increase sub_delta! */ |
498 | struct GNUNET_TIME_Relative corr; | 498 | struct GNUNET_TIME_Relative corr; |
499 | 499 | ||
500 | corr = GNUNET_TIME_relative_subtract (delta, | 500 | corr = GNUNET_TIME_relative_subtract (delta, |
501 | next_put_interval); | 501 | target_iteration_velocity_per_record); |
502 | sub_delta = GNUNET_TIME_relative_add (sub_delta, | 502 | sub_delta = GNUNET_TIME_relative_add (sub_delta, |
503 | corr); | 503 | corr); |
504 | if (sub_delta.rel_value_us > next_put_interval.rel_value_us) | 504 | if (sub_delta.rel_value_us > target_iteration_velocity_per_record.rel_value_us) |
505 | { | 505 | { |
506 | /* CPU overload detected, we cannot go at desired speed, | 506 | /* CPU overload detected, we cannot go at desired speed, |
507 | as this would mean using a negative delay. */ | 507 | as this would mean using a negative delay. */ |
508 | /* compute how much faster we would want to be for | 508 | /* compute how much faster we would want to be for |
509 | the desired velocity */ | 509 | the desired velocity */ |
510 | if (0 == next_put_interval.rel_value_us) | 510 | if (0 == target_iteration_velocity_per_record.rel_value_us) |
511 | pct = UINT64_MAX; /* desired speed is infinity ... */ | 511 | pct = UINT64_MAX; /* desired speed is infinity ... */ |
512 | else | 512 | else |
513 | pct = (sub_delta.rel_value_us - | 513 | pct = (sub_delta.rel_value_us - |
514 | next_put_interval.rel_value_us) * 100LLU | 514 | target_iteration_velocity_per_record.rel_value_us) * 100LLU |
515 | / next_put_interval.rel_value_us; | 515 | / target_iteration_velocity_per_record.rel_value_us; |
516 | sub_delta = next_put_interval; | 516 | sub_delta = target_iteration_velocity_per_record; |
517 | } | 517 | } |
518 | } | 518 | } |
519 | GNUNET_STATISTICS_set (statistics, | 519 | GNUNET_STATISTICS_set (statistics, |
@@ -548,7 +548,7 @@ check_zone_namestore_next () | |||
548 | return; /* current NAMESTORE iteration not yet done */ | 548 | return; /* current NAMESTORE iteration not yet done */ |
549 | update_velocity (put_cnt); | 549 | update_velocity (put_cnt); |
550 | put_cnt = 0; | 550 | put_cnt = 0; |
551 | delay = GNUNET_TIME_relative_subtract (next_put_interval, | 551 | delay = GNUNET_TIME_relative_subtract (target_iteration_velocity_per_record, |
552 | sub_delta); | 552 | sub_delta); |
553 | /* We delay *once* per #NS_BLOCK_SIZE, so we need to multiply the | 553 | /* We delay *once* per #NS_BLOCK_SIZE, so we need to multiply the |
554 | per-record delay calculated so far with the #NS_BLOCK_SIZE */ | 554 | per-record delay calculated so far with the #NS_BLOCK_SIZE */ |
@@ -755,11 +755,11 @@ zone_iteration_finished (void *cls) | |||
755 | PUBLISH_OPS_PER_EXPIRATION); | 755 | PUBLISH_OPS_PER_EXPIRATION); |
756 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 756 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, |
757 | "Zone iteration finished. Adjusted zone iteration interval to %s\n", | 757 | "Zone iteration finished. Adjusted zone iteration interval to %s\n", |
758 | GNUNET_STRINGS_relative_time_to_string (next_put_interval, | 758 | GNUNET_STRINGS_relative_time_to_string (target_iteration_velocity_per_record, |
759 | GNUNET_YES)); | 759 | GNUNET_YES)); |
760 | GNUNET_STATISTICS_set (statistics, | 760 | GNUNET_STATISTICS_set (statistics, |
761 | "Current zone iteration interval (in ms)", | 761 | "Target zone iteration velocity (μs)", |
762 | next_put_interval.rel_value_us / 1000LL, | 762 | target_iteration_velocity_per_record.rel_value_us, |
763 | GNUNET_NO); | 763 | GNUNET_NO); |
764 | GNUNET_STATISTICS_set (statistics, | 764 | GNUNET_STATISTICS_set (statistics, |
765 | "Number of public records in DHT", | 765 | "Number of public records in DHT", |
@@ -767,12 +767,16 @@ zone_iteration_finished (void *cls) | |||
767 | GNUNET_NO); | 767 | GNUNET_NO); |
768 | GNUNET_assert (NULL == zone_publish_task); | 768 | GNUNET_assert (NULL == zone_publish_task); |
769 | if (0 == last_num_public_records) | 769 | if (0 == last_num_public_records) |
770 | zone_publish_task = GNUNET_SCHEDULER_add_delayed (next_put_interval, | 770 | { |
771 | zone_publish_task = GNUNET_SCHEDULER_add_delayed (target_iteration_velocity_per_record, | ||
771 | &publish_zone_dht_start, | 772 | &publish_zone_dht_start, |
772 | NULL); | 773 | NULL); |
774 | } | ||
773 | else | 775 | else |
776 | { | ||
774 | zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start, | 777 | zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start, |
775 | NULL); | 778 | NULL); |
779 | } | ||
776 | } | 780 | } |
777 | 781 | ||
778 | 782 | ||
@@ -1028,7 +1032,7 @@ run (void *cls, | |||
1028 | min_relative_record_time | 1032 | min_relative_record_time |
1029 | = GNUNET_TIME_relative_multiply (GNUNET_DHT_DEFAULT_REPUBLISH_FREQUENCY, | 1033 | = GNUNET_TIME_relative_multiply (GNUNET_DHT_DEFAULT_REPUBLISH_FREQUENCY, |
1030 | PUBLISH_OPS_PER_EXPIRATION); | 1034 | PUBLISH_OPS_PER_EXPIRATION); |
1031 | next_put_interval = INITIAL_PUT_INTERVAL; | 1035 | target_iteration_velocity_per_record = INITIAL_ZONE_ITERATION_INTERVAL; |
1032 | namestore_handle = GNUNET_NAMESTORE_connect (c); | 1036 | namestore_handle = GNUNET_NAMESTORE_connect (c); |
1033 | if (NULL == namestore_handle) | 1037 | if (NULL == namestore_handle) |
1034 | { | 1038 | { |
@@ -1077,12 +1081,12 @@ run (void *cls, | |||
1077 | } | 1081 | } |
1078 | 1082 | ||
1079 | /* Schedule periodic put for our records. */ | 1083 | /* Schedule periodic put for our records. */ |
1080 | first_zone_iteration = GNUNET_YES;\ | 1084 | first_zone_iteration = GNUNET_YES; |
1081 | statistics = GNUNET_STATISTICS_create ("zonemaster", | 1085 | statistics = GNUNET_STATISTICS_create ("zonemaster", |
1082 | c); | 1086 | c); |
1083 | GNUNET_STATISTICS_set (statistics, | 1087 | GNUNET_STATISTICS_set (statistics, |
1084 | "Target zone iteration velocity (μs)", | 1088 | "Target zone iteration velocity (μs)", |
1085 | next_put_interval.rel_value_us, | 1089 | target_iteration_velocity_per_record.rel_value_us, |
1086 | GNUNET_NO); | 1090 | GNUNET_NO); |
1087 | zmon = GNUNET_NAMESTORE_zone_monitor_start (c, | 1091 | zmon = GNUNET_NAMESTORE_zone_monitor_start (c, |
1088 | NULL, | 1092 | NULL, |