diff options
-rw-r--r-- | src/fs/fs_test_lib_data.conf | 4 | ||||
-rw-r--r-- | src/fs/gnunet-service-fs.c | 23 | ||||
-rw-r--r-- | src/fs/perf_gnunet_service_fs_p2p.c | 2 | ||||
-rw-r--r-- | src/include/gnunet_load_lib.h | 9 | ||||
-rw-r--r-- | src/util/load.c | 126 |
5 files changed, 119 insertions, 45 deletions
diff --git a/src/fs/fs_test_lib_data.conf b/src/fs/fs_test_lib_data.conf index 0fd17390a..4e7dabd9d 100644 --- a/src/fs/fs_test_lib_data.conf +++ b/src/fs/fs_test_lib_data.conf | |||
@@ -41,8 +41,8 @@ PORT = 43470 | |||
41 | HOSTNAME = localhost | 41 | HOSTNAME = localhost |
42 | #TOTAL_QUOTA_IN = 9321 | 42 | #TOTAL_QUOTA_IN = 9321 |
43 | #TOTAL_QUOTA_OUT = 9321 | 43 | #TOTAL_QUOTA_OUT = 9321 |
44 | TOTAL_QUOTA_IN = 393216 | 44 | TOTAL_QUOTA_IN = 3932160 |
45 | TOTAL_QUOTA_OUT = 393216 | 45 | TOTAL_QUOTA_OUT = 3932160 |
46 | #DEBUG = YES | 46 | #DEBUG = YES |
47 | #PREFIX = valgrind --tool=memcheck --leak-check=yes | 47 | #PREFIX = valgrind --tool=memcheck --leak-check=yes |
48 | #BINARY = /home/grothoff/bin/gnunet-service-core | 48 | #BINARY = /home/grothoff/bin/gnunet-service-core |
diff --git a/src/fs/gnunet-service-fs.c b/src/fs/gnunet-service-fs.c index e46caa93f..746a69420 100644 --- a/src/fs/gnunet-service-fs.c +++ b/src/fs/gnunet-service-fs.c | |||
@@ -58,12 +58,19 @@ | |||
58 | #define SUPPORT_DELAYS GNUNET_NO | 58 | #define SUPPORT_DELAYS GNUNET_NO |
59 | 59 | ||
60 | /** | 60 | /** |
61 | * Currently experimental code... | ||
62 | */ | ||
63 | #define ENABLE_LOAD_MGMT GNUNET_YES | ||
64 | |||
65 | /** | ||
61 | * Size for the hash map for DHT requests from the FS | 66 | * Size for the hash map for DHT requests from the FS |
62 | * service. Should be about the number of concurrent | 67 | * service. Should be about the number of concurrent |
63 | * DHT requests we plan to make. | 68 | * DHT requests we plan to make. |
64 | */ | 69 | */ |
65 | #define FS_DHT_HT_SIZE 1024 | 70 | #define FS_DHT_HT_SIZE 1024 |
66 | 71 | ||
72 | #define DATASTORE_LOAD_AUTODECLINE GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS, 250) | ||
73 | |||
67 | /** | 74 | /** |
68 | * How often do we flush trust values to disk? | 75 | * How often do we flush trust values to disk? |
69 | */ | 76 | */ |
@@ -1553,7 +1560,7 @@ peer_connect_handler (void *cls, | |||
1553 | uint32_t trust; | 1560 | uint32_t trust; |
1554 | 1561 | ||
1555 | cp = GNUNET_malloc (sizeof (struct ConnectedPeer)); | 1562 | cp = GNUNET_malloc (sizeof (struct ConnectedPeer)); |
1556 | cp->transmission_delay = GNUNET_LOAD_value_init (); | 1563 | cp->transmission_delay = GNUNET_LOAD_value_init (GNUNET_CONSTANTS_IDLE_CONNECTION_TIMEOUT); |
1557 | cp->pid = GNUNET_PEER_intern (peer); | 1564 | cp->pid = GNUNET_PEER_intern (peer); |
1558 | 1565 | ||
1559 | fn = get_trust_filename (peer); | 1566 | fn = get_trust_filename (peer); |
@@ -3810,7 +3817,7 @@ process_local_reply (void *cls, | |||
3810 | GNUNET_NO); | 3817 | GNUNET_NO); |
3811 | /* FIXME: if this is activated, we might stall large downloads | 3818 | /* FIXME: if this is activated, we might stall large downloads |
3812 | indefinitely since (presumably) the load can never go down again! */ | 3819 | indefinitely since (presumably) the load can never go down again! */ |
3813 | #if 0 | 3820 | #if ENABLE_LOAD_MGMT |
3814 | GNUNET_DATASTORE_get_next (dsh, GNUNET_NO); | 3821 | GNUNET_DATASTORE_get_next (dsh, GNUNET_NO); |
3815 | return; | 3822 | return; |
3816 | #endif | 3823 | #endif |
@@ -4012,7 +4019,7 @@ handle_p2p_get (void *cls, | |||
4012 | gettext_noop ("# requests dropped due to high load"), | 4019 | gettext_noop ("# requests dropped due to high load"), |
4013 | 1, | 4020 | 1, |
4014 | GNUNET_NO); | 4021 | GNUNET_NO); |
4015 | #if 0 | 4022 | #if ENABLE_LOAD_MGMT |
4016 | /* FIXME: this causes problems... */ | 4023 | /* FIXME: this causes problems... */ |
4017 | return GNUNET_OK; | 4024 | return GNUNET_OK; |
4018 | #endif | 4025 | #endif |
@@ -4040,10 +4047,12 @@ handle_p2p_get (void *cls, | |||
4040 | /* don't have BW to send to peer, or would likely take longer than we have for it, | 4047 | /* don't have BW to send to peer, or would likely take longer than we have for it, |
4041 | so at best indirect the query */ | 4048 | so at best indirect the query */ |
4042 | priority = 0; | 4049 | priority = 0; |
4050 | #if ENABLE_LOAD_MGMT | ||
4043 | /* FIXME: if this line is enabled, the 'perf' test for larger files simply "hangs"; | 4051 | /* FIXME: if this line is enabled, the 'perf' test for larger files simply "hangs"; |
4044 | the cause seems to be that the load goes up (to the point where we do this) | 4052 | the cause seems to be that the load goes up (to the point where we do this) |
4045 | and then never goes down again... (outch) */ | 4053 | and then never goes down again... (outch) */ |
4046 | // pr->forward_only = GNUNET_YES; | 4054 | pr->forward_only = GNUNET_YES; |
4055 | #endif | ||
4047 | } | 4056 | } |
4048 | pr->type = type; | 4057 | pr->type = type; |
4049 | pr->mingle = ntohl (gm->filter_mutator); | 4058 | pr->mingle = ntohl (gm->filter_mutator); |
@@ -4442,7 +4451,7 @@ main_init (struct GNUNET_SCHEDULER_Handle *s, | |||
4442 | } | 4451 | } |
4443 | connected_peers = GNUNET_CONTAINER_multihashmap_create (enc); | 4452 | connected_peers = GNUNET_CONTAINER_multihashmap_create (enc); |
4444 | query_request_map = GNUNET_CONTAINER_multihashmap_create (max_pending_requests); | 4453 | query_request_map = GNUNET_CONTAINER_multihashmap_create (max_pending_requests); |
4445 | rt_entry_lifetime = GNUNET_LOAD_value_init (); | 4454 | rt_entry_lifetime = GNUNET_LOAD_value_init (GNUNET_TIME_UNIT_FOREVER_REL); |
4446 | peer_request_map = GNUNET_CONTAINER_multihashmap_create (enc); | 4455 | peer_request_map = GNUNET_CONTAINER_multihashmap_create (enc); |
4447 | requests_by_expiration_heap = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN); | 4456 | requests_by_expiration_heap = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN); |
4448 | core = GNUNET_CORE_connect (sched, | 4457 | core = GNUNET_CORE_connect (sched, |
@@ -4533,8 +4542,8 @@ run (void *cls, | |||
4533 | GNUNET_SCHEDULER_shutdown (sched); | 4542 | GNUNET_SCHEDULER_shutdown (sched); |
4534 | return; | 4543 | return; |
4535 | } | 4544 | } |
4536 | datastore_get_load = GNUNET_LOAD_value_init (); | 4545 | datastore_get_load = GNUNET_LOAD_value_init (DATASTORE_LOAD_AUTODECLINE); |
4537 | datastore_put_load = GNUNET_LOAD_value_init (); | 4546 | datastore_put_load = GNUNET_LOAD_value_init (DATASTORE_LOAD_AUTODECLINE); |
4538 | block_cfg = GNUNET_CONFIGURATION_create (); | 4547 | block_cfg = GNUNET_CONFIGURATION_create (); |
4539 | GNUNET_CONFIGURATION_set_value_string (block_cfg, | 4548 | GNUNET_CONFIGURATION_set_value_string (block_cfg, |
4540 | "block", | 4549 | "block", |
diff --git a/src/fs/perf_gnunet_service_fs_p2p.c b/src/fs/perf_gnunet_service_fs_p2p.c index ac4458994..1b1897ee2 100644 --- a/src/fs/perf_gnunet_service_fs_p2p.c +++ b/src/fs/perf_gnunet_service_fs_p2p.c | |||
@@ -32,7 +32,7 @@ | |||
32 | /** | 32 | /** |
33 | * File-size we use for testing. | 33 | * File-size we use for testing. |
34 | */ | 34 | */ |
35 | #define FILESIZE (1024 * 1024 * 1) | 35 | #define FILESIZE (1024 * 1024 * 500) |
36 | 36 | ||
37 | /** | 37 | /** |
38 | * How long until we give up on transmitting the message? | 38 | * How long until we give up on transmitting the message? |
diff --git a/src/include/gnunet_load_lib.h b/src/include/gnunet_load_lib.h index 7af00ccd3..7d3ec353e 100644 --- a/src/include/gnunet_load_lib.h +++ b/src/include/gnunet_load_lib.h | |||
@@ -46,10 +46,13 @@ struct GNUNET_LOAD_Value; | |||
46 | /** | 46 | /** |
47 | * Create a new load value. | 47 | * Create a new load value. |
48 | * | 48 | * |
49 | * @param autodecline speed at which this value should automatically | ||
50 | * decline in the absence of external events; at the given | ||
51 | * frequency, 0-load values will be added to the load | ||
49 | * @return the new load value | 52 | * @return the new load value |
50 | */ | 53 | */ |
51 | struct GNUNET_LOAD_Value * | 54 | struct GNUNET_LOAD_Value * |
52 | GNUNET_LOAD_value_init (void); | 55 | GNUNET_LOAD_value_init (struct GNUNET_TIME_Relative autodecline); |
53 | 56 | ||
54 | 57 | ||
55 | /** | 58 | /** |
@@ -70,7 +73,7 @@ GNUNET_LOAD_value_init (void); | |||
70 | * that we could not do proper calculations | 73 | * that we could not do proper calculations |
71 | */ | 74 | */ |
72 | double | 75 | double |
73 | GNUNET_LOAD_get_load (const struct GNUNET_LOAD_Value *load); | 76 | GNUNET_LOAD_get_load (struct GNUNET_LOAD_Value *load); |
74 | 77 | ||
75 | 78 | ||
76 | /** | 79 | /** |
@@ -80,7 +83,7 @@ GNUNET_LOAD_get_load (const struct GNUNET_LOAD_Value *load); | |||
80 | * @return zero if update was never called | 83 | * @return zero if update was never called |
81 | */ | 84 | */ |
82 | double | 85 | double |
83 | GNUNET_LOAD_get_average (const struct GNUNET_LOAD_Value *load); | 86 | GNUNET_LOAD_get_average (struct GNUNET_LOAD_Value *load); |
84 | 87 | ||
85 | 88 | ||
86 | /** | 89 | /** |
diff --git a/src/util/load.c b/src/util/load.c index 7eff40919..fd8a5afab 100644 --- a/src/util/load.c +++ b/src/util/load.c | |||
@@ -35,6 +35,16 @@ struct GNUNET_LOAD_Value | |||
35 | { | 35 | { |
36 | 36 | ||
37 | /** | 37 | /** |
38 | * How fast should the load decline if no values are added? | ||
39 | */ | ||
40 | struct GNUNET_TIME_Relative autodecline; | ||
41 | |||
42 | /** | ||
43 | * Last time this load value was updated by an event. | ||
44 | */ | ||
45 | struct GNUNET_TIME_Absolute last_update; | ||
46 | |||
47 | /** | ||
38 | * Sum of all datastore delays ever observed (in ms). Note that | 48 | * Sum of all datastore delays ever observed (in ms). Note that |
39 | * delays above 64k ms are excluded (to avoid overflow within | 49 | * delays above 64k ms are excluded (to avoid overflow within |
40 | * first 4 billion requests). | 50 | * first 4 billion requests). |
@@ -70,15 +80,89 @@ struct GNUNET_LOAD_Value | |||
70 | }; | 80 | }; |
71 | 81 | ||
72 | 82 | ||
83 | static void | ||
84 | internal_update (struct GNUNET_LOAD_Value *load) | ||
85 | { | ||
86 | struct GNUNET_TIME_Relative delta; | ||
87 | unsigned int n; | ||
88 | |||
89 | if (load->autodecline.value == GNUNET_TIME_UNIT_FOREVER_REL.value) | ||
90 | return; | ||
91 | delta = GNUNET_TIME_absolute_get_duration (load->last_update); | ||
92 | if (delta.value < load->autodecline.value) | ||
93 | return; | ||
94 | n = delta.value / load->autodecline.value; | ||
95 | if (n > 16) | ||
96 | { | ||
97 | load->runavg_delay = 0.0; | ||
98 | load->load = 0; | ||
99 | return; | ||
100 | } | ||
101 | while (n > 0) | ||
102 | { | ||
103 | n--; | ||
104 | load->runavg_delay = (load->runavg_delay * 7.0) / 8.0; | ||
105 | } | ||
106 | } | ||
107 | |||
108 | |||
73 | /** | 109 | /** |
74 | * Create a new load value. | 110 | * Create a new load value. |
75 | * | 111 | * |
112 | * @param autodecline speed at which this value should automatically | ||
113 | * decline in the absence of external events; at the given | ||
114 | * frequency, 0-load values will be added to the load | ||
76 | * @return the new load value | 115 | * @return the new load value |
77 | */ | 116 | */ |
78 | struct GNUNET_LOAD_Value * | 117 | struct GNUNET_LOAD_Value * |
79 | GNUNET_LOAD_value_init () | 118 | GNUNET_LOAD_value_init (struct GNUNET_TIME_Relative autodecline) |
80 | { | 119 | { |
81 | return GNUNET_malloc (sizeof (struct GNUNET_LOAD_Value)); | 120 | struct GNUNET_LOAD_Value *ret; |
121 | |||
122 | GNUNET_assert (autodecline.value != 0); | ||
123 | ret = GNUNET_malloc (sizeof (struct GNUNET_LOAD_Value)); | ||
124 | ret->autodecline = autodecline; | ||
125 | ret->last_update = GNUNET_TIME_absolute_get (); | ||
126 | return ret; | ||
127 | } | ||
128 | |||
129 | |||
130 | /** | ||
131 | * Recalculate our load value. | ||
132 | * | ||
133 | * @param load load to update | ||
134 | */ | ||
135 | static void | ||
136 | calculate_load (struct GNUNET_LOAD_Value *load) | ||
137 | { | ||
138 | double stddev; | ||
139 | double avgdel; | ||
140 | double sum_val_i; | ||
141 | double n; | ||
142 | double nm1; | ||
143 | |||
144 | if (load->cummulative_request_count == 0) | ||
145 | return; | ||
146 | /* calcuate std dev of latency; we have for n values of "i" that: | ||
147 | |||
148 | avg = (sum val_i) / n | ||
149 | stddev = (sum (val_i - avg)^2) / (n-1) | ||
150 | = (sum (val_i^2 - 2 avg val_i + avg^2) / (n-1) | ||
151 | = (sum (val_i^2) - 2 avg sum (val_i) + n * avg^2) / (n-1) | ||
152 | */ | ||
153 | sum_val_i = (double) load->cummulative_delay; | ||
154 | n = ((double) load->cummulative_request_count); | ||
155 | nm1 = n - 1.0; | ||
156 | avgdel = sum_val_i / n; | ||
157 | stddev = (((double) load->cummulative_squared_delay) - 2.0 * avgdel * sum_val_i + n * avgdel * avgdel) / nm1; | ||
158 | if (stddev <= 0) | ||
159 | stddev = 0.01; /* must have been rounding error or zero; prevent division by zero */ | ||
160 | /* now calculate load based on how far out we are from | ||
161 | std dev; or if we are below average, simply assume load zero */ | ||
162 | if (load->runavg_delay < avgdel) | ||
163 | load->load = 0.0; | ||
164 | else | ||
165 | load->load = (load->runavg_delay - avgdel) / stddev; | ||
82 | } | 166 | } |
83 | 167 | ||
84 | 168 | ||
@@ -92,8 +176,10 @@ GNUNET_LOAD_value_init () | |||
92 | * that we could not do proper calculations | 176 | * that we could not do proper calculations |
93 | */ | 177 | */ |
94 | double | 178 | double |
95 | GNUNET_LOAD_get_load (const struct GNUNET_LOAD_Value *load) | 179 | GNUNET_LOAD_get_load (struct GNUNET_LOAD_Value *load) |
96 | { | 180 | { |
181 | internal_update (load); | ||
182 | calculate_load (load); | ||
97 | return load->load; | 183 | return load->load; |
98 | } | 184 | } |
99 | 185 | ||
@@ -105,11 +191,12 @@ GNUNET_LOAD_get_load (const struct GNUNET_LOAD_Value *load) | |||
105 | * @return zero if update was never called | 191 | * @return zero if update was never called |
106 | */ | 192 | */ |
107 | double | 193 | double |
108 | GNUNET_LOAD_get_average (const struct GNUNET_LOAD_Value *load) | 194 | GNUNET_LOAD_get_average (struct GNUNET_LOAD_Value *load) |
109 | { | 195 | { |
110 | double n; | 196 | double n; |
111 | double sum_val_i; | 197 | double sum_val_i; |
112 | 198 | ||
199 | internal_update (load); | ||
113 | if (load->cummulative_request_count == 0) | 200 | if (load->cummulative_request_count == 0) |
114 | return 0.0; | 201 | return 0.0; |
115 | n = ((double) load->cummulative_request_count); | 202 | n = ((double) load->cummulative_request_count); |
@@ -129,12 +216,9 @@ GNUNET_LOAD_update (struct GNUNET_LOAD_Value *load, | |||
129 | uint64_t data) | 216 | uint64_t data) |
130 | { | 217 | { |
131 | uint32_t dv; | 218 | uint32_t dv; |
132 | double stddev; | ||
133 | double avgdel; | ||
134 | double sum_val_i; | ||
135 | double n; | ||
136 | double nm1; | ||
137 | 219 | ||
220 | internal_update (load); | ||
221 | load->last_update = GNUNET_TIME_absolute_get (); | ||
138 | if (data > 64 * 1024) | 222 | if (data > 64 * 1024) |
139 | { | 223 | { |
140 | /* very large */ | 224 | /* very large */ |
@@ -146,30 +230,8 @@ GNUNET_LOAD_update (struct GNUNET_LOAD_Value *load, | |||
146 | load->cummulative_squared_delay += dv * dv; | 230 | load->cummulative_squared_delay += dv * dv; |
147 | load->cummulative_request_count++; | 231 | load->cummulative_request_count++; |
148 | load->runavg_delay = ((load->runavg_delay * 7.0) + dv) / 8.0; | 232 | load->runavg_delay = ((load->runavg_delay * 7.0) + dv) / 8.0; |
149 | if (load->cummulative_request_count > 1) | ||
150 | { | ||
151 | /* calcuate std dev of latency; we have for n values of "i" that: | ||
152 | |||
153 | avg = (sum val_i) / n | ||
154 | stddev = (sum (val_i - avg)^2) / (n-1) | ||
155 | = (sum (val_i^2 - 2 avg val_i + avg^2) / (n-1) | ||
156 | = (sum (val_i^2) - 2 avg sum (val_i) + n * avg^2) / (n-1) | ||
157 | */ | ||
158 | sum_val_i = (double) load->cummulative_delay; | ||
159 | n = ((double) load->cummulative_request_count); | ||
160 | nm1 = n - 1.0; | ||
161 | avgdel = sum_val_i / n; | ||
162 | stddev = (((double) load->cummulative_squared_delay) - 2.0 * avgdel * sum_val_i + n * avgdel * avgdel) / nm1; | ||
163 | if (stddev <= 0) | ||
164 | stddev = 0.01; /* must have been rounding error or zero; prevent division by zero */ | ||
165 | /* now calculate load based on how far out we are from | ||
166 | std dev; or if we are below average, simply assume load zero */ | ||
167 | if (load->runavg_delay < avgdel) | ||
168 | load->load = 0.0; | ||
169 | else | ||
170 | load->load = (load->runavg_delay - avgdel) / stddev; | ||
171 | } | ||
172 | } | 233 | } |
173 | 234 | ||
174 | 235 | ||
236 | |||
175 | /* end of load.c */ | 237 | /* end of load.c */ |