aboutsummaryrefslogtreecommitdiff
path: root/src/datastore/perf_datastore_api.c
diff options
context:
space:
mode:
authorChristian Grothoff <christian@grothoff.org>2009-07-19 12:41:51 +0000
committerChristian Grothoff <christian@grothoff.org>2009-07-19 12:41:51 +0000
commit8de34260292a2e6f7bb980dcefd8d95f5763ad25 (patch)
treef60d024ec4f3defbb5b33398b6f32c8627caced7 /src/datastore/perf_datastore_api.c
parentbe8b1a087b70b2ee0dc905f7589199ebfa19c064 (diff)
downloadgnunet-8de34260292a2e6f7bb980dcefd8d95f5763ad25.tar.gz
gnunet-8de34260292a2e6f7bb980dcefd8d95f5763ad25.zip
passes
Diffstat (limited to 'src/datastore/perf_datastore_api.c')
-rw-r--r--src/datastore/perf_datastore_api.c64
1 files changed, 14 insertions, 50 deletions
diff --git a/src/datastore/perf_datastore_api.c b/src/datastore/perf_datastore_api.c
index a3cb7cb0f..2fa41b635 100644
--- a/src/datastore/perf_datastore_api.c
+++ b/src/datastore/perf_datastore_api.c
@@ -37,7 +37,7 @@
37#include "gnunet_protocols.h" 37#include "gnunet_protocols.h"
38#include "gnunet_datastore_service.h" 38#include "gnunet_datastore_service.h"
39 39
40#define VERBOSE GNUNET_YES 40#define VERBOSE GNUNET_NO
41 41
42/** 42/**
43 * How long until we give up on transmitting the message? 43 * How long until we give up on transmitting the message?
@@ -49,33 +49,6 @@ static struct GNUNET_DATASTORE_Handle *datastore;
49 49
50/** 50/**
51 * Target datastore size (in bytes). 51 * Target datastore size (in bytes).
52 * <p>
53 * Example impact of total size on the reported number
54 * of operations (insert and delete) per second (once
55 * roughly stabilized -- this is not "sound" experimental
56 * data but just a rough idea) for a particular machine:
57 * <pre>
58 * 4: 60 at 7k ops total
59 * 8: 50 at 3k ops total
60 * 16: 48 at 8k ops total
61 * 32: 46 at 8k ops total
62 * 64: 61 at 9k ops total
63 * 128: 89 at 9k ops total
64 * 4092: 11 at 383k ops total (12 GB stored, 14.8 GB DB size on disk, 2.5 GB reported)
65 * </pre>
66 * Pure insertion performance into an empty DB initially peaks
67 * at about 400 ops. The performance seems to drop especially
68 * once the existing (fragmented) ISAM space is filled up and
69 * the DB needs to grow on disk. This could be explained with
70 * ISAM looking more carefully for defragmentation opportunities.
71 * <p>
72 * MySQL disk space overheads (for otherwise unused database when
73 * run with 128 MB target data size; actual size 651 MB, useful
74 * data stored 520 MB) are quite large in the range of 25-30%.
75 * <p>
76 * This kind of processing seems to be IO bound (system is roughly
77 * at 90% wait, 10% CPU). This is with MySQL 5.0.
78 *
79 */ 52 */
80#define MAX_SIZE 1024LL * 1024 * 16 53#define MAX_SIZE 1024LL * 1024 * 16
81 54
@@ -83,24 +56,19 @@ static struct GNUNET_DATASTORE_Handle *datastore;
83 * Report progress outside of major reports? Should probably be GNUNET_YES if 56 * Report progress outside of major reports? Should probably be GNUNET_YES if
84 * size is > 16 MB. 57 * size is > 16 MB.
85 */ 58 */
86#define REPORT_ID GNUNET_NO 59#define REPORT_ID GNUNET_YES
87
88/**
89 * Number of put operations equivalent to 1/10th of MAX_SIZE
90 */
91#define PUT_10 MAX_SIZE / 32 / 1024 / 10
92 60
93/** 61/**
94 * Progress report frequency. 1/10th of a put operation block. 62 * Number of put operations equivalent to 1/3rd of MAX_SIZE
95 */ 63 */
96#define REP_FREQ PUT_10 / 10 64#define PUT_10 MAX_SIZE / 32 / 1024 / 3
97 65
98/** 66/**
99 * Total number of iterations (each iteration doing 67 * Total number of iterations (each iteration doing
100 * PUT_10 put operations); we report full status every 68 * PUT_10 put operations); we report full status every
101 * 10 iterations. Abort with CTRL-C. 69 * 10 iterations. Abort with CTRL-C.
102 */ 70 */
103#define ITERATIONS 100 71#define ITERATIONS 10
104 72
105 73
106static unsigned long long stored_bytes; 74static unsigned long long stored_bytes;
@@ -150,17 +118,13 @@ check_success (void *cls,
150 int success, 118 int success,
151 const char *msg) 119 const char *msg)
152{ 120{
153 static int ic;
154
155 struct CpsRunContext *crc = cls; 121 struct CpsRunContext *crc = cls;
156 if (GNUNET_OK != success) 122 if (GNUNET_OK != success)
157 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 123 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
158 "%s\n", msg); 124 "%s\n", msg);
159 GNUNET_assert (GNUNET_OK == success); 125 GNUNET_assert (GNUNET_OK == success);
160 ic++;
161#if REPORT_ID 126#if REPORT_ID
162 if (ic % REP_FREQ == 0) 127 fprintf (stderr, "I");
163 fprintf (stderr, "I");
164#endif 128#endif
165 stored_bytes += crc->size; 129 stored_bytes += crc->size;
166 stored_ops++; 130 stored_ops++;
@@ -198,11 +162,8 @@ remove_next(void *cls,
198{ 162{
199 struct CpsRunContext *crc = cls; 163 struct CpsRunContext *crc = cls;
200 164
201 static int dc;
202 dc++;
203#if REPORT_ID 165#if REPORT_ID
204 if (dc % REP_FREQ == 0) 166 fprintf (stderr, "D");
205 fprintf (stderr, "D");
206#endif 167#endif
207 GNUNET_assert (GNUNET_OK == success); 168 GNUNET_assert (GNUNET_OK == success);
208 GNUNET_SCHEDULER_add_continuation (crc->sched, 169 GNUNET_SCHEDULER_add_continuation (crc->sched,
@@ -222,6 +183,7 @@ do_delete (void *cls,
222 183
223 stored_bytes -= crc->esize; 184 stored_bytes -= crc->esize;
224 stored_entries--; 185 stored_entries--;
186 stored_ops++;
225 GNUNET_DATASTORE_remove (datastore, 187 GNUNET_DATASTORE_remove (datastore,
226 &crc->key, 188 &crc->key,
227 crc->esize, 189 crc->esize,
@@ -248,9 +210,9 @@ delete_value (void *cls,
248 210
249 if (key == NULL) 211 if (key == NULL)
250 { 212 {
251 crc->phase = RP_REPORT;
252 if (stored_bytes < MAX_SIZE) 213 if (stored_bytes < MAX_SIZE)
253 { 214 {
215 crc->phase = RP_REPORT;
254 GNUNET_SCHEDULER_add_continuation (crc->sched, 216 GNUNET_SCHEDULER_add_continuation (crc->sched,
255 GNUNET_NO, 217 GNUNET_NO,
256 &run_continuation, 218 &run_continuation,
@@ -266,6 +228,7 @@ delete_value (void *cls,
266 crc); 228 crc);
267 return; 229 return;
268 } 230 }
231 stored_ops++;
269 if (stored_bytes < MAX_SIZE) 232 if (stored_bytes < MAX_SIZE)
270 return; 233 return;
271 crc->key = *key; 234 crc->key = *key;
@@ -332,9 +295,10 @@ run_continuation (void *cls,
332#endif 295#endif
333 "Stored %llu kB / %lluk ops / %llu ops/s\n", 296 "Stored %llu kB / %lluk ops / %llu ops/s\n",
334 stored_bytes / 1024, /* used size in k */ 297 stored_bytes / 1024, /* used size in k */
335 (stored_ops * 2 - stored_entries) / 1024, /* total operations (in k) */ 298 stored_ops / 1024, /* total operations (in k) */
336 1000 * (stored_ops * 2 - stored_entries) / (1 + GNUNET_TIME_absolute_get_duration(start_time).value)); /* operations per second */ 299 1000 * stored_ops / (1 + GNUNET_TIME_absolute_get_duration(start_time).value));
337 crc->phase = RP_PUT; 300 crc->phase = RP_PUT;
301 crc->i = 0;
338 GNUNET_SCHEDULER_add_continuation (crc->sched, 302 GNUNET_SCHEDULER_add_continuation (crc->sched,
339 GNUNET_NO, 303 GNUNET_NO,
340 &run_continuation, 304 &run_continuation,
@@ -358,7 +322,7 @@ run (void *cls,
358 struct CpsRunContext *crc; 322 struct CpsRunContext *crc;
359 323
360 datastore = GNUNET_DATASTORE_connect (cfg, sched); 324 datastore = GNUNET_DATASTORE_connect (cfg, sched);
361 325 start_time = GNUNET_TIME_absolute_get ();
362 crc = GNUNET_malloc(sizeof(struct CpsRunContext)); 326 crc = GNUNET_malloc(sizeof(struct CpsRunContext));
363 crc->sched = sched; 327 crc->sched = sched;
364 crc->cfg = cfg; 328 crc->cfg = cfg;