diff options
author | Christian Grothoff <christian@grothoff.org> | 2011-08-03 19:53:43 +0000 |
---|---|---|
committer | Christian Grothoff <christian@grothoff.org> | 2011-08-03 19:53:43 +0000 |
commit | dd1dcb51fe9ad99b0f81598d4045d2e957834843 (patch) | |
tree | 4a983da7133685e86625f5d1579fe35d7507f34c /src/datastore/plugin_datastore_sqlite.c | |
parent | e8a32f22c8bceb389a3a6ad053d9d1043e9fa5d8 (diff) | |
download | gnunet-dd1dcb51fe9ad99b0f81598d4045d2e957834843.tar.gz gnunet-dd1dcb51fe9ad99b0f81598d4045d2e957834843.zip |
improving sqlite performance
Diffstat (limited to 'src/datastore/plugin_datastore_sqlite.c')
-rw-r--r-- | src/datastore/plugin_datastore_sqlite.c | 126 |
1 files changed, 82 insertions, 44 deletions
diff --git a/src/datastore/plugin_datastore_sqlite.c b/src/datastore/plugin_datastore_sqlite.c index fb4f2c0d9..5036004b2 100644 --- a/src/datastore/plugin_datastore_sqlite.c +++ b/src/datastore/plugin_datastore_sqlite.c | |||
@@ -93,6 +93,11 @@ struct Plugin | |||
93 | sqlite3_stmt *updPrio; | 93 | sqlite3_stmt *updPrio; |
94 | 94 | ||
95 | /** | 95 | /** |
96 | * Get maximum repl value in database. | ||
97 | */ | ||
98 | sqlite3_stmt *maxRepl; | ||
99 | |||
100 | /** | ||
96 | * Precompiled SQL for replication decrement. | 101 | * Precompiled SQL for replication decrement. |
97 | */ | 102 | */ |
98 | sqlite3_stmt *updRepl; | 103 | sqlite3_stmt *updRepl; |
@@ -181,7 +186,10 @@ create_indices (sqlite3 * dbh) | |||
181 | sqlite3_exec (dbh, "CREATE INDEX IF NOT EXISTS idx_comb ON gn090 (anonLevel ASC,expire ASC,prio,type,hash)", | 186 | sqlite3_exec (dbh, "CREATE INDEX IF NOT EXISTS idx_comb ON gn090 (anonLevel ASC,expire ASC,prio,type,hash)", |
182 | NULL, NULL, NULL)) || | 187 | NULL, NULL, NULL)) || |
183 | (SQLITE_OK != | 188 | (SQLITE_OK != |
184 | sqlite3_exec (dbh, "CREATE INDEX IF NOT EXISTS idx_expire ON gn090 (expire)", | 189 | sqlite3_exec (dbh, "CREATE INDEX IF NOT EXISTS idx_anon_type_exp ON gn090 (anonLevel ASC,type,hash)", |
190 | NULL, NULL, NULL)) || | ||
191 | (SQLITE_OK != | ||
192 | sqlite3_exec (dbh, "CREATE INDEX IF NOT EXISTS idx_expire ON gn090 (expire ASC)", | ||
185 | NULL, NULL, NULL)) || | 193 | NULL, NULL, NULL)) || |
186 | (SQLITE_OK != | 194 | (SQLITE_OK != |
187 | sqlite3_exec (dbh, "CREATE INDEX IF NOT EXISTS idx_repl_rvalue ON gn090 (repl,rvalue)", | 195 | sqlite3_exec (dbh, "CREATE INDEX IF NOT EXISTS idx_repl_rvalue ON gn090 (repl,rvalue)", |
@@ -321,61 +329,42 @@ database_setup (const struct GNUNET_CONFIGURATION_Handle *cfg, | |||
321 | create_indices (plugin->dbh); | 329 | create_indices (plugin->dbh); |
322 | 330 | ||
323 | if ((sq_prepare (plugin->dbh, | 331 | if ((sq_prepare (plugin->dbh, |
324 | "UPDATE gn090 SET prio = prio + ?, expire = MAX(expire,?) WHERE _ROWID_ = ?", | 332 | "UPDATE gn090 " |
333 | "SET prio = prio + ?, expire = MAX(expire,?) WHERE _ROWID_ = ?", | ||
325 | &plugin->updPrio) != SQLITE_OK) || | 334 | &plugin->updPrio) != SQLITE_OK) || |
326 | (sq_prepare (plugin->dbh, | 335 | (sq_prepare (plugin->dbh, |
327 | "UPDATE gn090 SET repl = MAX (0, repl - 1) WHERE _ROWID_ = ?", | 336 | "UPDATE gn090 " |
337 | "SET repl = MAX (0, repl - 1) WHERE _ROWID_ = ?", | ||
328 | &plugin->updRepl) != SQLITE_OK) || | 338 | &plugin->updRepl) != SQLITE_OK) || |
329 | #if 0 | ||
330 | /* FIXME: this is the O(n) version */ | ||
331 | (sq_prepare (plugin->dbh, | ||
332 | "SELECT type,prio,anonLevel,expire,hash,value,_ROWID_ FROM gn090" | ||
333 | " ORDER BY repl DESC, Random() LIMIT 1", | ||
334 | &plugin->selRepl) != SQLITE_OK) || | ||
335 | #elif 1 | ||
336 | /* FIXME: this gives O(n) queries, presumably because the LEFT JOIN generates | ||
337 | a temporary table with all matching expressions before the ORDER BY and LIMIT | ||
338 | clauses are applied */ | ||
339 | (sq_prepare (plugin->dbh, | 339 | (sq_prepare (plugin->dbh, |
340 | "SELECT type,prio,anonLevel,expire,hash,value,gn090._ROWID_ " | 340 | "SELECT type,prio,anonLevel,expire,hash,value,_ROWID_ " |
341 | "FROM (SELECT random() AS v) AS t1," | 341 | "FROM gn090 INDEXED BY idx_repl_rvalue " |
342 | " (SELECT MAX(repl) AS m FROM gn090 INDEXED BY idx_repl) AS t2 " | 342 | "WHERE repl=?2 AND" |
343 | " LEFT JOIN gn090 INDEXED BY idx_repl_rvalue" | 343 | " (rvalue>=?1 OR" |
344 | " ON repl=t2.m AND" | 344 | " NOT EXISTS (SELECT 1 FROM gn090 INDEXED BY idx_repl_rvalue WHERE repl=?2 AND rvalue>=?1 LIMIT 1))" |
345 | " (rvalue>=t1.v OR" | ||
346 | " NOT EXISTS (SELECT 1 FROM gn090 INDEXED BY idx_repl_rvalue WHERE repl=t2.m AND rvalue>=t1.v))" | ||
347 | " ORDER BY rvalue ASC" | 345 | " ORDER BY rvalue ASC" |
348 | " LIMIT 1 ", | 346 | " LIMIT 1", |
349 | &plugin->selRepl) != SQLITE_OK) || | 347 | &plugin->selRepl) != SQLITE_OK) || |
350 | #else | ||
351 | /* NOTE: this fails, because sqlite doesn't link the t2.m (or the t1.v) to the temporary result from before | ||
352 | (parse error during preparation) */ | ||
353 | (sq_prepare (plugin->dbh, | 348 | (sq_prepare (plugin->dbh, |
354 | "SELECT type,prio,anonLevel,expire,hash,value,_ROWID_ " | 349 | "SELECT MAX(repl) " |
355 | "FROM (SELECT random() AS v) AS t1," | 350 | "FROM gn090 INDEXED BY idx_repl_rvalue", |
356 | " (SELECT MAX(repl) AS m FROM gn090 INDEXED BY idx_repl) AS t2, " | 351 | &plugin->maxRepl) != SQLITE_OK) || |
357 | " (SELECT type,prio,anonLevel,expire,hash,value,_ROWID_ FROM gn090 INDEXED BY idx_repl_rvalue" | ||
358 | " WHERE repl=t2.m AND" /* "no such column: t2.m" */ | ||
359 | " (rvalue>=t1.v OR" | ||
360 | " NOT EXISTS (SELECT 1 FROM gn090 INDEXED BY idx_repl_rvalue WHERE repl=t2.m AND rvalue>=t1.v))" | ||
361 | " ORDER BY rvalue ASC" | ||
362 | " LIMIT 1)", | ||
363 | &plugin->selRepl) != SQLITE_OK) || | ||
364 | #endif | ||
365 | (sq_prepare (plugin->dbh, | 352 | (sq_prepare (plugin->dbh, |
366 | "SELECT type,prio,anonLevel,expire,hash,value,_ROWID_ FROM gn090 " | 353 | "SELECT type,prio,anonLevel,expire,hash,value,_ROWID_ " |
367 | " WHERE NOT EXISTS (SELECT 1 FROM gn090 WHERE expire < ?1 LIMIT 1) OR expire < ?1 " | 354 | "FROM gn090 INDEXED BY idx_expire" |
368 | " ORDER BY prio ASC LIMIT 1", | 355 | " WHERE NOT EXISTS (SELECT 1 FROM gn090 WHERE expire < ?1 LIMIT 1) OR (expire < ?1) " |
356 | " ORDER BY expire ASC LIMIT 1", | ||
369 | &plugin->selExpi) != SQLITE_OK) || | 357 | &plugin->selExpi) != SQLITE_OK) || |
370 | (sq_prepare (plugin->dbh, | 358 | (sq_prepare (plugin->dbh, |
371 | "SELECT type,prio,anonLevel,expire,hash,value,_ROWID_ FROM gn090 " | 359 | "SELECT type,prio,anonLevel,expire,hash,value,_ROWID_ " |
360 | "FROM gn090 INDEXED BY idx_anon_type_exp " | ||
372 | "WHERE (anonLevel = 0 AND type=?1) " | 361 | "WHERE (anonLevel = 0 AND type=?1) " |
373 | "ORDER BY hash DESC LIMIT 1 OFFSET ?2", | 362 | "ORDER BY hash DESC LIMIT 1 OFFSET ?2", |
374 | &plugin->selZeroAnon) != SQLITE_OK) || | 363 | &plugin->selZeroAnon) != SQLITE_OK) || |
375 | (sq_prepare (plugin->dbh, | 364 | (sq_prepare (plugin->dbh, |
376 | "INSERT INTO gn090 (repl, type, prio, " | 365 | "INSERT INTO gn090 (repl, type, prio, " |
377 | "anonLevel, expire, rvalue, hash, vhash, value) " | 366 | "anonLevel, expire, rvalue, hash, vhash, value) " |
378 | "VALUES (?, ?, ?, ?, ?, RANDOM(), ?, ?, ?)", | 367 | "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", |
379 | &plugin->insertContent) != SQLITE_OK) || | 368 | &plugin->insertContent) != SQLITE_OK) || |
380 | (sq_prepare (plugin->dbh, | 369 | (sq_prepare (plugin->dbh, |
381 | "DELETE FROM gn090 WHERE _ROWID_ = ?", | 370 | "DELETE FROM gn090 WHERE _ROWID_ = ?", |
@@ -411,6 +400,8 @@ database_shutdown (struct Plugin *plugin) | |||
411 | sqlite3_finalize (plugin->updRepl); | 400 | sqlite3_finalize (plugin->updRepl); |
412 | if (plugin->selRepl != NULL) | 401 | if (plugin->selRepl != NULL) |
413 | sqlite3_finalize (plugin->selRepl); | 402 | sqlite3_finalize (plugin->selRepl); |
403 | if (plugin->maxRepl != NULL) | ||
404 | sqlite3_finalize (plugin->maxRepl); | ||
414 | if (plugin->selExpi != NULL) | 405 | if (plugin->selExpi != NULL) |
415 | sqlite3_finalize (plugin->selExpi); | 406 | sqlite3_finalize (plugin->selExpi); |
416 | if (plugin->selZeroAnon != NULL) | 407 | if (plugin->selZeroAnon != NULL) |
@@ -524,6 +515,7 @@ sqlite_plugin_put (void *cls, | |||
524 | int ret; | 515 | int ret; |
525 | sqlite3_stmt *stmt; | 516 | sqlite3_stmt *stmt; |
526 | GNUNET_HashCode vhash; | 517 | GNUNET_HashCode vhash; |
518 | uint64_t rvalue; | ||
527 | 519 | ||
528 | if (size > MAX_ITEM_SIZE) | 520 | if (size > MAX_ITEM_SIZE) |
529 | return GNUNET_SYSERR; | 521 | return GNUNET_SYSERR; |
@@ -539,19 +531,21 @@ sqlite_plugin_put (void *cls, | |||
539 | #endif | 531 | #endif |
540 | GNUNET_CRYPTO_hash (data, size, &vhash); | 532 | GNUNET_CRYPTO_hash (data, size, &vhash); |
541 | stmt = plugin->insertContent; | 533 | stmt = plugin->insertContent; |
534 | rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, UINT64_MAX); | ||
542 | if ((SQLITE_OK != sqlite3_bind_int (stmt, 1, replication)) || | 535 | if ((SQLITE_OK != sqlite3_bind_int (stmt, 1, replication)) || |
543 | (SQLITE_OK != sqlite3_bind_int (stmt, 2, type)) || | 536 | (SQLITE_OK != sqlite3_bind_int (stmt, 2, type)) || |
544 | (SQLITE_OK != sqlite3_bind_int (stmt, 3, priority)) || | 537 | (SQLITE_OK != sqlite3_bind_int (stmt, 3, priority)) || |
545 | (SQLITE_OK != sqlite3_bind_int (stmt, 4, anonymity)) || | 538 | (SQLITE_OK != sqlite3_bind_int (stmt, 4, anonymity)) || |
546 | (SQLITE_OK != sqlite3_bind_int64 (stmt, 5, expiration.abs_value)) || | 539 | (SQLITE_OK != sqlite3_bind_int64 (stmt, 5, expiration.abs_value)) || |
540 | (SQLITE_OK != sqlite3_bind_int64 (stmt, 6, rvalue)) || | ||
547 | (SQLITE_OK != | 541 | (SQLITE_OK != |
548 | sqlite3_bind_blob (stmt, 6, key, sizeof (GNUNET_HashCode), | 542 | sqlite3_bind_blob (stmt, 7, key, sizeof (GNUNET_HashCode), |
549 | SQLITE_TRANSIENT)) || | 543 | SQLITE_TRANSIENT)) || |
550 | (SQLITE_OK != | 544 | (SQLITE_OK != |
551 | sqlite3_bind_blob (stmt, 7, &vhash, sizeof (GNUNET_HashCode), | 545 | sqlite3_bind_blob (stmt, 8, &vhash, sizeof (GNUNET_HashCode), |
552 | SQLITE_TRANSIENT)) | 546 | SQLITE_TRANSIENT)) |
553 | || (SQLITE_OK != | 547 | || (SQLITE_OK != |
554 | sqlite3_bind_blob (stmt, 8, data, size, | 548 | sqlite3_bind_blob (stmt, 9, data, size, |
555 | SQLITE_TRANSIENT))) | 549 | SQLITE_TRANSIENT))) |
556 | { | 550 | { |
557 | LOG_SQLITE (plugin, | 551 | LOG_SQLITE (plugin, |
@@ -1040,6 +1034,9 @@ sqlite_plugin_get_replication (void *cls, | |||
1040 | { | 1034 | { |
1041 | struct Plugin *plugin = cls; | 1035 | struct Plugin *plugin = cls; |
1042 | struct ReplCtx rc; | 1036 | struct ReplCtx rc; |
1037 | uint64_t rvalue; | ||
1038 | uint32_t repl; | ||
1039 | sqlite3_stmt *stmt; | ||
1043 | 1040 | ||
1044 | #if DEBUG_SQLITE | 1041 | #if DEBUG_SQLITE |
1045 | GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, | 1042 | GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, |
@@ -1049,7 +1046,48 @@ sqlite_plugin_get_replication (void *cls, | |||
1049 | rc.have_uid = GNUNET_NO; | 1046 | rc.have_uid = GNUNET_NO; |
1050 | rc.proc = proc; | 1047 | rc.proc = proc; |
1051 | rc.proc_cls = proc_cls; | 1048 | rc.proc_cls = proc_cls; |
1052 | execute_get (plugin, plugin->selRepl, &repl_proc, &rc); | 1049 | stmt = plugin->maxRepl; |
1050 | if (SQLITE_ROW != sqlite3_step (stmt)) | ||
1051 | { | ||
1052 | if (SQLITE_OK != sqlite3_reset (stmt)) | ||
1053 | LOG_SQLITE (plugin, NULL, | ||
1054 | GNUNET_ERROR_TYPE_ERROR | | ||
1055 | GNUNET_ERROR_TYPE_BULK, "sqlite3_reset"); | ||
1056 | /* DB empty */ | ||
1057 | proc (proc_cls, NULL, 0, NULL, 0, 0, 0, | ||
1058 | GNUNET_TIME_UNIT_ZERO_ABS, 0); | ||
1059 | return; | ||
1060 | } | ||
1061 | repl = sqlite3_column_int (stmt, 0); | ||
1062 | if (SQLITE_OK != sqlite3_reset (stmt)) | ||
1063 | LOG_SQLITE (plugin, NULL, | ||
1064 | GNUNET_ERROR_TYPE_ERROR | | ||
1065 | GNUNET_ERROR_TYPE_BULK, "sqlite3_reset"); | ||
1066 | stmt = plugin->selRepl; | ||
1067 | rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, UINT64_MAX); | ||
1068 | if (SQLITE_OK != sqlite3_bind_int64 (stmt, 1, rvalue)) | ||
1069 | { | ||
1070 | LOG_SQLITE (plugin, NULL, | ||
1071 | GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, "sqlite3_bind_XXXX"); | ||
1072 | if (SQLITE_OK != sqlite3_reset (stmt)) | ||
1073 | LOG_SQLITE (plugin, NULL, | ||
1074 | GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, "sqlite3_reset"); | ||
1075 | proc (proc_cls, NULL, 0, NULL, 0, 0, 0, | ||
1076 | GNUNET_TIME_UNIT_ZERO_ABS, 0); | ||
1077 | return; | ||
1078 | } | ||
1079 | if (SQLITE_OK != sqlite3_bind_int (stmt, 2, repl)) | ||
1080 | { | ||
1081 | LOG_SQLITE (plugin, NULL, | ||
1082 | GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, "sqlite3_bind_XXXX"); | ||
1083 | if (SQLITE_OK != sqlite3_reset (stmt)) | ||
1084 | LOG_SQLITE (plugin, NULL, | ||
1085 | GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, "sqlite3_reset"); | ||
1086 | proc (proc_cls, NULL, 0, NULL, 0, 0, 0, | ||
1087 | GNUNET_TIME_UNIT_ZERO_ABS, 0); | ||
1088 | return; | ||
1089 | } | ||
1090 | execute_get (plugin, stmt, &repl_proc, &rc); | ||
1053 | if (GNUNET_YES == rc.have_uid) | 1091 | if (GNUNET_YES == rc.have_uid) |
1054 | { | 1092 | { |
1055 | if (SQLITE_OK != | 1093 | if (SQLITE_OK != |