diff options
-rw-r--r-- | RATIONALE | 22 | ||||
-rw-r--r-- | src/Makefile.am | 1 | ||||
-rw-r--r-- | src/datastore/plugin_datastore.h | 8 | ||||
-rw-r--r-- | src/datastore/plugin_datastore_sqlite.c | 1195 | ||||
-rw-r--r-- | src/include/gnunet_common.h | 11 | ||||
-rw-r--r-- | src/statistics/statistics_api.c | 3 | ||||
-rw-r--r-- | src/util/common_logging.c | 18 |
7 files changed, 1217 insertions, 41 deletions
@@ -244,3 +244,25 @@ SOLUTION (draft, not done yet, details missing...): | |||
244 | => Open question: how to do this without | 244 | => Open question: how to do this without |
245 | compromising state/scalability? | 245 | compromising state/scalability? |
246 | 246 | ||
247 | PROBLEM GROUP 7 (User experience): | ||
248 | * Searches often do not return a sufficient / significant number of | ||
249 | results | ||
250 | * Sharing a directory with thousands of similar files (image/jpeg) | ||
251 | creates thousands of search results for the mime-type keyword | ||
252 | (problem with DB performance, network transmission, caching, | ||
253 | end-user display, etc.) | ||
254 | |||
255 | SOLUTION (draft, not done yet, details missing...): | ||
256 | * Canonicalize keywords (see suggestion on mailinglist end of | ||
257 | June 2009: keep consonants and sort those alphabetically); | ||
258 | while I think we must have an option to disable this feature | ||
259 | (for more private sharing), I do think it would make a reasonable | ||
260 | default | ||
261 | * When sharing directories, extract keywords first and then | ||
262 | push keywords that are common in all files up to the | ||
263 | directory level; when processing an AND-ed query and a directory | ||
264 | is found to match the result, do an inspection on the metadata | ||
265 | of the files in the directory to possibly produce further results | ||
266 | (requires downloading of the directory in the background) | ||
267 | |||
268 | |||
diff --git a/src/Makefile.am b/src/Makefile.am index 78e48e110..c366ce20c 100644 --- a/src/Makefile.am +++ b/src/Makefile.am | |||
@@ -18,7 +18,6 @@ SUBDIRS = \ | |||
18 | resolver \ | 18 | resolver \ |
19 | statistics \ | 19 | statistics \ |
20 | template \ | 20 | template \ |
21 | testing \ | ||
22 | transport \ | 21 | transport \ |
23 | core \ | 22 | core \ |
24 | $(HOSTLIST_DIR) \ | 23 | $(HOSTLIST_DIR) \ |
diff --git a/src/datastore/plugin_datastore.h b/src/datastore/plugin_datastore.h index 8c42b8b98..849188ce1 100644 --- a/src/datastore/plugin_datastore.h +++ b/src/datastore/plugin_datastore.h | |||
@@ -34,6 +34,14 @@ | |||
34 | #include "gnunet_scheduler_lib.h" | 34 | #include "gnunet_scheduler_lib.h" |
35 | #include "gnunet_datastore_service.h" | 35 | #include "gnunet_datastore_service.h" |
36 | 36 | ||
37 | |||
38 | /** | ||
39 | * How many bytes of overhead will we assume per entry | ||
40 | * in the SQlite DB? | ||
41 | */ | ||
42 | #define GNUNET_DATASTORE_ENTRY_OVERHEAD 256 | ||
43 | |||
44 | |||
37 | /** | 45 | /** |
38 | * The datastore service will pass a pointer to a struct | 46 | * The datastore service will pass a pointer to a struct |
39 | * of this type as the first and only argument to the | 47 | * of this type as the first and only argument to the |
diff --git a/src/datastore/plugin_datastore_sqlite.c b/src/datastore/plugin_datastore_sqlite.c index 6bee1a0c9..5ae03b9bb 100644 --- a/src/datastore/plugin_datastore_sqlite.c +++ b/src/datastore/plugin_datastore_sqlite.c | |||
@@ -25,7 +25,78 @@ | |||
25 | */ | 25 | */ |
26 | 26 | ||
27 | #include "platform.h" | 27 | #include "platform.h" |
28 | #include "gnunet_statistics_service.h" | ||
28 | #include "plugin_datastore.h" | 29 | #include "plugin_datastore.h" |
30 | #include <sqlite3.h> | ||
31 | |||
32 | #define DEBUG_SQLITE GNUNET_YES | ||
33 | |||
34 | /** | ||
35 | * After how many payload-changing operations | ||
36 | * do we sync our statistics? | ||
37 | */ | ||
38 | #define MAX_STAT_SYNC_LAG 50 | ||
39 | |||
40 | #define QUOTA_STAT_NAME gettext_noop ("file-sharing datastore utilization (in bytes)") | ||
41 | |||
42 | |||
43 | /** | ||
44 | * Die with an error message that indicates | ||
45 | * a failure of the command 'cmd' with the message given | ||
46 | * by strerror(errno). | ||
47 | */ | ||
48 | #define DIE_SQLITE(db, cmd) do { GNUNET_log_from(GNUNET_ERROR_TYPE_ERROR, "sqlite", _("`%s' failed at %s:%d with error: %s\n"), cmd, __FILE__, __LINE__, sqlite3_errmsg(db->dbh)); abort(); } while(0) | ||
49 | |||
50 | /** | ||
51 | * Log an error message at log-level 'level' that indicates | ||
52 | * a failure of the command 'cmd' on file 'filename' | ||
53 | * with the message given by strerror(errno). | ||
54 | */ | ||
55 | #define LOG_SQLITE(db, msg, level, cmd) do { GNUNET_log_from (level, "sqlite", _("`%s' failed at %s:%d with error: %s\n"), cmd, __FILE__, __LINE__, sqlite3_errmsg(db->dbh)); if (msg != NULL) GNUNET_asprintf(msg, _("`%s' failed with error: %s\n"), cmd, sqlite3_errmsg(db->dbh)); } while(0) | ||
56 | |||
57 | #define SELECT_IT_LOW_PRIORITY_1 \ | ||
58 | "SELECT size,type,prio,anonLevel,expire,hash,value,_ROWID_ FROM gn080 WHERE (prio = ? AND hash > ?) "\ | ||
59 | "ORDER BY hash ASC LIMIT 1" | ||
60 | |||
61 | #define SELECT_IT_LOW_PRIORITY_2 \ | ||
62 | "SELECT size,type,prio,anonLevel,expire,hash,value,_ROWID_ FROM gn080 WHERE (prio > ?) "\ | ||
63 | "ORDER BY prio ASC, hash ASC LIMIT 1" | ||
64 | |||
65 | #define SELECT_IT_NON_ANONYMOUS_1 \ | ||
66 | "SELECT size,type,prio,anonLevel,expire,hash,value,_ROWID_ FROM gn080 WHERE (prio = ? AND hash < ? AND anonLevel = 0) "\ | ||
67 | " ORDER BY hash DESC LIMIT 1" | ||
68 | |||
69 | #define SELECT_IT_NON_ANONYMOUS_2 \ | ||
70 | "SELECT size,type,prio,anonLevel,expire,hash,value,_ROWID_ FROM gn080 WHERE (prio < ? AND anonLevel = 0)"\ | ||
71 | " ORDER BY prio DESC, hash DESC LIMIT 1" | ||
72 | |||
73 | #define SELECT_IT_EXPIRATION_TIME_1 \ | ||
74 | "SELECT size,type,prio,anonLevel,expire,hash,value,_ROWID_ FROM gn080 WHERE (expire = ? AND hash > ?) "\ | ||
75 | " ORDER BY hash ASC LIMIT 1" | ||
76 | |||
77 | #define SELECT_IT_EXPIRATION_TIME_2 \ | ||
78 | "SELECT size,type,prio,anonLevel,expire,hash,value,_ROWID_ FROM gn080 WHERE (expire > ?) "\ | ||
79 | " ORDER BY expire ASC, hash ASC LIMIT 1" | ||
80 | |||
81 | #define SELECT_IT_MIGRATION_ORDER_1 \ | ||
82 | "SELECT size,type,prio,anonLevel,expire,hash,value,_ROWID_ FROM gn080 WHERE (expire = ? AND hash < ?) "\ | ||
83 | " ORDER BY hash DESC LIMIT 1" | ||
84 | |||
85 | #define SELECT_IT_MIGRATION_ORDER_2 \ | ||
86 | "SELECT size,type,prio,anonLevel,expire,hash,value,_ROWID_ FROM gn080 WHERE (expire < ?) "\ | ||
87 | " ORDER BY expire DESC, hash DESC LIMIT 1" | ||
88 | |||
89 | /** | ||
90 | * After how many ms "busy" should a DB operation fail for good? | ||
91 | * A low value makes sure that we are more responsive to requests | ||
92 | * (especially PUTs). A high value guarantees a higher success | ||
93 | * rate (SELECTs in iterate can take several seconds despite LIMIT=1). | ||
94 | * | ||
95 | * The default value of 250ms should ensure that users do not experience | ||
96 | * huge latencies while at the same time allowing operations to succeed | ||
97 | * with reasonable probability. | ||
98 | */ | ||
99 | #define BUSY_TIMEOUT_MS 250 | ||
29 | 100 | ||
30 | 101 | ||
31 | /** | 102 | /** |
@@ -37,20 +108,401 @@ struct Plugin | |||
37 | * Our execution environment. | 108 | * Our execution environment. |
38 | */ | 109 | */ |
39 | struct GNUNET_DATASTORE_PluginEnvironment *env; | 110 | struct GNUNET_DATASTORE_PluginEnvironment *env; |
111 | |||
112 | /** | ||
113 | * Database filename. | ||
114 | */ | ||
115 | char *fn; | ||
116 | |||
117 | /** | ||
118 | * Native SQLite database handle. | ||
119 | */ | ||
120 | sqlite3 *dbh; | ||
121 | |||
122 | /** | ||
123 | * Precompiled SQL for update. | ||
124 | */ | ||
125 | sqlite3_stmt *updPrio; | ||
126 | |||
127 | /** | ||
128 | * Precompiled SQL for insertion. | ||
129 | */ | ||
130 | sqlite3_stmt *insertContent; | ||
131 | |||
132 | /** | ||
133 | * Handle to the statistics service. | ||
134 | */ | ||
135 | struct GNUNET_STATISTICS_Handle *statistics; | ||
136 | |||
137 | /** | ||
138 | * How much data are we currently storing | ||
139 | * in the database? | ||
140 | */ | ||
141 | unsigned long long payload; | ||
142 | |||
143 | /** | ||
144 | * Number of updates that were made to the | ||
145 | * payload value since we last synchronized | ||
146 | * it with the statistics service. | ||
147 | */ | ||
148 | unsigned int lastSync; | ||
149 | |||
150 | /** | ||
151 | * Should the database be dropped on shutdown? | ||
152 | */ | ||
153 | int drop_on_shutdown; | ||
40 | }; | 154 | }; |
41 | 155 | ||
42 | 156 | ||
43 | /** | 157 | /** |
158 | * @brief Prepare a SQL statement | ||
159 | * | ||
160 | * @param zSql SQL statement, UTF-8 encoded | ||
161 | */ | ||
162 | static int | ||
163 | sq_prepare (sqlite3 * dbh, const char *zSql, | ||
164 | sqlite3_stmt ** ppStmt) | ||
165 | { | ||
166 | char *dummy; | ||
167 | return sqlite3_prepare (dbh, | ||
168 | zSql, | ||
169 | strlen (zSql), ppStmt, (const char **) &dummy); | ||
170 | } | ||
171 | |||
172 | |||
173 | /** | ||
174 | * Create our database indices. | ||
175 | */ | ||
176 | static void | ||
177 | create_indices (sqlite3 * dbh) | ||
178 | { | ||
179 | /* create indices */ | ||
180 | sqlite3_exec (dbh, | ||
181 | "CREATE INDEX idx_hash ON gn080 (hash)", NULL, NULL, NULL); | ||
182 | sqlite3_exec (dbh, | ||
183 | "CREATE INDEX idx_hash_vhash ON gn080 (hash,vhash)", NULL, | ||
184 | NULL, NULL); | ||
185 | sqlite3_exec (dbh, "CREATE INDEX idx_prio ON gn080 (prio)", NULL, NULL, | ||
186 | NULL); | ||
187 | sqlite3_exec (dbh, "CREATE INDEX idx_expire ON gn080 (expire)", NULL, NULL, | ||
188 | NULL); | ||
189 | sqlite3_exec (dbh, "CREATE INDEX idx_comb3 ON gn080 (prio,anonLevel)", NULL, | ||
190 | NULL, NULL); | ||
191 | sqlite3_exec (dbh, "CREATE INDEX idx_comb4 ON gn080 (prio,hash,anonLevel)", | ||
192 | NULL, NULL, NULL); | ||
193 | sqlite3_exec (dbh, "CREATE INDEX idx_comb7 ON gn080 (expire,hash)", NULL, | ||
194 | NULL, NULL); | ||
195 | } | ||
196 | |||
197 | |||
198 | |||
199 | #if 1 | ||
200 | #define CHECK(a) GNUNET_break(a) | ||
201 | #define ENULL NULL | ||
202 | #else | ||
203 | #define ENULL &e | ||
204 | #define ENULL_DEFINED 1 | ||
205 | #define CHECK(a) if (! a) { GNUNET_log(GNUNET_ERROR_TYPE_ERRROR, "%s\n", e); sqlite3_free(e); } | ||
206 | #endif | ||
207 | |||
208 | |||
209 | |||
210 | |||
211 | /** | ||
212 | * Initialize the database connections and associated | ||
213 | * data structures (create tables and indices | ||
214 | * as needed as well). | ||
215 | * | ||
216 | * @return GNUNET_OK on success | ||
217 | */ | ||
218 | static int | ||
219 | database_setup (struct GNUNET_CONFIGURATION_Handle *cfg, | ||
220 | struct Plugin *plugin) | ||
221 | { | ||
222 | sqlite3_stmt *stmt; | ||
223 | char *dir; | ||
224 | char *afsdir; | ||
225 | #if ENULL_DEFINED | ||
226 | char *e; | ||
227 | #endif | ||
228 | |||
229 | if (GNUNET_OK != | ||
230 | GNUNET_CONFIGURATION_get_value_filename (cfg, | ||
231 | "FS", | ||
232 | "DIR", | ||
233 | &afsdir)) | ||
234 | { | ||
235 | GNUNET_log_from (GNUNET_ERROR_TYPE_ERROR, | ||
236 | "sqlite", | ||
237 | _("Option `%s' in section `%s' missing in configuration!\n"), | ||
238 | "DIR", | ||
239 | "FS"); | ||
240 | return GNUNET_SYSERR; | ||
241 | } | ||
242 | GNUNET_asprintf (&dir, "%s/content/gnunet.dat", afsdir); | ||
243 | GNUNET_free (afsdir); | ||
244 | if (GNUNET_OK != GNUNET_DISK_directory_create_for_file (dir)) | ||
245 | { | ||
246 | GNUNET_break (0); | ||
247 | GNUNET_free (dir); | ||
248 | return GNUNET_SYSERR; | ||
249 | } | ||
250 | plugin->fn = GNUNET_STRINGS_to_utf8 (dir, strlen (dir), | ||
251 | #ifdef ENABLE_NLS | ||
252 | nl_langinfo (CODESET) | ||
253 | #else | ||
254 | "UTF-8" /* good luck */ | ||
255 | #endif | ||
256 | ); | ||
257 | GNUNET_free (dir); | ||
258 | |||
259 | /* Open database and precompile statements */ | ||
260 | if (sqlite3_open (plugin->fn, &plugin->dbh) != SQLITE_OK) | ||
261 | { | ||
262 | GNUNET_log_from (GNUNET_ERROR_TYPE_ERROR, | ||
263 | "sqlite", | ||
264 | _("Unable to initialize SQLite: %s.\n"), | ||
265 | sqlite3_errmsg (plugin->dbh)); | ||
266 | return GNUNET_SYSERR; | ||
267 | } | ||
268 | CHECK (SQLITE_OK == | ||
269 | sqlite3_exec (plugin->dbh, | ||
270 | "PRAGMA temp_store=MEMORY", NULL, NULL, ENULL)); | ||
271 | CHECK (SQLITE_OK == | ||
272 | sqlite3_exec (plugin->dbh, | ||
273 | "PRAGMA synchronous=OFF", NULL, NULL, ENULL)); | ||
274 | CHECK (SQLITE_OK == | ||
275 | sqlite3_exec (plugin->dbh, | ||
276 | "PRAGMA count_changes=OFF", NULL, NULL, ENULL)); | ||
277 | CHECK (SQLITE_OK == | ||
278 | sqlite3_exec (plugin->dbh, "PRAGMA page_size=4092", NULL, NULL, ENULL)); | ||
279 | |||
280 | CHECK (SQLITE_OK == sqlite3_busy_timeout (plugin->dbh, BUSY_TIMEOUT_MS)); | ||
281 | |||
282 | |||
283 | /* We have to do it here, because otherwise precompiling SQL might fail */ | ||
284 | CHECK (SQLITE_OK == | ||
285 | sq_prepare (plugin->dbh, | ||
286 | "SELECT 1 FROM sqlite_master WHERE tbl_name = 'gn080'", | ||
287 | &stmt)); | ||
288 | if ( (sqlite3_step (stmt) == SQLITE_DONE) && | ||
289 | (sqlite3_exec (plugin->dbh, | ||
290 | "CREATE TABLE gn080 (" | ||
291 | " size INTEGER NOT NULL DEFAULT 0," | ||
292 | " type INTEGER NOT NULL DEFAULT 0," | ||
293 | " prio INTEGER NOT NULL DEFAULT 0," | ||
294 | " anonLevel INTEGER NOT NULL DEFAULT 0," | ||
295 | " expire INTEGER NOT NULL DEFAULT 0," | ||
296 | " hash TEXT NOT NULL DEFAULT ''," | ||
297 | " vhash TEXT NOT NULL DEFAULT ''," | ||
298 | " value BLOB NOT NULL DEFAULT '')", NULL, NULL, | ||
299 | NULL) != SQLITE_OK) ) | ||
300 | { | ||
301 | LOG_SQLITE (plugin, NULL, | ||
302 | GNUNET_ERROR_TYPE_ERROR, | ||
303 | "sqlite3_exec"); | ||
304 | sqlite3_finalize (stmt); | ||
305 | return GNUNET_SYSERR; | ||
306 | } | ||
307 | sqlite3_finalize (stmt); | ||
308 | create_indices (plugin->dbh); | ||
309 | |||
310 | CHECK (SQLITE_OK == | ||
311 | sq_prepare (plugin->dbh, | ||
312 | "SELECT 1 FROM sqlite_master WHERE tbl_name = 'gn071'", | ||
313 | &stmt)); | ||
314 | if ( (sqlite3_step (stmt) == SQLITE_DONE) && | ||
315 | (sqlite3_exec (plugin->dbh, | ||
316 | "CREATE TABLE gn071 (" | ||
317 | " key TEXT NOT NULL DEFAULT ''," | ||
318 | " value INTEGER NOT NULL DEFAULT 0)", NULL, NULL, | ||
319 | NULL) != SQLITE_OK) ) | ||
320 | { | ||
321 | LOG_SQLITE (plugin, NULL, | ||
322 | GNUNET_ERROR_TYPE_ERROR, "sqlite3_exec"); | ||
323 | sqlite3_finalize (stmt); | ||
324 | return GNUNET_SYSERR; | ||
325 | } | ||
326 | sqlite3_finalize (stmt); | ||
327 | |||
328 | if ((sq_prepare (plugin->dbh, | ||
329 | "UPDATE gn080 SET prio = prio + ?, expire = MAX(expire,?) WHERE " | ||
330 | "_ROWID_ = ?", | ||
331 | &plugin->updPrio) != SQLITE_OK) || | ||
332 | (sq_prepare (plugin->dbh, | ||
333 | "INSERT INTO gn080 (size, type, prio, " | ||
334 | "anonLevel, expire, hash, vhash, value) VALUES " | ||
335 | "(?, ?, ?, ?, ?, ?, ?, ?)", | ||
336 | &plugin->insertContent) != SQLITE_OK)) | ||
337 | { | ||
338 | LOG_SQLITE (plugin, NULL, | ||
339 | GNUNET_ERROR_TYPE_ERROR, "precompiling"); | ||
340 | return GNUNET_SYSERR; | ||
341 | } | ||
342 | return GNUNET_OK; | ||
343 | } | ||
344 | |||
345 | |||
346 | /** | ||
347 | * Synchronize our utilization statistics with the | ||
348 | * statistics service. | ||
349 | */ | ||
350 | static void | ||
351 | sync_stats (struct Plugin *plugin) | ||
352 | { | ||
353 | GNUNET_STATISTICS_set (plugin->statistics, | ||
354 | QUOTA_STAT_NAME, | ||
355 | plugin->payload, | ||
356 | GNUNET_YES); | ||
357 | plugin->lastSync = 0; | ||
358 | } | ||
359 | |||
360 | |||
361 | /** | ||
362 | * Shutdown database connection and associate data | ||
363 | * structures. | ||
364 | */ | ||
365 | static void | ||
366 | database_shutdown (struct Plugin *plugin) | ||
367 | { | ||
368 | if (plugin->lastSync > 0) | ||
369 | sync_stats (plugin); | ||
370 | if (plugin->updPrio != NULL) | ||
371 | sqlite3_finalize (plugin->updPrio); | ||
372 | if (plugin->insertContent != NULL) | ||
373 | sqlite3_finalize (plugin->insertContent); | ||
374 | sqlite3_close (plugin->dbh); | ||
375 | GNUNET_free_non_null (plugin->fn); | ||
376 | } | ||
377 | |||
378 | |||
379 | /** | ||
44 | * Get an estimate of how much space the database is | 380 | * Get an estimate of how much space the database is |
45 | * currently using. | 381 | * currently using. |
46 | * @return number of bytes used on disk | 382 | * @return number of bytes used on disk |
47 | */ | 383 | */ |
48 | static unsigned long long sqlite_plugin_get_size (void *cls) | 384 | static unsigned long long sqlite_plugin_get_size (void *cls) |
49 | { | 385 | { |
50 | return 0; | 386 | struct Plugin *plugin = cls; |
387 | return plugin->payload; | ||
388 | } | ||
389 | |||
390 | |||
391 | /** | ||
392 | * Delete the database entry with the given | ||
393 | * row identifier. | ||
394 | */ | ||
395 | static int | ||
396 | delete_by_rowid (struct Plugin* plugin, | ||
397 | unsigned long long rid) | ||
398 | { | ||
399 | sqlite3_stmt *stmt; | ||
400 | |||
401 | if (sq_prepare (plugin->dbh, | ||
402 | "DELETE FROM gn080 WHERE _ROWID_ = ?", &stmt) != SQLITE_OK) | ||
403 | { | ||
404 | LOG_SQLITE (plugin, NULL, | ||
405 | GNUNET_ERROR_TYPE_ERROR | | ||
406 | GNUNET_ERROR_TYPE_BULK, "sq_prepare"); | ||
407 | return GNUNET_SYSERR; | ||
408 | } | ||
409 | sqlite3_bind_int64 (stmt, 1, rid); | ||
410 | if (SQLITE_DONE != sqlite3_step (stmt)) | ||
411 | { | ||
412 | LOG_SQLITE (plugin, NULL, | ||
413 | GNUNET_ERROR_TYPE_ERROR | | ||
414 | GNUNET_ERROR_TYPE_BULK, "sqlite3_step"); | ||
415 | sqlite3_finalize (stmt); | ||
416 | return GNUNET_SYSERR; | ||
417 | } | ||
418 | sqlite3_finalize (stmt); | ||
419 | return GNUNET_OK; | ||
51 | } | 420 | } |
52 | 421 | ||
53 | 422 | ||
423 | /** | ||
424 | * Context for the universal iterator. | ||
425 | */ | ||
426 | struct NextContext; | ||
427 | |||
428 | /** | ||
429 | * Type of a function that will prepare | ||
430 | * the next iteration. | ||
431 | * | ||
432 | * @param cls closure | ||
433 | * @param nc the next context; NULL for the last | ||
434 | * call which gives the callback a chance to | ||
435 | * clean up the closure | ||
436 | * @return GNUNET_OK on success, GNUNET_NO if there are | ||
437 | * no more values, GNUNET_SYSERR on error | ||
438 | */ | ||
439 | typedef int (*PrepareFunction)(void *cls, | ||
440 | struct NextContext *nc); | ||
441 | |||
442 | |||
443 | /** | ||
444 | * Context we keep for the "next request" callback. | ||
445 | */ | ||
446 | struct NextContext | ||
447 | { | ||
448 | /** | ||
449 | * Internal state. | ||
450 | */ | ||
451 | struct Plugin *plugin; | ||
452 | |||
453 | /** | ||
454 | * Function to call on the next value. | ||
455 | */ | ||
456 | PluginIterator iter; | ||
457 | |||
458 | /** | ||
459 | * Closure for iter. | ||
460 | */ | ||
461 | void *iter_cls; | ||
462 | |||
463 | /** | ||
464 | * Function to call to prepare the next | ||
465 | * iteration. | ||
466 | */ | ||
467 | PrepareFunction prep; | ||
468 | |||
469 | /** | ||
470 | * Closure for prep. | ||
471 | */ | ||
472 | void *prep_cls; | ||
473 | |||
474 | /** | ||
475 | * Statement that the iterator will get the data | ||
476 | * from (updated or set by prep). | ||
477 | */ | ||
478 | sqlite3_stmt *stmt; | ||
479 | |||
480 | /** | ||
481 | * Row ID of the last result. | ||
482 | */ | ||
483 | unsigned long long last_rowid; | ||
484 | |||
485 | /** | ||
486 | * Expiration time of the last value visited. | ||
487 | */ | ||
488 | struct GNUNET_TIME_Absolute lastExpiration; | ||
489 | |||
490 | /** | ||
491 | * Priority of the last value visited. | ||
492 | */ | ||
493 | unsigned int lastPriority; | ||
494 | |||
495 | /** | ||
496 | * Number of results processed so far. | ||
497 | */ | ||
498 | unsigned int count; | ||
499 | |||
500 | /** | ||
501 | * Set to GNUNET_YES if we must stop now. | ||
502 | */ | ||
503 | int end_it; | ||
504 | }; | ||
505 | |||
54 | 506 | ||
55 | /** | 507 | /** |
56 | * Function invoked on behalf of a "PluginIterator" | 508 | * Function invoked on behalf of a "PluginIterator" |
@@ -68,6 +520,107 @@ static void | |||
68 | sqlite_next_request (void *next_cls, | 520 | sqlite_next_request (void *next_cls, |
69 | int end_it) | 521 | int end_it) |
70 | { | 522 | { |
523 | static struct GNUNET_TIME_Absolute zero; | ||
524 | struct NextContext * nc= next_cls; | ||
525 | struct Plugin *plugin = nc->plugin; | ||
526 | unsigned long long rowid; | ||
527 | sqlite3_stmt *stmtd; | ||
528 | int ret; | ||
529 | unsigned int type; | ||
530 | unsigned int size; | ||
531 | unsigned int priority; | ||
532 | unsigned int anonymity; | ||
533 | struct GNUNET_TIME_Absolute expiration; | ||
534 | const GNUNET_HashCode *key; | ||
535 | const void *data; | ||
536 | |||
537 | sqlite3_reset (nc->stmt); | ||
538 | if ( (GNUNET_YES == end_it) || | ||
539 | (GNUNET_YES == nc->end_it) || | ||
540 | (GNUNET_OK != (nc->prep(nc->prep_cls, | ||
541 | nc))) || | ||
542 | (SQLITE_ROW != sqlite3_step (nc->stmt)) ) | ||
543 | { | ||
544 | END: | ||
545 | nc->iter (nc->iter_cls, | ||
546 | NULL, NULL, 0, NULL, 0, 0, 0, | ||
547 | zero, 0); | ||
548 | nc->prep (nc->prep_cls, NULL); | ||
549 | GNUNET_free (nc); | ||
550 | return; | ||
551 | } | ||
552 | |||
553 | rowid = sqlite3_column_int64 (nc->stmt, 7); | ||
554 | nc->last_rowid = rowid; | ||
555 | type = sqlite3_column_int (nc->stmt, 1); | ||
556 | size = sqlite3_column_bytes (nc->stmt, 6); | ||
557 | if (sqlite3_column_bytes (nc->stmt, 5) != sizeof (GNUNET_HashCode)) | ||
558 | { | ||
559 | GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING, | ||
560 | "sqlite", | ||
561 | _("Invalid data in database. Trying to fix (by deletion).\n")); | ||
562 | if (SQLITE_OK != sqlite3_reset (nc->stmt)) | ||
563 | LOG_SQLITE (nc->plugin, NULL, | ||
564 | GNUNET_ERROR_TYPE_ERROR | | ||
565 | GNUNET_ERROR_TYPE_BULK, "sqlite3_reset"); | ||
566 | if (sq_prepare | ||
567 | (nc->plugin->dbh, | ||
568 | "DELETE FROM gn080 WHERE NOT LENGTH(hash) = ?", | ||
569 | &stmtd) != SQLITE_OK) | ||
570 | { | ||
571 | LOG_SQLITE (nc->plugin, NULL, | ||
572 | GNUNET_ERROR_TYPE_ERROR | | ||
573 | GNUNET_ERROR_TYPE_BULK, | ||
574 | "sq_prepare"); | ||
575 | goto END; | ||
576 | } | ||
577 | |||
578 | if (SQLITE_OK != sqlite3_bind_int (stmtd, 1, sizeof (GNUNET_HashCode))) | ||
579 | LOG_SQLITE (nc->plugin, NULL, | ||
580 | GNUNET_ERROR_TYPE_ERROR | | ||
581 | GNUNET_ERROR_TYPE_BULK, "sqlite3_bind_int"); | ||
582 | if (SQLITE_DONE != sqlite3_step (stmtd)) | ||
583 | LOG_SQLITE (nc->plugin, NULL, | ||
584 | GNUNET_ERROR_TYPE_ERROR | | ||
585 | GNUNET_ERROR_TYPE_BULK, "sqlite3_step"); | ||
586 | if (SQLITE_OK != sqlite3_finalize (stmtd)) | ||
587 | LOG_SQLITE (nc->plugin, NULL, | ||
588 | GNUNET_ERROR_TYPE_ERROR | | ||
589 | GNUNET_ERROR_TYPE_BULK, "sqlite3_finalize"); | ||
590 | goto END; | ||
591 | } | ||
592 | |||
593 | priority = sqlite3_column_int (nc->stmt, 2); | ||
594 | nc->lastPriority = priority; | ||
595 | anonymity = sqlite3_column_int (nc->stmt, 3); | ||
596 | expiration.value = sqlite3_column_int64 (nc->stmt, 4); | ||
597 | nc->lastExpiration = expiration; | ||
598 | key = sqlite3_column_blob (nc->stmt, 5); | ||
599 | data = sqlite3_column_blob (nc->stmt, 6); | ||
600 | nc->count++; | ||
601 | ret = nc->iter (nc->iter_cls, | ||
602 | nc, | ||
603 | key, | ||
604 | size, | ||
605 | data, | ||
606 | type, | ||
607 | priority, | ||
608 | anonymity, | ||
609 | expiration, | ||
610 | rowid); | ||
611 | if (ret == GNUNET_SYSERR) | ||
612 | { | ||
613 | nc->end_it = GNUNET_YES; | ||
614 | return; | ||
615 | } | ||
616 | if ( (ret == GNUNET_NO) && | ||
617 | (GNUNET_OK == delete_by_rowid (plugin, rowid)) ) | ||
618 | { | ||
619 | plugin->payload -= (size + GNUNET_DATASTORE_ENTRY_OVERHEAD); | ||
620 | plugin->lastSync++; | ||
621 | if (plugin->lastSync >= MAX_STAT_SYNC_LAG) | ||
622 | sync_stats (plugin); | ||
623 | } | ||
71 | } | 624 | } |
72 | 625 | ||
73 | 626 | ||
@@ -96,37 +649,76 @@ sqlite_plugin_put (void *cls, | |||
96 | struct GNUNET_TIME_Absolute expiration, | 649 | struct GNUNET_TIME_Absolute expiration, |
97 | char ** msg) | 650 | char ** msg) |
98 | { | 651 | { |
99 | *msg = GNUNET_strdup("not implemented"); | 652 | struct Plugin *plugin = cls; |
100 | return GNUNET_SYSERR; | 653 | int n; |
101 | } | 654 | sqlite3_stmt *stmt; |
655 | GNUNET_HashCode vhash; | ||
102 | 656 | ||
657 | #if DEBUG_SQLITE | ||
658 | GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, | ||
659 | "sqlite", | ||
660 | "Storing in database block with type %u/key `%s'/priority %u/expiration %llu.\n", | ||
661 | type, | ||
662 | GNUNET_h2s(key), | ||
663 | priority, | ||
664 | GNUNET_TIME_absolute_get_remaining (expiration).value); | ||
665 | #endif | ||
666 | GNUNET_CRYPTO_hash (data, size, &vhash); | ||
667 | stmt = plugin->insertContent; | ||
668 | if ((SQLITE_OK != sqlite3_bind_int (stmt, 1, size)) || | ||
669 | (SQLITE_OK != sqlite3_bind_int (stmt, 2, type)) || | ||
670 | (SQLITE_OK != sqlite3_bind_int (stmt, 3, priority)) || | ||
671 | (SQLITE_OK != sqlite3_bind_int (stmt, 4, anonymity)) || | ||
672 | (SQLITE_OK != sqlite3_bind_int64 (stmt, 5, expiration.value)) || | ||
673 | (SQLITE_OK != | ||
674 | sqlite3_bind_blob (stmt, 6, key, sizeof (GNUNET_HashCode), | ||
675 | SQLITE_TRANSIENT)) || | ||
676 | (SQLITE_OK != | ||
677 | sqlite3_bind_blob (stmt, 7, &vhash, sizeof (GNUNET_HashCode), | ||
678 | SQLITE_TRANSIENT)) | ||
679 | || (SQLITE_OK != | ||
680 | sqlite3_bind_blob (stmt, 8, data, size, | ||
681 | SQLITE_TRANSIENT))) | ||
682 | { | ||
683 | LOG_SQLITE (plugin, | ||
684 | msg, | ||
685 | GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, "sqlite3_bind_XXXX"); | ||
686 | if (SQLITE_OK != sqlite3_reset (stmt)) | ||
687 | LOG_SQLITE (plugin, NULL, | ||
688 | GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, "sqlite3_reset"); | ||
689 | return GNUNET_SYSERR; | ||
690 | } | ||
103 | 691 | ||
104 | /** | 692 | n = sqlite3_step (stmt); |
105 | * Iterate over the results for a particular key | 693 | if (n != SQLITE_DONE) |
106 | * in the datastore. | 694 | { |
107 | * | 695 | if (n == SQLITE_BUSY) |
108 | * @param cls closure | 696 | { |
109 | * @param key maybe NULL (to match all entries) | 697 | LOG_SQLITE (plugin, msg, |
110 | * @param vhash hash of the value, maybe NULL (to | 698 | GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, "sqlite3_step"); |
111 | * match all values that have the right key). | 699 | sqlite3_reset (stmt); |
112 | * Note that for DBlocks there is no difference | 700 | GNUNET_break (0); |
113 | * betwen key and vhash, but for other blocks | 701 | return GNUNET_NO; |
114 | * there may be! | 702 | } |
115 | * @param type entries of which type are relevant? | 703 | LOG_SQLITE (plugin, msg, |
116 | * Use 0 for any type. | 704 | GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, "sqlite3_step"); |
117 | * @param iter function to call on each matching value; | 705 | sqlite3_reset (stmt); |
118 | * will be called once with a NULL value at the end | 706 | return GNUNET_SYSERR; |
119 | * @param iter_cls closure for iter | 707 | } |
120 | */ | 708 | if (SQLITE_OK != sqlite3_reset (stmt)) |
121 | static void | 709 | LOG_SQLITE (plugin, NULL, |
122 | sqlite_plugin_get (void *cls, | 710 | GNUNET_ERROR_TYPE_ERROR | |
123 | const GNUNET_HashCode * key, | 711 | GNUNET_ERROR_TYPE_BULK, "sqlite3_reset"); |
124 | const GNUNET_HashCode * vhash, | 712 | plugin->lastSync++; |
125 | uint32_t type, | 713 | plugin->payload += size + GNUNET_DATASTORE_ENTRY_OVERHEAD; |
126 | PluginIterator iter, void *iter_cls) | 714 | if (plugin->lastSync >= MAX_STAT_SYNC_LAG) |
127 | { | 715 | sync_stats (plugin); |
128 | static struct GNUNET_TIME_Absolute zero; | 716 | #if DEBUG_SQLITE |
129 | iter (iter_cls, NULL, NULL, 0, NULL, 0, 0, 0, zero, 0); | 717 | GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, |
718 | "sqlite", | ||
719 | "done writing content\n"); | ||
720 | #endif | ||
721 | return GNUNET_OK; | ||
130 | } | 722 | } |
131 | 723 | ||
132 | 724 | ||
@@ -158,8 +750,281 @@ sqlite_plugin_update (void *cls, | |||
158 | int delta, struct GNUNET_TIME_Absolute expire, | 750 | int delta, struct GNUNET_TIME_Absolute expire, |
159 | char **msg) | 751 | char **msg) |
160 | { | 752 | { |
161 | *msg = GNUNET_strdup ("not implemented"); | 753 | struct Plugin *plugin = cls; |
162 | return GNUNET_SYSERR; | 754 | int n; |
755 | |||
756 | sqlite3_bind_int (plugin->updPrio, 1, delta); | ||
757 | sqlite3_bind_int64 (plugin->updPrio, 2, expire.value); | ||
758 | sqlite3_bind_int64 (plugin->updPrio, 3, uid); | ||
759 | n = sqlite3_step (plugin->updPrio); | ||
760 | if (n != SQLITE_OK) | ||
761 | LOG_SQLITE (plugin, msg, | ||
762 | GNUNET_ERROR_TYPE_WARNING | GNUNET_ERROR_TYPE_BULK, | ||
763 | "sqlite3_step"); | ||
764 | #if DEBUG_SQLITE | ||
765 | else | ||
766 | GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, | ||
767 | "sqlite", | ||
768 | "Block updated\n"); | ||
769 | #endif | ||
770 | sqlite3_reset (plugin->updPrio); | ||
771 | |||
772 | if (n == SQLITE_BUSY) | ||
773 | return GNUNET_NO; | ||
774 | return n == SQLITE_OK ? GNUNET_OK : GNUNET_SYSERR; | ||
775 | } | ||
776 | |||
777 | |||
778 | struct IterContext | ||
779 | { | ||
780 | sqlite3_stmt *stmt_1; | ||
781 | sqlite3_stmt *stmt_2; | ||
782 | int is_asc; | ||
783 | int is_prio; | ||
784 | int is_migr; | ||
785 | int limit_nonanonymous; | ||
786 | uint32_t type; | ||
787 | GNUNET_HashCode key; | ||
788 | }; | ||
789 | |||
790 | |||
791 | static int | ||
792 | iter_next_prepare (void *cls, | ||
793 | struct NextContext *nc) | ||
794 | { | ||
795 | struct IterContext *ic = cls; | ||
796 | struct Plugin *plugin = nc->plugin; | ||
797 | struct GNUNET_TIME_Absolute now; | ||
798 | |||
799 | if (nc == NULL) | ||
800 | { | ||
801 | sqlite3_finalize (ic->stmt_1); | ||
802 | sqlite3_finalize (ic->stmt_2); | ||
803 | return GNUNET_SYSERR; | ||
804 | } | ||
805 | now = GNUNET_TIME_absolute_get (); | ||
806 | if (ic->is_prio) | ||
807 | { | ||
808 | sqlite3_bind_int (ic->stmt_1, 1, nc->lastPriority); | ||
809 | sqlite3_bind_int (ic->stmt_2, 1, nc->lastPriority); | ||
810 | } | ||
811 | else | ||
812 | { | ||
813 | sqlite3_bind_int64 (ic->stmt_1, 1, nc->lastExpiration.value); | ||
814 | sqlite3_bind_int64 (ic->stmt_2, 1, nc->lastExpiration.value); | ||
815 | } | ||
816 | sqlite3_bind_blob (ic->stmt_1, 2, | ||
817 | &ic->key, | ||
818 | sizeof (GNUNET_HashCode), | ||
819 | SQLITE_TRANSIENT); | ||
820 | datum_1 = NULL; | ||
821 | datum_2 = last_datum_2; | ||
822 | last_datum_2 = NULL; | ||
823 | |||
824 | if ( (SQLITE_ROW == (ret = sqlite3_step (stmt_1))) && | ||
825 | ( (GNUNET_NO == ic->is_migr) || | ||
826 | (sqlite3_column_int64 (stmt_1, 4) >= now.value) ) ) | ||
827 | { | ||
828 | nc->stmt = ic->stmt_1; | ||
829 | return GNUNET_OK; | ||
830 | } | ||
831 | if (ret != SQLITE_DONE) | ||
832 | { | ||
833 | LOG_SQLITE (plugin, NULL, | ||
834 | GNUNET_ERROR_TYPE_ERROR | | ||
835 | GNUNET_ERROR_TYPE_BULK, | ||
836 | "sqlite3_step"); | ||
837 | return GNUNET_SYSERR; | ||
838 | } | ||
839 | if (SQLITE_OK != sqlite3_reset (stmt_1)) | ||
840 | LOG_SQLITE (handle, NULL, | ||
841 | GNUNET_ERROR_TYPE_ERROR | | ||
842 | GNUNET_ERROR_TYPE_BULK, | ||
843 | "sqlite3_reset"); | ||
844 | |||
845 | if (datum_2 == NULL) | ||
846 | { | ||
847 | if ( (SQLITE_ROW == (ret = sqlite3_step (ic->stmt_2))) && | ||
848 | ( (GNUNET_NO == ic->is_migr) || | ||
849 | sqlite3_column_int64 (stmt_2, 4) >= now.value) ) | ||
850 | { | ||
851 | nc->stmt = ic->stmt_2; | ||
852 | return GNUNET_OK; | ||
853 | } | ||
854 | if (ret != SQLITE_DONE) | ||
855 | { | ||
856 | LOG_SQLITE (plugin, NULL, | ||
857 | GNUNET_ERROR_TYPE_ERROR | | ||
858 | GNUNET_ERROR_TYPE_BULK, | ||
859 | "sqlite3_step"); | ||
860 | return GNUNET_SYSERR; | ||
861 | } | ||
862 | if (SQLITE_OK != sqlite3_reset (stmt_2)) | ||
863 | LOG_SQLITE (plugin, NULL, | ||
864 | GNUNET_ERROR_TYPE_ERROR | | ||
865 | GNUNET_ERROR_TYPE_BULK, | ||
866 | "sqlite3_reset"); | ||
867 | } | ||
868 | datum = NULL; | ||
869 | if (datum_1 == NULL) | ||
870 | { | ||
871 | datum = datum_2; | ||
872 | rowid = rowid_2; | ||
873 | key = key_2; | ||
874 | } | ||
875 | else if (datum_2 == NULL) | ||
876 | { | ||
877 | datum = datum_1; | ||
878 | rowid = rowid_1; | ||
879 | key = key_1; | ||
880 | } | ||
881 | else | ||
882 | { | ||
883 | /* have to pick between 1 and 2 */ | ||
884 | if (is_prio) | ||
885 | { | ||
886 | if ((ntohl (datum_1->priority) < ntohl (datum_2->priority)) == | ||
887 | is_asc) | ||
888 | { | ||
889 | datum = datum_1; | ||
890 | rowid = rowid_1; | ||
891 | key = key_1; | ||
892 | last_datum_2 = datum_2; | ||
893 | } | ||
894 | else | ||
895 | { | ||
896 | datum = datum_2; | ||
897 | rowid = rowid_2; | ||
898 | key = key_2; | ||
899 | GNUNET_free (datum_1); | ||
900 | } | ||
901 | } | ||
902 | else | ||
903 | { | ||
904 | if ((GNUNET_ntohll (datum_1->expiration_time) < | ||
905 | GNUNET_ntohll (datum_2->expiration_time)) == is_asc) | ||
906 | { | ||
907 | datum = datum_1; | ||
908 | rowid = rowid_1; | ||
909 | key = key_1; | ||
910 | last_datum_2 = datum_2; | ||
911 | } | ||
912 | else | ||
913 | { | ||
914 | datum = datum_2; | ||
915 | rowid = rowid_2; | ||
916 | key = key_2; | ||
917 | GNUNET_free (datum_1); | ||
918 | } | ||
919 | } | ||
920 | } | ||
921 | if (datum == NULL) | ||
922 | break; | ||
923 | #if 0 | ||
924 | printf ("FOUND %4u prio %4u exp %20llu old: %4u, %20llu\n", | ||
925 | (ntohl (datum->size) - sizeof (GNUNET_DatastoreValue)), | ||
926 | ntohl (datum->priority), | ||
927 | GNUNET_ntohll (datum->expiration_time), lastPrio, lastExp); | ||
928 | #endif | ||
929 | if (((GNUNET_NO == limit_nonanonymous) || | ||
930 | (ntohl (datum->anonymity_level) == 0)) && | ||
931 | ((type == GNUNET_ECRS_BLOCKTYPE_ANY) || | ||
932 | (type == ntohl (datum->type)))) | ||
933 | { | ||
934 | count++; | ||
935 | if (iter != NULL) | ||
936 | { | ||
937 | ret = iter (&key, datum, closure, rowid); | ||
938 | if (ret == GNUNET_SYSERR) | ||
939 | { | ||
940 | GNUNET_free (datum); | ||
941 | break; | ||
942 | } | ||
943 | if (ret == GNUNET_NO) | ||
944 | { | ||
945 | payload -= getContentDatastoreSize (datum); | ||
946 | delete_by_rowid (handle, rowid); | ||
947 | } | ||
948 | } | ||
949 | } | ||
950 | } | ||
951 | |||
952 | |||
953 | /** | ||
954 | * Call a method for each key in the database and | ||
955 | * call the callback method on it. | ||
956 | * | ||
957 | * @param type entries of which type should be considered? | ||
958 | * @param iter function to call on each matching value; | ||
959 | * will be called once with a NULL value at the end | ||
960 | * @param iter_cls closure for iter | ||
961 | * @return the number of results processed, | ||
962 | * GNUNET_SYSERR on error | ||
963 | */ | ||
964 | static void | ||
965 | basic_iter (struct Plugin *plugin, | ||
966 | uint32_t type, | ||
967 | int is_asc, | ||
968 | int is_prio, | ||
969 | int is_migr, | ||
970 | int limit_nonanonymous, | ||
971 | const char *stmt_str_1, | ||
972 | const char *stmt_str_2, | ||
973 | PluginIterator iter, | ||
974 | void *iter_cls) | ||
975 | { | ||
976 | static struct GNUNET_TIME_Absolute zero; | ||
977 | struct NextContext *nc; | ||
978 | struct IterContext *ic; | ||
979 | sqlite3_stmt *stmt_1; | ||
980 | sqlite3_stmt *stmt_2; | ||
981 | |||
982 | if (sq_prepare (plugin->dbh, stmt_str_1, &stmt_1) != SQLITE_OK) | ||
983 | { | ||
984 | LOG_SQLITE (plugin, NULL, | ||
985 | GNUNET_ERROR_TYPE_ERROR | | ||
986 | GNUNET_ERROR_TYPE_BULK, "sqlite3_prepare"); | ||
987 | iter (iter_cls, NULL, NULL, 0, NULL, 0, 0, 0, zero, 0); | ||
988 | return; | ||
989 | } | ||
990 | if (sq_prepare (plugin->dbh, stmt_str_2, &stmt_2) != SQLITE_OK) | ||
991 | { | ||
992 | LOG_SQLITE (plugin, NULL, | ||
993 | GNUNET_ERROR_TYPE_ERROR | | ||
994 | GNUNET_ERROR_TYPE_BULK, "sqlite3_prepare"); | ||
995 | sqlite3_finalize (stmt_1); | ||
996 | iter (iter_cls, NULL, NULL, 0, NULL, 0, 0, 0, zero, 0); | ||
997 | return; | ||
998 | } | ||
999 | nc = GNUNET_malloc (sizeof(struct NextContext) + | ||
1000 | sizeof(struct IterContext)); | ||
1001 | nc->plugin = plugin; | ||
1002 | nc->iter = iter; | ||
1003 | nc->iter_cls = iter_cls; | ||
1004 | nc->stmt = NULL; | ||
1005 | ic = (struct IterContext*) &nc[1]; | ||
1006 | ic->stmt_1 = stmt_1; | ||
1007 | ic->stmt_2 = stmt_2; | ||
1008 | ic->type = type; | ||
1009 | ic->is_asc = is_asc; | ||
1010 | ic->is_prio = is_prio; | ||
1011 | ic->is_migr = is_migr; | ||
1012 | ic->limit_nonanonymous = limit_nonanonymous; | ||
1013 | nc->prep = &iter_next_prepare; | ||
1014 | nc->prep_cls = ic; | ||
1015 | if (is_asc) | ||
1016 | { | ||
1017 | nc->lastPriority = 0; | ||
1018 | nc->lastExpiration.value = 0; | ||
1019 | memset (&ic->key, 0, sizeof (GNUNET_HashCode)); | ||
1020 | } | ||
1021 | else | ||
1022 | { | ||
1023 | nc->lastPriority = 0x7FFFFFFF; | ||
1024 | nc->lastExpiration.value = 0x7FFFFFFFFFFFFFFFLL; | ||
1025 | memset (&ic->key, 255, sizeof (GNUNET_HashCode)); | ||
1026 | } | ||
1027 | sqlite_next_request (nc, GNUNET_NO); | ||
163 | } | 1028 | } |
164 | 1029 | ||
165 | 1030 | ||
@@ -179,8 +1044,13 @@ sqlite_plugin_iter_low_priority (void *cls, | |||
179 | PluginIterator iter, | 1044 | PluginIterator iter, |
180 | void *iter_cls) | 1045 | void *iter_cls) |
181 | { | 1046 | { |
182 | static struct GNUNET_TIME_Absolute zero; | 1047 | basic_iter (cls, |
183 | iter (iter_cls, NULL, NULL, 0, NULL, 0, 0, 0, zero, 0); | 1048 | type, |
1049 | GNUNET_YES, GNUNET_YES, | ||
1050 | GNUNET_NO, GNUNET_NO, | ||
1051 | SELECT_IT_LOW_PRIORITY_1, | ||
1052 | SELECT_IT_LOW_PRIORITY_2, | ||
1053 | iter, iter_cls); | ||
184 | } | 1054 | } |
185 | 1055 | ||
186 | 1056 | ||
@@ -271,29 +1141,262 @@ sqlite_plugin_iter_all_now (void *cls, | |||
271 | } | 1141 | } |
272 | 1142 | ||
273 | 1143 | ||
1144 | struct GetNextContext | ||
1145 | { | ||
1146 | int total; | ||
1147 | int off; | ||
1148 | int have_vhash; | ||
1149 | unsigned int type; | ||
1150 | sqlite3_stmt *stmt; | ||
1151 | GNUNET_HashCode key; | ||
1152 | GNUNET_HashCode vhash; | ||
1153 | }; | ||
1154 | |||
1155 | |||
1156 | static int | ||
1157 | get_next_prepare (void *cls, | ||
1158 | struct NextContext *nc) | ||
1159 | { | ||
1160 | struct GetNextContext *gnc = cls; | ||
1161 | int sqoff; | ||
1162 | int ret; | ||
1163 | int limit_off; | ||
1164 | |||
1165 | if (nc == NULL) | ||
1166 | { | ||
1167 | sqlite3_finalize (gnc->stmt); | ||
1168 | return GNUNET_SYSERR; | ||
1169 | } | ||
1170 | if (nc->count == gnc->total) | ||
1171 | return GNUNET_NO; | ||
1172 | if (nc->count + gnc->off == gnc->total) | ||
1173 | nc->last_rowid = 0; | ||
1174 | if (nc->count == 0) | ||
1175 | limit_off = gnc->off; | ||
1176 | else | ||
1177 | limit_off = 0; | ||
1178 | sqoff = 1; | ||
1179 | ret = sqlite3_bind_blob (nc->stmt, | ||
1180 | sqoff++, | ||
1181 | &gnc->key, | ||
1182 | sizeof (GNUNET_HashCode), | ||
1183 | SQLITE_TRANSIENT); | ||
1184 | if ((gnc->have_vhash) && (ret == SQLITE_OK)) | ||
1185 | ret = sqlite3_bind_blob (nc->stmt, | ||
1186 | sqoff++, | ||
1187 | &gnc->vhash, | ||
1188 | sizeof (GNUNET_HashCode), SQLITE_TRANSIENT); | ||
1189 | if ((gnc->type != 0) && (ret == SQLITE_OK)) | ||
1190 | ret = sqlite3_bind_int (nc->stmt, sqoff++, gnc->type); | ||
1191 | if (ret == SQLITE_OK) | ||
1192 | ret = sqlite3_bind_int64 (nc->stmt, sqoff++, nc->last_rowid + 1); | ||
1193 | if (ret == SQLITE_OK) | ||
1194 | ret = sqlite3_bind_int (nc->stmt, sqoff++, limit_off); | ||
1195 | if (ret != SQLITE_OK) | ||
1196 | return GNUNET_SYSERR; | ||
1197 | if (SQLITE_ROW != sqlite3_step (nc->stmt)) | ||
1198 | return GNUNET_NO; | ||
1199 | return GNUNET_OK; | ||
1200 | } | ||
1201 | |||
1202 | |||
1203 | /** | ||
1204 | * Iterate over the results for a particular key | ||
1205 | * in the datastore. | ||
1206 | * | ||
1207 | * @param cls closure | ||
1208 | * @param key maybe NULL (to match all entries) | ||
1209 | * @param vhash hash of the value, maybe NULL (to | ||
1210 | * match all values that have the right key). | ||
1211 | * Note that for DBlocks there is no difference | ||
1212 | * betwen key and vhash, but for other blocks | ||
1213 | * there may be! | ||
1214 | * @param type entries of which type are relevant? | ||
1215 | * Use 0 for any type. | ||
1216 | * @param iter function to call on each matching value; | ||
1217 | * will be called once with a NULL value at the end | ||
1218 | * @param iter_cls closure for iter | ||
1219 | */ | ||
1220 | static void | ||
1221 | sqlite_plugin_get (void *cls, | ||
1222 | const GNUNET_HashCode * key, | ||
1223 | const GNUNET_HashCode * vhash, | ||
1224 | uint32_t type, | ||
1225 | PluginIterator iter, void *iter_cls) | ||
1226 | { | ||
1227 | static struct GNUNET_TIME_Absolute zero; | ||
1228 | struct Plugin *plugin = cls; | ||
1229 | struct GetNextContext *gpc; | ||
1230 | struct NextContext *nc; | ||
1231 | int ret; | ||
1232 | int total; | ||
1233 | sqlite3_stmt *stmt; | ||
1234 | char scratch[256]; | ||
1235 | int sqoff; | ||
1236 | |||
1237 | GNUNET_assert (iter != NULL); | ||
1238 | if (key == NULL) | ||
1239 | { | ||
1240 | sqlite_plugin_iter_low_priority (cls, type, iter, iter_cls); | ||
1241 | return; | ||
1242 | } | ||
1243 | GNUNET_snprintf (scratch, 256, | ||
1244 | "SELECT count(*) FROM gn080 WHERE hash=:1%s%s", | ||
1245 | vhash == NULL ? "" : " AND vhash=:2", | ||
1246 | type == 0 ? "" : (vhash == | ||
1247 | NULL) ? " AND type=:2" : " AND type=:3"); | ||
1248 | if (sq_prepare (plugin->dbh, scratch, &stmt) != SQLITE_OK) | ||
1249 | { | ||
1250 | LOG_SQLITE (plugin, NULL, | ||
1251 | GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, "sqlite_prepare"); | ||
1252 | iter (iter_cls, NULL, NULL, 0, NULL, 0, 0, 0, zero, 0); | ||
1253 | return; | ||
1254 | } | ||
1255 | sqoff = 1; | ||
1256 | ret = sqlite3_bind_blob (stmt, | ||
1257 | sqoff++, | ||
1258 | key, sizeof (GNUNET_HashCode), SQLITE_TRANSIENT); | ||
1259 | if ((vhash != NULL) && (ret == SQLITE_OK)) | ||
1260 | ret = sqlite3_bind_blob (stmt, | ||
1261 | sqoff++, | ||
1262 | vhash, | ||
1263 | sizeof (GNUNET_HashCode), SQLITE_TRANSIENT); | ||
1264 | if ((type != 0) && (ret == SQLITE_OK)) | ||
1265 | ret = sqlite3_bind_int (stmt, sqoff++, type); | ||
1266 | if (SQLITE_OK != ret) | ||
1267 | { | ||
1268 | LOG_SQLITE (plugin, NULL, | ||
1269 | GNUNET_ERROR_TYPE_ERROR, "sqlite_bind"); | ||
1270 | sqlite3_reset (stmt); | ||
1271 | sqlite3_finalize (stmt); | ||
1272 | iter (iter_cls, NULL, NULL, 0, NULL, 0, 0, 0, zero, 0); | ||
1273 | return; | ||
1274 | } | ||
1275 | ret = sqlite3_step (stmt); | ||
1276 | if (ret != SQLITE_ROW) | ||
1277 | { | ||
1278 | LOG_SQLITE (plugin, NULL, | ||
1279 | GNUNET_ERROR_TYPE_ERROR| GNUNET_ERROR_TYPE_BULK, | ||
1280 | "sqlite_step"); | ||
1281 | sqlite3_reset (stmt); | ||
1282 | sqlite3_finalize (stmt); | ||
1283 | iter (iter_cls, NULL, NULL, 0, NULL, 0, 0, 0, zero, 0); | ||
1284 | return; | ||
1285 | } | ||
1286 | total = sqlite3_column_int (stmt, 0); | ||
1287 | sqlite3_reset (stmt); | ||
1288 | sqlite3_finalize (stmt); | ||
1289 | if (0 == total) | ||
1290 | { | ||
1291 | iter (iter_cls, NULL, NULL, 0, NULL, 0, 0, 0, zero, 0); | ||
1292 | return; | ||
1293 | } | ||
1294 | |||
1295 | GNUNET_snprintf (scratch, 256, | ||
1296 | "SELECT size, type, prio, anonLevel, expire, hash, value, _ROWID_ " | ||
1297 | "FROM gn080 WHERE hash=:1%s%s AND _ROWID_ >= :%d " | ||
1298 | "ORDER BY _ROWID_ ASC LIMIT 1 OFFSET :d", | ||
1299 | vhash == NULL ? "" : " AND vhash=:2", | ||
1300 | type == 0 ? "" : (vhash == | ||
1301 | NULL) ? " AND type=:2" : " AND type=:3", | ||
1302 | sqoff, sqoff + 1); | ||
1303 | if (sq_prepare (plugin->dbh, scratch, &stmt) != SQLITE_OK) | ||
1304 | { | ||
1305 | LOG_SQLITE (plugin, NULL, | ||
1306 | GNUNET_ERROR_TYPE_ERROR | | ||
1307 | GNUNET_ERROR_TYPE_BULK, "sqlite_prepare"); | ||
1308 | iter (iter_cls, NULL, NULL, 0, NULL, 0, 0, 0, zero, 0); | ||
1309 | return; | ||
1310 | } | ||
1311 | nc = GNUNET_malloc (sizeof(struct NextContext) + | ||
1312 | sizeof(struct GetNextContext)); | ||
1313 | nc->plugin = plugin; | ||
1314 | nc->iter = iter; | ||
1315 | nc->iter_cls = iter_cls; | ||
1316 | nc->stmt = stmt; | ||
1317 | gpc = (struct GetNextContext*) &nc[1]; | ||
1318 | gpc->total = total; | ||
1319 | gpc->type = type; | ||
1320 | gpc->key = *key; | ||
1321 | gpc->stmt = stmt; /* alias used for freeing at the end! */ | ||
1322 | if (NULL != vhash) | ||
1323 | { | ||
1324 | gpc->have_vhash = GNUNET_YES; | ||
1325 | gpc->vhash = *vhash; | ||
1326 | } | ||
1327 | gpc->off = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, total); | ||
1328 | nc->prep = &get_next_prepare; | ||
1329 | nc->prep_cls = gpc; | ||
1330 | sqlite_next_request (nc, GNUNET_NO); | ||
1331 | } | ||
1332 | |||
1333 | |||
274 | /** | 1334 | /** |
275 | * Drop database. | 1335 | * Drop database. |
276 | */ | 1336 | */ |
277 | static void | 1337 | static void |
278 | sqlite_plugin_drop (void *cls) | 1338 | sqlite_plugin_drop (void *cls) |
279 | { | 1339 | { |
1340 | struct Plugin *plugin = cls; | ||
1341 | plugin->drop_on_shutdown = GNUNET_YES; | ||
280 | } | 1342 | } |
281 | 1343 | ||
282 | 1344 | ||
283 | /** | 1345 | /** |
1346 | * Callback function to process statistic values. | ||
1347 | * | ||
1348 | * @param cls closure | ||
1349 | * @param subsystem name of subsystem that created the statistic | ||
1350 | * @param name the name of the datum | ||
1351 | * @param value the current value | ||
1352 | * @param is_persistent GNUNET_YES if the value is persistent, GNUNET_NO if not | ||
1353 | * @return GNUNET_OK to continue, GNUNET_SYSERR to abort iteration | ||
1354 | */ | ||
1355 | static int | ||
1356 | process_stat_in (void *cls, | ||
1357 | const char *subsystem, | ||
1358 | const char *name, | ||
1359 | unsigned long long value, | ||
1360 | int is_persistent) | ||
1361 | { | ||
1362 | struct Plugin *plugin = cls; | ||
1363 | plugin->payload += value; | ||
1364 | return GNUNET_OK; | ||
1365 | } | ||
1366 | |||
1367 | |||
1368 | /** | ||
284 | * Entry point for the plugin. | 1369 | * Entry point for the plugin. |
285 | */ | 1370 | */ |
286 | void * | 1371 | void * |
287 | libgnunet_plugin_datastore_sqlite_init (void *cls) | 1372 | libgnunet_plugin_datastore_sqlite_init (void *cls) |
288 | { | 1373 | { |
1374 | static struct Plugin plugin; | ||
289 | struct GNUNET_DATASTORE_PluginEnvironment *env = cls; | 1375 | struct GNUNET_DATASTORE_PluginEnvironment *env = cls; |
290 | struct GNUNET_DATASTORE_PluginFunctions *api; | 1376 | struct GNUNET_DATASTORE_PluginFunctions *api; |
291 | struct Plugin *plugin; | ||
292 | 1377 | ||
293 | plugin = GNUNET_malloc (sizeof (struct Plugin)); | 1378 | if (plugin.env != NULL) |
294 | plugin->env = env; | 1379 | return NULL; /* can only initialize once! */ |
1380 | memset (&plugin, 0, sizeof(struct Plugin)); | ||
1381 | plugin.env = env; | ||
1382 | plugin.statistics = GNUNET_STATISTICS_create (env->sched, | ||
1383 | "sqlite", | ||
1384 | env->cfg); | ||
1385 | GNUNET_STATISTICS_get (plugin.statistics, | ||
1386 | "sqlite", | ||
1387 | QUOTA_STAT_NAME, | ||
1388 | GNUNET_TIME_UNIT_MINUTES, | ||
1389 | NULL, | ||
1390 | &process_stat_in, | ||
1391 | &plugin); | ||
1392 | if (GNUNET_OK != | ||
1393 | database_setup (env->cfg, &plugin)) | ||
1394 | { | ||
1395 | database_shutdown (&plugin); | ||
1396 | return NULL; | ||
1397 | } | ||
295 | api = GNUNET_malloc (sizeof (struct GNUNET_DATASTORE_PluginFunctions)); | 1398 | api = GNUNET_malloc (sizeof (struct GNUNET_DATASTORE_PluginFunctions)); |
296 | api->cls = plugin; | 1399 | api->cls = &plugin; |
297 | api->get_size = &sqlite_plugin_get_size; | 1400 | api->get_size = &sqlite_plugin_get_size; |
298 | api->put = &sqlite_plugin_put; | 1401 | api->put = &sqlite_plugin_put; |
299 | api->next_request = &sqlite_next_request; | 1402 | api->next_request = &sqlite_next_request; |
@@ -317,11 +1420,25 @@ libgnunet_plugin_datastore_sqlite_init (void *cls) | |||
317 | void * | 1420 | void * |
318 | libgnunet_plugin_datastore_sqlite_done (void *cls) | 1421 | libgnunet_plugin_datastore_sqlite_done (void *cls) |
319 | { | 1422 | { |
1423 | char *fn; | ||
320 | struct GNUNET_DATASTORE_PluginFunctions *api = cls; | 1424 | struct GNUNET_DATASTORE_PluginFunctions *api = cls; |
321 | struct Plugin *plugin = api->cls; | 1425 | struct Plugin *plugin = api->cls; |
322 | 1426 | ||
323 | GNUNET_free (plugin); | 1427 | fn = NULL; |
1428 | if (plugin->drop_on_shutdown) | ||
1429 | fn = GNUNET_strdup (plugin->fn); | ||
1430 | database_shutdown (plugin); | ||
1431 | plugin->env = NULL; | ||
1432 | plugin->payload = 0; | ||
324 | GNUNET_free (api); | 1433 | GNUNET_free (api); |
1434 | if (fn != NULL) | ||
1435 | { | ||
1436 | if (0 != UNLINK(fn)) | ||
1437 | GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, | ||
1438 | "unlink", | ||
1439 | fn); | ||
1440 | GNUNET_free (fn); | ||
1441 | } | ||
325 | return NULL; | 1442 | return NULL; |
326 | } | 1443 | } |
327 | 1444 | ||
diff --git a/src/include/gnunet_common.h b/src/include/gnunet_common.h index 18358c029..aacf6328a 100644 --- a/src/include/gnunet_common.h +++ b/src/include/gnunet_common.h | |||
@@ -207,6 +207,17 @@ void GNUNET_logger_remove (GNUNET_Logger logger, void *logger_cls); | |||
207 | 207 | ||
208 | 208 | ||
209 | /** | 209 | /** |
210 | * Convert a hash value to a string (for printing debug messages). | ||
211 | * This is one of the very few calls in the entire API that is | ||
212 | * NOT reentrant! | ||
213 | * | ||
214 | * @param hc the hash code | ||
215 | * @return string | ||
216 | */ | ||
217 | const char *GNUNET_h2s (const GNUNET_HashCode *hc); | ||
218 | |||
219 | |||
220 | /** | ||
210 | * Convert a peer identity to a string (for printing debug messages). | 221 | * Convert a peer identity to a string (for printing debug messages). |
211 | * This is one of the very few calls in the entire API that is | 222 | * This is one of the very few calls in the entire API that is |
212 | * NOT reentrant! | 223 | * NOT reentrant! |
diff --git a/src/statistics/statistics_api.c b/src/statistics/statistics_api.c index be78d260a..13a091955 100644 --- a/src/statistics/statistics_api.c +++ b/src/statistics/statistics_api.c | |||
@@ -587,7 +587,8 @@ GNUNET_STATISTICS_get (struct GNUNET_STATISTICS_Handle *handle, | |||
587 | "Failed to connect to statistics service, can not get value `%s:%s'.\n", | 587 | "Failed to connect to statistics service, can not get value `%s:%s'.\n", |
588 | strlen (subsystem) ? subsystem : "*", | 588 | strlen (subsystem) ? subsystem : "*", |
589 | strlen (name) ? name : "*"); | 589 | strlen (name) ? name : "*"); |
590 | cont (cls, GNUNET_SYSERR); | 590 | if (cont != NULL) |
591 | cont (cls, GNUNET_SYSERR); | ||
591 | return; | 592 | return; |
592 | } | 593 | } |
593 | if (subsystem == NULL) | 594 | if (subsystem == NULL) |
diff --git a/src/util/common_logging.c b/src/util/common_logging.c index c77464eb5..a1a60b800 100644 --- a/src/util/common_logging.c +++ b/src/util/common_logging.c | |||
@@ -381,6 +381,24 @@ GNUNET_error_type_to_string (enum GNUNET_ErrorType kind) | |||
381 | 381 | ||
382 | 382 | ||
383 | /** | 383 | /** |
384 | * Convert a hash to a string (for printing debug messages). | ||
385 | * This is one of the very few calls in the entire API that is | ||
386 | * NOT reentrant! | ||
387 | * | ||
388 | * @param pid the peer identity | ||
389 | * @return string form; will be overwritten by next call to GNUNET_h2s. | ||
390 | */ | ||
391 | const char * | ||
392 | GNUNET_h2s (const GNUNET_HashCode *pid) | ||
393 | { | ||
394 | static struct GNUNET_CRYPTO_HashAsciiEncoded ret; | ||
395 | GNUNET_CRYPTO_hash_to_enc (pid, &ret); | ||
396 | ret.encoding[8] = '\0'; | ||
397 | return (const char *) ret.encoding; | ||
398 | } | ||
399 | |||
400 | |||
401 | /** | ||
384 | * Convert a peer identity to a string (for printing debug messages). | 402 | * Convert a peer identity to a string (for printing debug messages). |
385 | * This is one of the very few calls in the entire API that is | 403 | * This is one of the very few calls in the entire API that is |
386 | * NOT reentrant! | 404 | * NOT reentrant! |