diff options
Diffstat (limited to 'src/plugin/datastore')
17 files changed, 4918 insertions, 0 deletions
diff --git a/src/plugin/datastore/Makefile.am b/src/plugin/datastore/Makefile.am new file mode 100644 index 000000000..bde7321dd --- /dev/null +++ b/src/plugin/datastore/Makefile.am | |||
@@ -0,0 +1,144 @@ | |||
1 | # This Makefile.am is in the public domain | ||
2 | AM_CPPFLAGS = -I$(top_srcdir)/src/include | ||
3 | |||
4 | plugindir = $(libdir)/gnunet | ||
5 | |||
6 | pkgcfgdir= $(pkgdatadir)/config.d/ | ||
7 | |||
8 | libexecdir= $(pkglibdir)/libexec/ | ||
9 | |||
10 | sqldir = $(prefix)/share/gnunet/sql/ | ||
11 | |||
12 | sql_DATA = \ | ||
13 | datastore-0001.sql \ | ||
14 | datastore-drop.sql | ||
15 | |||
16 | if USE_COVERAGE | ||
17 | AM_CFLAGS = --coverage -O0 | ||
18 | XLIBS = -lgcov | ||
19 | endif | ||
20 | |||
21 | |||
22 | if HAVE_SQLITE | ||
23 | SQLITE_PLUGIN = libgnunet_plugin_datastore_sqlite.la | ||
24 | if HAVE_BENCHMARKS | ||
25 | SQLITE_BENCHMARKS = \ | ||
26 | perf_plugin_datastore_sqlite | ||
27 | endif | ||
28 | SQLITE_TESTS = \ | ||
29 | test_plugin_datastore_sqlite \ | ||
30 | $(SQLITE_BENCHMARKS) | ||
31 | endif | ||
32 | if HAVE_POSTGRESQL | ||
33 | POSTGRES_PLUGIN = libgnunet_plugin_datastore_postgres.la | ||
34 | if HAVE_BENCHMARKS | ||
35 | POSTGRES_BENCHMARKS = \ | ||
36 | perf_plugin_datastore_postgres | ||
37 | endif | ||
38 | POSTGRES_TESTS = \ | ||
39 | test_plugin_datastore_postgres \ | ||
40 | $(POSTGRES_BENCHMARKS) | ||
41 | endif | ||
42 | |||
43 | plugin_LTLIBRARIES = \ | ||
44 | $(SQLITE_PLUGIN) \ | ||
45 | $(POSTGRES_PLUGIN) \ | ||
46 | libgnunet_plugin_datastore_heap.la | ||
47 | |||
48 | # Real plugins should of course go into | ||
49 | # plugin_LTLIBRARIES | ||
50 | noinst_LTLIBRARIES = \ | ||
51 | libgnunet_plugin_datastore_template.la | ||
52 | |||
53 | |||
54 | libgnunet_plugin_datastore_sqlite_la_SOURCES = \ | ||
55 | plugin_datastore_sqlite.c | ||
56 | libgnunet_plugin_datastore_sqlite_la_LIBADD = \ | ||
57 | $(top_builddir)/src/lib/sq/libgnunetsq.la \ | ||
58 | $(top_builddir)/src/lib/util/libgnunetutil.la $(XLIBS) -lsqlite3 \ | ||
59 | $(LTLIBINTL) | ||
60 | libgnunet_plugin_datastore_sqlite_la_LDFLAGS = \ | ||
61 | $(GN_PLUGIN_LDFLAGS) | ||
62 | |||
63 | |||
64 | libgnunet_plugin_datastore_heap_la_SOURCES = \ | ||
65 | plugin_datastore_heap.c | ||
66 | libgnunet_plugin_datastore_heap_la_LIBADD = \ | ||
67 | $(top_builddir)/src/lib/util/libgnunetutil.la $(XLIBS) \ | ||
68 | $(LTLIBINTL) | ||
69 | libgnunet_plugin_datastore_heap_la_LDFLAGS = \ | ||
70 | $(GN_PLUGIN_LDFLAGS) | ||
71 | |||
72 | |||
73 | libgnunet_plugin_datastore_postgres_la_SOURCES = \ | ||
74 | plugin_datastore_postgres.c | ||
75 | libgnunet_plugin_datastore_postgres_la_LIBADD = \ | ||
76 | $(top_builddir)/src/lib/pq/libgnunetpq.la \ | ||
77 | $(top_builddir)/src/lib/util/libgnunetutil.la $(XLIBS) -lpq | ||
78 | libgnunet_plugin_datastore_postgres_la_LDFLAGS = \ | ||
79 | $(GN_PLUGIN_LDFLAGS) $(POSTGRESQL_LDFLAGS) | ||
80 | libgnunet_plugin_datastore_postgres_la_CPPFLAGS = \ | ||
81 | $(POSTGRESQL_CPPFLAGS) $(AM_CPPFLAGS) | ||
82 | |||
83 | |||
84 | libgnunet_plugin_datastore_template_la_SOURCES = \ | ||
85 | plugin_datastore_template.c | ||
86 | libgnunet_plugin_datastore_template_la_LIBADD = \ | ||
87 | $(top_builddir)/src/lib/util/libgnunetutil.la $(XLIBS) \ | ||
88 | $(LTLIBINTL) | ||
89 | libgnunet_plugin_datastore_template_la_LDFLAGS = \ | ||
90 | $(GN_PLUGIN_LDFLAGS) | ||
91 | |||
92 | check_PROGRAMS = \ | ||
93 | perf_plugin_datastore_heap \ | ||
94 | test_plugin_datastore_heap \ | ||
95 | $(SQLITE_TESTS) \ | ||
96 | $(POSTGRES_TESTS) | ||
97 | |||
98 | if ENABLE_TEST_RUN | ||
99 | AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME; | ||
100 | TESTS = $(check_PROGRAMS) | ||
101 | endif | ||
102 | |||
103 | perf_plugin_datastore_heap_SOURCES = \ | ||
104 | perf_plugin_datastore.c | ||
105 | perf_plugin_datastore_heap_LDADD = \ | ||
106 | $(top_builddir)/src/lib/util/libgnunetutil.la | ||
107 | |||
108 | test_plugin_datastore_heap_SOURCES = \ | ||
109 | test_plugin_datastore.c | ||
110 | test_plugin_datastore_heap_LDADD = \ | ||
111 | $(top_builddir)/src/lib/util/libgnunetutil.la | ||
112 | |||
113 | |||
114 | perf_plugin_datastore_sqlite_SOURCES = \ | ||
115 | perf_plugin_datastore.c | ||
116 | perf_plugin_datastore_sqlite_LDADD = \ | ||
117 | $(top_builddir)/src/lib/util/libgnunetutil.la | ||
118 | |||
119 | test_plugin_datastore_sqlite_SOURCES = \ | ||
120 | test_plugin_datastore.c | ||
121 | test_plugin_datastore_sqlite_LDADD = \ | ||
122 | $(top_builddir)/src/lib/util/libgnunetutil.la | ||
123 | |||
124 | |||
125 | test_plugin_datastore_postgres_SOURCES = \ | ||
126 | test_plugin_datastore.c | ||
127 | test_plugin_datastore_postgres_LDADD = \ | ||
128 | $(top_builddir)/src/lib/util/libgnunetutil.la | ||
129 | |||
130 | perf_plugin_datastore_postgres_SOURCES = \ | ||
131 | perf_plugin_datastore.c | ||
132 | perf_plugin_datastore_postgres_LDADD = \ | ||
133 | $(top_builddir)/src/lib/util/libgnunetutil.la | ||
134 | |||
135 | |||
136 | EXTRA_DIST = \ | ||
137 | test_defaults.conf \ | ||
138 | perf_plugin_datastore_data_sqlite.conf \ | ||
139 | test_plugin_datastore_data_sqlite.conf \ | ||
140 | perf_plugin_datastore_data_heap.conf \ | ||
141 | test_plugin_datastore_data_heap.conf \ | ||
142 | perf_plugin_datastore_data_postgres.conf \ | ||
143 | test_plugin_datastore_data_postgres.conf \ | ||
144 | $(sql_DATA) | ||
diff --git a/src/plugin/datastore/datastore-0001.sql b/src/plugin/datastore/datastore-0001.sql new file mode 100644 index 000000000..0d4758be2 --- /dev/null +++ b/src/plugin/datastore/datastore-0001.sql | |||
@@ -0,0 +1,49 @@ | |||
1 | -- | ||
2 | -- This file is part of GNUnet | ||
3 | -- Copyright (C) 2014--2022 GNUnet e.V. | ||
4 | -- | ||
5 | -- GNUnet is free software; you can redistribute it and/or modify it under the | ||
6 | -- terms of the GNU General Public License as published by the Free Software | ||
7 | -- Foundation; either version 3, or (at your option) any later version. | ||
8 | -- | ||
9 | -- GNUnet is distributed in the hope that it will be useful, but WITHOUT ANY | ||
10 | -- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR | ||
11 | -- A PARTICULAR PURPOSE. See the GNU General Public License for more details. | ||
12 | -- | ||
13 | -- You should have received a copy of the GNU General Public License along with | ||
14 | -- GNUnet; see the file COPYING. If not, see <http://www.gnu.org/licenses/> | ||
15 | -- | ||
16 | |||
17 | -- Everything in one big transaction | ||
18 | BEGIN; | ||
19 | |||
20 | -- Check patch versioning is in place. | ||
21 | SELECT _v.register_patch('datastore-0001', NULL, NULL); | ||
22 | |||
23 | -------------------- Schema ---------------------------- | ||
24 | |||
25 | CREATE SCHEMA datastore; | ||
26 | COMMENT ON SCHEMA datastore IS 'gnunet-datastore data'; | ||
27 | |||
28 | SET search_path TO datastore; | ||
29 | |||
30 | CREATE TABLE IF NOT EXISTS gn090 ( | ||
31 | repl INTEGER NOT NULL DEFAULT 0, | ||
32 | type INTEGER NOT NULL DEFAULT 0, | ||
33 | prio INTEGER NOT NULL DEFAULT 0, | ||
34 | anonLevel INTEGER NOT NULL DEFAULT 0, | ||
35 | expire BIGINT NOT NULL DEFAULT 0, | ||
36 | rvalue BIGINT NOT NULL DEFAULT 0, | ||
37 | hash BYTEA NOT NULL DEFAULT '', | ||
38 | vhash BYTEA NOT NULL DEFAULT '', | ||
39 | value BYTEA NOT NULL DEFAULT '', | ||
40 | oid BIGINT GENERATED BY DEFAULT AS IDENTITY); | ||
41 | |||
42 | CREATE INDEX IF NOT EXISTS oid_hash ON gn090 (oid); | ||
43 | CREATE INDEX IF NOT EXISTS idx_hash ON gn090 (hash); | ||
44 | CREATE INDEX IF NOT EXISTS idx_prio_anon ON gn090 (prio,anonLevel); | ||
45 | CREATE INDEX IF NOT EXISTS idx_prio_hash_anon ON gn090 (prio,hash,anonLevel); | ||
46 | CREATE INDEX IF NOT EXISTS idx_repl_rvalue ON gn090 (repl,rvalue); | ||
47 | CREATE INDEX IF NOT EXISTS idx_expire_hash ON gn090 (expire,hash); | ||
48 | |||
49 | COMMIT; | ||
diff --git a/src/plugin/datastore/datastore-drop.sql b/src/plugin/datastore/datastore-drop.sql new file mode 100644 index 000000000..67fee303d --- /dev/null +++ b/src/plugin/datastore/datastore-drop.sql | |||
@@ -0,0 +1,25 @@ | |||
1 | -- | ||
2 | -- This file is part of GNUnet | ||
3 | -- Copyright (C) 2014--2022 GNUnet e.V. | ||
4 | -- | ||
5 | -- GNUnet is free software; you can redistribute it and/or modify it under the | ||
6 | -- terms of the GNU General Public License as published by the Free Software | ||
7 | -- Foundation; either version 3, or (at your option) any later version. | ||
8 | -- | ||
9 | -- GNUnet is distributed in the hope that it will be useful, but WITHOUT ANY | ||
10 | -- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR | ||
11 | -- A PARTICULAR PURPOSE. See the GNU General Public License for more details. | ||
12 | -- | ||
13 | -- You should have received a copy of the GNU General Public License along with | ||
14 | -- GNUnet; see the file COPYING. If not, see <http://www.gnu.org/licenses/> | ||
15 | -- | ||
16 | |||
17 | -- Everything in one big transaction | ||
18 | BEGIN; | ||
19 | |||
20 | |||
21 | SELECT _v.unregister_patch('datastore-0001'); | ||
22 | |||
23 | DROP SCHEMA datastore CASCADE; | ||
24 | |||
25 | COMMIT; | ||
diff --git a/src/plugin/datastore/meson.build b/src/plugin/datastore/meson.build new file mode 100644 index 000000000..6769cb78b --- /dev/null +++ b/src/plugin/datastore/meson.build | |||
@@ -0,0 +1,75 @@ | |||
1 | install_data ('datastore-0001.sql', | ||
2 | 'datastore-drop.sql', | ||
3 | install_dir: get_option('datadir')/'gnunet'/'sql') | ||
4 | |||
5 | shared_module('gnunet_plugin_datastore_sqlite', | ||
6 | ['plugin_datastore_sqlite.c'], | ||
7 | dependencies: [libgnunetutil_dep, | ||
8 | sqlite_dep, | ||
9 | libgnunetsq_dep], | ||
10 | include_directories: [incdir, configuration_inc], | ||
11 | install: true, | ||
12 | install_dir: get_option('libdir')/'gnunet') | ||
13 | |||
14 | shared_module('gnunet_plugin_datastore_heap', | ||
15 | ['plugin_datastore_heap.c'], | ||
16 | dependencies: [libgnunetutil_dep], | ||
17 | include_directories: [incdir, configuration_inc], | ||
18 | install: true, | ||
19 | install_dir: get_option('libdir')/'gnunet') | ||
20 | |||
21 | if pq_dep.found() | ||
22 | shared_module('gnunet_plugin_datastore_postgres', | ||
23 | ['plugin_datastore_postgres.c'], | ||
24 | dependencies: [libgnunetutil_dep, | ||
25 | pq_dep, | ||
26 | libgnunetpq_dep], | ||
27 | include_directories: [incdir, configuration_inc], | ||
28 | install: true, | ||
29 | install_dir: get_option('libdir')/'gnunet') | ||
30 | endif | ||
31 | |||
32 | testds_plugin_sqlite = executable ('test_plugin_datastore_sqlite', | ||
33 | ['test_plugin_datastore.c'], | ||
34 | dependencies: [ | ||
35 | libgnunetutil_dep, | ||
36 | ], | ||
37 | include_directories: [incdir, configuration_inc], | ||
38 | install: false) | ||
39 | |||
40 | testds_plugin_heap = executable ('test_plugin_datastore_heap', | ||
41 | ['test_plugin_datastore.c'], | ||
42 | dependencies: [ | ||
43 | libgnunetutil_dep, | ||
44 | ], | ||
45 | include_directories: [incdir, configuration_inc], | ||
46 | install: false) | ||
47 | |||
48 | testds_plugin_pq = executable ('test_plugin_datastore_postgres', | ||
49 | ['test_plugin_datastore.c'], | ||
50 | dependencies: [ | ||
51 | libgnunetutil_dep, | ||
52 | ], | ||
53 | include_directories: [incdir, configuration_inc], | ||
54 | install: false) | ||
55 | |||
56 | configure_file(input : 'test_defaults.conf', | ||
57 | output : 'test_defaults.conf', | ||
58 | copy: true) | ||
59 | configure_file(input : 'test_plugin_datastore_data_sqlite.conf', | ||
60 | output : 'test_plugin_datastore_data_sqlite.conf', | ||
61 | copy: true) | ||
62 | configure_file(input : 'test_plugin_datastore_data_heap.conf', | ||
63 | output : 'test_plugin_datastore_data_heap.conf', | ||
64 | copy: true) | ||
65 | configure_file(input : 'test_plugin_datastore_data_postgres.conf', | ||
66 | output : 'test_plugin_datastore_data_postgres.conf', | ||
67 | copy: true) | ||
68 | |||
69 | test('test_plugin_datastore_sqlite', testds_plugin_sqlite, | ||
70 | suite: 'datastore', workdir: meson.current_build_dir()) | ||
71 | test('test_plugin_datastore_heap', testds_plugin_heap, | ||
72 | suite: 'datastore', workdir: meson.current_build_dir()) | ||
73 | test('test_plugin_datastore_postgres', testds_plugin_pq, | ||
74 | suite: 'datastore', workdir: meson.current_build_dir()) | ||
75 | |||
diff --git a/src/plugin/datastore/perf_plugin_datastore.c b/src/plugin/datastore/perf_plugin_datastore.c new file mode 100644 index 000000000..8e63b08e6 --- /dev/null +++ b/src/plugin/datastore/perf_plugin_datastore.c | |||
@@ -0,0 +1,573 @@ | |||
1 | /* | ||
2 | This file is part of GNUnet. | ||
3 | Copyright (C) 2004, 2005, 2006, 2007, 2009, 2011 GNUnet e.V. | ||
4 | |||
5 | GNUnet is free software: you can redistribute it and/or modify it | ||
6 | under the terms of the GNU Affero General Public License as published | ||
7 | by the Free Software Foundation, either version 3 of the License, | ||
8 | or (at your option) any later version. | ||
9 | |||
10 | GNUnet is distributed in the hope that it will be useful, but | ||
11 | WITHOUT ANY WARRANTY; without even the implied warranty of | ||
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
13 | Affero General Public License for more details. | ||
14 | |||
15 | You should have received a copy of the GNU Affero General Public License | ||
16 | along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
17 | |||
18 | SPDX-License-Identifier: AGPL3.0-or-later | ||
19 | */ | ||
20 | /* | ||
21 | * @file perf_plugin_datastore.c | ||
22 | * @brief Profile database plugin directly, focusing on iterators. | ||
23 | * @author Christian Grothoff | ||
24 | */ | ||
25 | |||
26 | #include "platform.h" | ||
27 | #include "gnunet_util_lib.h" | ||
28 | #include "gnunet_protocols.h" | ||
29 | #include "gnunet_datastore_plugin.h" | ||
30 | #include "gnunet_testing_lib.h" | ||
31 | #include <gauger.h> | ||
32 | |||
33 | /** | ||
34 | * Target datastore size (in bytes). Realistic sizes are | ||
35 | * more like 16 GB (not the default of 16 MB); however, | ||
36 | * those take too long to run them in the usual "make check" | ||
37 | * sequence. Hence the value used for shipping is tiny. | ||
38 | */ | ||
39 | #define MAX_SIZE 1024LL * 1024 * 16 * 1 | ||
40 | |||
41 | #define ITERATIONS 2 | ||
42 | |||
43 | /** | ||
44 | * Number of put operations equivalent to 1/10th of MAX_SIZE | ||
45 | */ | ||
46 | #define PUT_10 (MAX_SIZE / 32 / 1024 / ITERATIONS) | ||
47 | |||
48 | static char category[256]; | ||
49 | |||
50 | static unsigned int hits[PUT_10 / 8 + 1]; | ||
51 | |||
52 | static unsigned long long stored_bytes; | ||
53 | |||
54 | static unsigned long long stored_entries; | ||
55 | |||
56 | static unsigned long long stored_ops; | ||
57 | |||
58 | static const char *plugin_name; | ||
59 | |||
60 | static int ok; | ||
61 | |||
62 | enum RunPhase | ||
63 | { | ||
64 | RP_ERROR = 0, | ||
65 | RP_PUT, | ||
66 | RP_REP_GET, | ||
67 | RP_ZA_GET, | ||
68 | RP_EXP_GET, | ||
69 | RP_DONE | ||
70 | }; | ||
71 | |||
72 | |||
73 | struct CpsRunContext | ||
74 | { | ||
75 | unsigned int i; | ||
76 | struct GNUNET_TIME_Absolute start; | ||
77 | struct GNUNET_TIME_Absolute end; | ||
78 | const struct GNUNET_CONFIGURATION_Handle *cfg; | ||
79 | struct GNUNET_DATASTORE_PluginFunctions *api; | ||
80 | enum RunPhase phase; | ||
81 | unsigned int cnt; | ||
82 | unsigned int iter; | ||
83 | uint64_t offset; | ||
84 | }; | ||
85 | |||
86 | |||
87 | /** | ||
88 | * Function called by plugins to notify us about a | ||
89 | * change in their disk utilization. | ||
90 | * | ||
91 | * @param cls closure (NULL) | ||
92 | * @param delta change in disk utilization, | ||
93 | * 0 for "reset to empty" | ||
94 | */ | ||
95 | static void | ||
96 | disk_utilization_change_cb (void *cls, int delta) | ||
97 | { | ||
98 | } | ||
99 | |||
100 | |||
101 | static void | ||
102 | test (void *cls); | ||
103 | |||
104 | |||
105 | /** | ||
106 | * Put continuation. | ||
107 | * | ||
108 | * @param cls closure | ||
109 | * @param key key for the item stored | ||
110 | * @param size size of the item stored | ||
111 | * @param status #GNUNET_OK or #GNUNET_SYSERROR | ||
112 | * @param msg error message on error | ||
113 | */ | ||
114 | static void | ||
115 | put_continuation (void *cls, | ||
116 | const struct GNUNET_HashCode *key, | ||
117 | uint32_t size, | ||
118 | int status, | ||
119 | const char *msg) | ||
120 | { | ||
121 | struct CpsRunContext *crc = cls; | ||
122 | |||
123 | if (GNUNET_OK != status) | ||
124 | { | ||
125 | fprintf (stderr, "ERROR: `%s'\n", msg); | ||
126 | } | ||
127 | else | ||
128 | { | ||
129 | stored_bytes += size; | ||
130 | stored_ops++; | ||
131 | stored_entries++; | ||
132 | } | ||
133 | GNUNET_SCHEDULER_add_now (&test, crc); | ||
134 | } | ||
135 | |||
136 | |||
137 | static void | ||
138 | do_put (struct CpsRunContext *crc) | ||
139 | { | ||
140 | char value[65536]; | ||
141 | size_t size; | ||
142 | static struct GNUNET_HashCode key; | ||
143 | static int i; | ||
144 | unsigned int prio; | ||
145 | |||
146 | if (0 == i) | ||
147 | crc->start = GNUNET_TIME_absolute_get (); | ||
148 | if (PUT_10 == i) | ||
149 | { | ||
150 | i = 0; | ||
151 | crc->end = GNUNET_TIME_absolute_get (); | ||
152 | { | ||
153 | printf ("%s took %s for %llu items\n", "Storing an item", | ||
154 | GNUNET_STRINGS_relative_time_to_string ( | ||
155 | GNUNET_TIME_absolute_get_difference (crc->start, | ||
156 | crc | ||
157 | ->end), | ||
158 | GNUNET_YES), | ||
159 | PUT_10); | ||
160 | if (PUT_10 > 0) | ||
161 | GAUGER (category, "Storing an item", | ||
162 | (crc->end.abs_value_us - crc->start.abs_value_us) / 1000LL | ||
163 | / PUT_10, | ||
164 | "ms/item"); | ||
165 | } | ||
166 | crc->i++; | ||
167 | crc->start = GNUNET_TIME_absolute_get (); | ||
168 | crc->phase++; | ||
169 | GNUNET_SCHEDULER_add_now (&test, crc); | ||
170 | return; | ||
171 | } | ||
172 | /* most content is 32k */ | ||
173 | size = 32 * 1024; | ||
174 | if (GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 16) == 0) /* but some of it is less! */ | ||
175 | size = 8 + GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 32 * 1024); | ||
176 | size = size - (size & 7); /* always multiple of 8 */ | ||
177 | |||
178 | /* generate random key */ | ||
179 | key.bits[0] = (unsigned int) GNUNET_TIME_absolute_get ().abs_value_us; | ||
180 | GNUNET_CRYPTO_hash (&key, sizeof(struct GNUNET_HashCode), &key); | ||
181 | memset (value, i, size); | ||
182 | if (i > 255) | ||
183 | memset (value, i - 255, size / 2); | ||
184 | value[0] = crc->i; | ||
185 | GNUNET_memcpy (&value[4], &i, sizeof(i)); | ||
186 | prio = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 100); | ||
187 | crc->api->put (crc->api->cls, | ||
188 | &key, | ||
189 | false /* absent */, | ||
190 | size, | ||
191 | value, | ||
192 | 1 + i % 4 /* type */, | ||
193 | prio, | ||
194 | i % 4 /* anonymity */, | ||
195 | 0 /* replication */, | ||
196 | GNUNET_TIME_relative_to_absolute | ||
197 | (GNUNET_TIME_relative_multiply | ||
198 | (GNUNET_TIME_UNIT_MILLISECONDS, | ||
199 | 60 * 60 * 60 * 1000 | ||
200 | + GNUNET_CRYPTO_random_u32 | ||
201 | (GNUNET_CRYPTO_QUALITY_WEAK, 1000))), | ||
202 | put_continuation, | ||
203 | crc); | ||
204 | i++; | ||
205 | } | ||
206 | |||
207 | |||
208 | static int | ||
209 | iterate_zeros (void *cls, | ||
210 | const struct GNUNET_HashCode *key, | ||
211 | uint32_t size, | ||
212 | const void *data, | ||
213 | enum GNUNET_BLOCK_Type type, | ||
214 | uint32_t priority, | ||
215 | uint32_t anonymity, | ||
216 | uint32_t replication, | ||
217 | struct GNUNET_TIME_Absolute expiration, | ||
218 | uint64_t uid) | ||
219 | { | ||
220 | struct CpsRunContext *crc = cls; | ||
221 | int i; | ||
222 | const char *cdata = data; | ||
223 | |||
224 | GNUNET_assert (key != NULL); | ||
225 | GNUNET_assert (size >= 8); | ||
226 | GNUNET_memcpy (&i, &cdata[4], sizeof(i)); | ||
227 | hits[i / 8] |= (1 << (i % 8)); | ||
228 | |||
229 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
230 | "Found result %d type=%u, priority=%u, size=%u, expire=%s\n", | ||
231 | i, | ||
232 | type, priority, size, | ||
233 | GNUNET_STRINGS_absolute_time_to_string (expiration)); | ||
234 | crc->cnt++; | ||
235 | if (crc->cnt == PUT_10 / 4 - 1) | ||
236 | { | ||
237 | unsigned int bc; | ||
238 | |||
239 | bc = 0; | ||
240 | for (i = 0; i < PUT_10; i++) | ||
241 | if (0 != (hits[i / 8] & (1 << (i % 8)))) | ||
242 | bc++; | ||
243 | |||
244 | crc->end = GNUNET_TIME_absolute_get (); | ||
245 | printf ("%s took %s yielding %u/%u items\n", | ||
246 | "Select random zero-anonymity item", | ||
247 | GNUNET_STRINGS_relative_time_to_string ( | ||
248 | GNUNET_TIME_absolute_get_difference (crc->start, | ||
249 | crc | ||
250 | ->end), | ||
251 | GNUNET_YES), | ||
252 | bc, crc->cnt); | ||
253 | if (crc->cnt > 0) | ||
254 | GAUGER (category, "Select random zero-anonymity item", | ||
255 | (crc->end.abs_value_us - crc->start.abs_value_us) / 1000LL | ||
256 | / crc->cnt, | ||
257 | "ms/item"); | ||
258 | memset (hits, 0, sizeof(hits)); | ||
259 | crc->phase++; | ||
260 | crc->cnt = 0; | ||
261 | crc->start = GNUNET_TIME_absolute_get (); | ||
262 | } | ||
263 | GNUNET_SCHEDULER_add_now (&test, crc); | ||
264 | return GNUNET_OK; | ||
265 | } | ||
266 | |||
267 | |||
268 | static int | ||
269 | expiration_get (void *cls, | ||
270 | const struct GNUNET_HashCode *key, | ||
271 | uint32_t size, | ||
272 | const void *data, | ||
273 | enum GNUNET_BLOCK_Type type, | ||
274 | uint32_t priority, | ||
275 | uint32_t anonymity, | ||
276 | uint32_t replication, | ||
277 | struct GNUNET_TIME_Absolute expiration, | ||
278 | uint64_t uid) | ||
279 | { | ||
280 | struct CpsRunContext *crc = cls; | ||
281 | int i; | ||
282 | const char *cdata = data; | ||
283 | |||
284 | GNUNET_assert (size >= 8); | ||
285 | GNUNET_memcpy (&i, &cdata[4], sizeof(i)); | ||
286 | hits[i / 8] |= (1 << (i % 8)); | ||
287 | crc->cnt++; | ||
288 | if (PUT_10 <= crc->cnt) | ||
289 | { | ||
290 | unsigned int bc; | ||
291 | |||
292 | bc = 0; | ||
293 | for (i = 0; i < PUT_10; i++) | ||
294 | if (0 != (hits[i / 8] & (1 << (i % 8)))) | ||
295 | bc++; | ||
296 | |||
297 | crc->end = GNUNET_TIME_absolute_get (); | ||
298 | printf ("%s took %s yielding %u/%u items\n", | ||
299 | "Selecting and deleting by expiration", | ||
300 | GNUNET_STRINGS_relative_time_to_string ( | ||
301 | GNUNET_TIME_absolute_get_difference (crc->start, | ||
302 | crc | ||
303 | ->end), | ||
304 | GNUNET_YES), | ||
305 | bc, (unsigned int) PUT_10); | ||
306 | if (crc->cnt > 0) | ||
307 | GAUGER (category, "Selecting and deleting by expiration", | ||
308 | (crc->end.abs_value_us - crc->start.abs_value_us) / 1000LL | ||
309 | / crc->cnt, | ||
310 | "ms/item"); | ||
311 | memset (hits, 0, sizeof(hits)); | ||
312 | if (++crc->iter == ITERATIONS) | ||
313 | crc->phase++; | ||
314 | else | ||
315 | crc->phase = RP_PUT; | ||
316 | crc->cnt = 0; | ||
317 | crc->start = GNUNET_TIME_absolute_get (); | ||
318 | } | ||
319 | GNUNET_SCHEDULER_add_now (&test, crc); | ||
320 | return GNUNET_NO; | ||
321 | } | ||
322 | |||
323 | |||
324 | static int | ||
325 | replication_get (void *cls, | ||
326 | const struct GNUNET_HashCode *key, | ||
327 | uint32_t size, | ||
328 | const void *data, | ||
329 | enum GNUNET_BLOCK_Type type, | ||
330 | uint32_t priority, | ||
331 | uint32_t anonymity, | ||
332 | uint32_t replication, | ||
333 | struct GNUNET_TIME_Absolute expiration, | ||
334 | uint64_t uid) | ||
335 | { | ||
336 | struct CpsRunContext *crc = cls; | ||
337 | int i; | ||
338 | const char *cdata = data; | ||
339 | |||
340 | GNUNET_assert (NULL != key); | ||
341 | GNUNET_assert (size >= 8); | ||
342 | GNUNET_memcpy (&i, &cdata[4], sizeof(i)); | ||
343 | hits[i / 8] |= (1 << (i % 8)); | ||
344 | crc->cnt++; | ||
345 | if (PUT_10 <= crc->cnt) | ||
346 | { | ||
347 | unsigned int bc; | ||
348 | |||
349 | bc = 0; | ||
350 | for (i = 0; i < PUT_10; i++) | ||
351 | if (0 != (hits[i / 8] & (1 << (i % 8)))) | ||
352 | bc++; | ||
353 | |||
354 | crc->end = GNUNET_TIME_absolute_get (); | ||
355 | printf ("%s took %s yielding %u/%u items\n", | ||
356 | "Selecting random item for replication", | ||
357 | GNUNET_STRINGS_relative_time_to_string ( | ||
358 | GNUNET_TIME_absolute_get_difference (crc->start, | ||
359 | crc | ||
360 | ->end), | ||
361 | GNUNET_YES), | ||
362 | bc, (unsigned int) PUT_10); | ||
363 | if (crc->cnt > 0) | ||
364 | GAUGER (category, "Selecting random item for replication", | ||
365 | (crc->end.abs_value_us - crc->start.abs_value_us) / 1000LL | ||
366 | / crc->cnt, | ||
367 | "ms/item"); | ||
368 | memset (hits, 0, sizeof(hits)); | ||
369 | crc->phase++; | ||
370 | crc->offset = 0; | ||
371 | crc->cnt = 0; | ||
372 | crc->start = GNUNET_TIME_absolute_get (); | ||
373 | } | ||
374 | |||
375 | GNUNET_SCHEDULER_add_now (&test, crc); | ||
376 | return GNUNET_OK; | ||
377 | } | ||
378 | |||
379 | |||
380 | /** | ||
381 | * Function called when the service shuts | ||
382 | * down. Unloads our datastore plugin. | ||
383 | * | ||
384 | * @param api api to unload | ||
385 | * @param cfg configuration to use | ||
386 | */ | ||
387 | static void | ||
388 | unload_plugin (struct GNUNET_DATASTORE_PluginFunctions *api, | ||
389 | const struct GNUNET_CONFIGURATION_Handle *cfg) | ||
390 | { | ||
391 | char *name; | ||
392 | char *libname; | ||
393 | |||
394 | if (GNUNET_OK != | ||
395 | GNUNET_CONFIGURATION_get_value_string (cfg, "DATASTORE", "DATABASE", | ||
396 | &name)) | ||
397 | { | ||
398 | GNUNET_log (GNUNET_ERROR_TYPE_ERROR, | ||
399 | _ ("No `%s' specified for `%s' in configuration!\n"), | ||
400 | "DATABASE", | ||
401 | "DATASTORE"); | ||
402 | return; | ||
403 | } | ||
404 | GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", name); | ||
405 | GNUNET_break (NULL == GNUNET_PLUGIN_unload (libname, api)); | ||
406 | GNUNET_free (libname); | ||
407 | GNUNET_free (name); | ||
408 | } | ||
409 | |||
410 | |||
411 | /** | ||
412 | * Last task run during shutdown. Disconnects us from | ||
413 | * the transport and core. | ||
414 | */ | ||
415 | static void | ||
416 | cleaning_task (void *cls) | ||
417 | { | ||
418 | struct CpsRunContext *crc = cls; | ||
419 | |||
420 | unload_plugin (crc->api, crc->cfg); | ||
421 | GNUNET_free (crc); | ||
422 | } | ||
423 | |||
424 | |||
425 | static void | ||
426 | test (void *cls) | ||
427 | { | ||
428 | struct CpsRunContext *crc = cls; | ||
429 | |||
430 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
431 | "In phase %d, iteration %u\n", crc->phase, crc->cnt); | ||
432 | switch (crc->phase) | ||
433 | { | ||
434 | case RP_ERROR: | ||
435 | GNUNET_break (0); | ||
436 | crc->api->drop (crc->api->cls); | ||
437 | ok = 1; | ||
438 | GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_IDLE, | ||
439 | &cleaning_task, crc); | ||
440 | break; | ||
441 | |||
442 | case RP_PUT: | ||
443 | do_put (crc); | ||
444 | break; | ||
445 | |||
446 | case RP_REP_GET: | ||
447 | crc->api->get_replication (crc->api->cls, &replication_get, crc); | ||
448 | break; | ||
449 | |||
450 | case RP_ZA_GET: | ||
451 | crc->api->get_zero_anonymity (crc->api->cls, crc->offset++, 1, | ||
452 | &iterate_zeros, crc); | ||
453 | break; | ||
454 | |||
455 | case RP_EXP_GET: | ||
456 | crc->api->get_expiration (crc->api->cls, &expiration_get, crc); | ||
457 | break; | ||
458 | |||
459 | case RP_DONE: | ||
460 | crc->api->drop (crc->api->cls); | ||
461 | ok = 0; | ||
462 | GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_IDLE, | ||
463 | &cleaning_task, crc); | ||
464 | break; | ||
465 | } | ||
466 | } | ||
467 | |||
468 | |||
469 | /** | ||
470 | * Load the datastore plugin. | ||
471 | */ | ||
472 | static struct GNUNET_DATASTORE_PluginFunctions * | ||
473 | load_plugin (const struct GNUNET_CONFIGURATION_Handle *cfg) | ||
474 | { | ||
475 | static struct GNUNET_DATASTORE_PluginEnvironment env; | ||
476 | struct GNUNET_DATASTORE_PluginFunctions *ret; | ||
477 | char *name; | ||
478 | char *libname; | ||
479 | |||
480 | if (GNUNET_OK != | ||
481 | GNUNET_CONFIGURATION_get_value_string (cfg, "DATASTORE", "DATABASE", | ||
482 | &name)) | ||
483 | { | ||
484 | GNUNET_log (GNUNET_ERROR_TYPE_ERROR, | ||
485 | _ ("No `%s' specified for `%s' in configuration!\n"), | ||
486 | "DATABASE", | ||
487 | "DATASTORE"); | ||
488 | return NULL; | ||
489 | } | ||
490 | env.cfg = cfg; | ||
491 | env.duc = &disk_utilization_change_cb; | ||
492 | env.cls = NULL; | ||
493 | GNUNET_log (GNUNET_ERROR_TYPE_INFO, _ ("Loading `%s' datastore plugin\n"), | ||
494 | name); | ||
495 | GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", name); | ||
496 | if (NULL == (ret = GNUNET_PLUGIN_load (libname, &env))) | ||
497 | { | ||
498 | fprintf (stderr, "Failed to load plugin `%s'!\n", name); | ||
499 | GNUNET_free (name); | ||
500 | GNUNET_free (libname); | ||
501 | return NULL; | ||
502 | } | ||
503 | GNUNET_free (libname); | ||
504 | GNUNET_free (name); | ||
505 | return ret; | ||
506 | } | ||
507 | |||
508 | |||
509 | static void | ||
510 | run (void *cls, char *const *args, const char *cfgfile, | ||
511 | const struct GNUNET_CONFIGURATION_Handle *c) | ||
512 | { | ||
513 | struct GNUNET_DATASTORE_PluginFunctions *api; | ||
514 | struct CpsRunContext *crc; | ||
515 | |||
516 | if (NULL == c) | ||
517 | { | ||
518 | GNUNET_break (0); | ||
519 | return; | ||
520 | } | ||
521 | api = load_plugin (c); | ||
522 | if (api == NULL) | ||
523 | { | ||
524 | fprintf (stderr, | ||
525 | "%s", | ||
526 | "Could not initialize plugin, assuming database not configured. Test not run!\n"); | ||
527 | return; | ||
528 | } | ||
529 | crc = GNUNET_new (struct CpsRunContext); | ||
530 | crc->api = api; | ||
531 | crc->cfg = c; | ||
532 | crc->phase = RP_PUT; | ||
533 | ok = 2; | ||
534 | GNUNET_SCHEDULER_add_now (&test, crc); | ||
535 | } | ||
536 | |||
537 | |||
538 | int | ||
539 | main (int argc, char *argv[]) | ||
540 | { | ||
541 | char dir_name[PATH_MAX]; | ||
542 | char cfg_name[PATH_MAX]; | ||
543 | char *const xargv[] = { | ||
544 | "perf-plugin-datastore", | ||
545 | "-c", | ||
546 | cfg_name, | ||
547 | NULL | ||
548 | }; | ||
549 | struct GNUNET_GETOPT_CommandLineOption options[] = { | ||
550 | GNUNET_GETOPT_OPTION_END | ||
551 | }; | ||
552 | |||
553 | plugin_name = GNUNET_STRINGS_get_suffix_from_binary_name (argv[0]); | ||
554 | GNUNET_snprintf (dir_name, sizeof(dir_name), "/tmp/perf-gnunet-datastore-%s", | ||
555 | plugin_name); | ||
556 | GNUNET_DISK_directory_remove (dir_name); | ||
557 | GNUNET_log_setup ("perf-plugin-datastore", | ||
558 | "WARNING", | ||
559 | NULL); | ||
560 | GNUNET_snprintf (category, sizeof(category), "DATASTORE-%s", plugin_name); | ||
561 | GNUNET_snprintf (cfg_name, sizeof(cfg_name), | ||
562 | "perf_plugin_datastore_data_%s.conf", plugin_name); | ||
563 | GNUNET_PROGRAM_run ((sizeof(xargv) / sizeof(char *)) - 1, xargv, | ||
564 | "perf-plugin-datastore", "nohelp", options, &run, NULL); | ||
565 | if (ok != 0) | ||
566 | fprintf (stderr, "Missed some testcases: %u\n", ok); | ||
567 | GNUNET_DISK_directory_remove (dir_name); | ||
568 | |||
569 | return ok; | ||
570 | } | ||
571 | |||
572 | |||
573 | /* end of perf_plugin_datastore.c */ | ||
diff --git a/src/plugin/datastore/perf_plugin_datastore_data_heap.conf b/src/plugin/datastore/perf_plugin_datastore_data_heap.conf new file mode 100644 index 000000000..873cf9606 --- /dev/null +++ b/src/plugin/datastore/perf_plugin_datastore_data_heap.conf | |||
@@ -0,0 +1,7 @@ | |||
1 | @INLINE@ test_defaults.conf | ||
2 | [PATHS] | ||
3 | GNUNET_TEST_HOME = $GNUNET_TMP/perf-gnunet-datastore-heap/ | ||
4 | |||
5 | |||
6 | [datastore] | ||
7 | DATABASE = heap | ||
diff --git a/src/plugin/datastore/perf_plugin_datastore_data_postgres.conf b/src/plugin/datastore/perf_plugin_datastore_data_postgres.conf new file mode 100644 index 000000000..7683887a8 --- /dev/null +++ b/src/plugin/datastore/perf_plugin_datastore_data_postgres.conf | |||
@@ -0,0 +1,10 @@ | |||
1 | @INLINE@ test_defaults.conf | ||
2 | [PATHS] | ||
3 | GNUNET_TEST_HOME = $GNUNET_TMP/perf-gnunet-datastore-postgres/ | ||
4 | |||
5 | [datastore] | ||
6 | DATABASE = postgres | ||
7 | |||
8 | [datastore-postgres] | ||
9 | CONFIG = dbname=gnunetcheck | ||
10 | |||
diff --git a/src/plugin/datastore/perf_plugin_datastore_data_sqlite.conf b/src/plugin/datastore/perf_plugin_datastore_data_sqlite.conf new file mode 100644 index 000000000..888e020a6 --- /dev/null +++ b/src/plugin/datastore/perf_plugin_datastore_data_sqlite.conf | |||
@@ -0,0 +1,4 @@ | |||
1 | @INLINE@ test_defaults.conf | ||
2 | [PATHS] | ||
3 | GNUNET_TEST_HOME = $GNUNET_TMP/perf-gnunet-datastore-sqlite/ | ||
4 | |||
diff --git a/src/plugin/datastore/plugin_datastore_heap.c b/src/plugin/datastore/plugin_datastore_heap.c new file mode 100644 index 000000000..a827a2763 --- /dev/null +++ b/src/plugin/datastore/plugin_datastore_heap.c | |||
@@ -0,0 +1,944 @@ | |||
1 | /* | ||
2 | This file is part of GNUnet | ||
3 | Copyright (C) 2012 GNUnet e.V. | ||
4 | |||
5 | GNUnet is free software: you can redistribute it and/or modify it | ||
6 | under the terms of the GNU Affero General Public License as published | ||
7 | by the Free Software Foundation, either version 3 of the License, | ||
8 | or (at your option) any later version. | ||
9 | |||
10 | GNUnet is distributed in the hope that it will be useful, but | ||
11 | WITHOUT ANY WARRANTY; without even the implied warranty of | ||
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
13 | Affero General Public License for more details. | ||
14 | |||
15 | You should have received a copy of the GNU Affero General Public License | ||
16 | along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
17 | |||
18 | SPDX-License-Identifier: AGPL3.0-or-later | ||
19 | */ | ||
20 | |||
21 | /** | ||
22 | * @file datastore/plugin_datastore_heap.c | ||
23 | * @brief heap-based datastore backend; usually we want the datastore | ||
24 | * to be persistent, and storing data in the heap is obviously | ||
25 | * NOT going to be persistent; still, this plugin is useful for | ||
26 | * testing/benchmarking --- but never for production! | ||
27 | * @author Christian Grothoff | ||
28 | */ | ||
29 | |||
30 | #include "platform.h" | ||
31 | #include "gnunet_datastore_plugin.h" | ||
32 | |||
33 | |||
34 | /** | ||
35 | * A value that we are storing. | ||
36 | */ | ||
37 | struct Value | ||
38 | { | ||
39 | /** | ||
40 | * Key for the value. | ||
41 | */ | ||
42 | struct GNUNET_HashCode key; | ||
43 | |||
44 | /** | ||
45 | * Pointer to the value's data (allocated at the end of this struct). | ||
46 | */ | ||
47 | const void *data; | ||
48 | |||
49 | /** | ||
50 | * Entry for this value in the 'expire' heap. | ||
51 | */ | ||
52 | struct GNUNET_CONTAINER_HeapNode *expire_heap; | ||
53 | |||
54 | /** | ||
55 | * Entry for this value in the 'replication' heap. | ||
56 | */ | ||
57 | struct GNUNET_CONTAINER_HeapNode *replication_heap; | ||
58 | |||
59 | /** | ||
60 | * Expiration time for this value. | ||
61 | */ | ||
62 | struct GNUNET_TIME_Absolute expiration; | ||
63 | |||
64 | /** | ||
65 | * Offset of this value in the array of the 'struct ZeroAnonByType'; | ||
66 | * only used if anonymity is zero. | ||
67 | */ | ||
68 | unsigned int zero_anon_offset; | ||
69 | |||
70 | /** | ||
71 | * Number of bytes in 'data'. | ||
72 | */ | ||
73 | uint32_t size; | ||
74 | |||
75 | /** | ||
76 | * Priority of the value. | ||
77 | */ | ||
78 | uint32_t priority; | ||
79 | |||
80 | /** | ||
81 | * Anonymity level for the value. | ||
82 | */ | ||
83 | uint32_t anonymity; | ||
84 | |||
85 | /** | ||
86 | * Replication level for the value. | ||
87 | */ | ||
88 | uint32_t replication; | ||
89 | |||
90 | /** | ||
91 | * Type of 'data'. | ||
92 | */ | ||
93 | enum GNUNET_BLOCK_Type type; | ||
94 | }; | ||
95 | |||
96 | |||
97 | /** | ||
98 | * We organize 0-anonymity values in arrays "by type". | ||
99 | */ | ||
100 | struct ZeroAnonByType | ||
101 | { | ||
102 | /** | ||
103 | * We keep these in a DLL. | ||
104 | */ | ||
105 | struct ZeroAnonByType *next; | ||
106 | |||
107 | /** | ||
108 | * We keep these in a DLL. | ||
109 | */ | ||
110 | struct ZeroAnonByType *prev; | ||
111 | |||
112 | /** | ||
113 | * Array of 0-anonymity items of the given type. | ||
114 | */ | ||
115 | struct Value **array; | ||
116 | |||
117 | /** | ||
118 | * Allocated size of the array. | ||
119 | */ | ||
120 | unsigned int array_size; | ||
121 | |||
122 | /** | ||
123 | * First unused offset in 'array'. | ||
124 | */ | ||
125 | unsigned int array_pos; | ||
126 | |||
127 | /** | ||
128 | * Type of all of the values in 'array'. | ||
129 | */ | ||
130 | enum GNUNET_BLOCK_Type type; | ||
131 | }; | ||
132 | |||
133 | |||
134 | /** | ||
135 | * Context for all functions in this plugin. | ||
136 | */ | ||
137 | struct Plugin | ||
138 | { | ||
139 | /** | ||
140 | * Our execution environment. | ||
141 | */ | ||
142 | struct GNUNET_DATASTORE_PluginEnvironment *env; | ||
143 | |||
144 | /** | ||
145 | * Mapping from keys to 'struct Value's. | ||
146 | */ | ||
147 | struct GNUNET_CONTAINER_MultiHashMap *keyvalue; | ||
148 | |||
149 | /** | ||
150 | * Heap organized by minimum expiration time. | ||
151 | */ | ||
152 | struct GNUNET_CONTAINER_Heap *by_expiration; | ||
153 | |||
154 | /** | ||
155 | * Heap organized by maximum replication value. | ||
156 | */ | ||
157 | struct GNUNET_CONTAINER_Heap *by_replication; | ||
158 | |||
159 | /** | ||
160 | * Head of list of arrays containing zero-anonymity values by type. | ||
161 | */ | ||
162 | struct ZeroAnonByType *zero_head; | ||
163 | |||
164 | /** | ||
165 | * Tail of list of arrays containing zero-anonymity values by type. | ||
166 | */ | ||
167 | struct ZeroAnonByType *zero_tail; | ||
168 | |||
169 | /** | ||
170 | * Size of all values we're storing. | ||
171 | */ | ||
172 | unsigned long long size; | ||
173 | }; | ||
174 | |||
175 | |||
176 | /** | ||
177 | * Get an estimate of how much space the database is | ||
178 | * currently using. | ||
179 | * | ||
180 | * @param cls our "struct Plugin*" | ||
181 | * @return number of bytes used on disk | ||
182 | */ | ||
183 | static void | ||
184 | heap_plugin_estimate_size (void *cls, unsigned long long *estimate) | ||
185 | { | ||
186 | struct Plugin *plugin = cls; | ||
187 | |||
188 | if (NULL != estimate) | ||
189 | *estimate = plugin->size; | ||
190 | } | ||
191 | |||
192 | |||
193 | /** | ||
194 | * Closure for iterator for updating. | ||
195 | */ | ||
196 | struct UpdateContext | ||
197 | { | ||
198 | /** | ||
199 | * Number of bytes in 'data'. | ||
200 | */ | ||
201 | uint32_t size; | ||
202 | |||
203 | /** | ||
204 | * Pointer to the data. | ||
205 | */ | ||
206 | const void *data; | ||
207 | |||
208 | /** | ||
209 | * Priority of the value. | ||
210 | */ | ||
211 | uint32_t priority; | ||
212 | |||
213 | /** | ||
214 | * Replication level for the value. | ||
215 | */ | ||
216 | uint32_t replication; | ||
217 | |||
218 | /** | ||
219 | * Expiration time for this value. | ||
220 | */ | ||
221 | struct GNUNET_TIME_Absolute expiration; | ||
222 | |||
223 | /** | ||
224 | * True if the value was found and updated. | ||
225 | */ | ||
226 | bool updated; | ||
227 | }; | ||
228 | |||
229 | |||
230 | /** | ||
231 | * Update the matching value. | ||
232 | * | ||
233 | * @param cls the 'struct UpdateContext' | ||
234 | * @param key unused | ||
235 | * @param val the 'struct Value' | ||
236 | * @return GNUNET_YES (continue iteration), GNUNET_NO if value was found | ||
237 | */ | ||
238 | static int | ||
239 | update_iterator (void *cls, | ||
240 | const struct GNUNET_HashCode *key, | ||
241 | void *val) | ||
242 | { | ||
243 | struct UpdateContext *uc = cls; | ||
244 | struct Value *value = val; | ||
245 | |||
246 | if (value->size != uc->size) | ||
247 | return GNUNET_YES; | ||
248 | if (0 != memcmp (value->data, uc->data, uc->size)) | ||
249 | return GNUNET_YES; | ||
250 | uc->expiration = GNUNET_TIME_absolute_max (value->expiration, | ||
251 | uc->expiration); | ||
252 | if (value->expiration.abs_value_us != uc->expiration.abs_value_us) | ||
253 | { | ||
254 | value->expiration = uc->expiration; | ||
255 | GNUNET_CONTAINER_heap_update_cost (value->expire_heap, | ||
256 | value->expiration.abs_value_us); | ||
257 | } | ||
258 | /* Saturating adds, don't overflow */ | ||
259 | if (value->priority > UINT32_MAX - uc->priority) | ||
260 | value->priority = UINT32_MAX; | ||
261 | else | ||
262 | value->priority += uc->priority; | ||
263 | if (value->replication > UINT32_MAX - uc->replication) | ||
264 | value->replication = UINT32_MAX; | ||
265 | else | ||
266 | value->replication += uc->replication; | ||
267 | uc->updated = true; | ||
268 | return GNUNET_NO; | ||
269 | } | ||
270 | |||
271 | |||
272 | /** | ||
273 | * Store an item in the datastore. | ||
274 | * | ||
275 | * @param cls closure | ||
276 | * @param key key for the item | ||
277 | * @param absent true if the key was not found in the bloom filter | ||
278 | * @param size number of bytes in data | ||
279 | * @param data content stored | ||
280 | * @param type type of the content | ||
281 | * @param priority priority of the content | ||
282 | * @param anonymity anonymity-level for the content | ||
283 | * @param replication replication-level for the content | ||
284 | * @param expiration expiration time for the content | ||
285 | * @param cont continuation called with success or failure status | ||
286 | * @param cont_cls continuation closure | ||
287 | */ | ||
288 | static void | ||
289 | heap_plugin_put (void *cls, | ||
290 | const struct GNUNET_HashCode *key, | ||
291 | bool absent, | ||
292 | uint32_t size, | ||
293 | const void *data, | ||
294 | enum GNUNET_BLOCK_Type type, | ||
295 | uint32_t priority, | ||
296 | uint32_t anonymity, | ||
297 | uint32_t replication, | ||
298 | struct GNUNET_TIME_Absolute expiration, | ||
299 | PluginPutCont cont, | ||
300 | void *cont_cls) | ||
301 | { | ||
302 | struct Plugin *plugin = cls; | ||
303 | struct Value *value; | ||
304 | |||
305 | if (! absent) | ||
306 | { | ||
307 | struct UpdateContext uc; | ||
308 | |||
309 | uc.size = size; | ||
310 | uc.data = data; | ||
311 | uc.priority = priority; | ||
312 | uc.replication = replication; | ||
313 | uc.expiration = expiration; | ||
314 | uc.updated = false; | ||
315 | GNUNET_CONTAINER_multihashmap_get_multiple (plugin->keyvalue, | ||
316 | key, | ||
317 | &update_iterator, | ||
318 | &uc); | ||
319 | if (uc.updated) | ||
320 | { | ||
321 | cont (cont_cls, key, size, GNUNET_NO, NULL); | ||
322 | return; | ||
323 | } | ||
324 | } | ||
325 | value = GNUNET_malloc (sizeof(struct Value) + size); | ||
326 | value->key = *key; | ||
327 | value->data = &value[1]; | ||
328 | value->expire_heap = GNUNET_CONTAINER_heap_insert (plugin->by_expiration, | ||
329 | value, | ||
330 | expiration.abs_value_us); | ||
331 | value->replication_heap = GNUNET_CONTAINER_heap_insert ( | ||
332 | plugin->by_replication, | ||
333 | value, | ||
334 | replication); | ||
335 | value->expiration = expiration; | ||
336 | if (0 == anonymity) | ||
337 | { | ||
338 | struct ZeroAnonByType *zabt; | ||
339 | |||
340 | for (zabt = plugin->zero_head; NULL != zabt; zabt = zabt->next) | ||
341 | if (zabt->type == type) | ||
342 | break; | ||
343 | if (NULL == zabt) | ||
344 | { | ||
345 | zabt = GNUNET_new (struct ZeroAnonByType); | ||
346 | zabt->type = type; | ||
347 | GNUNET_CONTAINER_DLL_insert (plugin->zero_head, | ||
348 | plugin->zero_tail, | ||
349 | zabt); | ||
350 | } | ||
351 | if (zabt->array_size == zabt->array_pos) | ||
352 | { | ||
353 | GNUNET_array_grow (zabt->array, | ||
354 | zabt->array_size, | ||
355 | zabt->array_size * 2 + 4); | ||
356 | } | ||
357 | value->zero_anon_offset = zabt->array_pos; | ||
358 | zabt->array[zabt->array_pos++] = value; | ||
359 | } | ||
360 | value->size = size; | ||
361 | value->priority = priority; | ||
362 | value->anonymity = anonymity; | ||
363 | value->replication = replication; | ||
364 | value->type = type; | ||
365 | GNUNET_memcpy (&value[1], data, size); | ||
366 | GNUNET_CONTAINER_multihashmap_put (plugin->keyvalue, | ||
367 | &value->key, | ||
368 | value, | ||
369 | GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE); | ||
370 | plugin->size += size; | ||
371 | cont (cont_cls, key, size, GNUNET_OK, NULL); | ||
372 | } | ||
373 | |||
374 | |||
375 | /** | ||
376 | * Delete the given value, removing it from the plugin's data | ||
377 | * structures. | ||
378 | * | ||
379 | * @param plugin the plugin | ||
380 | * @param value value to delete | ||
381 | */ | ||
382 | static void | ||
383 | delete_value (struct Plugin *plugin, | ||
384 | struct Value *value) | ||
385 | { | ||
386 | GNUNET_assert (GNUNET_YES == | ||
387 | GNUNET_CONTAINER_multihashmap_remove (plugin->keyvalue, | ||
388 | &value->key, | ||
389 | value)); | ||
390 | GNUNET_assert (value == GNUNET_CONTAINER_heap_remove_node ( | ||
391 | value->expire_heap)); | ||
392 | GNUNET_assert (value == GNUNET_CONTAINER_heap_remove_node ( | ||
393 | value->replication_heap)); | ||
394 | if (0 == value->anonymity) | ||
395 | { | ||
396 | struct ZeroAnonByType *zabt; | ||
397 | |||
398 | for (zabt = plugin->zero_head; NULL != zabt; zabt = zabt->next) | ||
399 | if (zabt->type == value->type) | ||
400 | break; | ||
401 | GNUNET_assert (NULL != zabt); | ||
402 | zabt->array[value->zero_anon_offset] = zabt->array[--zabt->array_pos]; | ||
403 | zabt->array[value->zero_anon_offset]->zero_anon_offset = | ||
404 | value->zero_anon_offset; | ||
405 | if (0 == zabt->array_pos) | ||
406 | { | ||
407 | GNUNET_array_grow (zabt->array, | ||
408 | zabt->array_size, | ||
409 | 0); | ||
410 | GNUNET_CONTAINER_DLL_remove (plugin->zero_head, | ||
411 | plugin->zero_tail, | ||
412 | zabt); | ||
413 | GNUNET_free (zabt); | ||
414 | } | ||
415 | } | ||
416 | plugin->size -= value->size; | ||
417 | GNUNET_free (value); | ||
418 | } | ||
419 | |||
420 | |||
421 | /** | ||
422 | * Closure for iterator called during 'get_key'. | ||
423 | */ | ||
424 | struct GetContext | ||
425 | { | ||
426 | /** | ||
427 | * Lowest uid to consider. | ||
428 | */ | ||
429 | uint64_t next_uid; | ||
430 | |||
431 | /** | ||
432 | * Value with lowest uid >= next_uid found so far. | ||
433 | */ | ||
434 | struct Value *value; | ||
435 | |||
436 | /** | ||
437 | * Requested type. | ||
438 | */ | ||
439 | enum GNUNET_BLOCK_Type type; | ||
440 | |||
441 | /** | ||
442 | * If true, return a random value | ||
443 | */ | ||
444 | bool random; | ||
445 | }; | ||
446 | |||
447 | |||
448 | /** | ||
449 | * Obtain the matching value with the lowest uid >= next_uid. | ||
450 | * | ||
451 | * @param cls the 'struct GetContext' | ||
452 | * @param key unused | ||
453 | * @param val the 'struct Value' | ||
454 | * @return GNUNET_YES (continue iteration), GNUNET_NO if result was found | ||
455 | */ | ||
456 | static int | ||
457 | get_iterator (void *cls, | ||
458 | const struct GNUNET_HashCode *key, | ||
459 | void *val) | ||
460 | { | ||
461 | struct GetContext *gc = cls; | ||
462 | struct Value *value = val; | ||
463 | |||
464 | if ((gc->type != GNUNET_BLOCK_TYPE_ANY) && | ||
465 | (gc->type != value->type)) | ||
466 | return GNUNET_OK; | ||
467 | if (gc->random) | ||
468 | { | ||
469 | gc->value = value; | ||
470 | return GNUNET_NO; | ||
471 | } | ||
472 | if ((uint64_t) (intptr_t) value < gc->next_uid) | ||
473 | return GNUNET_OK; | ||
474 | if ((NULL != gc->value) && | ||
475 | (value > gc->value)) | ||
476 | return GNUNET_OK; | ||
477 | gc->value = value; | ||
478 | return GNUNET_OK; | ||
479 | } | ||
480 | |||
481 | |||
482 | /** | ||
483 | * Get one of the results for a particular key in the datastore. | ||
484 | * | ||
485 | * @param cls closure | ||
486 | * @param next_uid return the result with lowest uid >= next_uid | ||
487 | * @param random if true, return a random result instead of using next_uid | ||
488 | * @param key maybe NULL (to match all entries) | ||
489 | * @param type entries of which type are relevant? | ||
490 | * Use 0 for any type. | ||
491 | * @param proc function to call on the matching value; | ||
492 | * will be called with NULL if nothing matches | ||
493 | * @param proc_cls closure for @a proc | ||
494 | */ | ||
495 | static void | ||
496 | heap_plugin_get_key (void *cls, | ||
497 | uint64_t next_uid, | ||
498 | bool random, | ||
499 | const struct GNUNET_HashCode *key, | ||
500 | enum GNUNET_BLOCK_Type type, | ||
501 | PluginDatumProcessor proc, | ||
502 | void *proc_cls) | ||
503 | { | ||
504 | struct Plugin *plugin = cls; | ||
505 | struct GetContext gc; | ||
506 | |||
507 | gc.value = NULL; | ||
508 | gc.next_uid = next_uid; | ||
509 | gc.random = random; | ||
510 | gc.type = type; | ||
511 | if (NULL == key) | ||
512 | { | ||
513 | GNUNET_CONTAINER_multihashmap_iterate (plugin->keyvalue, | ||
514 | &get_iterator, | ||
515 | &gc); | ||
516 | } | ||
517 | else | ||
518 | { | ||
519 | GNUNET_CONTAINER_multihashmap_get_multiple (plugin->keyvalue, | ||
520 | key, | ||
521 | &get_iterator, | ||
522 | &gc); | ||
523 | } | ||
524 | if (NULL == gc.value) | ||
525 | { | ||
526 | proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); | ||
527 | return; | ||
528 | } | ||
529 | GNUNET_assert (GNUNET_OK == | ||
530 | proc (proc_cls, | ||
531 | &gc.value->key, | ||
532 | gc.value->size, | ||
533 | &gc.value[1], | ||
534 | gc.value->type, | ||
535 | gc.value->priority, | ||
536 | gc.value->anonymity, | ||
537 | gc.value->replication, | ||
538 | gc.value->expiration, | ||
539 | (uint64_t) (intptr_t) gc.value)); | ||
540 | } | ||
541 | |||
542 | |||
543 | /** | ||
544 | * Get a random item for replication. Returns a single, not expired, | ||
545 | * random item from those with the highest replication counters. The | ||
546 | * item's replication counter is decremented by one IF it was positive | ||
547 | * before. Call 'proc' with all values ZERO or NULL if the datastore | ||
548 | * is empty. | ||
549 | * | ||
550 | * @param cls closure | ||
551 | * @param proc function to call the value (once only). | ||
552 | * @param proc_cls closure for proc | ||
553 | */ | ||
554 | static void | ||
555 | heap_plugin_get_replication (void *cls, | ||
556 | PluginDatumProcessor proc, | ||
557 | void *proc_cls) | ||
558 | { | ||
559 | struct Plugin *plugin = cls; | ||
560 | struct Value *value; | ||
561 | |||
562 | value = GNUNET_CONTAINER_heap_remove_root (plugin->by_replication); | ||
563 | if (NULL == value) | ||
564 | { | ||
565 | proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); | ||
566 | return; | ||
567 | } | ||
568 | if (value->replication > 0) | ||
569 | { | ||
570 | value->replication--; | ||
571 | value->replication_heap = GNUNET_CONTAINER_heap_insert ( | ||
572 | plugin->by_replication, | ||
573 | value, | ||
574 | value->replication); | ||
575 | } | ||
576 | else | ||
577 | { | ||
578 | /* need a better way to pick a random item, replication level is always 0 */ | ||
579 | value->replication_heap = GNUNET_CONTAINER_heap_insert ( | ||
580 | plugin->by_replication, | ||
581 | value, | ||
582 | value->replication); | ||
583 | value = GNUNET_CONTAINER_heap_walk_get_next (plugin->by_replication); | ||
584 | } | ||
585 | GNUNET_assert (GNUNET_OK == | ||
586 | proc (proc_cls, | ||
587 | &value->key, | ||
588 | value->size, | ||
589 | &value[1], | ||
590 | value->type, | ||
591 | value->priority, | ||
592 | value->anonymity, | ||
593 | value->replication, | ||
594 | value->expiration, | ||
595 | (uint64_t) (intptr_t) value)); | ||
596 | } | ||
597 | |||
598 | |||
599 | /** | ||
600 | * Get a random item for expiration. Call 'proc' with all values ZERO | ||
601 | * or NULL if the datastore is empty. | ||
602 | * | ||
603 | * @param cls closure | ||
604 | * @param proc function to call the value (once only). | ||
605 | * @param proc_cls closure for proc | ||
606 | */ | ||
607 | static void | ||
608 | heap_plugin_get_expiration (void *cls, PluginDatumProcessor proc, | ||
609 | void *proc_cls) | ||
610 | { | ||
611 | struct Plugin *plugin = cls; | ||
612 | struct Value *value; | ||
613 | |||
614 | value = GNUNET_CONTAINER_heap_peek (plugin->by_expiration); | ||
615 | if (NULL == value) | ||
616 | { | ||
617 | proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); | ||
618 | return; | ||
619 | } | ||
620 | if (GNUNET_NO == | ||
621 | proc (proc_cls, | ||
622 | &value->key, | ||
623 | value->size, | ||
624 | &value[1], | ||
625 | value->type, | ||
626 | value->priority, | ||
627 | value->anonymity, | ||
628 | value->replication, | ||
629 | value->expiration, | ||
630 | (uint64_t) (intptr_t) value)) | ||
631 | delete_value (plugin, value); | ||
632 | } | ||
633 | |||
634 | |||
635 | /** | ||
636 | * Call the given processor on an item with zero anonymity. | ||
637 | * | ||
638 | * @param cls our "struct Plugin*" | ||
639 | * @param next_uid return the result with lowest uid >= next_uid | ||
640 | * @param type entries of which type should be considered? | ||
641 | * Must not be zero (ANY). | ||
642 | * @param proc function to call on each matching value; | ||
643 | * will be called with NULL if no value matches | ||
644 | * @param proc_cls closure for proc | ||
645 | */ | ||
646 | static void | ||
647 | heap_plugin_get_zero_anonymity (void *cls, uint64_t next_uid, | ||
648 | enum GNUNET_BLOCK_Type type, | ||
649 | PluginDatumProcessor proc, void *proc_cls) | ||
650 | { | ||
651 | struct Plugin *plugin = cls; | ||
652 | struct ZeroAnonByType *zabt; | ||
653 | struct Value *value = NULL; | ||
654 | |||
655 | for (zabt = plugin->zero_head; NULL != zabt; zabt = zabt->next) | ||
656 | { | ||
657 | if ((type != GNUNET_BLOCK_TYPE_ANY) && | ||
658 | (type != zabt->type)) | ||
659 | continue; | ||
660 | for (int i = 0; i < zabt->array_pos; ++i) | ||
661 | { | ||
662 | if ((uint64_t) (intptr_t) zabt->array[i] < next_uid) | ||
663 | continue; | ||
664 | if ((NULL != value) && | ||
665 | (zabt->array[i] > value)) | ||
666 | continue; | ||
667 | value = zabt->array[i]; | ||
668 | } | ||
669 | } | ||
670 | if (NULL == value) | ||
671 | { | ||
672 | proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); | ||
673 | return; | ||
674 | } | ||
675 | GNUNET_assert (GNUNET_OK == | ||
676 | proc (proc_cls, | ||
677 | &value->key, | ||
678 | value->size, | ||
679 | &value[1], | ||
680 | value->type, | ||
681 | value->priority, | ||
682 | value->anonymity, | ||
683 | value->replication, | ||
684 | value->expiration, | ||
685 | (uint64_t) (intptr_t) value)); | ||
686 | } | ||
687 | |||
688 | |||
689 | /** | ||
690 | * Drop database. | ||
691 | */ | ||
692 | static void | ||
693 | heap_plugin_drop (void *cls) | ||
694 | { | ||
695 | /* nothing needs to be done */ | ||
696 | } | ||
697 | |||
698 | |||
699 | /** | ||
700 | * Closure for the 'return_value' function. | ||
701 | */ | ||
702 | struct GetAllContext | ||
703 | { | ||
704 | /** | ||
705 | * Function to call. | ||
706 | */ | ||
707 | PluginKeyProcessor proc; | ||
708 | |||
709 | /** | ||
710 | * Closure for 'proc'. | ||
711 | */ | ||
712 | void *proc_cls; | ||
713 | }; | ||
714 | |||
715 | |||
716 | /** | ||
717 | * Callback invoked to call callback on each value. | ||
718 | * | ||
719 | * @param cls the plugin | ||
720 | * @param key unused | ||
721 | * @param val the value | ||
722 | * @return GNUNET_OK (continue to iterate) | ||
723 | */ | ||
724 | static int | ||
725 | return_value (void *cls, | ||
726 | const struct GNUNET_HashCode *key, | ||
727 | void *val) | ||
728 | { | ||
729 | struct GetAllContext *gac = cls; | ||
730 | |||
731 | gac->proc (gac->proc_cls, | ||
732 | key, | ||
733 | 1); | ||
734 | return GNUNET_OK; | ||
735 | } | ||
736 | |||
737 | |||
738 | /** | ||
739 | * Get all of the keys in the datastore. | ||
740 | * | ||
741 | * @param cls closure | ||
742 | * @param proc function to call on each key | ||
743 | * @param proc_cls closure for proc | ||
744 | */ | ||
745 | static void | ||
746 | heap_get_keys (void *cls, | ||
747 | PluginKeyProcessor proc, | ||
748 | void *proc_cls) | ||
749 | { | ||
750 | struct Plugin *plugin = cls; | ||
751 | struct GetAllContext gac; | ||
752 | |||
753 | gac.proc = proc; | ||
754 | gac.proc_cls = proc_cls; | ||
755 | GNUNET_CONTAINER_multihashmap_iterate (plugin->keyvalue, | ||
756 | &return_value, | ||
757 | &gac); | ||
758 | proc (proc_cls, NULL, 0); | ||
759 | } | ||
760 | |||
761 | |||
762 | /** | ||
763 | * Closure for iterator called during 'remove_key'. | ||
764 | */ | ||
765 | struct RemoveContext | ||
766 | { | ||
767 | /** | ||
768 | * Value found. | ||
769 | */ | ||
770 | struct Value *value; | ||
771 | |||
772 | /** | ||
773 | * Size of data. | ||
774 | */ | ||
775 | uint32_t size; | ||
776 | |||
777 | /** | ||
778 | * Data to remove. | ||
779 | */ | ||
780 | const void *data; | ||
781 | }; | ||
782 | |||
783 | |||
784 | /** | ||
785 | * Obtain the matching value with the lowest uid >= next_uid. | ||
786 | * | ||
787 | * @param cls the 'struct GetContext' | ||
788 | * @param key unused | ||
789 | * @param val the 'struct Value' | ||
790 | * @return GNUNET_YES (continue iteration), GNUNET_NO if result was found | ||
791 | */ | ||
792 | static int | ||
793 | remove_iterator (void *cls, | ||
794 | const struct GNUNET_HashCode *key, | ||
795 | void *val) | ||
796 | { | ||
797 | struct RemoveContext *rc = cls; | ||
798 | struct Value *value = val; | ||
799 | |||
800 | if (value->size != rc->size) | ||
801 | return GNUNET_YES; | ||
802 | if (0 != memcmp (value->data, rc->data, rc->size)) | ||
803 | return GNUNET_YES; | ||
804 | rc->value = value; | ||
805 | return GNUNET_NO; | ||
806 | } | ||
807 | |||
808 | |||
809 | /** | ||
810 | * Remove a particular key in the datastore. | ||
811 | * | ||
812 | * @param cls closure | ||
813 | * @param key key for the content | ||
814 | * @param size number of bytes in data | ||
815 | * @param data content stored | ||
816 | * @param cont continuation called with success or failure status | ||
817 | * @param cont_cls continuation closure for @a cont | ||
818 | */ | ||
819 | static void | ||
820 | heap_plugin_remove_key (void *cls, | ||
821 | const struct GNUNET_HashCode *key, | ||
822 | uint32_t size, | ||
823 | const void *data, | ||
824 | PluginRemoveCont cont, | ||
825 | void *cont_cls) | ||
826 | { | ||
827 | struct Plugin *plugin = cls; | ||
828 | struct RemoveContext rc; | ||
829 | |||
830 | rc.value = NULL; | ||
831 | rc.size = size; | ||
832 | rc.data = data; | ||
833 | GNUNET_CONTAINER_multihashmap_get_multiple (plugin->keyvalue, | ||
834 | key, | ||
835 | &remove_iterator, | ||
836 | &rc); | ||
837 | if (NULL == rc.value) | ||
838 | { | ||
839 | cont (cont_cls, | ||
840 | key, | ||
841 | size, | ||
842 | GNUNET_NO, | ||
843 | NULL); | ||
844 | return; | ||
845 | } | ||
846 | delete_value (plugin, | ||
847 | rc.value); | ||
848 | cont (cont_cls, | ||
849 | key, | ||
850 | size, | ||
851 | GNUNET_OK, | ||
852 | NULL); | ||
853 | } | ||
854 | |||
855 | |||
856 | /** | ||
857 | * Entry point for the plugin. | ||
858 | * | ||
859 | * @param cls the "struct GNUNET_DATASTORE_PluginEnvironment*" | ||
860 | * @return our "struct Plugin*" | ||
861 | */ | ||
862 | void * | ||
863 | libgnunet_plugin_datastore_heap_init (void *cls) | ||
864 | { | ||
865 | struct GNUNET_DATASTORE_PluginEnvironment *env = cls; | ||
866 | struct GNUNET_DATASTORE_PluginFunctions *api; | ||
867 | struct Plugin *plugin; | ||
868 | unsigned long long esize; | ||
869 | |||
870 | if (GNUNET_OK != | ||
871 | GNUNET_CONFIGURATION_get_value_number (env->cfg, | ||
872 | "datastore-heap", | ||
873 | "HASHMAPSIZE", | ||
874 | &esize)) | ||
875 | esize = 128 * 1024; | ||
876 | plugin = GNUNET_new (struct Plugin); | ||
877 | plugin->env = env; | ||
878 | plugin->keyvalue = GNUNET_CONTAINER_multihashmap_create (esize, GNUNET_YES); | ||
879 | plugin->by_expiration = GNUNET_CONTAINER_heap_create ( | ||
880 | GNUNET_CONTAINER_HEAP_ORDER_MIN); | ||
881 | plugin->by_replication = GNUNET_CONTAINER_heap_create ( | ||
882 | GNUNET_CONTAINER_HEAP_ORDER_MAX); | ||
883 | api = GNUNET_new (struct GNUNET_DATASTORE_PluginFunctions); | ||
884 | api->cls = plugin; | ||
885 | api->estimate_size = &heap_plugin_estimate_size; | ||
886 | api->put = &heap_plugin_put; | ||
887 | api->get_key = &heap_plugin_get_key; | ||
888 | api->get_replication = &heap_plugin_get_replication; | ||
889 | api->get_expiration = &heap_plugin_get_expiration; | ||
890 | api->get_zero_anonymity = &heap_plugin_get_zero_anonymity; | ||
891 | api->drop = &heap_plugin_drop; | ||
892 | api->get_keys = &heap_get_keys; | ||
893 | api->remove_key = &heap_plugin_remove_key; | ||
894 | GNUNET_log_from (GNUNET_ERROR_TYPE_INFO, "heap", | ||
895 | _ ("Heap database running\n")); | ||
896 | return api; | ||
897 | } | ||
898 | |||
899 | |||
900 | /** | ||
901 | * Callback invoked to free all value. | ||
902 | * | ||
903 | * @param cls the plugin | ||
904 | * @param key unused | ||
905 | * @param val the value | ||
906 | * @return GNUNET_OK (continue to iterate) | ||
907 | */ | ||
908 | static int | ||
909 | free_value (void *cls, | ||
910 | const struct GNUNET_HashCode *key, | ||
911 | void *val) | ||
912 | { | ||
913 | struct Plugin *plugin = cls; | ||
914 | struct Value *value = val; | ||
915 | |||
916 | delete_value (plugin, value); | ||
917 | return GNUNET_OK; | ||
918 | } | ||
919 | |||
920 | |||
921 | /** | ||
922 | * Exit point from the plugin. | ||
923 | * @param cls our "struct Plugin*" | ||
924 | * @return always NULL | ||
925 | */ | ||
926 | void * | ||
927 | libgnunet_plugin_datastore_heap_done (void *cls) | ||
928 | { | ||
929 | struct GNUNET_DATASTORE_PluginFunctions *api = cls; | ||
930 | struct Plugin *plugin = api->cls; | ||
931 | |||
932 | GNUNET_CONTAINER_multihashmap_iterate (plugin->keyvalue, | ||
933 | &free_value, | ||
934 | plugin); | ||
935 | GNUNET_CONTAINER_multihashmap_destroy (plugin->keyvalue); | ||
936 | GNUNET_CONTAINER_heap_destroy (plugin->by_expiration); | ||
937 | GNUNET_CONTAINER_heap_destroy (plugin->by_replication); | ||
938 | GNUNET_free (plugin); | ||
939 | GNUNET_free (api); | ||
940 | return NULL; | ||
941 | } | ||
942 | |||
943 | |||
944 | /* end of plugin_datastore_heap.c */ | ||
diff --git a/src/plugin/datastore/plugin_datastore_postgres.c b/src/plugin/datastore/plugin_datastore_postgres.c new file mode 100644 index 000000000..5fcacc17b --- /dev/null +++ b/src/plugin/datastore/plugin_datastore_postgres.c | |||
@@ -0,0 +1,930 @@ | |||
1 | /* | ||
2 | This file is part of GNUnet | ||
3 | Copyright (C) 2009-2017, 2022 GNUnet e.V. | ||
4 | |||
5 | GNUnet is free software: you can redistribute it and/or modify it | ||
6 | under the terms of the GNU Affero General Public License as published | ||
7 | by the Free Software Foundation, either version 3 of the License, | ||
8 | or (at your option) any later version. | ||
9 | |||
10 | GNUnet is distributed in the hope that it will be useful, but | ||
11 | WITHOUT ANY WARRANTY; without even the implied warranty of | ||
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
13 | Affero General Public License for more details. | ||
14 | |||
15 | You should have received a copy of the GNU Affero General Public License | ||
16 | along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
17 | |||
18 | SPDX-License-Identifier: AGPL3.0-or-later | ||
19 | */ | ||
20 | |||
21 | /** | ||
22 | * @file datastore/plugin_datastore_postgres.c | ||
23 | * @brief postgres-based datastore backend | ||
24 | * @author Christian Grothoff | ||
25 | */ | ||
26 | #include "platform.h" | ||
27 | #include "gnunet_datastore_plugin.h" | ||
28 | #include "gnunet_pq_lib.h" | ||
29 | |||
30 | |||
31 | /** | ||
32 | * After how many ms "busy" should a DB operation fail for good? | ||
33 | * A low value makes sure that we are more responsive to requests | ||
34 | * (especially PUTs). A high value guarantees a higher success | ||
35 | * rate (SELECTs in iterate can take several seconds despite LIMIT=1). | ||
36 | * | ||
37 | * The default value of 1s should ensure that users do not experience | ||
38 | * huge latencies while at the same time allowing operations to succeed | ||
39 | * with reasonable probability. | ||
40 | */ | ||
41 | #define BUSY_TIMEOUT GNUNET_TIME_UNIT_SECONDS | ||
42 | |||
43 | |||
44 | /** | ||
45 | * Context for all functions in this plugin. | ||
46 | */ | ||
47 | struct Plugin | ||
48 | { | ||
49 | /** | ||
50 | * Our execution environment. | ||
51 | */ | ||
52 | struct GNUNET_DATASTORE_PluginEnvironment *env; | ||
53 | |||
54 | /** | ||
55 | * Native Postgres database handle. | ||
56 | */ | ||
57 | struct GNUNET_PQ_Context *dbh; | ||
58 | }; | ||
59 | |||
60 | |||
61 | /** | ||
62 | * @brief Get a database handle | ||
63 | * | ||
64 | * @param plugin global context | ||
65 | * @return #GNUNET_OK on success, #GNUNET_SYSERR on error | ||
66 | */ | ||
67 | static enum GNUNET_GenericReturnValue | ||
68 | init_connection (struct Plugin *plugin) | ||
69 | { | ||
70 | #define RESULT_COLUMNS "repl, type, prio, anonLevel, expire, hash, value, oid" | ||
71 | struct GNUNET_PQ_PreparedStatement ps[] = { | ||
72 | GNUNET_PQ_make_prepare ("get", | ||
73 | "SELECT " RESULT_COLUMNS | ||
74 | " FROM datastore.gn090" | ||
75 | " WHERE oid >= $1::bigint AND" | ||
76 | " (rvalue >= $2 OR 0 = $3::smallint) AND" | ||
77 | " (hash = $4 OR 0 = $5::smallint) AND" | ||
78 | " (type = $6 OR 0 = $7::smallint)" | ||
79 | " ORDER BY oid ASC LIMIT 1"), | ||
80 | GNUNET_PQ_make_prepare ("put", | ||
81 | "INSERT INTO datastore.gn090" | ||
82 | " (repl, type, prio, anonLevel, expire, rvalue, hash, vhash, value) " | ||
83 | "VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)"), | ||
84 | GNUNET_PQ_make_prepare ("update", | ||
85 | "UPDATE datastore.gn090" | ||
86 | " SET prio = prio + $1," | ||
87 | " repl = repl + $2," | ||
88 | " expire = GREATEST(expire, $3)" | ||
89 | " WHERE hash = $4 AND vhash = $5"), | ||
90 | GNUNET_PQ_make_prepare ("decrepl", | ||
91 | "UPDATE datastore.gn090" | ||
92 | " SET repl = GREATEST (repl - 1, 0)" | ||
93 | " WHERE oid = $1"), | ||
94 | GNUNET_PQ_make_prepare ("select_non_anonymous", | ||
95 | "SELECT " RESULT_COLUMNS | ||
96 | " FROM datastore.gn090" | ||
97 | " WHERE anonLevel = 0 AND type = $1 AND oid >= $2::bigint" | ||
98 | " ORDER BY oid ASC LIMIT 1"), | ||
99 | GNUNET_PQ_make_prepare ("select_expiration_order", | ||
100 | "(SELECT " RESULT_COLUMNS | ||
101 | " FROM datastore.gn090" | ||
102 | " WHERE expire < $1 ORDER BY prio ASC LIMIT 1) " | ||
103 | "UNION " | ||
104 | "(SELECT " RESULT_COLUMNS | ||
105 | " FROM datastore.gn090" | ||
106 | " ORDER BY prio ASC LIMIT 1)" | ||
107 | " ORDER BY expire ASC LIMIT 1"), | ||
108 | GNUNET_PQ_make_prepare ("select_replication_order", | ||
109 | "SELECT " RESULT_COLUMNS | ||
110 | " FROM datastore.gn090" | ||
111 | " ORDER BY repl DESC,RANDOM() LIMIT 1"), | ||
112 | GNUNET_PQ_make_prepare ("delrow", | ||
113 | "DELETE FROM datastore.gn090" | ||
114 | " WHERE oid=$1"), | ||
115 | GNUNET_PQ_make_prepare ("remove", | ||
116 | "DELETE FROM datastore.gn090" | ||
117 | " WHERE hash = $1 AND" | ||
118 | " value = $2"), | ||
119 | GNUNET_PQ_make_prepare ("get_keys", | ||
120 | "SELECT hash" | ||
121 | " FROM datastore.gn090"), | ||
122 | GNUNET_PQ_make_prepare ("estimate_size", | ||
123 | "SELECT CASE WHEN NOT EXISTS" | ||
124 | " (SELECT 1 FROM datastore.gn090)" | ||
125 | " THEN 0" | ||
126 | " ELSE (SELECT SUM(LENGTH(value))+256*COUNT(*)" | ||
127 | " FROM datastore.gn090)" | ||
128 | "END AS total"), | ||
129 | GNUNET_PQ_PREPARED_STATEMENT_END | ||
130 | }; | ||
131 | #undef RESULT_COLUMNS | ||
132 | |||
133 | plugin->dbh = GNUNET_PQ_connect_with_cfg (plugin->env->cfg, | ||
134 | "datastore-postgres", | ||
135 | "datastore-", | ||
136 | NULL, | ||
137 | ps); | ||
138 | if (NULL == plugin->dbh) | ||
139 | return GNUNET_SYSERR; | ||
140 | return GNUNET_OK; | ||
141 | } | ||
142 | |||
143 | |||
144 | /** | ||
145 | * Get an estimate of how much space the database is | ||
146 | * currently using. | ||
147 | * | ||
148 | * @param cls our `struct Plugin *` | ||
149 | * @return number of bytes used on disk | ||
150 | */ | ||
151 | static void | ||
152 | postgres_plugin_estimate_size (void *cls, | ||
153 | unsigned long long *estimate) | ||
154 | { | ||
155 | struct Plugin *plugin = cls; | ||
156 | uint64_t total; | ||
157 | struct GNUNET_PQ_QueryParam params[] = { | ||
158 | GNUNET_PQ_query_param_end | ||
159 | }; | ||
160 | struct GNUNET_PQ_ResultSpec rs[] = { | ||
161 | GNUNET_PQ_result_spec_uint64 ("total", | ||
162 | &total), | ||
163 | GNUNET_PQ_result_spec_end | ||
164 | }; | ||
165 | enum GNUNET_DB_QueryStatus ret; | ||
166 | |||
167 | if (NULL == estimate) | ||
168 | return; | ||
169 | ret = GNUNET_PQ_eval_prepared_singleton_select (plugin->dbh, | ||
170 | "estimate_size", | ||
171 | params, | ||
172 | rs); | ||
173 | if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT != ret) | ||
174 | { | ||
175 | *estimate = 0LL; | ||
176 | return; | ||
177 | } | ||
178 | *estimate = total; | ||
179 | } | ||
180 | |||
181 | |||
182 | /** | ||
183 | * Store an item in the datastore. | ||
184 | * | ||
185 | * @param cls closure with the `struct Plugin` | ||
186 | * @param key key for the item | ||
187 | * @param absent true if the key was not found in the bloom filter | ||
188 | * @param size number of bytes in data | ||
189 | * @param data content stored | ||
190 | * @param type type of the content | ||
191 | * @param priority priority of the content | ||
192 | * @param anonymity anonymity-level for the content | ||
193 | * @param replication replication-level for the content | ||
194 | * @param expiration expiration time for the content | ||
195 | * @param cont continuation called with success or failure status | ||
196 | * @param cont_cls continuation closure | ||
197 | */ | ||
198 | static void | ||
199 | postgres_plugin_put (void *cls, | ||
200 | const struct GNUNET_HashCode *key, | ||
201 | bool absent, | ||
202 | uint32_t size, | ||
203 | const void *data, | ||
204 | enum GNUNET_BLOCK_Type type, | ||
205 | uint32_t priority, | ||
206 | uint32_t anonymity, | ||
207 | uint32_t replication, | ||
208 | struct GNUNET_TIME_Absolute expiration, | ||
209 | PluginPutCont cont, | ||
210 | void *cont_cls) | ||
211 | { | ||
212 | struct Plugin *plugin = cls; | ||
213 | struct GNUNET_HashCode vhash; | ||
214 | enum GNUNET_DB_QueryStatus ret; | ||
215 | |||
216 | GNUNET_CRYPTO_hash (data, | ||
217 | size, | ||
218 | &vhash); | ||
219 | if (! absent) | ||
220 | { | ||
221 | struct GNUNET_PQ_QueryParam params[] = { | ||
222 | GNUNET_PQ_query_param_uint32 (&priority), | ||
223 | GNUNET_PQ_query_param_uint32 (&replication), | ||
224 | GNUNET_PQ_query_param_absolute_time (&expiration), | ||
225 | GNUNET_PQ_query_param_auto_from_type (key), | ||
226 | GNUNET_PQ_query_param_auto_from_type (&vhash), | ||
227 | GNUNET_PQ_query_param_end | ||
228 | }; | ||
229 | ret = GNUNET_PQ_eval_prepared_non_select (plugin->dbh, | ||
230 | "update", | ||
231 | params); | ||
232 | if (0 > ret) | ||
233 | { | ||
234 | cont (cont_cls, | ||
235 | key, | ||
236 | size, | ||
237 | GNUNET_SYSERR, | ||
238 | _ ("Postgresql exec failure")); | ||
239 | return; | ||
240 | } | ||
241 | bool affected = (0 != ret); | ||
242 | if (affected) | ||
243 | { | ||
244 | cont (cont_cls, | ||
245 | key, | ||
246 | size, | ||
247 | GNUNET_NO, | ||
248 | NULL); | ||
249 | return; | ||
250 | } | ||
251 | } | ||
252 | |||
253 | { | ||
254 | uint32_t utype = (uint32_t) type; | ||
255 | uint64_t rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, | ||
256 | UINT64_MAX); | ||
257 | struct GNUNET_PQ_QueryParam params[] = { | ||
258 | GNUNET_PQ_query_param_uint32 (&replication), | ||
259 | GNUNET_PQ_query_param_uint32 (&utype), | ||
260 | GNUNET_PQ_query_param_uint32 (&priority), | ||
261 | GNUNET_PQ_query_param_uint32 (&anonymity), | ||
262 | GNUNET_PQ_query_param_absolute_time (&expiration), | ||
263 | GNUNET_PQ_query_param_uint64 (&rvalue), | ||
264 | GNUNET_PQ_query_param_auto_from_type (key), | ||
265 | GNUNET_PQ_query_param_auto_from_type (&vhash), | ||
266 | GNUNET_PQ_query_param_fixed_size (data, size), | ||
267 | GNUNET_PQ_query_param_end | ||
268 | }; | ||
269 | |||
270 | ret = GNUNET_PQ_eval_prepared_non_select (plugin->dbh, | ||
271 | "put", | ||
272 | params); | ||
273 | if (0 > ret) | ||
274 | { | ||
275 | cont (cont_cls, | ||
276 | key, | ||
277 | size, | ||
278 | GNUNET_SYSERR, | ||
279 | "Postgresql exec failure"); | ||
280 | return; | ||
281 | } | ||
282 | } | ||
283 | plugin->env->duc (plugin->env->cls, | ||
284 | size + GNUNET_DATASTORE_ENTRY_OVERHEAD); | ||
285 | GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, | ||
286 | "datastore-postgres", | ||
287 | "Stored %u bytes in database\n", | ||
288 | (unsigned int) size); | ||
289 | cont (cont_cls, | ||
290 | key, | ||
291 | size, | ||
292 | GNUNET_OK, | ||
293 | NULL); | ||
294 | } | ||
295 | |||
296 | |||
297 | /** | ||
298 | * Closure for #process_result. | ||
299 | */ | ||
300 | struct ProcessResultContext | ||
301 | { | ||
302 | /** | ||
303 | * The plugin handle. | ||
304 | */ | ||
305 | struct Plugin *plugin; | ||
306 | |||
307 | /** | ||
308 | * Function to call on each result. | ||
309 | */ | ||
310 | PluginDatumProcessor proc; | ||
311 | |||
312 | /** | ||
313 | * Closure for @e proc. | ||
314 | */ | ||
315 | void *proc_cls; | ||
316 | }; | ||
317 | |||
318 | |||
319 | /** | ||
320 | * Function invoked to process the result and call the processor of @a | ||
321 | * cls. | ||
322 | * | ||
323 | * @param cls our `struct ProcessResultContext` | ||
324 | * @param res result from exec | ||
325 | * @param num_results number of results in @a res | ||
326 | */ | ||
327 | static void | ||
328 | process_result (void *cls, | ||
329 | PGresult *res, | ||
330 | unsigned int num_results) | ||
331 | { | ||
332 | struct ProcessResultContext *prc = cls; | ||
333 | struct Plugin *plugin = prc->plugin; | ||
334 | |||
335 | if (0 == num_results) | ||
336 | { | ||
337 | /* no result */ | ||
338 | GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, | ||
339 | "datastore-postgres", | ||
340 | "Ending iteration (no more results)\n"); | ||
341 | prc->proc (prc->proc_cls, NULL, 0, NULL, 0, 0, 0, 0, | ||
342 | GNUNET_TIME_UNIT_ZERO_ABS, 0); | ||
343 | return; | ||
344 | } | ||
345 | if (1 != num_results) | ||
346 | { | ||
347 | GNUNET_break (0); | ||
348 | prc->proc (prc->proc_cls, NULL, 0, NULL, 0, 0, 0, 0, | ||
349 | GNUNET_TIME_UNIT_ZERO_ABS, 0); | ||
350 | return; | ||
351 | } | ||
352 | /* Technically we don't need the loop here, but nicer in case | ||
353 | we ever relax the condition above. */ | ||
354 | for (unsigned int i = 0; i < num_results; i++) | ||
355 | { | ||
356 | int iret; | ||
357 | uint64_t rowid; | ||
358 | uint32_t utype; | ||
359 | uint32_t anonymity; | ||
360 | uint32_t replication; | ||
361 | uint32_t priority; | ||
362 | size_t size; | ||
363 | void *data; | ||
364 | struct GNUNET_TIME_Absolute expiration_time; | ||
365 | struct GNUNET_HashCode key; | ||
366 | struct GNUNET_PQ_ResultSpec rs[] = { | ||
367 | GNUNET_PQ_result_spec_uint32 ("repl", &replication), | ||
368 | GNUNET_PQ_result_spec_uint32 ("type", &utype), | ||
369 | GNUNET_PQ_result_spec_uint32 ("prio", &priority), | ||
370 | GNUNET_PQ_result_spec_uint32 ("anonLevel", &anonymity), | ||
371 | GNUNET_PQ_result_spec_absolute_time ("expire", &expiration_time), | ||
372 | GNUNET_PQ_result_spec_auto_from_type ("hash", &key), | ||
373 | GNUNET_PQ_result_spec_variable_size ("value", &data, &size), | ||
374 | GNUNET_PQ_result_spec_uint64 ("oid", &rowid), | ||
375 | GNUNET_PQ_result_spec_end | ||
376 | }; | ||
377 | |||
378 | if (GNUNET_OK != | ||
379 | GNUNET_PQ_extract_result (res, | ||
380 | rs, | ||
381 | i)) | ||
382 | { | ||
383 | GNUNET_break (0); | ||
384 | prc->proc (prc->proc_cls, NULL, 0, NULL, 0, 0, 0, 0, | ||
385 | GNUNET_TIME_UNIT_ZERO_ABS, 0); | ||
386 | return; | ||
387 | } | ||
388 | |||
389 | GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, | ||
390 | "datastore-postgres", | ||
391 | "Found result of size %u bytes and type %u in database\n", | ||
392 | (unsigned int) size, | ||
393 | (unsigned int) utype); | ||
394 | iret = prc->proc (prc->proc_cls, | ||
395 | &key, | ||
396 | size, | ||
397 | data, | ||
398 | (enum GNUNET_BLOCK_Type) utype, | ||
399 | priority, | ||
400 | anonymity, | ||
401 | replication, | ||
402 | expiration_time, | ||
403 | rowid); | ||
404 | if (iret == GNUNET_NO) | ||
405 | { | ||
406 | struct GNUNET_PQ_QueryParam param[] = { | ||
407 | GNUNET_PQ_query_param_uint64 (&rowid), | ||
408 | GNUNET_PQ_query_param_end | ||
409 | }; | ||
410 | |||
411 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
412 | "Processor asked for item %u to be removed.\n", | ||
413 | (unsigned int) rowid); | ||
414 | if (0 < | ||
415 | GNUNET_PQ_eval_prepared_non_select (plugin->dbh, | ||
416 | "delrow", | ||
417 | param)) | ||
418 | { | ||
419 | GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, | ||
420 | "datastore-postgres", | ||
421 | "Deleting %u bytes from database\n", | ||
422 | (unsigned int) size); | ||
423 | plugin->env->duc (plugin->env->cls, | ||
424 | -(size + GNUNET_DATASTORE_ENTRY_OVERHEAD)); | ||
425 | GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, | ||
426 | "datastore-postgres", | ||
427 | "Deleted %u bytes from database\n", | ||
428 | (unsigned int) size); | ||
429 | } | ||
430 | } | ||
431 | GNUNET_PQ_cleanup_result (rs); | ||
432 | } /* for (i) */ | ||
433 | } | ||
434 | |||
435 | |||
436 | /** | ||
437 | * Get one of the results for a particular key in the datastore. | ||
438 | * | ||
439 | * @param cls closure with the `struct Plugin` | ||
440 | * @param next_uid return the result with lowest uid >= next_uid | ||
441 | * @param random if true, return a random result instead of using next_uid | ||
442 | * @param key maybe NULL (to match all entries) | ||
443 | * @param type entries of which type are relevant? | ||
444 | * Use 0 for any type. | ||
445 | * @param proc function to call on the matching value; | ||
446 | * will be called with NULL if nothing matches | ||
447 | * @param proc_cls closure for @a proc | ||
448 | */ | ||
449 | static void | ||
450 | postgres_plugin_get_key (void *cls, | ||
451 | uint64_t next_uid, | ||
452 | bool random, | ||
453 | const struct GNUNET_HashCode *key, | ||
454 | enum GNUNET_BLOCK_Type type, | ||
455 | PluginDatumProcessor proc, | ||
456 | void *proc_cls) | ||
457 | { | ||
458 | struct Plugin *plugin = cls; | ||
459 | uint32_t utype = type; | ||
460 | uint16_t use_rvalue = random; | ||
461 | uint16_t use_key = NULL != key; | ||
462 | uint16_t use_type = GNUNET_BLOCK_TYPE_ANY != type; | ||
463 | uint64_t rvalue; | ||
464 | struct GNUNET_PQ_QueryParam params[] = { | ||
465 | GNUNET_PQ_query_param_uint64 (&next_uid), | ||
466 | GNUNET_PQ_query_param_uint64 (&rvalue), | ||
467 | GNUNET_PQ_query_param_uint16 (&use_rvalue), | ||
468 | GNUNET_PQ_query_param_auto_from_type (key), | ||
469 | GNUNET_PQ_query_param_uint16 (&use_key), | ||
470 | GNUNET_PQ_query_param_uint32 (&utype), | ||
471 | GNUNET_PQ_query_param_uint16 (&use_type), | ||
472 | GNUNET_PQ_query_param_end | ||
473 | }; | ||
474 | struct ProcessResultContext prc; | ||
475 | enum GNUNET_DB_QueryStatus res; | ||
476 | |||
477 | if (random) | ||
478 | { | ||
479 | rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, | ||
480 | UINT64_MAX); | ||
481 | next_uid = 0; | ||
482 | } | ||
483 | else | ||
484 | { | ||
485 | rvalue = 0; | ||
486 | } | ||
487 | prc.plugin = plugin; | ||
488 | prc.proc = proc; | ||
489 | prc.proc_cls = proc_cls; | ||
490 | |||
491 | res = GNUNET_PQ_eval_prepared_multi_select (plugin->dbh, | ||
492 | "get", | ||
493 | params, | ||
494 | &process_result, | ||
495 | &prc); | ||
496 | if (0 > res) | ||
497 | proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, | ||
498 | GNUNET_TIME_UNIT_ZERO_ABS, 0); | ||
499 | } | ||
500 | |||
501 | |||
502 | /** | ||
503 | * Select a subset of the items in the datastore and call | ||
504 | * the given iterator for each of them. | ||
505 | * | ||
506 | * @param cls our `struct Plugin *` | ||
507 | * @param next_uid return the result with lowest uid >= next_uid | ||
508 | * @param type entries of which type should be considered? | ||
509 | * Must not be zero (ANY). | ||
510 | * @param proc function to call on the matching value; | ||
511 | * will be called with NULL if no value matches | ||
512 | * @param proc_cls closure for @a proc | ||
513 | */ | ||
514 | static void | ||
515 | postgres_plugin_get_zero_anonymity (void *cls, | ||
516 | uint64_t next_uid, | ||
517 | enum GNUNET_BLOCK_Type type, | ||
518 | PluginDatumProcessor proc, | ||
519 | void *proc_cls) | ||
520 | { | ||
521 | struct Plugin *plugin = cls; | ||
522 | uint32_t utype = type; | ||
523 | struct GNUNET_PQ_QueryParam params[] = { | ||
524 | GNUNET_PQ_query_param_uint32 (&utype), | ||
525 | GNUNET_PQ_query_param_uint64 (&next_uid), | ||
526 | GNUNET_PQ_query_param_end | ||
527 | }; | ||
528 | struct ProcessResultContext prc; | ||
529 | enum GNUNET_DB_QueryStatus res; | ||
530 | |||
531 | prc.plugin = plugin; | ||
532 | prc.proc = proc; | ||
533 | prc.proc_cls = proc_cls; | ||
534 | res = GNUNET_PQ_eval_prepared_multi_select (plugin->dbh, | ||
535 | "select_non_anonymous", | ||
536 | params, | ||
537 | &process_result, | ||
538 | &prc); | ||
539 | if (0 > res) | ||
540 | proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, | ||
541 | GNUNET_TIME_UNIT_ZERO_ABS, 0); | ||
542 | } | ||
543 | |||
544 | |||
545 | /** | ||
546 | * Context for #repl_iter() function. | ||
547 | */ | ||
548 | struct ReplCtx | ||
549 | { | ||
550 | /** | ||
551 | * Plugin handle. | ||
552 | */ | ||
553 | struct Plugin *plugin; | ||
554 | |||
555 | /** | ||
556 | * Function to call for the result (or the NULL). | ||
557 | */ | ||
558 | PluginDatumProcessor proc; | ||
559 | |||
560 | /** | ||
561 | * Closure for @e proc. | ||
562 | */ | ||
563 | void *proc_cls; | ||
564 | }; | ||
565 | |||
566 | |||
567 | /** | ||
568 | * Wrapper for the iterator for 'sqlite_plugin_replication_get'. | ||
569 | * Decrements the replication counter and calls the original | ||
570 | * iterator. | ||
571 | * | ||
572 | * @param cls closure with the `struct ReplCtx *` | ||
573 | * @param key key for the content | ||
574 | * @param size number of bytes in @a data | ||
575 | * @param data content stored | ||
576 | * @param type type of the content | ||
577 | * @param priority priority of the content | ||
578 | * @param anonymity anonymity-level for the content | ||
579 | * @param replication replication-level for the content | ||
580 | * @param expiration expiration time for the content | ||
581 | * @param uid unique identifier for the datum; | ||
582 | * maybe 0 if no unique identifier is available | ||
583 | * @return #GNUNET_SYSERR to abort the iteration, | ||
584 | * #GNUNET_OK to continue | ||
585 | * (continue on call to "next", of course), | ||
586 | * #GNUNET_NO to delete the item and continue (if supported) | ||
587 | */ | ||
588 | static int | ||
589 | repl_proc (void *cls, | ||
590 | const struct GNUNET_HashCode *key, | ||
591 | uint32_t size, | ||
592 | const void *data, | ||
593 | enum GNUNET_BLOCK_Type type, | ||
594 | uint32_t priority, | ||
595 | uint32_t anonymity, | ||
596 | uint32_t replication, | ||
597 | struct GNUNET_TIME_Absolute expiration, | ||
598 | uint64_t uid) | ||
599 | { | ||
600 | struct ReplCtx *rc = cls; | ||
601 | struct Plugin *plugin = rc->plugin; | ||
602 | int ret; | ||
603 | struct GNUNET_PQ_QueryParam params[] = { | ||
604 | GNUNET_PQ_query_param_uint64 (&uid), | ||
605 | GNUNET_PQ_query_param_end | ||
606 | }; | ||
607 | enum GNUNET_DB_QueryStatus qret; | ||
608 | |||
609 | ret = rc->proc (rc->proc_cls, | ||
610 | key, | ||
611 | size, | ||
612 | data, | ||
613 | type, | ||
614 | priority, | ||
615 | anonymity, | ||
616 | replication, | ||
617 | expiration, | ||
618 | uid); | ||
619 | if (NULL == key) | ||
620 | return ret; | ||
621 | qret = GNUNET_PQ_eval_prepared_non_select (plugin->dbh, | ||
622 | "decrepl", | ||
623 | params); | ||
624 | if (0 > qret) | ||
625 | return GNUNET_SYSERR; | ||
626 | return ret; | ||
627 | } | ||
628 | |||
629 | |||
630 | /** | ||
631 | * Get a random item for replication. Returns a single, not expired, | ||
632 | * random item from those with the highest replication counters. The | ||
633 | * item's replication counter is decremented by one IF it was positive | ||
634 | * before. Call @a proc with all values ZERO or NULL if the datastore | ||
635 | * is empty. | ||
636 | * | ||
637 | * @param cls closure with the `struct Plugin` | ||
638 | * @param proc function to call the value (once only). | ||
639 | * @param proc_cls closure for @a proc | ||
640 | */ | ||
641 | static void | ||
642 | postgres_plugin_get_replication (void *cls, | ||
643 | PluginDatumProcessor proc, | ||
644 | void *proc_cls) | ||
645 | { | ||
646 | struct Plugin *plugin = cls; | ||
647 | struct GNUNET_PQ_QueryParam params[] = { | ||
648 | GNUNET_PQ_query_param_end | ||
649 | }; | ||
650 | struct ReplCtx rc; | ||
651 | struct ProcessResultContext prc; | ||
652 | enum GNUNET_DB_QueryStatus res; | ||
653 | |||
654 | rc.plugin = plugin; | ||
655 | rc.proc = proc; | ||
656 | rc.proc_cls = proc_cls; | ||
657 | prc.plugin = plugin; | ||
658 | prc.proc = &repl_proc; | ||
659 | prc.proc_cls = &rc; | ||
660 | res = GNUNET_PQ_eval_prepared_multi_select (plugin->dbh, | ||
661 | "select_replication_order", | ||
662 | params, | ||
663 | &process_result, | ||
664 | &prc); | ||
665 | if (0 > res) | ||
666 | proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, | ||
667 | GNUNET_TIME_UNIT_ZERO_ABS, 0); | ||
668 | } | ||
669 | |||
670 | |||
671 | /** | ||
672 | * Get a random item for expiration. Call @a proc with all values | ||
673 | * ZERO or NULL if the datastore is empty. | ||
674 | * | ||
675 | * @param cls closure with the `struct Plugin` | ||
676 | * @param proc function to call the value (once only). | ||
677 | * @param proc_cls closure for @a proc | ||
678 | */ | ||
679 | static void | ||
680 | postgres_plugin_get_expiration (void *cls, | ||
681 | PluginDatumProcessor proc, | ||
682 | void *proc_cls) | ||
683 | { | ||
684 | struct Plugin *plugin = cls; | ||
685 | struct GNUNET_TIME_Absolute now = { 0 }; | ||
686 | struct GNUNET_PQ_QueryParam params[] = { | ||
687 | GNUNET_PQ_query_param_absolute_time (&now), | ||
688 | GNUNET_PQ_query_param_end | ||
689 | }; | ||
690 | struct ProcessResultContext prc; | ||
691 | |||
692 | now = GNUNET_TIME_absolute_get (); | ||
693 | prc.plugin = plugin; | ||
694 | prc.proc = proc; | ||
695 | prc.proc_cls = proc_cls; | ||
696 | (void) GNUNET_PQ_eval_prepared_multi_select (plugin->dbh, | ||
697 | "select_expiration_order", | ||
698 | params, | ||
699 | &process_result, | ||
700 | &prc); | ||
701 | } | ||
702 | |||
703 | |||
704 | /** | ||
705 | * Closure for #process_keys. | ||
706 | */ | ||
707 | struct ProcessKeysContext | ||
708 | { | ||
709 | /** | ||
710 | * Function to call for each key. | ||
711 | */ | ||
712 | PluginKeyProcessor proc; | ||
713 | |||
714 | /** | ||
715 | * Closure for @e proc. | ||
716 | */ | ||
717 | void *proc_cls; | ||
718 | }; | ||
719 | |||
720 | |||
721 | /** | ||
722 | * Function to be called with the results of a SELECT statement | ||
723 | * that has returned @a num_results results. | ||
724 | * | ||
725 | * @param cls closure with a `struct ProcessKeysContext` | ||
726 | * @param result the postgres result | ||
727 | * @param num_results the number of results in @a result | ||
728 | */ | ||
729 | static void | ||
730 | process_keys (void *cls, | ||
731 | PGresult *result, | ||
732 | unsigned int num_results) | ||
733 | { | ||
734 | struct ProcessKeysContext *pkc = cls; | ||
735 | |||
736 | for (unsigned i = 0; i < num_results; i++) | ||
737 | { | ||
738 | struct GNUNET_HashCode key; | ||
739 | struct GNUNET_PQ_ResultSpec rs[] = { | ||
740 | GNUNET_PQ_result_spec_auto_from_type ("hash", | ||
741 | &key), | ||
742 | GNUNET_PQ_result_spec_end | ||
743 | }; | ||
744 | |||
745 | if (GNUNET_OK != | ||
746 | GNUNET_PQ_extract_result (result, | ||
747 | rs, | ||
748 | i)) | ||
749 | { | ||
750 | GNUNET_break (0); | ||
751 | continue; | ||
752 | } | ||
753 | pkc->proc (pkc->proc_cls, | ||
754 | &key, | ||
755 | 1); | ||
756 | GNUNET_PQ_cleanup_result (rs); | ||
757 | } | ||
758 | } | ||
759 | |||
760 | |||
761 | /** | ||
762 | * Get all of the keys in the datastore. | ||
763 | * | ||
764 | * @param cls closure with the `struct Plugin *` | ||
765 | * @param proc function to call on each key | ||
766 | * @param proc_cls closure for @a proc | ||
767 | */ | ||
768 | static void | ||
769 | postgres_plugin_get_keys (void *cls, | ||
770 | PluginKeyProcessor proc, | ||
771 | void *proc_cls) | ||
772 | { | ||
773 | struct Plugin *plugin = cls; | ||
774 | struct GNUNET_PQ_QueryParam params[] = { | ||
775 | GNUNET_PQ_query_param_end | ||
776 | }; | ||
777 | struct ProcessKeysContext pkc; | ||
778 | |||
779 | pkc.proc = proc; | ||
780 | pkc.proc_cls = proc_cls; | ||
781 | (void) GNUNET_PQ_eval_prepared_multi_select (plugin->dbh, | ||
782 | "get_keys", | ||
783 | params, | ||
784 | &process_keys, | ||
785 | &pkc); | ||
786 | proc (proc_cls, | ||
787 | NULL, | ||
788 | 0); | ||
789 | } | ||
790 | |||
791 | |||
792 | /** | ||
793 | * Drop database. | ||
794 | * | ||
795 | * @param cls closure with the `struct Plugin *` | ||
796 | */ | ||
797 | static void | ||
798 | postgres_plugin_drop (void *cls) | ||
799 | { | ||
800 | struct Plugin *plugin = cls; | ||
801 | struct GNUNET_PQ_ExecuteStatement es[] = { | ||
802 | GNUNET_PQ_make_execute ("DROP TABLE gn090"), | ||
803 | GNUNET_PQ_EXECUTE_STATEMENT_END | ||
804 | }; | ||
805 | |||
806 | if (GNUNET_OK != | ||
807 | GNUNET_PQ_exec_statements (plugin->dbh, | ||
808 | es)) | ||
809 | GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING, | ||
810 | "postgres", | ||
811 | _ ("Failed to drop table from database.\n")); | ||
812 | } | ||
813 | |||
814 | |||
815 | /** | ||
816 | * Remove a particular key in the datastore. | ||
817 | * | ||
818 | * @param cls closure | ||
819 | * @param key key for the content | ||
820 | * @param size number of bytes in data | ||
821 | * @param data content stored | ||
822 | * @param cont continuation called with success or failure status | ||
823 | * @param cont_cls continuation closure for @a cont | ||
824 | */ | ||
825 | static void | ||
826 | postgres_plugin_remove_key (void *cls, | ||
827 | const struct GNUNET_HashCode *key, | ||
828 | uint32_t size, | ||
829 | const void *data, | ||
830 | PluginRemoveCont cont, | ||
831 | void *cont_cls) | ||
832 | { | ||
833 | struct Plugin *plugin = cls; | ||
834 | enum GNUNET_DB_QueryStatus ret; | ||
835 | struct GNUNET_PQ_QueryParam params[] = { | ||
836 | GNUNET_PQ_query_param_auto_from_type (key), | ||
837 | GNUNET_PQ_query_param_fixed_size (data, size), | ||
838 | GNUNET_PQ_query_param_end | ||
839 | }; | ||
840 | |||
841 | ret = GNUNET_PQ_eval_prepared_non_select (plugin->dbh, | ||
842 | "remove", | ||
843 | params); | ||
844 | if (0 > ret) | ||
845 | { | ||
846 | cont (cont_cls, | ||
847 | key, | ||
848 | size, | ||
849 | GNUNET_SYSERR, | ||
850 | _ ("Postgresql exec failure")); | ||
851 | return; | ||
852 | } | ||
853 | if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == ret) | ||
854 | { | ||
855 | cont (cont_cls, | ||
856 | key, | ||
857 | size, | ||
858 | GNUNET_NO, | ||
859 | NULL); | ||
860 | return; | ||
861 | } | ||
862 | plugin->env->duc (plugin->env->cls, | ||
863 | -(size + GNUNET_DATASTORE_ENTRY_OVERHEAD)); | ||
864 | GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, | ||
865 | "datastore-postgres", | ||
866 | "Deleted %u bytes from database\n", | ||
867 | (unsigned int) size); | ||
868 | cont (cont_cls, | ||
869 | key, | ||
870 | size, | ||
871 | GNUNET_OK, | ||
872 | NULL); | ||
873 | } | ||
874 | |||
875 | |||
876 | /** | ||
877 | * Entry point for the plugin. | ||
878 | * | ||
879 | * @param cls the `struct GNUNET_DATASTORE_PluginEnvironment*` | ||
880 | * @return our `struct Plugin *` | ||
881 | */ | ||
882 | void * | ||
883 | libgnunet_plugin_datastore_postgres_init (void *cls) | ||
884 | { | ||
885 | struct GNUNET_DATASTORE_PluginEnvironment *env = cls; | ||
886 | struct GNUNET_DATASTORE_PluginFunctions *api; | ||
887 | struct Plugin *plugin; | ||
888 | |||
889 | plugin = GNUNET_new (struct Plugin); | ||
890 | plugin->env = env; | ||
891 | if (GNUNET_OK != init_connection (plugin)) | ||
892 | { | ||
893 | GNUNET_free (plugin); | ||
894 | return NULL; | ||
895 | } | ||
896 | api = GNUNET_new (struct GNUNET_DATASTORE_PluginFunctions); | ||
897 | api->cls = plugin; | ||
898 | api->estimate_size = &postgres_plugin_estimate_size; | ||
899 | api->put = &postgres_plugin_put; | ||
900 | api->get_key = &postgres_plugin_get_key; | ||
901 | api->get_replication = &postgres_plugin_get_replication; | ||
902 | api->get_expiration = &postgres_plugin_get_expiration; | ||
903 | api->get_zero_anonymity = &postgres_plugin_get_zero_anonymity; | ||
904 | api->get_keys = &postgres_plugin_get_keys; | ||
905 | api->drop = &postgres_plugin_drop; | ||
906 | api->remove_key = &postgres_plugin_remove_key; | ||
907 | return api; | ||
908 | } | ||
909 | |||
910 | |||
911 | /** | ||
912 | * Exit point from the plugin. | ||
913 | * | ||
914 | * @param cls our `struct Plugin *` | ||
915 | * @return always NULL | ||
916 | */ | ||
917 | void * | ||
918 | libgnunet_plugin_datastore_postgres_done (void *cls) | ||
919 | { | ||
920 | struct GNUNET_DATASTORE_PluginFunctions *api = cls; | ||
921 | struct Plugin *plugin = api->cls; | ||
922 | |||
923 | GNUNET_PQ_disconnect (plugin->dbh); | ||
924 | GNUNET_free (plugin); | ||
925 | GNUNET_free (api); | ||
926 | return NULL; | ||
927 | } | ||
928 | |||
929 | |||
930 | /* end of plugin_datastore_postgres.c */ | ||
diff --git a/src/plugin/datastore/plugin_datastore_sqlite.c b/src/plugin/datastore/plugin_datastore_sqlite.c new file mode 100644 index 000000000..5ea9da4cb --- /dev/null +++ b/src/plugin/datastore/plugin_datastore_sqlite.c | |||
@@ -0,0 +1,1375 @@ | |||
1 | /* | ||
2 | * This file is part of GNUnet | ||
3 | * Copyright (C) 2009, 2011, 2017 GNUnet e.V. | ||
4 | * | ||
5 | * GNUnet is free software: you can redistribute it and/or modify it | ||
6 | * under the terms of the GNU Affero General Public License as published | ||
7 | * by the Free Software Foundation, either version 3 of the License, | ||
8 | * or (at your option) any later version. | ||
9 | * | ||
10 | * GNUnet is distributed in the hope that it will be useful, but | ||
11 | * WITHOUT ANY WARRANTY; without even the implied warranty of | ||
12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
13 | * Affero General Public License for more details. | ||
14 | * | ||
15 | * You should have received a copy of the GNU Affero General Public License | ||
16 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
17 | |||
18 | SPDX-License-Identifier: AGPL3.0-or-later | ||
19 | */ | ||
20 | |||
21 | /** | ||
22 | * @file datastore/plugin_datastore_sqlite.c | ||
23 | * @brief sqlite-based datastore backend | ||
24 | * @author Christian Grothoff | ||
25 | */ | ||
26 | |||
27 | #include "platform.h" | ||
28 | #include "gnunet_datastore_plugin.h" | ||
29 | #include "gnunet_sq_lib.h" | ||
30 | #include <sqlite3.h> | ||
31 | |||
32 | |||
33 | /** | ||
34 | * We allocate items on the stack at times. To prevent a stack | ||
35 | * overflow, we impose a limit on the maximum size for the data per | ||
36 | * item. 64k should be enough. | ||
37 | */ | ||
38 | #define MAX_ITEM_SIZE 65536 | ||
39 | |||
40 | /** | ||
41 | * After how many ms "busy" should a DB operation fail for good? | ||
42 | * A low value makes sure that we are more responsive to requests | ||
43 | * (especially PUTs). A high value guarantees a higher success | ||
44 | * rate (SELECTs in iterate can take several seconds despite LIMIT=1). | ||
45 | * | ||
46 | * The default value of 250ms should ensure that users do not experience | ||
47 | * huge latencies while at the same time allowing operations to succeed | ||
48 | * with reasonable probability. | ||
49 | */ | ||
50 | #define BUSY_TIMEOUT_MS 250 | ||
51 | |||
52 | |||
53 | /** | ||
54 | * Log an error message at log-level 'level' that indicates | ||
55 | * a failure of the command 'cmd' on file 'filename' | ||
56 | * with the message given by strerror(errno). | ||
57 | */ | ||
58 | #define LOG_SQLITE(db, level, cmd) \ | ||
59 | do \ | ||
60 | { \ | ||
61 | GNUNET_log_from (level, \ | ||
62 | "sqlite", \ | ||
63 | _ ("`%s' failed at %s:%d with error: %s\n"), \ | ||
64 | cmd, \ | ||
65 | __FILE__, \ | ||
66 | __LINE__, \ | ||
67 | sqlite3_errmsg (db->dbh)); \ | ||
68 | } while (0) | ||
69 | |||
70 | |||
71 | /** | ||
72 | * Log an error message at log-level 'level' that indicates | ||
73 | * a failure of the command 'cmd' on file 'filename' | ||
74 | * with the message given by strerror(errno). | ||
75 | */ | ||
76 | #define LOG_SQLITE_MSG(db, msg, level, cmd) \ | ||
77 | do \ | ||
78 | { \ | ||
79 | GNUNET_log_from (level, \ | ||
80 | "sqlite", \ | ||
81 | _ ("`%s' failed at %s:%d with error: %s\n"), \ | ||
82 | cmd, \ | ||
83 | __FILE__, \ | ||
84 | __LINE__, \ | ||
85 | sqlite3_errmsg (db->dbh)); \ | ||
86 | GNUNET_asprintf (msg, \ | ||
87 | _ ("`%s' failed at %s:%u with error: %s"), \ | ||
88 | cmd, \ | ||
89 | __FILE__, \ | ||
90 | __LINE__, \ | ||
91 | sqlite3_errmsg (db->dbh)); \ | ||
92 | } while (0) | ||
93 | |||
94 | |||
95 | /** | ||
96 | * Context for all functions in this plugin. | ||
97 | */ | ||
98 | struct Plugin | ||
99 | { | ||
100 | /** | ||
101 | * Our execution environment. | ||
102 | */ | ||
103 | struct GNUNET_DATASTORE_PluginEnvironment *env; | ||
104 | |||
105 | /** | ||
106 | * Database filename. | ||
107 | */ | ||
108 | char *fn; | ||
109 | |||
110 | /** | ||
111 | * Native SQLite database handle. | ||
112 | */ | ||
113 | sqlite3 *dbh; | ||
114 | |||
115 | /** | ||
116 | * Precompiled SQL for remove_key. | ||
117 | */ | ||
118 | sqlite3_stmt *remove; | ||
119 | |||
120 | /** | ||
121 | * Precompiled SQL for deletion. | ||
122 | */ | ||
123 | sqlite3_stmt *delRow; | ||
124 | |||
125 | /** | ||
126 | * Precompiled SQL for update. | ||
127 | */ | ||
128 | sqlite3_stmt *update; | ||
129 | |||
130 | /** | ||
131 | * Get maximum repl value in database. | ||
132 | */ | ||
133 | sqlite3_stmt *maxRepl; | ||
134 | |||
135 | /** | ||
136 | * Precompiled SQL for replication decrement. | ||
137 | */ | ||
138 | sqlite3_stmt *updRepl; | ||
139 | |||
140 | /** | ||
141 | * Precompiled SQL for replication selection. | ||
142 | */ | ||
143 | sqlite3_stmt *selRepl; | ||
144 | |||
145 | /** | ||
146 | * Precompiled SQL for expiration selection. | ||
147 | */ | ||
148 | sqlite3_stmt *selExpi; | ||
149 | |||
150 | /** | ||
151 | * Precompiled SQL for expiration selection. | ||
152 | */ | ||
153 | sqlite3_stmt *selZeroAnon; | ||
154 | |||
155 | /** | ||
156 | * Precompiled SQL for insertion. | ||
157 | */ | ||
158 | sqlite3_stmt *insertContent; | ||
159 | |||
160 | /** | ||
161 | * Precompiled SQL for selection | ||
162 | */ | ||
163 | sqlite3_stmt *get[8]; | ||
164 | |||
165 | /** | ||
166 | * Should the database be dropped on shutdown? | ||
167 | */ | ||
168 | int drop_on_shutdown; | ||
169 | }; | ||
170 | |||
171 | |||
172 | /** | ||
173 | * @brief Prepare a SQL statement | ||
174 | * | ||
175 | * @param dbh handle to the database | ||
176 | * @param zSql SQL statement, UTF-8 encoded | ||
177 | * @param ppStmt set to the prepared statement | ||
178 | * @return 0 on success | ||
179 | */ | ||
180 | static int | ||
181 | sq_prepare (sqlite3 *dbh, const char *zSql, sqlite3_stmt **ppStmt) | ||
182 | { | ||
183 | char *dummy; | ||
184 | int result; | ||
185 | |||
186 | result = sqlite3_prepare_v2 (dbh, | ||
187 | zSql, | ||
188 | strlen (zSql), | ||
189 | ppStmt, | ||
190 | (const char **) &dummy); | ||
191 | GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, | ||
192 | "sqlite", | ||
193 | "Prepared `%s' / %p: %d\n", | ||
194 | zSql, | ||
195 | *ppStmt, | ||
196 | result); | ||
197 | return result; | ||
198 | } | ||
199 | |||
200 | |||
201 | /** | ||
202 | * Create our database indices. | ||
203 | * | ||
204 | * @param dbh handle to the database | ||
205 | */ | ||
206 | static void | ||
207 | create_indices (sqlite3 *dbh) | ||
208 | { | ||
209 | /* create indices */ | ||
210 | if ( | ||
211 | 0 != | ||
212 | (SQLITE_OK != | ||
213 | sqlite3_exec (dbh, | ||
214 | "CREATE INDEX IF NOT EXISTS idx_hash ON gn091 (hash)", | ||
215 | NULL, | ||
216 | NULL, | ||
217 | NULL)) | ||
218 | + (SQLITE_OK != | ||
219 | sqlite3_exec ( | ||
220 | dbh, | ||
221 | "CREATE INDEX IF NOT EXISTS idx_anon_type ON gn091 (anonLevel ASC,type)", | ||
222 | NULL, | ||
223 | NULL, | ||
224 | NULL)) | ||
225 | + (SQLITE_OK != | ||
226 | sqlite3_exec (dbh, | ||
227 | "CREATE INDEX IF NOT EXISTS idx_expire ON gn091 (expire ASC)", | ||
228 | NULL, | ||
229 | NULL, | ||
230 | NULL)) | ||
231 | + (SQLITE_OK != | ||
232 | sqlite3_exec ( | ||
233 | dbh, | ||
234 | "CREATE INDEX IF NOT EXISTS idx_repl_rvalue ON gn091 (repl,rvalue)", | ||
235 | NULL, | ||
236 | NULL, | ||
237 | NULL))) | ||
238 | GNUNET_log_from (GNUNET_ERROR_TYPE_ERROR, | ||
239 | "sqlite", | ||
240 | "Failed to create indices: %s\n", | ||
241 | sqlite3_errmsg (dbh)); | ||
242 | } | ||
243 | |||
244 | |||
245 | #if 0 | ||
246 | #define CHECK(a) GNUNET_break (a) | ||
247 | #define ENULL NULL | ||
248 | #else | ||
249 | #define ENULL &e | ||
250 | #define ENULL_DEFINED 1 | ||
251 | #define CHECK(a) \ | ||
252 | if (! (a)) \ | ||
253 | { \ | ||
254 | GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "%s\n", e); \ | ||
255 | sqlite3_free (e); \ | ||
256 | e = NULL; \ | ||
257 | } | ||
258 | #endif | ||
259 | |||
260 | |||
261 | /** | ||
262 | * Initialize the database connections and associated | ||
263 | * data structures (create tables and indices | ||
264 | * as needed as well). | ||
265 | * | ||
266 | * @param cfg our configuration | ||
267 | * @param plugin the plugin context (state for this module) | ||
268 | * @return #GNUNET_OK on success | ||
269 | */ | ||
270 | static int | ||
271 | database_setup (const struct GNUNET_CONFIGURATION_Handle *cfg, | ||
272 | struct Plugin *plugin) | ||
273 | { | ||
274 | sqlite3_stmt *stmt; | ||
275 | char *afsdir; | ||
276 | |||
277 | #if ENULL_DEFINED | ||
278 | char *e; | ||
279 | #endif | ||
280 | |||
281 | if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_filename (cfg, | ||
282 | "datastore-sqlite", | ||
283 | "FILENAME", | ||
284 | &afsdir)) | ||
285 | { | ||
286 | GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR, | ||
287 | "datastore-sqlite", | ||
288 | "FILENAME"); | ||
289 | return GNUNET_SYSERR; | ||
290 | } | ||
291 | if (GNUNET_OK != GNUNET_DISK_file_test (afsdir)) | ||
292 | { | ||
293 | if (GNUNET_OK != GNUNET_DISK_directory_create_for_file (afsdir)) | ||
294 | { | ||
295 | GNUNET_break (0); | ||
296 | GNUNET_free (afsdir); | ||
297 | return GNUNET_SYSERR; | ||
298 | } | ||
299 | /* database is new or got deleted, reset payload to zero! */ | ||
300 | if (NULL != plugin->env->duc) | ||
301 | plugin->env->duc (plugin->env->cls, 0); | ||
302 | } | ||
303 | /* afsdir should be UTF-8-encoded. If it isn't, it's a bug */ | ||
304 | plugin->fn = afsdir; | ||
305 | |||
306 | /* Open database and precompile statements */ | ||
307 | if (SQLITE_OK != sqlite3_open (plugin->fn, &plugin->dbh)) | ||
308 | { | ||
309 | GNUNET_log_from (GNUNET_ERROR_TYPE_ERROR, | ||
310 | "sqlite", | ||
311 | _ ("Unable to initialize SQLite: %s.\n"), | ||
312 | sqlite3_errmsg (plugin->dbh)); | ||
313 | return GNUNET_SYSERR; | ||
314 | } | ||
315 | CHECK ( | ||
316 | SQLITE_OK == | ||
317 | sqlite3_exec (plugin->dbh, "PRAGMA temp_store=MEMORY", NULL, NULL, ENULL)); | ||
318 | CHECK ( | ||
319 | SQLITE_OK == | ||
320 | sqlite3_exec (plugin->dbh, "PRAGMA synchronous=OFF", NULL, NULL, ENULL)); | ||
321 | CHECK (SQLITE_OK == sqlite3_exec (plugin->dbh, | ||
322 | "PRAGMA legacy_file_format=OFF", | ||
323 | NULL, | ||
324 | NULL, | ||
325 | ENULL)); | ||
326 | CHECK (SQLITE_OK == sqlite3_exec (plugin->dbh, | ||
327 | "PRAGMA auto_vacuum=INCREMENTAL", | ||
328 | NULL, | ||
329 | NULL, | ||
330 | ENULL)); | ||
331 | CHECK (SQLITE_OK == sqlite3_exec (plugin->dbh, | ||
332 | "PRAGMA locking_mode=EXCLUSIVE", | ||
333 | NULL, | ||
334 | NULL, | ||
335 | ENULL)); | ||
336 | CHECK ( | ||
337 | SQLITE_OK == | ||
338 | sqlite3_exec (plugin->dbh, "PRAGMA page_size=4096", NULL, NULL, ENULL)); | ||
339 | |||
340 | CHECK (SQLITE_OK == sqlite3_busy_timeout (plugin->dbh, BUSY_TIMEOUT_MS)); | ||
341 | |||
342 | |||
343 | /* We have to do it here, because otherwise precompiling SQL might fail */ | ||
344 | CHECK (SQLITE_OK == | ||
345 | sq_prepare (plugin->dbh, | ||
346 | "SELECT 1 FROM sqlite_master WHERE tbl_name = 'gn091'", | ||
347 | &stmt)); | ||
348 | |||
349 | /* FIXME: SQLite does not have unsigned integers! This is ok for the type column because | ||
350 | * we only test equality on it and can cast it to/from uint32_t. For repl, prio, and anonLevel | ||
351 | * we do math or inequality tests, so we can't handle the entire range of uint32_t. | ||
352 | * This will also cause problems for expiration times after 294247-01-10-04:00:54 UTC. | ||
353 | */if ((SQLITE_DONE == sqlite3_step (stmt)) && | ||
354 | (SQLITE_OK != sqlite3_exec (plugin->dbh, | ||
355 | "CREATE TABLE gn091 (" | ||
356 | " repl INT4 NOT NULL DEFAULT 0," | ||
357 | " type INT4 NOT NULL DEFAULT 0," | ||
358 | " prio INT4 NOT NULL DEFAULT 0," | ||
359 | " anonLevel INT4 NOT NULL DEFAULT 0," | ||
360 | " expire INT8 NOT NULL DEFAULT 0," | ||
361 | " rvalue INT8 NOT NULL," | ||
362 | " hash TEXT NOT NULL DEFAULT ''," | ||
363 | " vhash TEXT NOT NULL DEFAULT ''," | ||
364 | " value BLOB NOT NULL DEFAULT '')", | ||
365 | NULL, | ||
366 | NULL, | ||
367 | NULL))) | ||
368 | { | ||
369 | LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "sqlite3_exec"); | ||
370 | sqlite3_finalize (stmt); | ||
371 | return GNUNET_SYSERR; | ||
372 | } | ||
373 | sqlite3_finalize (stmt); | ||
374 | create_indices (plugin->dbh); | ||
375 | |||
376 | #define RESULT_COLUMNS \ | ||
377 | "repl, type, prio, anonLevel, expire, hash, value, _ROWID_" | ||
378 | if ( | ||
379 | (SQLITE_OK != sq_prepare (plugin->dbh, | ||
380 | "UPDATE gn091 " | ||
381 | "SET prio = prio + ?, " | ||
382 | "repl = repl + ?, " | ||
383 | "expire = MAX(expire, ?) " | ||
384 | "WHERE hash = ? AND vhash = ?", | ||
385 | &plugin->update)) || | ||
386 | (SQLITE_OK != sq_prepare (plugin->dbh, | ||
387 | "UPDATE gn091 " | ||
388 | "SET repl = MAX (0, repl - 1) WHERE _ROWID_ = ?", | ||
389 | &plugin->updRepl)) || | ||
390 | (SQLITE_OK != sq_prepare (plugin->dbh, | ||
391 | "SELECT " RESULT_COLUMNS " FROM gn091 " | ||
392 | "WHERE repl=?2 AND " | ||
393 | " (rvalue>=?1 OR " | ||
394 | " NOT EXISTS (SELECT 1 FROM gn091 " | ||
395 | "WHERE repl=?2 AND rvalue>=?1 LIMIT 1) ) " | ||
396 | "ORDER BY rvalue ASC LIMIT 1", | ||
397 | &plugin->selRepl)) || | ||
398 | (SQLITE_OK != sq_prepare (plugin->dbh, | ||
399 | "SELECT MAX(repl) FROM gn091", | ||
400 | &plugin->maxRepl)) || | ||
401 | (SQLITE_OK != | ||
402 | sq_prepare (plugin->dbh, | ||
403 | "SELECT " RESULT_COLUMNS " FROM gn091 " | ||
404 | "WHERE NOT EXISTS (SELECT 1 FROM gn091 WHERE expire < ?1 LIMIT 1) OR (expire < ?1) " | ||
405 | "ORDER BY expire ASC LIMIT 1", | ||
406 | &plugin->selExpi)) || | ||
407 | (SQLITE_OK != sq_prepare (plugin->dbh, | ||
408 | "SELECT " RESULT_COLUMNS " FROM gn091 " | ||
409 | "WHERE _ROWID_ >= ? AND " | ||
410 | "anonLevel = 0 AND " | ||
411 | "type = ? " | ||
412 | "ORDER BY _ROWID_ ASC LIMIT 1", | ||
413 | &plugin->selZeroAnon)) || | ||
414 | (SQLITE_OK != | ||
415 | sq_prepare (plugin->dbh, | ||
416 | "INSERT INTO gn091 (repl, type, prio, anonLevel, expire, rvalue, hash, vhash, value) " | ||
417 | "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", | ||
418 | &plugin->insertContent)) || | ||
419 | (SQLITE_OK != sq_prepare (plugin->dbh, | ||
420 | "SELECT " RESULT_COLUMNS " FROM gn091 " | ||
421 | "WHERE _ROWID_ >= ?1 " | ||
422 | "ORDER BY _ROWID_ ASC LIMIT 1", | ||
423 | &plugin->get[0])) || | ||
424 | (SQLITE_OK != sq_prepare (plugin->dbh, | ||
425 | "SELECT " RESULT_COLUMNS " FROM gn091 " | ||
426 | "WHERE _ROWID_ >= ?1 AND " | ||
427 | "type = ?4 " | ||
428 | "ORDER BY _ROWID_ ASC LIMIT 1", | ||
429 | &plugin->get[1])) || | ||
430 | (SQLITE_OK != sq_prepare (plugin->dbh, | ||
431 | "SELECT " RESULT_COLUMNS " FROM gn091 " | ||
432 | "WHERE _ROWID_ >= ?1 AND " | ||
433 | "hash = ?3 " | ||
434 | "ORDER BY _ROWID_ ASC LIMIT 1", | ||
435 | &plugin->get[2])) || | ||
436 | (SQLITE_OK != sq_prepare (plugin->dbh, | ||
437 | "SELECT " RESULT_COLUMNS " FROM gn091 " | ||
438 | "WHERE _ROWID_ >= ?1 AND " | ||
439 | "hash = ?3 AND " | ||
440 | "type = ?4 " | ||
441 | "ORDER BY _ROWID_ ASC LIMIT 1", | ||
442 | &plugin->get[3])) || | ||
443 | (SQLITE_OK != sq_prepare (plugin->dbh, | ||
444 | "SELECT " RESULT_COLUMNS " FROM gn091 " | ||
445 | "WHERE _ROWID_ >= ?1 AND " | ||
446 | "rvalue >= ?2 " | ||
447 | "ORDER BY _ROWID_ ASC LIMIT 1", | ||
448 | &plugin->get[4])) || | ||
449 | (SQLITE_OK != sq_prepare (plugin->dbh, | ||
450 | "SELECT " RESULT_COLUMNS " FROM gn091 " | ||
451 | "WHERE _ROWID_ >= ?1 AND " | ||
452 | "rvalue >= ?2 AND " | ||
453 | "type = ?4 " | ||
454 | "ORDER BY _ROWID_ ASC LIMIT 1", | ||
455 | &plugin->get[5])) || | ||
456 | (SQLITE_OK != sq_prepare (plugin->dbh, | ||
457 | "SELECT " RESULT_COLUMNS " FROM gn091 " | ||
458 | "WHERE _ROWID_ >= ?1 AND " | ||
459 | "rvalue >= ?2 AND " | ||
460 | "hash = ?3 " | ||
461 | "ORDER BY _ROWID_ ASC LIMIT 1", | ||
462 | &plugin->get[6])) || | ||
463 | (SQLITE_OK != sq_prepare (plugin->dbh, | ||
464 | "SELECT " RESULT_COLUMNS " FROM gn091 " | ||
465 | "WHERE _ROWID_ >= ?1 AND " | ||
466 | "rvalue >= ?2 AND " | ||
467 | "hash = ?3 AND " | ||
468 | "type = ?4 " | ||
469 | "ORDER BY _ROWID_ ASC LIMIT 1", | ||
470 | &plugin->get[7])) || | ||
471 | (SQLITE_OK != sq_prepare (plugin->dbh, | ||
472 | "DELETE FROM gn091 WHERE _ROWID_ = ?", | ||
473 | &plugin->delRow)) || | ||
474 | (SQLITE_OK != sq_prepare (plugin->dbh, | ||
475 | "DELETE FROM gn091 " | ||
476 | "WHERE hash = ? AND " | ||
477 | "value = ? ", | ||
478 | &plugin->remove)) || | ||
479 | false) | ||
480 | { | ||
481 | LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "precompiling"); | ||
482 | return GNUNET_SYSERR; | ||
483 | } | ||
484 | return GNUNET_OK; | ||
485 | } | ||
486 | |||
487 | |||
488 | /** | ||
489 | * Shutdown database connection and associate data | ||
490 | * structures. | ||
491 | * | ||
492 | * @param plugin the plugin context (state for this module) | ||
493 | */ | ||
494 | static void | ||
495 | database_shutdown (struct Plugin *plugin) | ||
496 | { | ||
497 | int result; | ||
498 | |||
499 | #if SQLITE_VERSION_NUMBER >= 3007000 | ||
500 | sqlite3_stmt *stmt; | ||
501 | #endif | ||
502 | |||
503 | if (NULL != plugin->remove) | ||
504 | sqlite3_finalize (plugin->remove); | ||
505 | if (NULL != plugin->delRow) | ||
506 | sqlite3_finalize (plugin->delRow); | ||
507 | if (NULL != plugin->update) | ||
508 | sqlite3_finalize (plugin->update); | ||
509 | if (NULL != plugin->updRepl) | ||
510 | sqlite3_finalize (plugin->updRepl); | ||
511 | if (NULL != plugin->selRepl) | ||
512 | sqlite3_finalize (plugin->selRepl); | ||
513 | if (NULL != plugin->maxRepl) | ||
514 | sqlite3_finalize (plugin->maxRepl); | ||
515 | if (NULL != plugin->selExpi) | ||
516 | sqlite3_finalize (plugin->selExpi); | ||
517 | if (NULL != plugin->selZeroAnon) | ||
518 | sqlite3_finalize (plugin->selZeroAnon); | ||
519 | if (NULL != plugin->insertContent) | ||
520 | sqlite3_finalize (plugin->insertContent); | ||
521 | for (int i = 0; i < 8; ++i) | ||
522 | if (NULL != plugin->get[i]) | ||
523 | sqlite3_finalize (plugin->get[i]); | ||
524 | result = sqlite3_close (plugin->dbh); | ||
525 | #if SQLITE_VERSION_NUMBER >= 3007000 | ||
526 | if (result == SQLITE_BUSY) | ||
527 | { | ||
528 | GNUNET_log_from ( | ||
529 | GNUNET_ERROR_TYPE_WARNING, | ||
530 | "sqlite", | ||
531 | _ ( | ||
532 | "Tried to close sqlite without finalizing all prepared statements.\n")); | ||
533 | stmt = sqlite3_next_stmt (plugin->dbh, NULL); | ||
534 | while (NULL != stmt) | ||
535 | { | ||
536 | GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, | ||
537 | "sqlite", | ||
538 | "Closing statement %p\n", | ||
539 | stmt); | ||
540 | result = sqlite3_finalize (stmt); | ||
541 | if (result != SQLITE_OK) | ||
542 | GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING, | ||
543 | "sqlite", | ||
544 | "Failed to close statement %p: %d\n", | ||
545 | stmt, | ||
546 | result); | ||
547 | stmt = sqlite3_next_stmt (plugin->dbh, NULL); | ||
548 | } | ||
549 | result = sqlite3_close (plugin->dbh); | ||
550 | } | ||
551 | #endif | ||
552 | if (SQLITE_OK != result) | ||
553 | LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "sqlite3_close"); | ||
554 | GNUNET_free (plugin->fn); | ||
555 | } | ||
556 | |||
557 | |||
558 | /** | ||
559 | * Delete the database entry with the given | ||
560 | * row identifier. | ||
561 | * | ||
562 | * @param plugin the plugin context (state for this module) | ||
563 | * @param rid the ID of the row to delete | ||
564 | */ | ||
565 | static int | ||
566 | delete_by_rowid (struct Plugin *plugin, uint64_t rid) | ||
567 | { | ||
568 | struct GNUNET_SQ_QueryParam params[] = { GNUNET_SQ_query_param_uint64 (&rid), | ||
569 | GNUNET_SQ_query_param_end }; | ||
570 | |||
571 | if (GNUNET_OK != GNUNET_SQ_bind (plugin->delRow, params)) | ||
572 | return GNUNET_SYSERR; | ||
573 | if (SQLITE_DONE != sqlite3_step (plugin->delRow)) | ||
574 | { | ||
575 | LOG_SQLITE (plugin, | ||
576 | GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, | ||
577 | "sqlite3_step"); | ||
578 | GNUNET_SQ_reset (plugin->dbh, plugin->delRow); | ||
579 | return GNUNET_SYSERR; | ||
580 | } | ||
581 | GNUNET_SQ_reset (plugin->dbh, plugin->delRow); | ||
582 | return GNUNET_OK; | ||
583 | } | ||
584 | |||
585 | |||
586 | /** | ||
587 | * Store an item in the datastore. | ||
588 | * | ||
589 | * @param cls closure | ||
590 | * @param key key for the item | ||
591 | * @param absent true if the key was not found in the bloom filter | ||
592 | * @param size number of bytes in @a data | ||
593 | * @param data content stored | ||
594 | * @param type type of the content | ||
595 | * @param priority priority of the content | ||
596 | * @param anonymity anonymity-level for the content | ||
597 | * @param replication replication-level for the content | ||
598 | * @param expiration expiration time for the content | ||
599 | * @param cont continuation called with success or failure status | ||
600 | * @param cont_cls continuation closure | ||
601 | */ | ||
602 | static void | ||
603 | sqlite_plugin_put (void *cls, | ||
604 | const struct GNUNET_HashCode *key, | ||
605 | bool absent, | ||
606 | uint32_t size, | ||
607 | const void *data, | ||
608 | enum GNUNET_BLOCK_Type type, | ||
609 | uint32_t priority, | ||
610 | uint32_t anonymity, | ||
611 | uint32_t replication, | ||
612 | struct GNUNET_TIME_Absolute expiration, | ||
613 | PluginPutCont cont, | ||
614 | void *cont_cls) | ||
615 | { | ||
616 | struct Plugin *plugin = cls; | ||
617 | struct GNUNET_HashCode vhash; | ||
618 | char *msg = NULL; | ||
619 | |||
620 | GNUNET_CRYPTO_hash (data, size, &vhash); | ||
621 | |||
622 | if (! absent) | ||
623 | { | ||
624 | struct GNUNET_SQ_QueryParam params[] = | ||
625 | { GNUNET_SQ_query_param_uint32 (&priority), | ||
626 | GNUNET_SQ_query_param_uint32 (&replication), | ||
627 | GNUNET_SQ_query_param_absolute_time (&expiration), | ||
628 | GNUNET_SQ_query_param_auto_from_type (key), | ||
629 | GNUNET_SQ_query_param_auto_from_type (&vhash), | ||
630 | GNUNET_SQ_query_param_end }; | ||
631 | |||
632 | if (GNUNET_OK != GNUNET_SQ_bind (plugin->update, params)) | ||
633 | { | ||
634 | cont (cont_cls, key, size, GNUNET_SYSERR, _ ("sqlite bind failure")); | ||
635 | return; | ||
636 | } | ||
637 | if (SQLITE_DONE != sqlite3_step (plugin->update)) | ||
638 | { | ||
639 | LOG_SQLITE_MSG (plugin, | ||
640 | &msg, | ||
641 | GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, | ||
642 | "sqlite3_step"); | ||
643 | cont (cont_cls, key, size, GNUNET_SYSERR, msg); | ||
644 | GNUNET_free (msg); | ||
645 | return; | ||
646 | } | ||
647 | int changes = sqlite3_changes (plugin->dbh); | ||
648 | GNUNET_SQ_reset (plugin->dbh, plugin->update); | ||
649 | if (0 != changes) | ||
650 | { | ||
651 | cont (cont_cls, key, size, GNUNET_NO, NULL); | ||
652 | return; | ||
653 | } | ||
654 | } | ||
655 | |||
656 | uint64_t rvalue; | ||
657 | uint32_t type32 = (uint32_t) type; | ||
658 | struct GNUNET_SQ_QueryParam params[] = | ||
659 | { GNUNET_SQ_query_param_uint32 (&replication), | ||
660 | GNUNET_SQ_query_param_uint32 (&type32), | ||
661 | GNUNET_SQ_query_param_uint32 (&priority), | ||
662 | GNUNET_SQ_query_param_uint32 (&anonymity), | ||
663 | GNUNET_SQ_query_param_absolute_time (&expiration), | ||
664 | GNUNET_SQ_query_param_uint64 (&rvalue), | ||
665 | GNUNET_SQ_query_param_auto_from_type (key), | ||
666 | GNUNET_SQ_query_param_auto_from_type (&vhash), | ||
667 | GNUNET_SQ_query_param_fixed_size (data, size), | ||
668 | GNUNET_SQ_query_param_end }; | ||
669 | int n; | ||
670 | int ret; | ||
671 | sqlite3_stmt *stmt; | ||
672 | |||
673 | if (size > MAX_ITEM_SIZE) | ||
674 | { | ||
675 | cont (cont_cls, key, size, GNUNET_SYSERR, _ ("Data too large")); | ||
676 | return; | ||
677 | } | ||
678 | GNUNET_log_from ( | ||
679 | GNUNET_ERROR_TYPE_DEBUG, | ||
680 | "sqlite", | ||
681 | "Storing in database block with type %u/key `%s'/priority %u/expiration in %s (%s).\n", | ||
682 | type, | ||
683 | GNUNET_h2s (key), | ||
684 | priority, | ||
685 | GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_remaining ( | ||
686 | expiration), | ||
687 | GNUNET_YES), | ||
688 | GNUNET_STRINGS_absolute_time_to_string (expiration)); | ||
689 | stmt = plugin->insertContent; | ||
690 | rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, UINT64_MAX); | ||
691 | if (GNUNET_OK != GNUNET_SQ_bind (stmt, params)) | ||
692 | { | ||
693 | cont (cont_cls, key, size, GNUNET_SYSERR, NULL); | ||
694 | return; | ||
695 | } | ||
696 | n = sqlite3_step (stmt); | ||
697 | switch (n) | ||
698 | { | ||
699 | case SQLITE_DONE: | ||
700 | if (NULL != plugin->env->duc) | ||
701 | plugin->env->duc (plugin->env->cls, | ||
702 | size + GNUNET_DATASTORE_ENTRY_OVERHEAD); | ||
703 | GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, | ||
704 | "sqlite", | ||
705 | "Stored new entry (%u bytes)\n", | ||
706 | size + GNUNET_DATASTORE_ENTRY_OVERHEAD); | ||
707 | ret = GNUNET_OK; | ||
708 | break; | ||
709 | |||
710 | case SQLITE_BUSY: | ||
711 | GNUNET_break (0); | ||
712 | LOG_SQLITE_MSG (plugin, | ||
713 | &msg, | ||
714 | GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, | ||
715 | "sqlite3_step"); | ||
716 | ret = GNUNET_SYSERR; | ||
717 | break; | ||
718 | |||
719 | default: | ||
720 | LOG_SQLITE_MSG (plugin, | ||
721 | &msg, | ||
722 | GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, | ||
723 | "sqlite3_step"); | ||
724 | GNUNET_SQ_reset (plugin->dbh, stmt); | ||
725 | database_shutdown (plugin); | ||
726 | database_setup (plugin->env->cfg, plugin); | ||
727 | cont (cont_cls, key, size, GNUNET_SYSERR, msg); | ||
728 | GNUNET_free (msg); | ||
729 | return; | ||
730 | } | ||
731 | GNUNET_SQ_reset (plugin->dbh, stmt); | ||
732 | cont (cont_cls, key, size, ret, msg); | ||
733 | GNUNET_free (msg); | ||
734 | } | ||
735 | |||
736 | |||
737 | /** | ||
738 | * Execute statement that gets a row and call the callback | ||
739 | * with the result. Resets the statement afterwards. | ||
740 | * | ||
741 | * @param plugin the plugin | ||
742 | * @param stmt the statement | ||
743 | * @param proc processor to call | ||
744 | * @param proc_cls closure for @a proc | ||
745 | */ | ||
746 | static void | ||
747 | execute_get (struct Plugin *plugin, | ||
748 | sqlite3_stmt *stmt, | ||
749 | PluginDatumProcessor proc, | ||
750 | void *proc_cls) | ||
751 | { | ||
752 | int n; | ||
753 | struct GNUNET_TIME_Absolute expiration; | ||
754 | uint32_t replication; | ||
755 | uint32_t type; | ||
756 | uint32_t priority; | ||
757 | uint32_t anonymity; | ||
758 | uint64_t rowid; | ||
759 | void *value; | ||
760 | size_t value_size; | ||
761 | struct GNUNET_HashCode key; | ||
762 | int ret; | ||
763 | struct GNUNET_SQ_ResultSpec rs[] = | ||
764 | { GNUNET_SQ_result_spec_uint32 (&replication), | ||
765 | GNUNET_SQ_result_spec_uint32 (&type), | ||
766 | GNUNET_SQ_result_spec_uint32 (&priority), | ||
767 | GNUNET_SQ_result_spec_uint32 (&anonymity), | ||
768 | GNUNET_SQ_result_spec_absolute_time (&expiration), | ||
769 | GNUNET_SQ_result_spec_auto_from_type (&key), | ||
770 | GNUNET_SQ_result_spec_variable_size (&value, &value_size), | ||
771 | GNUNET_SQ_result_spec_uint64 (&rowid), | ||
772 | GNUNET_SQ_result_spec_end }; | ||
773 | |||
774 | n = sqlite3_step (stmt); | ||
775 | switch (n) | ||
776 | { | ||
777 | case SQLITE_ROW: | ||
778 | if (GNUNET_OK != GNUNET_SQ_extract_result (stmt, rs)) | ||
779 | { | ||
780 | GNUNET_break (0); | ||
781 | break; | ||
782 | } | ||
783 | GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, | ||
784 | "sqlite", | ||
785 | "Found reply in database with expiration %s\n", | ||
786 | GNUNET_STRINGS_absolute_time_to_string (expiration)); | ||
787 | ret = proc (proc_cls, | ||
788 | &key, | ||
789 | value_size, | ||
790 | value, | ||
791 | type, | ||
792 | priority, | ||
793 | anonymity, | ||
794 | replication, | ||
795 | expiration, | ||
796 | rowid); | ||
797 | GNUNET_SQ_cleanup_result (rs); | ||
798 | GNUNET_SQ_reset (plugin->dbh, stmt); | ||
799 | if ((GNUNET_NO == ret) && (GNUNET_OK == delete_by_rowid (plugin, rowid)) && | ||
800 | (NULL != plugin->env->duc)) | ||
801 | plugin->env->duc (plugin->env->cls, | ||
802 | -(value_size + GNUNET_DATASTORE_ENTRY_OVERHEAD)); | ||
803 | return; | ||
804 | |||
805 | case SQLITE_DONE: | ||
806 | /* database must be empty */ | ||
807 | break; | ||
808 | |||
809 | case SQLITE_BUSY: | ||
810 | case SQLITE_ERROR: | ||
811 | case SQLITE_MISUSE: | ||
812 | default: | ||
813 | LOG_SQLITE (plugin, | ||
814 | GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, | ||
815 | "sqlite3_step"); | ||
816 | if (SQLITE_OK != sqlite3_reset (stmt)) | ||
817 | LOG_SQLITE (plugin, | ||
818 | GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, | ||
819 | "sqlite3_reset"); | ||
820 | GNUNET_break (0); | ||
821 | proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); | ||
822 | database_shutdown (plugin); | ||
823 | database_setup (plugin->env->cfg, plugin); | ||
824 | return; | ||
825 | } | ||
826 | GNUNET_SQ_reset (plugin->dbh, stmt); | ||
827 | proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); | ||
828 | } | ||
829 | |||
830 | |||
831 | /** | ||
832 | * Select a subset of the items in the datastore and call | ||
833 | * the given processor for the item. | ||
834 | * | ||
835 | * @param cls our plugin context | ||
836 | * @param next_uid return the result with lowest uid >= next_uid | ||
837 | * @param type entries of which type should be considered? | ||
838 | * Must not be zero (ANY). | ||
839 | * @param proc function to call on the matching value; | ||
840 | * will be called with NULL if no value matches | ||
841 | * @param proc_cls closure for @a proc | ||
842 | */ | ||
843 | static void | ||
844 | sqlite_plugin_get_zero_anonymity (void *cls, | ||
845 | uint64_t next_uid, | ||
846 | enum GNUNET_BLOCK_Type type, | ||
847 | PluginDatumProcessor proc, | ||
848 | void *proc_cls) | ||
849 | { | ||
850 | struct Plugin *plugin = cls; | ||
851 | uint32_t type32 = type; | ||
852 | struct GNUNET_SQ_QueryParam params[] = { GNUNET_SQ_query_param_uint64 ( | ||
853 | &next_uid), | ||
854 | GNUNET_SQ_query_param_uint32 ( | ||
855 | &type32), | ||
856 | GNUNET_SQ_query_param_end }; | ||
857 | |||
858 | GNUNET_assert (type != GNUNET_BLOCK_TYPE_ANY); | ||
859 | if (GNUNET_OK != GNUNET_SQ_bind (plugin->selZeroAnon, params)) | ||
860 | { | ||
861 | proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); | ||
862 | return; | ||
863 | } | ||
864 | execute_get (plugin, plugin->selZeroAnon, proc, proc_cls); | ||
865 | } | ||
866 | |||
867 | |||
868 | /** | ||
869 | * Get results for a particular key in the datastore. | ||
870 | * | ||
871 | * @param cls closure | ||
872 | * @param next_uid return the result with lowest uid >= next_uid | ||
873 | * @param random if true, return a random result instead of using next_uid | ||
874 | * @param key maybe NULL (to match all entries) | ||
875 | * @param type entries of which type are relevant? | ||
876 | * Use 0 for any type. | ||
877 | * @param proc function to call on the matching value; | ||
878 | * will be called with NULL if nothing matches | ||
879 | * @param proc_cls closure for @a proc | ||
880 | */ | ||
881 | static void | ||
882 | sqlite_plugin_get_key (void *cls, | ||
883 | uint64_t next_uid, | ||
884 | bool random, | ||
885 | const struct GNUNET_HashCode *key, | ||
886 | enum GNUNET_BLOCK_Type type, | ||
887 | PluginDatumProcessor proc, | ||
888 | void *proc_cls) | ||
889 | { | ||
890 | struct Plugin *plugin = cls; | ||
891 | uint64_t rvalue; | ||
892 | int use_rvalue = random; | ||
893 | uint32_t type32 = (uint32_t) type; | ||
894 | int use_type = GNUNET_BLOCK_TYPE_ANY != type; | ||
895 | int use_key = NULL != key; | ||
896 | sqlite3_stmt *stmt = plugin->get[use_rvalue * 4 + use_key * 2 + use_type]; | ||
897 | struct GNUNET_SQ_QueryParam params[] = | ||
898 | { GNUNET_SQ_query_param_uint64 (&next_uid), | ||
899 | GNUNET_SQ_query_param_uint64 (&rvalue), | ||
900 | GNUNET_SQ_query_param_auto_from_type (key), | ||
901 | GNUNET_SQ_query_param_uint32 (&type32), | ||
902 | GNUNET_SQ_query_param_end }; | ||
903 | |||
904 | /* SQLite doesn't like it when you try to bind a parameter greater than the | ||
905 | * last numbered parameter, but unused parameters in the middle are OK. | ||
906 | */ | ||
907 | if (! use_type) | ||
908 | { | ||
909 | params[3] = (struct GNUNET_SQ_QueryParam) GNUNET_SQ_query_param_end; | ||
910 | if (! use_key) | ||
911 | { | ||
912 | params[2] = (struct GNUNET_SQ_QueryParam) GNUNET_SQ_query_param_end; | ||
913 | if (! use_rvalue) | ||
914 | params[1] = (struct GNUNET_SQ_QueryParam) GNUNET_SQ_query_param_end; | ||
915 | } | ||
916 | } | ||
917 | if (random) | ||
918 | { | ||
919 | rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, UINT64_MAX); | ||
920 | next_uid = 0; | ||
921 | } | ||
922 | else | ||
923 | rvalue = 0; | ||
924 | |||
925 | if (GNUNET_OK != GNUNET_SQ_bind (stmt, params)) | ||
926 | { | ||
927 | proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); | ||
928 | return; | ||
929 | } | ||
930 | execute_get (plugin, stmt, proc, proc_cls); | ||
931 | } | ||
932 | |||
933 | |||
934 | /** | ||
935 | * Context for #repl_proc() function. | ||
936 | */ | ||
937 | struct ReplCtx | ||
938 | { | ||
939 | /** | ||
940 | * Function to call for the result (or the NULL). | ||
941 | */ | ||
942 | PluginDatumProcessor proc; | ||
943 | |||
944 | /** | ||
945 | * Closure for @e proc. | ||
946 | */ | ||
947 | void *proc_cls; | ||
948 | |||
949 | /** | ||
950 | * UID to use. | ||
951 | */ | ||
952 | uint64_t uid; | ||
953 | |||
954 | /** | ||
955 | * Yes if UID was set. | ||
956 | */ | ||
957 | int have_uid; | ||
958 | }; | ||
959 | |||
960 | |||
961 | /** | ||
962 | * Wrapper for the processor for #sqlite_plugin_get_replication(). | ||
963 | * Decrements the replication counter and calls the original | ||
964 | * processor. | ||
965 | * | ||
966 | * @param cls closure | ||
967 | * @param key key for the content | ||
968 | * @param size number of bytes in @a data | ||
969 | * @param data content stored | ||
970 | * @param type type of the content | ||
971 | * @param priority priority of the content | ||
972 | * @param anonymity anonymity-level for the content | ||
973 | * @param replication replication-level for the content | ||
974 | * @param expiration expiration time for the content | ||
975 | * @param uid unique identifier for the datum; | ||
976 | * maybe 0 if no unique identifier is available | ||
977 | * @return #GNUNET_OK for normal return, | ||
978 | * #GNUNET_NO to delete the item | ||
979 | */ | ||
980 | static int | ||
981 | repl_proc (void *cls, | ||
982 | const struct GNUNET_HashCode *key, | ||
983 | uint32_t size, | ||
984 | const void *data, | ||
985 | enum GNUNET_BLOCK_Type type, | ||
986 | uint32_t priority, | ||
987 | uint32_t anonymity, | ||
988 | uint32_t replication, | ||
989 | struct GNUNET_TIME_Absolute expiration, | ||
990 | uint64_t uid) | ||
991 | { | ||
992 | struct ReplCtx *rc = cls; | ||
993 | int ret; | ||
994 | |||
995 | if (GNUNET_SYSERR == rc->have_uid) | ||
996 | rc->have_uid = GNUNET_NO; | ||
997 | ret = rc->proc (rc->proc_cls, | ||
998 | key, | ||
999 | size, | ||
1000 | data, | ||
1001 | type, | ||
1002 | priority, | ||
1003 | anonymity, | ||
1004 | replication, | ||
1005 | expiration, | ||
1006 | uid); | ||
1007 | if (NULL != key) | ||
1008 | { | ||
1009 | rc->uid = uid; | ||
1010 | rc->have_uid = GNUNET_YES; | ||
1011 | } | ||
1012 | return ret; | ||
1013 | } | ||
1014 | |||
1015 | |||
1016 | /** | ||
1017 | * Get a random item for replication. Returns a single random item | ||
1018 | * from those with the highest replication counters. The item's | ||
1019 | * replication counter is decremented by one IF it was positive before. | ||
1020 | * Call @a proc with all values ZERO or NULL if the datastore is empty. | ||
1021 | * | ||
1022 | * @param cls closure | ||
1023 | * @param proc function to call the value (once only). | ||
1024 | * @param proc_cls closure for @a proc | ||
1025 | */ | ||
1026 | static void | ||
1027 | sqlite_plugin_get_replication (void *cls, | ||
1028 | PluginDatumProcessor proc, | ||
1029 | void *proc_cls) | ||
1030 | { | ||
1031 | struct Plugin *plugin = cls; | ||
1032 | struct ReplCtx rc; | ||
1033 | uint64_t rvalue = 0; | ||
1034 | uint32_t repl; | ||
1035 | struct GNUNET_SQ_QueryParam params_sel_repl[] = | ||
1036 | { GNUNET_SQ_query_param_uint64 (&rvalue), | ||
1037 | GNUNET_SQ_query_param_uint32 (&repl), | ||
1038 | GNUNET_SQ_query_param_end }; | ||
1039 | struct GNUNET_SQ_QueryParam params_upd_repl[] = | ||
1040 | { GNUNET_SQ_query_param_uint64 (&rc.uid), GNUNET_SQ_query_param_end }; | ||
1041 | |||
1042 | GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, | ||
1043 | "datastore-sqlite", | ||
1044 | "Getting random block based on replication order.\n"); | ||
1045 | if (SQLITE_ROW != sqlite3_step (plugin->maxRepl)) | ||
1046 | { | ||
1047 | GNUNET_SQ_reset (plugin->dbh, plugin->maxRepl); | ||
1048 | /* DB empty */ | ||
1049 | proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); | ||
1050 | return; | ||
1051 | } | ||
1052 | repl = sqlite3_column_int (plugin->maxRepl, 0); | ||
1053 | GNUNET_SQ_reset (plugin->dbh, plugin->maxRepl); | ||
1054 | rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, UINT64_MAX); | ||
1055 | if (GNUNET_OK != GNUNET_SQ_bind (plugin->selRepl, params_sel_repl)) | ||
1056 | { | ||
1057 | proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); | ||
1058 | return; | ||
1059 | } | ||
1060 | rc.have_uid = GNUNET_SYSERR; | ||
1061 | rc.proc = proc; | ||
1062 | rc.proc_cls = proc_cls; | ||
1063 | execute_get (plugin, plugin->selRepl, &repl_proc, &rc); | ||
1064 | if (GNUNET_YES == rc.have_uid) | ||
1065 | { | ||
1066 | if (GNUNET_OK != GNUNET_SQ_bind (plugin->updRepl, params_upd_repl)) | ||
1067 | { | ||
1068 | proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); | ||
1069 | return; | ||
1070 | } | ||
1071 | if (SQLITE_DONE != sqlite3_step (plugin->updRepl)) | ||
1072 | LOG_SQLITE (plugin, | ||
1073 | GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, | ||
1074 | "sqlite3_step"); | ||
1075 | GNUNET_SQ_reset (plugin->dbh, plugin->updRepl); | ||
1076 | } | ||
1077 | if (GNUNET_SYSERR == rc.have_uid) | ||
1078 | { | ||
1079 | /* proc was not called at all so far, do it now. */ | ||
1080 | proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); | ||
1081 | } | ||
1082 | } | ||
1083 | |||
1084 | |||
1085 | /** | ||
1086 | * Get a random item that has expired or has low priority. | ||
1087 | * Call @a proc with all values ZERO or NULL if the datastore is empty. | ||
1088 | * | ||
1089 | * @param cls closure | ||
1090 | * @param proc function to call the value (once only). | ||
1091 | * @param proc_cls closure for @a proc | ||
1092 | */ | ||
1093 | static void | ||
1094 | sqlite_plugin_get_expiration (void *cls, | ||
1095 | PluginDatumProcessor proc, | ||
1096 | void *proc_cls) | ||
1097 | { | ||
1098 | struct Plugin *plugin = cls; | ||
1099 | sqlite3_stmt *stmt; | ||
1100 | struct GNUNET_TIME_Absolute now = { 0 }; | ||
1101 | struct GNUNET_SQ_QueryParam params[] = { GNUNET_SQ_query_param_absolute_time ( | ||
1102 | &now), | ||
1103 | GNUNET_SQ_query_param_end }; | ||
1104 | |||
1105 | GNUNET_log_from ( | ||
1106 | GNUNET_ERROR_TYPE_DEBUG, | ||
1107 | "sqlite", | ||
1108 | "Getting random block based on expiration and priority order.\n"); | ||
1109 | now = GNUNET_TIME_absolute_get (); | ||
1110 | stmt = plugin->selExpi; | ||
1111 | if (GNUNET_OK != GNUNET_SQ_bind (stmt, params)) | ||
1112 | { | ||
1113 | proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); | ||
1114 | return; | ||
1115 | } | ||
1116 | execute_get (plugin, stmt, proc, proc_cls); | ||
1117 | } | ||
1118 | |||
1119 | |||
1120 | /** | ||
1121 | * Get all of the keys in the datastore. | ||
1122 | * | ||
1123 | * @param cls closure | ||
1124 | * @param proc function to call on each key | ||
1125 | * @param proc_cls closure for @a proc | ||
1126 | */ | ||
1127 | static void | ||
1128 | sqlite_plugin_get_keys (void *cls, PluginKeyProcessor proc, void *proc_cls) | ||
1129 | { | ||
1130 | struct Plugin *plugin = cls; | ||
1131 | struct GNUNET_HashCode key; | ||
1132 | struct GNUNET_SQ_ResultSpec results[] = | ||
1133 | { GNUNET_SQ_result_spec_auto_from_type (&key), GNUNET_SQ_result_spec_end }; | ||
1134 | sqlite3_stmt *stmt; | ||
1135 | int ret; | ||
1136 | |||
1137 | GNUNET_assert (NULL != proc); | ||
1138 | if (SQLITE_OK != sq_prepare (plugin->dbh, "SELECT hash FROM gn091", &stmt)) | ||
1139 | { | ||
1140 | LOG_SQLITE (plugin, | ||
1141 | GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, | ||
1142 | "sqlite_prepare"); | ||
1143 | proc (proc_cls, NULL, 0); | ||
1144 | return; | ||
1145 | } | ||
1146 | while (SQLITE_ROW == (ret = sqlite3_step (stmt))) | ||
1147 | { | ||
1148 | if (GNUNET_OK == GNUNET_SQ_extract_result (stmt, results)) | ||
1149 | proc (proc_cls, &key, 1); | ||
1150 | else | ||
1151 | GNUNET_break (0); | ||
1152 | } | ||
1153 | if (SQLITE_DONE != ret) | ||
1154 | LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "sqlite_step"); | ||
1155 | sqlite3_finalize (stmt); | ||
1156 | proc (proc_cls, NULL, 0); | ||
1157 | } | ||
1158 | |||
1159 | |||
1160 | /** | ||
1161 | * Drop database. | ||
1162 | * | ||
1163 | * @param cls our plugin context | ||
1164 | */ | ||
1165 | static void | ||
1166 | sqlite_plugin_drop (void *cls) | ||
1167 | { | ||
1168 | struct Plugin *plugin = cls; | ||
1169 | |||
1170 | plugin->drop_on_shutdown = GNUNET_YES; | ||
1171 | } | ||
1172 | |||
1173 | |||
1174 | /** | ||
1175 | * Remove a particular key in the datastore. | ||
1176 | * | ||
1177 | * @param cls closure | ||
1178 | * @param key key for the content | ||
1179 | * @param size number of bytes in data | ||
1180 | * @param data content stored | ||
1181 | * @param cont continuation called with success or failure status | ||
1182 | * @param cont_cls continuation closure for @a cont | ||
1183 | */ | ||
1184 | static void | ||
1185 | sqlite_plugin_remove_key (void *cls, | ||
1186 | const struct GNUNET_HashCode *key, | ||
1187 | uint32_t size, | ||
1188 | const void *data, | ||
1189 | PluginRemoveCont cont, | ||
1190 | void *cont_cls) | ||
1191 | { | ||
1192 | struct Plugin *plugin = cls; | ||
1193 | struct GNUNET_SQ_QueryParam params[] = | ||
1194 | { GNUNET_SQ_query_param_auto_from_type (key), | ||
1195 | GNUNET_SQ_query_param_fixed_size (data, size), | ||
1196 | GNUNET_SQ_query_param_end }; | ||
1197 | |||
1198 | if (GNUNET_OK != GNUNET_SQ_bind (plugin->remove, params)) | ||
1199 | { | ||
1200 | cont (cont_cls, key, size, GNUNET_SYSERR, "bind failed"); | ||
1201 | return; | ||
1202 | } | ||
1203 | if (SQLITE_DONE != sqlite3_step (plugin->remove)) | ||
1204 | { | ||
1205 | LOG_SQLITE (plugin, | ||
1206 | GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, | ||
1207 | "sqlite3_step"); | ||
1208 | GNUNET_SQ_reset (plugin->dbh, plugin->remove); | ||
1209 | cont (cont_cls, key, size, GNUNET_SYSERR, "sqlite3_step failed"); | ||
1210 | return; | ||
1211 | } | ||
1212 | int changes = sqlite3_changes (plugin->dbh); | ||
1213 | GNUNET_SQ_reset (plugin->dbh, plugin->remove); | ||
1214 | if (0 == changes) | ||
1215 | { | ||
1216 | cont (cont_cls, key, size, GNUNET_NO, NULL); | ||
1217 | return; | ||
1218 | } | ||
1219 | if (NULL != plugin->env->duc) | ||
1220 | plugin->env->duc (plugin->env->cls, | ||
1221 | -(size + GNUNET_DATASTORE_ENTRY_OVERHEAD)); | ||
1222 | cont (cont_cls, key, size, GNUNET_OK, NULL); | ||
1223 | } | ||
1224 | |||
1225 | |||
1226 | /** | ||
1227 | * Get an estimate of how much space the database is | ||
1228 | * currently using. | ||
1229 | * | ||
1230 | * @param cls the `struct Plugin` | ||
1231 | * @return the size of the database on disk (estimate) | ||
1232 | */ | ||
1233 | static void | ||
1234 | sqlite_plugin_estimate_size (void *cls, unsigned long long *estimate) | ||
1235 | { | ||
1236 | struct Plugin *plugin = cls; | ||
1237 | sqlite3_stmt *stmt; | ||
1238 | uint64_t pages; | ||
1239 | uint64_t page_size; | ||
1240 | |||
1241 | #if ENULL_DEFINED | ||
1242 | char *e; | ||
1243 | #endif | ||
1244 | |||
1245 | if (NULL == estimate) | ||
1246 | return; | ||
1247 | if (SQLITE_VERSION_NUMBER < 3006000) | ||
1248 | { | ||
1249 | GNUNET_log_from ( | ||
1250 | GNUNET_ERROR_TYPE_WARNING, | ||
1251 | "datastore-sqlite", | ||
1252 | _ ("sqlite version to old to determine size, assuming zero\n")); | ||
1253 | *estimate = 0; | ||
1254 | return; | ||
1255 | } | ||
1256 | CHECK (SQLITE_OK == sqlite3_exec (plugin->dbh, "VACUUM", NULL, NULL, ENULL)); | ||
1257 | CHECK (SQLITE_OK == sqlite3_exec (plugin->dbh, | ||
1258 | "PRAGMA auto_vacuum=INCREMENTAL", | ||
1259 | NULL, | ||
1260 | NULL, | ||
1261 | ENULL)); | ||
1262 | if (SQLITE_OK != sq_prepare (plugin->dbh, "PRAGMA page_count", &stmt)) | ||
1263 | { | ||
1264 | GNUNET_log_from ( | ||
1265 | GNUNET_ERROR_TYPE_WARNING, | ||
1266 | "datastore-sqlite", | ||
1267 | _("error preparing statement\n")); | ||
1268 | return; | ||
1269 | } | ||
1270 | if (SQLITE_ROW == sqlite3_step (stmt)) | ||
1271 | pages = sqlite3_column_int64 (stmt, 0); | ||
1272 | else | ||
1273 | pages = 0; | ||
1274 | sqlite3_finalize (stmt); | ||
1275 | if (SQLITE_OK != sq_prepare (plugin->dbh, "PRAGMA page_size", &stmt)) | ||
1276 | { | ||
1277 | GNUNET_log_from ( | ||
1278 | GNUNET_ERROR_TYPE_WARNING, | ||
1279 | "datastore-sqlite", | ||
1280 | _("error preparing statement\n")); | ||
1281 | return; | ||
1282 | } | ||
1283 | if (SQLITE_ROW != sqlite3_step (stmt)) | ||
1284 | { | ||
1285 | GNUNET_log_from ( | ||
1286 | GNUNET_ERROR_TYPE_WARNING, | ||
1287 | "datastore-sqlite", | ||
1288 | _("error stepping\n")); | ||
1289 | return; | ||
1290 | } | ||
1291 | page_size = sqlite3_column_int64 (stmt, 0); | ||
1292 | sqlite3_finalize (stmt); | ||
1293 | GNUNET_log ( | ||
1294 | GNUNET_ERROR_TYPE_INFO, | ||
1295 | _ ( | ||
1296 | "Using sqlite page utilization to estimate payload (%llu pages of size %llu bytes)\n"), | ||
1297 | (unsigned long long) pages, | ||
1298 | (unsigned long long) page_size); | ||
1299 | *estimate = pages * page_size; | ||
1300 | } | ||
1301 | |||
1302 | |||
1303 | /** | ||
1304 | * Entry point for the plugin. | ||
1305 | * | ||
1306 | * @param cls the `struct GNUNET_DATASTORE_PluginEnvironment *` | ||
1307 | * @return NULL on error, othrewise the plugin context | ||
1308 | */ | ||
1309 | void * | ||
1310 | libgnunet_plugin_datastore_sqlite_init (void *cls) | ||
1311 | { | ||
1312 | static struct Plugin plugin; | ||
1313 | struct GNUNET_DATASTORE_PluginEnvironment *env = cls; | ||
1314 | struct GNUNET_DATASTORE_PluginFunctions *api; | ||
1315 | |||
1316 | if (NULL != plugin.env) | ||
1317 | return NULL; /* can only initialize once! */ | ||
1318 | memset (&plugin, 0, sizeof(struct Plugin)); | ||
1319 | plugin.env = env; | ||
1320 | if (GNUNET_OK != database_setup (env->cfg, &plugin)) | ||
1321 | { | ||
1322 | database_shutdown (&plugin); | ||
1323 | return NULL; | ||
1324 | } | ||
1325 | api = GNUNET_new (struct GNUNET_DATASTORE_PluginFunctions); | ||
1326 | api->cls = &plugin; | ||
1327 | api->estimate_size = &sqlite_plugin_estimate_size; | ||
1328 | api->put = &sqlite_plugin_put; | ||
1329 | api->get_key = &sqlite_plugin_get_key; | ||
1330 | api->get_replication = &sqlite_plugin_get_replication; | ||
1331 | api->get_expiration = &sqlite_plugin_get_expiration; | ||
1332 | api->get_zero_anonymity = &sqlite_plugin_get_zero_anonymity; | ||
1333 | api->get_keys = &sqlite_plugin_get_keys; | ||
1334 | api->drop = &sqlite_plugin_drop; | ||
1335 | api->remove_key = &sqlite_plugin_remove_key; | ||
1336 | GNUNET_log_from (GNUNET_ERROR_TYPE_INFO, | ||
1337 | "sqlite", | ||
1338 | _ ("Sqlite database running\n")); | ||
1339 | return api; | ||
1340 | } | ||
1341 | |||
1342 | |||
1343 | /** | ||
1344 | * Exit point from the plugin. | ||
1345 | * | ||
1346 | * @param cls the plugin context (as returned by "init") | ||
1347 | * @return always NULL | ||
1348 | */ | ||
1349 | void * | ||
1350 | libgnunet_plugin_datastore_sqlite_done (void *cls) | ||
1351 | { | ||
1352 | char *fn; | ||
1353 | struct GNUNET_DATASTORE_PluginFunctions *api = cls; | ||
1354 | struct Plugin *plugin = api->cls; | ||
1355 | |||
1356 | GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, | ||
1357 | "sqlite", | ||
1358 | "sqlite plugin is done\n"); | ||
1359 | fn = NULL; | ||
1360 | if (plugin->drop_on_shutdown) | ||
1361 | fn = GNUNET_strdup (plugin->fn); | ||
1362 | database_shutdown (plugin); | ||
1363 | plugin->env = NULL; | ||
1364 | GNUNET_free (api); | ||
1365 | if (NULL != fn) | ||
1366 | { | ||
1367 | if (0 != unlink (fn)) | ||
1368 | GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", fn); | ||
1369 | GNUNET_free (fn); | ||
1370 | } | ||
1371 | return NULL; | ||
1372 | } | ||
1373 | |||
1374 | |||
1375 | /* end of plugin_datastore_sqlite.c */ | ||
diff --git a/src/plugin/datastore/plugin_datastore_template.c b/src/plugin/datastore/plugin_datastore_template.c new file mode 100644 index 000000000..2b455f8cb --- /dev/null +++ b/src/plugin/datastore/plugin_datastore_template.c | |||
@@ -0,0 +1,274 @@ | |||
1 | /* | ||
2 | This file is part of GNUnet | ||
3 | Copyright (C) 2009, 2011 GNUnet e.V. | ||
4 | |||
5 | GNUnet is free software: you can redistribute it and/or modify it | ||
6 | under the terms of the GNU Affero General Public License as published | ||
7 | by the Free Software Foundation, either version 3 of the License, | ||
8 | or (at your option) any later version. | ||
9 | |||
10 | GNUnet is distributed in the hope that it will be useful, but | ||
11 | WITHOUT ANY WARRANTY; without even the implied warranty of | ||
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
13 | Affero General Public License for more details. | ||
14 | |||
15 | You should have received a copy of the GNU Affero General Public License | ||
16 | along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
17 | |||
18 | SPDX-License-Identifier: AGPL3.0-or-later | ||
19 | */ | ||
20 | |||
21 | /** | ||
22 | * @file datastore/plugin_datastore_template.c | ||
23 | * @brief template-based datastore backend | ||
24 | * @author Christian Grothoff | ||
25 | */ | ||
26 | |||
27 | #include "platform.h" | ||
28 | #include "gnunet_datastore_plugin.h" | ||
29 | |||
30 | |||
31 | /** | ||
32 | * Context for all functions in this plugin. | ||
33 | */ | ||
34 | struct Plugin | ||
35 | { | ||
36 | /** | ||
37 | * Our execution environment. | ||
38 | */ | ||
39 | struct GNUNET_DATASTORE_PluginEnvironment *env; | ||
40 | }; | ||
41 | |||
42 | |||
43 | /** | ||
44 | * Get an estimate of how much space the database is | ||
45 | * currently using. | ||
46 | * | ||
47 | * @param cls our "struct Plugin*" | ||
48 | * @return number of bytes used on disk | ||
49 | */ | ||
50 | static void | ||
51 | template_plugin_estimate_size (void *cls, unsigned long long *estimate) | ||
52 | { | ||
53 | if (NULL == estimate) | ||
54 | return; | ||
55 | GNUNET_break (0); | ||
56 | *estimate = 0; | ||
57 | } | ||
58 | |||
59 | |||
60 | /** | ||
61 | * Store an item in the datastore. | ||
62 | * | ||
63 | * @param cls closure | ||
64 | * @param key key for the item | ||
65 | * @param absent true if the key was not found in the bloom filter | ||
66 | * @param size number of bytes in data | ||
67 | * @param data content stored | ||
68 | * @param type type of the content | ||
69 | * @param priority priority of the content | ||
70 | * @param anonymity anonymity-level for the content | ||
71 | * @param replication replication-level for the content | ||
72 | * @param expiration expiration time for the content | ||
73 | * @param cont continuation called with success or failure status | ||
74 | * @param cont_cls continuation closure | ||
75 | */ | ||
76 | static void | ||
77 | template_plugin_put (void *cls, | ||
78 | const struct GNUNET_HashCode *key, | ||
79 | bool absent, | ||
80 | uint32_t size, | ||
81 | const void *data, | ||
82 | enum GNUNET_BLOCK_Type type, | ||
83 | uint32_t priority, | ||
84 | uint32_t anonymity, | ||
85 | uint32_t replication, | ||
86 | struct GNUNET_TIME_Absolute expiration, | ||
87 | PluginPutCont cont, | ||
88 | void *cont_cls) | ||
89 | { | ||
90 | GNUNET_break (0); | ||
91 | cont (cont_cls, key, size, GNUNET_SYSERR, "not implemented"); | ||
92 | } | ||
93 | |||
94 | |||
95 | /** | ||
96 | * Get one of the results for a particular key in the datastore. | ||
97 | * | ||
98 | * @param cls closure | ||
99 | * @param next_uid return the result with lowest uid >= next_uid | ||
100 | * @param random if true, return a random result instead of using next_uid | ||
101 | * @param key maybe NULL (to match all entries) | ||
102 | * @param type entries of which type are relevant? | ||
103 | * Use 0 for any type. | ||
104 | * @param proc function to call on each matching value; | ||
105 | * will be called with NULL if nothing matches | ||
106 | * @param proc_cls closure for proc | ||
107 | */ | ||
108 | static void | ||
109 | template_plugin_get_key (void *cls, | ||
110 | uint64_t next_uid, | ||
111 | bool random, | ||
112 | const struct GNUNET_HashCode *key, | ||
113 | enum GNUNET_BLOCK_Type type, | ||
114 | PluginDatumProcessor proc, | ||
115 | void *proc_cls) | ||
116 | { | ||
117 | GNUNET_break (0); | ||
118 | } | ||
119 | |||
120 | |||
121 | /** | ||
122 | * Get a random item for replication. Returns a single, not expired, | ||
123 | * random item from those with the highest replication counters. The | ||
124 | * item's replication counter is decremented by one IF it was positive | ||
125 | * before. Call 'proc' with all values ZERO or NULL if the datastore | ||
126 | * is empty. | ||
127 | * | ||
128 | * @param cls closure | ||
129 | * @param proc function to call the value (once only). | ||
130 | * @param proc_cls closure for proc | ||
131 | */ | ||
132 | static void | ||
133 | template_plugin_get_replication (void *cls, PluginDatumProcessor proc, | ||
134 | void *proc_cls) | ||
135 | { | ||
136 | GNUNET_break (0); | ||
137 | } | ||
138 | |||
139 | |||
140 | /** | ||
141 | * Get a random item for expiration. Call 'proc' with all values ZERO | ||
142 | * or NULL if the datastore is empty. | ||
143 | * | ||
144 | * @param cls closure | ||
145 | * @param proc function to call the value (once only). | ||
146 | * @param proc_cls closure for proc | ||
147 | */ | ||
148 | static void | ||
149 | template_plugin_get_expiration (void *cls, PluginDatumProcessor proc, | ||
150 | void *proc_cls) | ||
151 | { | ||
152 | GNUNET_break (0); | ||
153 | } | ||
154 | |||
155 | |||
156 | /** | ||
157 | * Call the given processor on an item with zero anonymity. | ||
158 | * | ||
159 | * @param cls our "struct Plugin*" | ||
160 | * @param next_uid return the result with lowest uid >= next_uid | ||
161 | * @param type entries of which type should be considered? | ||
162 | * Must not be zero (ANY). | ||
163 | * @param proc function to call on the matching value; | ||
164 | * will be called with NULL if no value matches | ||
165 | * @param proc_cls closure for proc | ||
166 | */ | ||
167 | static void | ||
168 | template_plugin_get_zero_anonymity (void *cls, uint64_t next_uid, | ||
169 | enum GNUNET_BLOCK_Type type, | ||
170 | PluginDatumProcessor proc, void *proc_cls) | ||
171 | { | ||
172 | GNUNET_break (0); | ||
173 | } | ||
174 | |||
175 | |||
176 | /** | ||
177 | * Drop database. | ||
178 | */ | ||
179 | static void | ||
180 | template_plugin_drop (void *cls) | ||
181 | { | ||
182 | GNUNET_break (0); | ||
183 | } | ||
184 | |||
185 | |||
186 | /** | ||
187 | * Get all of the keys in the datastore. | ||
188 | * | ||
189 | * @param cls closure | ||
190 | * @param proc function to call on each key | ||
191 | * @param proc_cls closure for proc | ||
192 | */ | ||
193 | static void | ||
194 | template_get_keys (void *cls, | ||
195 | PluginKeyProcessor proc, | ||
196 | void *proc_cls) | ||
197 | { | ||
198 | proc (proc_cls, NULL, 0); | ||
199 | } | ||
200 | |||
201 | |||
202 | /** | ||
203 | * Remove a particular key in the datastore. | ||
204 | * | ||
205 | * @param cls closure | ||
206 | * @param key key for the content | ||
207 | * @param size number of bytes in data | ||
208 | * @param data content stored | ||
209 | * @param cont continuation called with success or failure status | ||
210 | * @param cont_cls continuation closure for @a cont | ||
211 | */ | ||
212 | static void | ||
213 | template_plugin_remove_key (void *cls, | ||
214 | const struct GNUNET_HashCode *key, | ||
215 | uint32_t size, | ||
216 | const void *data, | ||
217 | PluginRemoveCont cont, | ||
218 | void *cont_cls) | ||
219 | { | ||
220 | GNUNET_break (0); | ||
221 | cont (cont_cls, key, size, GNUNET_SYSERR, "not implemented"); | ||
222 | } | ||
223 | |||
224 | |||
225 | /** | ||
226 | * Entry point for the plugin. | ||
227 | * | ||
228 | * @param cls the "struct GNUNET_DATASTORE_PluginEnvironment*" | ||
229 | * @return our "struct Plugin*" | ||
230 | */ | ||
231 | void * | ||
232 | libgnunet_plugin_datastore_template_init (void *cls) | ||
233 | { | ||
234 | struct GNUNET_DATASTORE_PluginEnvironment *env = cls; | ||
235 | struct GNUNET_DATASTORE_PluginFunctions *api; | ||
236 | struct Plugin *plugin; | ||
237 | |||
238 | plugin = GNUNET_new (struct Plugin); | ||
239 | plugin->env = env; | ||
240 | api = GNUNET_new (struct GNUNET_DATASTORE_PluginFunctions); | ||
241 | api->cls = plugin; | ||
242 | api->estimate_size = &template_plugin_estimate_size; | ||
243 | api->put = &template_plugin_put; | ||
244 | api->get_key = &template_plugin_get_key; | ||
245 | api->get_replication = &template_plugin_get_replication; | ||
246 | api->get_expiration = &template_plugin_get_expiration; | ||
247 | api->get_zero_anonymity = &template_plugin_get_zero_anonymity; | ||
248 | api->drop = &template_plugin_drop; | ||
249 | api->get_keys = &template_get_keys; | ||
250 | api->remove_key = &template_plugin_remove_key; | ||
251 | GNUNET_log_from (GNUNET_ERROR_TYPE_INFO, "template", | ||
252 | _ ("Template database running\n")); | ||
253 | return api; | ||
254 | } | ||
255 | |||
256 | |||
257 | /** | ||
258 | * Exit point from the plugin. | ||
259 | * @param cls our "struct Plugin*" | ||
260 | * @return always NULL | ||
261 | */ | ||
262 | void * | ||
263 | libgnunet_plugin_datastore_template_done (void *cls) | ||
264 | { | ||
265 | struct GNUNET_DATASTORE_PluginFunctions *api = cls; | ||
266 | struct Plugin *plugin = api->cls; | ||
267 | |||
268 | GNUNET_free (plugin); | ||
269 | GNUNET_free (api); | ||
270 | return NULL; | ||
271 | } | ||
272 | |||
273 | |||
274 | /* end of plugin_datastore_template.c */ | ||
diff --git a/src/plugin/datastore/test_defaults.conf b/src/plugin/datastore/test_defaults.conf new file mode 100644 index 000000000..1e6bbeeaf --- /dev/null +++ b/src/plugin/datastore/test_defaults.conf | |||
@@ -0,0 +1,10 @@ | |||
1 | @inline@ ../../../contrib/conf/gnunet/no_autostart_above_core.conf | ||
2 | @inline@ ../../../contrib/conf/gnunet/no_forcestart.conf | ||
3 | |||
4 | [datastore] | ||
5 | PORT = 22654 | ||
6 | QUOTA = 1 MB | ||
7 | START_ON_DEMAND = YES | ||
8 | |||
9 | [nse] | ||
10 | WORKBITS = 1 | ||
diff --git a/src/plugin/datastore/test_plugin_datastore.c b/src/plugin/datastore/test_plugin_datastore.c new file mode 100644 index 000000000..7de1acf2d --- /dev/null +++ b/src/plugin/datastore/test_plugin_datastore.c | |||
@@ -0,0 +1,478 @@ | |||
1 | /* | ||
2 | This file is part of GNUnet. | ||
3 | Copyright (C) 2011 GNUnet e.V. | ||
4 | |||
5 | GNUnet is free software: you can redistribute it and/or modify it | ||
6 | under the terms of the GNU Affero General Public License as published | ||
7 | by the Free Software Foundation, either version 3 of the License, | ||
8 | or (at your option) any later version. | ||
9 | |||
10 | GNUnet is distributed in the hope that it will be useful, but | ||
11 | WITHOUT ANY WARRANTY; without even the implied warranty of | ||
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
13 | Affero General Public License for more details. | ||
14 | |||
15 | You should have received a copy of the GNU Affero General Public License | ||
16 | along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
17 | |||
18 | SPDX-License-Identifier: AGPL3.0-or-later | ||
19 | */ | ||
20 | /* | ||
21 | * @file test_plugin_datastore.c | ||
22 | * @brief Test database plugin directly, calling each API function once | ||
23 | * @author Christian Grothoff | ||
24 | */ | ||
25 | |||
26 | #include "platform.h" | ||
27 | #include "gnunet_util_lib.h" | ||
28 | #include "gnunet_protocols.h" | ||
29 | #include "gnunet_datastore_plugin.h" | ||
30 | #include "gnunet_testing_lib.h" | ||
31 | |||
32 | /** | ||
33 | * Number of put operations to perform. | ||
34 | */ | ||
35 | #define PUT_10 10 | ||
36 | |||
37 | static unsigned long long stored_bytes; | ||
38 | |||
39 | static unsigned long long stored_entries; | ||
40 | |||
41 | static unsigned long long stored_ops; | ||
42 | |||
43 | static const char *plugin_name; | ||
44 | |||
45 | static int ok; | ||
46 | |||
47 | enum RunPhase | ||
48 | { | ||
49 | RP_ERROR = 0, | ||
50 | RP_PUT, | ||
51 | RP_GET, | ||
52 | RP_ITER_ZERO, | ||
53 | RP_REPL_GET, | ||
54 | RP_EXPI_GET, | ||
55 | RP_REMOVE, | ||
56 | RP_DROP | ||
57 | }; | ||
58 | |||
59 | |||
60 | struct CpsRunContext | ||
61 | { | ||
62 | const struct GNUNET_CONFIGURATION_Handle *cfg; | ||
63 | struct GNUNET_DATASTORE_PluginFunctions *api; | ||
64 | enum RunPhase phase; | ||
65 | unsigned int cnt; | ||
66 | unsigned int i; | ||
67 | }; | ||
68 | |||
69 | |||
70 | /** | ||
71 | * Function called by plugins to notify us about a | ||
72 | * change in their disk utilization. | ||
73 | * | ||
74 | * @param cls closure (NULL) | ||
75 | * @param delta change in disk utilization, | ||
76 | * 0 for "reset to empty" | ||
77 | */ | ||
78 | static void | ||
79 | disk_utilization_change_cb (void *cls, int delta) | ||
80 | { | ||
81 | /* do nothing */ | ||
82 | } | ||
83 | |||
84 | |||
85 | static void | ||
86 | test (void *cls); | ||
87 | |||
88 | |||
89 | /** | ||
90 | * Put continuation. | ||
91 | * | ||
92 | * @param cls closure | ||
93 | * @param key key for the item stored | ||
94 | * @param size size of the item stored | ||
95 | * @param status #GNUNET_OK or #GNUNET_SYSERROR | ||
96 | * @param msg error message on error | ||
97 | */ | ||
98 | static void | ||
99 | put_continuation (void *cls, | ||
100 | const struct GNUNET_HashCode *key, | ||
101 | uint32_t size, | ||
102 | int status, | ||
103 | const char *msg) | ||
104 | { | ||
105 | struct CpsRunContext *crc = cls; | ||
106 | static unsigned long long os; | ||
107 | unsigned long long cs; | ||
108 | |||
109 | if (GNUNET_OK != status) | ||
110 | { | ||
111 | fprintf (stderr, | ||
112 | "ERROR: `%s'\n", | ||
113 | msg); | ||
114 | } | ||
115 | else | ||
116 | { | ||
117 | crc->api->estimate_size (crc->api->cls, | ||
118 | &cs); | ||
119 | GNUNET_assert (os <= cs); | ||
120 | os = cs; | ||
121 | stored_bytes += size; | ||
122 | stored_ops++; | ||
123 | stored_entries++; | ||
124 | } | ||
125 | GNUNET_SCHEDULER_add_now (&test, crc); | ||
126 | } | ||
127 | |||
128 | |||
129 | static void | ||
130 | gen_key (int i, struct GNUNET_HashCode *key) | ||
131 | { | ||
132 | memset (key, 0, sizeof(struct GNUNET_HashCode)); | ||
133 | key->bits[0] = (unsigned int) i; | ||
134 | GNUNET_CRYPTO_hash (key, sizeof(struct GNUNET_HashCode), key); | ||
135 | } | ||
136 | |||
137 | |||
138 | static void | ||
139 | do_put (struct CpsRunContext *crc) | ||
140 | { | ||
141 | char value[65536]; | ||
142 | size_t size; | ||
143 | struct GNUNET_HashCode key; | ||
144 | unsigned int prio; | ||
145 | static int i; | ||
146 | |||
147 | if (PUT_10 == i) | ||
148 | { | ||
149 | i = 0; | ||
150 | crc->phase++; | ||
151 | GNUNET_SCHEDULER_add_now (&test, crc); | ||
152 | return; | ||
153 | } | ||
154 | /* most content is 32k */ | ||
155 | size = 32 * 1024; | ||
156 | |||
157 | if ((0 != i) && (GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 16) == | ||
158 | 0) ) /* but some of it is less! */ | ||
159 | size = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 32 * 1024); | ||
160 | size = size - (size & 7); /* always multiple of 8 */ | ||
161 | |||
162 | /* generate random key */ | ||
163 | gen_key (i, &key); | ||
164 | memset (value, i, size); | ||
165 | if (i > 255) | ||
166 | memset (value, i - 255, size / 2); | ||
167 | value[0] = crc->i; | ||
168 | prio = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 100); | ||
169 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
170 | "putting type %u, anon %u under key %s\n", i + 1, i, | ||
171 | GNUNET_h2s (&key)); | ||
172 | crc->api->put (crc->api->cls, | ||
173 | &key, | ||
174 | false /* absent */, | ||
175 | size, | ||
176 | value, i + 1 /* type */, | ||
177 | prio, | ||
178 | i /* anonymity */, | ||
179 | 0 /* replication */, | ||
180 | GNUNET_TIME_relative_to_absolute | ||
181 | (GNUNET_TIME_relative_multiply | ||
182 | (GNUNET_TIME_UNIT_MILLISECONDS, | ||
183 | 60 * 60 * 60 * 1000 | ||
184 | + GNUNET_CRYPTO_random_u32 | ||
185 | (GNUNET_CRYPTO_QUALITY_WEAK, 1000))), | ||
186 | put_continuation, | ||
187 | crc); | ||
188 | i++; | ||
189 | } | ||
190 | |||
191 | |||
192 | static uint64_t guid; | ||
193 | |||
194 | |||
195 | static int | ||
196 | iterate_one_shot (void *cls, | ||
197 | const struct GNUNET_HashCode *key, | ||
198 | uint32_t size, | ||
199 | const void *data, | ||
200 | enum GNUNET_BLOCK_Type type, | ||
201 | uint32_t priority, | ||
202 | uint32_t anonymity, | ||
203 | uint32_t replication, | ||
204 | struct GNUNET_TIME_Absolute expiration, | ||
205 | uint64_t uid) | ||
206 | { | ||
207 | struct CpsRunContext *crc = cls; | ||
208 | |||
209 | GNUNET_assert (NULL != key); | ||
210 | guid = uid; | ||
211 | crc->phase++; | ||
212 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
213 | "Found result type=%u, priority=%u, size=%u, expire=%s, key %s\n", | ||
214 | (unsigned int) type, | ||
215 | (unsigned int) priority, | ||
216 | (unsigned int) size, | ||
217 | GNUNET_STRINGS_absolute_time_to_string (expiration), | ||
218 | GNUNET_h2s (key)); | ||
219 | GNUNET_SCHEDULER_add_now (&test, | ||
220 | crc); | ||
221 | return GNUNET_OK; | ||
222 | } | ||
223 | |||
224 | |||
225 | static void | ||
226 | remove_continuation (void *cls, | ||
227 | const struct GNUNET_HashCode *key, | ||
228 | uint32_t size, | ||
229 | int status, | ||
230 | const char *msg) | ||
231 | { | ||
232 | struct CpsRunContext *crc = cls; | ||
233 | |||
234 | GNUNET_assert (NULL != key); | ||
235 | GNUNET_assert (32768 == size); | ||
236 | GNUNET_assert (GNUNET_OK == status); | ||
237 | GNUNET_assert (NULL == msg); | ||
238 | crc->phase++; | ||
239 | GNUNET_SCHEDULER_add_now (&test, | ||
240 | crc); | ||
241 | } | ||
242 | |||
243 | |||
244 | /** | ||
245 | * Function called when the service shuts | ||
246 | * down. Unloads our datastore plugin. | ||
247 | * | ||
248 | * @param api api to unload | ||
249 | * @param cfg configuration to use | ||
250 | */ | ||
251 | static void | ||
252 | unload_plugin (struct GNUNET_DATASTORE_PluginFunctions *api, | ||
253 | const struct GNUNET_CONFIGURATION_Handle *cfg) | ||
254 | { | ||
255 | char *name; | ||
256 | char *libname; | ||
257 | |||
258 | if (GNUNET_OK != | ||
259 | GNUNET_CONFIGURATION_get_value_string (cfg, | ||
260 | "DATASTORE", | ||
261 | "DATABASE", | ||
262 | &name)) | ||
263 | { | ||
264 | GNUNET_log (GNUNET_ERROR_TYPE_ERROR, | ||
265 | _ ("No `%s' specified for `%s' in configuration!\n"), | ||
266 | "DATABASE", | ||
267 | "DATASTORE"); | ||
268 | return; | ||
269 | } | ||
270 | GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", name); | ||
271 | GNUNET_break (NULL == GNUNET_PLUGIN_unload (libname, api)); | ||
272 | GNUNET_free (libname); | ||
273 | GNUNET_free (name); | ||
274 | } | ||
275 | |||
276 | |||
277 | /** | ||
278 | * Last task run during shutdown. Disconnects us from | ||
279 | * the transport and core. | ||
280 | */ | ||
281 | static void | ||
282 | cleaning_task (void *cls) | ||
283 | { | ||
284 | struct CpsRunContext *crc = cls; | ||
285 | |||
286 | unload_plugin (crc->api, crc->cfg); | ||
287 | GNUNET_free (crc); | ||
288 | } | ||
289 | |||
290 | |||
291 | static void | ||
292 | test (void *cls) | ||
293 | { | ||
294 | struct CpsRunContext *crc = cls; | ||
295 | struct GNUNET_HashCode key; | ||
296 | |||
297 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
298 | "In phase %d, iteration %u\n", crc->phase, crc->cnt); | ||
299 | switch (crc->phase) | ||
300 | { | ||
301 | case RP_ERROR: | ||
302 | ok = 1; | ||
303 | GNUNET_break (0); | ||
304 | crc->api->drop (crc->api->cls); | ||
305 | GNUNET_SCHEDULER_add_now (&cleaning_task, crc); | ||
306 | break; | ||
307 | |||
308 | case RP_PUT: | ||
309 | do_put (crc); | ||
310 | break; | ||
311 | |||
312 | case RP_GET: | ||
313 | if (crc->cnt == 1) | ||
314 | { | ||
315 | crc->cnt = 0; | ||
316 | crc->phase++; | ||
317 | GNUNET_SCHEDULER_add_now (&test, crc); | ||
318 | break; | ||
319 | } | ||
320 | gen_key (5, &key); | ||
321 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
322 | "Looking for %s\n", | ||
323 | GNUNET_h2s (&key)); | ||
324 | crc->api->get_key (crc->api->cls, | ||
325 | 0, | ||
326 | false, | ||
327 | &key, | ||
328 | GNUNET_BLOCK_TYPE_ANY, | ||
329 | &iterate_one_shot, | ||
330 | crc); | ||
331 | break; | ||
332 | |||
333 | case RP_ITER_ZERO: | ||
334 | if (crc->cnt == 1) | ||
335 | { | ||
336 | crc->cnt = 0; | ||
337 | crc->phase++; | ||
338 | GNUNET_SCHEDULER_add_now (&test, crc); | ||
339 | break; | ||
340 | } | ||
341 | crc->api->get_zero_anonymity (crc->api->cls, 0, 1, &iterate_one_shot, crc); | ||
342 | break; | ||
343 | |||
344 | case RP_REPL_GET: | ||
345 | crc->api->get_replication (crc->api->cls, &iterate_one_shot, crc); | ||
346 | break; | ||
347 | |||
348 | case RP_EXPI_GET: | ||
349 | crc->api->get_expiration (crc->api->cls, &iterate_one_shot, crc); | ||
350 | break; | ||
351 | |||
352 | case RP_REMOVE: | ||
353 | { | ||
354 | struct GNUNET_HashCode key; | ||
355 | uint32_t size = 32768; | ||
356 | char value[size]; | ||
357 | |||
358 | gen_key (0, &key); | ||
359 | memset (value, 0, size); | ||
360 | value[0] = crc->i; | ||
361 | crc->api->remove_key (crc->api->cls, | ||
362 | &key, | ||
363 | size, | ||
364 | value, | ||
365 | &remove_continuation, | ||
366 | crc); | ||
367 | break; | ||
368 | } | ||
369 | |||
370 | case RP_DROP: | ||
371 | crc->api->drop (crc->api->cls); | ||
372 | GNUNET_SCHEDULER_add_now (&cleaning_task, crc); | ||
373 | break; | ||
374 | } | ||
375 | } | ||
376 | |||
377 | |||
378 | /** | ||
379 | * Load the datastore plugin. | ||
380 | */ | ||
381 | static struct GNUNET_DATASTORE_PluginFunctions * | ||
382 | load_plugin (const struct GNUNET_CONFIGURATION_Handle *cfg) | ||
383 | { | ||
384 | static struct GNUNET_DATASTORE_PluginEnvironment env; | ||
385 | struct GNUNET_DATASTORE_PluginFunctions *ret; | ||
386 | char *name; | ||
387 | char *libname; | ||
388 | |||
389 | if (GNUNET_OK != | ||
390 | GNUNET_CONFIGURATION_get_value_string (cfg, | ||
391 | "DATASTORE", | ||
392 | "DATABASE", | ||
393 | &name)) | ||
394 | { | ||
395 | GNUNET_log (GNUNET_ERROR_TYPE_ERROR, | ||
396 | _ ("No `%s' specified for `%s' in configuration!\n"), | ||
397 | "DATABASE", | ||
398 | "DATASTORE"); | ||
399 | return NULL; | ||
400 | } | ||
401 | env.cfg = cfg; | ||
402 | env.duc = &disk_utilization_change_cb; | ||
403 | env.cls = NULL; | ||
404 | GNUNET_log (GNUNET_ERROR_TYPE_INFO, _ ("Loading `%s' datastore plugin\n"), | ||
405 | name); | ||
406 | GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", name); | ||
407 | if (NULL == (ret = GNUNET_PLUGIN_load (libname, &env))) | ||
408 | { | ||
409 | fprintf (stderr, "Failed to load plugin `%s'!\n", name); | ||
410 | GNUNET_free (libname); | ||
411 | GNUNET_free (name); | ||
412 | ok = 77; /* mark test as skipped */ | ||
413 | return NULL; | ||
414 | } | ||
415 | GNUNET_free (libname); | ||
416 | GNUNET_free (name); | ||
417 | return ret; | ||
418 | } | ||
419 | |||
420 | |||
421 | static void | ||
422 | run (void *cls, char *const *args, const char *cfgfile, | ||
423 | const struct GNUNET_CONFIGURATION_Handle *c) | ||
424 | { | ||
425 | struct GNUNET_DATASTORE_PluginFunctions *api; | ||
426 | struct CpsRunContext *crc; | ||
427 | |||
428 | api = load_plugin (c); | ||
429 | if (api == NULL) | ||
430 | { | ||
431 | fprintf (stderr, | ||
432 | "%s", | ||
433 | "Could not initialize plugin, assuming database not configured. Test not run!\n"); | ||
434 | return; | ||
435 | } | ||
436 | crc = GNUNET_new (struct CpsRunContext); | ||
437 | crc->api = api; | ||
438 | crc->cfg = c; | ||
439 | crc->phase = RP_PUT; | ||
440 | GNUNET_SCHEDULER_add_now (&test, crc); | ||
441 | } | ||
442 | |||
443 | |||
444 | int | ||
445 | main (int argc, char *argv[]) | ||
446 | { | ||
447 | char dir_name[PATH_MAX]; | ||
448 | char cfg_name[PATH_MAX]; | ||
449 | char *const xargv[] = { | ||
450 | "test-plugin-datastore", | ||
451 | "-c", | ||
452 | cfg_name, | ||
453 | NULL | ||
454 | }; | ||
455 | static struct GNUNET_GETOPT_CommandLineOption options[] = { | ||
456 | GNUNET_GETOPT_OPTION_END | ||
457 | }; | ||
458 | |||
459 | /* determine name of plugin to use */ | ||
460 | plugin_name = GNUNET_STRINGS_get_suffix_from_binary_name (argv[0]); | ||
461 | GNUNET_snprintf (dir_name, sizeof(dir_name), | ||
462 | "/tmp/test-gnunet-datastore-plugin-%s", plugin_name); | ||
463 | GNUNET_DISK_directory_remove (dir_name); | ||
464 | GNUNET_log_setup ("test-plugin-datastore", | ||
465 | "WARNING", | ||
466 | NULL); | ||
467 | GNUNET_snprintf (cfg_name, sizeof(cfg_name), | ||
468 | "test_plugin_datastore_data_%s.conf", plugin_name); | ||
469 | GNUNET_PROGRAM_run ((sizeof(xargv) / sizeof(char *)) - 1, xargv, | ||
470 | "test-plugin-datastore", "nohelp", options, &run, NULL); | ||
471 | if ((0 != ok) && (77 != ok)) | ||
472 | fprintf (stderr, "Missed some testcases: %u\n", ok); | ||
473 | GNUNET_DISK_directory_remove (dir_name); | ||
474 | return ok; | ||
475 | } | ||
476 | |||
477 | |||
478 | /* end of test_plugin_datastore.c */ | ||
diff --git a/src/plugin/datastore/test_plugin_datastore_data_heap.conf b/src/plugin/datastore/test_plugin_datastore_data_heap.conf new file mode 100644 index 000000000..b1ea8ff67 --- /dev/null +++ b/src/plugin/datastore/test_plugin_datastore_data_heap.conf | |||
@@ -0,0 +1,6 @@ | |||
1 | @INLINE@ test_defaults.conf | ||
2 | [PATHS] | ||
3 | GNUNET_TEST_HOME = $GNUNET_TMP/test-gnunet-datastore-plugin-heap/ | ||
4 | |||
5 | [datastore] | ||
6 | DATABASE = heap | ||
diff --git a/src/plugin/datastore/test_plugin_datastore_data_postgres.conf b/src/plugin/datastore/test_plugin_datastore_data_postgres.conf new file mode 100644 index 000000000..d0e29437f --- /dev/null +++ b/src/plugin/datastore/test_plugin_datastore_data_postgres.conf | |||
@@ -0,0 +1,10 @@ | |||
1 | @INLINE@ test_defaults.conf | ||
2 | [PATHS] | ||
3 | GNUNET_TEST_HOME = $GNUNET_TMP/test-gnunet-datastore-plugin-postgres/ | ||
4 | |||
5 | [datastore] | ||
6 | DATABASE = postgres | ||
7 | |||
8 | [datastore-postgres] | ||
9 | CONFIG = dbname=gnunetcheck | ||
10 | |||
diff --git a/src/plugin/datastore/test_plugin_datastore_data_sqlite.conf b/src/plugin/datastore/test_plugin_datastore_data_sqlite.conf new file mode 100644 index 000000000..ca837c77a --- /dev/null +++ b/src/plugin/datastore/test_plugin_datastore_data_sqlite.conf | |||
@@ -0,0 +1,4 @@ | |||
1 | @INLINE@ test_defaults.conf | ||
2 | [PATHS] | ||
3 | GNUNET_TEST_HOME = $GNUNET_TMP/test-gnunet-datastore-plugin-sqlite/ | ||
4 | |||