aboutsummaryrefslogtreecommitdiff
path: root/src/plugin/datastore
diff options
context:
space:
mode:
Diffstat (limited to 'src/plugin/datastore')
-rw-r--r--src/plugin/datastore/Makefile.am150
-rw-r--r--src/plugin/datastore/datastore-0001.sql49
-rw-r--r--src/plugin/datastore/datastore-drop.sql25
-rw-r--r--src/plugin/datastore/meson.build75
-rw-r--r--src/plugin/datastore/perf_plugin_datastore.c573
-rw-r--r--src/plugin/datastore/perf_plugin_datastore_data_heap.conf7
-rw-r--r--src/plugin/datastore/perf_plugin_datastore_data_postgres.conf10
-rw-r--r--src/plugin/datastore/perf_plugin_datastore_data_sqlite.conf4
-rw-r--r--src/plugin/datastore/plugin_datastore_heap.c944
-rw-r--r--src/plugin/datastore/plugin_datastore_postgres.c930
-rw-r--r--src/plugin/datastore/plugin_datastore_sqlite.c1375
-rw-r--r--src/plugin/datastore/plugin_datastore_template.c274
-rw-r--r--src/plugin/datastore/test_defaults.conf10
-rw-r--r--src/plugin/datastore/test_plugin_datastore.c478
-rw-r--r--src/plugin/datastore/test_plugin_datastore_data_heap.conf6
-rw-r--r--src/plugin/datastore/test_plugin_datastore_data_postgres.conf10
-rw-r--r--src/plugin/datastore/test_plugin_datastore_data_sqlite.conf4
17 files changed, 4924 insertions, 0 deletions
diff --git a/src/plugin/datastore/Makefile.am b/src/plugin/datastore/Makefile.am
new file mode 100644
index 000000000..1f4ab59c8
--- /dev/null
+++ b/src/plugin/datastore/Makefile.am
@@ -0,0 +1,150 @@
1# This Makefile.am is in the public domain
2AM_CPPFLAGS = -I$(top_srcdir)/src/include
3
4plugindir = $(libdir)/gnunet
5
6pkgcfgdir= $(pkgdatadir)/config.d/
7
8libexecdir= $(pkglibdir)/libexec/
9
10sqldir = $(prefix)/share/gnunet/sql/
11
12sql_DATA = \
13 datastore-0001.sql \
14 datastore-drop.sql
15
16if USE_COVERAGE
17 AM_CFLAGS = --coverage -O0
18 XLIBS = -lgcov
19endif
20
21
22if HAVE_SQLITE
23 SQLITE_PLUGIN = libgnunet_plugin_datastore_sqlite.la
24if HAVE_BENCHMARKS
25 SQLITE_BENCHMARKS = \
26 perf_plugin_datastore_sqlite
27endif
28 SQLITE_TESTS = \
29 test_plugin_datastore_sqlite \
30 $(SQLITE_BENCHMARKS)
31endif
32if HAVE_POSTGRESQL
33 POSTGRES_PLUGIN = libgnunet_plugin_datastore_postgres.la
34if HAVE_BENCHMARKS
35 POSTGRES_BENCHMARKS = \
36 perf_plugin_datastore_postgres
37endif
38 POSTGRES_TESTS = \
39 test_plugin_datastore_postgres \
40 $(POSTGRES_BENCHMARKS)
41endif
42
43plugin_LTLIBRARIES = \
44 $(SQLITE_PLUGIN) \
45 $(POSTGRES_PLUGIN) \
46 libgnunet_plugin_datastore_heap.la
47
48# Real plugins should of course go into
49# plugin_LTLIBRARIES
50noinst_LTLIBRARIES = \
51 libgnunet_plugin_datastore_template.la
52
53
54libgnunet_plugin_datastore_sqlite_la_SOURCES = \
55 plugin_datastore_sqlite.c
56libgnunet_plugin_datastore_sqlite_la_LIBADD = \
57 $(top_builddir)/src/lib/sq/libgnunetsq.la \
58 $(top_builddir)/src/lib/util/libgnunetutil.la $(XLIBS) -lsqlite3 \
59 $(LTLIBINTL)
60libgnunet_plugin_datastore_sqlite_la_LDFLAGS = \
61 $(GN_PLUGIN_LDFLAGS)
62
63
64libgnunet_plugin_datastore_heap_la_SOURCES = \
65 plugin_datastore_heap.c
66libgnunet_plugin_datastore_heap_la_LIBADD = \
67 $(top_builddir)/src/lib/util/libgnunetutil.la $(XLIBS) \
68 $(LTLIBINTL)
69libgnunet_plugin_datastore_heap_la_LDFLAGS = \
70 $(GN_PLUGIN_LDFLAGS)
71
72
73libgnunet_plugin_datastore_postgres_la_SOURCES = \
74 plugin_datastore_postgres.c
75libgnunet_plugin_datastore_postgres_la_LIBADD = \
76 $(top_builddir)/src/lib/pq/libgnunetpq.la \
77 $(top_builddir)/src/lib/util/libgnunetutil.la $(XLIBS) -lpq
78libgnunet_plugin_datastore_postgres_la_LDFLAGS = \
79 $(GN_PLUGIN_LDFLAGS) $(POSTGRESQL_LDFLAGS)
80libgnunet_plugin_datastore_postgres_la_CPPFLAGS = \
81 $(POSTGRESQL_CPPFLAGS) $(AM_CPPFLAGS)
82
83
84libgnunet_plugin_datastore_template_la_SOURCES = \
85 plugin_datastore_template.c
86libgnunet_plugin_datastore_template_la_LIBADD = \
87 $(top_builddir)/src/lib/util/libgnunetutil.la $(XLIBS) \
88 $(LTLIBINTL)
89libgnunet_plugin_datastore_template_la_LDFLAGS = \
90 $(GN_PLUGIN_LDFLAGS)
91
92check_PROGRAMS = \
93 perf_plugin_datastore_heap \
94 test_plugin_datastore_heap \
95 $(SQLITE_TESTS) \
96 $(POSTGRES_TESTS)
97
98if ENABLE_TEST_RUN
99AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
100TESTS = $(check_PROGRAMS)
101endif
102
103perf_plugin_datastore_heap_SOURCES = \
104 perf_plugin_datastore.c
105perf_plugin_datastore_heap_LDADD = \
106 $(top_builddir)/src/service/testing/libgnunettesting.la \
107 $(top_builddir)/src/lib/util/libgnunetutil.la
108
109test_plugin_datastore_heap_SOURCES = \
110 test_plugin_datastore.c
111test_plugin_datastore_heap_LDADD = \
112 $(top_builddir)/src/service/testing/libgnunettesting.la \
113 $(top_builddir)/src/lib/util/libgnunetutil.la
114
115
116perf_plugin_datastore_sqlite_SOURCES = \
117 perf_plugin_datastore.c
118perf_plugin_datastore_sqlite_LDADD = \
119 $(top_builddir)/src/service/testing/libgnunettesting.la \
120 $(top_builddir)/src/lib/util/libgnunetutil.la
121
122test_plugin_datastore_sqlite_SOURCES = \
123 test_plugin_datastore.c
124test_plugin_datastore_sqlite_LDADD = \
125 $(top_builddir)/src/service/testing/libgnunettesting.la \
126 $(top_builddir)/src/lib/util/libgnunetutil.la
127
128
129test_plugin_datastore_postgres_SOURCES = \
130 test_plugin_datastore.c
131test_plugin_datastore_postgres_LDADD = \
132 $(top_builddir)/src/service/testing/libgnunettesting.la \
133 $(top_builddir)/src/lib/util/libgnunetutil.la
134
135perf_plugin_datastore_postgres_SOURCES = \
136 perf_plugin_datastore.c
137perf_plugin_datastore_postgres_LDADD = \
138 $(top_builddir)/src/service/testing/libgnunettesting.la \
139 $(top_builddir)/src/lib/util/libgnunetutil.la
140
141
142EXTRA_DIST = \
143 test_defaults.conf \
144 perf_plugin_datastore_data_sqlite.conf \
145 test_plugin_datastore_data_sqlite.conf \
146 perf_plugin_datastore_data_heap.conf \
147 test_plugin_datastore_data_heap.conf \
148 perf_plugin_datastore_data_postgres.conf \
149 test_plugin_datastore_data_postgres.conf \
150 $(sql_DATA)
diff --git a/src/plugin/datastore/datastore-0001.sql b/src/plugin/datastore/datastore-0001.sql
new file mode 100644
index 000000000..0d4758be2
--- /dev/null
+++ b/src/plugin/datastore/datastore-0001.sql
@@ -0,0 +1,49 @@
1--
2-- This file is part of GNUnet
3-- Copyright (C) 2014--2022 GNUnet e.V.
4--
5-- GNUnet is free software; you can redistribute it and/or modify it under the
6-- terms of the GNU General Public License as published by the Free Software
7-- Foundation; either version 3, or (at your option) any later version.
8--
9-- GNUnet is distributed in the hope that it will be useful, but WITHOUT ANY
10-- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
11-- A PARTICULAR PURPOSE. See the GNU General Public License for more details.
12--
13-- You should have received a copy of the GNU General Public License along with
14-- GNUnet; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
15--
16
17-- Everything in one big transaction
18BEGIN;
19
20-- Check patch versioning is in place.
21SELECT _v.register_patch('datastore-0001', NULL, NULL);
22
23-------------------- Schema ----------------------------
24
25CREATE SCHEMA datastore;
26COMMENT ON SCHEMA datastore IS 'gnunet-datastore data';
27
28SET search_path TO datastore;
29
30CREATE TABLE IF NOT EXISTS gn090 (
31 repl INTEGER NOT NULL DEFAULT 0,
32 type INTEGER NOT NULL DEFAULT 0,
33 prio INTEGER NOT NULL DEFAULT 0,
34 anonLevel INTEGER NOT NULL DEFAULT 0,
35 expire BIGINT NOT NULL DEFAULT 0,
36 rvalue BIGINT NOT NULL DEFAULT 0,
37 hash BYTEA NOT NULL DEFAULT '',
38 vhash BYTEA NOT NULL DEFAULT '',
39 value BYTEA NOT NULL DEFAULT '',
40 oid BIGINT GENERATED BY DEFAULT AS IDENTITY);
41
42CREATE INDEX IF NOT EXISTS oid_hash ON gn090 (oid);
43CREATE INDEX IF NOT EXISTS idx_hash ON gn090 (hash);
44CREATE INDEX IF NOT EXISTS idx_prio_anon ON gn090 (prio,anonLevel);
45CREATE INDEX IF NOT EXISTS idx_prio_hash_anon ON gn090 (prio,hash,anonLevel);
46CREATE INDEX IF NOT EXISTS idx_repl_rvalue ON gn090 (repl,rvalue);
47CREATE INDEX IF NOT EXISTS idx_expire_hash ON gn090 (expire,hash);
48
49COMMIT;
diff --git a/src/plugin/datastore/datastore-drop.sql b/src/plugin/datastore/datastore-drop.sql
new file mode 100644
index 000000000..67fee303d
--- /dev/null
+++ b/src/plugin/datastore/datastore-drop.sql
@@ -0,0 +1,25 @@
1--
2-- This file is part of GNUnet
3-- Copyright (C) 2014--2022 GNUnet e.V.
4--
5-- GNUnet is free software; you can redistribute it and/or modify it under the
6-- terms of the GNU General Public License as published by the Free Software
7-- Foundation; either version 3, or (at your option) any later version.
8--
9-- GNUnet is distributed in the hope that it will be useful, but WITHOUT ANY
10-- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
11-- A PARTICULAR PURPOSE. See the GNU General Public License for more details.
12--
13-- You should have received a copy of the GNU General Public License along with
14-- GNUnet; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
15--
16
17-- Everything in one big transaction
18BEGIN;
19
20
21SELECT _v.unregister_patch('datastore-0001');
22
23DROP SCHEMA datastore CASCADE;
24
25COMMIT;
diff --git a/src/plugin/datastore/meson.build b/src/plugin/datastore/meson.build
new file mode 100644
index 000000000..6769cb78b
--- /dev/null
+++ b/src/plugin/datastore/meson.build
@@ -0,0 +1,75 @@
1install_data ('datastore-0001.sql',
2 'datastore-drop.sql',
3 install_dir: get_option('datadir')/'gnunet'/'sql')
4
5shared_module('gnunet_plugin_datastore_sqlite',
6 ['plugin_datastore_sqlite.c'],
7 dependencies: [libgnunetutil_dep,
8 sqlite_dep,
9 libgnunetsq_dep],
10 include_directories: [incdir, configuration_inc],
11 install: true,
12 install_dir: get_option('libdir')/'gnunet')
13
14shared_module('gnunet_plugin_datastore_heap',
15 ['plugin_datastore_heap.c'],
16 dependencies: [libgnunetutil_dep],
17 include_directories: [incdir, configuration_inc],
18 install: true,
19 install_dir: get_option('libdir')/'gnunet')
20
21if pq_dep.found()
22 shared_module('gnunet_plugin_datastore_postgres',
23 ['plugin_datastore_postgres.c'],
24 dependencies: [libgnunetutil_dep,
25 pq_dep,
26 libgnunetpq_dep],
27 include_directories: [incdir, configuration_inc],
28 install: true,
29 install_dir: get_option('libdir')/'gnunet')
30endif
31
32testds_plugin_sqlite = executable ('test_plugin_datastore_sqlite',
33 ['test_plugin_datastore.c'],
34 dependencies: [
35 libgnunetutil_dep,
36 ],
37 include_directories: [incdir, configuration_inc],
38 install: false)
39
40testds_plugin_heap = executable ('test_plugin_datastore_heap',
41 ['test_plugin_datastore.c'],
42 dependencies: [
43 libgnunetutil_dep,
44 ],
45 include_directories: [incdir, configuration_inc],
46 install: false)
47
48testds_plugin_pq = executable ('test_plugin_datastore_postgres',
49 ['test_plugin_datastore.c'],
50 dependencies: [
51 libgnunetutil_dep,
52 ],
53 include_directories: [incdir, configuration_inc],
54 install: false)
55
56configure_file(input : 'test_defaults.conf',
57 output : 'test_defaults.conf',
58 copy: true)
59configure_file(input : 'test_plugin_datastore_data_sqlite.conf',
60 output : 'test_plugin_datastore_data_sqlite.conf',
61 copy: true)
62configure_file(input : 'test_plugin_datastore_data_heap.conf',
63 output : 'test_plugin_datastore_data_heap.conf',
64 copy: true)
65configure_file(input : 'test_plugin_datastore_data_postgres.conf',
66 output : 'test_plugin_datastore_data_postgres.conf',
67 copy: true)
68
69test('test_plugin_datastore_sqlite', testds_plugin_sqlite,
70 suite: 'datastore', workdir: meson.current_build_dir())
71test('test_plugin_datastore_heap', testds_plugin_heap,
72 suite: 'datastore', workdir: meson.current_build_dir())
73test('test_plugin_datastore_postgres', testds_plugin_pq,
74 suite: 'datastore', workdir: meson.current_build_dir())
75
diff --git a/src/plugin/datastore/perf_plugin_datastore.c b/src/plugin/datastore/perf_plugin_datastore.c
new file mode 100644
index 000000000..8e63b08e6
--- /dev/null
+++ b/src/plugin/datastore/perf_plugin_datastore.c
@@ -0,0 +1,573 @@
1/*
2 This file is part of GNUnet.
3 Copyright (C) 2004, 2005, 2006, 2007, 2009, 2011 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20/*
21 * @file perf_plugin_datastore.c
22 * @brief Profile database plugin directly, focusing on iterators.
23 * @author Christian Grothoff
24 */
25
26#include "platform.h"
27#include "gnunet_util_lib.h"
28#include "gnunet_protocols.h"
29#include "gnunet_datastore_plugin.h"
30#include "gnunet_testing_lib.h"
31#include <gauger.h>
32
33/**
34 * Target datastore size (in bytes). Realistic sizes are
35 * more like 16 GB (not the default of 16 MB); however,
36 * those take too long to run them in the usual "make check"
37 * sequence. Hence the value used for shipping is tiny.
38 */
39#define MAX_SIZE 1024LL * 1024 * 16 * 1
40
41#define ITERATIONS 2
42
43/**
44 * Number of put operations equivalent to 1/10th of MAX_SIZE
45 */
46#define PUT_10 (MAX_SIZE / 32 / 1024 / ITERATIONS)
47
48static char category[256];
49
50static unsigned int hits[PUT_10 / 8 + 1];
51
52static unsigned long long stored_bytes;
53
54static unsigned long long stored_entries;
55
56static unsigned long long stored_ops;
57
58static const char *plugin_name;
59
60static int ok;
61
62enum RunPhase
63{
64 RP_ERROR = 0,
65 RP_PUT,
66 RP_REP_GET,
67 RP_ZA_GET,
68 RP_EXP_GET,
69 RP_DONE
70};
71
72
73struct CpsRunContext
74{
75 unsigned int i;
76 struct GNUNET_TIME_Absolute start;
77 struct GNUNET_TIME_Absolute end;
78 const struct GNUNET_CONFIGURATION_Handle *cfg;
79 struct GNUNET_DATASTORE_PluginFunctions *api;
80 enum RunPhase phase;
81 unsigned int cnt;
82 unsigned int iter;
83 uint64_t offset;
84};
85
86
87/**
88 * Function called by plugins to notify us about a
89 * change in their disk utilization.
90 *
91 * @param cls closure (NULL)
92 * @param delta change in disk utilization,
93 * 0 for "reset to empty"
94 */
95static void
96disk_utilization_change_cb (void *cls, int delta)
97{
98}
99
100
101static void
102test (void *cls);
103
104
105/**
106 * Put continuation.
107 *
108 * @param cls closure
109 * @param key key for the item stored
110 * @param size size of the item stored
111 * @param status #GNUNET_OK or #GNUNET_SYSERROR
112 * @param msg error message on error
113 */
114static void
115put_continuation (void *cls,
116 const struct GNUNET_HashCode *key,
117 uint32_t size,
118 int status,
119 const char *msg)
120{
121 struct CpsRunContext *crc = cls;
122
123 if (GNUNET_OK != status)
124 {
125 fprintf (stderr, "ERROR: `%s'\n", msg);
126 }
127 else
128 {
129 stored_bytes += size;
130 stored_ops++;
131 stored_entries++;
132 }
133 GNUNET_SCHEDULER_add_now (&test, crc);
134}
135
136
137static void
138do_put (struct CpsRunContext *crc)
139{
140 char value[65536];
141 size_t size;
142 static struct GNUNET_HashCode key;
143 static int i;
144 unsigned int prio;
145
146 if (0 == i)
147 crc->start = GNUNET_TIME_absolute_get ();
148 if (PUT_10 == i)
149 {
150 i = 0;
151 crc->end = GNUNET_TIME_absolute_get ();
152 {
153 printf ("%s took %s for %llu items\n", "Storing an item",
154 GNUNET_STRINGS_relative_time_to_string (
155 GNUNET_TIME_absolute_get_difference (crc->start,
156 crc
157 ->end),
158 GNUNET_YES),
159 PUT_10);
160 if (PUT_10 > 0)
161 GAUGER (category, "Storing an item",
162 (crc->end.abs_value_us - crc->start.abs_value_us) / 1000LL
163 / PUT_10,
164 "ms/item");
165 }
166 crc->i++;
167 crc->start = GNUNET_TIME_absolute_get ();
168 crc->phase++;
169 GNUNET_SCHEDULER_add_now (&test, crc);
170 return;
171 }
172 /* most content is 32k */
173 size = 32 * 1024;
174 if (GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 16) == 0) /* but some of it is less! */
175 size = 8 + GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 32 * 1024);
176 size = size - (size & 7); /* always multiple of 8 */
177
178 /* generate random key */
179 key.bits[0] = (unsigned int) GNUNET_TIME_absolute_get ().abs_value_us;
180 GNUNET_CRYPTO_hash (&key, sizeof(struct GNUNET_HashCode), &key);
181 memset (value, i, size);
182 if (i > 255)
183 memset (value, i - 255, size / 2);
184 value[0] = crc->i;
185 GNUNET_memcpy (&value[4], &i, sizeof(i));
186 prio = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 100);
187 crc->api->put (crc->api->cls,
188 &key,
189 false /* absent */,
190 size,
191 value,
192 1 + i % 4 /* type */,
193 prio,
194 i % 4 /* anonymity */,
195 0 /* replication */,
196 GNUNET_TIME_relative_to_absolute
197 (GNUNET_TIME_relative_multiply
198 (GNUNET_TIME_UNIT_MILLISECONDS,
199 60 * 60 * 60 * 1000
200 + GNUNET_CRYPTO_random_u32
201 (GNUNET_CRYPTO_QUALITY_WEAK, 1000))),
202 put_continuation,
203 crc);
204 i++;
205}
206
207
208static int
209iterate_zeros (void *cls,
210 const struct GNUNET_HashCode *key,
211 uint32_t size,
212 const void *data,
213 enum GNUNET_BLOCK_Type type,
214 uint32_t priority,
215 uint32_t anonymity,
216 uint32_t replication,
217 struct GNUNET_TIME_Absolute expiration,
218 uint64_t uid)
219{
220 struct CpsRunContext *crc = cls;
221 int i;
222 const char *cdata = data;
223
224 GNUNET_assert (key != NULL);
225 GNUNET_assert (size >= 8);
226 GNUNET_memcpy (&i, &cdata[4], sizeof(i));
227 hits[i / 8] |= (1 << (i % 8));
228
229 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
230 "Found result %d type=%u, priority=%u, size=%u, expire=%s\n",
231 i,
232 type, priority, size,
233 GNUNET_STRINGS_absolute_time_to_string (expiration));
234 crc->cnt++;
235 if (crc->cnt == PUT_10 / 4 - 1)
236 {
237 unsigned int bc;
238
239 bc = 0;
240 for (i = 0; i < PUT_10; i++)
241 if (0 != (hits[i / 8] & (1 << (i % 8))))
242 bc++;
243
244 crc->end = GNUNET_TIME_absolute_get ();
245 printf ("%s took %s yielding %u/%u items\n",
246 "Select random zero-anonymity item",
247 GNUNET_STRINGS_relative_time_to_string (
248 GNUNET_TIME_absolute_get_difference (crc->start,
249 crc
250 ->end),
251 GNUNET_YES),
252 bc, crc->cnt);
253 if (crc->cnt > 0)
254 GAUGER (category, "Select random zero-anonymity item",
255 (crc->end.abs_value_us - crc->start.abs_value_us) / 1000LL
256 / crc->cnt,
257 "ms/item");
258 memset (hits, 0, sizeof(hits));
259 crc->phase++;
260 crc->cnt = 0;
261 crc->start = GNUNET_TIME_absolute_get ();
262 }
263 GNUNET_SCHEDULER_add_now (&test, crc);
264 return GNUNET_OK;
265}
266
267
268static int
269expiration_get (void *cls,
270 const struct GNUNET_HashCode *key,
271 uint32_t size,
272 const void *data,
273 enum GNUNET_BLOCK_Type type,
274 uint32_t priority,
275 uint32_t anonymity,
276 uint32_t replication,
277 struct GNUNET_TIME_Absolute expiration,
278 uint64_t uid)
279{
280 struct CpsRunContext *crc = cls;
281 int i;
282 const char *cdata = data;
283
284 GNUNET_assert (size >= 8);
285 GNUNET_memcpy (&i, &cdata[4], sizeof(i));
286 hits[i / 8] |= (1 << (i % 8));
287 crc->cnt++;
288 if (PUT_10 <= crc->cnt)
289 {
290 unsigned int bc;
291
292 bc = 0;
293 for (i = 0; i < PUT_10; i++)
294 if (0 != (hits[i / 8] & (1 << (i % 8))))
295 bc++;
296
297 crc->end = GNUNET_TIME_absolute_get ();
298 printf ("%s took %s yielding %u/%u items\n",
299 "Selecting and deleting by expiration",
300 GNUNET_STRINGS_relative_time_to_string (
301 GNUNET_TIME_absolute_get_difference (crc->start,
302 crc
303 ->end),
304 GNUNET_YES),
305 bc, (unsigned int) PUT_10);
306 if (crc->cnt > 0)
307 GAUGER (category, "Selecting and deleting by expiration",
308 (crc->end.abs_value_us - crc->start.abs_value_us) / 1000LL
309 / crc->cnt,
310 "ms/item");
311 memset (hits, 0, sizeof(hits));
312 if (++crc->iter == ITERATIONS)
313 crc->phase++;
314 else
315 crc->phase = RP_PUT;
316 crc->cnt = 0;
317 crc->start = GNUNET_TIME_absolute_get ();
318 }
319 GNUNET_SCHEDULER_add_now (&test, crc);
320 return GNUNET_NO;
321}
322
323
324static int
325replication_get (void *cls,
326 const struct GNUNET_HashCode *key,
327 uint32_t size,
328 const void *data,
329 enum GNUNET_BLOCK_Type type,
330 uint32_t priority,
331 uint32_t anonymity,
332 uint32_t replication,
333 struct GNUNET_TIME_Absolute expiration,
334 uint64_t uid)
335{
336 struct CpsRunContext *crc = cls;
337 int i;
338 const char *cdata = data;
339
340 GNUNET_assert (NULL != key);
341 GNUNET_assert (size >= 8);
342 GNUNET_memcpy (&i, &cdata[4], sizeof(i));
343 hits[i / 8] |= (1 << (i % 8));
344 crc->cnt++;
345 if (PUT_10 <= crc->cnt)
346 {
347 unsigned int bc;
348
349 bc = 0;
350 for (i = 0; i < PUT_10; i++)
351 if (0 != (hits[i / 8] & (1 << (i % 8))))
352 bc++;
353
354 crc->end = GNUNET_TIME_absolute_get ();
355 printf ("%s took %s yielding %u/%u items\n",
356 "Selecting random item for replication",
357 GNUNET_STRINGS_relative_time_to_string (
358 GNUNET_TIME_absolute_get_difference (crc->start,
359 crc
360 ->end),
361 GNUNET_YES),
362 bc, (unsigned int) PUT_10);
363 if (crc->cnt > 0)
364 GAUGER (category, "Selecting random item for replication",
365 (crc->end.abs_value_us - crc->start.abs_value_us) / 1000LL
366 / crc->cnt,
367 "ms/item");
368 memset (hits, 0, sizeof(hits));
369 crc->phase++;
370 crc->offset = 0;
371 crc->cnt = 0;
372 crc->start = GNUNET_TIME_absolute_get ();
373 }
374
375 GNUNET_SCHEDULER_add_now (&test, crc);
376 return GNUNET_OK;
377}
378
379
380/**
381 * Function called when the service shuts
382 * down. Unloads our datastore plugin.
383 *
384 * @param api api to unload
385 * @param cfg configuration to use
386 */
387static void
388unload_plugin (struct GNUNET_DATASTORE_PluginFunctions *api,
389 const struct GNUNET_CONFIGURATION_Handle *cfg)
390{
391 char *name;
392 char *libname;
393
394 if (GNUNET_OK !=
395 GNUNET_CONFIGURATION_get_value_string (cfg, "DATASTORE", "DATABASE",
396 &name))
397 {
398 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
399 _ ("No `%s' specified for `%s' in configuration!\n"),
400 "DATABASE",
401 "DATASTORE");
402 return;
403 }
404 GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", name);
405 GNUNET_break (NULL == GNUNET_PLUGIN_unload (libname, api));
406 GNUNET_free (libname);
407 GNUNET_free (name);
408}
409
410
411/**
412 * Last task run during shutdown. Disconnects us from
413 * the transport and core.
414 */
415static void
416cleaning_task (void *cls)
417{
418 struct CpsRunContext *crc = cls;
419
420 unload_plugin (crc->api, crc->cfg);
421 GNUNET_free (crc);
422}
423
424
425static void
426test (void *cls)
427{
428 struct CpsRunContext *crc = cls;
429
430 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
431 "In phase %d, iteration %u\n", crc->phase, crc->cnt);
432 switch (crc->phase)
433 {
434 case RP_ERROR:
435 GNUNET_break (0);
436 crc->api->drop (crc->api->cls);
437 ok = 1;
438 GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_IDLE,
439 &cleaning_task, crc);
440 break;
441
442 case RP_PUT:
443 do_put (crc);
444 break;
445
446 case RP_REP_GET:
447 crc->api->get_replication (crc->api->cls, &replication_get, crc);
448 break;
449
450 case RP_ZA_GET:
451 crc->api->get_zero_anonymity (crc->api->cls, crc->offset++, 1,
452 &iterate_zeros, crc);
453 break;
454
455 case RP_EXP_GET:
456 crc->api->get_expiration (crc->api->cls, &expiration_get, crc);
457 break;
458
459 case RP_DONE:
460 crc->api->drop (crc->api->cls);
461 ok = 0;
462 GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_IDLE,
463 &cleaning_task, crc);
464 break;
465 }
466}
467
468
469/**
470 * Load the datastore plugin.
471 */
472static struct GNUNET_DATASTORE_PluginFunctions *
473load_plugin (const struct GNUNET_CONFIGURATION_Handle *cfg)
474{
475 static struct GNUNET_DATASTORE_PluginEnvironment env;
476 struct GNUNET_DATASTORE_PluginFunctions *ret;
477 char *name;
478 char *libname;
479
480 if (GNUNET_OK !=
481 GNUNET_CONFIGURATION_get_value_string (cfg, "DATASTORE", "DATABASE",
482 &name))
483 {
484 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
485 _ ("No `%s' specified for `%s' in configuration!\n"),
486 "DATABASE",
487 "DATASTORE");
488 return NULL;
489 }
490 env.cfg = cfg;
491 env.duc = &disk_utilization_change_cb;
492 env.cls = NULL;
493 GNUNET_log (GNUNET_ERROR_TYPE_INFO, _ ("Loading `%s' datastore plugin\n"),
494 name);
495 GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", name);
496 if (NULL == (ret = GNUNET_PLUGIN_load (libname, &env)))
497 {
498 fprintf (stderr, "Failed to load plugin `%s'!\n", name);
499 GNUNET_free (name);
500 GNUNET_free (libname);
501 return NULL;
502 }
503 GNUNET_free (libname);
504 GNUNET_free (name);
505 return ret;
506}
507
508
509static void
510run (void *cls, char *const *args, const char *cfgfile,
511 const struct GNUNET_CONFIGURATION_Handle *c)
512{
513 struct GNUNET_DATASTORE_PluginFunctions *api;
514 struct CpsRunContext *crc;
515
516 if (NULL == c)
517 {
518 GNUNET_break (0);
519 return;
520 }
521 api = load_plugin (c);
522 if (api == NULL)
523 {
524 fprintf (stderr,
525 "%s",
526 "Could not initialize plugin, assuming database not configured. Test not run!\n");
527 return;
528 }
529 crc = GNUNET_new (struct CpsRunContext);
530 crc->api = api;
531 crc->cfg = c;
532 crc->phase = RP_PUT;
533 ok = 2;
534 GNUNET_SCHEDULER_add_now (&test, crc);
535}
536
537
538int
539main (int argc, char *argv[])
540{
541 char dir_name[PATH_MAX];
542 char cfg_name[PATH_MAX];
543 char *const xargv[] = {
544 "perf-plugin-datastore",
545 "-c",
546 cfg_name,
547 NULL
548 };
549 struct GNUNET_GETOPT_CommandLineOption options[] = {
550 GNUNET_GETOPT_OPTION_END
551 };
552
553 plugin_name = GNUNET_STRINGS_get_suffix_from_binary_name (argv[0]);
554 GNUNET_snprintf (dir_name, sizeof(dir_name), "/tmp/perf-gnunet-datastore-%s",
555 plugin_name);
556 GNUNET_DISK_directory_remove (dir_name);
557 GNUNET_log_setup ("perf-plugin-datastore",
558 "WARNING",
559 NULL);
560 GNUNET_snprintf (category, sizeof(category), "DATASTORE-%s", plugin_name);
561 GNUNET_snprintf (cfg_name, sizeof(cfg_name),
562 "perf_plugin_datastore_data_%s.conf", plugin_name);
563 GNUNET_PROGRAM_run ((sizeof(xargv) / sizeof(char *)) - 1, xargv,
564 "perf-plugin-datastore", "nohelp", options, &run, NULL);
565 if (ok != 0)
566 fprintf (stderr, "Missed some testcases: %u\n", ok);
567 GNUNET_DISK_directory_remove (dir_name);
568
569 return ok;
570}
571
572
573/* end of perf_plugin_datastore.c */
diff --git a/src/plugin/datastore/perf_plugin_datastore_data_heap.conf b/src/plugin/datastore/perf_plugin_datastore_data_heap.conf
new file mode 100644
index 000000000..873cf9606
--- /dev/null
+++ b/src/plugin/datastore/perf_plugin_datastore_data_heap.conf
@@ -0,0 +1,7 @@
1@INLINE@ test_defaults.conf
2[PATHS]
3GNUNET_TEST_HOME = $GNUNET_TMP/perf-gnunet-datastore-heap/
4
5
6[datastore]
7DATABASE = heap
diff --git a/src/plugin/datastore/perf_plugin_datastore_data_postgres.conf b/src/plugin/datastore/perf_plugin_datastore_data_postgres.conf
new file mode 100644
index 000000000..7683887a8
--- /dev/null
+++ b/src/plugin/datastore/perf_plugin_datastore_data_postgres.conf
@@ -0,0 +1,10 @@
1@INLINE@ test_defaults.conf
2[PATHS]
3GNUNET_TEST_HOME = $GNUNET_TMP/perf-gnunet-datastore-postgres/
4
5[datastore]
6DATABASE = postgres
7
8[datastore-postgres]
9CONFIG = dbname=gnunetcheck
10
diff --git a/src/plugin/datastore/perf_plugin_datastore_data_sqlite.conf b/src/plugin/datastore/perf_plugin_datastore_data_sqlite.conf
new file mode 100644
index 000000000..888e020a6
--- /dev/null
+++ b/src/plugin/datastore/perf_plugin_datastore_data_sqlite.conf
@@ -0,0 +1,4 @@
1@INLINE@ test_defaults.conf
2[PATHS]
3GNUNET_TEST_HOME = $GNUNET_TMP/perf-gnunet-datastore-sqlite/
4
diff --git a/src/plugin/datastore/plugin_datastore_heap.c b/src/plugin/datastore/plugin_datastore_heap.c
new file mode 100644
index 000000000..a827a2763
--- /dev/null
+++ b/src/plugin/datastore/plugin_datastore_heap.c
@@ -0,0 +1,944 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2012 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file datastore/plugin_datastore_heap.c
23 * @brief heap-based datastore backend; usually we want the datastore
24 * to be persistent, and storing data in the heap is obviously
25 * NOT going to be persistent; still, this plugin is useful for
26 * testing/benchmarking --- but never for production!
27 * @author Christian Grothoff
28 */
29
30#include "platform.h"
31#include "gnunet_datastore_plugin.h"
32
33
34/**
35 * A value that we are storing.
36 */
37struct Value
38{
39 /**
40 * Key for the value.
41 */
42 struct GNUNET_HashCode key;
43
44 /**
45 * Pointer to the value's data (allocated at the end of this struct).
46 */
47 const void *data;
48
49 /**
50 * Entry for this value in the 'expire' heap.
51 */
52 struct GNUNET_CONTAINER_HeapNode *expire_heap;
53
54 /**
55 * Entry for this value in the 'replication' heap.
56 */
57 struct GNUNET_CONTAINER_HeapNode *replication_heap;
58
59 /**
60 * Expiration time for this value.
61 */
62 struct GNUNET_TIME_Absolute expiration;
63
64 /**
65 * Offset of this value in the array of the 'struct ZeroAnonByType';
66 * only used if anonymity is zero.
67 */
68 unsigned int zero_anon_offset;
69
70 /**
71 * Number of bytes in 'data'.
72 */
73 uint32_t size;
74
75 /**
76 * Priority of the value.
77 */
78 uint32_t priority;
79
80 /**
81 * Anonymity level for the value.
82 */
83 uint32_t anonymity;
84
85 /**
86 * Replication level for the value.
87 */
88 uint32_t replication;
89
90 /**
91 * Type of 'data'.
92 */
93 enum GNUNET_BLOCK_Type type;
94};
95
96
97/**
98 * We organize 0-anonymity values in arrays "by type".
99 */
100struct ZeroAnonByType
101{
102 /**
103 * We keep these in a DLL.
104 */
105 struct ZeroAnonByType *next;
106
107 /**
108 * We keep these in a DLL.
109 */
110 struct ZeroAnonByType *prev;
111
112 /**
113 * Array of 0-anonymity items of the given type.
114 */
115 struct Value **array;
116
117 /**
118 * Allocated size of the array.
119 */
120 unsigned int array_size;
121
122 /**
123 * First unused offset in 'array'.
124 */
125 unsigned int array_pos;
126
127 /**
128 * Type of all of the values in 'array'.
129 */
130 enum GNUNET_BLOCK_Type type;
131};
132
133
134/**
135 * Context for all functions in this plugin.
136 */
137struct Plugin
138{
139 /**
140 * Our execution environment.
141 */
142 struct GNUNET_DATASTORE_PluginEnvironment *env;
143
144 /**
145 * Mapping from keys to 'struct Value's.
146 */
147 struct GNUNET_CONTAINER_MultiHashMap *keyvalue;
148
149 /**
150 * Heap organized by minimum expiration time.
151 */
152 struct GNUNET_CONTAINER_Heap *by_expiration;
153
154 /**
155 * Heap organized by maximum replication value.
156 */
157 struct GNUNET_CONTAINER_Heap *by_replication;
158
159 /**
160 * Head of list of arrays containing zero-anonymity values by type.
161 */
162 struct ZeroAnonByType *zero_head;
163
164 /**
165 * Tail of list of arrays containing zero-anonymity values by type.
166 */
167 struct ZeroAnonByType *zero_tail;
168
169 /**
170 * Size of all values we're storing.
171 */
172 unsigned long long size;
173};
174
175
176/**
177 * Get an estimate of how much space the database is
178 * currently using.
179 *
180 * @param cls our "struct Plugin*"
181 * @return number of bytes used on disk
182 */
183static void
184heap_plugin_estimate_size (void *cls, unsigned long long *estimate)
185{
186 struct Plugin *plugin = cls;
187
188 if (NULL != estimate)
189 *estimate = plugin->size;
190}
191
192
193/**
194 * Closure for iterator for updating.
195 */
196struct UpdateContext
197{
198 /**
199 * Number of bytes in 'data'.
200 */
201 uint32_t size;
202
203 /**
204 * Pointer to the data.
205 */
206 const void *data;
207
208 /**
209 * Priority of the value.
210 */
211 uint32_t priority;
212
213 /**
214 * Replication level for the value.
215 */
216 uint32_t replication;
217
218 /**
219 * Expiration time for this value.
220 */
221 struct GNUNET_TIME_Absolute expiration;
222
223 /**
224 * True if the value was found and updated.
225 */
226 bool updated;
227};
228
229
230/**
231 * Update the matching value.
232 *
233 * @param cls the 'struct UpdateContext'
234 * @param key unused
235 * @param val the 'struct Value'
236 * @return GNUNET_YES (continue iteration), GNUNET_NO if value was found
237 */
238static int
239update_iterator (void *cls,
240 const struct GNUNET_HashCode *key,
241 void *val)
242{
243 struct UpdateContext *uc = cls;
244 struct Value *value = val;
245
246 if (value->size != uc->size)
247 return GNUNET_YES;
248 if (0 != memcmp (value->data, uc->data, uc->size))
249 return GNUNET_YES;
250 uc->expiration = GNUNET_TIME_absolute_max (value->expiration,
251 uc->expiration);
252 if (value->expiration.abs_value_us != uc->expiration.abs_value_us)
253 {
254 value->expiration = uc->expiration;
255 GNUNET_CONTAINER_heap_update_cost (value->expire_heap,
256 value->expiration.abs_value_us);
257 }
258 /* Saturating adds, don't overflow */
259 if (value->priority > UINT32_MAX - uc->priority)
260 value->priority = UINT32_MAX;
261 else
262 value->priority += uc->priority;
263 if (value->replication > UINT32_MAX - uc->replication)
264 value->replication = UINT32_MAX;
265 else
266 value->replication += uc->replication;
267 uc->updated = true;
268 return GNUNET_NO;
269}
270
271
272/**
273 * Store an item in the datastore.
274 *
275 * @param cls closure
276 * @param key key for the item
277 * @param absent true if the key was not found in the bloom filter
278 * @param size number of bytes in data
279 * @param data content stored
280 * @param type type of the content
281 * @param priority priority of the content
282 * @param anonymity anonymity-level for the content
283 * @param replication replication-level for the content
284 * @param expiration expiration time for the content
285 * @param cont continuation called with success or failure status
286 * @param cont_cls continuation closure
287 */
288static void
289heap_plugin_put (void *cls,
290 const struct GNUNET_HashCode *key,
291 bool absent,
292 uint32_t size,
293 const void *data,
294 enum GNUNET_BLOCK_Type type,
295 uint32_t priority,
296 uint32_t anonymity,
297 uint32_t replication,
298 struct GNUNET_TIME_Absolute expiration,
299 PluginPutCont cont,
300 void *cont_cls)
301{
302 struct Plugin *plugin = cls;
303 struct Value *value;
304
305 if (! absent)
306 {
307 struct UpdateContext uc;
308
309 uc.size = size;
310 uc.data = data;
311 uc.priority = priority;
312 uc.replication = replication;
313 uc.expiration = expiration;
314 uc.updated = false;
315 GNUNET_CONTAINER_multihashmap_get_multiple (plugin->keyvalue,
316 key,
317 &update_iterator,
318 &uc);
319 if (uc.updated)
320 {
321 cont (cont_cls, key, size, GNUNET_NO, NULL);
322 return;
323 }
324 }
325 value = GNUNET_malloc (sizeof(struct Value) + size);
326 value->key = *key;
327 value->data = &value[1];
328 value->expire_heap = GNUNET_CONTAINER_heap_insert (plugin->by_expiration,
329 value,
330 expiration.abs_value_us);
331 value->replication_heap = GNUNET_CONTAINER_heap_insert (
332 plugin->by_replication,
333 value,
334 replication);
335 value->expiration = expiration;
336 if (0 == anonymity)
337 {
338 struct ZeroAnonByType *zabt;
339
340 for (zabt = plugin->zero_head; NULL != zabt; zabt = zabt->next)
341 if (zabt->type == type)
342 break;
343 if (NULL == zabt)
344 {
345 zabt = GNUNET_new (struct ZeroAnonByType);
346 zabt->type = type;
347 GNUNET_CONTAINER_DLL_insert (plugin->zero_head,
348 plugin->zero_tail,
349 zabt);
350 }
351 if (zabt->array_size == zabt->array_pos)
352 {
353 GNUNET_array_grow (zabt->array,
354 zabt->array_size,
355 zabt->array_size * 2 + 4);
356 }
357 value->zero_anon_offset = zabt->array_pos;
358 zabt->array[zabt->array_pos++] = value;
359 }
360 value->size = size;
361 value->priority = priority;
362 value->anonymity = anonymity;
363 value->replication = replication;
364 value->type = type;
365 GNUNET_memcpy (&value[1], data, size);
366 GNUNET_CONTAINER_multihashmap_put (plugin->keyvalue,
367 &value->key,
368 value,
369 GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
370 plugin->size += size;
371 cont (cont_cls, key, size, GNUNET_OK, NULL);
372}
373
374
375/**
376 * Delete the given value, removing it from the plugin's data
377 * structures.
378 *
379 * @param plugin the plugin
380 * @param value value to delete
381 */
382static void
383delete_value (struct Plugin *plugin,
384 struct Value *value)
385{
386 GNUNET_assert (GNUNET_YES ==
387 GNUNET_CONTAINER_multihashmap_remove (plugin->keyvalue,
388 &value->key,
389 value));
390 GNUNET_assert (value == GNUNET_CONTAINER_heap_remove_node (
391 value->expire_heap));
392 GNUNET_assert (value == GNUNET_CONTAINER_heap_remove_node (
393 value->replication_heap));
394 if (0 == value->anonymity)
395 {
396 struct ZeroAnonByType *zabt;
397
398 for (zabt = plugin->zero_head; NULL != zabt; zabt = zabt->next)
399 if (zabt->type == value->type)
400 break;
401 GNUNET_assert (NULL != zabt);
402 zabt->array[value->zero_anon_offset] = zabt->array[--zabt->array_pos];
403 zabt->array[value->zero_anon_offset]->zero_anon_offset =
404 value->zero_anon_offset;
405 if (0 == zabt->array_pos)
406 {
407 GNUNET_array_grow (zabt->array,
408 zabt->array_size,
409 0);
410 GNUNET_CONTAINER_DLL_remove (plugin->zero_head,
411 plugin->zero_tail,
412 zabt);
413 GNUNET_free (zabt);
414 }
415 }
416 plugin->size -= value->size;
417 GNUNET_free (value);
418}
419
420
421/**
422 * Closure for iterator called during 'get_key'.
423 */
424struct GetContext
425{
426 /**
427 * Lowest uid to consider.
428 */
429 uint64_t next_uid;
430
431 /**
432 * Value with lowest uid >= next_uid found so far.
433 */
434 struct Value *value;
435
436 /**
437 * Requested type.
438 */
439 enum GNUNET_BLOCK_Type type;
440
441 /**
442 * If true, return a random value
443 */
444 bool random;
445};
446
447
448/**
449 * Obtain the matching value with the lowest uid >= next_uid.
450 *
451 * @param cls the 'struct GetContext'
452 * @param key unused
453 * @param val the 'struct Value'
454 * @return GNUNET_YES (continue iteration), GNUNET_NO if result was found
455 */
456static int
457get_iterator (void *cls,
458 const struct GNUNET_HashCode *key,
459 void *val)
460{
461 struct GetContext *gc = cls;
462 struct Value *value = val;
463
464 if ((gc->type != GNUNET_BLOCK_TYPE_ANY) &&
465 (gc->type != value->type))
466 return GNUNET_OK;
467 if (gc->random)
468 {
469 gc->value = value;
470 return GNUNET_NO;
471 }
472 if ((uint64_t) (intptr_t) value < gc->next_uid)
473 return GNUNET_OK;
474 if ((NULL != gc->value) &&
475 (value > gc->value))
476 return GNUNET_OK;
477 gc->value = value;
478 return GNUNET_OK;
479}
480
481
482/**
483 * Get one of the results for a particular key in the datastore.
484 *
485 * @param cls closure
486 * @param next_uid return the result with lowest uid >= next_uid
487 * @param random if true, return a random result instead of using next_uid
488 * @param key maybe NULL (to match all entries)
489 * @param type entries of which type are relevant?
490 * Use 0 for any type.
491 * @param proc function to call on the matching value;
492 * will be called with NULL if nothing matches
493 * @param proc_cls closure for @a proc
494 */
495static void
496heap_plugin_get_key (void *cls,
497 uint64_t next_uid,
498 bool random,
499 const struct GNUNET_HashCode *key,
500 enum GNUNET_BLOCK_Type type,
501 PluginDatumProcessor proc,
502 void *proc_cls)
503{
504 struct Plugin *plugin = cls;
505 struct GetContext gc;
506
507 gc.value = NULL;
508 gc.next_uid = next_uid;
509 gc.random = random;
510 gc.type = type;
511 if (NULL == key)
512 {
513 GNUNET_CONTAINER_multihashmap_iterate (plugin->keyvalue,
514 &get_iterator,
515 &gc);
516 }
517 else
518 {
519 GNUNET_CONTAINER_multihashmap_get_multiple (plugin->keyvalue,
520 key,
521 &get_iterator,
522 &gc);
523 }
524 if (NULL == gc.value)
525 {
526 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
527 return;
528 }
529 GNUNET_assert (GNUNET_OK ==
530 proc (proc_cls,
531 &gc.value->key,
532 gc.value->size,
533 &gc.value[1],
534 gc.value->type,
535 gc.value->priority,
536 gc.value->anonymity,
537 gc.value->replication,
538 gc.value->expiration,
539 (uint64_t) (intptr_t) gc.value));
540}
541
542
543/**
544 * Get a random item for replication. Returns a single, not expired,
545 * random item from those with the highest replication counters. The
546 * item's replication counter is decremented by one IF it was positive
547 * before. Call 'proc' with all values ZERO or NULL if the datastore
548 * is empty.
549 *
550 * @param cls closure
551 * @param proc function to call the value (once only).
552 * @param proc_cls closure for proc
553 */
554static void
555heap_plugin_get_replication (void *cls,
556 PluginDatumProcessor proc,
557 void *proc_cls)
558{
559 struct Plugin *plugin = cls;
560 struct Value *value;
561
562 value = GNUNET_CONTAINER_heap_remove_root (plugin->by_replication);
563 if (NULL == value)
564 {
565 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
566 return;
567 }
568 if (value->replication > 0)
569 {
570 value->replication--;
571 value->replication_heap = GNUNET_CONTAINER_heap_insert (
572 plugin->by_replication,
573 value,
574 value->replication);
575 }
576 else
577 {
578 /* need a better way to pick a random item, replication level is always 0 */
579 value->replication_heap = GNUNET_CONTAINER_heap_insert (
580 plugin->by_replication,
581 value,
582 value->replication);
583 value = GNUNET_CONTAINER_heap_walk_get_next (plugin->by_replication);
584 }
585 GNUNET_assert (GNUNET_OK ==
586 proc (proc_cls,
587 &value->key,
588 value->size,
589 &value[1],
590 value->type,
591 value->priority,
592 value->anonymity,
593 value->replication,
594 value->expiration,
595 (uint64_t) (intptr_t) value));
596}
597
598
599/**
600 * Get a random item for expiration. Call 'proc' with all values ZERO
601 * or NULL if the datastore is empty.
602 *
603 * @param cls closure
604 * @param proc function to call the value (once only).
605 * @param proc_cls closure for proc
606 */
607static void
608heap_plugin_get_expiration (void *cls, PluginDatumProcessor proc,
609 void *proc_cls)
610{
611 struct Plugin *plugin = cls;
612 struct Value *value;
613
614 value = GNUNET_CONTAINER_heap_peek (plugin->by_expiration);
615 if (NULL == value)
616 {
617 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
618 return;
619 }
620 if (GNUNET_NO ==
621 proc (proc_cls,
622 &value->key,
623 value->size,
624 &value[1],
625 value->type,
626 value->priority,
627 value->anonymity,
628 value->replication,
629 value->expiration,
630 (uint64_t) (intptr_t) value))
631 delete_value (plugin, value);
632}
633
634
635/**
636 * Call the given processor on an item with zero anonymity.
637 *
638 * @param cls our "struct Plugin*"
639 * @param next_uid return the result with lowest uid >= next_uid
640 * @param type entries of which type should be considered?
641 * Must not be zero (ANY).
642 * @param proc function to call on each matching value;
643 * will be called with NULL if no value matches
644 * @param proc_cls closure for proc
645 */
646static void
647heap_plugin_get_zero_anonymity (void *cls, uint64_t next_uid,
648 enum GNUNET_BLOCK_Type type,
649 PluginDatumProcessor proc, void *proc_cls)
650{
651 struct Plugin *plugin = cls;
652 struct ZeroAnonByType *zabt;
653 struct Value *value = NULL;
654
655 for (zabt = plugin->zero_head; NULL != zabt; zabt = zabt->next)
656 {
657 if ((type != GNUNET_BLOCK_TYPE_ANY) &&
658 (type != zabt->type))
659 continue;
660 for (int i = 0; i < zabt->array_pos; ++i)
661 {
662 if ((uint64_t) (intptr_t) zabt->array[i] < next_uid)
663 continue;
664 if ((NULL != value) &&
665 (zabt->array[i] > value))
666 continue;
667 value = zabt->array[i];
668 }
669 }
670 if (NULL == value)
671 {
672 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
673 return;
674 }
675 GNUNET_assert (GNUNET_OK ==
676 proc (proc_cls,
677 &value->key,
678 value->size,
679 &value[1],
680 value->type,
681 value->priority,
682 value->anonymity,
683 value->replication,
684 value->expiration,
685 (uint64_t) (intptr_t) value));
686}
687
688
689/**
690 * Drop database.
691 */
692static void
693heap_plugin_drop (void *cls)
694{
695 /* nothing needs to be done */
696}
697
698
699/**
700 * Closure for the 'return_value' function.
701 */
702struct GetAllContext
703{
704 /**
705 * Function to call.
706 */
707 PluginKeyProcessor proc;
708
709 /**
710 * Closure for 'proc'.
711 */
712 void *proc_cls;
713};
714
715
716/**
717 * Callback invoked to call callback on each value.
718 *
719 * @param cls the plugin
720 * @param key unused
721 * @param val the value
722 * @return GNUNET_OK (continue to iterate)
723 */
724static int
725return_value (void *cls,
726 const struct GNUNET_HashCode *key,
727 void *val)
728{
729 struct GetAllContext *gac = cls;
730
731 gac->proc (gac->proc_cls,
732 key,
733 1);
734 return GNUNET_OK;
735}
736
737
738/**
739 * Get all of the keys in the datastore.
740 *
741 * @param cls closure
742 * @param proc function to call on each key
743 * @param proc_cls closure for proc
744 */
745static void
746heap_get_keys (void *cls,
747 PluginKeyProcessor proc,
748 void *proc_cls)
749{
750 struct Plugin *plugin = cls;
751 struct GetAllContext gac;
752
753 gac.proc = proc;
754 gac.proc_cls = proc_cls;
755 GNUNET_CONTAINER_multihashmap_iterate (plugin->keyvalue,
756 &return_value,
757 &gac);
758 proc (proc_cls, NULL, 0);
759}
760
761
762/**
763 * Closure for iterator called during 'remove_key'.
764 */
765struct RemoveContext
766{
767 /**
768 * Value found.
769 */
770 struct Value *value;
771
772 /**
773 * Size of data.
774 */
775 uint32_t size;
776
777 /**
778 * Data to remove.
779 */
780 const void *data;
781};
782
783
784/**
785 * Obtain the matching value with the lowest uid >= next_uid.
786 *
787 * @param cls the 'struct GetContext'
788 * @param key unused
789 * @param val the 'struct Value'
790 * @return GNUNET_YES (continue iteration), GNUNET_NO if result was found
791 */
792static int
793remove_iterator (void *cls,
794 const struct GNUNET_HashCode *key,
795 void *val)
796{
797 struct RemoveContext *rc = cls;
798 struct Value *value = val;
799
800 if (value->size != rc->size)
801 return GNUNET_YES;
802 if (0 != memcmp (value->data, rc->data, rc->size))
803 return GNUNET_YES;
804 rc->value = value;
805 return GNUNET_NO;
806}
807
808
809/**
810 * Remove a particular key in the datastore.
811 *
812 * @param cls closure
813 * @param key key for the content
814 * @param size number of bytes in data
815 * @param data content stored
816 * @param cont continuation called with success or failure status
817 * @param cont_cls continuation closure for @a cont
818 */
819static void
820heap_plugin_remove_key (void *cls,
821 const struct GNUNET_HashCode *key,
822 uint32_t size,
823 const void *data,
824 PluginRemoveCont cont,
825 void *cont_cls)
826{
827 struct Plugin *plugin = cls;
828 struct RemoveContext rc;
829
830 rc.value = NULL;
831 rc.size = size;
832 rc.data = data;
833 GNUNET_CONTAINER_multihashmap_get_multiple (plugin->keyvalue,
834 key,
835 &remove_iterator,
836 &rc);
837 if (NULL == rc.value)
838 {
839 cont (cont_cls,
840 key,
841 size,
842 GNUNET_NO,
843 NULL);
844 return;
845 }
846 delete_value (plugin,
847 rc.value);
848 cont (cont_cls,
849 key,
850 size,
851 GNUNET_OK,
852 NULL);
853}
854
855
856/**
857 * Entry point for the plugin.
858 *
859 * @param cls the "struct GNUNET_DATASTORE_PluginEnvironment*"
860 * @return our "struct Plugin*"
861 */
862void *
863libgnunet_plugin_datastore_heap_init (void *cls)
864{
865 struct GNUNET_DATASTORE_PluginEnvironment *env = cls;
866 struct GNUNET_DATASTORE_PluginFunctions *api;
867 struct Plugin *plugin;
868 unsigned long long esize;
869
870 if (GNUNET_OK !=
871 GNUNET_CONFIGURATION_get_value_number (env->cfg,
872 "datastore-heap",
873 "HASHMAPSIZE",
874 &esize))
875 esize = 128 * 1024;
876 plugin = GNUNET_new (struct Plugin);
877 plugin->env = env;
878 plugin->keyvalue = GNUNET_CONTAINER_multihashmap_create (esize, GNUNET_YES);
879 plugin->by_expiration = GNUNET_CONTAINER_heap_create (
880 GNUNET_CONTAINER_HEAP_ORDER_MIN);
881 plugin->by_replication = GNUNET_CONTAINER_heap_create (
882 GNUNET_CONTAINER_HEAP_ORDER_MAX);
883 api = GNUNET_new (struct GNUNET_DATASTORE_PluginFunctions);
884 api->cls = plugin;
885 api->estimate_size = &heap_plugin_estimate_size;
886 api->put = &heap_plugin_put;
887 api->get_key = &heap_plugin_get_key;
888 api->get_replication = &heap_plugin_get_replication;
889 api->get_expiration = &heap_plugin_get_expiration;
890 api->get_zero_anonymity = &heap_plugin_get_zero_anonymity;
891 api->drop = &heap_plugin_drop;
892 api->get_keys = &heap_get_keys;
893 api->remove_key = &heap_plugin_remove_key;
894 GNUNET_log_from (GNUNET_ERROR_TYPE_INFO, "heap",
895 _ ("Heap database running\n"));
896 return api;
897}
898
899
900/**
901 * Callback invoked to free all value.
902 *
903 * @param cls the plugin
904 * @param key unused
905 * @param val the value
906 * @return GNUNET_OK (continue to iterate)
907 */
908static int
909free_value (void *cls,
910 const struct GNUNET_HashCode *key,
911 void *val)
912{
913 struct Plugin *plugin = cls;
914 struct Value *value = val;
915
916 delete_value (plugin, value);
917 return GNUNET_OK;
918}
919
920
921/**
922 * Exit point from the plugin.
923 * @param cls our "struct Plugin*"
924 * @return always NULL
925 */
926void *
927libgnunet_plugin_datastore_heap_done (void *cls)
928{
929 struct GNUNET_DATASTORE_PluginFunctions *api = cls;
930 struct Plugin *plugin = api->cls;
931
932 GNUNET_CONTAINER_multihashmap_iterate (plugin->keyvalue,
933 &free_value,
934 plugin);
935 GNUNET_CONTAINER_multihashmap_destroy (plugin->keyvalue);
936 GNUNET_CONTAINER_heap_destroy (plugin->by_expiration);
937 GNUNET_CONTAINER_heap_destroy (plugin->by_replication);
938 GNUNET_free (plugin);
939 GNUNET_free (api);
940 return NULL;
941}
942
943
944/* end of plugin_datastore_heap.c */
diff --git a/src/plugin/datastore/plugin_datastore_postgres.c b/src/plugin/datastore/plugin_datastore_postgres.c
new file mode 100644
index 000000000..5fcacc17b
--- /dev/null
+++ b/src/plugin/datastore/plugin_datastore_postgres.c
@@ -0,0 +1,930 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2009-2017, 2022 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file datastore/plugin_datastore_postgres.c
23 * @brief postgres-based datastore backend
24 * @author Christian Grothoff
25 */
26#include "platform.h"
27#include "gnunet_datastore_plugin.h"
28#include "gnunet_pq_lib.h"
29
30
31/**
32 * After how many ms "busy" should a DB operation fail for good?
33 * A low value makes sure that we are more responsive to requests
34 * (especially PUTs). A high value guarantees a higher success
35 * rate (SELECTs in iterate can take several seconds despite LIMIT=1).
36 *
37 * The default value of 1s should ensure that users do not experience
38 * huge latencies while at the same time allowing operations to succeed
39 * with reasonable probability.
40 */
41#define BUSY_TIMEOUT GNUNET_TIME_UNIT_SECONDS
42
43
44/**
45 * Context for all functions in this plugin.
46 */
47struct Plugin
48{
49 /**
50 * Our execution environment.
51 */
52 struct GNUNET_DATASTORE_PluginEnvironment *env;
53
54 /**
55 * Native Postgres database handle.
56 */
57 struct GNUNET_PQ_Context *dbh;
58};
59
60
61/**
62 * @brief Get a database handle
63 *
64 * @param plugin global context
65 * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
66 */
67static enum GNUNET_GenericReturnValue
68init_connection (struct Plugin *plugin)
69{
70#define RESULT_COLUMNS "repl, type, prio, anonLevel, expire, hash, value, oid"
71 struct GNUNET_PQ_PreparedStatement ps[] = {
72 GNUNET_PQ_make_prepare ("get",
73 "SELECT " RESULT_COLUMNS
74 " FROM datastore.gn090"
75 " WHERE oid >= $1::bigint AND"
76 " (rvalue >= $2 OR 0 = $3::smallint) AND"
77 " (hash = $4 OR 0 = $5::smallint) AND"
78 " (type = $6 OR 0 = $7::smallint)"
79 " ORDER BY oid ASC LIMIT 1"),
80 GNUNET_PQ_make_prepare ("put",
81 "INSERT INTO datastore.gn090"
82 " (repl, type, prio, anonLevel, expire, rvalue, hash, vhash, value) "
83 "VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)"),
84 GNUNET_PQ_make_prepare ("update",
85 "UPDATE datastore.gn090"
86 " SET prio = prio + $1,"
87 " repl = repl + $2,"
88 " expire = GREATEST(expire, $3)"
89 " WHERE hash = $4 AND vhash = $5"),
90 GNUNET_PQ_make_prepare ("decrepl",
91 "UPDATE datastore.gn090"
92 " SET repl = GREATEST (repl - 1, 0)"
93 " WHERE oid = $1"),
94 GNUNET_PQ_make_prepare ("select_non_anonymous",
95 "SELECT " RESULT_COLUMNS
96 " FROM datastore.gn090"
97 " WHERE anonLevel = 0 AND type = $1 AND oid >= $2::bigint"
98 " ORDER BY oid ASC LIMIT 1"),
99 GNUNET_PQ_make_prepare ("select_expiration_order",
100 "(SELECT " RESULT_COLUMNS
101 " FROM datastore.gn090"
102 " WHERE expire < $1 ORDER BY prio ASC LIMIT 1) "
103 "UNION "
104 "(SELECT " RESULT_COLUMNS
105 " FROM datastore.gn090"
106 " ORDER BY prio ASC LIMIT 1)"
107 " ORDER BY expire ASC LIMIT 1"),
108 GNUNET_PQ_make_prepare ("select_replication_order",
109 "SELECT " RESULT_COLUMNS
110 " FROM datastore.gn090"
111 " ORDER BY repl DESC,RANDOM() LIMIT 1"),
112 GNUNET_PQ_make_prepare ("delrow",
113 "DELETE FROM datastore.gn090"
114 " WHERE oid=$1"),
115 GNUNET_PQ_make_prepare ("remove",
116 "DELETE FROM datastore.gn090"
117 " WHERE hash = $1 AND"
118 " value = $2"),
119 GNUNET_PQ_make_prepare ("get_keys",
120 "SELECT hash"
121 " FROM datastore.gn090"),
122 GNUNET_PQ_make_prepare ("estimate_size",
123 "SELECT CASE WHEN NOT EXISTS"
124 " (SELECT 1 FROM datastore.gn090)"
125 " THEN 0"
126 " ELSE (SELECT SUM(LENGTH(value))+256*COUNT(*)"
127 " FROM datastore.gn090)"
128 "END AS total"),
129 GNUNET_PQ_PREPARED_STATEMENT_END
130 };
131#undef RESULT_COLUMNS
132
133 plugin->dbh = GNUNET_PQ_connect_with_cfg (plugin->env->cfg,
134 "datastore-postgres",
135 "datastore-",
136 NULL,
137 ps);
138 if (NULL == plugin->dbh)
139 return GNUNET_SYSERR;
140 return GNUNET_OK;
141}
142
143
144/**
145 * Get an estimate of how much space the database is
146 * currently using.
147 *
148 * @param cls our `struct Plugin *`
149 * @return number of bytes used on disk
150 */
151static void
152postgres_plugin_estimate_size (void *cls,
153 unsigned long long *estimate)
154{
155 struct Plugin *plugin = cls;
156 uint64_t total;
157 struct GNUNET_PQ_QueryParam params[] = {
158 GNUNET_PQ_query_param_end
159 };
160 struct GNUNET_PQ_ResultSpec rs[] = {
161 GNUNET_PQ_result_spec_uint64 ("total",
162 &total),
163 GNUNET_PQ_result_spec_end
164 };
165 enum GNUNET_DB_QueryStatus ret;
166
167 if (NULL == estimate)
168 return;
169 ret = GNUNET_PQ_eval_prepared_singleton_select (plugin->dbh,
170 "estimate_size",
171 params,
172 rs);
173 if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT != ret)
174 {
175 *estimate = 0LL;
176 return;
177 }
178 *estimate = total;
179}
180
181
182/**
183 * Store an item in the datastore.
184 *
185 * @param cls closure with the `struct Plugin`
186 * @param key key for the item
187 * @param absent true if the key was not found in the bloom filter
188 * @param size number of bytes in data
189 * @param data content stored
190 * @param type type of the content
191 * @param priority priority of the content
192 * @param anonymity anonymity-level for the content
193 * @param replication replication-level for the content
194 * @param expiration expiration time for the content
195 * @param cont continuation called with success or failure status
196 * @param cont_cls continuation closure
197 */
198static void
199postgres_plugin_put (void *cls,
200 const struct GNUNET_HashCode *key,
201 bool absent,
202 uint32_t size,
203 const void *data,
204 enum GNUNET_BLOCK_Type type,
205 uint32_t priority,
206 uint32_t anonymity,
207 uint32_t replication,
208 struct GNUNET_TIME_Absolute expiration,
209 PluginPutCont cont,
210 void *cont_cls)
211{
212 struct Plugin *plugin = cls;
213 struct GNUNET_HashCode vhash;
214 enum GNUNET_DB_QueryStatus ret;
215
216 GNUNET_CRYPTO_hash (data,
217 size,
218 &vhash);
219 if (! absent)
220 {
221 struct GNUNET_PQ_QueryParam params[] = {
222 GNUNET_PQ_query_param_uint32 (&priority),
223 GNUNET_PQ_query_param_uint32 (&replication),
224 GNUNET_PQ_query_param_absolute_time (&expiration),
225 GNUNET_PQ_query_param_auto_from_type (key),
226 GNUNET_PQ_query_param_auto_from_type (&vhash),
227 GNUNET_PQ_query_param_end
228 };
229 ret = GNUNET_PQ_eval_prepared_non_select (plugin->dbh,
230 "update",
231 params);
232 if (0 > ret)
233 {
234 cont (cont_cls,
235 key,
236 size,
237 GNUNET_SYSERR,
238 _ ("Postgresql exec failure"));
239 return;
240 }
241 bool affected = (0 != ret);
242 if (affected)
243 {
244 cont (cont_cls,
245 key,
246 size,
247 GNUNET_NO,
248 NULL);
249 return;
250 }
251 }
252
253 {
254 uint32_t utype = (uint32_t) type;
255 uint64_t rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
256 UINT64_MAX);
257 struct GNUNET_PQ_QueryParam params[] = {
258 GNUNET_PQ_query_param_uint32 (&replication),
259 GNUNET_PQ_query_param_uint32 (&utype),
260 GNUNET_PQ_query_param_uint32 (&priority),
261 GNUNET_PQ_query_param_uint32 (&anonymity),
262 GNUNET_PQ_query_param_absolute_time (&expiration),
263 GNUNET_PQ_query_param_uint64 (&rvalue),
264 GNUNET_PQ_query_param_auto_from_type (key),
265 GNUNET_PQ_query_param_auto_from_type (&vhash),
266 GNUNET_PQ_query_param_fixed_size (data, size),
267 GNUNET_PQ_query_param_end
268 };
269
270 ret = GNUNET_PQ_eval_prepared_non_select (plugin->dbh,
271 "put",
272 params);
273 if (0 > ret)
274 {
275 cont (cont_cls,
276 key,
277 size,
278 GNUNET_SYSERR,
279 "Postgresql exec failure");
280 return;
281 }
282 }
283 plugin->env->duc (plugin->env->cls,
284 size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
285 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
286 "datastore-postgres",
287 "Stored %u bytes in database\n",
288 (unsigned int) size);
289 cont (cont_cls,
290 key,
291 size,
292 GNUNET_OK,
293 NULL);
294}
295
296
297/**
298 * Closure for #process_result.
299 */
300struct ProcessResultContext
301{
302 /**
303 * The plugin handle.
304 */
305 struct Plugin *plugin;
306
307 /**
308 * Function to call on each result.
309 */
310 PluginDatumProcessor proc;
311
312 /**
313 * Closure for @e proc.
314 */
315 void *proc_cls;
316};
317
318
319/**
320 * Function invoked to process the result and call the processor of @a
321 * cls.
322 *
323 * @param cls our `struct ProcessResultContext`
324 * @param res result from exec
325 * @param num_results number of results in @a res
326 */
327static void
328process_result (void *cls,
329 PGresult *res,
330 unsigned int num_results)
331{
332 struct ProcessResultContext *prc = cls;
333 struct Plugin *plugin = prc->plugin;
334
335 if (0 == num_results)
336 {
337 /* no result */
338 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
339 "datastore-postgres",
340 "Ending iteration (no more results)\n");
341 prc->proc (prc->proc_cls, NULL, 0, NULL, 0, 0, 0, 0,
342 GNUNET_TIME_UNIT_ZERO_ABS, 0);
343 return;
344 }
345 if (1 != num_results)
346 {
347 GNUNET_break (0);
348 prc->proc (prc->proc_cls, NULL, 0, NULL, 0, 0, 0, 0,
349 GNUNET_TIME_UNIT_ZERO_ABS, 0);
350 return;
351 }
352 /* Technically we don't need the loop here, but nicer in case
353 we ever relax the condition above. */
354 for (unsigned int i = 0; i < num_results; i++)
355 {
356 int iret;
357 uint64_t rowid;
358 uint32_t utype;
359 uint32_t anonymity;
360 uint32_t replication;
361 uint32_t priority;
362 size_t size;
363 void *data;
364 struct GNUNET_TIME_Absolute expiration_time;
365 struct GNUNET_HashCode key;
366 struct GNUNET_PQ_ResultSpec rs[] = {
367 GNUNET_PQ_result_spec_uint32 ("repl", &replication),
368 GNUNET_PQ_result_spec_uint32 ("type", &utype),
369 GNUNET_PQ_result_spec_uint32 ("prio", &priority),
370 GNUNET_PQ_result_spec_uint32 ("anonLevel", &anonymity),
371 GNUNET_PQ_result_spec_absolute_time ("expire", &expiration_time),
372 GNUNET_PQ_result_spec_auto_from_type ("hash", &key),
373 GNUNET_PQ_result_spec_variable_size ("value", &data, &size),
374 GNUNET_PQ_result_spec_uint64 ("oid", &rowid),
375 GNUNET_PQ_result_spec_end
376 };
377
378 if (GNUNET_OK !=
379 GNUNET_PQ_extract_result (res,
380 rs,
381 i))
382 {
383 GNUNET_break (0);
384 prc->proc (prc->proc_cls, NULL, 0, NULL, 0, 0, 0, 0,
385 GNUNET_TIME_UNIT_ZERO_ABS, 0);
386 return;
387 }
388
389 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
390 "datastore-postgres",
391 "Found result of size %u bytes and type %u in database\n",
392 (unsigned int) size,
393 (unsigned int) utype);
394 iret = prc->proc (prc->proc_cls,
395 &key,
396 size,
397 data,
398 (enum GNUNET_BLOCK_Type) utype,
399 priority,
400 anonymity,
401 replication,
402 expiration_time,
403 rowid);
404 if (iret == GNUNET_NO)
405 {
406 struct GNUNET_PQ_QueryParam param[] = {
407 GNUNET_PQ_query_param_uint64 (&rowid),
408 GNUNET_PQ_query_param_end
409 };
410
411 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
412 "Processor asked for item %u to be removed.\n",
413 (unsigned int) rowid);
414 if (0 <
415 GNUNET_PQ_eval_prepared_non_select (plugin->dbh,
416 "delrow",
417 param))
418 {
419 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
420 "datastore-postgres",
421 "Deleting %u bytes from database\n",
422 (unsigned int) size);
423 plugin->env->duc (plugin->env->cls,
424 -(size + GNUNET_DATASTORE_ENTRY_OVERHEAD));
425 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
426 "datastore-postgres",
427 "Deleted %u bytes from database\n",
428 (unsigned int) size);
429 }
430 }
431 GNUNET_PQ_cleanup_result (rs);
432 } /* for (i) */
433}
434
435
436/**
437 * Get one of the results for a particular key in the datastore.
438 *
439 * @param cls closure with the `struct Plugin`
440 * @param next_uid return the result with lowest uid >= next_uid
441 * @param random if true, return a random result instead of using next_uid
442 * @param key maybe NULL (to match all entries)
443 * @param type entries of which type are relevant?
444 * Use 0 for any type.
445 * @param proc function to call on the matching value;
446 * will be called with NULL if nothing matches
447 * @param proc_cls closure for @a proc
448 */
449static void
450postgres_plugin_get_key (void *cls,
451 uint64_t next_uid,
452 bool random,
453 const struct GNUNET_HashCode *key,
454 enum GNUNET_BLOCK_Type type,
455 PluginDatumProcessor proc,
456 void *proc_cls)
457{
458 struct Plugin *plugin = cls;
459 uint32_t utype = type;
460 uint16_t use_rvalue = random;
461 uint16_t use_key = NULL != key;
462 uint16_t use_type = GNUNET_BLOCK_TYPE_ANY != type;
463 uint64_t rvalue;
464 struct GNUNET_PQ_QueryParam params[] = {
465 GNUNET_PQ_query_param_uint64 (&next_uid),
466 GNUNET_PQ_query_param_uint64 (&rvalue),
467 GNUNET_PQ_query_param_uint16 (&use_rvalue),
468 GNUNET_PQ_query_param_auto_from_type (key),
469 GNUNET_PQ_query_param_uint16 (&use_key),
470 GNUNET_PQ_query_param_uint32 (&utype),
471 GNUNET_PQ_query_param_uint16 (&use_type),
472 GNUNET_PQ_query_param_end
473 };
474 struct ProcessResultContext prc;
475 enum GNUNET_DB_QueryStatus res;
476
477 if (random)
478 {
479 rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
480 UINT64_MAX);
481 next_uid = 0;
482 }
483 else
484 {
485 rvalue = 0;
486 }
487 prc.plugin = plugin;
488 prc.proc = proc;
489 prc.proc_cls = proc_cls;
490
491 res = GNUNET_PQ_eval_prepared_multi_select (plugin->dbh,
492 "get",
493 params,
494 &process_result,
495 &prc);
496 if (0 > res)
497 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0,
498 GNUNET_TIME_UNIT_ZERO_ABS, 0);
499}
500
501
502/**
503 * Select a subset of the items in the datastore and call
504 * the given iterator for each of them.
505 *
506 * @param cls our `struct Plugin *`
507 * @param next_uid return the result with lowest uid >= next_uid
508 * @param type entries of which type should be considered?
509 * Must not be zero (ANY).
510 * @param proc function to call on the matching value;
511 * will be called with NULL if no value matches
512 * @param proc_cls closure for @a proc
513 */
514static void
515postgres_plugin_get_zero_anonymity (void *cls,
516 uint64_t next_uid,
517 enum GNUNET_BLOCK_Type type,
518 PluginDatumProcessor proc,
519 void *proc_cls)
520{
521 struct Plugin *plugin = cls;
522 uint32_t utype = type;
523 struct GNUNET_PQ_QueryParam params[] = {
524 GNUNET_PQ_query_param_uint32 (&utype),
525 GNUNET_PQ_query_param_uint64 (&next_uid),
526 GNUNET_PQ_query_param_end
527 };
528 struct ProcessResultContext prc;
529 enum GNUNET_DB_QueryStatus res;
530
531 prc.plugin = plugin;
532 prc.proc = proc;
533 prc.proc_cls = proc_cls;
534 res = GNUNET_PQ_eval_prepared_multi_select (plugin->dbh,
535 "select_non_anonymous",
536 params,
537 &process_result,
538 &prc);
539 if (0 > res)
540 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0,
541 GNUNET_TIME_UNIT_ZERO_ABS, 0);
542}
543
544
545/**
546 * Context for #repl_iter() function.
547 */
548struct ReplCtx
549{
550 /**
551 * Plugin handle.
552 */
553 struct Plugin *plugin;
554
555 /**
556 * Function to call for the result (or the NULL).
557 */
558 PluginDatumProcessor proc;
559
560 /**
561 * Closure for @e proc.
562 */
563 void *proc_cls;
564};
565
566
567/**
568 * Wrapper for the iterator for 'sqlite_plugin_replication_get'.
569 * Decrements the replication counter and calls the original
570 * iterator.
571 *
572 * @param cls closure with the `struct ReplCtx *`
573 * @param key key for the content
574 * @param size number of bytes in @a data
575 * @param data content stored
576 * @param type type of the content
577 * @param priority priority of the content
578 * @param anonymity anonymity-level for the content
579 * @param replication replication-level for the content
580 * @param expiration expiration time for the content
581 * @param uid unique identifier for the datum;
582 * maybe 0 if no unique identifier is available
583 * @return #GNUNET_SYSERR to abort the iteration,
584 * #GNUNET_OK to continue
585 * (continue on call to "next", of course),
586 * #GNUNET_NO to delete the item and continue (if supported)
587 */
588static int
589repl_proc (void *cls,
590 const struct GNUNET_HashCode *key,
591 uint32_t size,
592 const void *data,
593 enum GNUNET_BLOCK_Type type,
594 uint32_t priority,
595 uint32_t anonymity,
596 uint32_t replication,
597 struct GNUNET_TIME_Absolute expiration,
598 uint64_t uid)
599{
600 struct ReplCtx *rc = cls;
601 struct Plugin *plugin = rc->plugin;
602 int ret;
603 struct GNUNET_PQ_QueryParam params[] = {
604 GNUNET_PQ_query_param_uint64 (&uid),
605 GNUNET_PQ_query_param_end
606 };
607 enum GNUNET_DB_QueryStatus qret;
608
609 ret = rc->proc (rc->proc_cls,
610 key,
611 size,
612 data,
613 type,
614 priority,
615 anonymity,
616 replication,
617 expiration,
618 uid);
619 if (NULL == key)
620 return ret;
621 qret = GNUNET_PQ_eval_prepared_non_select (plugin->dbh,
622 "decrepl",
623 params);
624 if (0 > qret)
625 return GNUNET_SYSERR;
626 return ret;
627}
628
629
630/**
631 * Get a random item for replication. Returns a single, not expired,
632 * random item from those with the highest replication counters. The
633 * item's replication counter is decremented by one IF it was positive
634 * before. Call @a proc with all values ZERO or NULL if the datastore
635 * is empty.
636 *
637 * @param cls closure with the `struct Plugin`
638 * @param proc function to call the value (once only).
639 * @param proc_cls closure for @a proc
640 */
641static void
642postgres_plugin_get_replication (void *cls,
643 PluginDatumProcessor proc,
644 void *proc_cls)
645{
646 struct Plugin *plugin = cls;
647 struct GNUNET_PQ_QueryParam params[] = {
648 GNUNET_PQ_query_param_end
649 };
650 struct ReplCtx rc;
651 struct ProcessResultContext prc;
652 enum GNUNET_DB_QueryStatus res;
653
654 rc.plugin = plugin;
655 rc.proc = proc;
656 rc.proc_cls = proc_cls;
657 prc.plugin = plugin;
658 prc.proc = &repl_proc;
659 prc.proc_cls = &rc;
660 res = GNUNET_PQ_eval_prepared_multi_select (plugin->dbh,
661 "select_replication_order",
662 params,
663 &process_result,
664 &prc);
665 if (0 > res)
666 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0,
667 GNUNET_TIME_UNIT_ZERO_ABS, 0);
668}
669
670
671/**
672 * Get a random item for expiration. Call @a proc with all values
673 * ZERO or NULL if the datastore is empty.
674 *
675 * @param cls closure with the `struct Plugin`
676 * @param proc function to call the value (once only).
677 * @param proc_cls closure for @a proc
678 */
679static void
680postgres_plugin_get_expiration (void *cls,
681 PluginDatumProcessor proc,
682 void *proc_cls)
683{
684 struct Plugin *plugin = cls;
685 struct GNUNET_TIME_Absolute now = { 0 };
686 struct GNUNET_PQ_QueryParam params[] = {
687 GNUNET_PQ_query_param_absolute_time (&now),
688 GNUNET_PQ_query_param_end
689 };
690 struct ProcessResultContext prc;
691
692 now = GNUNET_TIME_absolute_get ();
693 prc.plugin = plugin;
694 prc.proc = proc;
695 prc.proc_cls = proc_cls;
696 (void) GNUNET_PQ_eval_prepared_multi_select (plugin->dbh,
697 "select_expiration_order",
698 params,
699 &process_result,
700 &prc);
701}
702
703
704/**
705 * Closure for #process_keys.
706 */
707struct ProcessKeysContext
708{
709 /**
710 * Function to call for each key.
711 */
712 PluginKeyProcessor proc;
713
714 /**
715 * Closure for @e proc.
716 */
717 void *proc_cls;
718};
719
720
721/**
722 * Function to be called with the results of a SELECT statement
723 * that has returned @a num_results results.
724 *
725 * @param cls closure with a `struct ProcessKeysContext`
726 * @param result the postgres result
727 * @param num_results the number of results in @a result
728 */
729static void
730process_keys (void *cls,
731 PGresult *result,
732 unsigned int num_results)
733{
734 struct ProcessKeysContext *pkc = cls;
735
736 for (unsigned i = 0; i < num_results; i++)
737 {
738 struct GNUNET_HashCode key;
739 struct GNUNET_PQ_ResultSpec rs[] = {
740 GNUNET_PQ_result_spec_auto_from_type ("hash",
741 &key),
742 GNUNET_PQ_result_spec_end
743 };
744
745 if (GNUNET_OK !=
746 GNUNET_PQ_extract_result (result,
747 rs,
748 i))
749 {
750 GNUNET_break (0);
751 continue;
752 }
753 pkc->proc (pkc->proc_cls,
754 &key,
755 1);
756 GNUNET_PQ_cleanup_result (rs);
757 }
758}
759
760
761/**
762 * Get all of the keys in the datastore.
763 *
764 * @param cls closure with the `struct Plugin *`
765 * @param proc function to call on each key
766 * @param proc_cls closure for @a proc
767 */
768static void
769postgres_plugin_get_keys (void *cls,
770 PluginKeyProcessor proc,
771 void *proc_cls)
772{
773 struct Plugin *plugin = cls;
774 struct GNUNET_PQ_QueryParam params[] = {
775 GNUNET_PQ_query_param_end
776 };
777 struct ProcessKeysContext pkc;
778
779 pkc.proc = proc;
780 pkc.proc_cls = proc_cls;
781 (void) GNUNET_PQ_eval_prepared_multi_select (plugin->dbh,
782 "get_keys",
783 params,
784 &process_keys,
785 &pkc);
786 proc (proc_cls,
787 NULL,
788 0);
789}
790
791
792/**
793 * Drop database.
794 *
795 * @param cls closure with the `struct Plugin *`
796 */
797static void
798postgres_plugin_drop (void *cls)
799{
800 struct Plugin *plugin = cls;
801 struct GNUNET_PQ_ExecuteStatement es[] = {
802 GNUNET_PQ_make_execute ("DROP TABLE gn090"),
803 GNUNET_PQ_EXECUTE_STATEMENT_END
804 };
805
806 if (GNUNET_OK !=
807 GNUNET_PQ_exec_statements (plugin->dbh,
808 es))
809 GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING,
810 "postgres",
811 _ ("Failed to drop table from database.\n"));
812}
813
814
815/**
816 * Remove a particular key in the datastore.
817 *
818 * @param cls closure
819 * @param key key for the content
820 * @param size number of bytes in data
821 * @param data content stored
822 * @param cont continuation called with success or failure status
823 * @param cont_cls continuation closure for @a cont
824 */
825static void
826postgres_plugin_remove_key (void *cls,
827 const struct GNUNET_HashCode *key,
828 uint32_t size,
829 const void *data,
830 PluginRemoveCont cont,
831 void *cont_cls)
832{
833 struct Plugin *plugin = cls;
834 enum GNUNET_DB_QueryStatus ret;
835 struct GNUNET_PQ_QueryParam params[] = {
836 GNUNET_PQ_query_param_auto_from_type (key),
837 GNUNET_PQ_query_param_fixed_size (data, size),
838 GNUNET_PQ_query_param_end
839 };
840
841 ret = GNUNET_PQ_eval_prepared_non_select (plugin->dbh,
842 "remove",
843 params);
844 if (0 > ret)
845 {
846 cont (cont_cls,
847 key,
848 size,
849 GNUNET_SYSERR,
850 _ ("Postgresql exec failure"));
851 return;
852 }
853 if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == ret)
854 {
855 cont (cont_cls,
856 key,
857 size,
858 GNUNET_NO,
859 NULL);
860 return;
861 }
862 plugin->env->duc (plugin->env->cls,
863 -(size + GNUNET_DATASTORE_ENTRY_OVERHEAD));
864 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
865 "datastore-postgres",
866 "Deleted %u bytes from database\n",
867 (unsigned int) size);
868 cont (cont_cls,
869 key,
870 size,
871 GNUNET_OK,
872 NULL);
873}
874
875
876/**
877 * Entry point for the plugin.
878 *
879 * @param cls the `struct GNUNET_DATASTORE_PluginEnvironment*`
880 * @return our `struct Plugin *`
881 */
882void *
883libgnunet_plugin_datastore_postgres_init (void *cls)
884{
885 struct GNUNET_DATASTORE_PluginEnvironment *env = cls;
886 struct GNUNET_DATASTORE_PluginFunctions *api;
887 struct Plugin *plugin;
888
889 plugin = GNUNET_new (struct Plugin);
890 plugin->env = env;
891 if (GNUNET_OK != init_connection (plugin))
892 {
893 GNUNET_free (plugin);
894 return NULL;
895 }
896 api = GNUNET_new (struct GNUNET_DATASTORE_PluginFunctions);
897 api->cls = plugin;
898 api->estimate_size = &postgres_plugin_estimate_size;
899 api->put = &postgres_plugin_put;
900 api->get_key = &postgres_plugin_get_key;
901 api->get_replication = &postgres_plugin_get_replication;
902 api->get_expiration = &postgres_plugin_get_expiration;
903 api->get_zero_anonymity = &postgres_plugin_get_zero_anonymity;
904 api->get_keys = &postgres_plugin_get_keys;
905 api->drop = &postgres_plugin_drop;
906 api->remove_key = &postgres_plugin_remove_key;
907 return api;
908}
909
910
911/**
912 * Exit point from the plugin.
913 *
914 * @param cls our `struct Plugin *`
915 * @return always NULL
916 */
917void *
918libgnunet_plugin_datastore_postgres_done (void *cls)
919{
920 struct GNUNET_DATASTORE_PluginFunctions *api = cls;
921 struct Plugin *plugin = api->cls;
922
923 GNUNET_PQ_disconnect (plugin->dbh);
924 GNUNET_free (plugin);
925 GNUNET_free (api);
926 return NULL;
927}
928
929
930/* end of plugin_datastore_postgres.c */
diff --git a/src/plugin/datastore/plugin_datastore_sqlite.c b/src/plugin/datastore/plugin_datastore_sqlite.c
new file mode 100644
index 000000000..5ea9da4cb
--- /dev/null
+++ b/src/plugin/datastore/plugin_datastore_sqlite.c
@@ -0,0 +1,1375 @@
1/*
2 * This file is part of GNUnet
3 * Copyright (C) 2009, 2011, 2017 GNUnet e.V.
4 *
5 * GNUnet is free software: you can redistribute it and/or modify it
6 * under the terms of the GNU Affero General Public License as published
7 * by the Free Software Foundation, either version 3 of the License,
8 * or (at your option) any later version.
9 *
10 * GNUnet is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Affero General Public License for more details.
14 *
15 * You should have received a copy of the GNU Affero General Public License
16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file datastore/plugin_datastore_sqlite.c
23 * @brief sqlite-based datastore backend
24 * @author Christian Grothoff
25 */
26
27#include "platform.h"
28#include "gnunet_datastore_plugin.h"
29#include "gnunet_sq_lib.h"
30#include <sqlite3.h>
31
32
33/**
34 * We allocate items on the stack at times. To prevent a stack
35 * overflow, we impose a limit on the maximum size for the data per
36 * item. 64k should be enough.
37 */
38#define MAX_ITEM_SIZE 65536
39
40/**
41 * After how many ms "busy" should a DB operation fail for good?
42 * A low value makes sure that we are more responsive to requests
43 * (especially PUTs). A high value guarantees a higher success
44 * rate (SELECTs in iterate can take several seconds despite LIMIT=1).
45 *
46 * The default value of 250ms should ensure that users do not experience
47 * huge latencies while at the same time allowing operations to succeed
48 * with reasonable probability.
49 */
50#define BUSY_TIMEOUT_MS 250
51
52
53/**
54 * Log an error message at log-level 'level' that indicates
55 * a failure of the command 'cmd' on file 'filename'
56 * with the message given by strerror(errno).
57 */
58#define LOG_SQLITE(db, level, cmd) \
59 do \
60 { \
61 GNUNET_log_from (level, \
62 "sqlite", \
63 _ ("`%s' failed at %s:%d with error: %s\n"), \
64 cmd, \
65 __FILE__, \
66 __LINE__, \
67 sqlite3_errmsg (db->dbh)); \
68 } while (0)
69
70
71/**
72 * Log an error message at log-level 'level' that indicates
73 * a failure of the command 'cmd' on file 'filename'
74 * with the message given by strerror(errno).
75 */
76#define LOG_SQLITE_MSG(db, msg, level, cmd) \
77 do \
78 { \
79 GNUNET_log_from (level, \
80 "sqlite", \
81 _ ("`%s' failed at %s:%d with error: %s\n"), \
82 cmd, \
83 __FILE__, \
84 __LINE__, \
85 sqlite3_errmsg (db->dbh)); \
86 GNUNET_asprintf (msg, \
87 _ ("`%s' failed at %s:%u with error: %s"), \
88 cmd, \
89 __FILE__, \
90 __LINE__, \
91 sqlite3_errmsg (db->dbh)); \
92 } while (0)
93
94
95/**
96 * Context for all functions in this plugin.
97 */
98struct Plugin
99{
100 /**
101 * Our execution environment.
102 */
103 struct GNUNET_DATASTORE_PluginEnvironment *env;
104
105 /**
106 * Database filename.
107 */
108 char *fn;
109
110 /**
111 * Native SQLite database handle.
112 */
113 sqlite3 *dbh;
114
115 /**
116 * Precompiled SQL for remove_key.
117 */
118 sqlite3_stmt *remove;
119
120 /**
121 * Precompiled SQL for deletion.
122 */
123 sqlite3_stmt *delRow;
124
125 /**
126 * Precompiled SQL for update.
127 */
128 sqlite3_stmt *update;
129
130 /**
131 * Get maximum repl value in database.
132 */
133 sqlite3_stmt *maxRepl;
134
135 /**
136 * Precompiled SQL for replication decrement.
137 */
138 sqlite3_stmt *updRepl;
139
140 /**
141 * Precompiled SQL for replication selection.
142 */
143 sqlite3_stmt *selRepl;
144
145 /**
146 * Precompiled SQL for expiration selection.
147 */
148 sqlite3_stmt *selExpi;
149
150 /**
151 * Precompiled SQL for expiration selection.
152 */
153 sqlite3_stmt *selZeroAnon;
154
155 /**
156 * Precompiled SQL for insertion.
157 */
158 sqlite3_stmt *insertContent;
159
160 /**
161 * Precompiled SQL for selection
162 */
163 sqlite3_stmt *get[8];
164
165 /**
166 * Should the database be dropped on shutdown?
167 */
168 int drop_on_shutdown;
169};
170
171
172/**
173 * @brief Prepare a SQL statement
174 *
175 * @param dbh handle to the database
176 * @param zSql SQL statement, UTF-8 encoded
177 * @param ppStmt set to the prepared statement
178 * @return 0 on success
179 */
180static int
181sq_prepare (sqlite3 *dbh, const char *zSql, sqlite3_stmt **ppStmt)
182{
183 char *dummy;
184 int result;
185
186 result = sqlite3_prepare_v2 (dbh,
187 zSql,
188 strlen (zSql),
189 ppStmt,
190 (const char **) &dummy);
191 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
192 "sqlite",
193 "Prepared `%s' / %p: %d\n",
194 zSql,
195 *ppStmt,
196 result);
197 return result;
198}
199
200
201/**
202 * Create our database indices.
203 *
204 * @param dbh handle to the database
205 */
206static void
207create_indices (sqlite3 *dbh)
208{
209 /* create indices */
210 if (
211 0 !=
212 (SQLITE_OK !=
213 sqlite3_exec (dbh,
214 "CREATE INDEX IF NOT EXISTS idx_hash ON gn091 (hash)",
215 NULL,
216 NULL,
217 NULL))
218 + (SQLITE_OK !=
219 sqlite3_exec (
220 dbh,
221 "CREATE INDEX IF NOT EXISTS idx_anon_type ON gn091 (anonLevel ASC,type)",
222 NULL,
223 NULL,
224 NULL))
225 + (SQLITE_OK !=
226 sqlite3_exec (dbh,
227 "CREATE INDEX IF NOT EXISTS idx_expire ON gn091 (expire ASC)",
228 NULL,
229 NULL,
230 NULL))
231 + (SQLITE_OK !=
232 sqlite3_exec (
233 dbh,
234 "CREATE INDEX IF NOT EXISTS idx_repl_rvalue ON gn091 (repl,rvalue)",
235 NULL,
236 NULL,
237 NULL)))
238 GNUNET_log_from (GNUNET_ERROR_TYPE_ERROR,
239 "sqlite",
240 "Failed to create indices: %s\n",
241 sqlite3_errmsg (dbh));
242}
243
244
245#if 0
246#define CHECK(a) GNUNET_break (a)
247#define ENULL NULL
248#else
249#define ENULL &e
250#define ENULL_DEFINED 1
251#define CHECK(a) \
252 if (! (a)) \
253 { \
254 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "%s\n", e); \
255 sqlite3_free (e); \
256 e = NULL; \
257 }
258#endif
259
260
261/**
262 * Initialize the database connections and associated
263 * data structures (create tables and indices
264 * as needed as well).
265 *
266 * @param cfg our configuration
267 * @param plugin the plugin context (state for this module)
268 * @return #GNUNET_OK on success
269 */
270static int
271database_setup (const struct GNUNET_CONFIGURATION_Handle *cfg,
272 struct Plugin *plugin)
273{
274 sqlite3_stmt *stmt;
275 char *afsdir;
276
277#if ENULL_DEFINED
278 char *e;
279#endif
280
281 if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_filename (cfg,
282 "datastore-sqlite",
283 "FILENAME",
284 &afsdir))
285 {
286 GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
287 "datastore-sqlite",
288 "FILENAME");
289 return GNUNET_SYSERR;
290 }
291 if (GNUNET_OK != GNUNET_DISK_file_test (afsdir))
292 {
293 if (GNUNET_OK != GNUNET_DISK_directory_create_for_file (afsdir))
294 {
295 GNUNET_break (0);
296 GNUNET_free (afsdir);
297 return GNUNET_SYSERR;
298 }
299 /* database is new or got deleted, reset payload to zero! */
300 if (NULL != plugin->env->duc)
301 plugin->env->duc (plugin->env->cls, 0);
302 }
303 /* afsdir should be UTF-8-encoded. If it isn't, it's a bug */
304 plugin->fn = afsdir;
305
306 /* Open database and precompile statements */
307 if (SQLITE_OK != sqlite3_open (plugin->fn, &plugin->dbh))
308 {
309 GNUNET_log_from (GNUNET_ERROR_TYPE_ERROR,
310 "sqlite",
311 _ ("Unable to initialize SQLite: %s.\n"),
312 sqlite3_errmsg (plugin->dbh));
313 return GNUNET_SYSERR;
314 }
315 CHECK (
316 SQLITE_OK ==
317 sqlite3_exec (plugin->dbh, "PRAGMA temp_store=MEMORY", NULL, NULL, ENULL));
318 CHECK (
319 SQLITE_OK ==
320 sqlite3_exec (plugin->dbh, "PRAGMA synchronous=OFF", NULL, NULL, ENULL));
321 CHECK (SQLITE_OK == sqlite3_exec (plugin->dbh,
322 "PRAGMA legacy_file_format=OFF",
323 NULL,
324 NULL,
325 ENULL));
326 CHECK (SQLITE_OK == sqlite3_exec (plugin->dbh,
327 "PRAGMA auto_vacuum=INCREMENTAL",
328 NULL,
329 NULL,
330 ENULL));
331 CHECK (SQLITE_OK == sqlite3_exec (plugin->dbh,
332 "PRAGMA locking_mode=EXCLUSIVE",
333 NULL,
334 NULL,
335 ENULL));
336 CHECK (
337 SQLITE_OK ==
338 sqlite3_exec (plugin->dbh, "PRAGMA page_size=4096", NULL, NULL, ENULL));
339
340 CHECK (SQLITE_OK == sqlite3_busy_timeout (plugin->dbh, BUSY_TIMEOUT_MS));
341
342
343 /* We have to do it here, because otherwise precompiling SQL might fail */
344 CHECK (SQLITE_OK ==
345 sq_prepare (plugin->dbh,
346 "SELECT 1 FROM sqlite_master WHERE tbl_name = 'gn091'",
347 &stmt));
348
349 /* FIXME: SQLite does not have unsigned integers! This is ok for the type column because
350 * we only test equality on it and can cast it to/from uint32_t. For repl, prio, and anonLevel
351 * we do math or inequality tests, so we can't handle the entire range of uint32_t.
352 * This will also cause problems for expiration times after 294247-01-10-04:00:54 UTC.
353 */if ((SQLITE_DONE == sqlite3_step (stmt)) &&
354 (SQLITE_OK != sqlite3_exec (plugin->dbh,
355 "CREATE TABLE gn091 ("
356 " repl INT4 NOT NULL DEFAULT 0,"
357 " type INT4 NOT NULL DEFAULT 0,"
358 " prio INT4 NOT NULL DEFAULT 0,"
359 " anonLevel INT4 NOT NULL DEFAULT 0,"
360 " expire INT8 NOT NULL DEFAULT 0,"
361 " rvalue INT8 NOT NULL,"
362 " hash TEXT NOT NULL DEFAULT '',"
363 " vhash TEXT NOT NULL DEFAULT '',"
364 " value BLOB NOT NULL DEFAULT '')",
365 NULL,
366 NULL,
367 NULL)))
368 {
369 LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "sqlite3_exec");
370 sqlite3_finalize (stmt);
371 return GNUNET_SYSERR;
372 }
373 sqlite3_finalize (stmt);
374 create_indices (plugin->dbh);
375
376#define RESULT_COLUMNS \
377 "repl, type, prio, anonLevel, expire, hash, value, _ROWID_"
378 if (
379 (SQLITE_OK != sq_prepare (plugin->dbh,
380 "UPDATE gn091 "
381 "SET prio = prio + ?, "
382 "repl = repl + ?, "
383 "expire = MAX(expire, ?) "
384 "WHERE hash = ? AND vhash = ?",
385 &plugin->update)) ||
386 (SQLITE_OK != sq_prepare (plugin->dbh,
387 "UPDATE gn091 "
388 "SET repl = MAX (0, repl - 1) WHERE _ROWID_ = ?",
389 &plugin->updRepl)) ||
390 (SQLITE_OK != sq_prepare (plugin->dbh,
391 "SELECT " RESULT_COLUMNS " FROM gn091 "
392 "WHERE repl=?2 AND "
393 " (rvalue>=?1 OR "
394 " NOT EXISTS (SELECT 1 FROM gn091 "
395 "WHERE repl=?2 AND rvalue>=?1 LIMIT 1) ) "
396 "ORDER BY rvalue ASC LIMIT 1",
397 &plugin->selRepl)) ||
398 (SQLITE_OK != sq_prepare (plugin->dbh,
399 "SELECT MAX(repl) FROM gn091",
400 &plugin->maxRepl)) ||
401 (SQLITE_OK !=
402 sq_prepare (plugin->dbh,
403 "SELECT " RESULT_COLUMNS " FROM gn091 "
404 "WHERE NOT EXISTS (SELECT 1 FROM gn091 WHERE expire < ?1 LIMIT 1) OR (expire < ?1) "
405 "ORDER BY expire ASC LIMIT 1",
406 &plugin->selExpi)) ||
407 (SQLITE_OK != sq_prepare (plugin->dbh,
408 "SELECT " RESULT_COLUMNS " FROM gn091 "
409 "WHERE _ROWID_ >= ? AND "
410 "anonLevel = 0 AND "
411 "type = ? "
412 "ORDER BY _ROWID_ ASC LIMIT 1",
413 &plugin->selZeroAnon)) ||
414 (SQLITE_OK !=
415 sq_prepare (plugin->dbh,
416 "INSERT INTO gn091 (repl, type, prio, anonLevel, expire, rvalue, hash, vhash, value) "
417 "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
418 &plugin->insertContent)) ||
419 (SQLITE_OK != sq_prepare (plugin->dbh,
420 "SELECT " RESULT_COLUMNS " FROM gn091 "
421 "WHERE _ROWID_ >= ?1 "
422 "ORDER BY _ROWID_ ASC LIMIT 1",
423 &plugin->get[0])) ||
424 (SQLITE_OK != sq_prepare (plugin->dbh,
425 "SELECT " RESULT_COLUMNS " FROM gn091 "
426 "WHERE _ROWID_ >= ?1 AND "
427 "type = ?4 "
428 "ORDER BY _ROWID_ ASC LIMIT 1",
429 &plugin->get[1])) ||
430 (SQLITE_OK != sq_prepare (plugin->dbh,
431 "SELECT " RESULT_COLUMNS " FROM gn091 "
432 "WHERE _ROWID_ >= ?1 AND "
433 "hash = ?3 "
434 "ORDER BY _ROWID_ ASC LIMIT 1",
435 &plugin->get[2])) ||
436 (SQLITE_OK != sq_prepare (plugin->dbh,
437 "SELECT " RESULT_COLUMNS " FROM gn091 "
438 "WHERE _ROWID_ >= ?1 AND "
439 "hash = ?3 AND "
440 "type = ?4 "
441 "ORDER BY _ROWID_ ASC LIMIT 1",
442 &plugin->get[3])) ||
443 (SQLITE_OK != sq_prepare (plugin->dbh,
444 "SELECT " RESULT_COLUMNS " FROM gn091 "
445 "WHERE _ROWID_ >= ?1 AND "
446 "rvalue >= ?2 "
447 "ORDER BY _ROWID_ ASC LIMIT 1",
448 &plugin->get[4])) ||
449 (SQLITE_OK != sq_prepare (plugin->dbh,
450 "SELECT " RESULT_COLUMNS " FROM gn091 "
451 "WHERE _ROWID_ >= ?1 AND "
452 "rvalue >= ?2 AND "
453 "type = ?4 "
454 "ORDER BY _ROWID_ ASC LIMIT 1",
455 &plugin->get[5])) ||
456 (SQLITE_OK != sq_prepare (plugin->dbh,
457 "SELECT " RESULT_COLUMNS " FROM gn091 "
458 "WHERE _ROWID_ >= ?1 AND "
459 "rvalue >= ?2 AND "
460 "hash = ?3 "
461 "ORDER BY _ROWID_ ASC LIMIT 1",
462 &plugin->get[6])) ||
463 (SQLITE_OK != sq_prepare (plugin->dbh,
464 "SELECT " RESULT_COLUMNS " FROM gn091 "
465 "WHERE _ROWID_ >= ?1 AND "
466 "rvalue >= ?2 AND "
467 "hash = ?3 AND "
468 "type = ?4 "
469 "ORDER BY _ROWID_ ASC LIMIT 1",
470 &plugin->get[7])) ||
471 (SQLITE_OK != sq_prepare (plugin->dbh,
472 "DELETE FROM gn091 WHERE _ROWID_ = ?",
473 &plugin->delRow)) ||
474 (SQLITE_OK != sq_prepare (plugin->dbh,
475 "DELETE FROM gn091 "
476 "WHERE hash = ? AND "
477 "value = ? ",
478 &plugin->remove)) ||
479 false)
480 {
481 LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "precompiling");
482 return GNUNET_SYSERR;
483 }
484 return GNUNET_OK;
485}
486
487
488/**
489 * Shutdown database connection and associate data
490 * structures.
491 *
492 * @param plugin the plugin context (state for this module)
493 */
494static void
495database_shutdown (struct Plugin *plugin)
496{
497 int result;
498
499#if SQLITE_VERSION_NUMBER >= 3007000
500 sqlite3_stmt *stmt;
501#endif
502
503 if (NULL != plugin->remove)
504 sqlite3_finalize (plugin->remove);
505 if (NULL != plugin->delRow)
506 sqlite3_finalize (plugin->delRow);
507 if (NULL != plugin->update)
508 sqlite3_finalize (plugin->update);
509 if (NULL != plugin->updRepl)
510 sqlite3_finalize (plugin->updRepl);
511 if (NULL != plugin->selRepl)
512 sqlite3_finalize (plugin->selRepl);
513 if (NULL != plugin->maxRepl)
514 sqlite3_finalize (plugin->maxRepl);
515 if (NULL != plugin->selExpi)
516 sqlite3_finalize (plugin->selExpi);
517 if (NULL != plugin->selZeroAnon)
518 sqlite3_finalize (plugin->selZeroAnon);
519 if (NULL != plugin->insertContent)
520 sqlite3_finalize (plugin->insertContent);
521 for (int i = 0; i < 8; ++i)
522 if (NULL != plugin->get[i])
523 sqlite3_finalize (plugin->get[i]);
524 result = sqlite3_close (plugin->dbh);
525#if SQLITE_VERSION_NUMBER >= 3007000
526 if (result == SQLITE_BUSY)
527 {
528 GNUNET_log_from (
529 GNUNET_ERROR_TYPE_WARNING,
530 "sqlite",
531 _ (
532 "Tried to close sqlite without finalizing all prepared statements.\n"));
533 stmt = sqlite3_next_stmt (plugin->dbh, NULL);
534 while (NULL != stmt)
535 {
536 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
537 "sqlite",
538 "Closing statement %p\n",
539 stmt);
540 result = sqlite3_finalize (stmt);
541 if (result != SQLITE_OK)
542 GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING,
543 "sqlite",
544 "Failed to close statement %p: %d\n",
545 stmt,
546 result);
547 stmt = sqlite3_next_stmt (plugin->dbh, NULL);
548 }
549 result = sqlite3_close (plugin->dbh);
550 }
551#endif
552 if (SQLITE_OK != result)
553 LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "sqlite3_close");
554 GNUNET_free (plugin->fn);
555}
556
557
558/**
559 * Delete the database entry with the given
560 * row identifier.
561 *
562 * @param plugin the plugin context (state for this module)
563 * @param rid the ID of the row to delete
564 */
565static int
566delete_by_rowid (struct Plugin *plugin, uint64_t rid)
567{
568 struct GNUNET_SQ_QueryParam params[] = { GNUNET_SQ_query_param_uint64 (&rid),
569 GNUNET_SQ_query_param_end };
570
571 if (GNUNET_OK != GNUNET_SQ_bind (plugin->delRow, params))
572 return GNUNET_SYSERR;
573 if (SQLITE_DONE != sqlite3_step (plugin->delRow))
574 {
575 LOG_SQLITE (plugin,
576 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
577 "sqlite3_step");
578 GNUNET_SQ_reset (plugin->dbh, plugin->delRow);
579 return GNUNET_SYSERR;
580 }
581 GNUNET_SQ_reset (plugin->dbh, plugin->delRow);
582 return GNUNET_OK;
583}
584
585
586/**
587 * Store an item in the datastore.
588 *
589 * @param cls closure
590 * @param key key for the item
591 * @param absent true if the key was not found in the bloom filter
592 * @param size number of bytes in @a data
593 * @param data content stored
594 * @param type type of the content
595 * @param priority priority of the content
596 * @param anonymity anonymity-level for the content
597 * @param replication replication-level for the content
598 * @param expiration expiration time for the content
599 * @param cont continuation called with success or failure status
600 * @param cont_cls continuation closure
601 */
602static void
603sqlite_plugin_put (void *cls,
604 const struct GNUNET_HashCode *key,
605 bool absent,
606 uint32_t size,
607 const void *data,
608 enum GNUNET_BLOCK_Type type,
609 uint32_t priority,
610 uint32_t anonymity,
611 uint32_t replication,
612 struct GNUNET_TIME_Absolute expiration,
613 PluginPutCont cont,
614 void *cont_cls)
615{
616 struct Plugin *plugin = cls;
617 struct GNUNET_HashCode vhash;
618 char *msg = NULL;
619
620 GNUNET_CRYPTO_hash (data, size, &vhash);
621
622 if (! absent)
623 {
624 struct GNUNET_SQ_QueryParam params[] =
625 { GNUNET_SQ_query_param_uint32 (&priority),
626 GNUNET_SQ_query_param_uint32 (&replication),
627 GNUNET_SQ_query_param_absolute_time (&expiration),
628 GNUNET_SQ_query_param_auto_from_type (key),
629 GNUNET_SQ_query_param_auto_from_type (&vhash),
630 GNUNET_SQ_query_param_end };
631
632 if (GNUNET_OK != GNUNET_SQ_bind (plugin->update, params))
633 {
634 cont (cont_cls, key, size, GNUNET_SYSERR, _ ("sqlite bind failure"));
635 return;
636 }
637 if (SQLITE_DONE != sqlite3_step (plugin->update))
638 {
639 LOG_SQLITE_MSG (plugin,
640 &msg,
641 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
642 "sqlite3_step");
643 cont (cont_cls, key, size, GNUNET_SYSERR, msg);
644 GNUNET_free (msg);
645 return;
646 }
647 int changes = sqlite3_changes (plugin->dbh);
648 GNUNET_SQ_reset (plugin->dbh, plugin->update);
649 if (0 != changes)
650 {
651 cont (cont_cls, key, size, GNUNET_NO, NULL);
652 return;
653 }
654 }
655
656 uint64_t rvalue;
657 uint32_t type32 = (uint32_t) type;
658 struct GNUNET_SQ_QueryParam params[] =
659 { GNUNET_SQ_query_param_uint32 (&replication),
660 GNUNET_SQ_query_param_uint32 (&type32),
661 GNUNET_SQ_query_param_uint32 (&priority),
662 GNUNET_SQ_query_param_uint32 (&anonymity),
663 GNUNET_SQ_query_param_absolute_time (&expiration),
664 GNUNET_SQ_query_param_uint64 (&rvalue),
665 GNUNET_SQ_query_param_auto_from_type (key),
666 GNUNET_SQ_query_param_auto_from_type (&vhash),
667 GNUNET_SQ_query_param_fixed_size (data, size),
668 GNUNET_SQ_query_param_end };
669 int n;
670 int ret;
671 sqlite3_stmt *stmt;
672
673 if (size > MAX_ITEM_SIZE)
674 {
675 cont (cont_cls, key, size, GNUNET_SYSERR, _ ("Data too large"));
676 return;
677 }
678 GNUNET_log_from (
679 GNUNET_ERROR_TYPE_DEBUG,
680 "sqlite",
681 "Storing in database block with type %u/key `%s'/priority %u/expiration in %s (%s).\n",
682 type,
683 GNUNET_h2s (key),
684 priority,
685 GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_remaining (
686 expiration),
687 GNUNET_YES),
688 GNUNET_STRINGS_absolute_time_to_string (expiration));
689 stmt = plugin->insertContent;
690 rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, UINT64_MAX);
691 if (GNUNET_OK != GNUNET_SQ_bind (stmt, params))
692 {
693 cont (cont_cls, key, size, GNUNET_SYSERR, NULL);
694 return;
695 }
696 n = sqlite3_step (stmt);
697 switch (n)
698 {
699 case SQLITE_DONE:
700 if (NULL != plugin->env->duc)
701 plugin->env->duc (plugin->env->cls,
702 size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
703 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
704 "sqlite",
705 "Stored new entry (%u bytes)\n",
706 size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
707 ret = GNUNET_OK;
708 break;
709
710 case SQLITE_BUSY:
711 GNUNET_break (0);
712 LOG_SQLITE_MSG (plugin,
713 &msg,
714 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
715 "sqlite3_step");
716 ret = GNUNET_SYSERR;
717 break;
718
719 default:
720 LOG_SQLITE_MSG (plugin,
721 &msg,
722 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
723 "sqlite3_step");
724 GNUNET_SQ_reset (plugin->dbh, stmt);
725 database_shutdown (plugin);
726 database_setup (plugin->env->cfg, plugin);
727 cont (cont_cls, key, size, GNUNET_SYSERR, msg);
728 GNUNET_free (msg);
729 return;
730 }
731 GNUNET_SQ_reset (plugin->dbh, stmt);
732 cont (cont_cls, key, size, ret, msg);
733 GNUNET_free (msg);
734}
735
736
737/**
738 * Execute statement that gets a row and call the callback
739 * with the result. Resets the statement afterwards.
740 *
741 * @param plugin the plugin
742 * @param stmt the statement
743 * @param proc processor to call
744 * @param proc_cls closure for @a proc
745 */
746static void
747execute_get (struct Plugin *plugin,
748 sqlite3_stmt *stmt,
749 PluginDatumProcessor proc,
750 void *proc_cls)
751{
752 int n;
753 struct GNUNET_TIME_Absolute expiration;
754 uint32_t replication;
755 uint32_t type;
756 uint32_t priority;
757 uint32_t anonymity;
758 uint64_t rowid;
759 void *value;
760 size_t value_size;
761 struct GNUNET_HashCode key;
762 int ret;
763 struct GNUNET_SQ_ResultSpec rs[] =
764 { GNUNET_SQ_result_spec_uint32 (&replication),
765 GNUNET_SQ_result_spec_uint32 (&type),
766 GNUNET_SQ_result_spec_uint32 (&priority),
767 GNUNET_SQ_result_spec_uint32 (&anonymity),
768 GNUNET_SQ_result_spec_absolute_time (&expiration),
769 GNUNET_SQ_result_spec_auto_from_type (&key),
770 GNUNET_SQ_result_spec_variable_size (&value, &value_size),
771 GNUNET_SQ_result_spec_uint64 (&rowid),
772 GNUNET_SQ_result_spec_end };
773
774 n = sqlite3_step (stmt);
775 switch (n)
776 {
777 case SQLITE_ROW:
778 if (GNUNET_OK != GNUNET_SQ_extract_result (stmt, rs))
779 {
780 GNUNET_break (0);
781 break;
782 }
783 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
784 "sqlite",
785 "Found reply in database with expiration %s\n",
786 GNUNET_STRINGS_absolute_time_to_string (expiration));
787 ret = proc (proc_cls,
788 &key,
789 value_size,
790 value,
791 type,
792 priority,
793 anonymity,
794 replication,
795 expiration,
796 rowid);
797 GNUNET_SQ_cleanup_result (rs);
798 GNUNET_SQ_reset (plugin->dbh, stmt);
799 if ((GNUNET_NO == ret) && (GNUNET_OK == delete_by_rowid (plugin, rowid)) &&
800 (NULL != plugin->env->duc))
801 plugin->env->duc (plugin->env->cls,
802 -(value_size + GNUNET_DATASTORE_ENTRY_OVERHEAD));
803 return;
804
805 case SQLITE_DONE:
806 /* database must be empty */
807 break;
808
809 case SQLITE_BUSY:
810 case SQLITE_ERROR:
811 case SQLITE_MISUSE:
812 default:
813 LOG_SQLITE (plugin,
814 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
815 "sqlite3_step");
816 if (SQLITE_OK != sqlite3_reset (stmt))
817 LOG_SQLITE (plugin,
818 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
819 "sqlite3_reset");
820 GNUNET_break (0);
821 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
822 database_shutdown (plugin);
823 database_setup (plugin->env->cfg, plugin);
824 return;
825 }
826 GNUNET_SQ_reset (plugin->dbh, stmt);
827 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
828}
829
830
831/**
832 * Select a subset of the items in the datastore and call
833 * the given processor for the item.
834 *
835 * @param cls our plugin context
836 * @param next_uid return the result with lowest uid >= next_uid
837 * @param type entries of which type should be considered?
838 * Must not be zero (ANY).
839 * @param proc function to call on the matching value;
840 * will be called with NULL if no value matches
841 * @param proc_cls closure for @a proc
842 */
843static void
844sqlite_plugin_get_zero_anonymity (void *cls,
845 uint64_t next_uid,
846 enum GNUNET_BLOCK_Type type,
847 PluginDatumProcessor proc,
848 void *proc_cls)
849{
850 struct Plugin *plugin = cls;
851 uint32_t type32 = type;
852 struct GNUNET_SQ_QueryParam params[] = { GNUNET_SQ_query_param_uint64 (
853 &next_uid),
854 GNUNET_SQ_query_param_uint32 (
855 &type32),
856 GNUNET_SQ_query_param_end };
857
858 GNUNET_assert (type != GNUNET_BLOCK_TYPE_ANY);
859 if (GNUNET_OK != GNUNET_SQ_bind (plugin->selZeroAnon, params))
860 {
861 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
862 return;
863 }
864 execute_get (plugin, plugin->selZeroAnon, proc, proc_cls);
865}
866
867
868/**
869 * Get results for a particular key in the datastore.
870 *
871 * @param cls closure
872 * @param next_uid return the result with lowest uid >= next_uid
873 * @param random if true, return a random result instead of using next_uid
874 * @param key maybe NULL (to match all entries)
875 * @param type entries of which type are relevant?
876 * Use 0 for any type.
877 * @param proc function to call on the matching value;
878 * will be called with NULL if nothing matches
879 * @param proc_cls closure for @a proc
880 */
881static void
882sqlite_plugin_get_key (void *cls,
883 uint64_t next_uid,
884 bool random,
885 const struct GNUNET_HashCode *key,
886 enum GNUNET_BLOCK_Type type,
887 PluginDatumProcessor proc,
888 void *proc_cls)
889{
890 struct Plugin *plugin = cls;
891 uint64_t rvalue;
892 int use_rvalue = random;
893 uint32_t type32 = (uint32_t) type;
894 int use_type = GNUNET_BLOCK_TYPE_ANY != type;
895 int use_key = NULL != key;
896 sqlite3_stmt *stmt = plugin->get[use_rvalue * 4 + use_key * 2 + use_type];
897 struct GNUNET_SQ_QueryParam params[] =
898 { GNUNET_SQ_query_param_uint64 (&next_uid),
899 GNUNET_SQ_query_param_uint64 (&rvalue),
900 GNUNET_SQ_query_param_auto_from_type (key),
901 GNUNET_SQ_query_param_uint32 (&type32),
902 GNUNET_SQ_query_param_end };
903
904 /* SQLite doesn't like it when you try to bind a parameter greater than the
905 * last numbered parameter, but unused parameters in the middle are OK.
906 */
907 if (! use_type)
908 {
909 params[3] = (struct GNUNET_SQ_QueryParam) GNUNET_SQ_query_param_end;
910 if (! use_key)
911 {
912 params[2] = (struct GNUNET_SQ_QueryParam) GNUNET_SQ_query_param_end;
913 if (! use_rvalue)
914 params[1] = (struct GNUNET_SQ_QueryParam) GNUNET_SQ_query_param_end;
915 }
916 }
917 if (random)
918 {
919 rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, UINT64_MAX);
920 next_uid = 0;
921 }
922 else
923 rvalue = 0;
924
925 if (GNUNET_OK != GNUNET_SQ_bind (stmt, params))
926 {
927 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
928 return;
929 }
930 execute_get (plugin, stmt, proc, proc_cls);
931}
932
933
934/**
935 * Context for #repl_proc() function.
936 */
937struct ReplCtx
938{
939 /**
940 * Function to call for the result (or the NULL).
941 */
942 PluginDatumProcessor proc;
943
944 /**
945 * Closure for @e proc.
946 */
947 void *proc_cls;
948
949 /**
950 * UID to use.
951 */
952 uint64_t uid;
953
954 /**
955 * Yes if UID was set.
956 */
957 int have_uid;
958};
959
960
961/**
962 * Wrapper for the processor for #sqlite_plugin_get_replication().
963 * Decrements the replication counter and calls the original
964 * processor.
965 *
966 * @param cls closure
967 * @param key key for the content
968 * @param size number of bytes in @a data
969 * @param data content stored
970 * @param type type of the content
971 * @param priority priority of the content
972 * @param anonymity anonymity-level for the content
973 * @param replication replication-level for the content
974 * @param expiration expiration time for the content
975 * @param uid unique identifier for the datum;
976 * maybe 0 if no unique identifier is available
977 * @return #GNUNET_OK for normal return,
978 * #GNUNET_NO to delete the item
979 */
980static int
981repl_proc (void *cls,
982 const struct GNUNET_HashCode *key,
983 uint32_t size,
984 const void *data,
985 enum GNUNET_BLOCK_Type type,
986 uint32_t priority,
987 uint32_t anonymity,
988 uint32_t replication,
989 struct GNUNET_TIME_Absolute expiration,
990 uint64_t uid)
991{
992 struct ReplCtx *rc = cls;
993 int ret;
994
995 if (GNUNET_SYSERR == rc->have_uid)
996 rc->have_uid = GNUNET_NO;
997 ret = rc->proc (rc->proc_cls,
998 key,
999 size,
1000 data,
1001 type,
1002 priority,
1003 anonymity,
1004 replication,
1005 expiration,
1006 uid);
1007 if (NULL != key)
1008 {
1009 rc->uid = uid;
1010 rc->have_uid = GNUNET_YES;
1011 }
1012 return ret;
1013}
1014
1015
1016/**
1017 * Get a random item for replication. Returns a single random item
1018 * from those with the highest replication counters. The item's
1019 * replication counter is decremented by one IF it was positive before.
1020 * Call @a proc with all values ZERO or NULL if the datastore is empty.
1021 *
1022 * @param cls closure
1023 * @param proc function to call the value (once only).
1024 * @param proc_cls closure for @a proc
1025 */
1026static void
1027sqlite_plugin_get_replication (void *cls,
1028 PluginDatumProcessor proc,
1029 void *proc_cls)
1030{
1031 struct Plugin *plugin = cls;
1032 struct ReplCtx rc;
1033 uint64_t rvalue = 0;
1034 uint32_t repl;
1035 struct GNUNET_SQ_QueryParam params_sel_repl[] =
1036 { GNUNET_SQ_query_param_uint64 (&rvalue),
1037 GNUNET_SQ_query_param_uint32 (&repl),
1038 GNUNET_SQ_query_param_end };
1039 struct GNUNET_SQ_QueryParam params_upd_repl[] =
1040 { GNUNET_SQ_query_param_uint64 (&rc.uid), GNUNET_SQ_query_param_end };
1041
1042 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
1043 "datastore-sqlite",
1044 "Getting random block based on replication order.\n");
1045 if (SQLITE_ROW != sqlite3_step (plugin->maxRepl))
1046 {
1047 GNUNET_SQ_reset (plugin->dbh, plugin->maxRepl);
1048 /* DB empty */
1049 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
1050 return;
1051 }
1052 repl = sqlite3_column_int (plugin->maxRepl, 0);
1053 GNUNET_SQ_reset (plugin->dbh, plugin->maxRepl);
1054 rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, UINT64_MAX);
1055 if (GNUNET_OK != GNUNET_SQ_bind (plugin->selRepl, params_sel_repl))
1056 {
1057 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
1058 return;
1059 }
1060 rc.have_uid = GNUNET_SYSERR;
1061 rc.proc = proc;
1062 rc.proc_cls = proc_cls;
1063 execute_get (plugin, plugin->selRepl, &repl_proc, &rc);
1064 if (GNUNET_YES == rc.have_uid)
1065 {
1066 if (GNUNET_OK != GNUNET_SQ_bind (plugin->updRepl, params_upd_repl))
1067 {
1068 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
1069 return;
1070 }
1071 if (SQLITE_DONE != sqlite3_step (plugin->updRepl))
1072 LOG_SQLITE (plugin,
1073 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
1074 "sqlite3_step");
1075 GNUNET_SQ_reset (plugin->dbh, plugin->updRepl);
1076 }
1077 if (GNUNET_SYSERR == rc.have_uid)
1078 {
1079 /* proc was not called at all so far, do it now. */
1080 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
1081 }
1082}
1083
1084
1085/**
1086 * Get a random item that has expired or has low priority.
1087 * Call @a proc with all values ZERO or NULL if the datastore is empty.
1088 *
1089 * @param cls closure
1090 * @param proc function to call the value (once only).
1091 * @param proc_cls closure for @a proc
1092 */
1093static void
1094sqlite_plugin_get_expiration (void *cls,
1095 PluginDatumProcessor proc,
1096 void *proc_cls)
1097{
1098 struct Plugin *plugin = cls;
1099 sqlite3_stmt *stmt;
1100 struct GNUNET_TIME_Absolute now = { 0 };
1101 struct GNUNET_SQ_QueryParam params[] = { GNUNET_SQ_query_param_absolute_time (
1102 &now),
1103 GNUNET_SQ_query_param_end };
1104
1105 GNUNET_log_from (
1106 GNUNET_ERROR_TYPE_DEBUG,
1107 "sqlite",
1108 "Getting random block based on expiration and priority order.\n");
1109 now = GNUNET_TIME_absolute_get ();
1110 stmt = plugin->selExpi;
1111 if (GNUNET_OK != GNUNET_SQ_bind (stmt, params))
1112 {
1113 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
1114 return;
1115 }
1116 execute_get (plugin, stmt, proc, proc_cls);
1117}
1118
1119
1120/**
1121 * Get all of the keys in the datastore.
1122 *
1123 * @param cls closure
1124 * @param proc function to call on each key
1125 * @param proc_cls closure for @a proc
1126 */
1127static void
1128sqlite_plugin_get_keys (void *cls, PluginKeyProcessor proc, void *proc_cls)
1129{
1130 struct Plugin *plugin = cls;
1131 struct GNUNET_HashCode key;
1132 struct GNUNET_SQ_ResultSpec results[] =
1133 { GNUNET_SQ_result_spec_auto_from_type (&key), GNUNET_SQ_result_spec_end };
1134 sqlite3_stmt *stmt;
1135 int ret;
1136
1137 GNUNET_assert (NULL != proc);
1138 if (SQLITE_OK != sq_prepare (plugin->dbh, "SELECT hash FROM gn091", &stmt))
1139 {
1140 LOG_SQLITE (plugin,
1141 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
1142 "sqlite_prepare");
1143 proc (proc_cls, NULL, 0);
1144 return;
1145 }
1146 while (SQLITE_ROW == (ret = sqlite3_step (stmt)))
1147 {
1148 if (GNUNET_OK == GNUNET_SQ_extract_result (stmt, results))
1149 proc (proc_cls, &key, 1);
1150 else
1151 GNUNET_break (0);
1152 }
1153 if (SQLITE_DONE != ret)
1154 LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "sqlite_step");
1155 sqlite3_finalize (stmt);
1156 proc (proc_cls, NULL, 0);
1157}
1158
1159
1160/**
1161 * Drop database.
1162 *
1163 * @param cls our plugin context
1164 */
1165static void
1166sqlite_plugin_drop (void *cls)
1167{
1168 struct Plugin *plugin = cls;
1169
1170 plugin->drop_on_shutdown = GNUNET_YES;
1171}
1172
1173
1174/**
1175 * Remove a particular key in the datastore.
1176 *
1177 * @param cls closure
1178 * @param key key for the content
1179 * @param size number of bytes in data
1180 * @param data content stored
1181 * @param cont continuation called with success or failure status
1182 * @param cont_cls continuation closure for @a cont
1183 */
1184static void
1185sqlite_plugin_remove_key (void *cls,
1186 const struct GNUNET_HashCode *key,
1187 uint32_t size,
1188 const void *data,
1189 PluginRemoveCont cont,
1190 void *cont_cls)
1191{
1192 struct Plugin *plugin = cls;
1193 struct GNUNET_SQ_QueryParam params[] =
1194 { GNUNET_SQ_query_param_auto_from_type (key),
1195 GNUNET_SQ_query_param_fixed_size (data, size),
1196 GNUNET_SQ_query_param_end };
1197
1198 if (GNUNET_OK != GNUNET_SQ_bind (plugin->remove, params))
1199 {
1200 cont (cont_cls, key, size, GNUNET_SYSERR, "bind failed");
1201 return;
1202 }
1203 if (SQLITE_DONE != sqlite3_step (plugin->remove))
1204 {
1205 LOG_SQLITE (plugin,
1206 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
1207 "sqlite3_step");
1208 GNUNET_SQ_reset (plugin->dbh, plugin->remove);
1209 cont (cont_cls, key, size, GNUNET_SYSERR, "sqlite3_step failed");
1210 return;
1211 }
1212 int changes = sqlite3_changes (plugin->dbh);
1213 GNUNET_SQ_reset (plugin->dbh, plugin->remove);
1214 if (0 == changes)
1215 {
1216 cont (cont_cls, key, size, GNUNET_NO, NULL);
1217 return;
1218 }
1219 if (NULL != plugin->env->duc)
1220 plugin->env->duc (plugin->env->cls,
1221 -(size + GNUNET_DATASTORE_ENTRY_OVERHEAD));
1222 cont (cont_cls, key, size, GNUNET_OK, NULL);
1223}
1224
1225
1226/**
1227 * Get an estimate of how much space the database is
1228 * currently using.
1229 *
1230 * @param cls the `struct Plugin`
1231 * @return the size of the database on disk (estimate)
1232 */
1233static void
1234sqlite_plugin_estimate_size (void *cls, unsigned long long *estimate)
1235{
1236 struct Plugin *plugin = cls;
1237 sqlite3_stmt *stmt;
1238 uint64_t pages;
1239 uint64_t page_size;
1240
1241#if ENULL_DEFINED
1242 char *e;
1243#endif
1244
1245 if (NULL == estimate)
1246 return;
1247 if (SQLITE_VERSION_NUMBER < 3006000)
1248 {
1249 GNUNET_log_from (
1250 GNUNET_ERROR_TYPE_WARNING,
1251 "datastore-sqlite",
1252 _ ("sqlite version to old to determine size, assuming zero\n"));
1253 *estimate = 0;
1254 return;
1255 }
1256 CHECK (SQLITE_OK == sqlite3_exec (plugin->dbh, "VACUUM", NULL, NULL, ENULL));
1257 CHECK (SQLITE_OK == sqlite3_exec (plugin->dbh,
1258 "PRAGMA auto_vacuum=INCREMENTAL",
1259 NULL,
1260 NULL,
1261 ENULL));
1262 if (SQLITE_OK != sq_prepare (plugin->dbh, "PRAGMA page_count", &stmt))
1263 {
1264 GNUNET_log_from (
1265 GNUNET_ERROR_TYPE_WARNING,
1266 "datastore-sqlite",
1267 _("error preparing statement\n"));
1268 return;
1269 }
1270 if (SQLITE_ROW == sqlite3_step (stmt))
1271 pages = sqlite3_column_int64 (stmt, 0);
1272 else
1273 pages = 0;
1274 sqlite3_finalize (stmt);
1275 if (SQLITE_OK != sq_prepare (plugin->dbh, "PRAGMA page_size", &stmt))
1276 {
1277 GNUNET_log_from (
1278 GNUNET_ERROR_TYPE_WARNING,
1279 "datastore-sqlite",
1280 _("error preparing statement\n"));
1281 return;
1282 }
1283 if (SQLITE_ROW != sqlite3_step (stmt))
1284 {
1285 GNUNET_log_from (
1286 GNUNET_ERROR_TYPE_WARNING,
1287 "datastore-sqlite",
1288 _("error stepping\n"));
1289 return;
1290 }
1291 page_size = sqlite3_column_int64 (stmt, 0);
1292 sqlite3_finalize (stmt);
1293 GNUNET_log (
1294 GNUNET_ERROR_TYPE_INFO,
1295 _ (
1296 "Using sqlite page utilization to estimate payload (%llu pages of size %llu bytes)\n"),
1297 (unsigned long long) pages,
1298 (unsigned long long) page_size);
1299 *estimate = pages * page_size;
1300}
1301
1302
1303/**
1304 * Entry point for the plugin.
1305 *
1306 * @param cls the `struct GNUNET_DATASTORE_PluginEnvironment *`
1307 * @return NULL on error, othrewise the plugin context
1308 */
1309void *
1310libgnunet_plugin_datastore_sqlite_init (void *cls)
1311{
1312 static struct Plugin plugin;
1313 struct GNUNET_DATASTORE_PluginEnvironment *env = cls;
1314 struct GNUNET_DATASTORE_PluginFunctions *api;
1315
1316 if (NULL != plugin.env)
1317 return NULL; /* can only initialize once! */
1318 memset (&plugin, 0, sizeof(struct Plugin));
1319 plugin.env = env;
1320 if (GNUNET_OK != database_setup (env->cfg, &plugin))
1321 {
1322 database_shutdown (&plugin);
1323 return NULL;
1324 }
1325 api = GNUNET_new (struct GNUNET_DATASTORE_PluginFunctions);
1326 api->cls = &plugin;
1327 api->estimate_size = &sqlite_plugin_estimate_size;
1328 api->put = &sqlite_plugin_put;
1329 api->get_key = &sqlite_plugin_get_key;
1330 api->get_replication = &sqlite_plugin_get_replication;
1331 api->get_expiration = &sqlite_plugin_get_expiration;
1332 api->get_zero_anonymity = &sqlite_plugin_get_zero_anonymity;
1333 api->get_keys = &sqlite_plugin_get_keys;
1334 api->drop = &sqlite_plugin_drop;
1335 api->remove_key = &sqlite_plugin_remove_key;
1336 GNUNET_log_from (GNUNET_ERROR_TYPE_INFO,
1337 "sqlite",
1338 _ ("Sqlite database running\n"));
1339 return api;
1340}
1341
1342
1343/**
1344 * Exit point from the plugin.
1345 *
1346 * @param cls the plugin context (as returned by "init")
1347 * @return always NULL
1348 */
1349void *
1350libgnunet_plugin_datastore_sqlite_done (void *cls)
1351{
1352 char *fn;
1353 struct GNUNET_DATASTORE_PluginFunctions *api = cls;
1354 struct Plugin *plugin = api->cls;
1355
1356 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
1357 "sqlite",
1358 "sqlite plugin is done\n");
1359 fn = NULL;
1360 if (plugin->drop_on_shutdown)
1361 fn = GNUNET_strdup (plugin->fn);
1362 database_shutdown (plugin);
1363 plugin->env = NULL;
1364 GNUNET_free (api);
1365 if (NULL != fn)
1366 {
1367 if (0 != unlink (fn))
1368 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", fn);
1369 GNUNET_free (fn);
1370 }
1371 return NULL;
1372}
1373
1374
1375/* end of plugin_datastore_sqlite.c */
diff --git a/src/plugin/datastore/plugin_datastore_template.c b/src/plugin/datastore/plugin_datastore_template.c
new file mode 100644
index 000000000..2b455f8cb
--- /dev/null
+++ b/src/plugin/datastore/plugin_datastore_template.c
@@ -0,0 +1,274 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2009, 2011 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file datastore/plugin_datastore_template.c
23 * @brief template-based datastore backend
24 * @author Christian Grothoff
25 */
26
27#include "platform.h"
28#include "gnunet_datastore_plugin.h"
29
30
31/**
32 * Context for all functions in this plugin.
33 */
34struct Plugin
35{
36 /**
37 * Our execution environment.
38 */
39 struct GNUNET_DATASTORE_PluginEnvironment *env;
40};
41
42
43/**
44 * Get an estimate of how much space the database is
45 * currently using.
46 *
47 * @param cls our "struct Plugin*"
48 * @return number of bytes used on disk
49 */
50static void
51template_plugin_estimate_size (void *cls, unsigned long long *estimate)
52{
53 if (NULL == estimate)
54 return;
55 GNUNET_break (0);
56 *estimate = 0;
57}
58
59
60/**
61 * Store an item in the datastore.
62 *
63 * @param cls closure
64 * @param key key for the item
65 * @param absent true if the key was not found in the bloom filter
66 * @param size number of bytes in data
67 * @param data content stored
68 * @param type type of the content
69 * @param priority priority of the content
70 * @param anonymity anonymity-level for the content
71 * @param replication replication-level for the content
72 * @param expiration expiration time for the content
73 * @param cont continuation called with success or failure status
74 * @param cont_cls continuation closure
75 */
76static void
77template_plugin_put (void *cls,
78 const struct GNUNET_HashCode *key,
79 bool absent,
80 uint32_t size,
81 const void *data,
82 enum GNUNET_BLOCK_Type type,
83 uint32_t priority,
84 uint32_t anonymity,
85 uint32_t replication,
86 struct GNUNET_TIME_Absolute expiration,
87 PluginPutCont cont,
88 void *cont_cls)
89{
90 GNUNET_break (0);
91 cont (cont_cls, key, size, GNUNET_SYSERR, "not implemented");
92}
93
94
95/**
96 * Get one of the results for a particular key in the datastore.
97 *
98 * @param cls closure
99 * @param next_uid return the result with lowest uid >= next_uid
100 * @param random if true, return a random result instead of using next_uid
101 * @param key maybe NULL (to match all entries)
102 * @param type entries of which type are relevant?
103 * Use 0 for any type.
104 * @param proc function to call on each matching value;
105 * will be called with NULL if nothing matches
106 * @param proc_cls closure for proc
107 */
108static void
109template_plugin_get_key (void *cls,
110 uint64_t next_uid,
111 bool random,
112 const struct GNUNET_HashCode *key,
113 enum GNUNET_BLOCK_Type type,
114 PluginDatumProcessor proc,
115 void *proc_cls)
116{
117 GNUNET_break (0);
118}
119
120
121/**
122 * Get a random item for replication. Returns a single, not expired,
123 * random item from those with the highest replication counters. The
124 * item's replication counter is decremented by one IF it was positive
125 * before. Call 'proc' with all values ZERO or NULL if the datastore
126 * is empty.
127 *
128 * @param cls closure
129 * @param proc function to call the value (once only).
130 * @param proc_cls closure for proc
131 */
132static void
133template_plugin_get_replication (void *cls, PluginDatumProcessor proc,
134 void *proc_cls)
135{
136 GNUNET_break (0);
137}
138
139
140/**
141 * Get a random item for expiration. Call 'proc' with all values ZERO
142 * or NULL if the datastore is empty.
143 *
144 * @param cls closure
145 * @param proc function to call the value (once only).
146 * @param proc_cls closure for proc
147 */
148static void
149template_plugin_get_expiration (void *cls, PluginDatumProcessor proc,
150 void *proc_cls)
151{
152 GNUNET_break (0);
153}
154
155
156/**
157 * Call the given processor on an item with zero anonymity.
158 *
159 * @param cls our "struct Plugin*"
160 * @param next_uid return the result with lowest uid >= next_uid
161 * @param type entries of which type should be considered?
162 * Must not be zero (ANY).
163 * @param proc function to call on the matching value;
164 * will be called with NULL if no value matches
165 * @param proc_cls closure for proc
166 */
167static void
168template_plugin_get_zero_anonymity (void *cls, uint64_t next_uid,
169 enum GNUNET_BLOCK_Type type,
170 PluginDatumProcessor proc, void *proc_cls)
171{
172 GNUNET_break (0);
173}
174
175
176/**
177 * Drop database.
178 */
179static void
180template_plugin_drop (void *cls)
181{
182 GNUNET_break (0);
183}
184
185
186/**
187 * Get all of the keys in the datastore.
188 *
189 * @param cls closure
190 * @param proc function to call on each key
191 * @param proc_cls closure for proc
192 */
193static void
194template_get_keys (void *cls,
195 PluginKeyProcessor proc,
196 void *proc_cls)
197{
198 proc (proc_cls, NULL, 0);
199}
200
201
202/**
203 * Remove a particular key in the datastore.
204 *
205 * @param cls closure
206 * @param key key for the content
207 * @param size number of bytes in data
208 * @param data content stored
209 * @param cont continuation called with success or failure status
210 * @param cont_cls continuation closure for @a cont
211 */
212static void
213template_plugin_remove_key (void *cls,
214 const struct GNUNET_HashCode *key,
215 uint32_t size,
216 const void *data,
217 PluginRemoveCont cont,
218 void *cont_cls)
219{
220 GNUNET_break (0);
221 cont (cont_cls, key, size, GNUNET_SYSERR, "not implemented");
222}
223
224
225/**
226 * Entry point for the plugin.
227 *
228 * @param cls the "struct GNUNET_DATASTORE_PluginEnvironment*"
229 * @return our "struct Plugin*"
230 */
231void *
232libgnunet_plugin_datastore_template_init (void *cls)
233{
234 struct GNUNET_DATASTORE_PluginEnvironment *env = cls;
235 struct GNUNET_DATASTORE_PluginFunctions *api;
236 struct Plugin *plugin;
237
238 plugin = GNUNET_new (struct Plugin);
239 plugin->env = env;
240 api = GNUNET_new (struct GNUNET_DATASTORE_PluginFunctions);
241 api->cls = plugin;
242 api->estimate_size = &template_plugin_estimate_size;
243 api->put = &template_plugin_put;
244 api->get_key = &template_plugin_get_key;
245 api->get_replication = &template_plugin_get_replication;
246 api->get_expiration = &template_plugin_get_expiration;
247 api->get_zero_anonymity = &template_plugin_get_zero_anonymity;
248 api->drop = &template_plugin_drop;
249 api->get_keys = &template_get_keys;
250 api->remove_key = &template_plugin_remove_key;
251 GNUNET_log_from (GNUNET_ERROR_TYPE_INFO, "template",
252 _ ("Template database running\n"));
253 return api;
254}
255
256
257/**
258 * Exit point from the plugin.
259 * @param cls our "struct Plugin*"
260 * @return always NULL
261 */
262void *
263libgnunet_plugin_datastore_template_done (void *cls)
264{
265 struct GNUNET_DATASTORE_PluginFunctions *api = cls;
266 struct Plugin *plugin = api->cls;
267
268 GNUNET_free (plugin);
269 GNUNET_free (api);
270 return NULL;
271}
272
273
274/* end of plugin_datastore_template.c */
diff --git a/src/plugin/datastore/test_defaults.conf b/src/plugin/datastore/test_defaults.conf
new file mode 100644
index 000000000..1e6bbeeaf
--- /dev/null
+++ b/src/plugin/datastore/test_defaults.conf
@@ -0,0 +1,10 @@
1@inline@ ../../../contrib/conf/gnunet/no_autostart_above_core.conf
2@inline@ ../../../contrib/conf/gnunet/no_forcestart.conf
3
4[datastore]
5PORT = 22654
6QUOTA = 1 MB
7START_ON_DEMAND = YES
8
9[nse]
10WORKBITS = 1
diff --git a/src/plugin/datastore/test_plugin_datastore.c b/src/plugin/datastore/test_plugin_datastore.c
new file mode 100644
index 000000000..7de1acf2d
--- /dev/null
+++ b/src/plugin/datastore/test_plugin_datastore.c
@@ -0,0 +1,478 @@
1/*
2 This file is part of GNUnet.
3 Copyright (C) 2011 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20/*
21 * @file test_plugin_datastore.c
22 * @brief Test database plugin directly, calling each API function once
23 * @author Christian Grothoff
24 */
25
26#include "platform.h"
27#include "gnunet_util_lib.h"
28#include "gnunet_protocols.h"
29#include "gnunet_datastore_plugin.h"
30#include "gnunet_testing_lib.h"
31
32/**
33 * Number of put operations to perform.
34 */
35#define PUT_10 10
36
37static unsigned long long stored_bytes;
38
39static unsigned long long stored_entries;
40
41static unsigned long long stored_ops;
42
43static const char *plugin_name;
44
45static int ok;
46
47enum RunPhase
48{
49 RP_ERROR = 0,
50 RP_PUT,
51 RP_GET,
52 RP_ITER_ZERO,
53 RP_REPL_GET,
54 RP_EXPI_GET,
55 RP_REMOVE,
56 RP_DROP
57};
58
59
60struct CpsRunContext
61{
62 const struct GNUNET_CONFIGURATION_Handle *cfg;
63 struct GNUNET_DATASTORE_PluginFunctions *api;
64 enum RunPhase phase;
65 unsigned int cnt;
66 unsigned int i;
67};
68
69
70/**
71 * Function called by plugins to notify us about a
72 * change in their disk utilization.
73 *
74 * @param cls closure (NULL)
75 * @param delta change in disk utilization,
76 * 0 for "reset to empty"
77 */
78static void
79disk_utilization_change_cb (void *cls, int delta)
80{
81 /* do nothing */
82}
83
84
85static void
86test (void *cls);
87
88
89/**
90 * Put continuation.
91 *
92 * @param cls closure
93 * @param key key for the item stored
94 * @param size size of the item stored
95 * @param status #GNUNET_OK or #GNUNET_SYSERROR
96 * @param msg error message on error
97 */
98static void
99put_continuation (void *cls,
100 const struct GNUNET_HashCode *key,
101 uint32_t size,
102 int status,
103 const char *msg)
104{
105 struct CpsRunContext *crc = cls;
106 static unsigned long long os;
107 unsigned long long cs;
108
109 if (GNUNET_OK != status)
110 {
111 fprintf (stderr,
112 "ERROR: `%s'\n",
113 msg);
114 }
115 else
116 {
117 crc->api->estimate_size (crc->api->cls,
118 &cs);
119 GNUNET_assert (os <= cs);
120 os = cs;
121 stored_bytes += size;
122 stored_ops++;
123 stored_entries++;
124 }
125 GNUNET_SCHEDULER_add_now (&test, crc);
126}
127
128
129static void
130gen_key (int i, struct GNUNET_HashCode *key)
131{
132 memset (key, 0, sizeof(struct GNUNET_HashCode));
133 key->bits[0] = (unsigned int) i;
134 GNUNET_CRYPTO_hash (key, sizeof(struct GNUNET_HashCode), key);
135}
136
137
138static void
139do_put (struct CpsRunContext *crc)
140{
141 char value[65536];
142 size_t size;
143 struct GNUNET_HashCode key;
144 unsigned int prio;
145 static int i;
146
147 if (PUT_10 == i)
148 {
149 i = 0;
150 crc->phase++;
151 GNUNET_SCHEDULER_add_now (&test, crc);
152 return;
153 }
154 /* most content is 32k */
155 size = 32 * 1024;
156
157 if ((0 != i) && (GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 16) ==
158 0) ) /* but some of it is less! */
159 size = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 32 * 1024);
160 size = size - (size & 7); /* always multiple of 8 */
161
162 /* generate random key */
163 gen_key (i, &key);
164 memset (value, i, size);
165 if (i > 255)
166 memset (value, i - 255, size / 2);
167 value[0] = crc->i;
168 prio = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 100);
169 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
170 "putting type %u, anon %u under key %s\n", i + 1, i,
171 GNUNET_h2s (&key));
172 crc->api->put (crc->api->cls,
173 &key,
174 false /* absent */,
175 size,
176 value, i + 1 /* type */,
177 prio,
178 i /* anonymity */,
179 0 /* replication */,
180 GNUNET_TIME_relative_to_absolute
181 (GNUNET_TIME_relative_multiply
182 (GNUNET_TIME_UNIT_MILLISECONDS,
183 60 * 60 * 60 * 1000
184 + GNUNET_CRYPTO_random_u32
185 (GNUNET_CRYPTO_QUALITY_WEAK, 1000))),
186 put_continuation,
187 crc);
188 i++;
189}
190
191
192static uint64_t guid;
193
194
195static int
196iterate_one_shot (void *cls,
197 const struct GNUNET_HashCode *key,
198 uint32_t size,
199 const void *data,
200 enum GNUNET_BLOCK_Type type,
201 uint32_t priority,
202 uint32_t anonymity,
203 uint32_t replication,
204 struct GNUNET_TIME_Absolute expiration,
205 uint64_t uid)
206{
207 struct CpsRunContext *crc = cls;
208
209 GNUNET_assert (NULL != key);
210 guid = uid;
211 crc->phase++;
212 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
213 "Found result type=%u, priority=%u, size=%u, expire=%s, key %s\n",
214 (unsigned int) type,
215 (unsigned int) priority,
216 (unsigned int) size,
217 GNUNET_STRINGS_absolute_time_to_string (expiration),
218 GNUNET_h2s (key));
219 GNUNET_SCHEDULER_add_now (&test,
220 crc);
221 return GNUNET_OK;
222}
223
224
225static void
226remove_continuation (void *cls,
227 const struct GNUNET_HashCode *key,
228 uint32_t size,
229 int status,
230 const char *msg)
231{
232 struct CpsRunContext *crc = cls;
233
234 GNUNET_assert (NULL != key);
235 GNUNET_assert (32768 == size);
236 GNUNET_assert (GNUNET_OK == status);
237 GNUNET_assert (NULL == msg);
238 crc->phase++;
239 GNUNET_SCHEDULER_add_now (&test,
240 crc);
241}
242
243
244/**
245 * Function called when the service shuts
246 * down. Unloads our datastore plugin.
247 *
248 * @param api api to unload
249 * @param cfg configuration to use
250 */
251static void
252unload_plugin (struct GNUNET_DATASTORE_PluginFunctions *api,
253 const struct GNUNET_CONFIGURATION_Handle *cfg)
254{
255 char *name;
256 char *libname;
257
258 if (GNUNET_OK !=
259 GNUNET_CONFIGURATION_get_value_string (cfg,
260 "DATASTORE",
261 "DATABASE",
262 &name))
263 {
264 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
265 _ ("No `%s' specified for `%s' in configuration!\n"),
266 "DATABASE",
267 "DATASTORE");
268 return;
269 }
270 GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", name);
271 GNUNET_break (NULL == GNUNET_PLUGIN_unload (libname, api));
272 GNUNET_free (libname);
273 GNUNET_free (name);
274}
275
276
277/**
278 * Last task run during shutdown. Disconnects us from
279 * the transport and core.
280 */
281static void
282cleaning_task (void *cls)
283{
284 struct CpsRunContext *crc = cls;
285
286 unload_plugin (crc->api, crc->cfg);
287 GNUNET_free (crc);
288}
289
290
291static void
292test (void *cls)
293{
294 struct CpsRunContext *crc = cls;
295 struct GNUNET_HashCode key;
296
297 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
298 "In phase %d, iteration %u\n", crc->phase, crc->cnt);
299 switch (crc->phase)
300 {
301 case RP_ERROR:
302 ok = 1;
303 GNUNET_break (0);
304 crc->api->drop (crc->api->cls);
305 GNUNET_SCHEDULER_add_now (&cleaning_task, crc);
306 break;
307
308 case RP_PUT:
309 do_put (crc);
310 break;
311
312 case RP_GET:
313 if (crc->cnt == 1)
314 {
315 crc->cnt = 0;
316 crc->phase++;
317 GNUNET_SCHEDULER_add_now (&test, crc);
318 break;
319 }
320 gen_key (5, &key);
321 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
322 "Looking for %s\n",
323 GNUNET_h2s (&key));
324 crc->api->get_key (crc->api->cls,
325 0,
326 false,
327 &key,
328 GNUNET_BLOCK_TYPE_ANY,
329 &iterate_one_shot,
330 crc);
331 break;
332
333 case RP_ITER_ZERO:
334 if (crc->cnt == 1)
335 {
336 crc->cnt = 0;
337 crc->phase++;
338 GNUNET_SCHEDULER_add_now (&test, crc);
339 break;
340 }
341 crc->api->get_zero_anonymity (crc->api->cls, 0, 1, &iterate_one_shot, crc);
342 break;
343
344 case RP_REPL_GET:
345 crc->api->get_replication (crc->api->cls, &iterate_one_shot, crc);
346 break;
347
348 case RP_EXPI_GET:
349 crc->api->get_expiration (crc->api->cls, &iterate_one_shot, crc);
350 break;
351
352 case RP_REMOVE:
353 {
354 struct GNUNET_HashCode key;
355 uint32_t size = 32768;
356 char value[size];
357
358 gen_key (0, &key);
359 memset (value, 0, size);
360 value[0] = crc->i;
361 crc->api->remove_key (crc->api->cls,
362 &key,
363 size,
364 value,
365 &remove_continuation,
366 crc);
367 break;
368 }
369
370 case RP_DROP:
371 crc->api->drop (crc->api->cls);
372 GNUNET_SCHEDULER_add_now (&cleaning_task, crc);
373 break;
374 }
375}
376
377
378/**
379 * Load the datastore plugin.
380 */
381static struct GNUNET_DATASTORE_PluginFunctions *
382load_plugin (const struct GNUNET_CONFIGURATION_Handle *cfg)
383{
384 static struct GNUNET_DATASTORE_PluginEnvironment env;
385 struct GNUNET_DATASTORE_PluginFunctions *ret;
386 char *name;
387 char *libname;
388
389 if (GNUNET_OK !=
390 GNUNET_CONFIGURATION_get_value_string (cfg,
391 "DATASTORE",
392 "DATABASE",
393 &name))
394 {
395 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
396 _ ("No `%s' specified for `%s' in configuration!\n"),
397 "DATABASE",
398 "DATASTORE");
399 return NULL;
400 }
401 env.cfg = cfg;
402 env.duc = &disk_utilization_change_cb;
403 env.cls = NULL;
404 GNUNET_log (GNUNET_ERROR_TYPE_INFO, _ ("Loading `%s' datastore plugin\n"),
405 name);
406 GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", name);
407 if (NULL == (ret = GNUNET_PLUGIN_load (libname, &env)))
408 {
409 fprintf (stderr, "Failed to load plugin `%s'!\n", name);
410 GNUNET_free (libname);
411 GNUNET_free (name);
412 ok = 77; /* mark test as skipped */
413 return NULL;
414 }
415 GNUNET_free (libname);
416 GNUNET_free (name);
417 return ret;
418}
419
420
421static void
422run (void *cls, char *const *args, const char *cfgfile,
423 const struct GNUNET_CONFIGURATION_Handle *c)
424{
425 struct GNUNET_DATASTORE_PluginFunctions *api;
426 struct CpsRunContext *crc;
427
428 api = load_plugin (c);
429 if (api == NULL)
430 {
431 fprintf (stderr,
432 "%s",
433 "Could not initialize plugin, assuming database not configured. Test not run!\n");
434 return;
435 }
436 crc = GNUNET_new (struct CpsRunContext);
437 crc->api = api;
438 crc->cfg = c;
439 crc->phase = RP_PUT;
440 GNUNET_SCHEDULER_add_now (&test, crc);
441}
442
443
444int
445main (int argc, char *argv[])
446{
447 char dir_name[PATH_MAX];
448 char cfg_name[PATH_MAX];
449 char *const xargv[] = {
450 "test-plugin-datastore",
451 "-c",
452 cfg_name,
453 NULL
454 };
455 static struct GNUNET_GETOPT_CommandLineOption options[] = {
456 GNUNET_GETOPT_OPTION_END
457 };
458
459 /* determine name of plugin to use */
460 plugin_name = GNUNET_STRINGS_get_suffix_from_binary_name (argv[0]);
461 GNUNET_snprintf (dir_name, sizeof(dir_name),
462 "/tmp/test-gnunet-datastore-plugin-%s", plugin_name);
463 GNUNET_DISK_directory_remove (dir_name);
464 GNUNET_log_setup ("test-plugin-datastore",
465 "WARNING",
466 NULL);
467 GNUNET_snprintf (cfg_name, sizeof(cfg_name),
468 "test_plugin_datastore_data_%s.conf", plugin_name);
469 GNUNET_PROGRAM_run ((sizeof(xargv) / sizeof(char *)) - 1, xargv,
470 "test-plugin-datastore", "nohelp", options, &run, NULL);
471 if ((0 != ok) && (77 != ok))
472 fprintf (stderr, "Missed some testcases: %u\n", ok);
473 GNUNET_DISK_directory_remove (dir_name);
474 return ok;
475}
476
477
478/* end of test_plugin_datastore.c */
diff --git a/src/plugin/datastore/test_plugin_datastore_data_heap.conf b/src/plugin/datastore/test_plugin_datastore_data_heap.conf
new file mode 100644
index 000000000..b1ea8ff67
--- /dev/null
+++ b/src/plugin/datastore/test_plugin_datastore_data_heap.conf
@@ -0,0 +1,6 @@
1@INLINE@ test_defaults.conf
2[PATHS]
3GNUNET_TEST_HOME = $GNUNET_TMP/test-gnunet-datastore-plugin-heap/
4
5[datastore]
6DATABASE = heap
diff --git a/src/plugin/datastore/test_plugin_datastore_data_postgres.conf b/src/plugin/datastore/test_plugin_datastore_data_postgres.conf
new file mode 100644
index 000000000..d0e29437f
--- /dev/null
+++ b/src/plugin/datastore/test_plugin_datastore_data_postgres.conf
@@ -0,0 +1,10 @@
1@INLINE@ test_defaults.conf
2[PATHS]
3GNUNET_TEST_HOME = $GNUNET_TMP/test-gnunet-datastore-plugin-postgres/
4
5[datastore]
6DATABASE = postgres
7
8[datastore-postgres]
9CONFIG = dbname=gnunetcheck
10
diff --git a/src/plugin/datastore/test_plugin_datastore_data_sqlite.conf b/src/plugin/datastore/test_plugin_datastore_data_sqlite.conf
new file mode 100644
index 000000000..ca837c77a
--- /dev/null
+++ b/src/plugin/datastore/test_plugin_datastore_data_sqlite.conf
@@ -0,0 +1,4 @@
1@INLINE@ test_defaults.conf
2[PATHS]
3GNUNET_TEST_HOME = $GNUNET_TMP/test-gnunet-datastore-plugin-sqlite/
4
generated by cgit v1.2.3 (git 2.39.1) at 2024-05-20 07:33:58 +0000