aboutsummaryrefslogtreecommitdiff
path: root/src/datastore
diff options
context:
space:
mode:
Diffstat (limited to 'src/datastore')
-rw-r--r--src/datastore/.gitignore16
-rw-r--r--src/datastore/Makefile.am328
-rw-r--r--src/datastore/datastore-0001.sql49
-rw-r--r--src/datastore/datastore-drop.sql25
-rw-r--r--src/datastore/datastore.conf.in33
-rw-r--r--src/datastore/datastore.h257
-rw-r--r--src/datastore/datastore_api.c1393
-rw-r--r--src/datastore/gnunet-datastore.c508
-rw-r--r--src/datastore/gnunet-service-datastore.c1647
-rw-r--r--src/datastore/perf_datastore_api.c629
-rw-r--r--src/datastore/perf_plugin_datastore.c573
-rw-r--r--src/datastore/perf_plugin_datastore_data_heap.conf7
-rw-r--r--src/datastore/perf_plugin_datastore_data_mysql.conf10
-rw-r--r--src/datastore/perf_plugin_datastore_data_postgres.conf10
-rw-r--r--src/datastore/perf_plugin_datastore_data_sqlite.conf4
-rw-r--r--src/datastore/plugin_datastore_heap.c944
-rw-r--r--src/datastore/plugin_datastore_mysql.c1204
-rw-r--r--src/datastore/plugin_datastore_postgres.c930
-rw-r--r--src/datastore/plugin_datastore_sqlite.c1375
-rw-r--r--src/datastore/plugin_datastore_template.c274
-rw-r--r--src/datastore/selectrandom.sql9
-rw-r--r--src/datastore/test_datastore_api.c732
-rw-r--r--src/datastore/test_datastore_api_data_heap.conf19
-rw-r--r--src/datastore/test_datastore_api_data_mysql.conf10
-rw-r--r--src/datastore/test_datastore_api_data_postgres.conf10
-rw-r--r--src/datastore/test_datastore_api_data_sqlite.conf7
-rw-r--r--src/datastore/test_datastore_api_management.c408
-rw-r--r--src/datastore/test_defaults.conf10
-rw-r--r--src/datastore/test_plugin_datastore.c478
-rw-r--r--src/datastore/test_plugin_datastore_data_heap.conf6
-rw-r--r--src/datastore/test_plugin_datastore_data_mysql.conf9
-rw-r--r--src/datastore/test_plugin_datastore_data_postgres.conf10
-rw-r--r--src/datastore/test_plugin_datastore_data_sqlite.conf4
33 files changed, 0 insertions, 11928 deletions
diff --git a/src/datastore/.gitignore b/src/datastore/.gitignore
deleted file mode 100644
index 51d3391b9..000000000
--- a/src/datastore/.gitignore
+++ /dev/null
@@ -1,16 +0,0 @@
1gnunet-service-datastore
2gnunet-datastore
3perf_datastore_api_heap
4perf_plugin_datastore_heap
5test_datastore_api_heap
6test_datastore_api_management_heap
7test_datastore_api_management_mysql
8test_datastore_api_management_postgres
9test_datastore_api_management_sqlite
10test_datastore_api_mysql
11test_datastore_api_postgres
12test_datastore_api_sqlite
13test_plugin_datastore_heap
14test_plugin_datastore_mysql
15test_plugin_datastore_postgres
16test_plugin_datastore_sqlite
diff --git a/src/datastore/Makefile.am b/src/datastore/Makefile.am
deleted file mode 100644
index b73a0497b..000000000
--- a/src/datastore/Makefile.am
+++ /dev/null
@@ -1,328 +0,0 @@
1# This Makefile.am is in the public domain
2AM_CPPFLAGS = -I$(top_srcdir)/src/include
3
4plugindir = $(libdir)/gnunet
5
6pkgcfgdir= $(pkgdatadir)/config.d/
7
8libexecdir= $(pkglibdir)/libexec/
9
10pkgcfg_DATA = \
11 datastore.conf
12
13sqldir = $(prefix)/share/gnunet/sql/
14
15sql_DATA = \
16 datastore-0001.sql \
17 datastore-drop.sql
18
19if USE_COVERAGE
20 AM_CFLAGS = --coverage -O0
21 XLIBS = -lgcov
22endif
23
24
25lib_LTLIBRARIES = \
26 libgnunetdatastore.la
27
28libgnunetdatastore_la_SOURCES = \
29 datastore_api.c datastore.h
30libgnunetdatastore_la_LIBADD = \
31 $(top_builddir)/src/statistics/libgnunetstatistics.la \
32 $(top_builddir)/src/util/libgnunetutil.la \
33 $(GN_LIBINTL)
34libgnunetdatastore_la_LDFLAGS = \
35 $(GN_LIB_LDFLAGS) \
36 -version-info 1:0:0
37
38bin_PROGRAMS = \
39 gnunet-datastore
40
41libexec_PROGRAMS = \
42 gnunet-service-datastore
43
44gnunet_service_datastore_SOURCES = \
45 gnunet-service-datastore.c
46gnunet_service_datastore_LDADD = \
47 $(top_builddir)/src/statistics/libgnunetstatistics.la \
48 $(top_builddir)/src/util/libgnunetutil.la \
49 $(GN_LIBINTL)
50
51gnunet_datastore_SOURCES = \
52 gnunet-datastore.c
53gnunet_datastore_LDADD = \
54 libgnunetdatastore.la \
55 $(top_builddir)/src/util/libgnunetutil.la \
56 $(GN_LIBINTL)
57
58
59if HAVE_MYSQL
60 MYSQL_PLUGIN = libgnunet_plugin_datastore_mysql.la
61if HAVE_BENCHMARKS
62 MYSQL_BENCHMARKS = \
63 perf_datastore_api_mysql \
64 perf_plugin_datastore_mysql
65endif
66 MYSQL_TESTS = \
67 test_datastore_api_mysql \
68 test_datastore_api_management_mysql \
69 test_plugin_datastore_mysql \
70 $(MYSQL_BENCHMARKS)
71endif
72if HAVE_SQLITE
73 SQLITE_PLUGIN = libgnunet_plugin_datastore_sqlite.la
74if HAVE_BENCHMARKS
75 SQLITE_BENCHMARKS = \
76 perf_datastore_api_sqlite \
77 perf_plugin_datastore_sqlite
78endif
79 SQLITE_TESTS = \
80 test_datastore_api_sqlite \
81 test_datastore_api_management_sqlite \
82 test_plugin_datastore_sqlite \
83 $(SQLITE_BENCHMARKS)
84endif
85if HAVE_POSTGRESQL
86 POSTGRES_PLUGIN = libgnunet_plugin_datastore_postgres.la
87if HAVE_BENCHMARKS
88 POSTGRES_BENCHMARKS = \
89 perf_datastore_api_postgres \
90 perf_plugin_datastore_postgres
91endif
92 POSTGRES_TESTS = \
93 test_datastore_api_postgres \
94 test_datastore_api_management_postgres \
95 test_plugin_datastore_postgres \
96 $(POSTGRES_BENCHMARKS)
97endif
98
99plugin_LTLIBRARIES = \
100 $(SQLITE_PLUGIN) \
101 $(MYSQL_PLUGIN) \
102 $(POSTGRES_PLUGIN) \
103 libgnunet_plugin_datastore_heap.la
104
105# Real plugins should of course go into
106# plugin_LTLIBRARIES
107noinst_LTLIBRARIES = \
108 libgnunet_plugin_datastore_template.la
109
110
111libgnunet_plugin_datastore_sqlite_la_SOURCES = \
112 plugin_datastore_sqlite.c
113libgnunet_plugin_datastore_sqlite_la_LIBADD = \
114 $(top_builddir)/src/sq/libgnunetsq.la \
115 $(top_builddir)/src/statistics/libgnunetstatistics.la \
116 $(top_builddir)/src/util/libgnunetutil.la $(XLIBS) -lsqlite3 \
117 $(LTLIBINTL)
118libgnunet_plugin_datastore_sqlite_la_LDFLAGS = \
119 $(GN_PLUGIN_LDFLAGS)
120
121
122libgnunet_plugin_datastore_heap_la_SOURCES = \
123 plugin_datastore_heap.c
124libgnunet_plugin_datastore_heap_la_LIBADD = \
125 $(top_builddir)/src/util/libgnunetutil.la $(XLIBS) \
126 $(LTLIBINTL)
127libgnunet_plugin_datastore_heap_la_LDFLAGS = \
128 $(GN_PLUGIN_LDFLAGS)
129
130
131libgnunet_plugin_datastore_mysql_la_SOURCES = \
132 plugin_datastore_mysql.c
133libgnunet_plugin_datastore_mysql_la_LIBADD = \
134 $(top_builddir)/src/my/libgnunetmy.la \
135 $(top_builddir)/src/mysql/libgnunetmysql.la \
136 $(top_builddir)/src/statistics/libgnunetstatistics.la \
137 $(top_builddir)/src/util/libgnunetutil.la $(XLIBS) $(Z_LIBS) -lmysqlclient
138libgnunet_plugin_datastore_mysql_la_LDFLAGS = \
139 $(GN_PLUGIN_LDFLAGS) $(MYSQL_LDFLAGS) -lmysqlclient
140libgnunet_plugin_datastore_mysql_la_CPPFLAGS = \
141 $(MYSQL_CPPFLAGS) $(AM_CPPFLAGS)
142
143libgnunet_plugin_datastore_postgres_la_SOURCES = \
144 plugin_datastore_postgres.c
145libgnunet_plugin_datastore_postgres_la_LIBADD = \
146 $(top_builddir)/src/statistics/libgnunetstatistics.la \
147 $(top_builddir)/src/pq/libgnunetpq.la \
148 $(top_builddir)/src/util/libgnunetutil.la $(XLIBS) -lpq
149libgnunet_plugin_datastore_postgres_la_LDFLAGS = \
150 $(GN_PLUGIN_LDFLAGS) $(POSTGRESQL_LDFLAGS)
151libgnunet_plugin_datastore_postgres_la_CPPFLAGS = \
152 $(POSTGRESQL_CPPFLAGS) $(AM_CPPFLAGS)
153
154
155libgnunet_plugin_datastore_template_la_SOURCES = \
156 plugin_datastore_template.c
157libgnunet_plugin_datastore_template_la_LIBADD = \
158 $(top_builddir)/src/util/libgnunetutil.la $(XLIBS) \
159 $(LTLIBINTL)
160libgnunet_plugin_datastore_template_la_LDFLAGS = \
161 $(GN_PLUGIN_LDFLAGS)
162
163check_PROGRAMS = \
164 test_datastore_api_heap \
165 test_datastore_api_management_heap \
166 perf_datastore_api_heap \
167 perf_plugin_datastore_heap \
168 test_plugin_datastore_heap \
169 $(SQLITE_TESTS) \
170 $(MYSQL_TESTS) \
171 $(POSTGRES_TESTS)
172
173if ENABLE_TEST_RUN
174AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
175TESTS = $(check_PROGRAMS)
176endif
177
178test_datastore_api_heap_SOURCES = \
179 test_datastore_api.c
180test_datastore_api_heap_LDADD = \
181 $(top_builddir)/src/testing/libgnunettesting.la \
182 libgnunetdatastore.la \
183 $(top_builddir)/src/util/libgnunetutil.la
184
185test_datastore_api_management_heap_SOURCES = \
186 test_datastore_api_management.c
187test_datastore_api_management_heap_LDADD = \
188 $(top_builddir)/src/testing/libgnunettesting.la \
189 libgnunetdatastore.la \
190 $(top_builddir)/src/util/libgnunetutil.la
191
192perf_datastore_api_heap_SOURCES = \
193 perf_datastore_api.c
194perf_datastore_api_heap_LDADD = \
195 $(top_builddir)/src/testing/libgnunettesting.la \
196 libgnunetdatastore.la \
197 $(top_builddir)/src/util/libgnunetutil.la
198
199perf_plugin_datastore_heap_SOURCES = \
200 perf_plugin_datastore.c
201perf_plugin_datastore_heap_LDADD = \
202 $(top_builddir)/src/testing/libgnunettesting.la \
203 $(top_builddir)/src/util/libgnunetutil.la
204
205test_plugin_datastore_heap_SOURCES = \
206 test_plugin_datastore.c
207test_plugin_datastore_heap_LDADD = \
208 $(top_builddir)/src/testing/libgnunettesting.la \
209 $(top_builddir)/src/util/libgnunetutil.la
210
211
212test_datastore_api_sqlite_SOURCES = \
213 test_datastore_api.c
214test_datastore_api_sqlite_LDADD = \
215 $(top_builddir)/src/testing/libgnunettesting.la \
216 libgnunetdatastore.la \
217 $(top_builddir)/src/util/libgnunetutil.la
218
219test_datastore_api_management_sqlite_SOURCES = \
220 test_datastore_api_management.c
221test_datastore_api_management_sqlite_LDADD = \
222 $(top_builddir)/src/testing/libgnunettesting.la \
223 libgnunetdatastore.la \
224 $(top_builddir)/src/util/libgnunetutil.la
225
226perf_datastore_api_sqlite_SOURCES = \
227 perf_datastore_api.c
228perf_datastore_api_sqlite_LDADD = \
229 $(top_builddir)/src/testing/libgnunettesting.la \
230 libgnunetdatastore.la \
231 $(top_builddir)/src/util/libgnunetutil.la
232
233perf_plugin_datastore_sqlite_SOURCES = \
234 perf_plugin_datastore.c
235perf_plugin_datastore_sqlite_LDADD = \
236 $(top_builddir)/src/testing/libgnunettesting.la \
237 $(top_builddir)/src/util/libgnunetutil.la
238
239test_plugin_datastore_sqlite_SOURCES = \
240 test_plugin_datastore.c
241test_plugin_datastore_sqlite_LDADD = \
242 $(top_builddir)/src/testing/libgnunettesting.la \
243 $(top_builddir)/src/util/libgnunetutil.la
244
245
246test_datastore_api_mysql_SOURCES = \
247 test_datastore_api.c
248test_datastore_api_mysql_LDADD = \
249 $(top_builddir)/src/testing/libgnunettesting.la \
250 libgnunetdatastore.la \
251 $(top_builddir)/src/util/libgnunetutil.la
252
253test_datastore_api_management_mysql_SOURCES = \
254 test_datastore_api_management.c
255test_datastore_api_management_mysql_LDADD = \
256 $(top_builddir)/src/testing/libgnunettesting.la \
257 libgnunetdatastore.la \
258 $(top_builddir)/src/util/libgnunetutil.la
259
260perf_datastore_api_mysql_SOURCES = \
261 perf_datastore_api.c
262perf_datastore_api_mysql_LDADD = \
263 $(top_builddir)/src/testing/libgnunettesting.la \
264 libgnunetdatastore.la \
265 $(top_builddir)/src/util/libgnunetutil.la
266
267test_plugin_datastore_mysql_SOURCES = \
268 test_plugin_datastore.c
269test_plugin_datastore_mysql_LDADD = \
270 $(top_builddir)/src/testing/libgnunettesting.la \
271 $(top_builddir)/src/util/libgnunetutil.la
272
273perf_plugin_datastore_mysql_SOURCES = \
274 perf_plugin_datastore.c
275perf_plugin_datastore_mysql_LDADD = \
276 $(top_builddir)/src/testing/libgnunettesting.la \
277 $(top_builddir)/src/util/libgnunetutil.la
278
279
280test_datastore_api_postgres_SOURCES = \
281 test_datastore_api.c
282test_datastore_api_postgres_LDADD = \
283 $(top_builddir)/src/testing/libgnunettesting.la \
284 libgnunetdatastore.la \
285 $(top_builddir)/src/util/libgnunetutil.la
286
287test_datastore_api_management_postgres_SOURCES = \
288 test_datastore_api_management.c
289test_datastore_api_management_postgres_LDADD = \
290 $(top_builddir)/src/testing/libgnunettesting.la \
291 libgnunetdatastore.la \
292 $(top_builddir)/src/util/libgnunetutil.la
293
294perf_datastore_api_postgres_SOURCES = \
295 perf_datastore_api.c
296perf_datastore_api_postgres_LDADD = \
297 $(top_builddir)/src/testing/libgnunettesting.la \
298 libgnunetdatastore.la \
299 $(top_builddir)/src/util/libgnunetutil.la
300
301test_plugin_datastore_postgres_SOURCES = \
302 test_plugin_datastore.c
303test_plugin_datastore_postgres_LDADD = \
304 $(top_builddir)/src/testing/libgnunettesting.la \
305 $(top_builddir)/src/util/libgnunetutil.la
306
307perf_plugin_datastore_postgres_SOURCES = \
308 perf_plugin_datastore.c
309perf_plugin_datastore_postgres_LDADD = \
310 $(top_builddir)/src/testing/libgnunettesting.la \
311 $(top_builddir)/src/util/libgnunetutil.la
312
313
314EXTRA_DIST = \
315 test_defaults.conf \
316 test_datastore_api_data_sqlite.conf \
317 perf_plugin_datastore_data_sqlite.conf \
318 test_plugin_datastore_data_sqlite.conf \
319 test_datastore_api_data_heap.conf \
320 perf_plugin_datastore_data_heap.conf \
321 test_plugin_datastore_data_heap.conf \
322 test_datastore_api_data_mysql.conf \
323 perf_plugin_datastore_data_mysql.conf \
324 test_plugin_datastore_data_mysql.conf \
325 test_datastore_api_data_postgres.conf \
326 perf_plugin_datastore_data_postgres.conf \
327 test_plugin_datastore_data_postgres.conf \
328 $(sql_DATA)
diff --git a/src/datastore/datastore-0001.sql b/src/datastore/datastore-0001.sql
deleted file mode 100644
index 0d4758be2..000000000
--- a/src/datastore/datastore-0001.sql
+++ /dev/null
@@ -1,49 +0,0 @@
1--
2-- This file is part of GNUnet
3-- Copyright (C) 2014--2022 GNUnet e.V.
4--
5-- GNUnet is free software; you can redistribute it and/or modify it under the
6-- terms of the GNU General Public License as published by the Free Software
7-- Foundation; either version 3, or (at your option) any later version.
8--
9-- GNUnet is distributed in the hope that it will be useful, but WITHOUT ANY
10-- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
11-- A PARTICULAR PURPOSE. See the GNU General Public License for more details.
12--
13-- You should have received a copy of the GNU General Public License along with
14-- GNUnet; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
15--
16
17-- Everything in one big transaction
18BEGIN;
19
20-- Check patch versioning is in place.
21SELECT _v.register_patch('datastore-0001', NULL, NULL);
22
23-------------------- Schema ----------------------------
24
25CREATE SCHEMA datastore;
26COMMENT ON SCHEMA datastore IS 'gnunet-datastore data';
27
28SET search_path TO datastore;
29
30CREATE TABLE IF NOT EXISTS gn090 (
31 repl INTEGER NOT NULL DEFAULT 0,
32 type INTEGER NOT NULL DEFAULT 0,
33 prio INTEGER NOT NULL DEFAULT 0,
34 anonLevel INTEGER NOT NULL DEFAULT 0,
35 expire BIGINT NOT NULL DEFAULT 0,
36 rvalue BIGINT NOT NULL DEFAULT 0,
37 hash BYTEA NOT NULL DEFAULT '',
38 vhash BYTEA NOT NULL DEFAULT '',
39 value BYTEA NOT NULL DEFAULT '',
40 oid BIGINT GENERATED BY DEFAULT AS IDENTITY);
41
42CREATE INDEX IF NOT EXISTS oid_hash ON gn090 (oid);
43CREATE INDEX IF NOT EXISTS idx_hash ON gn090 (hash);
44CREATE INDEX IF NOT EXISTS idx_prio_anon ON gn090 (prio,anonLevel);
45CREATE INDEX IF NOT EXISTS idx_prio_hash_anon ON gn090 (prio,hash,anonLevel);
46CREATE INDEX IF NOT EXISTS idx_repl_rvalue ON gn090 (repl,rvalue);
47CREATE INDEX IF NOT EXISTS idx_expire_hash ON gn090 (expire,hash);
48
49COMMIT;
diff --git a/src/datastore/datastore-drop.sql b/src/datastore/datastore-drop.sql
deleted file mode 100644
index 67fee303d..000000000
--- a/src/datastore/datastore-drop.sql
+++ /dev/null
@@ -1,25 +0,0 @@
1--
2-- This file is part of GNUnet
3-- Copyright (C) 2014--2022 GNUnet e.V.
4--
5-- GNUnet is free software; you can redistribute it and/or modify it under the
6-- terms of the GNU General Public License as published by the Free Software
7-- Foundation; either version 3, or (at your option) any later version.
8--
9-- GNUnet is distributed in the hope that it will be useful, but WITHOUT ANY
10-- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
11-- A PARTICULAR PURPOSE. See the GNU General Public License for more details.
12--
13-- You should have received a copy of the GNU General Public License along with
14-- GNUnet; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
15--
16
17-- Everything in one big transaction
18BEGIN;
19
20
21SELECT _v.unregister_patch('datastore-0001');
22
23DROP SCHEMA datastore CASCADE;
24
25COMMIT;
diff --git a/src/datastore/datastore.conf.in b/src/datastore/datastore.conf.in
deleted file mode 100644
index bcd495c8f..000000000
--- a/src/datastore/datastore.conf.in
+++ /dev/null
@@ -1,33 +0,0 @@
1[datastore]
2START_ON_DEMAND = @START_ON_DEMAND@
3UNIXPATH = $GNUNET_RUNTIME_DIR/gnunet-service-datastore.sock
4UNIX_MATCH_UID = NO
5UNIX_MATCH_GID = YES
6@UNIXONLY@ PORT = 2093
7HOSTNAME = localhost
8BINARY = gnunet-service-datastore
9ACCEPT_FROM = 127.0.0.1;
10ACCEPT_FROM6 = ::1;
11QUOTA = 5 GB
12BLOOMFILTER = $GNUNET_DATA_HOME/datastore/bloomfilter
13DATABASE = sqlite
14# DISABLE_SOCKET_FORWARDING = NO
15
16[datastore-sqlite]
17FILENAME = $GNUNET_DATA_HOME/datastore/sqlite.db
18
19[datastore-postgres]
20CONFIG = postgres:///gnunet
21SQL_DIR = ${DATADIR}/sql/
22
23[datastore-mysql]
24DATABASE = gnunet
25CONFIG = ~/.my.cnf
26# USER = gnunet
27# PASSWORD =
28# HOST = localhost
29# PORT = 3306
30
31
32[datastore-heap]
33HASHMAPSIZE = 1024
diff --git a/src/datastore/datastore.h b/src/datastore/datastore.h
deleted file mode 100644
index 7af926617..000000000
--- a/src/datastore/datastore.h
+++ /dev/null
@@ -1,257 +0,0 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2004, 2005, 2006, 2007, 2009 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file datastore/datastore.h
23 * @brief structs for communication between datastore service and API
24 * @author Christian Grothoff
25 */
26
27#ifndef DATASTORE_H
28#define DATASTORE_H
29
30
31#include "gnunet_util_lib.h"
32
33GNUNET_NETWORK_STRUCT_BEGIN
34
35/**
36 * Message from datastore service informing client about
37 * the current size of the datastore.
38 */
39struct ReserveMessage
40{
41 /**
42 * Type is GNUNET_MESSAGE_TYPE_DATASTORE_RESERVE.
43 */
44 struct GNUNET_MessageHeader header;
45
46 /**
47 * Number of items to reserve.
48 */
49 uint32_t entries GNUNET_PACKED;
50
51 /**
52 * Number of bytes to reserve.
53 */
54 uint64_t amount GNUNET_PACKED;
55};
56
57
58/**
59 * Message from datastore service informing client about
60 * the success or failure of a requested operation.
61 * This header is optionally followed by a variable-size,
62 * 0-terminated error message.
63 */
64struct StatusMessage
65{
66 /**
67 * Type is GNUNET_MESSAGE_TYPE_DATASTORE_STATUS.
68 */
69 struct GNUNET_MessageHeader header;
70
71 /**
72 * Status code, -1 for errors.
73 */
74 int32_t status GNUNET_PACKED;
75
76 /**
77 * Minimum expiration time required for content to be stored
78 * by the datacache at this time, zero for unknown or no limit.
79 */
80 struct GNUNET_TIME_AbsoluteNBO min_expiration;
81};
82
83
84/**
85 * Message from datastore client informing service that
86 * the remainder of the reserved bytes can now be released
87 * for other requests.
88 */
89struct ReleaseReserveMessage
90{
91 /**
92 * Type is GNUNET_MESSAGE_TYPE_DATASTORE_RELEASE_RESERVE.
93 */
94 struct GNUNET_MessageHeader header;
95
96 /**
97 * Reservation id.
98 */
99 int32_t rid GNUNET_PACKED;
100};
101
102
103/**
104 * Message to the datastore service asking about specific
105 * content.
106 */
107struct GetKeyMessage
108{
109 /**
110 * Type is #GNUNET_MESSAGE_TYPE_DATASTORE_GET_KEY.
111 */
112 struct GNUNET_MessageHeader header;
113
114 /**
115 * Desired content type. (actually an enum GNUNET_BLOCK_Type)
116 */
117 uint32_t type GNUNET_PACKED;
118
119 /**
120 * UID at which to start the search
121 */
122 uint64_t next_uid GNUNET_PACKED;
123
124 /**
125 * If true return a random result
126 */
127 uint32_t random GNUNET_PACKED;
128
129 /**
130 * Desired key.
131 */
132 struct GNUNET_HashCode key;
133};
134
135
136/**
137 * Message to the datastore service asking about specific
138 * content.
139 */
140struct GetMessage
141{
142 /**
143 * Type is #GNUNET_MESSAGE_TYPE_DATASTORE_GET.
144 */
145 struct GNUNET_MessageHeader header;
146
147 /**
148 * Desired content type. (actually an enum GNUNET_BLOCK_Type)
149 */
150 uint32_t type GNUNET_PACKED;
151
152 /**
153 * UID at which to start the search
154 */
155 uint64_t next_uid GNUNET_PACKED;
156
157 /**
158 * If true return a random result
159 */
160 uint32_t random GNUNET_PACKED;
161};
162
163
164/**
165 * Message to the datastore service asking about zero
166 * anonymity content.
167 */
168struct GetZeroAnonymityMessage
169{
170 /**
171 * Type is GNUNET_MESSAGE_TYPE_DATASTORE_GET_ZERO_ANONYMITY.
172 */
173 struct GNUNET_MessageHeader header;
174
175 /**
176 * Desired content type (actually an enum GNUNET_BLOCK_Type)
177 */
178 uint32_t type GNUNET_PACKED;
179
180 /**
181 * UID at which to start the search
182 */
183 uint64_t next_uid GNUNET_PACKED;
184};
185
186
187/**
188 * Message transmitting content from or to the datastore
189 * service.
190 */
191struct DataMessage
192{
193 /**
194 * Type is either GNUNET_MESSAGE_TYPE_DATASTORE_PUT,
195 * GNUNET_MESSAGE_TYPE_DATASTORE_REMOVE or
196 * GNUNET_MESSAGE_TYPE_DATASTORE_DATA. Depending on the message
197 * type, some fields may simply have values of zero.
198 */
199 struct GNUNET_MessageHeader header;
200
201 /**
202 * Reservation ID to use; use zero for none.
203 */
204 uint32_t rid GNUNET_PACKED;
205
206 /**
207 * Number of bytes in the item (NBO).
208 */
209 uint32_t size GNUNET_PACKED;
210
211 /**
212 * Type of the item (NBO), zero for remove, (actually an enum GNUNET_BLOCK_Type)
213 */
214 uint32_t type GNUNET_PACKED;
215
216 /**
217 * Priority of the item (NBO), zero for remove.
218 */
219 uint32_t priority GNUNET_PACKED;
220
221 /**
222 * Desired anonymity level (NBO), zero for remove.
223 */
224 uint32_t anonymity GNUNET_PACKED;
225
226 /**
227 * Desired replication level.
228 */
229 uint32_t replication GNUNET_PACKED;
230
231 /**
232 * For alignment.
233 */
234 uint32_t reserved GNUNET_PACKED;
235
236 /**
237 * Unique ID for the content (can be used for UPDATE);
238 * can be zero for remove (which indicates that
239 * the datastore should use whatever UID matches
240 * the key and content).
241 */
242 uint64_t uid;
243
244 /**
245 * Expiration time (NBO); zero for remove.
246 */
247 struct GNUNET_TIME_AbsoluteNBO expiration;
248
249 /**
250 * Key under which the item can be found.
251 */
252 struct GNUNET_HashCode key;
253};
254GNUNET_NETWORK_STRUCT_END
255
256
257#endif
diff --git a/src/datastore/datastore_api.c b/src/datastore/datastore_api.c
deleted file mode 100644
index 4d27efb4e..000000000
--- a/src/datastore/datastore_api.c
+++ /dev/null
@@ -1,1393 +0,0 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2004-2013, 2016 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file datastore/datastore_api.c
23 * @brief Management for the datastore for files stored on a GNUnet node. Implements
24 * a priority queue for requests
25 * @author Christian Grothoff
26 */
27#include "platform.h"
28#include "gnunet_arm_service.h"
29#include "gnunet_constants.h"
30#include "gnunet_datastore_service.h"
31#include "gnunet_statistics_service.h"
32#include "datastore.h"
33
34#define LOG(kind, ...) GNUNET_log_from (kind, "datastore-api", __VA_ARGS__)
35
36#define DELAY_WARN_TIMEOUT GNUNET_TIME_UNIT_MINUTES
37
38/**
39 * Collect an instance number of statistics? May cause excessive IPC.
40 */
41#define INSANE_STATISTICS GNUNET_NO
42
43/**
44 * If a client stopped asking for more results, how many more do
45 * we receive from the DB before killing the connection? Trade-off
46 * between re-doing TCP handshakes and (needlessly) receiving
47 * useless results.
48 */
49#define MAX_EXCESS_RESULTS 8
50
51/**
52 * Context for processing status messages.
53 */
54struct StatusContext
55{
56 /**
57 * Continuation to call with the status.
58 */
59 GNUNET_DATASTORE_ContinuationWithStatus cont;
60
61 /**
62 * Closure for @e cont.
63 */
64 void *cont_cls;
65};
66
67
68/**
69 * Context for processing result messages.
70 */
71struct ResultContext
72{
73 /**
74 * Function to call with the result.
75 */
76 GNUNET_DATASTORE_DatumProcessor proc;
77
78 /**
79 * Closure for @e proc.
80 */
81 void *proc_cls;
82};
83
84
85/**
86 * Context for a queue operation.
87 */
88union QueueContext
89{
90 struct StatusContext sc;
91
92 struct ResultContext rc;
93};
94
95
96/**
97 * Entry in our priority queue.
98 */
99struct GNUNET_DATASTORE_QueueEntry
100{
101 /**
102 * This is a linked list.
103 */
104 struct GNUNET_DATASTORE_QueueEntry *next;
105
106 /**
107 * This is a linked list.
108 */
109 struct GNUNET_DATASTORE_QueueEntry *prev;
110
111 /**
112 * Handle to the master context.
113 */
114 struct GNUNET_DATASTORE_Handle *h;
115
116 /**
117 * Function to call after transmission of the request.
118 */
119 GNUNET_DATASTORE_ContinuationWithStatus cont;
120
121 /**
122 * Closure for @e cont.
123 */
124 void *cont_cls;
125
126 /**
127 * Context for the operation.
128 */
129 union QueueContext qc;
130
131 /**
132 * Envelope of the request to transmit, NULL after
133 * transmission.
134 */
135 struct GNUNET_MQ_Envelope *env;
136
137 /**
138 * Task we run if this entry stalls the queue and we
139 * need to warn the user.
140 */
141 struct GNUNET_SCHEDULER_Task *delay_warn_task;
142
143 /**
144 * Priority in the queue.
145 */
146 unsigned int priority;
147
148 /**
149 * Maximum allowed length of queue (otherwise
150 * this request should be discarded).
151 */
152 unsigned int max_queue;
153
154 /**
155 * Expected response type.
156 */
157 uint16_t response_type;
158};
159
160
161/**
162 * Handle to the datastore service.
163 */
164struct GNUNET_DATASTORE_Handle
165{
166 /**
167 * Our configuration.
168 */
169 const struct GNUNET_CONFIGURATION_Handle *cfg;
170
171 /**
172 * Current connection to the datastore service.
173 */
174 struct GNUNET_MQ_Handle *mq;
175
176 /**
177 * Handle for statistics.
178 */
179 struct GNUNET_STATISTICS_Handle *stats;
180
181 /**
182 * Current head of priority queue.
183 */
184 struct GNUNET_DATASTORE_QueueEntry *queue_head;
185
186 /**
187 * Current tail of priority queue.
188 */
189 struct GNUNET_DATASTORE_QueueEntry *queue_tail;
190
191 /**
192 * Task for trying to reconnect.
193 */
194 struct GNUNET_SCHEDULER_Task *reconnect_task;
195
196 /**
197 * How quickly should we retry? Used for exponential back-off on
198 * connect-errors.
199 */
200 struct GNUNET_TIME_Relative retry_time;
201
202 /**
203 * Number of entries in the queue.
204 */
205 unsigned int queue_size;
206
207 /**
208 * Number of results we're receiving for the current query
209 * after application stopped to care. Used to determine when
210 * to reset the connection.
211 */
212 unsigned int result_count;
213
214 /**
215 * We should ignore the next message(s) from the service.
216 */
217 unsigned int skip_next_messages;
218};
219
220
221/**
222 * Try reconnecting to the datastore service.
223 *
224 * @param cls the `struct GNUNET_DATASTORE_Handle`
225 */
226static void
227try_reconnect (void *cls);
228
229
230/**
231 * Disconnect from the service and then try reconnecting to the datastore service
232 * after some delay.
233 *
234 * @param h handle to datastore to disconnect and reconnect
235 */
236static void
237do_disconnect (struct GNUNET_DATASTORE_Handle *h)
238{
239 if (NULL == h->mq)
240 {
241 GNUNET_break (0);
242 return;
243 }
244 GNUNET_MQ_destroy (h->mq);
245 h->mq = NULL;
246 h->skip_next_messages = 0;
247 h->reconnect_task
248 = GNUNET_SCHEDULER_add_delayed (h->retry_time,
249 &try_reconnect,
250 h);
251}
252
253
254/**
255 * Free a queue entry. Removes the given entry from the
256 * queue and releases associated resources. Does NOT
257 * call the callback.
258 *
259 * @param qe entry to free.
260 */
261static void
262free_queue_entry (struct GNUNET_DATASTORE_QueueEntry *qe)
263{
264 struct GNUNET_DATASTORE_Handle *h = qe->h;
265
266 GNUNET_CONTAINER_DLL_remove (h->queue_head,
267 h->queue_tail,
268 qe);
269 h->queue_size--;
270 if (NULL != qe->env)
271 GNUNET_MQ_discard (qe->env);
272 if (NULL != qe->delay_warn_task)
273 GNUNET_SCHEDULER_cancel (qe->delay_warn_task);
274 GNUNET_free (qe);
275}
276
277
278/**
279 * Task that logs an error after some time.
280 *
281 * @param cls `struct GNUNET_DATASTORE_QueueEntry` about which the error is
282 */
283static void
284delay_warning (void *cls)
285{
286 struct GNUNET_DATASTORE_QueueEntry *qe = cls;
287
288 qe->delay_warn_task = NULL;
289 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
290 "Request %p of type %u at head of datastore queue for more than %s\n",
291 qe,
292 (unsigned int) qe->response_type,
293 GNUNET_STRINGS_relative_time_to_string (DELAY_WARN_TIMEOUT,
294 GNUNET_YES));
295 qe->delay_warn_task = GNUNET_SCHEDULER_add_delayed (DELAY_WARN_TIMEOUT,
296 &delay_warning,
297 qe);
298}
299
300
301/**
302 * Handle error in sending drop request to datastore.
303 *
304 * @param cls closure with the datastore handle
305 * @param error error code
306 */
307static void
308mq_error_handler (void *cls,
309 enum GNUNET_MQ_Error error)
310{
311 struct GNUNET_DATASTORE_Handle *h = cls;
312 struct GNUNET_DATASTORE_QueueEntry *qe;
313
314 LOG (GNUNET_ERROR_TYPE_DEBUG,
315 "MQ error, reconnecting to DATASTORE\n");
316 do_disconnect (h);
317 qe = h->queue_head;
318 if (NULL == qe)
319 return;
320 if (NULL != qe->delay_warn_task)
321 {
322 GNUNET_SCHEDULER_cancel (qe->delay_warn_task);
323 qe->delay_warn_task = NULL;
324 }
325 if (NULL == qe->env)
326 {
327 union QueueContext qc = qe->qc;
328 uint16_t rt = qe->response_type;
329
330 LOG (GNUNET_ERROR_TYPE_DEBUG,
331 "Failed to receive response from database.\n");
332 free_queue_entry (qe);
333 switch (rt)
334 {
335 case GNUNET_MESSAGE_TYPE_DATASTORE_STATUS:
336 if (NULL != qc.sc.cont)
337 qc.sc.cont (qc.sc.cont_cls,
338 GNUNET_SYSERR,
339 GNUNET_TIME_UNIT_ZERO_ABS,
340 _ ("DATASTORE disconnected"));
341 break;
342
343 case GNUNET_MESSAGE_TYPE_DATASTORE_DATA:
344 if (NULL != qc.rc.proc)
345 qc.rc.proc (qc.rc.proc_cls,
346 NULL,
347 0,
348 NULL,
349 0,
350 0,
351 0,
352 0,
353 GNUNET_TIME_UNIT_ZERO_ABS,
354 0);
355 break;
356
357 default:
358 GNUNET_break (0);
359 }
360 }
361}
362
363
364/**
365 * Connect to the datastore service.
366 *
367 * @param cfg configuration to use
368 * @return handle to use to access the service
369 */
370struct GNUNET_DATASTORE_Handle *
371GNUNET_DATASTORE_connect (const struct GNUNET_CONFIGURATION_Handle *cfg)
372{
373 struct GNUNET_DATASTORE_Handle *h;
374
375 LOG (GNUNET_ERROR_TYPE_DEBUG,
376 "Establishing DATASTORE connection!\n");
377 h = GNUNET_new (struct GNUNET_DATASTORE_Handle);
378 h->cfg = cfg;
379 try_reconnect (h);
380 if (NULL == h->mq)
381 {
382 GNUNET_free (h);
383 return NULL;
384 }
385 h->stats = GNUNET_STATISTICS_create ("datastore-api",
386 cfg);
387 return h;
388}
389
390
391/**
392 * Task used by to disconnect from the datastore after
393 * we send the #GNUNET_MESSAGE_TYPE_DATASTORE_DROP message.
394 *
395 * @param cls the datastore handle
396 */
397static void
398disconnect_after_drop (void *cls)
399{
400 struct GNUNET_DATASTORE_Handle *h = cls;
401
402 LOG (GNUNET_ERROR_TYPE_DEBUG,
403 "Drop sent, disconnecting\n");
404 GNUNET_DATASTORE_disconnect (h,
405 GNUNET_NO);
406}
407
408
409/**
410 * Handle error in sending drop request to datastore.
411 *
412 * @param cls closure with the datastore handle
413 * @param error error code
414 */
415static void
416disconnect_on_mq_error (void *cls,
417 enum GNUNET_MQ_Error error)
418{
419 struct GNUNET_DATASTORE_Handle *h = cls;
420
421 LOG (GNUNET_ERROR_TYPE_ERROR,
422 "Failed to ask datastore to drop tables\n");
423 GNUNET_DATASTORE_disconnect (h,
424 GNUNET_NO);
425}
426
427
428/**
429 * Disconnect from the datastore service (and free
430 * associated resources).
431 *
432 * @param h handle to the datastore
433 * @param drop set to #GNUNET_YES to delete all data in datastore (!)
434 */
435void
436GNUNET_DATASTORE_disconnect (struct GNUNET_DATASTORE_Handle *h,
437 int drop)
438{
439 struct GNUNET_DATASTORE_QueueEntry *qe;
440
441 LOG (GNUNET_ERROR_TYPE_DEBUG,
442 "Datastore disconnect\n");
443 if (NULL != h->mq)
444 {
445 GNUNET_MQ_destroy (h->mq);
446 h->mq = NULL;
447 }
448 if (NULL != h->reconnect_task)
449 {
450 GNUNET_SCHEDULER_cancel (h->reconnect_task);
451 h->reconnect_task = NULL;
452 }
453 while (NULL != (qe = h->queue_head))
454 {
455 switch (qe->response_type)
456 {
457 case GNUNET_MESSAGE_TYPE_DATASTORE_STATUS:
458 if (NULL != qe->qc.sc.cont)
459 qe->qc.sc.cont (qe->qc.sc.cont_cls,
460 GNUNET_SYSERR,
461 GNUNET_TIME_UNIT_ZERO_ABS,
462 _ ("Disconnected from DATASTORE"));
463 break;
464
465 case GNUNET_MESSAGE_TYPE_DATASTORE_DATA:
466 if (NULL != qe->qc.rc.proc)
467 qe->qc.rc.proc (qe->qc.rc.proc_cls,
468 NULL,
469 0,
470 NULL,
471 0,
472 0,
473 0,
474 0,
475 GNUNET_TIME_UNIT_ZERO_ABS,
476 0);
477 break;
478
479 default:
480 GNUNET_break (0);
481 }
482 free_queue_entry (qe);
483 }
484 if (GNUNET_YES == drop)
485 {
486 LOG (GNUNET_ERROR_TYPE_DEBUG,
487 "Re-connecting to issue DROP!\n");
488 GNUNET_assert (NULL == h->mq);
489 h->mq = GNUNET_CLIENT_connect (h->cfg,
490 "datastore",
491 NULL,
492 &disconnect_on_mq_error,
493 h);
494 if (NULL != h->mq)
495 {
496 struct GNUNET_MessageHeader *hdr;
497 struct GNUNET_MQ_Envelope *env;
498
499 env = GNUNET_MQ_msg (hdr,
500 GNUNET_MESSAGE_TYPE_DATASTORE_DROP);
501 GNUNET_MQ_notify_sent (env,
502 &disconnect_after_drop,
503 h);
504 GNUNET_MQ_send (h->mq,
505 env);
506 return;
507 }
508 GNUNET_break (0);
509 }
510 GNUNET_STATISTICS_destroy (h->stats,
511 GNUNET_NO);
512 h->stats = NULL;
513 GNUNET_free (h);
514}
515
516
517/**
518 * Create a new entry for our priority queue (and possibly discard other entries if
519 * the queue is getting too long).
520 *
521 * @param h handle to the datastore
522 * @param env envelope with the message to queue
523 * @param queue_priority priority of the entry
524 * @param max_queue_size at what queue size should this request be dropped
525 * (if other requests of higher priority are in the queue)
526 * @param expected_type which type of response do we expect,
527 * #GNUNET_MESSAGE_TYPE_DATASTORE_STATUS or
528 * #GNUNET_MESSAGE_TYPE_DATASTORE_DATA
529 * @param qc client context (NOT a closure for @a response_proc)
530 * @return NULL if the queue is full
531 */
532static struct GNUNET_DATASTORE_QueueEntry *
533make_queue_entry (struct GNUNET_DATASTORE_Handle *h,
534 struct GNUNET_MQ_Envelope *env,
535 unsigned int queue_priority,
536 unsigned int max_queue_size,
537 uint16_t expected_type,
538 const union QueueContext *qc)
539{
540 struct GNUNET_DATASTORE_QueueEntry *qe;
541 struct GNUNET_DATASTORE_QueueEntry *pos;
542 unsigned int c;
543
544 if ((NULL != h->queue_tail) &&
545 (h->queue_tail->priority >= queue_priority))
546 {
547 c = h->queue_size;
548 pos = NULL;
549 }
550 else
551 {
552 c = 0;
553 pos = h->queue_head;
554 }
555 while ((NULL != pos) &&
556 (c < max_queue_size) &&
557 (pos->priority >= queue_priority))
558 {
559 c++;
560 pos = pos->next;
561 }
562 if (c >= max_queue_size)
563 {
564 GNUNET_STATISTICS_update (h->stats,
565 gettext_noop ("# queue overflows"),
566 1,
567 GNUNET_NO);
568 GNUNET_MQ_discard (env);
569 return NULL;
570 }
571 qe = GNUNET_new (struct GNUNET_DATASTORE_QueueEntry);
572 qe->h = h;
573 qe->env = env;
574 qe->response_type = expected_type;
575 qe->qc = *qc;
576 qe->priority = queue_priority;
577 qe->max_queue = max_queue_size;
578 if (NULL == pos)
579 {
580 /* append at the tail */
581 pos = h->queue_tail;
582 }
583 else
584 {
585 pos = pos->prev;
586 /* do not insert at HEAD if HEAD query was already
587 * transmitted and we are still receiving replies! */
588 if ((NULL == pos) &&
589 (NULL == h->queue_head->env))
590 pos = h->queue_head;
591 }
592 c++;
593#if INSANE_STATISTICS
594 GNUNET_STATISTICS_update (h->stats,
595 gettext_noop ("# queue entries created"),
596 1,
597 GNUNET_NO);
598#endif
599 GNUNET_CONTAINER_DLL_insert_after (h->queue_head,
600 h->queue_tail,
601 pos,
602 qe);
603 h->queue_size++;
604 return qe;
605}
606
607
608/**
609 * Process entries in the queue (or do nothing if we are already
610 * doing so).
611 *
612 * @param h handle to the datastore
613 */
614static void
615process_queue (struct GNUNET_DATASTORE_Handle *h)
616{
617 struct GNUNET_DATASTORE_QueueEntry *qe;
618
619 if (NULL == (qe = h->queue_head))
620 {
621 /* no entry in queue */
622 LOG (GNUNET_ERROR_TYPE_DEBUG,
623 "Queue empty\n");
624 return;
625 }
626 if (NULL == qe->env)
627 {
628 /* waiting for replies */
629 LOG (GNUNET_ERROR_TYPE_DEBUG,
630 "Head request already transmitted\n");
631 return;
632 }
633 if (NULL == h->mq)
634 {
635 /* waiting for reconnect */
636 LOG (GNUNET_ERROR_TYPE_DEBUG,
637 "Not connected\n");
638 return;
639 }
640 GNUNET_assert (NULL == qe->delay_warn_task);
641 qe->delay_warn_task = GNUNET_SCHEDULER_add_delayed (DELAY_WARN_TIMEOUT,
642 &delay_warning,
643 qe);
644 GNUNET_MQ_send (h->mq,
645 qe->env);
646 qe->env = NULL;
647}
648
649
650/**
651 * Get the entry at the head of the message queue.
652 *
653 * @param h handle to the datastore
654 * @param response_type the expected response type
655 * @return the queue entry
656 */
657static struct GNUNET_DATASTORE_QueueEntry *
658get_queue_head (struct GNUNET_DATASTORE_Handle *h,
659 uint16_t response_type)
660{
661 struct GNUNET_DATASTORE_QueueEntry *qe;
662
663 if (h->skip_next_messages > 0)
664 {
665 h->skip_next_messages--;
666 process_queue (h);
667 return NULL;
668 }
669 qe = h->queue_head;
670 if (NULL == qe)
671 {
672 GNUNET_break (0);
673 do_disconnect (h);
674 return NULL;
675 }
676 if (NULL != qe->env)
677 {
678 GNUNET_break (0);
679 do_disconnect (h);
680 return NULL;
681 }
682 if (response_type != qe->response_type)
683 {
684 GNUNET_break (0);
685 do_disconnect (h);
686 return NULL;
687 }
688 return qe;
689}
690
691
692/**
693 * Function called to check status message from the service.
694 *
695 * @param cls closure
696 * @param sm status message received
697 * @return #GNUNET_OK if the message is well-formed
698 */
699static int
700check_status (void *cls,
701 const struct StatusMessage *sm)
702{
703 uint16_t msize = ntohs (sm->header.size) - sizeof(*sm);
704 int32_t status = ntohl (sm->status);
705
706 if (msize > 0)
707 {
708 const char *emsg = (const char *) &sm[1];
709
710 if ('\0' != emsg[msize - 1])
711 {
712 GNUNET_break (0);
713 return GNUNET_SYSERR;
714 }
715 }
716 else if (GNUNET_SYSERR == status)
717 {
718 GNUNET_break (0);
719 return GNUNET_SYSERR;
720 }
721 return GNUNET_OK;
722}
723
724
725/**
726 * Function called to handle status message from the service.
727 *
728 * @param cls closure
729 * @param sm status message received
730 */
731static void
732handle_status (void *cls,
733 const struct StatusMessage *sm)
734{
735 struct GNUNET_DATASTORE_Handle *h = cls;
736 struct GNUNET_DATASTORE_QueueEntry *qe;
737 struct StatusContext rc;
738 const char *emsg;
739 int32_t status = ntohl (sm->status);
740
741 qe = get_queue_head (h,
742 GNUNET_MESSAGE_TYPE_DATASTORE_STATUS);
743 if (NULL == qe)
744 return;
745 rc = qe->qc.sc;
746 free_queue_entry (qe);
747 if (ntohs (sm->header.size) > sizeof(struct StatusMessage))
748 emsg = (const char *) &sm[1];
749 else
750 emsg = NULL;
751 LOG (GNUNET_ERROR_TYPE_DEBUG,
752 "Received status %d/%s\n",
753 (int) status,
754 emsg);
755 GNUNET_STATISTICS_update (h->stats,
756 gettext_noop ("# status messages received"),
757 1,
758 GNUNET_NO);
759 h->retry_time = GNUNET_TIME_UNIT_ZERO;
760 process_queue (h);
761 if (NULL != rc.cont)
762 rc.cont (rc.cont_cls,
763 status,
764 GNUNET_TIME_absolute_ntoh (sm->min_expiration),
765 emsg);
766}
767
768
769/**
770 * Check data message we received from the service.
771 *
772 * @param cls closure with the `struct GNUNET_DATASTORE_Handle *`
773 * @param dm message received
774 */
775static int
776check_data (void *cls,
777 const struct DataMessage *dm)
778{
779 uint16_t msize = ntohs (dm->header.size) - sizeof(*dm);
780
781 if (msize != ntohl (dm->size))
782 {
783 GNUNET_break (0);
784 return GNUNET_SYSERR;
785 }
786 return GNUNET_OK;
787}
788
789
790/**
791 * Handle data message we got from the service.
792 *
793 * @param cls closure with the `struct GNUNET_DATASTORE_Handle *`
794 * @param dm message received
795 */
796static void
797handle_data (void *cls,
798 const struct DataMessage *dm)
799{
800 struct GNUNET_DATASTORE_Handle *h = cls;
801 struct GNUNET_DATASTORE_QueueEntry *qe;
802 struct ResultContext rc;
803
804 qe = get_queue_head (h,
805 GNUNET_MESSAGE_TYPE_DATASTORE_DATA);
806 if (NULL == qe)
807 return;
808#if INSANE_STATISTICS
809 GNUNET_STATISTICS_update (h->stats,
810 gettext_noop ("# Results received"),
811 1,
812 GNUNET_NO);
813#endif
814 LOG (GNUNET_ERROR_TYPE_DEBUG,
815 "Received result %llu with type %u and size %u with key %s\n",
816 (unsigned long long) GNUNET_ntohll (dm->uid),
817 ntohl (dm->type),
818 ntohl (dm->size),
819 GNUNET_h2s (&dm->key));
820 rc = qe->qc.rc;
821 free_queue_entry (qe);
822 h->retry_time = GNUNET_TIME_UNIT_ZERO;
823 process_queue (h);
824 if (NULL != rc.proc)
825 rc.proc (rc.proc_cls,
826 &dm->key,
827 ntohl (dm->size),
828 &dm[1],
829 ntohl (dm->type),
830 ntohl (dm->priority),
831 ntohl (dm->anonymity),
832 ntohl (dm->replication),
833 GNUNET_TIME_absolute_ntoh (dm->expiration),
834 GNUNET_ntohll (dm->uid));
835}
836
837
838/**
839 * Type of a function to call when we receive a
840 * #GNUNET_MESSAGE_TYPE_DATASTORE_DATA_END message from the service.
841 *
842 * @param cls closure with the `struct GNUNET_DATASTORE_Handle *`
843 * @param msg message received
844 */
845static void
846handle_data_end (void *cls,
847 const struct GNUNET_MessageHeader *msg)
848{
849 struct GNUNET_DATASTORE_Handle *h = cls;
850 struct GNUNET_DATASTORE_QueueEntry *qe;
851 struct ResultContext rc;
852
853 qe = get_queue_head (h,
854 GNUNET_MESSAGE_TYPE_DATASTORE_DATA);
855 if (NULL == qe)
856 return;
857 rc = qe->qc.rc;
858 free_queue_entry (qe);
859 LOG (GNUNET_ERROR_TYPE_DEBUG,
860 "Received end of result set, new queue size is %u\n",
861 h->queue_size);
862 h->retry_time = GNUNET_TIME_UNIT_ZERO;
863 h->result_count = 0;
864 process_queue (h);
865 /* signal end of iteration */
866 if (NULL != rc.proc)
867 rc.proc (rc.proc_cls,
868 NULL,
869 0,
870 NULL,
871 0,
872 0,
873 0,
874 0,
875 GNUNET_TIME_UNIT_ZERO_ABS,
876 0);
877}
878
879
880/**
881 * Try reconnecting to the datastore service.
882 *
883 * @param cls the `struct GNUNET_DATASTORE_Handle`
884 */
885static void
886try_reconnect (void *cls)
887{
888 struct GNUNET_DATASTORE_Handle *h = cls;
889 struct GNUNET_MQ_MessageHandler handlers[] = {
890 GNUNET_MQ_hd_var_size (status,
891 GNUNET_MESSAGE_TYPE_DATASTORE_STATUS,
892 struct StatusMessage,
893 h),
894 GNUNET_MQ_hd_var_size (data,
895 GNUNET_MESSAGE_TYPE_DATASTORE_DATA,
896 struct DataMessage,
897 h),
898 GNUNET_MQ_hd_fixed_size (data_end,
899 GNUNET_MESSAGE_TYPE_DATASTORE_DATA_END,
900 struct GNUNET_MessageHeader,
901 h),
902 GNUNET_MQ_handler_end ()
903 };
904
905 h->retry_time = GNUNET_TIME_STD_BACKOFF (h->retry_time);
906 h->reconnect_task = NULL;
907 GNUNET_assert (NULL == h->mq);
908 h->mq = GNUNET_CLIENT_connect (h->cfg,
909 "datastore",
910 handlers,
911 &mq_error_handler,
912 h);
913 if (NULL == h->mq)
914 return;
915 GNUNET_STATISTICS_update (h->stats,
916 gettext_noop (
917 "# datastore connections (re)created"),
918 1,
919 GNUNET_NO);
920 LOG (GNUNET_ERROR_TYPE_DEBUG,
921 "Reconnected to DATASTORE\n");
922 process_queue (h);
923}
924
925
926/**
927 * Dummy continuation used to do nothing (but be non-zero).
928 *
929 * @param cls closure
930 * @param result result
931 * @param min_expiration expiration time
932 * @param emsg error message
933 */
934static void
935drop_status_cont (void *cls,
936 int32_t result,
937 struct GNUNET_TIME_Absolute min_expiration,
938 const char *emsg)
939{
940 /* do nothing */
941}
942
943
944struct GNUNET_DATASTORE_QueueEntry *
945GNUNET_DATASTORE_put (struct GNUNET_DATASTORE_Handle *h,
946 uint32_t rid,
947 const struct GNUNET_HashCode *key,
948 size_t size,
949 const void *data,
950 enum GNUNET_BLOCK_Type type,
951 uint32_t priority,
952 uint32_t anonymity,
953 uint32_t replication,
954 struct GNUNET_TIME_Absolute expiration,
955 unsigned int queue_priority,
956 unsigned int max_queue_size,
957 GNUNET_DATASTORE_ContinuationWithStatus cont,
958 void *cont_cls)
959{
960 struct GNUNET_DATASTORE_QueueEntry *qe;
961 struct GNUNET_MQ_Envelope *env;
962 struct DataMessage *dm;
963 union QueueContext qc;
964
965 if (size + sizeof(*dm) >= GNUNET_MAX_MESSAGE_SIZE)
966 {
967 GNUNET_break (0);
968 return NULL;
969 }
970
971 LOG (GNUNET_ERROR_TYPE_DEBUG,
972 "Asked to put %lu bytes of data under key `%s' for %s\n",
973 (unsigned long) size,
974 GNUNET_h2s (key),
975 GNUNET_STRINGS_relative_time_to_string (
976 GNUNET_TIME_absolute_get_remaining (expiration),
977 GNUNET_YES));
978 env = GNUNET_MQ_msg_extra (dm,
979 size,
980 GNUNET_MESSAGE_TYPE_DATASTORE_PUT);
981 dm->rid = htonl (rid);
982 dm->size = htonl ((uint32_t) size);
983 dm->type = htonl (type);
984 dm->priority = htonl (priority);
985 dm->anonymity = htonl (anonymity);
986 dm->replication = htonl (replication);
987 dm->expiration = GNUNET_TIME_absolute_hton (expiration);
988 dm->key = *key;
989 GNUNET_memcpy (&dm[1],
990 data,
991 size);
992 qc.sc.cont = cont;
993 qc.sc.cont_cls = cont_cls;
994 qe = make_queue_entry (h,
995 env,
996 queue_priority,
997 max_queue_size,
998 GNUNET_MESSAGE_TYPE_DATASTORE_STATUS,
999 &qc);
1000 if (NULL == qe)
1001 {
1002 LOG (GNUNET_ERROR_TYPE_DEBUG,
1003 "Could not create queue entry for PUT\n");
1004 return NULL;
1005 }
1006 GNUNET_STATISTICS_update (h->stats,
1007 gettext_noop ("# PUT requests executed"),
1008 1,
1009 GNUNET_NO);
1010 process_queue (h);
1011 return qe;
1012}
1013
1014
1015/**
1016 * Reserve space in the datastore. This function should be used
1017 * to avoid "out of space" failures during a longer sequence of "put"
1018 * operations (for example, when a file is being inserted).
1019 *
1020 * @param h handle to the datastore
1021 * @param amount how much space (in bytes) should be reserved (for content only)
1022 * @param entries how many entries will be created (to calculate per-entry overhead)
1023 * @param cont continuation to call when done; "success" will be set to
1024 * a positive reservation value if space could be reserved.
1025 * @param cont_cls closure for @a cont
1026 * @return NULL if the entry was not queued, otherwise a handle that can be used to
1027 * cancel; note that even if NULL is returned, the callback will be invoked
1028 * (or rather, will already have been invoked)
1029 */
1030struct GNUNET_DATASTORE_QueueEntry *
1031GNUNET_DATASTORE_reserve (struct GNUNET_DATASTORE_Handle *h,
1032 uint64_t amount,
1033 uint32_t entries,
1034 GNUNET_DATASTORE_ContinuationWithStatus cont,
1035 void *cont_cls)
1036{
1037 struct GNUNET_DATASTORE_QueueEntry *qe;
1038 struct GNUNET_MQ_Envelope *env;
1039 struct ReserveMessage *rm;
1040 union QueueContext qc;
1041
1042 if (NULL == cont)
1043 cont = &drop_status_cont;
1044 LOG (GNUNET_ERROR_TYPE_DEBUG,
1045 "Asked to reserve %llu bytes of data and %u entries\n",
1046 (unsigned long long) amount,
1047 (unsigned int) entries);
1048 env = GNUNET_MQ_msg (rm,
1049 GNUNET_MESSAGE_TYPE_DATASTORE_RESERVE);
1050 rm->entries = htonl (entries);
1051 rm->amount = GNUNET_htonll (amount);
1052
1053 qc.sc.cont = cont;
1054 qc.sc.cont_cls = cont_cls;
1055 qe = make_queue_entry (h,
1056 env,
1057 UINT_MAX,
1058 UINT_MAX,
1059 GNUNET_MESSAGE_TYPE_DATASTORE_STATUS,
1060 &qc);
1061 if (NULL == qe)
1062 {
1063 LOG (GNUNET_ERROR_TYPE_DEBUG,
1064 "Could not create queue entry to reserve\n");
1065 return NULL;
1066 }
1067 GNUNET_STATISTICS_update (h->stats,
1068 gettext_noop ("# RESERVE requests executed"),
1069 1,
1070 GNUNET_NO);
1071 process_queue (h);
1072 return qe;
1073}
1074
1075
1076struct GNUNET_DATASTORE_QueueEntry *
1077GNUNET_DATASTORE_release_reserve (struct GNUNET_DATASTORE_Handle *h,
1078 uint32_t rid,
1079 unsigned int queue_priority,
1080 unsigned int max_queue_size,
1081 GNUNET_DATASTORE_ContinuationWithStatus cont,
1082 void *cont_cls)
1083{
1084 struct GNUNET_DATASTORE_QueueEntry *qe;
1085 struct GNUNET_MQ_Envelope *env;
1086 struct ReleaseReserveMessage *rrm;
1087 union QueueContext qc;
1088
1089 if (NULL == cont)
1090 cont = &drop_status_cont;
1091 LOG (GNUNET_ERROR_TYPE_DEBUG,
1092 "Asked to release reserve %d\n",
1093 rid);
1094 env = GNUNET_MQ_msg (rrm,
1095 GNUNET_MESSAGE_TYPE_DATASTORE_RELEASE_RESERVE);
1096 rrm->rid = htonl (rid);
1097 qc.sc.cont = cont;
1098 qc.sc.cont_cls = cont_cls;
1099 qe = make_queue_entry (h,
1100 env,
1101 queue_priority,
1102 max_queue_size,
1103 GNUNET_MESSAGE_TYPE_DATASTORE_STATUS,
1104 &qc);
1105 if (NULL == qe)
1106 {
1107 LOG (GNUNET_ERROR_TYPE_DEBUG,
1108 "Could not create queue entry to release reserve\n");
1109 return NULL;
1110 }
1111 GNUNET_STATISTICS_update (h->stats,
1112 gettext_noop
1113 ("# RELEASE RESERVE requests executed"), 1,
1114 GNUNET_NO);
1115 process_queue (h);
1116 return qe;
1117}
1118
1119
1120struct GNUNET_DATASTORE_QueueEntry *
1121GNUNET_DATASTORE_remove (struct GNUNET_DATASTORE_Handle *h,
1122 const struct GNUNET_HashCode *key,
1123 size_t size,
1124 const void *data,
1125 unsigned int queue_priority,
1126 unsigned int max_queue_size,
1127 GNUNET_DATASTORE_ContinuationWithStatus cont,
1128 void *cont_cls)
1129{
1130 struct GNUNET_DATASTORE_QueueEntry *qe;
1131 struct DataMessage *dm;
1132 struct GNUNET_MQ_Envelope *env;
1133 union QueueContext qc;
1134
1135 if (sizeof(*dm) + size >= GNUNET_MAX_MESSAGE_SIZE)
1136 {
1137 GNUNET_break (0);
1138 return NULL;
1139 }
1140 if (NULL == cont)
1141 cont = &drop_status_cont;
1142 LOG (GNUNET_ERROR_TYPE_DEBUG,
1143 "Asked to remove %lu bytes under key `%s'\n",
1144 (unsigned long) size,
1145 GNUNET_h2s (key));
1146 env = GNUNET_MQ_msg_extra (dm,
1147 size,
1148 GNUNET_MESSAGE_TYPE_DATASTORE_REMOVE);
1149 dm->size = htonl (size);
1150 dm->key = *key;
1151 GNUNET_memcpy (&dm[1],
1152 data,
1153 size);
1154
1155 qc.sc.cont = cont;
1156 qc.sc.cont_cls = cont_cls;
1157
1158 qe = make_queue_entry (h,
1159 env,
1160 queue_priority,
1161 max_queue_size,
1162 GNUNET_MESSAGE_TYPE_DATASTORE_STATUS,
1163 &qc);
1164 if (NULL == qe)
1165 {
1166 LOG (GNUNET_ERROR_TYPE_DEBUG,
1167 "Could not create queue entry for REMOVE\n");
1168 return NULL;
1169 }
1170 GNUNET_STATISTICS_update (h->stats,
1171 gettext_noop ("# REMOVE requests executed"),
1172 1,
1173 GNUNET_NO);
1174 process_queue (h);
1175 return qe;
1176}
1177
1178
1179/**
1180 * Get a random value from the datastore for content replication.
1181 * Returns a single, random value among those with the highest
1182 * replication score, lowering positive replication scores by one for
1183 * the chosen value (if only content with a replication score exists,
1184 * a random value is returned and replication scores are not changed).
1185 *
1186 * @param h handle to the datastore
1187 * @param queue_priority ranking of this request in the priority queue
1188 * @param max_queue_size at what queue size should this request be dropped
1189 * (if other requests of higher priority are in the queue)
1190 * @param proc function to call on a random value; it
1191 * will be called once with a value (if available)
1192 * and always once with a value of NULL.
1193 * @param proc_cls closure for @a proc
1194 * @return NULL if the entry was not queued, otherwise a handle that can be used to
1195 * cancel
1196 */
1197struct GNUNET_DATASTORE_QueueEntry *
1198GNUNET_DATASTORE_get_for_replication (struct GNUNET_DATASTORE_Handle *h,
1199 unsigned int queue_priority,
1200 unsigned int max_queue_size,
1201 GNUNET_DATASTORE_DatumProcessor proc,
1202 void *proc_cls)
1203{
1204 struct GNUNET_DATASTORE_QueueEntry *qe;
1205 struct GNUNET_MQ_Envelope *env;
1206 struct GNUNET_MessageHeader *m;
1207 union QueueContext qc;
1208
1209 GNUNET_assert (NULL != proc);
1210 LOG (GNUNET_ERROR_TYPE_DEBUG,
1211 "Asked to get replication entry\n");
1212 env = GNUNET_MQ_msg (m,
1213 GNUNET_MESSAGE_TYPE_DATASTORE_GET_REPLICATION);
1214 qc.rc.proc = proc;
1215 qc.rc.proc_cls = proc_cls;
1216 qe = make_queue_entry (h,
1217 env,
1218 queue_priority,
1219 max_queue_size,
1220 GNUNET_MESSAGE_TYPE_DATASTORE_DATA,
1221 &qc);
1222 if (NULL == qe)
1223 {
1224 LOG (GNUNET_ERROR_TYPE_DEBUG,
1225 "Could not create queue entry for GET REPLICATION\n");
1226 return NULL;
1227 }
1228 GNUNET_STATISTICS_update (h->stats,
1229 gettext_noop
1230 ("# GET REPLICATION requests executed"), 1,
1231 GNUNET_NO);
1232 process_queue (h);
1233 return qe;
1234}
1235
1236
1237struct GNUNET_DATASTORE_QueueEntry *
1238GNUNET_DATASTORE_get_zero_anonymity (struct GNUNET_DATASTORE_Handle *h,
1239 uint64_t next_uid,
1240 unsigned int queue_priority,
1241 unsigned int max_queue_size,
1242 enum GNUNET_BLOCK_Type type,
1243 GNUNET_DATASTORE_DatumProcessor proc,
1244 void *proc_cls)
1245{
1246 struct GNUNET_DATASTORE_QueueEntry *qe;
1247 struct GNUNET_MQ_Envelope *env;
1248 struct GetZeroAnonymityMessage *m;
1249 union QueueContext qc;
1250
1251 GNUNET_assert (NULL != proc);
1252 GNUNET_assert (type != GNUNET_BLOCK_TYPE_ANY);
1253 LOG (GNUNET_ERROR_TYPE_DEBUG,
1254 "Asked to get a zero-anonymity entry of type %d\n",
1255 type);
1256 env = GNUNET_MQ_msg (m,
1257 GNUNET_MESSAGE_TYPE_DATASTORE_GET_ZERO_ANONYMITY);
1258 m->type = htonl ((uint32_t) type);
1259 m->next_uid = GNUNET_htonll (next_uid);
1260 qc.rc.proc = proc;
1261 qc.rc.proc_cls = proc_cls;
1262 qe = make_queue_entry (h,
1263 env,
1264 queue_priority,
1265 max_queue_size,
1266 GNUNET_MESSAGE_TYPE_DATASTORE_DATA,
1267 &qc);
1268 if (NULL == qe)
1269 {
1270 LOG (GNUNET_ERROR_TYPE_DEBUG,
1271 "Could not create queue entry for zero-anonymity procation\n");
1272 return NULL;
1273 }
1274 GNUNET_STATISTICS_update (h->stats,
1275 gettext_noop
1276 ("# GET ZERO ANONYMITY requests executed"), 1,
1277 GNUNET_NO);
1278 process_queue (h);
1279 return qe;
1280}
1281
1282
1283/**
1284 * Get a result for a particular key from the datastore. The processor
1285 * will only be called once.
1286 *
1287 * @param h handle to the datastore
1288 * @param next_uid return the result with lowest uid >= next_uid
1289 * @param random if true, return a random result instead of using next_uid
1290 * @param key maybe NULL (to match all entries)
1291 * @param type desired type, 0 for any
1292 * @param queue_priority ranking of this request in the priority queue
1293 * @param max_queue_size at what queue size should this request be dropped
1294 * (if other requests of higher priority are in the queue)
1295 * @param proc function to call on each matching value;
1296 * will be called once with a NULL value at the end
1297 * @param proc_cls closure for @a proc
1298 * @return NULL if the entry was not queued, otherwise a handle that can be used to
1299 * cancel
1300 */
1301struct GNUNET_DATASTORE_QueueEntry *
1302GNUNET_DATASTORE_get_key (struct GNUNET_DATASTORE_Handle *h,
1303 uint64_t next_uid,
1304 bool random,
1305 const struct GNUNET_HashCode *key,
1306 enum GNUNET_BLOCK_Type type,
1307 unsigned int queue_priority,
1308 unsigned int max_queue_size,
1309 GNUNET_DATASTORE_DatumProcessor proc,
1310 void *proc_cls)
1311{
1312 struct GNUNET_DATASTORE_QueueEntry *qe;
1313 struct GNUNET_MQ_Envelope *env;
1314 struct GetKeyMessage *gkm;
1315 struct GetMessage *gm;
1316 union QueueContext qc;
1317
1318 GNUNET_assert (NULL != proc);
1319 LOG (GNUNET_ERROR_TYPE_DEBUG,
1320 "Asked to look for data of type %u under key `%s'\n",
1321 (unsigned int) type,
1322 (NULL != key) ? GNUNET_h2s (key) : "NULL");
1323 if (NULL == key)
1324 {
1325 env = GNUNET_MQ_msg (gm,
1326 GNUNET_MESSAGE_TYPE_DATASTORE_GET);
1327 gm->type = htonl (type);
1328 gm->next_uid = GNUNET_htonll (next_uid);
1329 gm->random = random;
1330 }
1331 else
1332 {
1333 env = GNUNET_MQ_msg (gkm,
1334 GNUNET_MESSAGE_TYPE_DATASTORE_GET_KEY);
1335 gkm->type = htonl (type);
1336 gkm->next_uid = GNUNET_htonll (next_uid);
1337 gkm->random = random;
1338 gkm->key = *key;
1339 }
1340 qc.rc.proc = proc;
1341 qc.rc.proc_cls = proc_cls;
1342 qe = make_queue_entry (h,
1343 env,
1344 queue_priority,
1345 max_queue_size,
1346 GNUNET_MESSAGE_TYPE_DATASTORE_DATA,
1347 &qc);
1348 if (NULL == qe)
1349 {
1350 LOG (GNUNET_ERROR_TYPE_DEBUG,
1351 "Could not queue request for `%s'\n",
1352 (NULL != key) ? GNUNET_h2s (key): "NULL");
1353 return NULL;
1354 }
1355#if INSANE_STATISTICS
1356 GNUNET_STATISTICS_update (h->stats,
1357 gettext_noop ("# GET requests executed"),
1358 1,
1359 GNUNET_NO);
1360#endif
1361 process_queue (h);
1362 return qe;
1363}
1364
1365
1366/**
1367 * Cancel a datastore operation. The final callback from the
1368 * operation must not have been done yet.
1369 *
1370 * @param qe operation to cancel
1371 */
1372void
1373GNUNET_DATASTORE_cancel (struct GNUNET_DATASTORE_QueueEntry *qe)
1374{
1375 struct GNUNET_DATASTORE_Handle *h = qe->h;
1376
1377 LOG (GNUNET_ERROR_TYPE_DEBUG,
1378 "Pending DATASTORE request %p cancelled (%d, %d)\n",
1379 qe,
1380 NULL == qe->env,
1381 h->queue_head == qe);
1382 if (NULL == qe->env)
1383 {
1384 free_queue_entry (qe);
1385 h->skip_next_messages++;
1386 return;
1387 }
1388 free_queue_entry (qe);
1389 process_queue (h);
1390}
1391
1392
1393/* end of datastore_api.c */
diff --git a/src/datastore/gnunet-datastore.c b/src/datastore/gnunet-datastore.c
deleted file mode 100644
index 9a76d1160..000000000
--- a/src/datastore/gnunet-datastore.c
+++ /dev/null
@@ -1,508 +0,0 @@
1/*
2 This file is part of GNUnet.
3 Copyright (C) 2010, 2013 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file datastore/gnunet-datastore.c
23 * @brief tool to manipulate datastores
24 * @author Christian Grothoff
25 */
26#include "platform.h"
27#include <inttypes.h>
28#include "gnunet_util_lib.h"
29#include "gnunet_datastore_service.h"
30
31GNUNET_NETWORK_STRUCT_BEGIN
32
33struct DataRecord
34{
35 /**
36 * Number of bytes in the item (NBO).
37 */
38 uint32_t size GNUNET_PACKED;
39
40 /**
41 * Type of the item (NBO) (actually an enum GNUNET_BLOCK_Type)
42 */
43 uint32_t type GNUNET_PACKED;
44
45 /**
46 * Priority of the item (NBO).
47 */
48 uint32_t priority GNUNET_PACKED;
49
50 /**
51 * Desired anonymity level (NBO).
52 */
53 uint32_t anonymity GNUNET_PACKED;
54
55 /**
56 * Desired replication level (NBO).
57 */
58 uint32_t replication GNUNET_PACKED;
59
60 /**
61 * Expiration time (NBO).
62 */
63 struct GNUNET_TIME_AbsoluteNBO expiration;
64
65 /**
66 * Key under which the item can be found.
67 */
68 struct GNUNET_HashCode key;
69};
70GNUNET_NETWORK_STRUCT_END
71
72
73/**
74 * Length of our magic header.
75 */
76static const size_t MAGIC_LEN = 16;
77
78/**
79 * Magic header bytes.
80 */
81static const uint8_t MAGIC_BYTES[16] = "GNUNETDATASTORE1";
82
83/**
84 * Dump the database.
85 */
86static int dump;
87
88/**
89 * Insert into the database.
90 */
91static int insert;
92
93/**
94 * Dump file name.
95 */
96static char *file_name;
97
98/**
99 * Dump file handle.
100 */
101static struct GNUNET_DISK_FileHandle *file_handle;
102
103/**
104 * Global return value.
105 */
106static int ret;
107
108/**
109 * Handle for datastore.
110 */
111static struct GNUNET_DATASTORE_Handle *datastore;
112
113/**
114 * Current operation.
115 */
116static struct GNUNET_DATASTORE_QueueEntry *qe;
117
118/**
119 * Record count.
120 */
121static uint64_t record_count;
122
123
124static void
125do_shutdown (void *cls)
126{
127 if (NULL != qe)
128 GNUNET_DATASTORE_cancel (qe);
129 if (NULL != datastore)
130 GNUNET_DATASTORE_disconnect (datastore, GNUNET_NO);
131 if (NULL != file_handle)
132 GNUNET_DISK_file_close (file_handle);
133}
134
135
136/**
137 * Begin dumping the database.
138 */
139static void
140start_dump (void);
141
142
143/**
144 * Begin inserting into the database.
145 */
146static void
147start_insert (void);
148
149
150/**
151 * Perform next GET operation.
152 */
153static void
154do_get (const uint64_t next_uid);
155
156
157/**
158 * Process a datum that was stored in the datastore.
159 *
160 * @param cls closure
161 * @param key key for the content
162 * @param size number of bytes in data
163 * @param data content stored
164 * @param type type of the content
165 * @param priority priority of the content
166 * @param anonymity anonymity-level for the content
167 * @param replication replication-level for the content
168 * @param expiration expiration time for the content
169 * @param uid unique identifier for the datum;
170 * maybe 0 if no unique identifier is available
171 */
172static void
173get_cb (void *cls,
174 const struct GNUNET_HashCode *key,
175 size_t size,
176 const void *data,
177 enum GNUNET_BLOCK_Type type,
178 uint32_t priority,
179 uint32_t anonymity,
180 uint32_t replication,
181 struct GNUNET_TIME_Absolute expiration,
182 uint64_t uid)
183{
184 qe = NULL;
185 if (NULL == key)
186 {
187 fprintf (stderr, _ ("Dumped %" PRIu64 " records\n"), record_count);
188 GNUNET_DISK_file_close (file_handle);
189 file_handle = NULL;
190 if (insert)
191 start_insert ();
192 else
193 {
194 ret = 0;
195 GNUNET_SCHEDULER_shutdown ();
196 }
197 return;
198 }
199
200 struct DataRecord dr;
201 dr.size = htonl ((uint32_t) size);
202 dr.type = htonl (type);
203 dr.priority = htonl (priority);
204 dr.anonymity = htonl (anonymity);
205 dr.replication = htonl (replication);
206 dr.expiration = GNUNET_TIME_absolute_hton (expiration);
207 dr.key = *key;
208
209 ssize_t len;
210 len = GNUNET_DISK_file_write (file_handle, &dr, sizeof(dr));
211 if (sizeof(dr) != len)
212 {
213 fprintf (stderr,
214 _ ("Short write to file: %zd bytes expecting %zd\n"),
215 len,
216 sizeof(dr));
217 ret = 1;
218 GNUNET_SCHEDULER_shutdown ();
219 return;
220 }
221
222 len = GNUNET_DISK_file_write (file_handle, data, size);
223 if (size != len)
224 {
225 fprintf (stderr,
226 _ ("Short write to file: %zd bytes expecting %zd\n"),
227 len,
228 size);
229 ret = 1;
230 GNUNET_SCHEDULER_shutdown ();
231 return;
232 }
233
234 record_count++;
235 do_get (uid + 1);
236}
237
238
239/**
240 * Perform next GET operation.
241 */
242static void
243do_get (const uint64_t next_uid)
244{
245 GNUNET_assert (NULL == qe);
246 qe = GNUNET_DATASTORE_get_key (datastore,
247 next_uid,
248 false /* random */,
249 NULL /* key */,
250 GNUNET_BLOCK_TYPE_ANY,
251 0 /* queue_priority */,
252 1 /* max_queue_size */,
253 &get_cb,
254 NULL /* proc_cls */);
255 if (NULL == qe)
256 {
257 fprintf (stderr, _ ("Error queueing datastore GET operation\n"));
258 ret = 1;
259 GNUNET_SCHEDULER_shutdown ();
260 }
261}
262
263
264/**
265 * Begin dumping the database.
266 */
267static void
268start_dump ()
269{
270 record_count = 0;
271
272 if (NULL != file_name)
273 {
274 file_handle = GNUNET_DISK_file_open (file_name,
275 GNUNET_DISK_OPEN_WRITE
276 | GNUNET_DISK_OPEN_TRUNCATE
277 | GNUNET_DISK_OPEN_CREATE,
278 GNUNET_DISK_PERM_USER_READ
279 | GNUNET_DISK_PERM_USER_WRITE);
280 if (NULL == file_handle)
281 {
282 fprintf (stderr, _ ("Unable to open dump file: %s\n"), file_name);
283 ret = 1;
284 GNUNET_SCHEDULER_shutdown ();
285 return;
286 }
287 }
288 else
289 {
290 file_handle = GNUNET_DISK_get_handle_from_int_fd (STDOUT_FILENO);
291 }
292 GNUNET_DISK_file_write (file_handle, MAGIC_BYTES, MAGIC_LEN);
293 do_get (0);
294}
295
296
297/**
298 * Continuation called to notify client about result of the
299 * operation.
300 *
301 * @param cls closure
302 * @param success GNUNET_SYSERR on failure (including timeout/queue drop)
303 * GNUNET_NO if content was already there
304 * GNUNET_YES (or other positive value) on success
305 * @param min_expiration minimum expiration time required for 0-priority content to be stored
306 * by the datacache at this time, zero for unknown, forever if we have no
307 * space for 0-priority content
308 * @param msg NULL on success, otherwise an error message
309 */
310static void
311put_cb (void *cls,
312 int32_t success,
313 struct GNUNET_TIME_Absolute min_expiration,
314 const char *msg)
315{
316 qe = NULL;
317 if (GNUNET_SYSERR == success)
318 {
319 fprintf (stderr, _ ("Failed to store item: %s, aborting\n"), msg);
320 ret = 1;
321 GNUNET_SCHEDULER_shutdown ();
322 return;
323 }
324
325 struct DataRecord dr;
326 ssize_t len;
327
328 len = GNUNET_DISK_file_read (file_handle, &dr, sizeof(dr));
329 if (0 == len)
330 {
331 fprintf (stderr, _ ("Inserted %" PRIu64 " records\n"), record_count);
332 ret = 0;
333 GNUNET_SCHEDULER_shutdown ();
334 return;
335 }
336 else if (sizeof(dr) != len)
337 {
338 fprintf (stderr,
339 _ ("Short read from file: %zd bytes expecting %zd\n"),
340 len,
341 sizeof(dr));
342 ret = 1;
343 GNUNET_SCHEDULER_shutdown ();
344 return;
345 }
346
347 const size_t size = ntohl (dr.size);
348 uint8_t data[size];
349 len = GNUNET_DISK_file_read (file_handle, data, size);
350 if (size != len)
351 {
352 fprintf (stderr,
353 _ ("Short read from file: %zd bytes expecting %zd\n"),
354 len,
355 size);
356 ret = 1;
357 GNUNET_SCHEDULER_shutdown ();
358 return;
359 }
360
361 record_count++;
362 qe = GNUNET_DATASTORE_put (datastore,
363 0,
364 &dr.key,
365 size,
366 data,
367 ntohl (dr.type),
368 ntohl (dr.priority),
369 ntohl (dr.anonymity),
370 ntohl (dr.replication),
371 GNUNET_TIME_absolute_ntoh (dr.expiration),
372 0,
373 1,
374 &put_cb,
375 NULL);
376 if (NULL == qe)
377 {
378 fprintf (stderr, _ ("Error queueing datastore PUT operation\n"));
379 ret = 1;
380 GNUNET_SCHEDULER_shutdown ();
381 }
382}
383
384
385/**
386 * Begin inserting into the database.
387 */
388static void
389start_insert ()
390{
391 record_count = 0;
392
393 if (NULL != file_name)
394 {
395 file_handle = GNUNET_DISK_file_open (file_name,
396 GNUNET_DISK_OPEN_READ,
397 GNUNET_DISK_PERM_NONE);
398 if (NULL == file_handle)
399 {
400 fprintf (stderr, _ ("Unable to open dump file: %s\n"), file_name);
401 ret = 1;
402 GNUNET_SCHEDULER_shutdown ();
403 return;
404 }
405 }
406 else
407 {
408 file_handle = GNUNET_DISK_get_handle_from_int_fd (STDIN_FILENO);
409 }
410
411 uint8_t buf[MAGIC_LEN];
412 ssize_t len;
413
414 len = GNUNET_DISK_file_read (file_handle, buf, MAGIC_LEN);
415 if ((len != MAGIC_LEN) || (0 != memcmp (buf, MAGIC_BYTES, MAGIC_LEN)))
416 {
417 fprintf (stderr, _ ("Input file is not of a supported format\n"));
418 return;
419 }
420 put_cb (NULL, GNUNET_YES, GNUNET_TIME_UNIT_ZERO_ABS, NULL);
421}
422
423
424/**
425 * Main function that will be run by the scheduler.
426 *
427 * @param cls closure
428 * @param args remaining command-line arguments
429 * @param cfgfile name of the configuration file used
430 * @param cfg configuration
431 */
432static void
433run (void *cls,
434 char *const *args,
435 const char *cfgfile,
436 const struct GNUNET_CONFIGURATION_Handle *cfg)
437{
438 GNUNET_SCHEDULER_add_shutdown (&do_shutdown, NULL);
439 datastore = GNUNET_DATASTORE_connect (cfg);
440 if (NULL == datastore)
441 {
442 fprintf (stderr, _ ("Failed connecting to the datastore.\n"));
443 ret = 1;
444 GNUNET_SCHEDULER_shutdown ();
445 return;
446 }
447 if (dump)
448 start_dump ();
449 else if (insert)
450 start_insert ();
451 else
452 {
453 fprintf (stderr,
454 _ ("Please choose at least one operation: %s, %s\n"),
455 "dump",
456 "insert");
457 ret = 1;
458 GNUNET_SCHEDULER_shutdown ();
459 }
460}
461
462
463/**
464 * The main function to manipulate datastores.
465 *
466 * @param argc number of arguments from the command line
467 * @param argv command line arguments
468 * @return 0 ok, 1 on error
469 */
470int
471main (int argc, char *const *argv)
472{
473 struct GNUNET_GETOPT_CommandLineOption options[] =
474 { GNUNET_GETOPT_option_flag ('d',
475 "dump",
476 gettext_noop (
477 "Dump all records from the datastore"),
478 &dump),
479 GNUNET_GETOPT_option_flag ('i',
480 "insert",
481 gettext_noop (
482 "Insert records into the datastore"),
483 &insert),
484 GNUNET_GETOPT_option_filename ('f',
485 "file",
486 "FILENAME",
487 gettext_noop ("File to dump or insert"),
488 &file_name),
489 GNUNET_GETOPT_OPTION_END };
490
491 if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
492 return 2;
493
494 if (GNUNET_OK !=
495 GNUNET_PROGRAM_run (argc,
496 argv,
497 "gnunet-datastore",
498 gettext_noop ("Manipulate GNUnet datastore"),
499 options,
500 &run,
501 NULL))
502 ret = 1;
503 GNUNET_free_nz ((void *) argv);
504 return ret;
505}
506
507
508/* end of gnunet-datastore.c */
diff --git a/src/datastore/gnunet-service-datastore.c b/src/datastore/gnunet-service-datastore.c
deleted file mode 100644
index f45e71ee9..000000000
--- a/src/datastore/gnunet-service-datastore.c
+++ /dev/null
@@ -1,1647 +0,0 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2004-2014, 2016 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file datastore/gnunet-service-datastore.c
23 * @brief Management for the datastore for files stored on a GNUnet node
24 * @author Christian Grothoff
25 */
26
27#include "platform.h"
28#include "gnunet_util_lib.h"
29#include "gnunet_protocols.h"
30#include "gnunet_statistics_service.h"
31#include "gnunet_datastore_plugin.h"
32#include "datastore.h"
33
34/**
35 * How many messages do we queue at most per client?
36 */
37#define MAX_PENDING 1024
38
39/**
40 * Limit size of bloom filter to 2 GB.
41 */
42#define MAX_BF_SIZE ((uint32_t) (1LL << 31))
43
44/**
45 * How long are we at most keeping "expired" content
46 * past the expiration date in the database?
47 */
48#define MAX_EXPIRE_DELAY \
49 GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15)
50
51/**
52 * How fast are we allowed to query the database for deleting
53 * expired content? (1 item per second).
54 */
55#define MIN_EXPIRE_DELAY \
56 GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 1)
57
58/**
59 * Name under which we store current space consumption.
60 */
61static char *quota_stat_name;
62
63/**
64 * Task to timeout stat GET.
65 */
66static struct GNUNET_SCHEDULER_Task *stat_timeout_task;
67
68/**
69 * After how many payload-changing operations
70 * do we sync our statistics?
71 */
72#define MAX_STAT_SYNC_LAG 50
73
74
75/**
76 * Our datastore plugin.
77 */
78struct DatastorePlugin
79{
80 /**
81 * API of the transport as returned by the plugin's
82 * initialization function.
83 */
84 struct GNUNET_DATASTORE_PluginFunctions *api;
85
86 /**
87 * Short name for the plugin (e.g. "sqlite").
88 */
89 char *short_name;
90
91 /**
92 * Name of the library (e.g. "gnunet_plugin_datastore_sqlite").
93 */
94 char *lib_name;
95
96 /**
97 * Environment this transport service is using
98 * for this plugin.
99 */
100 struct GNUNET_DATASTORE_PluginEnvironment env;
101};
102
103
104/**
105 * Linked list of active reservations.
106 */
107struct ReservationList
108{
109 /**
110 * This is a linked list.
111 */
112 struct ReservationList *next;
113
114 /**
115 * Client that made the reservation.
116 */
117 struct GNUNET_SERVICE_Client *client;
118
119 /**
120 * Number of bytes (still) reserved.
121 */
122 uint64_t amount;
123
124 /**
125 * Number of items (still) reserved.
126 */
127 uint64_t entries;
128
129 /**
130 * Reservation identifier.
131 */
132 int32_t rid;
133};
134
135
136/**
137 * Our datastore plugin (NULL if not available).
138 */
139static struct DatastorePlugin *plugin;
140
141/**
142 * Linked list of space reservations made by clients.
143 */
144static struct ReservationList *reservations;
145
146/**
147 * Bloomfilter to quickly tell if we don't have the content.
148 */
149static struct GNUNET_CONTAINER_BloomFilter *filter;
150
151/**
152 * Name of our plugin.
153 */
154static char *plugin_name;
155
156/**
157 * Our configuration.
158 */
159static const struct GNUNET_CONFIGURATION_Handle *cfg;
160
161/**
162 * Handle for reporting statistics.
163 */
164static struct GNUNET_STATISTICS_Handle *stats;
165
166/**
167 * How much space are we using for the cache? (space available for
168 * insertions that will be instantly reclaimed by discarding less
169 * important content --- or possibly whatever we just inserted into
170 * the "cache").
171 */
172static unsigned long long cache_size;
173
174/**
175 * How much space have we currently reserved?
176 */
177static unsigned long long reserved;
178
179/**
180 * How much data are we currently storing
181 * in the database?
182 */
183static unsigned long long payload;
184
185/**
186 * Identity of the task that is used to delete
187 * expired content.
188 */
189static struct GNUNET_SCHEDULER_Task *expired_kill_task;
190
191/**
192 * Minimum time that content should have to not be discarded instantly
193 * (time stamp of any content that we've been discarding recently to
194 * stay below the quota). FOREVER if we had to expire content with
195 * non-zero priority.
196 */
197static struct GNUNET_TIME_Absolute min_expiration;
198
199/**
200 * How much space are we allowed to use?
201 */
202static unsigned long long quota;
203
204/**
205 * Should the database be dropped on exit?
206 */
207static int do_drop;
208
209/**
210 * Should we refresh the BF when the DB is loaded?
211 */
212static int refresh_bf;
213
214/**
215 * Number of updates that were made to the
216 * payload value since we last synchronized
217 * it with the statistics service.
218 */
219static unsigned int last_sync;
220
221/**
222 * Did we get an answer from statistics?
223 */
224static int stats_worked;
225
226
227/**
228 * Synchronize our utilization statistics with the
229 * statistics service.
230 */
231static void
232sync_stats ()
233{
234 GNUNET_STATISTICS_set (stats, quota_stat_name, payload, GNUNET_YES);
235 GNUNET_STATISTICS_set (stats,
236 "# utilization by current datastore",
237 payload,
238 GNUNET_NO);
239 last_sync = 0;
240}
241
242
243/**
244 * Have we already cleaned up the TCCs and are hence no longer
245 * willing (or able) to transmit anything to anyone?
246 */
247static int cleaning_done;
248
249/**
250 * Handle for pending get request.
251 */
252static struct GNUNET_STATISTICS_GetHandle *stat_get;
253
254/**
255 * Handle to our server.
256 */
257static struct GNUNET_SERVICE_Handle *service;
258
259/**
260 * Task that is used to remove expired entries from
261 * the datastore. This task will schedule itself
262 * again automatically to always delete all expired
263 * content quickly.
264 *
265 * @param cls not used
266 */
267static void
268delete_expired (void *cls);
269
270
271/**
272 * Iterate over the expired items stored in the datastore.
273 * Delete all expired items; once we have processed all
274 * expired items, re-schedule the "delete_expired" task.
275 *
276 * @param cls not used
277 * @param key key for the content
278 * @param size number of bytes in data
279 * @param data content stored
280 * @param type type of the content
281 * @param priority priority of the content
282 * @param anonymity anonymity-level for the content
283 * @param replication replication-level for the content
284 * @param expiration expiration time for the content
285 * @param uid unique identifier for the datum;
286 * maybe 0 if no unique identifier is available
287 *
288 * @return #GNUNET_SYSERR to abort the iteration, #GNUNET_OK to continue
289 * (continue on call to "next", of course),
290 * #GNUNET_NO to delete the item and continue (if supported)
291 */
292static int
293expired_processor (void *cls,
294 const struct GNUNET_HashCode *key,
295 uint32_t size,
296 const void *data,
297 enum GNUNET_BLOCK_Type type,
298 uint32_t priority,
299 uint32_t anonymity,
300 uint32_t replication,
301 struct GNUNET_TIME_Absolute expiration,
302 uint64_t uid)
303{
304 struct GNUNET_TIME_Absolute now;
305
306 if (NULL == key)
307 {
308 expired_kill_task =
309 GNUNET_SCHEDULER_add_delayed_with_priority (MAX_EXPIRE_DELAY,
310 GNUNET_SCHEDULER_PRIORITY_IDLE,
311 &delete_expired,
312 NULL);
313 return GNUNET_SYSERR;
314 }
315 now = GNUNET_TIME_absolute_get ();
316 if (expiration.abs_value_us > now.abs_value_us)
317 {
318 /* finished processing */
319 expired_kill_task =
320 GNUNET_SCHEDULER_add_delayed_with_priority (MAX_EXPIRE_DELAY,
321 GNUNET_SCHEDULER_PRIORITY_IDLE,
322 &delete_expired,
323 NULL);
324 return GNUNET_SYSERR;
325 }
326 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
327 "Deleting content `%s' of type %u that expired %s ago\n",
328 GNUNET_h2s (key),
329 type,
330 GNUNET_STRINGS_relative_time_to_string (
331 GNUNET_TIME_absolute_get_difference (expiration, now),
332 GNUNET_YES));
333 min_expiration = now;
334 GNUNET_STATISTICS_update (stats,
335 gettext_noop ("# bytes expired"),
336 size,
337 GNUNET_YES);
338 GNUNET_CONTAINER_bloomfilter_remove (filter, key);
339 expired_kill_task =
340 GNUNET_SCHEDULER_add_delayed_with_priority (MIN_EXPIRE_DELAY,
341 GNUNET_SCHEDULER_PRIORITY_IDLE,
342 &delete_expired,
343 NULL);
344 return GNUNET_NO;
345}
346
347
348/**
349 * Task that is used to remove expired entries from
350 * the datastore. This task will schedule itself
351 * again automatically to always delete all expired
352 * content quickly.
353 *
354 * @param cls not used
355 */
356static void
357delete_expired (void *cls)
358{
359 expired_kill_task = NULL;
360 plugin->api->get_expiration (plugin->api->cls, &expired_processor, NULL);
361}
362
363
364/**
365 * An iterator over a set of items stored in the datastore
366 * that deletes until we're happy with respect to our quota.
367 *
368 * @param cls closure
369 * @param key key for the content
370 * @param size number of bytes in data
371 * @param data content stored
372 * @param type type of the content
373 * @param priority priority of the content
374 * @param anonymity anonymity-level for the content
375 * @param replication replication-level for the content
376 * @param expiration expiration time for the content
377 * @param uid unique identifier for the datum;
378 * maybe 0 if no unique identifier is available
379 * @return #GNUNET_SYSERR to abort the iteration, #GNUNET_OK to continue
380 * (continue on call to "next", of course),
381 * #GNUNET_NO to delete the item and continue (if supported)
382 */
383static int
384quota_processor (void *cls,
385 const struct GNUNET_HashCode *key,
386 uint32_t size,
387 const void *data,
388 enum GNUNET_BLOCK_Type type,
389 uint32_t priority,
390 uint32_t anonymity,
391 uint32_t replication,
392 struct GNUNET_TIME_Absolute expiration,
393 uint64_t uid)
394{
395 unsigned long long *need = cls;
396
397 if (NULL == key)
398 return GNUNET_SYSERR;
399 GNUNET_log (
400 GNUNET_ERROR_TYPE_DEBUG,
401 "Deleting %llu bytes of low-priority (%u) content `%s' of type %u at %s prior to expiration (still trying to free another %llu bytes)\n",
402 (unsigned long long) (size + GNUNET_DATASTORE_ENTRY_OVERHEAD),
403 (unsigned int) priority,
404 GNUNET_h2s (key),
405 type,
406 GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_remaining (
407 expiration),
408 GNUNET_YES),
409 *need);
410 if (size + GNUNET_DATASTORE_ENTRY_OVERHEAD > *need)
411 *need = 0;
412 else
413 *need -= size + GNUNET_DATASTORE_ENTRY_OVERHEAD;
414 if (priority > 0)
415 min_expiration = GNUNET_TIME_UNIT_FOREVER_ABS;
416 else
417 min_expiration = expiration;
418 GNUNET_STATISTICS_update (stats,
419 gettext_noop ("# bytes purged (low-priority)"),
420 size,
421 GNUNET_YES);
422 GNUNET_CONTAINER_bloomfilter_remove (filter, key);
423 return GNUNET_NO;
424}
425
426
427/**
428 * Manage available disk space by running tasks
429 * that will discard content if necessary. This
430 * function will be run whenever a request for
431 * "need" bytes of storage could only be satisfied
432 * by eating into the "cache" (and we want our cache
433 * space back).
434 *
435 * @param need number of bytes of content that were
436 * placed into the "cache" (and hence the
437 * number of bytes that should be removed).
438 */
439static void
440manage_space (unsigned long long need)
441{
442 unsigned long long last;
443
444 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
445 "Asked to free up %llu bytes of cache space\n",
446 need);
447 last = 0;
448 while ((need > 0) && (last != need))
449 {
450 last = need;
451 plugin->api->get_expiration (plugin->api->cls, &quota_processor, &need);
452 }
453}
454
455
456/**
457 * Transmit a status code to the client.
458 *
459 * @param client receiver of the response
460 * @param code status code
461 * @param msg optional error message (can be NULL)
462 */
463static void
464transmit_status (struct GNUNET_SERVICE_Client *client,
465 int code,
466 const char *msg)
467{
468 struct GNUNET_MQ_Envelope *env;
469 struct StatusMessage *sm;
470 size_t slen;
471
472 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
473 "Transmitting `%s' message with value %d and message `%s'\n",
474 "STATUS",
475 code,
476 msg != NULL ? msg : "(none)");
477 slen = (msg == NULL) ? 0 : strlen (msg) + 1;
478 env = GNUNET_MQ_msg_extra (sm, slen, GNUNET_MESSAGE_TYPE_DATASTORE_STATUS);
479 sm->status = htonl (code);
480 sm->min_expiration = GNUNET_TIME_absolute_hton (min_expiration);
481 GNUNET_memcpy (&sm[1], msg, slen);
482 GNUNET_MQ_send (GNUNET_SERVICE_client_get_mq (client), env);
483}
484
485
486/**
487 * Function that will transmit the given datastore entry
488 * to the client.
489 *
490 * @param cls closure, pointer to the client (of type `struct GNUNET_SERVICE_Client`).
491 * @param key key for the content
492 * @param size number of bytes in data
493 * @param data content stored
494 * @param type type of the content
495 * @param priority priority of the content
496 * @param anonymity anonymity-level for the content
497 * @param replication replication-level for the content
498 * @param expiration expiration time for the content
499 * @param uid unique identifier for the datum;
500 * maybe 0 if no unique identifier is available
501 * @return #GNUNET_SYSERR to abort the iteration, #GNUNET_OK to continue,
502 * #GNUNET_NO to delete the item and continue (if supported)
503 */
504static int
505transmit_item (void *cls,
506 const struct GNUNET_HashCode *key,
507 uint32_t size,
508 const void *data,
509 enum GNUNET_BLOCK_Type type,
510 uint32_t priority,
511 uint32_t anonymity,
512 uint32_t replication,
513 struct GNUNET_TIME_Absolute expiration,
514 uint64_t uid)
515{
516 struct GNUNET_SERVICE_Client *client = cls;
517 struct GNUNET_MQ_Envelope *env;
518 struct GNUNET_MessageHeader *end;
519 struct DataMessage *dm;
520
521 if (NULL == key)
522 {
523 /* transmit 'DATA_END' */
524 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Transmitting DATA_END message\n");
525 env = GNUNET_MQ_msg (end, GNUNET_MESSAGE_TYPE_DATASTORE_DATA_END);
526 GNUNET_MQ_send (GNUNET_SERVICE_client_get_mq (client), env);
527 return GNUNET_OK;
528 }
529 GNUNET_assert (sizeof(struct DataMessage) + size < GNUNET_MAX_MESSAGE_SIZE);
530 env = GNUNET_MQ_msg_extra (dm, size, GNUNET_MESSAGE_TYPE_DATASTORE_DATA);
531 dm->rid = htonl (0);
532 dm->size = htonl (size);
533 dm->type = htonl (type);
534 dm->priority = htonl (priority);
535 dm->anonymity = htonl (anonymity);
536 dm->replication = htonl (replication);
537 dm->expiration = GNUNET_TIME_absolute_hton (expiration);
538 dm->uid = GNUNET_htonll (uid);
539 dm->key = *key;
540 GNUNET_memcpy (&dm[1], data, size);
541 GNUNET_log (
542 GNUNET_ERROR_TYPE_DEBUG,
543 "Transmitting DATA message for `%s' of type %u with expiration %s (in: %s)\n",
544 GNUNET_h2s (key),
545 type,
546 GNUNET_STRINGS_absolute_time_to_string (expiration),
547 GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_remaining (
548 expiration),
549 GNUNET_YES));
550 GNUNET_STATISTICS_update (stats,
551 gettext_noop ("# results found"),
552 1,
553 GNUNET_NO);
554 GNUNET_MQ_send (GNUNET_SERVICE_client_get_mq (client), env);
555 return GNUNET_OK;
556}
557
558
559/**
560 * Handle RESERVE-message.
561 *
562 * @param cls identification of the client
563 * @param msg the actual message
564 */
565static void
566handle_reserve (void *cls, const struct ReserveMessage *msg)
567{
568 /**
569 * Static counter to produce reservation identifiers.
570 */
571 static int reservation_gen;
572 struct GNUNET_SERVICE_Client *client = cls;
573 struct ReservationList *e;
574 unsigned long long used;
575 unsigned long long req;
576 uint64_t amount;
577 uint32_t entries;
578
579 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Processing RESERVE request\n");
580 amount = GNUNET_ntohll (msg->amount);
581 entries = ntohl (msg->entries);
582 used = payload + reserved;
583 req =
584 amount + ((unsigned long long) GNUNET_DATASTORE_ENTRY_OVERHEAD) * entries;
585 if (used + req > quota)
586 {
587 if (quota < used)
588 used =
589 quota; /* cheat a bit for error message (to avoid negative numbers) */
590 GNUNET_log (
591 GNUNET_ERROR_TYPE_WARNING,
592 _ (
593 "Insufficient space (%llu bytes are available) to satisfy RESERVE request for %llu bytes\n"),
594 quota - used,
595 req);
596 if (cache_size < req)
597 {
598 /* TODO: document this in the FAQ; essentially, if this
599 * message happens, the insertion request could be blocked
600 * by less-important content from migration because it is
601 * larger than 1/8th of the overall available space, and
602 * we only reserve 1/8th for "fresh" insertions */
603 GNUNET_log (
604 GNUNET_ERROR_TYPE_WARNING,
605 _ (
606 "The requested amount (%llu bytes) is larger than the cache size (%llu bytes)\n"),
607 req,
608 cache_size);
609 transmit_status (client,
610 0,
611 gettext_noop (
612 "Insufficient space to satisfy request and "
613 "requested amount is larger than cache size"));
614 }
615 else
616 {
617 transmit_status (client,
618 0,
619 gettext_noop ("Insufficient space to satisfy request"));
620 }
621 GNUNET_SERVICE_client_continue (client);
622 return;
623 }
624 reserved += req;
625 GNUNET_STATISTICS_set (stats,
626 gettext_noop ("# reserved"),
627 reserved,
628 GNUNET_NO);
629 e = GNUNET_new (struct ReservationList);
630 e->next = reservations;
631 reservations = e;
632 e->client = client;
633 e->amount = amount;
634 e->entries = entries;
635 e->rid = ++reservation_gen;
636 if (reservation_gen < 0)
637 reservation_gen = 0; /* wrap around */
638 transmit_status (client, e->rid, NULL);
639 GNUNET_SERVICE_client_continue (client);
640}
641
642
643/**
644 * Handle RELEASE_RESERVE-message.
645 *
646 * @param cls identification of the client
647 * @param msg the actual message
648 */
649static void
650handle_release_reserve (void *cls, const struct ReleaseReserveMessage *msg)
651{
652 struct GNUNET_SERVICE_Client *client = cls;
653 struct ReservationList *pos;
654 struct ReservationList *prev;
655 struct ReservationList *next;
656 int rid = ntohl (msg->rid);
657 unsigned long long rem;
658
659 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Processing RELEASE_RESERVE request\n");
660 next = reservations;
661 prev = NULL;
662 while (NULL != (pos = next))
663 {
664 next = pos->next;
665 if (rid == pos->rid)
666 {
667 if (prev == NULL)
668 reservations = next;
669 else
670 prev->next = next;
671 rem =
672 pos->amount
673 + ((unsigned long long) GNUNET_DATASTORE_ENTRY_OVERHEAD) * pos->entries;
674 GNUNET_assert (reserved >= rem);
675 reserved -= rem;
676 GNUNET_STATISTICS_set (stats,
677 gettext_noop ("# reserved"),
678 reserved,
679 GNUNET_NO);
680 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
681 "Returning %llu remaining reserved bytes to storage pool\n",
682 rem);
683 GNUNET_free (pos);
684 transmit_status (client, GNUNET_OK, NULL);
685 GNUNET_SERVICE_client_continue (client);
686 return;
687 }
688 prev = pos;
689 }
690 GNUNET_break (0);
691 transmit_status (client,
692 GNUNET_SYSERR,
693 gettext_noop ("Could not find matching reservation"));
694 GNUNET_SERVICE_client_continue (client);
695}
696
697
698/**
699 * Check that the given message is a valid data message.
700 *
701 * @param dm message to check
702 * @return #GNUNET_SYSERR is not well-formed, otherwise #GNUNET_OK
703 */
704static int
705check_data (const struct DataMessage *dm)
706{
707 uint16_t size;
708 uint32_t dsize;
709
710 size = ntohs (dm->header.size);
711 dsize = ntohl (dm->size);
712 if (size != dsize + sizeof(struct DataMessage))
713 {
714 GNUNET_break (0);
715 return GNUNET_SYSERR;
716 }
717 return GNUNET_OK;
718}
719
720
721/**
722 * Put continuation.
723 *
724 * @param cls closure
725 * @param key key for the item stored
726 * @param size size of the item stored
727 * @param status #GNUNET_OK if inserted, #GNUNET_NO if updated,
728 * or #GNUNET_SYSERROR if error
729 * @param msg error message on error
730 */
731static void
732put_continuation (void *cls,
733 const struct GNUNET_HashCode *key,
734 uint32_t size,
735 int status,
736 const char *msg)
737{
738 struct GNUNET_SERVICE_Client *client = cls;
739
740 if (GNUNET_OK == status)
741 {
742 GNUNET_STATISTICS_update (stats,
743 gettext_noop ("# bytes stored"),
744 size,
745 GNUNET_YES);
746 GNUNET_CONTAINER_bloomfilter_add (filter, key);
747 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
748 "Successfully stored %u bytes under key `%s'\n",
749 size,
750 GNUNET_h2s (key));
751 }
752 transmit_status (client,
753 GNUNET_SYSERR == status ? GNUNET_SYSERR : GNUNET_OK,
754 msg);
755 if (quota - reserved - cache_size < payload)
756 {
757 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
758 _ ("Need %llu bytes more space (%llu allowed, using %llu)\n"),
759 (unsigned long long) size + GNUNET_DATASTORE_ENTRY_OVERHEAD,
760 (unsigned long long) (quota - reserved - cache_size),
761 (unsigned long long) payload);
762 manage_space (size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
763 }
764}
765
766
767/**
768 * Verify PUT-message.
769 *
770 * @param cls identification of the client
771 * @param dm the actual message
772 * @return #GNUNET_OK if @a dm is well-formed
773 */
774static int
775check_put (void *cls, const struct DataMessage *dm)
776{
777 if (GNUNET_OK != check_data (dm))
778 {
779 GNUNET_break (0);
780 return GNUNET_SYSERR;
781 }
782 return GNUNET_OK;
783}
784
785
786/**
787 * Handle PUT-message.
788 *
789 * @param cls identification of the client
790 * @param dm the actual message
791 */
792static void
793handle_put (void *cls, const struct DataMessage *dm)
794{
795 struct GNUNET_SERVICE_Client *client = cls;
796 int rid;
797 struct ReservationList *pos;
798 uint32_t size;
799
800 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
801 "Processing PUT request for `%s' of type %u\n",
802 GNUNET_h2s (&dm->key),
803 (uint32_t) ntohl (dm->type));
804 rid = ntohl (dm->rid);
805 size = ntohl (dm->size);
806 if (rid > 0)
807 {
808 pos = reservations;
809 while ((NULL != pos) && (rid != pos->rid))
810 pos = pos->next;
811 GNUNET_break (pos != NULL);
812 if (NULL != pos)
813 {
814 GNUNET_break (pos->entries > 0);
815 GNUNET_break (pos->amount >= size);
816 pos->entries--;
817 pos->amount -= size;
818 reserved -= (size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
819 GNUNET_STATISTICS_set (stats,
820 gettext_noop ("# reserved"),
821 reserved,
822 GNUNET_NO);
823 }
824 }
825 bool absent =
826 GNUNET_NO == GNUNET_CONTAINER_bloomfilter_test (filter, &dm->key);
827 plugin->api->put (plugin->api->cls,
828 &dm->key,
829 absent,
830 ntohl (dm->size),
831 &dm[1],
832 ntohl (dm->type),
833 ntohl (dm->priority),
834 ntohl (dm->anonymity),
835 ntohl (dm->replication),
836 GNUNET_TIME_absolute_ntoh (dm->expiration),
837 &put_continuation,
838 client);
839 GNUNET_SERVICE_client_continue (client);
840}
841
842
843/**
844 * Handle #GNUNET_MESSAGE_TYPE_DATASTORE_GET-message.
845 *
846 * @param cls identification of the client
847 * @param msg the actual message
848 */
849static void
850handle_get (void *cls, const struct GetMessage *msg)
851{
852 struct GNUNET_SERVICE_Client *client = cls;
853
854 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
855 "Processing GET request of type %u\n",
856 (uint32_t) ntohl (msg->type));
857 GNUNET_STATISTICS_update (stats,
858 gettext_noop ("# GET requests received"),
859 1,
860 GNUNET_NO);
861 plugin->api->get_key (plugin->api->cls,
862 GNUNET_ntohll (msg->next_uid),
863 msg->random,
864 NULL,
865 ntohl (msg->type),
866 &transmit_item,
867 client);
868 GNUNET_SERVICE_client_continue (client);
869}
870
871
872/**
873 * Handle #GNUNET_MESSAGE_TYPE_DATASTORE_GET_KEY-message.
874 *
875 * @param cls closure
876 * @param msg the actual message
877 */
878static void
879handle_get_key (void *cls, const struct GetKeyMessage *msg)
880{
881 struct GNUNET_SERVICE_Client *client = cls;
882
883 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
884 "Processing GET request for `%s' of type %u\n",
885 GNUNET_h2s (&msg->key),
886 (uint32_t) ntohl (msg->type));
887 GNUNET_STATISTICS_update (stats,
888 gettext_noop ("# GET KEY requests received"),
889 1,
890 GNUNET_NO);
891 if (GNUNET_YES != GNUNET_CONTAINER_bloomfilter_test (filter, &msg->key))
892 {
893 /* don't bother database... */
894 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
895 "Empty result set for GET request for `%s' (bloomfilter).\n",
896 GNUNET_h2s (&msg->key));
897 GNUNET_STATISTICS_update (stats,
898 gettext_noop (
899 "# requests filtered by bloomfilter"),
900 1,
901 GNUNET_NO);
902 transmit_item (client,
903 NULL,
904 0,
905 NULL,
906 0,
907 0,
908 0,
909 0,
910 GNUNET_TIME_UNIT_ZERO_ABS,
911 0);
912 GNUNET_SERVICE_client_continue (client);
913 return;
914 }
915 plugin->api->get_key (plugin->api->cls,
916 GNUNET_ntohll (msg->next_uid),
917 msg->random,
918 &msg->key,
919 ntohl (msg->type),
920 &transmit_item,
921 client);
922 GNUNET_SERVICE_client_continue (client);
923}
924
925
926/**
927 * Handle GET_REPLICATION-message.
928 *
929 * @param cls identification of the client
930 * @param message the actual message
931 */
932static void
933handle_get_replication (void *cls, const struct GNUNET_MessageHeader *message)
934{
935 struct GNUNET_SERVICE_Client *client = cls;
936
937 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Processing GET_REPLICATION request\n");
938 GNUNET_STATISTICS_update (stats,
939 gettext_noop (
940 "# GET REPLICATION requests received"),
941 1,
942 GNUNET_NO);
943 plugin->api->get_replication (plugin->api->cls, &transmit_item, client);
944 GNUNET_SERVICE_client_continue (client);
945}
946
947
948/**
949 * Handle GET_ZERO_ANONYMITY-message.
950 *
951 * @param cls client identification of the client
952 * @param msg the actual message
953 */
954static void
955handle_get_zero_anonymity (void *cls, const struct GetZeroAnonymityMessage *msg)
956{
957 struct GNUNET_SERVICE_Client *client = cls;
958 enum GNUNET_BLOCK_Type type;
959
960 type = (enum GNUNET_BLOCK_Type) ntohl (msg->type);
961 if (type == GNUNET_BLOCK_TYPE_ANY)
962 {
963 GNUNET_break (0);
964 GNUNET_SERVICE_client_drop (client);
965 return;
966 }
967 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
968 "Processing GET_ZERO_ANONYMITY request\n");
969 GNUNET_STATISTICS_update (stats,
970 gettext_noop (
971 "# GET ZERO ANONYMITY requests received"),
972 1,
973 GNUNET_NO);
974 plugin->api->get_zero_anonymity (plugin->api->cls,
975 GNUNET_ntohll (msg->next_uid),
976 type,
977 &transmit_item,
978 client);
979 GNUNET_SERVICE_client_continue (client);
980}
981
982
983/**
984 * Remove continuation.
985 *
986 * @param cls closure
987 * @param key key for the content
988 * @param size number of bytes in data
989 * @param status #GNUNET_OK if removed, #GNUNET_NO if not found,
990 * or #GNUNET_SYSERROR if error
991 * @param msg error message on error
992 */
993static void
994remove_continuation (void *cls,
995 const struct GNUNET_HashCode *key,
996 uint32_t size,
997 int status,
998 const char *msg)
999{
1000 struct GNUNET_SERVICE_Client *client = cls;
1001
1002 if (GNUNET_SYSERR == status)
1003 {
1004 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "REMOVE request failed: %s.\n", msg);
1005 transmit_status (client, GNUNET_NO, msg);
1006 return;
1007 }
1008 if (GNUNET_NO == status)
1009 {
1010 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1011 "Content not found for REMOVE request.\n");
1012 transmit_status (client, GNUNET_NO, _ ("Content not found"));
1013 return;
1014 }
1015 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1016 "Item matches REMOVE request for key `%s'.\n",
1017 GNUNET_h2s (key));
1018 GNUNET_STATISTICS_update (stats,
1019 gettext_noop ("# bytes removed (explicit request)"),
1020 size,
1021 GNUNET_YES);
1022 GNUNET_CONTAINER_bloomfilter_remove (filter, key);
1023 transmit_status (client, GNUNET_OK, NULL);
1024}
1025
1026
1027/**
1028 * Verify REMOVE-message.
1029 *
1030 * @param cls identification of the client
1031 * @param dm the actual message
1032 * @return #GNUNET_OK if @a dm is well-formed
1033 */
1034static int
1035check_remove (void *cls, const struct DataMessage *dm)
1036{
1037 if (GNUNET_OK != check_data (dm))
1038 {
1039 GNUNET_break (0);
1040 return GNUNET_SYSERR;
1041 }
1042 return GNUNET_OK;
1043}
1044
1045
1046/**
1047 * Handle REMOVE-message.
1048 *
1049 * @param cls closure
1050 * @param dm the actual message
1051 */
1052static void
1053handle_remove (void *cls, const struct DataMessage *dm)
1054{
1055 struct GNUNET_SERVICE_Client *client = cls;
1056
1057 GNUNET_STATISTICS_update (stats,
1058 gettext_noop ("# REMOVE requests received"),
1059 1,
1060 GNUNET_NO);
1061 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1062 "Processing REMOVE request for `%s'\n",
1063 GNUNET_h2s (&dm->key));
1064 plugin->api->remove_key (plugin->api->cls,
1065 &dm->key,
1066 ntohl (dm->size),
1067 &dm[1],
1068 &remove_continuation,
1069 client);
1070 GNUNET_SERVICE_client_continue (client);
1071}
1072
1073
1074/**
1075 * Handle DROP-message.
1076 *
1077 * @param cls identification of the client
1078 * @param message the actual message
1079 */
1080static void
1081handle_drop (void *cls, const struct GNUNET_MessageHeader *message)
1082{
1083 struct GNUNET_SERVICE_Client *client = cls;
1084
1085 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Processing DROP request\n");
1086 do_drop = GNUNET_YES;
1087 GNUNET_SERVICE_client_continue (client);
1088}
1089
1090
1091/**
1092 * Function called by plugins to notify us about a
1093 * change in their disk utilization.
1094 *
1095 * @param cls closure (NULL)
1096 * @param delta change in disk utilization,
1097 * 0 for "reset to empty"
1098 */
1099static void
1100disk_utilization_change_cb (void *cls, int delta)
1101{
1102 if ((delta < 0) && (payload < -delta))
1103 {
1104 GNUNET_log (
1105 GNUNET_ERROR_TYPE_WARNING,
1106 _ (
1107 "Datastore payload must have been inaccurate (%lld < %lld). Recomputing it.\n"),
1108 (long long) payload,
1109 (long long) -delta);
1110 plugin->api->estimate_size (plugin->api->cls, &payload);
1111 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1112 _ ("New payload: %lld\n"),
1113 (long long) payload);
1114 sync_stats ();
1115 return;
1116 }
1117 payload += delta;
1118 last_sync++;
1119 if (last_sync >= MAX_STAT_SYNC_LAG)
1120 sync_stats ();
1121}
1122
1123
1124/**
1125 * Callback function to process statistic values.
1126 *
1127 * @param cls closure (struct Plugin*)
1128 * @param subsystem name of subsystem that created the statistic
1129 * @param name the name of the datum
1130 * @param value the current value
1131 * @param is_persistent #GNUNET_YES if the value is persistent, #GNUNET_NO if not
1132 * @return #GNUNET_OK to continue, #GNUNET_SYSERR to abort iteration
1133 */
1134static int
1135process_stat_in (void *cls,
1136 const char *subsystem,
1137 const char *name,
1138 uint64_t value,
1139 int is_persistent)
1140{
1141 GNUNET_assert (GNUNET_NO == stats_worked);
1142 stats_worked = GNUNET_YES;
1143 payload += value;
1144 GNUNET_log (
1145 GNUNET_ERROR_TYPE_DEBUG,
1146 "Notification from statistics about existing payload (%llu), new payload is %llu\n",
1147 (unsigned long long) value,
1148 (unsigned long long) payload);
1149 return GNUNET_OK;
1150}
1151
1152
1153/**
1154 * Load the datastore plugin.
1155 */
1156static struct DatastorePlugin *
1157load_plugin ()
1158{
1159 struct DatastorePlugin *ret;
1160 char *libname;
1161
1162 ret = GNUNET_new (struct DatastorePlugin);
1163 ret->env.cfg = cfg;
1164 ret->env.duc = &disk_utilization_change_cb;
1165 ret->env.cls = NULL;
1166 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1167 _ ("Loading `%s' datastore plugin\n"),
1168 plugin_name);
1169 GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", plugin_name);
1170 ret->short_name = GNUNET_strdup (plugin_name);
1171 ret->lib_name = libname;
1172 ret->api = GNUNET_PLUGIN_load (libname, &ret->env);
1173 if (NULL == ret->api)
1174 {
1175 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1176 _ ("Failed to load datastore plugin for `%s'\n"),
1177 plugin_name);
1178 GNUNET_free (ret->short_name);
1179 GNUNET_free (libname);
1180 GNUNET_free (ret);
1181 return NULL;
1182 }
1183 return ret;
1184}
1185
1186
1187/**
1188 * Function called when the service shuts
1189 * down. Unloads our datastore plugin.
1190 *
1191 * @param plug plugin to unload
1192 */
1193static void
1194unload_plugin (struct DatastorePlugin *plug)
1195{
1196 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1197 "Datastore service is unloading plugin...\n");
1198 GNUNET_break (NULL == GNUNET_PLUGIN_unload (plug->lib_name, plug->api));
1199 GNUNET_free (plug->lib_name);
1200 GNUNET_free (plug->short_name);
1201 GNUNET_free (plug);
1202}
1203
1204
1205/**
1206 * Initialization complete, start operating the service.
1207 */
1208static void
1209begin_service ()
1210{
1211 GNUNET_SERVICE_resume (service);
1212 expired_kill_task =
1213 GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_IDLE,
1214 &delete_expired,
1215 NULL);
1216}
1217
1218
1219/**
1220 * Adds a given @a key to the bloomfilter in @a cls @a count times.
1221 *
1222 * @param cls the bloomfilter
1223 * @param key key to add
1224 * @param count number of times to add key
1225 */
1226static void
1227add_key_to_bloomfilter (void *cls,
1228 const struct GNUNET_HashCode *key,
1229 unsigned int count)
1230{
1231 struct GNUNET_CONTAINER_BloomFilter *bf = cls;
1232
1233 if (NULL == key)
1234 {
1235 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1236 _ ("Bloomfilter construction complete.\n"));
1237 begin_service ();
1238 return;
1239 }
1240
1241 while (0 < count--)
1242 GNUNET_CONTAINER_bloomfilter_add (bf, key);
1243}
1244
1245
1246/**
1247 * We finished receiving the statistic. Initialize the plugin; if
1248 * loading the statistic failed, run the estimator.
1249 *
1250 * @param cls NULL
1251 * @param success #GNUNET_NO if we failed to read the stat
1252 */
1253static void
1254process_stat_done (void *cls, int success)
1255{
1256 stat_get = NULL;
1257 if (NULL != stat_timeout_task)
1258 {
1259 GNUNET_SCHEDULER_cancel (stat_timeout_task);
1260 stat_timeout_task = NULL;
1261 }
1262 plugin = load_plugin ();
1263 if (NULL == plugin)
1264 {
1265 GNUNET_CONTAINER_bloomfilter_free (filter);
1266 filter = NULL;
1267 if (NULL != stats)
1268 {
1269 GNUNET_STATISTICS_destroy (stats, GNUNET_YES);
1270 stats = NULL;
1271 }
1272 return;
1273 }
1274
1275 if (GNUNET_NO == stats_worked)
1276 {
1277 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1278 "Failed to obtain value from statistics service, recomputing it\n");
1279 plugin->api->estimate_size (plugin->api->cls, &payload);
1280 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1281 _ ("New payload: %lld\n"),
1282 (long long) payload);
1283 }
1284
1285 if (GNUNET_YES == refresh_bf)
1286 {
1287 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1288 _ ("Rebuilding bloomfilter. Please be patient.\n"));
1289 if (NULL != plugin->api->get_keys)
1290 {
1291 plugin->api->get_keys (plugin->api->cls, &add_key_to_bloomfilter, filter);
1292 return;
1293 }
1294 else
1295 {
1296 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1297 _ (
1298 "Plugin does not support get_keys function. Please fix!\n"));
1299 }
1300 }
1301 begin_service ();
1302}
1303
1304
1305/**
1306 * Fetching stats took to long, run without.
1307 *
1308 * @param cls NULL
1309 */
1310static void
1311stat_timeout (void *cls)
1312{
1313 stat_timeout_task = NULL;
1314 GNUNET_STATISTICS_get_cancel (stat_get);
1315 process_stat_done (NULL, GNUNET_NO);
1316}
1317
1318
1319/**
1320 * Task run during shutdown.
1321 */
1322static void
1323cleaning_task (void *cls)
1324{
1325 cleaning_done = GNUNET_YES;
1326 if (NULL != expired_kill_task)
1327 {
1328 GNUNET_SCHEDULER_cancel (expired_kill_task);
1329 expired_kill_task = NULL;
1330 }
1331 if (GNUNET_YES == do_drop)
1332 {
1333 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Dropping database!\n");
1334 plugin->api->drop (plugin->api->cls);
1335 payload = 0;
1336 last_sync++;
1337 }
1338 if (NULL != plugin)
1339 {
1340 unload_plugin (plugin);
1341 plugin = NULL;
1342 }
1343 if (NULL != filter)
1344 {
1345 GNUNET_CONTAINER_bloomfilter_free (filter);
1346 filter = NULL;
1347 }
1348 if (NULL != stat_get)
1349 {
1350 GNUNET_STATISTICS_get_cancel (stat_get);
1351 stat_get = NULL;
1352 }
1353 if (NULL != stat_timeout_task)
1354 {
1355 GNUNET_SCHEDULER_cancel (stat_timeout_task);
1356 stat_timeout_task = NULL;
1357 }
1358 GNUNET_free (plugin_name);
1359 plugin_name = NULL;
1360 if (last_sync > 0)
1361 sync_stats ();
1362 if (NULL != stats)
1363 {
1364 GNUNET_STATISTICS_destroy (stats, GNUNET_YES);
1365 stats = NULL;
1366 }
1367 GNUNET_free (quota_stat_name);
1368 quota_stat_name = NULL;
1369}
1370
1371
1372/**
1373 * Add a client to our list of active clients.
1374 *
1375 * @param cls NULL
1376 * @param client client to add
1377 * @param mq message queue for @a client
1378 * @return @a client
1379 */
1380static void *
1381client_connect_cb (void *cls,
1382 struct GNUNET_SERVICE_Client *client,
1383 struct GNUNET_MQ_Handle *mq)
1384{
1385 return client;
1386}
1387
1388
1389/**
1390 * Called whenever a client is disconnected.
1391 * Frees our resources associated with that client.
1392 *
1393 * @param cls closure
1394 * @param client identification of the client
1395 * @param app_ctx must match @a client
1396 */
1397static void
1398client_disconnect_cb (void *cls,
1399 struct GNUNET_SERVICE_Client *client,
1400 void *app_ctx)
1401{
1402 struct ReservationList *pos;
1403 struct ReservationList *prev;
1404 struct ReservationList *next;
1405
1406 GNUNET_assert (app_ctx == client);
1407 prev = NULL;
1408 pos = reservations;
1409 while (NULL != pos)
1410 {
1411 next = pos->next;
1412 if (pos->client == client)
1413 {
1414 if (NULL == prev)
1415 reservations = next;
1416 else
1417 prev->next = next;
1418 reserved -= pos->amount + pos->entries * GNUNET_DATASTORE_ENTRY_OVERHEAD;
1419 GNUNET_free (pos);
1420 }
1421 else
1422 {
1423 prev = pos;
1424 }
1425 pos = next;
1426 }
1427 GNUNET_STATISTICS_set (stats,
1428 gettext_noop ("# reserved"),
1429 reserved,
1430 GNUNET_NO);
1431}
1432
1433
1434/**
1435 * Process datastore requests.
1436 *
1437 * @param cls closure
1438 * @param serv the initialized service
1439 * @param c configuration to use
1440 */
1441static void
1442run (void *cls,
1443 const struct GNUNET_CONFIGURATION_Handle *c,
1444 struct GNUNET_SERVICE_Handle *serv)
1445{
1446 char *fn;
1447 char *pfn;
1448 unsigned int bf_size;
1449
1450 service = serv;
1451 cfg = c;
1452 if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_string (cfg,
1453 "DATASTORE",
1454 "DATABASE",
1455 &plugin_name))
1456 {
1457 GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
1458 "DATABASE",
1459 "DATASTORE");
1460 return;
1461 }
1462 GNUNET_asprintf (&quota_stat_name,
1463 _ ("# bytes used in file-sharing datastore `%s'"),
1464 plugin_name);
1465 if (GNUNET_OK !=
1466 GNUNET_CONFIGURATION_get_value_size (cfg, "DATASTORE", "QUOTA", &quota))
1467 {
1468 GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR, "QUOTA", "DATASTORE");
1469 return;
1470 }
1471 stats = GNUNET_STATISTICS_create ("datastore", cfg);
1472 GNUNET_STATISTICS_set (stats, gettext_noop ("# quota"), quota, GNUNET_NO);
1473 cache_size = quota / 8; /* Or should we make this an option? */
1474 GNUNET_STATISTICS_set (stats,
1475 gettext_noop ("# cache size"),
1476 cache_size,
1477 GNUNET_NO);
1478 if (quota / (32 * 1024LL) > MAX_BF_SIZE)
1479 bf_size = MAX_BF_SIZE;
1480 else
1481 bf_size =
1482 quota / (32 * 1024LL); /* 8 bit per entry, 1 bit per 32 kb in DB */
1483 fn = NULL;
1484 if ((GNUNET_OK != GNUNET_CONFIGURATION_get_value_filename (cfg,
1485 "DATASTORE",
1486 "BLOOMFILTER",
1487 &fn)) ||
1488 (GNUNET_OK != GNUNET_DISK_directory_create_for_file (fn)))
1489 {
1490 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1491 _ ("Could not use specified filename `%s' for bloomfilter.\n"),
1492 NULL != fn ? fn : "");
1493 GNUNET_free (fn);
1494 fn = NULL;
1495 }
1496 if (NULL != fn)
1497 {
1498 GNUNET_asprintf (&pfn, "%s.%s", fn, plugin_name);
1499 if (GNUNET_YES == GNUNET_DISK_file_test (pfn))
1500 {
1501 filter =
1502 GNUNET_CONTAINER_bloomfilter_load (pfn,
1503 bf_size,
1504 5); /* approx. 3% false positives at max use */
1505 if (NULL == filter)
1506 {
1507 /* file exists but not valid, remove and try again, but refresh */
1508 if (0 != unlink (pfn))
1509 {
1510 /* failed to remove, run without file */
1511 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1512 _ ("Failed to remove bogus bloomfilter file `%s'\n"),
1513 pfn);
1514 GNUNET_free (pfn);
1515 pfn = NULL;
1516 filter = GNUNET_CONTAINER_bloomfilter_load (
1517 NULL,
1518 bf_size,
1519 5); /* approx. 3% false positives at max use */
1520 refresh_bf = GNUNET_YES;
1521 }
1522 else
1523 {
1524 /* try again after remove */
1525 filter = GNUNET_CONTAINER_bloomfilter_load (
1526 pfn,
1527 bf_size,
1528 5); /* approx. 3% false positives at max use */
1529 refresh_bf = GNUNET_YES;
1530 if (NULL == filter)
1531 {
1532 /* failed yet again, give up on using file */
1533 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1534 _ ("Failed to remove bogus bloomfilter file `%s'\n"),
1535 pfn);
1536 GNUNET_free (pfn);
1537 pfn = NULL;
1538 filter = GNUNET_CONTAINER_bloomfilter_init (
1539 NULL,
1540 bf_size,
1541 5); /* approx. 3% false positives at max use */
1542 }
1543 }
1544 }
1545 else
1546 {
1547 /* normal case: have an existing valid bf file, no need to refresh */
1548 refresh_bf = GNUNET_NO;
1549 }
1550 }
1551 else
1552 {
1553 filter =
1554 GNUNET_CONTAINER_bloomfilter_load (pfn,
1555 bf_size,
1556 5); /* approx. 3% false positives at max use */
1557 refresh_bf = GNUNET_YES;
1558 }
1559 GNUNET_free (pfn);
1560 }
1561 else
1562 {
1563 filter =
1564 GNUNET_CONTAINER_bloomfilter_init (NULL,
1565 bf_size,
1566 5); /* approx. 3% false positives at max use */
1567 refresh_bf = GNUNET_YES;
1568 }
1569 GNUNET_free (fn);
1570 if (NULL == filter)
1571 {
1572 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1573 _ ("Failed to initialize bloomfilter.\n"));
1574 if (NULL != stats)
1575 {
1576 GNUNET_STATISTICS_destroy (stats, GNUNET_YES);
1577 stats = NULL;
1578 }
1579 return;
1580 }
1581 GNUNET_SERVICE_suspend (service);
1582 stat_get = GNUNET_STATISTICS_get (stats,
1583 "datastore",
1584 quota_stat_name,
1585 &process_stat_done,
1586 &process_stat_in,
1587 NULL);
1588 if (NULL == stat_get)
1589 process_stat_done (NULL, GNUNET_SYSERR);
1590 else
1591 stat_timeout_task = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_SECONDS,
1592 &stat_timeout,
1593 NULL);
1594 GNUNET_SCHEDULER_add_shutdown (&cleaning_task, NULL);
1595}
1596
1597
1598/**
1599 * Define "main" method using service macro.
1600 */
1601GNUNET_SERVICE_MAIN (
1602 "datastore",
1603 GNUNET_SERVICE_OPTION_NONE,
1604 &run,
1605 &client_connect_cb,
1606 &client_disconnect_cb,
1607 NULL,
1608 GNUNET_MQ_hd_fixed_size (reserve,
1609 GNUNET_MESSAGE_TYPE_DATASTORE_RESERVE,
1610 struct ReserveMessage,
1611 NULL),
1612 GNUNET_MQ_hd_fixed_size (release_reserve,
1613 GNUNET_MESSAGE_TYPE_DATASTORE_RELEASE_RESERVE,
1614 struct ReleaseReserveMessage,
1615 NULL),
1616 GNUNET_MQ_hd_var_size (put,
1617 GNUNET_MESSAGE_TYPE_DATASTORE_PUT,
1618 struct DataMessage,
1619 NULL),
1620 GNUNET_MQ_hd_fixed_size (get,
1621 GNUNET_MESSAGE_TYPE_DATASTORE_GET,
1622 struct GetMessage,
1623 NULL),
1624 GNUNET_MQ_hd_fixed_size (get_key,
1625 GNUNET_MESSAGE_TYPE_DATASTORE_GET_KEY,
1626 struct GetKeyMessage,
1627 NULL),
1628 GNUNET_MQ_hd_fixed_size (get_replication,
1629 GNUNET_MESSAGE_TYPE_DATASTORE_GET_REPLICATION,
1630 struct GNUNET_MessageHeader,
1631 NULL),
1632 GNUNET_MQ_hd_fixed_size (get_zero_anonymity,
1633 GNUNET_MESSAGE_TYPE_DATASTORE_GET_ZERO_ANONYMITY,
1634 struct GetZeroAnonymityMessage,
1635 NULL),
1636 GNUNET_MQ_hd_var_size (remove,
1637 GNUNET_MESSAGE_TYPE_DATASTORE_REMOVE,
1638 struct DataMessage,
1639 NULL),
1640 GNUNET_MQ_hd_fixed_size (drop,
1641 GNUNET_MESSAGE_TYPE_DATASTORE_DROP,
1642 struct GNUNET_MessageHeader,
1643 NULL),
1644 GNUNET_MQ_handler_end ());
1645
1646
1647/* end of gnunet-service-datastore.c */
diff --git a/src/datastore/perf_datastore_api.c b/src/datastore/perf_datastore_api.c
deleted file mode 100644
index fef38891e..000000000
--- a/src/datastore/perf_datastore_api.c
+++ /dev/null
@@ -1,629 +0,0 @@
1/*
2 This file is part of GNUnet.
3 Copyright (C) 2004, 2005, 2006, 2007, 2009, 2011, 2015 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20/*
21 * @file datastore/perf_datastore_api.c
22 * @brief performance measurement for the datastore implementation
23 * @author Christian Grothoff
24 *
25 * This testcase inserts a bunch of (variable size) data and then
26 * deletes data until the (reported) database size drops below a given
27 * threshold. This is iterated 10 times, with the actual size of the
28 * content stored and the number of operations performed being printed
29 * for each iteration. The code also prints a "I" for every 40 blocks
30 * inserted and a "D" for every 40 blocks deleted. The deletion
31 * strategy uses the "random" iterator. Priorities and expiration
32 * dates are set using a pseudo-random value within a realistic range.
33 */
34#include "platform.h"
35#include "gnunet_util_lib.h"
36#include "gnunet_protocols.h"
37#include "gnunet_datastore_service.h"
38#include "gnunet_testing_lib.h"
39#include <gauger.h>
40
41/**
42 * How long until we give up on transmitting the message?
43 */
44#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 15)
45
46/**
47 * Target datastore size (in bytes).
48 */
49#define MAX_SIZE (1024LL * 1024 * 4)
50
51/**
52 * Report progress outside of major reports? Should probably be #GNUNET_YES if
53 * size is > 16 MB.
54 */
55#define REPORT_ID GNUNET_YES
56
57/**
58 * Number of put operations equivalent to 1/3rd of #MAX_SIZE
59 */
60#define PUT_10 MAX_SIZE / 32 / 1024 / 3
61
62/**
63 * Total number of iterations (each iteration doing
64 * PUT_10 put operations); we report full status every
65 * 10 iterations. Abort with CTRL-C.
66 */
67#define ITERATIONS 8
68
69/**
70 * Total number of iterations to do to go beyond the quota.
71 * The quota is set to 10 MB or 2.5 times #MAX_SIZE,
72 * so we got 16 times #MAX_SIZE to be sure to hit it a LOT.
73 */
74#define QUOTA_PUTS (MAX_SIZE / 32 / 1024 * 16LL)
75
76
77/**
78 * Number of bytes stored in the datastore in total.
79 */
80static unsigned long long stored_bytes;
81
82/**
83 * Number of entries stored in the datastore in total.
84 */
85static unsigned long long stored_entries;
86
87/**
88 * Number of database operations performed. Inserting
89 * counts as one operation, deleting as two (as deletion
90 * requires selecting a value for deletion first).
91 */
92static unsigned long long stored_ops;
93
94/**
95 * Start time of the benchmark.
96 */
97static struct GNUNET_TIME_Absolute start_time;
98
99/**
100 * Database backend we use.
101 */
102static const char *plugin_name;
103
104/**
105 * Handle to the datastore.
106 */
107static struct GNUNET_DATASTORE_Handle *datastore;
108
109/**
110 * Value we return from #main().
111 */
112static int ok;
113
114/**
115 * Which phase of the process are we in?
116 */
117enum RunPhase
118{
119 /**
120 * We are done (shutting down normally).
121 */
122 RP_DONE = 0,
123
124 /**
125 * We are adding new entries to the datastore.
126 */
127 RP_PUT,
128
129 /**
130 * We are deleting entries from the datastore.
131 */
132 RP_CUT,
133
134 /**
135 * We are putting as much as we can to see how the database performs
136 * when it reaches the quota and has to auto-delete (see #3903).
137 */
138 RP_PUT_QUOTA,
139
140 /**
141 * We are generating a report.
142 */
143 RP_REPORT,
144
145 /**
146 * Execution failed with some kind of error.
147 */
148 RP_ERROR
149};
150
151
152/**
153 * Closure we give to all of the functions executing the
154 * benchmark. Could right now be global, but this allows
155 * us to theoretically run multiple clients "in parallel".
156 */
157struct CpsRunContext
158{
159 /**
160 * Execution phase we are in.
161 */
162 enum RunPhase phase;
163
164 /**
165 * Size of the value we are currently storing (during #RP_PUT).
166 */
167 size_t size;
168
169 /**
170 * Current iteration counter, we are done with the benchmark
171 * once it hits #ITERATIONS.
172 */
173 unsigned int i;
174
175 /**
176 * Counts the number of items put in the current phase.
177 * Once it hits #PUT_10, we progress to the #RP_CUT phase
178 * or are done if @e i reaches #ITERATIONS.
179 */
180 unsigned int j;
181};
182
183
184/**
185 * Main state machine. Executes the next step of the benchmark
186 * depending on the current state.
187 *
188 * @param cls the `struct CpsRunContext`
189 */
190static void
191run_continuation (void *cls);
192
193
194/**
195 * Continuation called to notify client about result of the insertion
196 * operation. Checks for errors, updates our iteration counters and
197 * continues execution with #run_continuation().
198 *
199 * @param cls the `struct CpsRunContext`
200 * @param success #GNUNET_SYSERR on failure
201 * @param min_expiration minimum expiration time required for content to be stored
202 * by the datacache at this time, zero for unknown
203 * @param msg NULL on success, otherwise an error message
204 */
205static void
206check_success (void *cls,
207 int success,
208 struct GNUNET_TIME_Absolute min_expiration,
209 const char *msg)
210{
211 struct CpsRunContext *crc = cls;
212
213#if REPORT_ID
214 fprintf (stderr, "%s", (GNUNET_OK == success) ? "I" : "i");
215#endif
216 if (GNUNET_OK != success)
217 {
218 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
219 "Check success failed: `%s'\n",
220 msg);
221 crc->phase = RP_ERROR;
222 GNUNET_SCHEDULER_add_now (&run_continuation,
223 crc);
224 return;
225 }
226 stored_bytes += crc->size;
227 stored_ops++;
228 stored_entries++;
229 crc->j++;
230 switch (crc->phase)
231 {
232 case RP_PUT:
233 if (crc->j >= PUT_10)
234 {
235 crc->j = 0;
236 crc->i++;
237 if (crc->i == ITERATIONS)
238 crc->phase = RP_PUT_QUOTA;
239 else
240 crc->phase = RP_CUT;
241 }
242 break;
243
244 case RP_PUT_QUOTA:
245 if (crc->j >= QUOTA_PUTS)
246 {
247 crc->j = 0;
248 crc->phase = RP_DONE;
249 }
250 break;
251
252 default:
253 GNUNET_assert (0);
254 }
255 GNUNET_SCHEDULER_add_now (&run_continuation,
256 crc);
257}
258
259
260/**
261 * Continuation called to notify client about result of the
262 * deletion operation. Checks for errors and continues
263 * execution with #run_continuation().
264 *
265 * @param cls the `struct CpsRunContext`
266 * @param success #GNUNET_SYSERR on failure
267 * @param min_expiration minimum expiration time required for content to be stored
268 * by the datacache at this time, zero for unknown
269 * @param msg NULL on success, otherwise an error message
270 */
271static void
272remove_next (void *cls,
273 int success,
274 struct GNUNET_TIME_Absolute min_expiration,
275 const char *msg)
276{
277 struct CpsRunContext *crc = cls;
278
279 if (GNUNET_OK != success)
280 {
281 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
282 "remove_next failed: `%s'\n",
283 msg);
284 crc->phase = RP_ERROR;
285 GNUNET_SCHEDULER_add_now (&run_continuation,
286 crc);
287 return;
288 }
289#if REPORT_ID
290 fprintf (stderr, "%s", "D");
291#endif
292 GNUNET_assert (GNUNET_OK == success);
293 GNUNET_SCHEDULER_add_now (&run_continuation,
294 crc);
295}
296
297
298/**
299 * We have selected a value for deletion, trigger removal.
300 *
301 * @param cls the `struct CpsRunContext`
302 * @param key key for the content
303 * @param size number of bytes in data
304 * @param data content stored
305 * @param type type of the content
306 * @param priority priority of the content
307 * @param anonymity anonymity-level for the content
308 * @param replication replication-level for the content
309 * @param expiration expiration time for the content
310 * @param uid unique identifier for the datum;
311 * maybe 0 if no unique identifier is available
312 */
313static void
314delete_value (void *cls,
315 const struct GNUNET_HashCode *key,
316 size_t size,
317 const void *data,
318 enum GNUNET_BLOCK_Type type,
319 uint32_t priority,
320 uint32_t anonymity,
321 uint32_t replication,
322 struct GNUNET_TIME_Absolute expiration,
323 uint64_t uid)
324{
325 struct CpsRunContext *crc = cls;
326
327 GNUNET_assert (NULL != key);
328 stored_ops++;
329 stored_bytes -= size;
330 stored_entries--;
331 stored_ops++;
332 if (stored_bytes < MAX_SIZE)
333 crc->phase = RP_PUT;
334 GNUNET_assert (NULL !=
335 GNUNET_DATASTORE_remove (datastore,
336 key,
337 size,
338 data, 1, 1,
339 &remove_next, crc));
340}
341
342
343/**
344 * Main state machine. Executes the next step of the benchmark
345 * depending on the current state.
346 *
347 * @param cls the `struct CpsRunContext`
348 */
349static void
350run_continuation (void *cls)
351{
352 struct CpsRunContext *crc = cls;
353 size_t size;
354 static struct GNUNET_HashCode key;
355 static char data[65536];
356 char gstr[128];
357
358 ok = (int) crc->phase;
359 switch (crc->phase)
360 {
361 case RP_PUT:
362 memset (&key,
363 256 - crc->i,
364 sizeof(struct GNUNET_HashCode));
365 /* most content is 32k */
366 size = 32 * 1024;
367 if (0 ==
368 GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
369 16)) /* but some of it is less! */
370 size = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
371 32 * 1024);
372 crc->size = size = size - (size & 7); /* always multiple of 8 */
373 GNUNET_CRYPTO_hash (&key,
374 sizeof(struct GNUNET_HashCode),
375 &key);
376 memset (data,
377 (int) crc->j,
378 size);
379 if (crc->j > 255)
380 memset (data,
381 (int) (crc->j - 255),
382 size / 2);
383 data[0] = crc->i;
384 GNUNET_assert (NULL !=
385 GNUNET_DATASTORE_put (datastore,
386 0,
387 &key,
388 size,
389 data,
390 crc->j + 1,
391 GNUNET_CRYPTO_random_u32
392 (GNUNET_CRYPTO_QUALITY_WEAK, 100),
393 crc->j,
394 0,
395 GNUNET_TIME_relative_to_absolute
396 (GNUNET_TIME_relative_multiply
397 (GNUNET_TIME_UNIT_SECONDS,
398 GNUNET_CRYPTO_random_u32
399 (GNUNET_CRYPTO_QUALITY_WEAK,
400 1000))),
401 1,
402 1,
403 &check_success, crc));
404 break;
405
406 case RP_CUT:
407 /* trim down below MAX_SIZE again */
408 GNUNET_assert (NULL !=
409 GNUNET_DATASTORE_get_for_replication (datastore,
410 1, 1,
411 &delete_value,
412 crc));
413 break;
414
415 case RP_REPORT:
416 printf (
417#if REPORT_ID
418 "\n"
419#endif
420 "Stored %llu kB / %lluk ops / %llu ops/s\n",
421 stored_bytes / 1024, /* used size in k */
422 stored_ops / 1024, /* total operations (in k) */
423 1000LL * 1000LL * stored_ops / (1
424 + GNUNET_TIME_absolute_get_duration
425 (start_time).rel_value_us));
426 crc->phase = RP_PUT;
427 crc->j = 0;
428 GNUNET_SCHEDULER_add_now (&run_continuation,
429 crc);
430 break;
431
432 case RP_PUT_QUOTA:
433 memset (&key,
434 256 - crc->i,
435 sizeof(struct GNUNET_HashCode));
436 /* most content is 32k */
437 size = 32 * 1024;
438 if (0 ==
439 GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
440 16)) /* but some of it is less! */
441 size = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
442 32 * 1024);
443 crc->size = size = size - (size & 7); /* always multiple of 8 */
444 GNUNET_CRYPTO_hash (&key,
445 sizeof(struct GNUNET_HashCode),
446 &key);
447 memset (data,
448 (int) crc->j,
449 size);
450 if (crc->j > 255)
451 memset (data,
452 (int) (crc->j - 255),
453 size / 2);
454 data[0] = crc->i;
455 GNUNET_assert (NULL !=
456 GNUNET_DATASTORE_put (datastore,
457 0, /* reservation ID */
458 &key,
459 size,
460 data,
461 crc->j + 1, /* type */
462 GNUNET_CRYPTO_random_u32
463 (GNUNET_CRYPTO_QUALITY_WEAK,
464 100), /* priority */
465 crc->j, /* anonymity */
466 0, /* replication */
467 GNUNET_TIME_relative_to_absolute
468 (GNUNET_TIME_relative_multiply
469 (GNUNET_TIME_UNIT_SECONDS,
470 GNUNET_CRYPTO_random_u32
471 (GNUNET_CRYPTO_QUALITY_WEAK,
472 1000))),
473 1,
474 1,
475 &check_success, crc));
476 break;
477
478 case RP_DONE:
479 GNUNET_snprintf (gstr,
480 sizeof(gstr),
481 "DATASTORE-%s",
482 plugin_name);
483 if ((crc->i == ITERATIONS) && (stored_ops > 0))
484 {
485 GAUGER (gstr,
486 "PUT operation duration",
487 GNUNET_TIME_absolute_get_duration (start_time).rel_value_us
488 / 1000LL
489 / stored_ops,
490 "ms/operation");
491 fprintf (stdout,
492 "\nPUT performance: %s for %llu operations\n",
493 GNUNET_STRINGS_relative_time_to_string (
494 GNUNET_TIME_absolute_get_duration (start_time),
495 GNUNET_YES),
496 stored_ops);
497 fprintf (stdout,
498 "PUT performance: %llu ms/operation\n",
499 GNUNET_TIME_absolute_get_duration (start_time).rel_value_us
500 / 1000LL
501 / stored_ops);
502 }
503 GNUNET_DATASTORE_disconnect (datastore,
504 GNUNET_YES);
505 GNUNET_free (crc);
506 ok = 0;
507 break;
508
509 case RP_ERROR:
510 GNUNET_DATASTORE_disconnect (datastore, GNUNET_YES);
511 GNUNET_free (crc);
512 ok = 1;
513 break;
514
515 default:
516 GNUNET_assert (0);
517 }
518}
519
520
521/**
522 * Function called with the result of the initial PUT operation. If
523 * the PUT succeeded, we start the actual benchmark loop, otherwise we
524 * bail out with an error.
525 *
526 *
527 * @param cls closure
528 * @param success #GNUNET_SYSERR on failure
529 * @param min_expiration minimum expiration time required for content to be stored
530 * by the datacache at this time, zero for unknown
531 * @param msg NULL on success, otherwise an error message
532 */
533static void
534run_tests (void *cls,
535 int success,
536 struct GNUNET_TIME_Absolute min_expiration,
537 const char *msg)
538{
539 struct CpsRunContext *crc = cls;
540
541 if (success != GNUNET_YES)
542 {
543 fprintf (stderr,
544 "Test 'put' operation failed with error `%s' database likely not setup, skipping test.\n",
545 msg);
546 GNUNET_DATASTORE_disconnect (datastore,
547 GNUNET_YES);
548 GNUNET_free (crc);
549 return;
550 }
551 GNUNET_SCHEDULER_add_now (&run_continuation,
552 crc);
553}
554
555
556/**
557 * Beginning of the actual execution of the benchmark.
558 * Performs a first test operation (PUT) to verify that
559 * the plugin works at all.
560 *
561 * @param cls NULL
562 * @param cfg configuration to use
563 * @param peer peer handle (unused)
564 */
565static void
566run (void *cls,
567 const struct GNUNET_CONFIGURATION_Handle *cfg,
568 struct GNUNET_TESTING_Peer *peer)
569{
570 struct CpsRunContext *crc;
571 static struct GNUNET_HashCode zkey;
572
573 datastore = GNUNET_DATASTORE_connect (cfg);
574 start_time = GNUNET_TIME_absolute_get ();
575 crc = GNUNET_new (struct CpsRunContext);
576 crc->phase = RP_PUT;
577 if (NULL ==
578 GNUNET_DATASTORE_put (datastore,
579 0,
580 &zkey,
581 4, "TEST",
582 GNUNET_BLOCK_TYPE_TEST,
583 0, 0, 0,
584 GNUNET_TIME_relative_to_absolute (
585 GNUNET_TIME_UNIT_SECONDS),
586 0, 1,
587 &run_tests, crc))
588 {
589 fprintf (stderr,
590 "%s",
591 "Test 'put' operation failed.\n");
592 ok = 1;
593 GNUNET_free (crc);
594 }
595}
596
597
598/**
599 * Entry point into the test. Determines which configuration / plugin
600 * we are running with based on the name of the binary and starts
601 * the peer.
602 *
603 * @param argc should be 1
604 * @param argv used to determine plugin / configuration name.
605 * @return 0 on success
606 */
607int
608main (int argc,
609 char *argv[])
610{
611 char cfg_name[PATH_MAX];
612
613 plugin_name = GNUNET_TESTING_get_testname_from_underscore (argv[0]);
614 GNUNET_snprintf (cfg_name,
615 sizeof(cfg_name),
616 "test_datastore_api_data_%s.conf",
617 plugin_name);
618 if (0 !=
619 GNUNET_TESTING_peer_run ("perf-gnunet-datastore",
620 cfg_name,
621 &run,
622 NULL))
623 return 1;
624 fprintf (stderr, "%s", "\n");
625 return ok;
626}
627
628
629/* end of perf_datastore_api.c */
diff --git a/src/datastore/perf_plugin_datastore.c b/src/datastore/perf_plugin_datastore.c
deleted file mode 100644
index d7488d4e7..000000000
--- a/src/datastore/perf_plugin_datastore.c
+++ /dev/null
@@ -1,573 +0,0 @@
1/*
2 This file is part of GNUnet.
3 Copyright (C) 2004, 2005, 2006, 2007, 2009, 2011 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20/*
21 * @file perf_plugin_datastore.c
22 * @brief Profile database plugin directly, focusing on iterators.
23 * @author Christian Grothoff
24 */
25
26#include "platform.h"
27#include "gnunet_util_lib.h"
28#include "gnunet_protocols.h"
29#include "gnunet_datastore_plugin.h"
30#include "gnunet_testing_lib.h"
31#include <gauger.h>
32
33/**
34 * Target datastore size (in bytes). Realistic sizes are
35 * more like 16 GB (not the default of 16 MB); however,
36 * those take too long to run them in the usual "make check"
37 * sequence. Hence the value used for shipping is tiny.
38 */
39#define MAX_SIZE 1024LL * 1024 * 16 * 1
40
41#define ITERATIONS 2
42
43/**
44 * Number of put operations equivalent to 1/10th of MAX_SIZE
45 */
46#define PUT_10 (MAX_SIZE / 32 / 1024 / ITERATIONS)
47
48static char category[256];
49
50static unsigned int hits[PUT_10 / 8 + 1];
51
52static unsigned long long stored_bytes;
53
54static unsigned long long stored_entries;
55
56static unsigned long long stored_ops;
57
58static const char *plugin_name;
59
60static int ok;
61
62enum RunPhase
63{
64 RP_ERROR = 0,
65 RP_PUT,
66 RP_REP_GET,
67 RP_ZA_GET,
68 RP_EXP_GET,
69 RP_DONE
70};
71
72
73struct CpsRunContext
74{
75 unsigned int i;
76 struct GNUNET_TIME_Absolute start;
77 struct GNUNET_TIME_Absolute end;
78 const struct GNUNET_CONFIGURATION_Handle *cfg;
79 struct GNUNET_DATASTORE_PluginFunctions *api;
80 enum RunPhase phase;
81 unsigned int cnt;
82 unsigned int iter;
83 uint64_t offset;
84};
85
86
87/**
88 * Function called by plugins to notify us about a
89 * change in their disk utilization.
90 *
91 * @param cls closure (NULL)
92 * @param delta change in disk utilization,
93 * 0 for "reset to empty"
94 */
95static void
96disk_utilization_change_cb (void *cls, int delta)
97{
98}
99
100
101static void
102test (void *cls);
103
104
105/**
106 * Put continuation.
107 *
108 * @param cls closure
109 * @param key key for the item stored
110 * @param size size of the item stored
111 * @param status #GNUNET_OK or #GNUNET_SYSERROR
112 * @param msg error message on error
113 */
114static void
115put_continuation (void *cls,
116 const struct GNUNET_HashCode *key,
117 uint32_t size,
118 int status,
119 const char *msg)
120{
121 struct CpsRunContext *crc = cls;
122
123 if (GNUNET_OK != status)
124 {
125 fprintf (stderr, "ERROR: `%s'\n", msg);
126 }
127 else
128 {
129 stored_bytes += size;
130 stored_ops++;
131 stored_entries++;
132 }
133 GNUNET_SCHEDULER_add_now (&test, crc);
134}
135
136
137static void
138do_put (struct CpsRunContext *crc)
139{
140 char value[65536];
141 size_t size;
142 static struct GNUNET_HashCode key;
143 static int i;
144 unsigned int prio;
145
146 if (0 == i)
147 crc->start = GNUNET_TIME_absolute_get ();
148 if (PUT_10 == i)
149 {
150 i = 0;
151 crc->end = GNUNET_TIME_absolute_get ();
152 {
153 printf ("%s took %s for %llu items\n", "Storing an item",
154 GNUNET_STRINGS_relative_time_to_string (
155 GNUNET_TIME_absolute_get_difference (crc->start,
156 crc
157 ->end),
158 GNUNET_YES),
159 PUT_10);
160 if (PUT_10 > 0)
161 GAUGER (category, "Storing an item",
162 (crc->end.abs_value_us - crc->start.abs_value_us) / 1000LL
163 / PUT_10,
164 "ms/item");
165 }
166 crc->i++;
167 crc->start = GNUNET_TIME_absolute_get ();
168 crc->phase++;
169 GNUNET_SCHEDULER_add_now (&test, crc);
170 return;
171 }
172 /* most content is 32k */
173 size = 32 * 1024;
174 if (GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 16) == 0) /* but some of it is less! */
175 size = 8 + GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 32 * 1024);
176 size = size - (size & 7); /* always multiple of 8 */
177
178 /* generate random key */
179 key.bits[0] = (unsigned int) GNUNET_TIME_absolute_get ().abs_value_us;
180 GNUNET_CRYPTO_hash (&key, sizeof(struct GNUNET_HashCode), &key);
181 memset (value, i, size);
182 if (i > 255)
183 memset (value, i - 255, size / 2);
184 value[0] = crc->i;
185 GNUNET_memcpy (&value[4], &i, sizeof(i));
186 prio = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 100);
187 crc->api->put (crc->api->cls,
188 &key,
189 false /* absent */,
190 size,
191 value,
192 1 + i % 4 /* type */,
193 prio,
194 i % 4 /* anonymity */,
195 0 /* replication */,
196 GNUNET_TIME_relative_to_absolute
197 (GNUNET_TIME_relative_multiply
198 (GNUNET_TIME_UNIT_MILLISECONDS,
199 60 * 60 * 60 * 1000
200 + GNUNET_CRYPTO_random_u32
201 (GNUNET_CRYPTO_QUALITY_WEAK, 1000))),
202 put_continuation,
203 crc);
204 i++;
205}
206
207
208static int
209iterate_zeros (void *cls,
210 const struct GNUNET_HashCode *key,
211 uint32_t size,
212 const void *data,
213 enum GNUNET_BLOCK_Type type,
214 uint32_t priority,
215 uint32_t anonymity,
216 uint32_t replication,
217 struct GNUNET_TIME_Absolute expiration,
218 uint64_t uid)
219{
220 struct CpsRunContext *crc = cls;
221 int i;
222 const char *cdata = data;
223
224 GNUNET_assert (key != NULL);
225 GNUNET_assert (size >= 8);
226 GNUNET_memcpy (&i, &cdata[4], sizeof(i));
227 hits[i / 8] |= (1 << (i % 8));
228
229 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
230 "Found result %d type=%u, priority=%u, size=%u, expire=%s\n",
231 i,
232 type, priority, size,
233 GNUNET_STRINGS_absolute_time_to_string (expiration));
234 crc->cnt++;
235 if (crc->cnt == PUT_10 / 4 - 1)
236 {
237 unsigned int bc;
238
239 bc = 0;
240 for (i = 0; i < PUT_10; i++)
241 if (0 != (hits[i / 8] & (1 << (i % 8))))
242 bc++;
243
244 crc->end = GNUNET_TIME_absolute_get ();
245 printf ("%s took %s yielding %u/%u items\n",
246 "Select random zero-anonymity item",
247 GNUNET_STRINGS_relative_time_to_string (
248 GNUNET_TIME_absolute_get_difference (crc->start,
249 crc
250 ->end),
251 GNUNET_YES),
252 bc, crc->cnt);
253 if (crc->cnt > 0)
254 GAUGER (category, "Select random zero-anonymity item",
255 (crc->end.abs_value_us - crc->start.abs_value_us) / 1000LL
256 / crc->cnt,
257 "ms/item");
258 memset (hits, 0, sizeof(hits));
259 crc->phase++;
260 crc->cnt = 0;
261 crc->start = GNUNET_TIME_absolute_get ();
262 }
263 GNUNET_SCHEDULER_add_now (&test, crc);
264 return GNUNET_OK;
265}
266
267
268static int
269expiration_get (void *cls,
270 const struct GNUNET_HashCode *key,
271 uint32_t size,
272 const void *data,
273 enum GNUNET_BLOCK_Type type,
274 uint32_t priority,
275 uint32_t anonymity,
276 uint32_t replication,
277 struct GNUNET_TIME_Absolute expiration,
278 uint64_t uid)
279{
280 struct CpsRunContext *crc = cls;
281 int i;
282 const char *cdata = data;
283
284 GNUNET_assert (size >= 8);
285 GNUNET_memcpy (&i, &cdata[4], sizeof(i));
286 hits[i / 8] |= (1 << (i % 8));
287 crc->cnt++;
288 if (PUT_10 <= crc->cnt)
289 {
290 unsigned int bc;
291
292 bc = 0;
293 for (i = 0; i < PUT_10; i++)
294 if (0 != (hits[i / 8] & (1 << (i % 8))))
295 bc++;
296
297 crc->end = GNUNET_TIME_absolute_get ();
298 printf ("%s took %s yielding %u/%u items\n",
299 "Selecting and deleting by expiration",
300 GNUNET_STRINGS_relative_time_to_string (
301 GNUNET_TIME_absolute_get_difference (crc->start,
302 crc
303 ->end),
304 GNUNET_YES),
305 bc, (unsigned int) PUT_10);
306 if (crc->cnt > 0)
307 GAUGER (category, "Selecting and deleting by expiration",
308 (crc->end.abs_value_us - crc->start.abs_value_us) / 1000LL
309 / crc->cnt,
310 "ms/item");
311 memset (hits, 0, sizeof(hits));
312 if (++crc->iter == ITERATIONS)
313 crc->phase++;
314 else
315 crc->phase = RP_PUT;
316 crc->cnt = 0;
317 crc->start = GNUNET_TIME_absolute_get ();
318 }
319 GNUNET_SCHEDULER_add_now (&test, crc);
320 return GNUNET_NO;
321}
322
323
324static int
325replication_get (void *cls,
326 const struct GNUNET_HashCode *key,
327 uint32_t size,
328 const void *data,
329 enum GNUNET_BLOCK_Type type,
330 uint32_t priority,
331 uint32_t anonymity,
332 uint32_t replication,
333 struct GNUNET_TIME_Absolute expiration,
334 uint64_t uid)
335{
336 struct CpsRunContext *crc = cls;
337 int i;
338 const char *cdata = data;
339
340 GNUNET_assert (NULL != key);
341 GNUNET_assert (size >= 8);
342 GNUNET_memcpy (&i, &cdata[4], sizeof(i));
343 hits[i / 8] |= (1 << (i % 8));
344 crc->cnt++;
345 if (PUT_10 <= crc->cnt)
346 {
347 unsigned int bc;
348
349 bc = 0;
350 for (i = 0; i < PUT_10; i++)
351 if (0 != (hits[i / 8] & (1 << (i % 8))))
352 bc++;
353
354 crc->end = GNUNET_TIME_absolute_get ();
355 printf ("%s took %s yielding %u/%u items\n",
356 "Selecting random item for replication",
357 GNUNET_STRINGS_relative_time_to_string (
358 GNUNET_TIME_absolute_get_difference (crc->start,
359 crc
360 ->end),
361 GNUNET_YES),
362 bc, (unsigned int) PUT_10);
363 if (crc->cnt > 0)
364 GAUGER (category, "Selecting random item for replication",
365 (crc->end.abs_value_us - crc->start.abs_value_us) / 1000LL
366 / crc->cnt,
367 "ms/item");
368 memset (hits, 0, sizeof(hits));
369 crc->phase++;
370 crc->offset = 0;
371 crc->cnt = 0;
372 crc->start = GNUNET_TIME_absolute_get ();
373 }
374
375 GNUNET_SCHEDULER_add_now (&test, crc);
376 return GNUNET_OK;
377}
378
379
380/**
381 * Function called when the service shuts
382 * down. Unloads our datastore plugin.
383 *
384 * @param api api to unload
385 * @param cfg configuration to use
386 */
387static void
388unload_plugin (struct GNUNET_DATASTORE_PluginFunctions *api,
389 const struct GNUNET_CONFIGURATION_Handle *cfg)
390{
391 char *name;
392 char *libname;
393
394 if (GNUNET_OK !=
395 GNUNET_CONFIGURATION_get_value_string (cfg, "DATASTORE", "DATABASE",
396 &name))
397 {
398 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
399 _ ("No `%s' specified for `%s' in configuration!\n"),
400 "DATABASE",
401 "DATASTORE");
402 return;
403 }
404 GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", name);
405 GNUNET_break (NULL == GNUNET_PLUGIN_unload (libname, api));
406 GNUNET_free (libname);
407 GNUNET_free (name);
408}
409
410
411/**
412 * Last task run during shutdown. Disconnects us from
413 * the transport and core.
414 */
415static void
416cleaning_task (void *cls)
417{
418 struct CpsRunContext *crc = cls;
419
420 unload_plugin (crc->api, crc->cfg);
421 GNUNET_free (crc);
422}
423
424
425static void
426test (void *cls)
427{
428 struct CpsRunContext *crc = cls;
429
430 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
431 "In phase %d, iteration %u\n", crc->phase, crc->cnt);
432 switch (crc->phase)
433 {
434 case RP_ERROR:
435 GNUNET_break (0);
436 crc->api->drop (crc->api->cls);
437 ok = 1;
438 GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_IDLE,
439 &cleaning_task, crc);
440 break;
441
442 case RP_PUT:
443 do_put (crc);
444 break;
445
446 case RP_REP_GET:
447 crc->api->get_replication (crc->api->cls, &replication_get, crc);
448 break;
449
450 case RP_ZA_GET:
451 crc->api->get_zero_anonymity (crc->api->cls, crc->offset++, 1,
452 &iterate_zeros, crc);
453 break;
454
455 case RP_EXP_GET:
456 crc->api->get_expiration (crc->api->cls, &expiration_get, crc);
457 break;
458
459 case RP_DONE:
460 crc->api->drop (crc->api->cls);
461 ok = 0;
462 GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_IDLE,
463 &cleaning_task, crc);
464 break;
465 }
466}
467
468
469/**
470 * Load the datastore plugin.
471 */
472static struct GNUNET_DATASTORE_PluginFunctions *
473load_plugin (const struct GNUNET_CONFIGURATION_Handle *cfg)
474{
475 static struct GNUNET_DATASTORE_PluginEnvironment env;
476 struct GNUNET_DATASTORE_PluginFunctions *ret;
477 char *name;
478 char *libname;
479
480 if (GNUNET_OK !=
481 GNUNET_CONFIGURATION_get_value_string (cfg, "DATASTORE", "DATABASE",
482 &name))
483 {
484 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
485 _ ("No `%s' specified for `%s' in configuration!\n"),
486 "DATABASE",
487 "DATASTORE");
488 return NULL;
489 }
490 env.cfg = cfg;
491 env.duc = &disk_utilization_change_cb;
492 env.cls = NULL;
493 GNUNET_log (GNUNET_ERROR_TYPE_INFO, _ ("Loading `%s' datastore plugin\n"),
494 name);
495 GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", name);
496 if (NULL == (ret = GNUNET_PLUGIN_load (libname, &env)))
497 {
498 fprintf (stderr, "Failed to load plugin `%s'!\n", name);
499 GNUNET_free (name);
500 GNUNET_free (libname);
501 return NULL;
502 }
503 GNUNET_free (libname);
504 GNUNET_free (name);
505 return ret;
506}
507
508
509static void
510run (void *cls, char *const *args, const char *cfgfile,
511 const struct GNUNET_CONFIGURATION_Handle *c)
512{
513 struct GNUNET_DATASTORE_PluginFunctions *api;
514 struct CpsRunContext *crc;
515
516 if (NULL == c)
517 {
518 GNUNET_break (0);
519 return;
520 }
521 api = load_plugin (c);
522 if (api == NULL)
523 {
524 fprintf (stderr,
525 "%s",
526 "Could not initialize plugin, assuming database not configured. Test not run!\n");
527 return;
528 }
529 crc = GNUNET_new (struct CpsRunContext);
530 crc->api = api;
531 crc->cfg = c;
532 crc->phase = RP_PUT;
533 ok = 2;
534 GNUNET_SCHEDULER_add_now (&test, crc);
535}
536
537
538int
539main (int argc, char *argv[])
540{
541 char dir_name[PATH_MAX];
542 char cfg_name[PATH_MAX];
543 char *const xargv[] = {
544 "perf-plugin-datastore",
545 "-c",
546 cfg_name,
547 NULL
548 };
549 struct GNUNET_GETOPT_CommandLineOption options[] = {
550 GNUNET_GETOPT_OPTION_END
551 };
552
553 plugin_name = GNUNET_TESTING_get_testname_from_underscore (argv[0]);
554 GNUNET_snprintf (dir_name, sizeof(dir_name), "/tmp/perf-gnunet-datastore-%s",
555 plugin_name);
556 GNUNET_DISK_directory_remove (dir_name);
557 GNUNET_log_setup ("perf-plugin-datastore",
558 "WARNING",
559 NULL);
560 GNUNET_snprintf (category, sizeof(category), "DATASTORE-%s", plugin_name);
561 GNUNET_snprintf (cfg_name, sizeof(cfg_name),
562 "perf_plugin_datastore_data_%s.conf", plugin_name);
563 GNUNET_PROGRAM_run ((sizeof(xargv) / sizeof(char *)) - 1, xargv,
564 "perf-plugin-datastore", "nohelp", options, &run, NULL);
565 if (ok != 0)
566 fprintf (stderr, "Missed some testcases: %u\n", ok);
567 GNUNET_DISK_directory_remove (dir_name);
568
569 return ok;
570}
571
572
573/* end of perf_plugin_datastore.c */
diff --git a/src/datastore/perf_plugin_datastore_data_heap.conf b/src/datastore/perf_plugin_datastore_data_heap.conf
deleted file mode 100644
index 873cf9606..000000000
--- a/src/datastore/perf_plugin_datastore_data_heap.conf
+++ /dev/null
@@ -1,7 +0,0 @@
1@INLINE@ test_defaults.conf
2[PATHS]
3GNUNET_TEST_HOME = $GNUNET_TMP/perf-gnunet-datastore-heap/
4
5
6[datastore]
7DATABASE = heap
diff --git a/src/datastore/perf_plugin_datastore_data_mysql.conf b/src/datastore/perf_plugin_datastore_data_mysql.conf
deleted file mode 100644
index a32b830c3..000000000
--- a/src/datastore/perf_plugin_datastore_data_mysql.conf
+++ /dev/null
@@ -1,10 +0,0 @@
1@INLINE@ test_defaults.conf
2[PATHS]
3GNUNET_TEST_HOME = $GNUNET_TMP/perf-gnunet-datastore-mysql/
4
5[datastore]
6DATABASE = mysql
7
8[datastore-mysql]
9DATABASE = gnunetcheck
10
diff --git a/src/datastore/perf_plugin_datastore_data_postgres.conf b/src/datastore/perf_plugin_datastore_data_postgres.conf
deleted file mode 100644
index 7683887a8..000000000
--- a/src/datastore/perf_plugin_datastore_data_postgres.conf
+++ /dev/null
@@ -1,10 +0,0 @@
1@INLINE@ test_defaults.conf
2[PATHS]
3GNUNET_TEST_HOME = $GNUNET_TMP/perf-gnunet-datastore-postgres/
4
5[datastore]
6DATABASE = postgres
7
8[datastore-postgres]
9CONFIG = dbname=gnunetcheck
10
diff --git a/src/datastore/perf_plugin_datastore_data_sqlite.conf b/src/datastore/perf_plugin_datastore_data_sqlite.conf
deleted file mode 100644
index 888e020a6..000000000
--- a/src/datastore/perf_plugin_datastore_data_sqlite.conf
+++ /dev/null
@@ -1,4 +0,0 @@
1@INLINE@ test_defaults.conf
2[PATHS]
3GNUNET_TEST_HOME = $GNUNET_TMP/perf-gnunet-datastore-sqlite/
4
diff --git a/src/datastore/plugin_datastore_heap.c b/src/datastore/plugin_datastore_heap.c
deleted file mode 100644
index a827a2763..000000000
--- a/src/datastore/plugin_datastore_heap.c
+++ /dev/null
@@ -1,944 +0,0 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2012 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file datastore/plugin_datastore_heap.c
23 * @brief heap-based datastore backend; usually we want the datastore
24 * to be persistent, and storing data in the heap is obviously
25 * NOT going to be persistent; still, this plugin is useful for
26 * testing/benchmarking --- but never for production!
27 * @author Christian Grothoff
28 */
29
30#include "platform.h"
31#include "gnunet_datastore_plugin.h"
32
33
34/**
35 * A value that we are storing.
36 */
37struct Value
38{
39 /**
40 * Key for the value.
41 */
42 struct GNUNET_HashCode key;
43
44 /**
45 * Pointer to the value's data (allocated at the end of this struct).
46 */
47 const void *data;
48
49 /**
50 * Entry for this value in the 'expire' heap.
51 */
52 struct GNUNET_CONTAINER_HeapNode *expire_heap;
53
54 /**
55 * Entry for this value in the 'replication' heap.
56 */
57 struct GNUNET_CONTAINER_HeapNode *replication_heap;
58
59 /**
60 * Expiration time for this value.
61 */
62 struct GNUNET_TIME_Absolute expiration;
63
64 /**
65 * Offset of this value in the array of the 'struct ZeroAnonByType';
66 * only used if anonymity is zero.
67 */
68 unsigned int zero_anon_offset;
69
70 /**
71 * Number of bytes in 'data'.
72 */
73 uint32_t size;
74
75 /**
76 * Priority of the value.
77 */
78 uint32_t priority;
79
80 /**
81 * Anonymity level for the value.
82 */
83 uint32_t anonymity;
84
85 /**
86 * Replication level for the value.
87 */
88 uint32_t replication;
89
90 /**
91 * Type of 'data'.
92 */
93 enum GNUNET_BLOCK_Type type;
94};
95
96
97/**
98 * We organize 0-anonymity values in arrays "by type".
99 */
100struct ZeroAnonByType
101{
102 /**
103 * We keep these in a DLL.
104 */
105 struct ZeroAnonByType *next;
106
107 /**
108 * We keep these in a DLL.
109 */
110 struct ZeroAnonByType *prev;
111
112 /**
113 * Array of 0-anonymity items of the given type.
114 */
115 struct Value **array;
116
117 /**
118 * Allocated size of the array.
119 */
120 unsigned int array_size;
121
122 /**
123 * First unused offset in 'array'.
124 */
125 unsigned int array_pos;
126
127 /**
128 * Type of all of the values in 'array'.
129 */
130 enum GNUNET_BLOCK_Type type;
131};
132
133
134/**
135 * Context for all functions in this plugin.
136 */
137struct Plugin
138{
139 /**
140 * Our execution environment.
141 */
142 struct GNUNET_DATASTORE_PluginEnvironment *env;
143
144 /**
145 * Mapping from keys to 'struct Value's.
146 */
147 struct GNUNET_CONTAINER_MultiHashMap *keyvalue;
148
149 /**
150 * Heap organized by minimum expiration time.
151 */
152 struct GNUNET_CONTAINER_Heap *by_expiration;
153
154 /**
155 * Heap organized by maximum replication value.
156 */
157 struct GNUNET_CONTAINER_Heap *by_replication;
158
159 /**
160 * Head of list of arrays containing zero-anonymity values by type.
161 */
162 struct ZeroAnonByType *zero_head;
163
164 /**
165 * Tail of list of arrays containing zero-anonymity values by type.
166 */
167 struct ZeroAnonByType *zero_tail;
168
169 /**
170 * Size of all values we're storing.
171 */
172 unsigned long long size;
173};
174
175
176/**
177 * Get an estimate of how much space the database is
178 * currently using.
179 *
180 * @param cls our "struct Plugin*"
181 * @return number of bytes used on disk
182 */
183static void
184heap_plugin_estimate_size (void *cls, unsigned long long *estimate)
185{
186 struct Plugin *plugin = cls;
187
188 if (NULL != estimate)
189 *estimate = plugin->size;
190}
191
192
193/**
194 * Closure for iterator for updating.
195 */
196struct UpdateContext
197{
198 /**
199 * Number of bytes in 'data'.
200 */
201 uint32_t size;
202
203 /**
204 * Pointer to the data.
205 */
206 const void *data;
207
208 /**
209 * Priority of the value.
210 */
211 uint32_t priority;
212
213 /**
214 * Replication level for the value.
215 */
216 uint32_t replication;
217
218 /**
219 * Expiration time for this value.
220 */
221 struct GNUNET_TIME_Absolute expiration;
222
223 /**
224 * True if the value was found and updated.
225 */
226 bool updated;
227};
228
229
230/**
231 * Update the matching value.
232 *
233 * @param cls the 'struct UpdateContext'
234 * @param key unused
235 * @param val the 'struct Value'
236 * @return GNUNET_YES (continue iteration), GNUNET_NO if value was found
237 */
238static int
239update_iterator (void *cls,
240 const struct GNUNET_HashCode *key,
241 void *val)
242{
243 struct UpdateContext *uc = cls;
244 struct Value *value = val;
245
246 if (value->size != uc->size)
247 return GNUNET_YES;
248 if (0 != memcmp (value->data, uc->data, uc->size))
249 return GNUNET_YES;
250 uc->expiration = GNUNET_TIME_absolute_max (value->expiration,
251 uc->expiration);
252 if (value->expiration.abs_value_us != uc->expiration.abs_value_us)
253 {
254 value->expiration = uc->expiration;
255 GNUNET_CONTAINER_heap_update_cost (value->expire_heap,
256 value->expiration.abs_value_us);
257 }
258 /* Saturating adds, don't overflow */
259 if (value->priority > UINT32_MAX - uc->priority)
260 value->priority = UINT32_MAX;
261 else
262 value->priority += uc->priority;
263 if (value->replication > UINT32_MAX - uc->replication)
264 value->replication = UINT32_MAX;
265 else
266 value->replication += uc->replication;
267 uc->updated = true;
268 return GNUNET_NO;
269}
270
271
272/**
273 * Store an item in the datastore.
274 *
275 * @param cls closure
276 * @param key key for the item
277 * @param absent true if the key was not found in the bloom filter
278 * @param size number of bytes in data
279 * @param data content stored
280 * @param type type of the content
281 * @param priority priority of the content
282 * @param anonymity anonymity-level for the content
283 * @param replication replication-level for the content
284 * @param expiration expiration time for the content
285 * @param cont continuation called with success or failure status
286 * @param cont_cls continuation closure
287 */
288static void
289heap_plugin_put (void *cls,
290 const struct GNUNET_HashCode *key,
291 bool absent,
292 uint32_t size,
293 const void *data,
294 enum GNUNET_BLOCK_Type type,
295 uint32_t priority,
296 uint32_t anonymity,
297 uint32_t replication,
298 struct GNUNET_TIME_Absolute expiration,
299 PluginPutCont cont,
300 void *cont_cls)
301{
302 struct Plugin *plugin = cls;
303 struct Value *value;
304
305 if (! absent)
306 {
307 struct UpdateContext uc;
308
309 uc.size = size;
310 uc.data = data;
311 uc.priority = priority;
312 uc.replication = replication;
313 uc.expiration = expiration;
314 uc.updated = false;
315 GNUNET_CONTAINER_multihashmap_get_multiple (plugin->keyvalue,
316 key,
317 &update_iterator,
318 &uc);
319 if (uc.updated)
320 {
321 cont (cont_cls, key, size, GNUNET_NO, NULL);
322 return;
323 }
324 }
325 value = GNUNET_malloc (sizeof(struct Value) + size);
326 value->key = *key;
327 value->data = &value[1];
328 value->expire_heap = GNUNET_CONTAINER_heap_insert (plugin->by_expiration,
329 value,
330 expiration.abs_value_us);
331 value->replication_heap = GNUNET_CONTAINER_heap_insert (
332 plugin->by_replication,
333 value,
334 replication);
335 value->expiration = expiration;
336 if (0 == anonymity)
337 {
338 struct ZeroAnonByType *zabt;
339
340 for (zabt = plugin->zero_head; NULL != zabt; zabt = zabt->next)
341 if (zabt->type == type)
342 break;
343 if (NULL == zabt)
344 {
345 zabt = GNUNET_new (struct ZeroAnonByType);
346 zabt->type = type;
347 GNUNET_CONTAINER_DLL_insert (plugin->zero_head,
348 plugin->zero_tail,
349 zabt);
350 }
351 if (zabt->array_size == zabt->array_pos)
352 {
353 GNUNET_array_grow (zabt->array,
354 zabt->array_size,
355 zabt->array_size * 2 + 4);
356 }
357 value->zero_anon_offset = zabt->array_pos;
358 zabt->array[zabt->array_pos++] = value;
359 }
360 value->size = size;
361 value->priority = priority;
362 value->anonymity = anonymity;
363 value->replication = replication;
364 value->type = type;
365 GNUNET_memcpy (&value[1], data, size);
366 GNUNET_CONTAINER_multihashmap_put (plugin->keyvalue,
367 &value->key,
368 value,
369 GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
370 plugin->size += size;
371 cont (cont_cls, key, size, GNUNET_OK, NULL);
372}
373
374
375/**
376 * Delete the given value, removing it from the plugin's data
377 * structures.
378 *
379 * @param plugin the plugin
380 * @param value value to delete
381 */
382static void
383delete_value (struct Plugin *plugin,
384 struct Value *value)
385{
386 GNUNET_assert (GNUNET_YES ==
387 GNUNET_CONTAINER_multihashmap_remove (plugin->keyvalue,
388 &value->key,
389 value));
390 GNUNET_assert (value == GNUNET_CONTAINER_heap_remove_node (
391 value->expire_heap));
392 GNUNET_assert (value == GNUNET_CONTAINER_heap_remove_node (
393 value->replication_heap));
394 if (0 == value->anonymity)
395 {
396 struct ZeroAnonByType *zabt;
397
398 for (zabt = plugin->zero_head; NULL != zabt; zabt = zabt->next)
399 if (zabt->type == value->type)
400 break;
401 GNUNET_assert (NULL != zabt);
402 zabt->array[value->zero_anon_offset] = zabt->array[--zabt->array_pos];
403 zabt->array[value->zero_anon_offset]->zero_anon_offset =
404 value->zero_anon_offset;
405 if (0 == zabt->array_pos)
406 {
407 GNUNET_array_grow (zabt->array,
408 zabt->array_size,
409 0);
410 GNUNET_CONTAINER_DLL_remove (plugin->zero_head,
411 plugin->zero_tail,
412 zabt);
413 GNUNET_free (zabt);
414 }
415 }
416 plugin->size -= value->size;
417 GNUNET_free (value);
418}
419
420
421/**
422 * Closure for iterator called during 'get_key'.
423 */
424struct GetContext
425{
426 /**
427 * Lowest uid to consider.
428 */
429 uint64_t next_uid;
430
431 /**
432 * Value with lowest uid >= next_uid found so far.
433 */
434 struct Value *value;
435
436 /**
437 * Requested type.
438 */
439 enum GNUNET_BLOCK_Type type;
440
441 /**
442 * If true, return a random value
443 */
444 bool random;
445};
446
447
448/**
449 * Obtain the matching value with the lowest uid >= next_uid.
450 *
451 * @param cls the 'struct GetContext'
452 * @param key unused
453 * @param val the 'struct Value'
454 * @return GNUNET_YES (continue iteration), GNUNET_NO if result was found
455 */
456static int
457get_iterator (void *cls,
458 const struct GNUNET_HashCode *key,
459 void *val)
460{
461 struct GetContext *gc = cls;
462 struct Value *value = val;
463
464 if ((gc->type != GNUNET_BLOCK_TYPE_ANY) &&
465 (gc->type != value->type))
466 return GNUNET_OK;
467 if (gc->random)
468 {
469 gc->value = value;
470 return GNUNET_NO;
471 }
472 if ((uint64_t) (intptr_t) value < gc->next_uid)
473 return GNUNET_OK;
474 if ((NULL != gc->value) &&
475 (value > gc->value))
476 return GNUNET_OK;
477 gc->value = value;
478 return GNUNET_OK;
479}
480
481
482/**
483 * Get one of the results for a particular key in the datastore.
484 *
485 * @param cls closure
486 * @param next_uid return the result with lowest uid >= next_uid
487 * @param random if true, return a random result instead of using next_uid
488 * @param key maybe NULL (to match all entries)
489 * @param type entries of which type are relevant?
490 * Use 0 for any type.
491 * @param proc function to call on the matching value;
492 * will be called with NULL if nothing matches
493 * @param proc_cls closure for @a proc
494 */
495static void
496heap_plugin_get_key (void *cls,
497 uint64_t next_uid,
498 bool random,
499 const struct GNUNET_HashCode *key,
500 enum GNUNET_BLOCK_Type type,
501 PluginDatumProcessor proc,
502 void *proc_cls)
503{
504 struct Plugin *plugin = cls;
505 struct GetContext gc;
506
507 gc.value = NULL;
508 gc.next_uid = next_uid;
509 gc.random = random;
510 gc.type = type;
511 if (NULL == key)
512 {
513 GNUNET_CONTAINER_multihashmap_iterate (plugin->keyvalue,
514 &get_iterator,
515 &gc);
516 }
517 else
518 {
519 GNUNET_CONTAINER_multihashmap_get_multiple (plugin->keyvalue,
520 key,
521 &get_iterator,
522 &gc);
523 }
524 if (NULL == gc.value)
525 {
526 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
527 return;
528 }
529 GNUNET_assert (GNUNET_OK ==
530 proc (proc_cls,
531 &gc.value->key,
532 gc.value->size,
533 &gc.value[1],
534 gc.value->type,
535 gc.value->priority,
536 gc.value->anonymity,
537 gc.value->replication,
538 gc.value->expiration,
539 (uint64_t) (intptr_t) gc.value));
540}
541
542
543/**
544 * Get a random item for replication. Returns a single, not expired,
545 * random item from those with the highest replication counters. The
546 * item's replication counter is decremented by one IF it was positive
547 * before. Call 'proc' with all values ZERO or NULL if the datastore
548 * is empty.
549 *
550 * @param cls closure
551 * @param proc function to call the value (once only).
552 * @param proc_cls closure for proc
553 */
554static void
555heap_plugin_get_replication (void *cls,
556 PluginDatumProcessor proc,
557 void *proc_cls)
558{
559 struct Plugin *plugin = cls;
560 struct Value *value;
561
562 value = GNUNET_CONTAINER_heap_remove_root (plugin->by_replication);
563 if (NULL == value)
564 {
565 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
566 return;
567 }
568 if (value->replication > 0)
569 {
570 value->replication--;
571 value->replication_heap = GNUNET_CONTAINER_heap_insert (
572 plugin->by_replication,
573 value,
574 value->replication);
575 }
576 else
577 {
578 /* need a better way to pick a random item, replication level is always 0 */
579 value->replication_heap = GNUNET_CONTAINER_heap_insert (
580 plugin->by_replication,
581 value,
582 value->replication);
583 value = GNUNET_CONTAINER_heap_walk_get_next (plugin->by_replication);
584 }
585 GNUNET_assert (GNUNET_OK ==
586 proc (proc_cls,
587 &value->key,
588 value->size,
589 &value[1],
590 value->type,
591 value->priority,
592 value->anonymity,
593 value->replication,
594 value->expiration,
595 (uint64_t) (intptr_t) value));
596}
597
598
599/**
600 * Get a random item for expiration. Call 'proc' with all values ZERO
601 * or NULL if the datastore is empty.
602 *
603 * @param cls closure
604 * @param proc function to call the value (once only).
605 * @param proc_cls closure for proc
606 */
607static void
608heap_plugin_get_expiration (void *cls, PluginDatumProcessor proc,
609 void *proc_cls)
610{
611 struct Plugin *plugin = cls;
612 struct Value *value;
613
614 value = GNUNET_CONTAINER_heap_peek (plugin->by_expiration);
615 if (NULL == value)
616 {
617 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
618 return;
619 }
620 if (GNUNET_NO ==
621 proc (proc_cls,
622 &value->key,
623 value->size,
624 &value[1],
625 value->type,
626 value->priority,
627 value->anonymity,
628 value->replication,
629 value->expiration,
630 (uint64_t) (intptr_t) value))
631 delete_value (plugin, value);
632}
633
634
635/**
636 * Call the given processor on an item with zero anonymity.
637 *
638 * @param cls our "struct Plugin*"
639 * @param next_uid return the result with lowest uid >= next_uid
640 * @param type entries of which type should be considered?
641 * Must not be zero (ANY).
642 * @param proc function to call on each matching value;
643 * will be called with NULL if no value matches
644 * @param proc_cls closure for proc
645 */
646static void
647heap_plugin_get_zero_anonymity (void *cls, uint64_t next_uid,
648 enum GNUNET_BLOCK_Type type,
649 PluginDatumProcessor proc, void *proc_cls)
650{
651 struct Plugin *plugin = cls;
652 struct ZeroAnonByType *zabt;
653 struct Value *value = NULL;
654
655 for (zabt = plugin->zero_head; NULL != zabt; zabt = zabt->next)
656 {
657 if ((type != GNUNET_BLOCK_TYPE_ANY) &&
658 (type != zabt->type))
659 continue;
660 for (int i = 0; i < zabt->array_pos; ++i)
661 {
662 if ((uint64_t) (intptr_t) zabt->array[i] < next_uid)
663 continue;
664 if ((NULL != value) &&
665 (zabt->array[i] > value))
666 continue;
667 value = zabt->array[i];
668 }
669 }
670 if (NULL == value)
671 {
672 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
673 return;
674 }
675 GNUNET_assert (GNUNET_OK ==
676 proc (proc_cls,
677 &value->key,
678 value->size,
679 &value[1],
680 value->type,
681 value->priority,
682 value->anonymity,
683 value->replication,
684 value->expiration,
685 (uint64_t) (intptr_t) value));
686}
687
688
689/**
690 * Drop database.
691 */
692static void
693heap_plugin_drop (void *cls)
694{
695 /* nothing needs to be done */
696}
697
698
699/**
700 * Closure for the 'return_value' function.
701 */
702struct GetAllContext
703{
704 /**
705 * Function to call.
706 */
707 PluginKeyProcessor proc;
708
709 /**
710 * Closure for 'proc'.
711 */
712 void *proc_cls;
713};
714
715
716/**
717 * Callback invoked to call callback on each value.
718 *
719 * @param cls the plugin
720 * @param key unused
721 * @param val the value
722 * @return GNUNET_OK (continue to iterate)
723 */
724static int
725return_value (void *cls,
726 const struct GNUNET_HashCode *key,
727 void *val)
728{
729 struct GetAllContext *gac = cls;
730
731 gac->proc (gac->proc_cls,
732 key,
733 1);
734 return GNUNET_OK;
735}
736
737
738/**
739 * Get all of the keys in the datastore.
740 *
741 * @param cls closure
742 * @param proc function to call on each key
743 * @param proc_cls closure for proc
744 */
745static void
746heap_get_keys (void *cls,
747 PluginKeyProcessor proc,
748 void *proc_cls)
749{
750 struct Plugin *plugin = cls;
751 struct GetAllContext gac;
752
753 gac.proc = proc;
754 gac.proc_cls = proc_cls;
755 GNUNET_CONTAINER_multihashmap_iterate (plugin->keyvalue,
756 &return_value,
757 &gac);
758 proc (proc_cls, NULL, 0);
759}
760
761
762/**
763 * Closure for iterator called during 'remove_key'.
764 */
765struct RemoveContext
766{
767 /**
768 * Value found.
769 */
770 struct Value *value;
771
772 /**
773 * Size of data.
774 */
775 uint32_t size;
776
777 /**
778 * Data to remove.
779 */
780 const void *data;
781};
782
783
784/**
785 * Obtain the matching value with the lowest uid >= next_uid.
786 *
787 * @param cls the 'struct GetContext'
788 * @param key unused
789 * @param val the 'struct Value'
790 * @return GNUNET_YES (continue iteration), GNUNET_NO if result was found
791 */
792static int
793remove_iterator (void *cls,
794 const struct GNUNET_HashCode *key,
795 void *val)
796{
797 struct RemoveContext *rc = cls;
798 struct Value *value = val;
799
800 if (value->size != rc->size)
801 return GNUNET_YES;
802 if (0 != memcmp (value->data, rc->data, rc->size))
803 return GNUNET_YES;
804 rc->value = value;
805 return GNUNET_NO;
806}
807
808
809/**
810 * Remove a particular key in the datastore.
811 *
812 * @param cls closure
813 * @param key key for the content
814 * @param size number of bytes in data
815 * @param data content stored
816 * @param cont continuation called with success or failure status
817 * @param cont_cls continuation closure for @a cont
818 */
819static void
820heap_plugin_remove_key (void *cls,
821 const struct GNUNET_HashCode *key,
822 uint32_t size,
823 const void *data,
824 PluginRemoveCont cont,
825 void *cont_cls)
826{
827 struct Plugin *plugin = cls;
828 struct RemoveContext rc;
829
830 rc.value = NULL;
831 rc.size = size;
832 rc.data = data;
833 GNUNET_CONTAINER_multihashmap_get_multiple (plugin->keyvalue,
834 key,
835 &remove_iterator,
836 &rc);
837 if (NULL == rc.value)
838 {
839 cont (cont_cls,
840 key,
841 size,
842 GNUNET_NO,
843 NULL);
844 return;
845 }
846 delete_value (plugin,
847 rc.value);
848 cont (cont_cls,
849 key,
850 size,
851 GNUNET_OK,
852 NULL);
853}
854
855
856/**
857 * Entry point for the plugin.
858 *
859 * @param cls the "struct GNUNET_DATASTORE_PluginEnvironment*"
860 * @return our "struct Plugin*"
861 */
862void *
863libgnunet_plugin_datastore_heap_init (void *cls)
864{
865 struct GNUNET_DATASTORE_PluginEnvironment *env = cls;
866 struct GNUNET_DATASTORE_PluginFunctions *api;
867 struct Plugin *plugin;
868 unsigned long long esize;
869
870 if (GNUNET_OK !=
871 GNUNET_CONFIGURATION_get_value_number (env->cfg,
872 "datastore-heap",
873 "HASHMAPSIZE",
874 &esize))
875 esize = 128 * 1024;
876 plugin = GNUNET_new (struct Plugin);
877 plugin->env = env;
878 plugin->keyvalue = GNUNET_CONTAINER_multihashmap_create (esize, GNUNET_YES);
879 plugin->by_expiration = GNUNET_CONTAINER_heap_create (
880 GNUNET_CONTAINER_HEAP_ORDER_MIN);
881 plugin->by_replication = GNUNET_CONTAINER_heap_create (
882 GNUNET_CONTAINER_HEAP_ORDER_MAX);
883 api = GNUNET_new (struct GNUNET_DATASTORE_PluginFunctions);
884 api->cls = plugin;
885 api->estimate_size = &heap_plugin_estimate_size;
886 api->put = &heap_plugin_put;
887 api->get_key = &heap_plugin_get_key;
888 api->get_replication = &heap_plugin_get_replication;
889 api->get_expiration = &heap_plugin_get_expiration;
890 api->get_zero_anonymity = &heap_plugin_get_zero_anonymity;
891 api->drop = &heap_plugin_drop;
892 api->get_keys = &heap_get_keys;
893 api->remove_key = &heap_plugin_remove_key;
894 GNUNET_log_from (GNUNET_ERROR_TYPE_INFO, "heap",
895 _ ("Heap database running\n"));
896 return api;
897}
898
899
900/**
901 * Callback invoked to free all value.
902 *
903 * @param cls the plugin
904 * @param key unused
905 * @param val the value
906 * @return GNUNET_OK (continue to iterate)
907 */
908static int
909free_value (void *cls,
910 const struct GNUNET_HashCode *key,
911 void *val)
912{
913 struct Plugin *plugin = cls;
914 struct Value *value = val;
915
916 delete_value (plugin, value);
917 return GNUNET_OK;
918}
919
920
921/**
922 * Exit point from the plugin.
923 * @param cls our "struct Plugin*"
924 * @return always NULL
925 */
926void *
927libgnunet_plugin_datastore_heap_done (void *cls)
928{
929 struct GNUNET_DATASTORE_PluginFunctions *api = cls;
930 struct Plugin *plugin = api->cls;
931
932 GNUNET_CONTAINER_multihashmap_iterate (plugin->keyvalue,
933 &free_value,
934 plugin);
935 GNUNET_CONTAINER_multihashmap_destroy (plugin->keyvalue);
936 GNUNET_CONTAINER_heap_destroy (plugin->by_expiration);
937 GNUNET_CONTAINER_heap_destroy (plugin->by_replication);
938 GNUNET_free (plugin);
939 GNUNET_free (api);
940 return NULL;
941}
942
943
944/* end of plugin_datastore_heap.c */
diff --git a/src/datastore/plugin_datastore_mysql.c b/src/datastore/plugin_datastore_mysql.c
deleted file mode 100644
index f62c51778..000000000
--- a/src/datastore/plugin_datastore_mysql.c
+++ /dev/null
@@ -1,1204 +0,0 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2009, 2010, 2011 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file datastore/plugin_datastore_mysql.c
23 * @brief mysql-based datastore backend
24 * @author Igor Wronsky
25 * @author Christian Grothoff
26 * @author Christophe Genevey
27 *
28 * NOTE: This db module does NOT work with mysql prior to 4.1 since
29 * it uses prepared statements. MySQL 5.0.46 promises to fix a bug
30 * in MyISAM that is causing us grief. At the time of this writing,
31 * that version is yet to be released. In anticipation, the code
32 * will use MyISAM with 5.0.46 (and higher). If you run such a
33 * version, please run "make check" to verify that the MySQL bug
34 * was actually fixed in your version (and if not, change the
35 * code below to use MyISAM for gn071).
36 *
37 * HIGHLIGHTS
38 *
39 * Pros
40 * + On up-to-date hardware where mysql can be used comfortably, this
41 * module will have better performance than the other db choices
42 * (according to our tests).
43 * + Its often possible to recover the mysql database from internal
44 * inconsistencies. The other db choices do not support repair!
45 * Cons
46 * - Memory usage (Comment: "I have 1G and it never caused me trouble")
47 * - Manual setup
48 *
49 * MANUAL SETUP INSTRUCTIONS
50 *
51 * 1) in gnunet.conf, set
52 * @verbatim
53 [datastore]
54 DATABASE = "mysql"
55 @endverbatim
56 * 2) Then access mysql as root,
57 * @verbatim
58 $ mysql -u root -p
59 @endverbatim
60 * and do the following. [You should replace $USER with the username
61 * that will be running the gnunetd process].
62 * @verbatim
63 CREATE DATABASE gnunet;
64 GRANT select,insert,update,delete,create,alter,drop,create temporary tables
65 ON gnunet.* TO $USER@localhost;
66 SET PASSWORD FOR $USER@localhost=PASSWORD('$the_password_you_like');
67 FLUSH PRIVILEGES;
68 @endverbatim
69 * 3) In the $HOME directory of $USER, create a ".my.cnf" file
70 * with the following lines
71 * @verbatim
72 [client]
73 user=$USER
74 password=$the_password_you_like
75 @endverbatim
76 *
77 * That's it. Note that .my.cnf file is a security risk unless its on
78 * a safe partition etc. The $HOME/.my.cnf can of course be a symbolic
79 * link. Even greater security risk can be achieved by setting no
80 * password for $USER. Luckily $USER has only privileges to mess
81 * up GNUnet's tables, nothing else (unless you give them more,
82 * of course).<p>
83 *
84 * 4) Still, perhaps you should briefly try if the DB connection
85 * works. First, login as $USER. Then use,
86 *
87 * @verbatim
88 $ mysql -u $USER -p $the_password_you_like
89 mysql> use gnunet;
90 @endverbatim
91 *
92 * If you get the message &quot;Database changed&quot; it probably works.
93 *
94 * [If you get &quot;ERROR 2002: Can't connect to local MySQL server
95 * through socket '/tmp/mysql.sock' (2)&quot; it may be resolvable by
96 * &quot;ln -s /var/run/mysqld/mysqld.sock /tmp/mysql.sock&quot;
97 * so there may be some additional trouble depending on your mysql setup.]
98 *
99 * REPAIRING TABLES
100 *
101 * - Its probably healthy to check your tables for inconsistencies
102 * every now and then.
103 * - If you get odd SEGVs on gnunetd startup, it might be that the mysql
104 * databases have been corrupted.
105 * - The tables can be verified/fixed in two ways;
106 * 1) by running mysqlcheck -A, or
107 * 2) by executing (inside of mysql using the GNUnet database):
108 * @verbatim
109 mysql> REPAIR TABLE gn090;
110 @endverbatim
111 *
112 * PROBLEMS?
113 *
114 * If you have problems related to the mysql module, your best
115 * friend is probably the mysql manual. The first thing to check
116 * is that mysql is basically operational, that you can connect
117 * to it, create tables, issue queries etc.
118 */
119
120#include "platform.h"
121#include "gnunet_datastore_plugin.h"
122#include "gnunet_util_lib.h"
123#include "gnunet_mysql_lib.h"
124#include "gnunet_mysql_compat.h"
125#include "gnunet_my_lib.h"
126
127#define MAX_DATUM_SIZE 65536
128
129
130/**
131 * Context for all functions in this plugin.
132 */
133struct Plugin
134{
135 /**
136 * Our execution environment.
137 */
138 struct GNUNET_DATASTORE_PluginEnvironment *env;
139
140 /**
141 * Handle to talk to MySQL.
142 */
143 struct GNUNET_MYSQL_Context *mc;
144
145 /**
146 * Prepared statements.
147 */
148#define INSERT_ENTRY \
149 "INSERT INTO gn090 (repl,type,prio,anonLevel,expire,rvalue,hash,vhash,value) VALUES (?,?,?,?,?,?,?,?,?)"
150 struct GNUNET_MYSQL_StatementHandle *insert_entry;
151
152#define DELETE_ENTRY_BY_UID "DELETE FROM gn090 WHERE uid=?"
153 struct GNUNET_MYSQL_StatementHandle *delete_entry_by_uid;
154
155#define DELETE_ENTRY_BY_HASH_VALUE "DELETE FROM gn090 " \
156 "WHERE hash = ? AND " \
157 "value = ? " \
158 "LIMIT 1"
159 struct GNUNET_MYSQL_StatementHandle *delete_entry_by_hash_value;
160
161#define RESULT_COLUMNS "repl, type, prio, anonLevel, expire, hash, value, uid"
162
163#define SELECT_ENTRY "SELECT " RESULT_COLUMNS " FROM gn090 " \
164 "WHERE uid >= ? AND " \
165 "(rvalue >= ? OR 0 = ?) " \
166 "ORDER BY uid LIMIT 1"
167 struct GNUNET_MYSQL_StatementHandle *select_entry;
168
169#define SELECT_ENTRY_BY_HASH "SELECT " RESULT_COLUMNS " FROM gn090 " \
170 "FORCE INDEX (idx_hash_type_uid) " \
171 "WHERE hash=? AND " \
172 "uid >= ? AND " \
173 "(rvalue >= ? OR 0 = ?) " \
174 "ORDER BY uid LIMIT 1"
175 struct GNUNET_MYSQL_StatementHandle *select_entry_by_hash;
176
177#define SELECT_ENTRY_BY_HASH_AND_TYPE "SELECT " RESULT_COLUMNS " FROM gn090 " \
178 "FORCE INDEX (idx_hash_type_uid) " \
179 "WHERE hash = ? AND " \
180 "type = ? AND " \
181 "uid >= ? AND " \
182 "(rvalue >= ? OR 0 = ?) " \
183 "ORDER BY uid LIMIT 1"
184 struct GNUNET_MYSQL_StatementHandle *select_entry_by_hash_and_type;
185
186#define UPDATE_ENTRY "UPDATE gn090 SET " \
187 "prio = prio + ?, " \
188 "repl = repl + ?, " \
189 "expire = GREATEST(expire, ?) " \
190 "WHERE hash = ? AND vhash = ?"
191 struct GNUNET_MYSQL_StatementHandle *update_entry;
192
193#define DEC_REPL "UPDATE gn090 SET repl=GREATEST (1, repl) - 1 WHERE uid=?"
194 struct GNUNET_MYSQL_StatementHandle *dec_repl;
195
196#define SELECT_SIZE "SELECT SUM(LENGTH(value)+256) FROM gn090"
197 struct GNUNET_MYSQL_StatementHandle *get_size;
198
199#define SELECT_IT_NON_ANONYMOUS "SELECT " RESULT_COLUMNS " FROM gn090 " \
200 "FORCE INDEX (idx_anonLevel_type_rvalue) " \
201 "WHERE anonLevel=0 AND " \
202 "type=? AND " \
203 "uid >= ? " \
204 "ORDER BY uid LIMIT 1"
205 struct GNUNET_MYSQL_StatementHandle *zero_iter;
206
207#define SELECT_IT_EXPIRATION "SELECT " RESULT_COLUMNS " FROM gn090 " \
208 "FORCE INDEX (idx_expire) " \
209 "WHERE expire < ? " \
210 "ORDER BY expire ASC LIMIT 1"
211 struct GNUNET_MYSQL_StatementHandle *select_expiration;
212
213#define SELECT_IT_PRIORITY "SELECT " RESULT_COLUMNS " FROM gn090 " \
214 "FORCE INDEX (idx_prio) " \
215 "ORDER BY prio ASC LIMIT 1"
216 struct GNUNET_MYSQL_StatementHandle *select_priority;
217
218#define SELECT_IT_REPLICATION "SELECT " RESULT_COLUMNS " FROM gn090 " \
219 "FORCE INDEX (idx_repl_rvalue) " \
220 "WHERE repl=? AND " \
221 " (rvalue>=? OR" \
222 " NOT EXISTS (SELECT 1 FROM gn090 FORCE INDEX (idx_repl_rvalue) WHERE repl=? AND rvalue>=?)) " \
223 "ORDER BY rvalue ASC " \
224 "LIMIT 1"
225 struct GNUNET_MYSQL_StatementHandle *select_replication;
226
227#define SELECT_MAX_REPL "SELECT MAX(repl) FROM gn090"
228 struct GNUNET_MYSQL_StatementHandle *max_repl;
229
230#define GET_ALL_KEYS "SELECT hash from gn090"
231 struct GNUNET_MYSQL_StatementHandle *get_all_keys;
232};
233
234#define MAX_PARAM 16
235
236/**
237 * Delete an entry from the gn090 table.
238 *
239 * @param plugin plugin context
240 * @param uid unique ID of the entry to delete
241 * @return #GNUNET_OK on success, #GNUNET_NO if no such value exists, #GNUNET_SYSERR on error
242 */
243static int
244do_delete_entry (struct Plugin *plugin,
245 unsigned long long uid)
246{
247 int ret;
248 uint64_t uid64 = (uint64_t) uid;
249 struct GNUNET_MY_QueryParam params_delete[] = {
250 GNUNET_MY_query_param_uint64 (&uid64),
251 GNUNET_MY_query_param_end
252 };
253
254 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
255 "Deleting value %llu from gn090 table\n",
256 uid);
257 ret = GNUNET_MY_exec_prepared (plugin->mc,
258 plugin->delete_entry_by_uid,
259 params_delete);
260 if (ret >= 0)
261 {
262 return GNUNET_OK;
263 }
264 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
265 "Deleting value %llu from gn090 table failed\n",
266 (unsigned long long) uid);
267 return ret;
268}
269
270
271/**
272 * Get an estimate of how much space the database is
273 * currently using.
274 *
275 * @param cls our `struct Plugin *`
276 * @return number of bytes used on disk
277 */
278static void
279mysql_plugin_estimate_size (void *cls,
280 unsigned long long *estimate)
281{
282 struct Plugin *plugin = cls;
283 uint64_t total;
284 int ret;
285 struct GNUNET_MY_QueryParam params_get[] = {
286 GNUNET_MY_query_param_end
287 };
288 struct GNUNET_MY_ResultSpec results_get[] = {
289 GNUNET_MY_result_spec_uint64 (&total),
290 GNUNET_MY_result_spec_end
291 };
292
293 ret = GNUNET_MY_exec_prepared (plugin->mc,
294 plugin->get_size,
295 params_get);
296 *estimate = 0;
297 total = UINT64_MAX;
298 if ((GNUNET_OK == ret) &&
299 (GNUNET_OK ==
300 GNUNET_MY_extract_result (plugin->get_size,
301 results_get)))
302 {
303 *estimate = (unsigned long long) total;
304 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
305 "Size estimate for MySQL payload is %lld\n",
306 (long long) total);
307 GNUNET_assert (UINT64_MAX != total);
308 GNUNET_break (GNUNET_NO ==
309 GNUNET_MY_extract_result (plugin->get_size,
310 NULL));
311 }
312}
313
314
315/**
316 * Store an item in the datastore.
317 *
318 * @param cls closure
319 * @param key key for the item
320 * @param absent true if the key was not found in the bloom filter
321 * @param size number of bytes in @a data
322 * @param data content stored
323 * @param type type of the content
324 * @param priority priority of the content
325 * @param anonymity anonymity-level for the content
326 * @param replication replication-level for the content
327 * @param expiration expiration time for the content
328 * @param cont continuation called with success or failure status
329 * @param cont_cls closure for @a cont
330 */
331static void
332mysql_plugin_put (void *cls,
333 const struct GNUNET_HashCode *key,
334 bool absent,
335 uint32_t size,
336 const void *data,
337 enum GNUNET_BLOCK_Type type,
338 uint32_t priority,
339 uint32_t anonymity,
340 uint32_t replication,
341 struct GNUNET_TIME_Absolute expiration,
342 PluginPutCont cont,
343 void *cont_cls)
344{
345 struct Plugin *plugin = cls;
346 uint64_t lexpiration = expiration.abs_value_us;
347 struct GNUNET_HashCode vhash;
348
349 GNUNET_CRYPTO_hash (data,
350 size,
351 &vhash);
352 if (! absent)
353 {
354 struct GNUNET_MY_QueryParam params_update[] = {
355 GNUNET_MY_query_param_uint32 (&priority),
356 GNUNET_MY_query_param_uint32 (&replication),
357 GNUNET_MY_query_param_uint64 (&lexpiration),
358 GNUNET_MY_query_param_auto_from_type (key),
359 GNUNET_MY_query_param_auto_from_type (&vhash),
360 GNUNET_MY_query_param_end
361 };
362
363 if (GNUNET_OK !=
364 GNUNET_MY_exec_prepared (plugin->mc,
365 plugin->update_entry,
366 params_update))
367 {
368 cont (cont_cls,
369 key,
370 size,
371 GNUNET_SYSERR,
372 _ ("MySQL statement run failure"));
373 return;
374 }
375
376 MYSQL_STMT *stmt = GNUNET_MYSQL_statement_get_stmt (plugin->update_entry);
377 my_ulonglong rows = mysql_stmt_affected_rows (stmt);
378
379 GNUNET_break (GNUNET_NO ==
380 GNUNET_MY_extract_result (plugin->update_entry,
381 NULL));
382 if (0 != rows)
383 {
384 cont (cont_cls,
385 key,
386 size,
387 GNUNET_NO,
388 NULL);
389 return;
390 }
391 }
392
393 uint64_t lrvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
394 UINT64_MAX);
395 struct GNUNET_MY_QueryParam params_insert[] = {
396 GNUNET_MY_query_param_uint32 (&replication),
397 GNUNET_MY_query_param_uint32 (&type),
398 GNUNET_MY_query_param_uint32 (&priority),
399 GNUNET_MY_query_param_uint32 (&anonymity),
400 GNUNET_MY_query_param_uint64 (&lexpiration),
401 GNUNET_MY_query_param_uint64 (&lrvalue),
402 GNUNET_MY_query_param_auto_from_type (key),
403 GNUNET_MY_query_param_auto_from_type (&vhash),
404 GNUNET_MY_query_param_fixed_size (data, size),
405 GNUNET_MY_query_param_end
406 };
407
408 if (size > MAX_DATUM_SIZE)
409 {
410 GNUNET_break (0);
411 cont (cont_cls, key, size, GNUNET_SYSERR, _ ("Data too large"));
412 return;
413 }
414
415 if (GNUNET_OK !=
416 GNUNET_MY_exec_prepared (plugin->mc,
417 plugin->insert_entry,
418 params_insert))
419 {
420 cont (cont_cls,
421 key,
422 size,
423 GNUNET_SYSERR,
424 _ ("MySQL statement run failure"));
425 return;
426 }
427 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
428 "Inserted value `%s' with size %u into gn090 table\n",
429 GNUNET_h2s (key),
430 (unsigned int) size);
431 if (size > 0)
432 plugin->env->duc (plugin->env->cls,
433 size);
434 GNUNET_break (GNUNET_NO ==
435 GNUNET_MY_extract_result (plugin->insert_entry,
436 NULL));
437 cont (cont_cls,
438 key,
439 size,
440 GNUNET_OK,
441 NULL);
442}
443
444
445/**
446 * Run the given select statement and call 'proc' on the resulting
447 * values (which must be in particular positions).
448 *
449 * @param plugin the plugin handle
450 * @param stmt select statement to run
451 * @param proc function to call on result
452 * @param proc_cls closure for @a proc
453 * @param params_select arguments to initialize stmt
454 */
455static void
456execute_select (struct Plugin *plugin,
457 struct GNUNET_MYSQL_StatementHandle *stmt,
458 PluginDatumProcessor proc,
459 void *proc_cls,
460 struct GNUNET_MY_QueryParam *params_select)
461{
462 int ret;
463 uint32_t replication;
464 uint32_t type;
465 uint32_t priority;
466 uint32_t anonymity;
467 uint64_t uid;
468 size_t value_size;
469 void *value;
470 struct GNUNET_HashCode key;
471 struct GNUNET_TIME_Absolute expiration;
472 struct GNUNET_MY_ResultSpec results_select[] = {
473 GNUNET_MY_result_spec_uint32 (&replication),
474 GNUNET_MY_result_spec_uint32 (&type),
475 GNUNET_MY_result_spec_uint32 (&priority),
476 GNUNET_MY_result_spec_uint32 (&anonymity),
477 GNUNET_MY_result_spec_absolute_time (&expiration),
478 GNUNET_MY_result_spec_auto_from_type (&key),
479 GNUNET_MY_result_spec_variable_size (&value, &value_size),
480 GNUNET_MY_result_spec_uint64 (&uid),
481 GNUNET_MY_result_spec_end
482 };
483
484 ret = GNUNET_MY_exec_prepared (plugin->mc,
485 stmt,
486 params_select);
487 if (GNUNET_OK != ret)
488 {
489 proc (proc_cls,
490 NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
491 return;
492 }
493
494 ret = GNUNET_MY_extract_result (stmt,
495 results_select);
496 if (GNUNET_OK != ret)
497 {
498 proc (proc_cls,
499 NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
500 return;
501 }
502
503 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
504 "Found %u-byte value under key `%s' with prio %u, anon %u, expire %s selecting from gn090 table\n",
505 (unsigned int) value_size,
506 GNUNET_h2s (&key),
507 (unsigned int) priority,
508 (unsigned int) anonymity,
509 GNUNET_STRINGS_absolute_time_to_string (expiration));
510 GNUNET_assert (value_size < MAX_DATUM_SIZE);
511 GNUNET_break (GNUNET_NO ==
512 GNUNET_MY_extract_result (stmt,
513 NULL));
514 ret = proc (proc_cls,
515 &key,
516 value_size,
517 value,
518 type,
519 priority,
520 anonymity,
521 replication,
522 expiration,
523 uid);
524 GNUNET_MY_cleanup_result (results_select);
525 if (GNUNET_NO == ret)
526 {
527 do_delete_entry (plugin, uid);
528 if (0 != value_size)
529 plugin->env->duc (plugin->env->cls,
530 -value_size);
531 }
532}
533
534
535/**
536 * Get one of the results for a particular key in the datastore.
537 *
538 * @param cls closure
539 * @param next_uid return the result with lowest uid >= next_uid
540 * @param random if true, return a random result instead of using next_uid
541 * @param key key to match, never NULL
542 * @param type entries of which type are relevant?
543 * Use 0 for any type.
544 * @param proc function to call on the matching value,
545 * with NULL for if no value matches
546 * @param proc_cls closure for @a proc
547 */
548static void
549mysql_plugin_get_key (void *cls,
550 uint64_t next_uid,
551 bool random,
552 const struct GNUNET_HashCode *key,
553 enum GNUNET_BLOCK_Type type,
554 PluginDatumProcessor proc,
555 void *proc_cls)
556{
557 struct Plugin *plugin = cls;
558 uint64_t rvalue;
559
560 if (random)
561 {
562 rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
563 UINT64_MAX);
564 next_uid = 0;
565 }
566 else
567 rvalue = 0;
568
569 if (NULL == key)
570 {
571 struct GNUNET_MY_QueryParam params_select[] = {
572 GNUNET_MY_query_param_uint64 (&next_uid),
573 GNUNET_MY_query_param_uint64 (&rvalue),
574 GNUNET_MY_query_param_uint64 (&rvalue),
575 GNUNET_MY_query_param_end
576 };
577
578 execute_select (plugin,
579 plugin->select_entry,
580 proc,
581 proc_cls,
582 params_select);
583 }
584 else if (type != GNUNET_BLOCK_TYPE_ANY)
585 {
586 struct GNUNET_MY_QueryParam params_select[] = {
587 GNUNET_MY_query_param_auto_from_type (key),
588 GNUNET_MY_query_param_uint32 (&type),
589 GNUNET_MY_query_param_uint64 (&next_uid),
590 GNUNET_MY_query_param_uint64 (&rvalue),
591 GNUNET_MY_query_param_uint64 (&rvalue),
592 GNUNET_MY_query_param_end
593 };
594
595 execute_select (plugin,
596 plugin->select_entry_by_hash_and_type,
597 proc,
598 proc_cls,
599 params_select);
600 }
601 else
602 {
603 struct GNUNET_MY_QueryParam params_select[] = {
604 GNUNET_MY_query_param_auto_from_type (key),
605 GNUNET_MY_query_param_uint64 (&next_uid),
606 GNUNET_MY_query_param_uint64 (&rvalue),
607 GNUNET_MY_query_param_uint64 (&rvalue),
608 GNUNET_MY_query_param_end
609 };
610
611 execute_select (plugin,
612 plugin->select_entry_by_hash,
613 proc,
614 proc_cls,
615 params_select);
616 }
617}
618
619
620/**
621 * Get a zero-anonymity datum from the datastore.
622 *
623 * @param cls our `struct Plugin *`
624 * @param next_uid return the result with lowest uid >= next_uid
625 * @param type entries of which type should be considered?
626 * Must not be zero (ANY).
627 * @param proc function to call on a matching value;
628 * will be called with NULL if no value matches
629 * @param proc_cls closure for @a proc
630 */
631static void
632mysql_plugin_get_zero_anonymity (void *cls,
633 uint64_t next_uid,
634 enum GNUNET_BLOCK_Type type,
635 PluginDatumProcessor proc,
636 void *proc_cls)
637{
638 struct Plugin *plugin = cls;
639 uint32_t typei = (uint32_t) type;
640
641 struct GNUNET_MY_QueryParam params_zero_iter[] = {
642 GNUNET_MY_query_param_uint32 (&typei),
643 GNUNET_MY_query_param_uint64 (&next_uid),
644 GNUNET_MY_query_param_end
645 };
646
647 execute_select (plugin,
648 plugin->zero_iter,
649 proc,
650 proc_cls,
651 params_zero_iter);
652}
653
654
655/**
656 * Context for #repl_proc() function.
657 */
658struct ReplCtx
659{
660 /**
661 * Plugin handle.
662 */
663 struct Plugin *plugin;
664
665 /**
666 * Function to call for the result (or the NULL).
667 */
668 PluginDatumProcessor proc;
669
670 /**
671 * Closure for @e proc.
672 */
673 void *proc_cls;
674};
675
676
677/**
678 * Wrapper for the processor for #mysql_plugin_get_replication().
679 * Decrements the replication counter and calls the original
680 * iterator.
681 *
682 * @param cls closure
683 * @param key key for the content
684 * @param size number of bytes in @a data
685 * @param data content stored
686 * @param type type of the content
687 * @param priority priority of the content
688 * @param anonymity anonymity-level for the content
689 * @param replication replication-level for the content
690 * @param expiration expiration time for the content
691 * @param uid unique identifier for the datum;
692 * maybe 0 if no unique identifier is available
693 * @return #GNUNET_SYSERR to abort the iteration, #GNUNET_OK to continue
694 * (continue on call to "next", of course),
695 * #GNUNET_NO to delete the item and continue (if supported)
696 */
697static int
698repl_proc (void *cls,
699 const struct GNUNET_HashCode *key,
700 uint32_t size,
701 const void *data,
702 enum GNUNET_BLOCK_Type type,
703 uint32_t priority,
704 uint32_t anonymity,
705 uint32_t replication,
706 struct GNUNET_TIME_Absolute expiration,
707 uint64_t uid)
708{
709 struct ReplCtx *rc = cls;
710 struct Plugin *plugin = rc->plugin;
711 int ret;
712 int iret;
713
714 ret = rc->proc (rc->proc_cls,
715 key,
716 size,
717 data,
718 type,
719 priority,
720 anonymity,
721 replication,
722 expiration,
723 uid);
724 if (NULL != key)
725 {
726 struct GNUNET_MY_QueryParam params_proc[] = {
727 GNUNET_MY_query_param_uint64 (&uid),
728 GNUNET_MY_query_param_end
729 };
730
731 iret = GNUNET_MY_exec_prepared (plugin->mc,
732 plugin->dec_repl,
733 params_proc);
734 if (GNUNET_SYSERR == iret)
735 {
736 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
737 "Failed to reduce replication counter\n");
738 return GNUNET_SYSERR;
739 }
740 }
741 return ret;
742}
743
744
745/**
746 * Get a random item for replication. Returns a single, not expired,
747 * random item from those with the highest replication counters. The
748 * item's replication counter is decremented by one IF it was positive
749 * before. Call @a proc with all values ZERO or NULL if the datastore
750 * is empty.
751 *
752 * @param cls closure
753 * @param proc function to call the value (once only).
754 * @param proc_cls closure for @a proc
755 */
756static void
757mysql_plugin_get_replication (void *cls,
758 PluginDatumProcessor proc,
759 void *proc_cls)
760{
761 struct Plugin *plugin = cls;
762 uint64_t rvalue;
763 uint32_t repl;
764 struct ReplCtx rc;
765 struct GNUNET_MY_QueryParam params_get[] = {
766 GNUNET_MY_query_param_end
767 };
768 struct GNUNET_MY_ResultSpec results_get[] = {
769 GNUNET_MY_result_spec_uint32 (&repl),
770 GNUNET_MY_result_spec_end
771 };
772 struct GNUNET_MY_QueryParam params_select[] = {
773 GNUNET_MY_query_param_uint32 (&repl),
774 GNUNET_MY_query_param_uint64 (&rvalue),
775 GNUNET_MY_query_param_uint32 (&repl),
776 GNUNET_MY_query_param_uint64 (&rvalue),
777 GNUNET_MY_query_param_end
778 };
779
780 rc.plugin = plugin;
781 rc.proc = proc;
782 rc.proc_cls = proc_cls;
783
784 if (1 !=
785 GNUNET_MY_exec_prepared (plugin->mc,
786 plugin->max_repl,
787 params_get))
788 {
789 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
790 return;
791 }
792
793 if (GNUNET_OK !=
794 GNUNET_MY_extract_result (plugin->max_repl,
795 results_get))
796 {
797 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
798 return;
799 }
800 GNUNET_break (GNUNET_NO ==
801 GNUNET_MY_extract_result (plugin->max_repl,
802 NULL));
803 rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
804 UINT64_MAX);
805
806 execute_select (plugin,
807 plugin->select_replication,
808 &repl_proc,
809 &rc,
810 params_select);
811}
812
813
814/**
815 * Get all of the keys in the datastore.
816 *
817 * @param cls closure
818 * @param proc function to call on each key
819 * @param proc_cls closure for @a proc
820 */
821static void
822mysql_plugin_get_keys (void *cls,
823 PluginKeyProcessor proc,
824 void *proc_cls)
825{
826 struct Plugin *plugin = cls;
827 int ret;
828 MYSQL_STMT *statement;
829 unsigned int cnt;
830 struct GNUNET_HashCode key;
831 struct GNUNET_HashCode last;
832 struct GNUNET_MY_QueryParam params_select[] = {
833 GNUNET_MY_query_param_end
834 };
835 struct GNUNET_MY_ResultSpec results_select[] = {
836 GNUNET_MY_result_spec_auto_from_type (&key),
837 GNUNET_MY_result_spec_end
838 };
839
840 GNUNET_assert (NULL != proc);
841 statement = GNUNET_MYSQL_statement_get_stmt (plugin->get_all_keys);
842 if (GNUNET_OK !=
843 GNUNET_MY_exec_prepared (plugin->mc,
844 plugin->get_all_keys,
845 params_select))
846 {
847 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
848 _ ("`%s' for `%s' failed at %s:%d with error: %s\n"),
849 "mysql_stmt_execute",
850 GET_ALL_KEYS,
851 __FILE__,
852 __LINE__,
853 mysql_stmt_error (statement));
854 GNUNET_MYSQL_statements_invalidate (plugin->mc);
855 proc (proc_cls, NULL, 0);
856 return;
857 }
858 memset (&last, 0, sizeof(last)); /* make static analysis happy */
859 ret = GNUNET_YES;
860 cnt = 0;
861 while (ret == GNUNET_YES)
862 {
863 ret = GNUNET_MY_extract_result (plugin->get_all_keys,
864 results_select);
865 if (0 != GNUNET_memcmp (&last,
866 &key))
867 {
868 if (0 != cnt)
869 proc (proc_cls,
870 &last,
871 cnt);
872 cnt = 1;
873 last = key;
874 }
875 else
876 {
877 cnt++;
878 }
879 }
880 if (0 != cnt)
881 proc (proc_cls,
882 &last,
883 cnt);
884 /* finally, let app know we are done */
885 proc (proc_cls,
886 NULL,
887 0);
888 if (GNUNET_SYSERR == ret)
889 {
890 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
891 _ ("`%s' failed at %s:%d with error: %s\n"),
892 "mysql_stmt_fetch",
893 __FILE__,
894 __LINE__,
895 mysql_stmt_error (statement));
896 GNUNET_MYSQL_statements_invalidate (plugin->mc);
897 return;
898 }
899}
900
901
902/**
903 * Context for #expi_proc() function.
904 */
905struct ExpiCtx
906{
907 /**
908 * Plugin handle.
909 */
910 struct Plugin *plugin;
911
912 /**
913 * Function to call for the result (or the NULL).
914 */
915 PluginDatumProcessor proc;
916
917 /**
918 * Closure for @e proc.
919 */
920 void *proc_cls;
921};
922
923
924/**
925 * Wrapper for the processor for #mysql_plugin_get_expiration().
926 * If no expired value was found, we do a second query for
927 * low-priority content.
928 *
929 * @param cls closure
930 * @param key key for the content
931 * @param size number of bytes in data
932 * @param data content stored
933 * @param type type of the content
934 * @param priority priority of the content
935 * @param anonymity anonymity-level for the content
936 * @param replication replication-level for the content
937 * @param expiration expiration time for the content
938 * @param uid unique identifier for the datum;
939 * maybe 0 if no unique identifier is available
940 * @return #GNUNET_SYSERR to abort the iteration, #GNUNET_OK to continue
941 * (continue on call to "next", of course),
942 * #GNUNET_NO to delete the item and continue (if supported)
943 */
944static int
945expi_proc (void *cls,
946 const struct GNUNET_HashCode *key,
947 uint32_t size,
948 const void *data,
949 enum GNUNET_BLOCK_Type type,
950 uint32_t priority,
951 uint32_t anonymity,
952 uint32_t replication,
953 struct GNUNET_TIME_Absolute expiration,
954 uint64_t uid)
955{
956 struct ExpiCtx *rc = cls;
957 struct Plugin *plugin = rc->plugin;
958 struct GNUNET_MY_QueryParam params_select[] = {
959 GNUNET_MY_query_param_end
960 };
961
962 if (NULL == key)
963 {
964 execute_select (plugin,
965 plugin->select_priority,
966 rc->proc,
967 rc->proc_cls,
968 params_select);
969 return GNUNET_SYSERR;
970 }
971 return rc->proc (rc->proc_cls,
972 key,
973 size,
974 data,
975 type,
976 priority,
977 anonymity,
978 replication,
979 expiration,
980 uid);
981}
982
983
984/**
985 * Get a random item for expiration.
986 * Call @a proc with all values ZERO or NULL if the datastore is empty.
987 *
988 * @param cls closure
989 * @param proc function to call the value (once only).
990 * @param proc_cls closure for @a proc
991 */
992static void
993mysql_plugin_get_expiration (void *cls,
994 PluginDatumProcessor proc,
995 void *proc_cls)
996{
997 struct Plugin *plugin = cls;
998 struct GNUNET_TIME_Absolute now = { 0 };
999 struct GNUNET_MY_QueryParam params_select[] = {
1000 GNUNET_MY_query_param_absolute_time (&now),
1001 GNUNET_MY_query_param_end
1002 };
1003 struct ExpiCtx rc;
1004
1005 rc.plugin = plugin;
1006 rc.proc = proc;
1007 rc.proc_cls = proc_cls;
1008 now = GNUNET_TIME_absolute_get ();
1009 execute_select (plugin,
1010 plugin->select_expiration,
1011 expi_proc,
1012 &rc,
1013 params_select);
1014}
1015
1016
1017/**
1018 * Drop database.
1019 *
1020 * @param cls the `struct Plugin *`
1021 */
1022static void
1023mysql_plugin_drop (void *cls)
1024{
1025 struct Plugin *plugin = cls;
1026
1027 if (GNUNET_OK !=
1028 GNUNET_MYSQL_statement_run (plugin->mc,
1029 "DROP TABLE gn090"))
1030 return; /* error */
1031 plugin->env->duc (plugin->env->cls, 0);
1032}
1033
1034
1035/**
1036 * Remove a particular key in the datastore.
1037 *
1038 * @param cls closure
1039 * @param key key for the content
1040 * @param size number of bytes in data
1041 * @param data content stored
1042 * @param cont continuation called with success or failure status
1043 * @param cont_cls continuation closure for @a cont
1044 */
1045static void
1046mysql_plugin_remove_key (void *cls,
1047 const struct GNUNET_HashCode *key,
1048 uint32_t size,
1049 const void *data,
1050 PluginRemoveCont cont,
1051 void *cont_cls)
1052{
1053 struct Plugin *plugin = cls;
1054 struct GNUNET_MY_QueryParam params_delete[] = {
1055 GNUNET_MY_query_param_auto_from_type (key),
1056 GNUNET_MY_query_param_fixed_size (data, size),
1057 GNUNET_MY_query_param_end
1058 };
1059
1060 if (GNUNET_OK !=
1061 GNUNET_MY_exec_prepared (plugin->mc,
1062 plugin->delete_entry_by_hash_value,
1063 params_delete))
1064 {
1065 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1066 "Removing key `%s' from gn090 table failed\n",
1067 GNUNET_h2s (key));
1068 cont (cont_cls,
1069 key,
1070 size,
1071 GNUNET_SYSERR,
1072 _ ("MySQL statement run failure"));
1073 return;
1074 }
1075
1076 MYSQL_STMT *stmt = GNUNET_MYSQL_statement_get_stmt (
1077 plugin->delete_entry_by_hash_value);
1078 my_ulonglong rows = mysql_stmt_affected_rows (stmt);
1079
1080 if (0 == rows)
1081 {
1082 cont (cont_cls,
1083 key,
1084 size,
1085 GNUNET_NO,
1086 NULL);
1087 return;
1088 }
1089 plugin->env->duc (plugin->env->cls,
1090 -size);
1091 cont (cont_cls,
1092 key,
1093 size,
1094 GNUNET_OK,
1095 NULL);
1096}
1097
1098
1099/**
1100 * Entry point for the plugin.
1101 *
1102 * @param cls the `struct GNUNET_DATASTORE_PluginEnvironment *`
1103 * @return our `struct Plugin *`
1104 */
1105void *
1106libgnunet_plugin_datastore_mysql_init (void *cls)
1107{
1108 struct GNUNET_DATASTORE_PluginEnvironment *env = cls;
1109 struct GNUNET_DATASTORE_PluginFunctions *api;
1110 struct Plugin *plugin;
1111
1112 plugin = GNUNET_new (struct Plugin);
1113 plugin->env = env;
1114 plugin->mc = GNUNET_MYSQL_context_create (env->cfg,
1115 "datastore-mysql");
1116 if (NULL == plugin->mc)
1117 {
1118 GNUNET_free (plugin);
1119 return NULL;
1120 }
1121#define MRUNS(a) (GNUNET_OK != GNUNET_MYSQL_statement_run (plugin->mc, a))
1122#define PINIT(a, b) (NULL == (a = GNUNET_MYSQL_statement_prepare (plugin->mc, \
1123 b)))
1124 if (MRUNS
1125 ("CREATE TABLE IF NOT EXISTS gn090 ("
1126 " repl INT(11) UNSIGNED NOT NULL DEFAULT 0,"
1127 " type INT(11) UNSIGNED NOT NULL DEFAULT 0,"
1128 " prio INT(11) UNSIGNED NOT NULL DEFAULT 0,"
1129 " anonLevel INT(11) UNSIGNED NOT NULL DEFAULT 0,"
1130 " expire BIGINT UNSIGNED NOT NULL DEFAULT 0,"
1131 " rvalue BIGINT UNSIGNED NOT NULL,"
1132 " hash BINARY(64) NOT NULL DEFAULT '',"
1133 " vhash BINARY(64) NOT NULL DEFAULT '',"
1134 " value BLOB NOT NULL DEFAULT '',"
1135 " uid BIGINT NOT NULL AUTO_INCREMENT,"
1136 " PRIMARY KEY (uid),"
1137 " INDEX idx_hash_type_uid (hash(64),type,rvalue),"
1138 " INDEX idx_prio (prio),"
1139 " INDEX idx_repl_rvalue (repl,rvalue),"
1140 " INDEX idx_expire (expire),"
1141 " INDEX idx_anonLevel_type_rvalue (anonLevel,type,rvalue)"
1142 ") ENGINE=InnoDB") || MRUNS ("SET AUTOCOMMIT = 1") ||
1143 PINIT (plugin->insert_entry, INSERT_ENTRY) ||
1144 PINIT (plugin->delete_entry_by_uid, DELETE_ENTRY_BY_UID) ||
1145 PINIT (plugin->delete_entry_by_hash_value, DELETE_ENTRY_BY_HASH_VALUE) ||
1146 PINIT (plugin->select_entry, SELECT_ENTRY) ||
1147 PINIT (plugin->select_entry_by_hash, SELECT_ENTRY_BY_HASH) ||
1148 PINIT (plugin->select_entry_by_hash_and_type,
1149 SELECT_ENTRY_BY_HASH_AND_TYPE) ||
1150 PINIT (plugin->get_size, SELECT_SIZE) ||
1151 PINIT (plugin->update_entry, UPDATE_ENTRY) ||
1152 PINIT (plugin->dec_repl, DEC_REPL) ||
1153 PINIT (plugin->zero_iter, SELECT_IT_NON_ANONYMOUS) ||
1154 PINIT (plugin->select_expiration, SELECT_IT_EXPIRATION) ||
1155 PINIT (plugin->select_priority, SELECT_IT_PRIORITY) ||
1156 PINIT (plugin->max_repl, SELECT_MAX_REPL) ||
1157 PINIT (plugin->get_all_keys, GET_ALL_KEYS) ||
1158 PINIT (plugin->select_replication, SELECT_IT_REPLICATION) ||
1159 false)
1160 {
1161 GNUNET_MYSQL_context_destroy (plugin->mc);
1162 GNUNET_free (plugin);
1163 return NULL;
1164 }
1165#undef PINIT
1166#undef MRUNS
1167
1168 api = GNUNET_new (struct GNUNET_DATASTORE_PluginFunctions);
1169 api->cls = plugin;
1170 api->estimate_size = &mysql_plugin_estimate_size;
1171 api->put = &mysql_plugin_put;
1172 api->get_key = &mysql_plugin_get_key;
1173 api->get_replication = &mysql_plugin_get_replication;
1174 api->get_expiration = &mysql_plugin_get_expiration;
1175 api->get_zero_anonymity = &mysql_plugin_get_zero_anonymity;
1176 api->get_keys = &mysql_plugin_get_keys;
1177 api->drop = &mysql_plugin_drop;
1178 api->remove_key = &mysql_plugin_remove_key;
1179 GNUNET_log_from (GNUNET_ERROR_TYPE_INFO, "mysql",
1180 _ ("Mysql database running\n"));
1181 return api;
1182}
1183
1184
1185/**
1186 * Exit point from the plugin.
1187 *
1188 * @param cls our `struct Plugin *`
1189 * @return always NULL
1190 */
1191void *
1192libgnunet_plugin_datastore_mysql_done (void *cls)
1193{
1194 struct GNUNET_DATASTORE_PluginFunctions *api = cls;
1195 struct Plugin *plugin = api->cls;
1196
1197 GNUNET_MYSQL_context_destroy (plugin->mc);
1198 GNUNET_free (plugin);
1199 GNUNET_free (api);
1200 return NULL;
1201}
1202
1203
1204/* end of plugin_datastore_mysql.c */
diff --git a/src/datastore/plugin_datastore_postgres.c b/src/datastore/plugin_datastore_postgres.c
deleted file mode 100644
index 5fcacc17b..000000000
--- a/src/datastore/plugin_datastore_postgres.c
+++ /dev/null
@@ -1,930 +0,0 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2009-2017, 2022 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file datastore/plugin_datastore_postgres.c
23 * @brief postgres-based datastore backend
24 * @author Christian Grothoff
25 */
26#include "platform.h"
27#include "gnunet_datastore_plugin.h"
28#include "gnunet_pq_lib.h"
29
30
31/**
32 * After how many ms "busy" should a DB operation fail for good?
33 * A low value makes sure that we are more responsive to requests
34 * (especially PUTs). A high value guarantees a higher success
35 * rate (SELECTs in iterate can take several seconds despite LIMIT=1).
36 *
37 * The default value of 1s should ensure that users do not experience
38 * huge latencies while at the same time allowing operations to succeed
39 * with reasonable probability.
40 */
41#define BUSY_TIMEOUT GNUNET_TIME_UNIT_SECONDS
42
43
44/**
45 * Context for all functions in this plugin.
46 */
47struct Plugin
48{
49 /**
50 * Our execution environment.
51 */
52 struct GNUNET_DATASTORE_PluginEnvironment *env;
53
54 /**
55 * Native Postgres database handle.
56 */
57 struct GNUNET_PQ_Context *dbh;
58};
59
60
61/**
62 * @brief Get a database handle
63 *
64 * @param plugin global context
65 * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
66 */
67static enum GNUNET_GenericReturnValue
68init_connection (struct Plugin *plugin)
69{
70#define RESULT_COLUMNS "repl, type, prio, anonLevel, expire, hash, value, oid"
71 struct GNUNET_PQ_PreparedStatement ps[] = {
72 GNUNET_PQ_make_prepare ("get",
73 "SELECT " RESULT_COLUMNS
74 " FROM datastore.gn090"
75 " WHERE oid >= $1::bigint AND"
76 " (rvalue >= $2 OR 0 = $3::smallint) AND"
77 " (hash = $4 OR 0 = $5::smallint) AND"
78 " (type = $6 OR 0 = $7::smallint)"
79 " ORDER BY oid ASC LIMIT 1"),
80 GNUNET_PQ_make_prepare ("put",
81 "INSERT INTO datastore.gn090"
82 " (repl, type, prio, anonLevel, expire, rvalue, hash, vhash, value) "
83 "VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)"),
84 GNUNET_PQ_make_prepare ("update",
85 "UPDATE datastore.gn090"
86 " SET prio = prio + $1,"
87 " repl = repl + $2,"
88 " expire = GREATEST(expire, $3)"
89 " WHERE hash = $4 AND vhash = $5"),
90 GNUNET_PQ_make_prepare ("decrepl",
91 "UPDATE datastore.gn090"
92 " SET repl = GREATEST (repl - 1, 0)"
93 " WHERE oid = $1"),
94 GNUNET_PQ_make_prepare ("select_non_anonymous",
95 "SELECT " RESULT_COLUMNS
96 " FROM datastore.gn090"
97 " WHERE anonLevel = 0 AND type = $1 AND oid >= $2::bigint"
98 " ORDER BY oid ASC LIMIT 1"),
99 GNUNET_PQ_make_prepare ("select_expiration_order",
100 "(SELECT " RESULT_COLUMNS
101 " FROM datastore.gn090"
102 " WHERE expire < $1 ORDER BY prio ASC LIMIT 1) "
103 "UNION "
104 "(SELECT " RESULT_COLUMNS
105 " FROM datastore.gn090"
106 " ORDER BY prio ASC LIMIT 1)"
107 " ORDER BY expire ASC LIMIT 1"),
108 GNUNET_PQ_make_prepare ("select_replication_order",
109 "SELECT " RESULT_COLUMNS
110 " FROM datastore.gn090"
111 " ORDER BY repl DESC,RANDOM() LIMIT 1"),
112 GNUNET_PQ_make_prepare ("delrow",
113 "DELETE FROM datastore.gn090"
114 " WHERE oid=$1"),
115 GNUNET_PQ_make_prepare ("remove",
116 "DELETE FROM datastore.gn090"
117 " WHERE hash = $1 AND"
118 " value = $2"),
119 GNUNET_PQ_make_prepare ("get_keys",
120 "SELECT hash"
121 " FROM datastore.gn090"),
122 GNUNET_PQ_make_prepare ("estimate_size",
123 "SELECT CASE WHEN NOT EXISTS"
124 " (SELECT 1 FROM datastore.gn090)"
125 " THEN 0"
126 " ELSE (SELECT SUM(LENGTH(value))+256*COUNT(*)"
127 " FROM datastore.gn090)"
128 "END AS total"),
129 GNUNET_PQ_PREPARED_STATEMENT_END
130 };
131#undef RESULT_COLUMNS
132
133 plugin->dbh = GNUNET_PQ_connect_with_cfg (plugin->env->cfg,
134 "datastore-postgres",
135 "datastore-",
136 NULL,
137 ps);
138 if (NULL == plugin->dbh)
139 return GNUNET_SYSERR;
140 return GNUNET_OK;
141}
142
143
144/**
145 * Get an estimate of how much space the database is
146 * currently using.
147 *
148 * @param cls our `struct Plugin *`
149 * @return number of bytes used on disk
150 */
151static void
152postgres_plugin_estimate_size (void *cls,
153 unsigned long long *estimate)
154{
155 struct Plugin *plugin = cls;
156 uint64_t total;
157 struct GNUNET_PQ_QueryParam params[] = {
158 GNUNET_PQ_query_param_end
159 };
160 struct GNUNET_PQ_ResultSpec rs[] = {
161 GNUNET_PQ_result_spec_uint64 ("total",
162 &total),
163 GNUNET_PQ_result_spec_end
164 };
165 enum GNUNET_DB_QueryStatus ret;
166
167 if (NULL == estimate)
168 return;
169 ret = GNUNET_PQ_eval_prepared_singleton_select (plugin->dbh,
170 "estimate_size",
171 params,
172 rs);
173 if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT != ret)
174 {
175 *estimate = 0LL;
176 return;
177 }
178 *estimate = total;
179}
180
181
182/**
183 * Store an item in the datastore.
184 *
185 * @param cls closure with the `struct Plugin`
186 * @param key key for the item
187 * @param absent true if the key was not found in the bloom filter
188 * @param size number of bytes in data
189 * @param data content stored
190 * @param type type of the content
191 * @param priority priority of the content
192 * @param anonymity anonymity-level for the content
193 * @param replication replication-level for the content
194 * @param expiration expiration time for the content
195 * @param cont continuation called with success or failure status
196 * @param cont_cls continuation closure
197 */
198static void
199postgres_plugin_put (void *cls,
200 const struct GNUNET_HashCode *key,
201 bool absent,
202 uint32_t size,
203 const void *data,
204 enum GNUNET_BLOCK_Type type,
205 uint32_t priority,
206 uint32_t anonymity,
207 uint32_t replication,
208 struct GNUNET_TIME_Absolute expiration,
209 PluginPutCont cont,
210 void *cont_cls)
211{
212 struct Plugin *plugin = cls;
213 struct GNUNET_HashCode vhash;
214 enum GNUNET_DB_QueryStatus ret;
215
216 GNUNET_CRYPTO_hash (data,
217 size,
218 &vhash);
219 if (! absent)
220 {
221 struct GNUNET_PQ_QueryParam params[] = {
222 GNUNET_PQ_query_param_uint32 (&priority),
223 GNUNET_PQ_query_param_uint32 (&replication),
224 GNUNET_PQ_query_param_absolute_time (&expiration),
225 GNUNET_PQ_query_param_auto_from_type (key),
226 GNUNET_PQ_query_param_auto_from_type (&vhash),
227 GNUNET_PQ_query_param_end
228 };
229 ret = GNUNET_PQ_eval_prepared_non_select (plugin->dbh,
230 "update",
231 params);
232 if (0 > ret)
233 {
234 cont (cont_cls,
235 key,
236 size,
237 GNUNET_SYSERR,
238 _ ("Postgresql exec failure"));
239 return;
240 }
241 bool affected = (0 != ret);
242 if (affected)
243 {
244 cont (cont_cls,
245 key,
246 size,
247 GNUNET_NO,
248 NULL);
249 return;
250 }
251 }
252
253 {
254 uint32_t utype = (uint32_t) type;
255 uint64_t rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
256 UINT64_MAX);
257 struct GNUNET_PQ_QueryParam params[] = {
258 GNUNET_PQ_query_param_uint32 (&replication),
259 GNUNET_PQ_query_param_uint32 (&utype),
260 GNUNET_PQ_query_param_uint32 (&priority),
261 GNUNET_PQ_query_param_uint32 (&anonymity),
262 GNUNET_PQ_query_param_absolute_time (&expiration),
263 GNUNET_PQ_query_param_uint64 (&rvalue),
264 GNUNET_PQ_query_param_auto_from_type (key),
265 GNUNET_PQ_query_param_auto_from_type (&vhash),
266 GNUNET_PQ_query_param_fixed_size (data, size),
267 GNUNET_PQ_query_param_end
268 };
269
270 ret = GNUNET_PQ_eval_prepared_non_select (plugin->dbh,
271 "put",
272 params);
273 if (0 > ret)
274 {
275 cont (cont_cls,
276 key,
277 size,
278 GNUNET_SYSERR,
279 "Postgresql exec failure");
280 return;
281 }
282 }
283 plugin->env->duc (plugin->env->cls,
284 size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
285 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
286 "datastore-postgres",
287 "Stored %u bytes in database\n",
288 (unsigned int) size);
289 cont (cont_cls,
290 key,
291 size,
292 GNUNET_OK,
293 NULL);
294}
295
296
297/**
298 * Closure for #process_result.
299 */
300struct ProcessResultContext
301{
302 /**
303 * The plugin handle.
304 */
305 struct Plugin *plugin;
306
307 /**
308 * Function to call on each result.
309 */
310 PluginDatumProcessor proc;
311
312 /**
313 * Closure for @e proc.
314 */
315 void *proc_cls;
316};
317
318
319/**
320 * Function invoked to process the result and call the processor of @a
321 * cls.
322 *
323 * @param cls our `struct ProcessResultContext`
324 * @param res result from exec
325 * @param num_results number of results in @a res
326 */
327static void
328process_result (void *cls,
329 PGresult *res,
330 unsigned int num_results)
331{
332 struct ProcessResultContext *prc = cls;
333 struct Plugin *plugin = prc->plugin;
334
335 if (0 == num_results)
336 {
337 /* no result */
338 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
339 "datastore-postgres",
340 "Ending iteration (no more results)\n");
341 prc->proc (prc->proc_cls, NULL, 0, NULL, 0, 0, 0, 0,
342 GNUNET_TIME_UNIT_ZERO_ABS, 0);
343 return;
344 }
345 if (1 != num_results)
346 {
347 GNUNET_break (0);
348 prc->proc (prc->proc_cls, NULL, 0, NULL, 0, 0, 0, 0,
349 GNUNET_TIME_UNIT_ZERO_ABS, 0);
350 return;
351 }
352 /* Technically we don't need the loop here, but nicer in case
353 we ever relax the condition above. */
354 for (unsigned int i = 0; i < num_results; i++)
355 {
356 int iret;
357 uint64_t rowid;
358 uint32_t utype;
359 uint32_t anonymity;
360 uint32_t replication;
361 uint32_t priority;
362 size_t size;
363 void *data;
364 struct GNUNET_TIME_Absolute expiration_time;
365 struct GNUNET_HashCode key;
366 struct GNUNET_PQ_ResultSpec rs[] = {
367 GNUNET_PQ_result_spec_uint32 ("repl", &replication),
368 GNUNET_PQ_result_spec_uint32 ("type", &utype),
369 GNUNET_PQ_result_spec_uint32 ("prio", &priority),
370 GNUNET_PQ_result_spec_uint32 ("anonLevel", &anonymity),
371 GNUNET_PQ_result_spec_absolute_time ("expire", &expiration_time),
372 GNUNET_PQ_result_spec_auto_from_type ("hash", &key),
373 GNUNET_PQ_result_spec_variable_size ("value", &data, &size),
374 GNUNET_PQ_result_spec_uint64 ("oid", &rowid),
375 GNUNET_PQ_result_spec_end
376 };
377
378 if (GNUNET_OK !=
379 GNUNET_PQ_extract_result (res,
380 rs,
381 i))
382 {
383 GNUNET_break (0);
384 prc->proc (prc->proc_cls, NULL, 0, NULL, 0, 0, 0, 0,
385 GNUNET_TIME_UNIT_ZERO_ABS, 0);
386 return;
387 }
388
389 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
390 "datastore-postgres",
391 "Found result of size %u bytes and type %u in database\n",
392 (unsigned int) size,
393 (unsigned int) utype);
394 iret = prc->proc (prc->proc_cls,
395 &key,
396 size,
397 data,
398 (enum GNUNET_BLOCK_Type) utype,
399 priority,
400 anonymity,
401 replication,
402 expiration_time,
403 rowid);
404 if (iret == GNUNET_NO)
405 {
406 struct GNUNET_PQ_QueryParam param[] = {
407 GNUNET_PQ_query_param_uint64 (&rowid),
408 GNUNET_PQ_query_param_end
409 };
410
411 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
412 "Processor asked for item %u to be removed.\n",
413 (unsigned int) rowid);
414 if (0 <
415 GNUNET_PQ_eval_prepared_non_select (plugin->dbh,
416 "delrow",
417 param))
418 {
419 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
420 "datastore-postgres",
421 "Deleting %u bytes from database\n",
422 (unsigned int) size);
423 plugin->env->duc (plugin->env->cls,
424 -(size + GNUNET_DATASTORE_ENTRY_OVERHEAD));
425 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
426 "datastore-postgres",
427 "Deleted %u bytes from database\n",
428 (unsigned int) size);
429 }
430 }
431 GNUNET_PQ_cleanup_result (rs);
432 } /* for (i) */
433}
434
435
436/**
437 * Get one of the results for a particular key in the datastore.
438 *
439 * @param cls closure with the `struct Plugin`
440 * @param next_uid return the result with lowest uid >= next_uid
441 * @param random if true, return a random result instead of using next_uid
442 * @param key maybe NULL (to match all entries)
443 * @param type entries of which type are relevant?
444 * Use 0 for any type.
445 * @param proc function to call on the matching value;
446 * will be called with NULL if nothing matches
447 * @param proc_cls closure for @a proc
448 */
449static void
450postgres_plugin_get_key (void *cls,
451 uint64_t next_uid,
452 bool random,
453 const struct GNUNET_HashCode *key,
454 enum GNUNET_BLOCK_Type type,
455 PluginDatumProcessor proc,
456 void *proc_cls)
457{
458 struct Plugin *plugin = cls;
459 uint32_t utype = type;
460 uint16_t use_rvalue = random;
461 uint16_t use_key = NULL != key;
462 uint16_t use_type = GNUNET_BLOCK_TYPE_ANY != type;
463 uint64_t rvalue;
464 struct GNUNET_PQ_QueryParam params[] = {
465 GNUNET_PQ_query_param_uint64 (&next_uid),
466 GNUNET_PQ_query_param_uint64 (&rvalue),
467 GNUNET_PQ_query_param_uint16 (&use_rvalue),
468 GNUNET_PQ_query_param_auto_from_type (key),
469 GNUNET_PQ_query_param_uint16 (&use_key),
470 GNUNET_PQ_query_param_uint32 (&utype),
471 GNUNET_PQ_query_param_uint16 (&use_type),
472 GNUNET_PQ_query_param_end
473 };
474 struct ProcessResultContext prc;
475 enum GNUNET_DB_QueryStatus res;
476
477 if (random)
478 {
479 rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
480 UINT64_MAX);
481 next_uid = 0;
482 }
483 else
484 {
485 rvalue = 0;
486 }
487 prc.plugin = plugin;
488 prc.proc = proc;
489 prc.proc_cls = proc_cls;
490
491 res = GNUNET_PQ_eval_prepared_multi_select (plugin->dbh,
492 "get",
493 params,
494 &process_result,
495 &prc);
496 if (0 > res)
497 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0,
498 GNUNET_TIME_UNIT_ZERO_ABS, 0);
499}
500
501
502/**
503 * Select a subset of the items in the datastore and call
504 * the given iterator for each of them.
505 *
506 * @param cls our `struct Plugin *`
507 * @param next_uid return the result with lowest uid >= next_uid
508 * @param type entries of which type should be considered?
509 * Must not be zero (ANY).
510 * @param proc function to call on the matching value;
511 * will be called with NULL if no value matches
512 * @param proc_cls closure for @a proc
513 */
514static void
515postgres_plugin_get_zero_anonymity (void *cls,
516 uint64_t next_uid,
517 enum GNUNET_BLOCK_Type type,
518 PluginDatumProcessor proc,
519 void *proc_cls)
520{
521 struct Plugin *plugin = cls;
522 uint32_t utype = type;
523 struct GNUNET_PQ_QueryParam params[] = {
524 GNUNET_PQ_query_param_uint32 (&utype),
525 GNUNET_PQ_query_param_uint64 (&next_uid),
526 GNUNET_PQ_query_param_end
527 };
528 struct ProcessResultContext prc;
529 enum GNUNET_DB_QueryStatus res;
530
531 prc.plugin = plugin;
532 prc.proc = proc;
533 prc.proc_cls = proc_cls;
534 res = GNUNET_PQ_eval_prepared_multi_select (plugin->dbh,
535 "select_non_anonymous",
536 params,
537 &process_result,
538 &prc);
539 if (0 > res)
540 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0,
541 GNUNET_TIME_UNIT_ZERO_ABS, 0);
542}
543
544
545/**
546 * Context for #repl_iter() function.
547 */
548struct ReplCtx
549{
550 /**
551 * Plugin handle.
552 */
553 struct Plugin *plugin;
554
555 /**
556 * Function to call for the result (or the NULL).
557 */
558 PluginDatumProcessor proc;
559
560 /**
561 * Closure for @e proc.
562 */
563 void *proc_cls;
564};
565
566
567/**
568 * Wrapper for the iterator for 'sqlite_plugin_replication_get'.
569 * Decrements the replication counter and calls the original
570 * iterator.
571 *
572 * @param cls closure with the `struct ReplCtx *`
573 * @param key key for the content
574 * @param size number of bytes in @a data
575 * @param data content stored
576 * @param type type of the content
577 * @param priority priority of the content
578 * @param anonymity anonymity-level for the content
579 * @param replication replication-level for the content
580 * @param expiration expiration time for the content
581 * @param uid unique identifier for the datum;
582 * maybe 0 if no unique identifier is available
583 * @return #GNUNET_SYSERR to abort the iteration,
584 * #GNUNET_OK to continue
585 * (continue on call to "next", of course),
586 * #GNUNET_NO to delete the item and continue (if supported)
587 */
588static int
589repl_proc (void *cls,
590 const struct GNUNET_HashCode *key,
591 uint32_t size,
592 const void *data,
593 enum GNUNET_BLOCK_Type type,
594 uint32_t priority,
595 uint32_t anonymity,
596 uint32_t replication,
597 struct GNUNET_TIME_Absolute expiration,
598 uint64_t uid)
599{
600 struct ReplCtx *rc = cls;
601 struct Plugin *plugin = rc->plugin;
602 int ret;
603 struct GNUNET_PQ_QueryParam params[] = {
604 GNUNET_PQ_query_param_uint64 (&uid),
605 GNUNET_PQ_query_param_end
606 };
607 enum GNUNET_DB_QueryStatus qret;
608
609 ret = rc->proc (rc->proc_cls,
610 key,
611 size,
612 data,
613 type,
614 priority,
615 anonymity,
616 replication,
617 expiration,
618 uid);
619 if (NULL == key)
620 return ret;
621 qret = GNUNET_PQ_eval_prepared_non_select (plugin->dbh,
622 "decrepl",
623 params);
624 if (0 > qret)
625 return GNUNET_SYSERR;
626 return ret;
627}
628
629
630/**
631 * Get a random item for replication. Returns a single, not expired,
632 * random item from those with the highest replication counters. The
633 * item's replication counter is decremented by one IF it was positive
634 * before. Call @a proc with all values ZERO or NULL if the datastore
635 * is empty.
636 *
637 * @param cls closure with the `struct Plugin`
638 * @param proc function to call the value (once only).
639 * @param proc_cls closure for @a proc
640 */
641static void
642postgres_plugin_get_replication (void *cls,
643 PluginDatumProcessor proc,
644 void *proc_cls)
645{
646 struct Plugin *plugin = cls;
647 struct GNUNET_PQ_QueryParam params[] = {
648 GNUNET_PQ_query_param_end
649 };
650 struct ReplCtx rc;
651 struct ProcessResultContext prc;
652 enum GNUNET_DB_QueryStatus res;
653
654 rc.plugin = plugin;
655 rc.proc = proc;
656 rc.proc_cls = proc_cls;
657 prc.plugin = plugin;
658 prc.proc = &repl_proc;
659 prc.proc_cls = &rc;
660 res = GNUNET_PQ_eval_prepared_multi_select (plugin->dbh,
661 "select_replication_order",
662 params,
663 &process_result,
664 &prc);
665 if (0 > res)
666 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0,
667 GNUNET_TIME_UNIT_ZERO_ABS, 0);
668}
669
670
671/**
672 * Get a random item for expiration. Call @a proc with all values
673 * ZERO or NULL if the datastore is empty.
674 *
675 * @param cls closure with the `struct Plugin`
676 * @param proc function to call the value (once only).
677 * @param proc_cls closure for @a proc
678 */
679static void
680postgres_plugin_get_expiration (void *cls,
681 PluginDatumProcessor proc,
682 void *proc_cls)
683{
684 struct Plugin *plugin = cls;
685 struct GNUNET_TIME_Absolute now = { 0 };
686 struct GNUNET_PQ_QueryParam params[] = {
687 GNUNET_PQ_query_param_absolute_time (&now),
688 GNUNET_PQ_query_param_end
689 };
690 struct ProcessResultContext prc;
691
692 now = GNUNET_TIME_absolute_get ();
693 prc.plugin = plugin;
694 prc.proc = proc;
695 prc.proc_cls = proc_cls;
696 (void) GNUNET_PQ_eval_prepared_multi_select (plugin->dbh,
697 "select_expiration_order",
698 params,
699 &process_result,
700 &prc);
701}
702
703
704/**
705 * Closure for #process_keys.
706 */
707struct ProcessKeysContext
708{
709 /**
710 * Function to call for each key.
711 */
712 PluginKeyProcessor proc;
713
714 /**
715 * Closure for @e proc.
716 */
717 void *proc_cls;
718};
719
720
721/**
722 * Function to be called with the results of a SELECT statement
723 * that has returned @a num_results results.
724 *
725 * @param cls closure with a `struct ProcessKeysContext`
726 * @param result the postgres result
727 * @param num_results the number of results in @a result
728 */
729static void
730process_keys (void *cls,
731 PGresult *result,
732 unsigned int num_results)
733{
734 struct ProcessKeysContext *pkc = cls;
735
736 for (unsigned i = 0; i < num_results; i++)
737 {
738 struct GNUNET_HashCode key;
739 struct GNUNET_PQ_ResultSpec rs[] = {
740 GNUNET_PQ_result_spec_auto_from_type ("hash",
741 &key),
742 GNUNET_PQ_result_spec_end
743 };
744
745 if (GNUNET_OK !=
746 GNUNET_PQ_extract_result (result,
747 rs,
748 i))
749 {
750 GNUNET_break (0);
751 continue;
752 }
753 pkc->proc (pkc->proc_cls,
754 &key,
755 1);
756 GNUNET_PQ_cleanup_result (rs);
757 }
758}
759
760
761/**
762 * Get all of the keys in the datastore.
763 *
764 * @param cls closure with the `struct Plugin *`
765 * @param proc function to call on each key
766 * @param proc_cls closure for @a proc
767 */
768static void
769postgres_plugin_get_keys (void *cls,
770 PluginKeyProcessor proc,
771 void *proc_cls)
772{
773 struct Plugin *plugin = cls;
774 struct GNUNET_PQ_QueryParam params[] = {
775 GNUNET_PQ_query_param_end
776 };
777 struct ProcessKeysContext pkc;
778
779 pkc.proc = proc;
780 pkc.proc_cls = proc_cls;
781 (void) GNUNET_PQ_eval_prepared_multi_select (plugin->dbh,
782 "get_keys",
783 params,
784 &process_keys,
785 &pkc);
786 proc (proc_cls,
787 NULL,
788 0);
789}
790
791
792/**
793 * Drop database.
794 *
795 * @param cls closure with the `struct Plugin *`
796 */
797static void
798postgres_plugin_drop (void *cls)
799{
800 struct Plugin *plugin = cls;
801 struct GNUNET_PQ_ExecuteStatement es[] = {
802 GNUNET_PQ_make_execute ("DROP TABLE gn090"),
803 GNUNET_PQ_EXECUTE_STATEMENT_END
804 };
805
806 if (GNUNET_OK !=
807 GNUNET_PQ_exec_statements (plugin->dbh,
808 es))
809 GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING,
810 "postgres",
811 _ ("Failed to drop table from database.\n"));
812}
813
814
815/**
816 * Remove a particular key in the datastore.
817 *
818 * @param cls closure
819 * @param key key for the content
820 * @param size number of bytes in data
821 * @param data content stored
822 * @param cont continuation called with success or failure status
823 * @param cont_cls continuation closure for @a cont
824 */
825static void
826postgres_plugin_remove_key (void *cls,
827 const struct GNUNET_HashCode *key,
828 uint32_t size,
829 const void *data,
830 PluginRemoveCont cont,
831 void *cont_cls)
832{
833 struct Plugin *plugin = cls;
834 enum GNUNET_DB_QueryStatus ret;
835 struct GNUNET_PQ_QueryParam params[] = {
836 GNUNET_PQ_query_param_auto_from_type (key),
837 GNUNET_PQ_query_param_fixed_size (data, size),
838 GNUNET_PQ_query_param_end
839 };
840
841 ret = GNUNET_PQ_eval_prepared_non_select (plugin->dbh,
842 "remove",
843 params);
844 if (0 > ret)
845 {
846 cont (cont_cls,
847 key,
848 size,
849 GNUNET_SYSERR,
850 _ ("Postgresql exec failure"));
851 return;
852 }
853 if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == ret)
854 {
855 cont (cont_cls,
856 key,
857 size,
858 GNUNET_NO,
859 NULL);
860 return;
861 }
862 plugin->env->duc (plugin->env->cls,
863 -(size + GNUNET_DATASTORE_ENTRY_OVERHEAD));
864 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
865 "datastore-postgres",
866 "Deleted %u bytes from database\n",
867 (unsigned int) size);
868 cont (cont_cls,
869 key,
870 size,
871 GNUNET_OK,
872 NULL);
873}
874
875
876/**
877 * Entry point for the plugin.
878 *
879 * @param cls the `struct GNUNET_DATASTORE_PluginEnvironment*`
880 * @return our `struct Plugin *`
881 */
882void *
883libgnunet_plugin_datastore_postgres_init (void *cls)
884{
885 struct GNUNET_DATASTORE_PluginEnvironment *env = cls;
886 struct GNUNET_DATASTORE_PluginFunctions *api;
887 struct Plugin *plugin;
888
889 plugin = GNUNET_new (struct Plugin);
890 plugin->env = env;
891 if (GNUNET_OK != init_connection (plugin))
892 {
893 GNUNET_free (plugin);
894 return NULL;
895 }
896 api = GNUNET_new (struct GNUNET_DATASTORE_PluginFunctions);
897 api->cls = plugin;
898 api->estimate_size = &postgres_plugin_estimate_size;
899 api->put = &postgres_plugin_put;
900 api->get_key = &postgres_plugin_get_key;
901 api->get_replication = &postgres_plugin_get_replication;
902 api->get_expiration = &postgres_plugin_get_expiration;
903 api->get_zero_anonymity = &postgres_plugin_get_zero_anonymity;
904 api->get_keys = &postgres_plugin_get_keys;
905 api->drop = &postgres_plugin_drop;
906 api->remove_key = &postgres_plugin_remove_key;
907 return api;
908}
909
910
911/**
912 * Exit point from the plugin.
913 *
914 * @param cls our `struct Plugin *`
915 * @return always NULL
916 */
917void *
918libgnunet_plugin_datastore_postgres_done (void *cls)
919{
920 struct GNUNET_DATASTORE_PluginFunctions *api = cls;
921 struct Plugin *plugin = api->cls;
922
923 GNUNET_PQ_disconnect (plugin->dbh);
924 GNUNET_free (plugin);
925 GNUNET_free (api);
926 return NULL;
927}
928
929
930/* end of plugin_datastore_postgres.c */
diff --git a/src/datastore/plugin_datastore_sqlite.c b/src/datastore/plugin_datastore_sqlite.c
deleted file mode 100644
index 5ea9da4cb..000000000
--- a/src/datastore/plugin_datastore_sqlite.c
+++ /dev/null
@@ -1,1375 +0,0 @@
1/*
2 * This file is part of GNUnet
3 * Copyright (C) 2009, 2011, 2017 GNUnet e.V.
4 *
5 * GNUnet is free software: you can redistribute it and/or modify it
6 * under the terms of the GNU Affero General Public License as published
7 * by the Free Software Foundation, either version 3 of the License,
8 * or (at your option) any later version.
9 *
10 * GNUnet is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Affero General Public License for more details.
14 *
15 * You should have received a copy of the GNU Affero General Public License
16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file datastore/plugin_datastore_sqlite.c
23 * @brief sqlite-based datastore backend
24 * @author Christian Grothoff
25 */
26
27#include "platform.h"
28#include "gnunet_datastore_plugin.h"
29#include "gnunet_sq_lib.h"
30#include <sqlite3.h>
31
32
33/**
34 * We allocate items on the stack at times. To prevent a stack
35 * overflow, we impose a limit on the maximum size for the data per
36 * item. 64k should be enough.
37 */
38#define MAX_ITEM_SIZE 65536
39
40/**
41 * After how many ms "busy" should a DB operation fail for good?
42 * A low value makes sure that we are more responsive to requests
43 * (especially PUTs). A high value guarantees a higher success
44 * rate (SELECTs in iterate can take several seconds despite LIMIT=1).
45 *
46 * The default value of 250ms should ensure that users do not experience
47 * huge latencies while at the same time allowing operations to succeed
48 * with reasonable probability.
49 */
50#define BUSY_TIMEOUT_MS 250
51
52
53/**
54 * Log an error message at log-level 'level' that indicates
55 * a failure of the command 'cmd' on file 'filename'
56 * with the message given by strerror(errno).
57 */
58#define LOG_SQLITE(db, level, cmd) \
59 do \
60 { \
61 GNUNET_log_from (level, \
62 "sqlite", \
63 _ ("`%s' failed at %s:%d with error: %s\n"), \
64 cmd, \
65 __FILE__, \
66 __LINE__, \
67 sqlite3_errmsg (db->dbh)); \
68 } while (0)
69
70
71/**
72 * Log an error message at log-level 'level' that indicates
73 * a failure of the command 'cmd' on file 'filename'
74 * with the message given by strerror(errno).
75 */
76#define LOG_SQLITE_MSG(db, msg, level, cmd) \
77 do \
78 { \
79 GNUNET_log_from (level, \
80 "sqlite", \
81 _ ("`%s' failed at %s:%d with error: %s\n"), \
82 cmd, \
83 __FILE__, \
84 __LINE__, \
85 sqlite3_errmsg (db->dbh)); \
86 GNUNET_asprintf (msg, \
87 _ ("`%s' failed at %s:%u with error: %s"), \
88 cmd, \
89 __FILE__, \
90 __LINE__, \
91 sqlite3_errmsg (db->dbh)); \
92 } while (0)
93
94
95/**
96 * Context for all functions in this plugin.
97 */
98struct Plugin
99{
100 /**
101 * Our execution environment.
102 */
103 struct GNUNET_DATASTORE_PluginEnvironment *env;
104
105 /**
106 * Database filename.
107 */
108 char *fn;
109
110 /**
111 * Native SQLite database handle.
112 */
113 sqlite3 *dbh;
114
115 /**
116 * Precompiled SQL for remove_key.
117 */
118 sqlite3_stmt *remove;
119
120 /**
121 * Precompiled SQL for deletion.
122 */
123 sqlite3_stmt *delRow;
124
125 /**
126 * Precompiled SQL for update.
127 */
128 sqlite3_stmt *update;
129
130 /**
131 * Get maximum repl value in database.
132 */
133 sqlite3_stmt *maxRepl;
134
135 /**
136 * Precompiled SQL for replication decrement.
137 */
138 sqlite3_stmt *updRepl;
139
140 /**
141 * Precompiled SQL for replication selection.
142 */
143 sqlite3_stmt *selRepl;
144
145 /**
146 * Precompiled SQL for expiration selection.
147 */
148 sqlite3_stmt *selExpi;
149
150 /**
151 * Precompiled SQL for expiration selection.
152 */
153 sqlite3_stmt *selZeroAnon;
154
155 /**
156 * Precompiled SQL for insertion.
157 */
158 sqlite3_stmt *insertContent;
159
160 /**
161 * Precompiled SQL for selection
162 */
163 sqlite3_stmt *get[8];
164
165 /**
166 * Should the database be dropped on shutdown?
167 */
168 int drop_on_shutdown;
169};
170
171
172/**
173 * @brief Prepare a SQL statement
174 *
175 * @param dbh handle to the database
176 * @param zSql SQL statement, UTF-8 encoded
177 * @param ppStmt set to the prepared statement
178 * @return 0 on success
179 */
180static int
181sq_prepare (sqlite3 *dbh, const char *zSql, sqlite3_stmt **ppStmt)
182{
183 char *dummy;
184 int result;
185
186 result = sqlite3_prepare_v2 (dbh,
187 zSql,
188 strlen (zSql),
189 ppStmt,
190 (const char **) &dummy);
191 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
192 "sqlite",
193 "Prepared `%s' / %p: %d\n",
194 zSql,
195 *ppStmt,
196 result);
197 return result;
198}
199
200
201/**
202 * Create our database indices.
203 *
204 * @param dbh handle to the database
205 */
206static void
207create_indices (sqlite3 *dbh)
208{
209 /* create indices */
210 if (
211 0 !=
212 (SQLITE_OK !=
213 sqlite3_exec (dbh,
214 "CREATE INDEX IF NOT EXISTS idx_hash ON gn091 (hash)",
215 NULL,
216 NULL,
217 NULL))
218 + (SQLITE_OK !=
219 sqlite3_exec (
220 dbh,
221 "CREATE INDEX IF NOT EXISTS idx_anon_type ON gn091 (anonLevel ASC,type)",
222 NULL,
223 NULL,
224 NULL))
225 + (SQLITE_OK !=
226 sqlite3_exec (dbh,
227 "CREATE INDEX IF NOT EXISTS idx_expire ON gn091 (expire ASC)",
228 NULL,
229 NULL,
230 NULL))
231 + (SQLITE_OK !=
232 sqlite3_exec (
233 dbh,
234 "CREATE INDEX IF NOT EXISTS idx_repl_rvalue ON gn091 (repl,rvalue)",
235 NULL,
236 NULL,
237 NULL)))
238 GNUNET_log_from (GNUNET_ERROR_TYPE_ERROR,
239 "sqlite",
240 "Failed to create indices: %s\n",
241 sqlite3_errmsg (dbh));
242}
243
244
245#if 0
246#define CHECK(a) GNUNET_break (a)
247#define ENULL NULL
248#else
249#define ENULL &e
250#define ENULL_DEFINED 1
251#define CHECK(a) \
252 if (! (a)) \
253 { \
254 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "%s\n", e); \
255 sqlite3_free (e); \
256 e = NULL; \
257 }
258#endif
259
260
261/**
262 * Initialize the database connections and associated
263 * data structures (create tables and indices
264 * as needed as well).
265 *
266 * @param cfg our configuration
267 * @param plugin the plugin context (state for this module)
268 * @return #GNUNET_OK on success
269 */
270static int
271database_setup (const struct GNUNET_CONFIGURATION_Handle *cfg,
272 struct Plugin *plugin)
273{
274 sqlite3_stmt *stmt;
275 char *afsdir;
276
277#if ENULL_DEFINED
278 char *e;
279#endif
280
281 if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_filename (cfg,
282 "datastore-sqlite",
283 "FILENAME",
284 &afsdir))
285 {
286 GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
287 "datastore-sqlite",
288 "FILENAME");
289 return GNUNET_SYSERR;
290 }
291 if (GNUNET_OK != GNUNET_DISK_file_test (afsdir))
292 {
293 if (GNUNET_OK != GNUNET_DISK_directory_create_for_file (afsdir))
294 {
295 GNUNET_break (0);
296 GNUNET_free (afsdir);
297 return GNUNET_SYSERR;
298 }
299 /* database is new or got deleted, reset payload to zero! */
300 if (NULL != plugin->env->duc)
301 plugin->env->duc (plugin->env->cls, 0);
302 }
303 /* afsdir should be UTF-8-encoded. If it isn't, it's a bug */
304 plugin->fn = afsdir;
305
306 /* Open database and precompile statements */
307 if (SQLITE_OK != sqlite3_open (plugin->fn, &plugin->dbh))
308 {
309 GNUNET_log_from (GNUNET_ERROR_TYPE_ERROR,
310 "sqlite",
311 _ ("Unable to initialize SQLite: %s.\n"),
312 sqlite3_errmsg (plugin->dbh));
313 return GNUNET_SYSERR;
314 }
315 CHECK (
316 SQLITE_OK ==
317 sqlite3_exec (plugin->dbh, "PRAGMA temp_store=MEMORY", NULL, NULL, ENULL));
318 CHECK (
319 SQLITE_OK ==
320 sqlite3_exec (plugin->dbh, "PRAGMA synchronous=OFF", NULL, NULL, ENULL));
321 CHECK (SQLITE_OK == sqlite3_exec (plugin->dbh,
322 "PRAGMA legacy_file_format=OFF",
323 NULL,
324 NULL,
325 ENULL));
326 CHECK (SQLITE_OK == sqlite3_exec (plugin->dbh,
327 "PRAGMA auto_vacuum=INCREMENTAL",
328 NULL,
329 NULL,
330 ENULL));
331 CHECK (SQLITE_OK == sqlite3_exec (plugin->dbh,
332 "PRAGMA locking_mode=EXCLUSIVE",
333 NULL,
334 NULL,
335 ENULL));
336 CHECK (
337 SQLITE_OK ==
338 sqlite3_exec (plugin->dbh, "PRAGMA page_size=4096", NULL, NULL, ENULL));
339
340 CHECK (SQLITE_OK == sqlite3_busy_timeout (plugin->dbh, BUSY_TIMEOUT_MS));
341
342
343 /* We have to do it here, because otherwise precompiling SQL might fail */
344 CHECK (SQLITE_OK ==
345 sq_prepare (plugin->dbh,
346 "SELECT 1 FROM sqlite_master WHERE tbl_name = 'gn091'",
347 &stmt));
348
349 /* FIXME: SQLite does not have unsigned integers! This is ok for the type column because
350 * we only test equality on it and can cast it to/from uint32_t. For repl, prio, and anonLevel
351 * we do math or inequality tests, so we can't handle the entire range of uint32_t.
352 * This will also cause problems for expiration times after 294247-01-10-04:00:54 UTC.
353 */if ((SQLITE_DONE == sqlite3_step (stmt)) &&
354 (SQLITE_OK != sqlite3_exec (plugin->dbh,
355 "CREATE TABLE gn091 ("
356 " repl INT4 NOT NULL DEFAULT 0,"
357 " type INT4 NOT NULL DEFAULT 0,"
358 " prio INT4 NOT NULL DEFAULT 0,"
359 " anonLevel INT4 NOT NULL DEFAULT 0,"
360 " expire INT8 NOT NULL DEFAULT 0,"
361 " rvalue INT8 NOT NULL,"
362 " hash TEXT NOT NULL DEFAULT '',"
363 " vhash TEXT NOT NULL DEFAULT '',"
364 " value BLOB NOT NULL DEFAULT '')",
365 NULL,
366 NULL,
367 NULL)))
368 {
369 LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "sqlite3_exec");
370 sqlite3_finalize (stmt);
371 return GNUNET_SYSERR;
372 }
373 sqlite3_finalize (stmt);
374 create_indices (plugin->dbh);
375
376#define RESULT_COLUMNS \
377 "repl, type, prio, anonLevel, expire, hash, value, _ROWID_"
378 if (
379 (SQLITE_OK != sq_prepare (plugin->dbh,
380 "UPDATE gn091 "
381 "SET prio = prio + ?, "
382 "repl = repl + ?, "
383 "expire = MAX(expire, ?) "
384 "WHERE hash = ? AND vhash = ?",
385 &plugin->update)) ||
386 (SQLITE_OK != sq_prepare (plugin->dbh,
387 "UPDATE gn091 "
388 "SET repl = MAX (0, repl - 1) WHERE _ROWID_ = ?",
389 &plugin->updRepl)) ||
390 (SQLITE_OK != sq_prepare (plugin->dbh,
391 "SELECT " RESULT_COLUMNS " FROM gn091 "
392 "WHERE repl=?2 AND "
393 " (rvalue>=?1 OR "
394 " NOT EXISTS (SELECT 1 FROM gn091 "
395 "WHERE repl=?2 AND rvalue>=?1 LIMIT 1) ) "
396 "ORDER BY rvalue ASC LIMIT 1",
397 &plugin->selRepl)) ||
398 (SQLITE_OK != sq_prepare (plugin->dbh,
399 "SELECT MAX(repl) FROM gn091",
400 &plugin->maxRepl)) ||
401 (SQLITE_OK !=
402 sq_prepare (plugin->dbh,
403 "SELECT " RESULT_COLUMNS " FROM gn091 "
404 "WHERE NOT EXISTS (SELECT 1 FROM gn091 WHERE expire < ?1 LIMIT 1) OR (expire < ?1) "
405 "ORDER BY expire ASC LIMIT 1",
406 &plugin->selExpi)) ||
407 (SQLITE_OK != sq_prepare (plugin->dbh,
408 "SELECT " RESULT_COLUMNS " FROM gn091 "
409 "WHERE _ROWID_ >= ? AND "
410 "anonLevel = 0 AND "
411 "type = ? "
412 "ORDER BY _ROWID_ ASC LIMIT 1",
413 &plugin->selZeroAnon)) ||
414 (SQLITE_OK !=
415 sq_prepare (plugin->dbh,
416 "INSERT INTO gn091 (repl, type, prio, anonLevel, expire, rvalue, hash, vhash, value) "
417 "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
418 &plugin->insertContent)) ||
419 (SQLITE_OK != sq_prepare (plugin->dbh,
420 "SELECT " RESULT_COLUMNS " FROM gn091 "
421 "WHERE _ROWID_ >= ?1 "
422 "ORDER BY _ROWID_ ASC LIMIT 1",
423 &plugin->get[0])) ||
424 (SQLITE_OK != sq_prepare (plugin->dbh,
425 "SELECT " RESULT_COLUMNS " FROM gn091 "
426 "WHERE _ROWID_ >= ?1 AND "
427 "type = ?4 "
428 "ORDER BY _ROWID_ ASC LIMIT 1",
429 &plugin->get[1])) ||
430 (SQLITE_OK != sq_prepare (plugin->dbh,
431 "SELECT " RESULT_COLUMNS " FROM gn091 "
432 "WHERE _ROWID_ >= ?1 AND "
433 "hash = ?3 "
434 "ORDER BY _ROWID_ ASC LIMIT 1",
435 &plugin->get[2])) ||
436 (SQLITE_OK != sq_prepare (plugin->dbh,
437 "SELECT " RESULT_COLUMNS " FROM gn091 "
438 "WHERE _ROWID_ >= ?1 AND "
439 "hash = ?3 AND "
440 "type = ?4 "
441 "ORDER BY _ROWID_ ASC LIMIT 1",
442 &plugin->get[3])) ||
443 (SQLITE_OK != sq_prepare (plugin->dbh,
444 "SELECT " RESULT_COLUMNS " FROM gn091 "
445 "WHERE _ROWID_ >= ?1 AND "
446 "rvalue >= ?2 "
447 "ORDER BY _ROWID_ ASC LIMIT 1",
448 &plugin->get[4])) ||
449 (SQLITE_OK != sq_prepare (plugin->dbh,
450 "SELECT " RESULT_COLUMNS " FROM gn091 "
451 "WHERE _ROWID_ >= ?1 AND "
452 "rvalue >= ?2 AND "
453 "type = ?4 "
454 "ORDER BY _ROWID_ ASC LIMIT 1",
455 &plugin->get[5])) ||
456 (SQLITE_OK != sq_prepare (plugin->dbh,
457 "SELECT " RESULT_COLUMNS " FROM gn091 "
458 "WHERE _ROWID_ >= ?1 AND "
459 "rvalue >= ?2 AND "
460 "hash = ?3 "
461 "ORDER BY _ROWID_ ASC LIMIT 1",
462 &plugin->get[6])) ||
463 (SQLITE_OK != sq_prepare (plugin->dbh,
464 "SELECT " RESULT_COLUMNS " FROM gn091 "
465 "WHERE _ROWID_ >= ?1 AND "
466 "rvalue >= ?2 AND "
467 "hash = ?3 AND "
468 "type = ?4 "
469 "ORDER BY _ROWID_ ASC LIMIT 1",
470 &plugin->get[7])) ||
471 (SQLITE_OK != sq_prepare (plugin->dbh,
472 "DELETE FROM gn091 WHERE _ROWID_ = ?",
473 &plugin->delRow)) ||
474 (SQLITE_OK != sq_prepare (plugin->dbh,
475 "DELETE FROM gn091 "
476 "WHERE hash = ? AND "
477 "value = ? ",
478 &plugin->remove)) ||
479 false)
480 {
481 LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "precompiling");
482 return GNUNET_SYSERR;
483 }
484 return GNUNET_OK;
485}
486
487
488/**
489 * Shutdown database connection and associate data
490 * structures.
491 *
492 * @param plugin the plugin context (state for this module)
493 */
494static void
495database_shutdown (struct Plugin *plugin)
496{
497 int result;
498
499#if SQLITE_VERSION_NUMBER >= 3007000
500 sqlite3_stmt *stmt;
501#endif
502
503 if (NULL != plugin->remove)
504 sqlite3_finalize (plugin->remove);
505 if (NULL != plugin->delRow)
506 sqlite3_finalize (plugin->delRow);
507 if (NULL != plugin->update)
508 sqlite3_finalize (plugin->update);
509 if (NULL != plugin->updRepl)
510 sqlite3_finalize (plugin->updRepl);
511 if (NULL != plugin->selRepl)
512 sqlite3_finalize (plugin->selRepl);
513 if (NULL != plugin->maxRepl)
514 sqlite3_finalize (plugin->maxRepl);
515 if (NULL != plugin->selExpi)
516 sqlite3_finalize (plugin->selExpi);
517 if (NULL != plugin->selZeroAnon)
518 sqlite3_finalize (plugin->selZeroAnon);
519 if (NULL != plugin->insertContent)
520 sqlite3_finalize (plugin->insertContent);
521 for (int i = 0; i < 8; ++i)
522 if (NULL != plugin->get[i])
523 sqlite3_finalize (plugin->get[i]);
524 result = sqlite3_close (plugin->dbh);
525#if SQLITE_VERSION_NUMBER >= 3007000
526 if (result == SQLITE_BUSY)
527 {
528 GNUNET_log_from (
529 GNUNET_ERROR_TYPE_WARNING,
530 "sqlite",
531 _ (
532 "Tried to close sqlite without finalizing all prepared statements.\n"));
533 stmt = sqlite3_next_stmt (plugin->dbh, NULL);
534 while (NULL != stmt)
535 {
536 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
537 "sqlite",
538 "Closing statement %p\n",
539 stmt);
540 result = sqlite3_finalize (stmt);
541 if (result != SQLITE_OK)
542 GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING,
543 "sqlite",
544 "Failed to close statement %p: %d\n",
545 stmt,
546 result);
547 stmt = sqlite3_next_stmt (plugin->dbh, NULL);
548 }
549 result = sqlite3_close (plugin->dbh);
550 }
551#endif
552 if (SQLITE_OK != result)
553 LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "sqlite3_close");
554 GNUNET_free (plugin->fn);
555}
556
557
558/**
559 * Delete the database entry with the given
560 * row identifier.
561 *
562 * @param plugin the plugin context (state for this module)
563 * @param rid the ID of the row to delete
564 */
565static int
566delete_by_rowid (struct Plugin *plugin, uint64_t rid)
567{
568 struct GNUNET_SQ_QueryParam params[] = { GNUNET_SQ_query_param_uint64 (&rid),
569 GNUNET_SQ_query_param_end };
570
571 if (GNUNET_OK != GNUNET_SQ_bind (plugin->delRow, params))
572 return GNUNET_SYSERR;
573 if (SQLITE_DONE != sqlite3_step (plugin->delRow))
574 {
575 LOG_SQLITE (plugin,
576 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
577 "sqlite3_step");
578 GNUNET_SQ_reset (plugin->dbh, plugin->delRow);
579 return GNUNET_SYSERR;
580 }
581 GNUNET_SQ_reset (plugin->dbh, plugin->delRow);
582 return GNUNET_OK;
583}
584
585
586/**
587 * Store an item in the datastore.
588 *
589 * @param cls closure
590 * @param key key for the item
591 * @param absent true if the key was not found in the bloom filter
592 * @param size number of bytes in @a data
593 * @param data content stored
594 * @param type type of the content
595 * @param priority priority of the content
596 * @param anonymity anonymity-level for the content
597 * @param replication replication-level for the content
598 * @param expiration expiration time for the content
599 * @param cont continuation called with success or failure status
600 * @param cont_cls continuation closure
601 */
602static void
603sqlite_plugin_put (void *cls,
604 const struct GNUNET_HashCode *key,
605 bool absent,
606 uint32_t size,
607 const void *data,
608 enum GNUNET_BLOCK_Type type,
609 uint32_t priority,
610 uint32_t anonymity,
611 uint32_t replication,
612 struct GNUNET_TIME_Absolute expiration,
613 PluginPutCont cont,
614 void *cont_cls)
615{
616 struct Plugin *plugin = cls;
617 struct GNUNET_HashCode vhash;
618 char *msg = NULL;
619
620 GNUNET_CRYPTO_hash (data, size, &vhash);
621
622 if (! absent)
623 {
624 struct GNUNET_SQ_QueryParam params[] =
625 { GNUNET_SQ_query_param_uint32 (&priority),
626 GNUNET_SQ_query_param_uint32 (&replication),
627 GNUNET_SQ_query_param_absolute_time (&expiration),
628 GNUNET_SQ_query_param_auto_from_type (key),
629 GNUNET_SQ_query_param_auto_from_type (&vhash),
630 GNUNET_SQ_query_param_end };
631
632 if (GNUNET_OK != GNUNET_SQ_bind (plugin->update, params))
633 {
634 cont (cont_cls, key, size, GNUNET_SYSERR, _ ("sqlite bind failure"));
635 return;
636 }
637 if (SQLITE_DONE != sqlite3_step (plugin->update))
638 {
639 LOG_SQLITE_MSG (plugin,
640 &msg,
641 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
642 "sqlite3_step");
643 cont (cont_cls, key, size, GNUNET_SYSERR, msg);
644 GNUNET_free (msg);
645 return;
646 }
647 int changes = sqlite3_changes (plugin->dbh);
648 GNUNET_SQ_reset (plugin->dbh, plugin->update);
649 if (0 != changes)
650 {
651 cont (cont_cls, key, size, GNUNET_NO, NULL);
652 return;
653 }
654 }
655
656 uint64_t rvalue;
657 uint32_t type32 = (uint32_t) type;
658 struct GNUNET_SQ_QueryParam params[] =
659 { GNUNET_SQ_query_param_uint32 (&replication),
660 GNUNET_SQ_query_param_uint32 (&type32),
661 GNUNET_SQ_query_param_uint32 (&priority),
662 GNUNET_SQ_query_param_uint32 (&anonymity),
663 GNUNET_SQ_query_param_absolute_time (&expiration),
664 GNUNET_SQ_query_param_uint64 (&rvalue),
665 GNUNET_SQ_query_param_auto_from_type (key),
666 GNUNET_SQ_query_param_auto_from_type (&vhash),
667 GNUNET_SQ_query_param_fixed_size (data, size),
668 GNUNET_SQ_query_param_end };
669 int n;
670 int ret;
671 sqlite3_stmt *stmt;
672
673 if (size > MAX_ITEM_SIZE)
674 {
675 cont (cont_cls, key, size, GNUNET_SYSERR, _ ("Data too large"));
676 return;
677 }
678 GNUNET_log_from (
679 GNUNET_ERROR_TYPE_DEBUG,
680 "sqlite",
681 "Storing in database block with type %u/key `%s'/priority %u/expiration in %s (%s).\n",
682 type,
683 GNUNET_h2s (key),
684 priority,
685 GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_remaining (
686 expiration),
687 GNUNET_YES),
688 GNUNET_STRINGS_absolute_time_to_string (expiration));
689 stmt = plugin->insertContent;
690 rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, UINT64_MAX);
691 if (GNUNET_OK != GNUNET_SQ_bind (stmt, params))
692 {
693 cont (cont_cls, key, size, GNUNET_SYSERR, NULL);
694 return;
695 }
696 n = sqlite3_step (stmt);
697 switch (n)
698 {
699 case SQLITE_DONE:
700 if (NULL != plugin->env->duc)
701 plugin->env->duc (plugin->env->cls,
702 size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
703 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
704 "sqlite",
705 "Stored new entry (%u bytes)\n",
706 size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
707 ret = GNUNET_OK;
708 break;
709
710 case SQLITE_BUSY:
711 GNUNET_break (0);
712 LOG_SQLITE_MSG (plugin,
713 &msg,
714 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
715 "sqlite3_step");
716 ret = GNUNET_SYSERR;
717 break;
718
719 default:
720 LOG_SQLITE_MSG (plugin,
721 &msg,
722 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
723 "sqlite3_step");
724 GNUNET_SQ_reset (plugin->dbh, stmt);
725 database_shutdown (plugin);
726 database_setup (plugin->env->cfg, plugin);
727 cont (cont_cls, key, size, GNUNET_SYSERR, msg);
728 GNUNET_free (msg);
729 return;
730 }
731 GNUNET_SQ_reset (plugin->dbh, stmt);
732 cont (cont_cls, key, size, ret, msg);
733 GNUNET_free (msg);
734}
735
736
737/**
738 * Execute statement that gets a row and call the callback
739 * with the result. Resets the statement afterwards.
740 *
741 * @param plugin the plugin
742 * @param stmt the statement
743 * @param proc processor to call
744 * @param proc_cls closure for @a proc
745 */
746static void
747execute_get (struct Plugin *plugin,
748 sqlite3_stmt *stmt,
749 PluginDatumProcessor proc,
750 void *proc_cls)
751{
752 int n;
753 struct GNUNET_TIME_Absolute expiration;
754 uint32_t replication;
755 uint32_t type;
756 uint32_t priority;
757 uint32_t anonymity;
758 uint64_t rowid;
759 void *value;
760 size_t value_size;
761 struct GNUNET_HashCode key;
762 int ret;
763 struct GNUNET_SQ_ResultSpec rs[] =
764 { GNUNET_SQ_result_spec_uint32 (&replication),
765 GNUNET_SQ_result_spec_uint32 (&type),
766 GNUNET_SQ_result_spec_uint32 (&priority),
767 GNUNET_SQ_result_spec_uint32 (&anonymity),
768 GNUNET_SQ_result_spec_absolute_time (&expiration),
769 GNUNET_SQ_result_spec_auto_from_type (&key),
770 GNUNET_SQ_result_spec_variable_size (&value, &value_size),
771 GNUNET_SQ_result_spec_uint64 (&rowid),
772 GNUNET_SQ_result_spec_end };
773
774 n = sqlite3_step (stmt);
775 switch (n)
776 {
777 case SQLITE_ROW:
778 if (GNUNET_OK != GNUNET_SQ_extract_result (stmt, rs))
779 {
780 GNUNET_break (0);
781 break;
782 }
783 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
784 "sqlite",
785 "Found reply in database with expiration %s\n",
786 GNUNET_STRINGS_absolute_time_to_string (expiration));
787 ret = proc (proc_cls,
788 &key,
789 value_size,
790 value,
791 type,
792 priority,
793 anonymity,
794 replication,
795 expiration,
796 rowid);
797 GNUNET_SQ_cleanup_result (rs);
798 GNUNET_SQ_reset (plugin->dbh, stmt);
799 if ((GNUNET_NO == ret) && (GNUNET_OK == delete_by_rowid (plugin, rowid)) &&
800 (NULL != plugin->env->duc))
801 plugin->env->duc (plugin->env->cls,
802 -(value_size + GNUNET_DATASTORE_ENTRY_OVERHEAD));
803 return;
804
805 case SQLITE_DONE:
806 /* database must be empty */
807 break;
808
809 case SQLITE_BUSY:
810 case SQLITE_ERROR:
811 case SQLITE_MISUSE:
812 default:
813 LOG_SQLITE (plugin,
814 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
815 "sqlite3_step");
816 if (SQLITE_OK != sqlite3_reset (stmt))
817 LOG_SQLITE (plugin,
818 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
819 "sqlite3_reset");
820 GNUNET_break (0);
821 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
822 database_shutdown (plugin);
823 database_setup (plugin->env->cfg, plugin);
824 return;
825 }
826 GNUNET_SQ_reset (plugin->dbh, stmt);
827 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
828}
829
830
831/**
832 * Select a subset of the items in the datastore and call
833 * the given processor for the item.
834 *
835 * @param cls our plugin context
836 * @param next_uid return the result with lowest uid >= next_uid
837 * @param type entries of which type should be considered?
838 * Must not be zero (ANY).
839 * @param proc function to call on the matching value;
840 * will be called with NULL if no value matches
841 * @param proc_cls closure for @a proc
842 */
843static void
844sqlite_plugin_get_zero_anonymity (void *cls,
845 uint64_t next_uid,
846 enum GNUNET_BLOCK_Type type,
847 PluginDatumProcessor proc,
848 void *proc_cls)
849{
850 struct Plugin *plugin = cls;
851 uint32_t type32 = type;
852 struct GNUNET_SQ_QueryParam params[] = { GNUNET_SQ_query_param_uint64 (
853 &next_uid),
854 GNUNET_SQ_query_param_uint32 (
855 &type32),
856 GNUNET_SQ_query_param_end };
857
858 GNUNET_assert (type != GNUNET_BLOCK_TYPE_ANY);
859 if (GNUNET_OK != GNUNET_SQ_bind (plugin->selZeroAnon, params))
860 {
861 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
862 return;
863 }
864 execute_get (plugin, plugin->selZeroAnon, proc, proc_cls);
865}
866
867
868/**
869 * Get results for a particular key in the datastore.
870 *
871 * @param cls closure
872 * @param next_uid return the result with lowest uid >= next_uid
873 * @param random if true, return a random result instead of using next_uid
874 * @param key maybe NULL (to match all entries)
875 * @param type entries of which type are relevant?
876 * Use 0 for any type.
877 * @param proc function to call on the matching value;
878 * will be called with NULL if nothing matches
879 * @param proc_cls closure for @a proc
880 */
881static void
882sqlite_plugin_get_key (void *cls,
883 uint64_t next_uid,
884 bool random,
885 const struct GNUNET_HashCode *key,
886 enum GNUNET_BLOCK_Type type,
887 PluginDatumProcessor proc,
888 void *proc_cls)
889{
890 struct Plugin *plugin = cls;
891 uint64_t rvalue;
892 int use_rvalue = random;
893 uint32_t type32 = (uint32_t) type;
894 int use_type = GNUNET_BLOCK_TYPE_ANY != type;
895 int use_key = NULL != key;
896 sqlite3_stmt *stmt = plugin->get[use_rvalue * 4 + use_key * 2 + use_type];
897 struct GNUNET_SQ_QueryParam params[] =
898 { GNUNET_SQ_query_param_uint64 (&next_uid),
899 GNUNET_SQ_query_param_uint64 (&rvalue),
900 GNUNET_SQ_query_param_auto_from_type (key),
901 GNUNET_SQ_query_param_uint32 (&type32),
902 GNUNET_SQ_query_param_end };
903
904 /* SQLite doesn't like it when you try to bind a parameter greater than the
905 * last numbered parameter, but unused parameters in the middle are OK.
906 */
907 if (! use_type)
908 {
909 params[3] = (struct GNUNET_SQ_QueryParam) GNUNET_SQ_query_param_end;
910 if (! use_key)
911 {
912 params[2] = (struct GNUNET_SQ_QueryParam) GNUNET_SQ_query_param_end;
913 if (! use_rvalue)
914 params[1] = (struct GNUNET_SQ_QueryParam) GNUNET_SQ_query_param_end;
915 }
916 }
917 if (random)
918 {
919 rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, UINT64_MAX);
920 next_uid = 0;
921 }
922 else
923 rvalue = 0;
924
925 if (GNUNET_OK != GNUNET_SQ_bind (stmt, params))
926 {
927 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
928 return;
929 }
930 execute_get (plugin, stmt, proc, proc_cls);
931}
932
933
934/**
935 * Context for #repl_proc() function.
936 */
937struct ReplCtx
938{
939 /**
940 * Function to call for the result (or the NULL).
941 */
942 PluginDatumProcessor proc;
943
944 /**
945 * Closure for @e proc.
946 */
947 void *proc_cls;
948
949 /**
950 * UID to use.
951 */
952 uint64_t uid;
953
954 /**
955 * Yes if UID was set.
956 */
957 int have_uid;
958};
959
960
961/**
962 * Wrapper for the processor for #sqlite_plugin_get_replication().
963 * Decrements the replication counter and calls the original
964 * processor.
965 *
966 * @param cls closure
967 * @param key key for the content
968 * @param size number of bytes in @a data
969 * @param data content stored
970 * @param type type of the content
971 * @param priority priority of the content
972 * @param anonymity anonymity-level for the content
973 * @param replication replication-level for the content
974 * @param expiration expiration time for the content
975 * @param uid unique identifier for the datum;
976 * maybe 0 if no unique identifier is available
977 * @return #GNUNET_OK for normal return,
978 * #GNUNET_NO to delete the item
979 */
980static int
981repl_proc (void *cls,
982 const struct GNUNET_HashCode *key,
983 uint32_t size,
984 const void *data,
985 enum GNUNET_BLOCK_Type type,
986 uint32_t priority,
987 uint32_t anonymity,
988 uint32_t replication,
989 struct GNUNET_TIME_Absolute expiration,
990 uint64_t uid)
991{
992 struct ReplCtx *rc = cls;
993 int ret;
994
995 if (GNUNET_SYSERR == rc->have_uid)
996 rc->have_uid = GNUNET_NO;
997 ret = rc->proc (rc->proc_cls,
998 key,
999 size,
1000 data,
1001 type,
1002 priority,
1003 anonymity,
1004 replication,
1005 expiration,
1006 uid);
1007 if (NULL != key)
1008 {
1009 rc->uid = uid;
1010 rc->have_uid = GNUNET_YES;
1011 }
1012 return ret;
1013}
1014
1015
1016/**
1017 * Get a random item for replication. Returns a single random item
1018 * from those with the highest replication counters. The item's
1019 * replication counter is decremented by one IF it was positive before.
1020 * Call @a proc with all values ZERO or NULL if the datastore is empty.
1021 *
1022 * @param cls closure
1023 * @param proc function to call the value (once only).
1024 * @param proc_cls closure for @a proc
1025 */
1026static void
1027sqlite_plugin_get_replication (void *cls,
1028 PluginDatumProcessor proc,
1029 void *proc_cls)
1030{
1031 struct Plugin *plugin = cls;
1032 struct ReplCtx rc;
1033 uint64_t rvalue = 0;
1034 uint32_t repl;
1035 struct GNUNET_SQ_QueryParam params_sel_repl[] =
1036 { GNUNET_SQ_query_param_uint64 (&rvalue),
1037 GNUNET_SQ_query_param_uint32 (&repl),
1038 GNUNET_SQ_query_param_end };
1039 struct GNUNET_SQ_QueryParam params_upd_repl[] =
1040 { GNUNET_SQ_query_param_uint64 (&rc.uid), GNUNET_SQ_query_param_end };
1041
1042 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
1043 "datastore-sqlite",
1044 "Getting random block based on replication order.\n");
1045 if (SQLITE_ROW != sqlite3_step (plugin->maxRepl))
1046 {
1047 GNUNET_SQ_reset (plugin->dbh, plugin->maxRepl);
1048 /* DB empty */
1049 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
1050 return;
1051 }
1052 repl = sqlite3_column_int (plugin->maxRepl, 0);
1053 GNUNET_SQ_reset (plugin->dbh, plugin->maxRepl);
1054 rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, UINT64_MAX);
1055 if (GNUNET_OK != GNUNET_SQ_bind (plugin->selRepl, params_sel_repl))
1056 {
1057 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
1058 return;
1059 }
1060 rc.have_uid = GNUNET_SYSERR;
1061 rc.proc = proc;
1062 rc.proc_cls = proc_cls;
1063 execute_get (plugin, plugin->selRepl, &repl_proc, &rc);
1064 if (GNUNET_YES == rc.have_uid)
1065 {
1066 if (GNUNET_OK != GNUNET_SQ_bind (plugin->updRepl, params_upd_repl))
1067 {
1068 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
1069 return;
1070 }
1071 if (SQLITE_DONE != sqlite3_step (plugin->updRepl))
1072 LOG_SQLITE (plugin,
1073 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
1074 "sqlite3_step");
1075 GNUNET_SQ_reset (plugin->dbh, plugin->updRepl);
1076 }
1077 if (GNUNET_SYSERR == rc.have_uid)
1078 {
1079 /* proc was not called at all so far, do it now. */
1080 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
1081 }
1082}
1083
1084
1085/**
1086 * Get a random item that has expired or has low priority.
1087 * Call @a proc with all values ZERO or NULL if the datastore is empty.
1088 *
1089 * @param cls closure
1090 * @param proc function to call the value (once only).
1091 * @param proc_cls closure for @a proc
1092 */
1093static void
1094sqlite_plugin_get_expiration (void *cls,
1095 PluginDatumProcessor proc,
1096 void *proc_cls)
1097{
1098 struct Plugin *plugin = cls;
1099 sqlite3_stmt *stmt;
1100 struct GNUNET_TIME_Absolute now = { 0 };
1101 struct GNUNET_SQ_QueryParam params[] = { GNUNET_SQ_query_param_absolute_time (
1102 &now),
1103 GNUNET_SQ_query_param_end };
1104
1105 GNUNET_log_from (
1106 GNUNET_ERROR_TYPE_DEBUG,
1107 "sqlite",
1108 "Getting random block based on expiration and priority order.\n");
1109 now = GNUNET_TIME_absolute_get ();
1110 stmt = plugin->selExpi;
1111 if (GNUNET_OK != GNUNET_SQ_bind (stmt, params))
1112 {
1113 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
1114 return;
1115 }
1116 execute_get (plugin, stmt, proc, proc_cls);
1117}
1118
1119
1120/**
1121 * Get all of the keys in the datastore.
1122 *
1123 * @param cls closure
1124 * @param proc function to call on each key
1125 * @param proc_cls closure for @a proc
1126 */
1127static void
1128sqlite_plugin_get_keys (void *cls, PluginKeyProcessor proc, void *proc_cls)
1129{
1130 struct Plugin *plugin = cls;
1131 struct GNUNET_HashCode key;
1132 struct GNUNET_SQ_ResultSpec results[] =
1133 { GNUNET_SQ_result_spec_auto_from_type (&key), GNUNET_SQ_result_spec_end };
1134 sqlite3_stmt *stmt;
1135 int ret;
1136
1137 GNUNET_assert (NULL != proc);
1138 if (SQLITE_OK != sq_prepare (plugin->dbh, "SELECT hash FROM gn091", &stmt))
1139 {
1140 LOG_SQLITE (plugin,
1141 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
1142 "sqlite_prepare");
1143 proc (proc_cls, NULL, 0);
1144 return;
1145 }
1146 while (SQLITE_ROW == (ret = sqlite3_step (stmt)))
1147 {
1148 if (GNUNET_OK == GNUNET_SQ_extract_result (stmt, results))
1149 proc (proc_cls, &key, 1);
1150 else
1151 GNUNET_break (0);
1152 }
1153 if (SQLITE_DONE != ret)
1154 LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "sqlite_step");
1155 sqlite3_finalize (stmt);
1156 proc (proc_cls, NULL, 0);
1157}
1158
1159
1160/**
1161 * Drop database.
1162 *
1163 * @param cls our plugin context
1164 */
1165static void
1166sqlite_plugin_drop (void *cls)
1167{
1168 struct Plugin *plugin = cls;
1169
1170 plugin->drop_on_shutdown = GNUNET_YES;
1171}
1172
1173
1174/**
1175 * Remove a particular key in the datastore.
1176 *
1177 * @param cls closure
1178 * @param key key for the content
1179 * @param size number of bytes in data
1180 * @param data content stored
1181 * @param cont continuation called with success or failure status
1182 * @param cont_cls continuation closure for @a cont
1183 */
1184static void
1185sqlite_plugin_remove_key (void *cls,
1186 const struct GNUNET_HashCode *key,
1187 uint32_t size,
1188 const void *data,
1189 PluginRemoveCont cont,
1190 void *cont_cls)
1191{
1192 struct Plugin *plugin = cls;
1193 struct GNUNET_SQ_QueryParam params[] =
1194 { GNUNET_SQ_query_param_auto_from_type (key),
1195 GNUNET_SQ_query_param_fixed_size (data, size),
1196 GNUNET_SQ_query_param_end };
1197
1198 if (GNUNET_OK != GNUNET_SQ_bind (plugin->remove, params))
1199 {
1200 cont (cont_cls, key, size, GNUNET_SYSERR, "bind failed");
1201 return;
1202 }
1203 if (SQLITE_DONE != sqlite3_step (plugin->remove))
1204 {
1205 LOG_SQLITE (plugin,
1206 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
1207 "sqlite3_step");
1208 GNUNET_SQ_reset (plugin->dbh, plugin->remove);
1209 cont (cont_cls, key, size, GNUNET_SYSERR, "sqlite3_step failed");
1210 return;
1211 }
1212 int changes = sqlite3_changes (plugin->dbh);
1213 GNUNET_SQ_reset (plugin->dbh, plugin->remove);
1214 if (0 == changes)
1215 {
1216 cont (cont_cls, key, size, GNUNET_NO, NULL);
1217 return;
1218 }
1219 if (NULL != plugin->env->duc)
1220 plugin->env->duc (plugin->env->cls,
1221 -(size + GNUNET_DATASTORE_ENTRY_OVERHEAD));
1222 cont (cont_cls, key, size, GNUNET_OK, NULL);
1223}
1224
1225
1226/**
1227 * Get an estimate of how much space the database is
1228 * currently using.
1229 *
1230 * @param cls the `struct Plugin`
1231 * @return the size of the database on disk (estimate)
1232 */
1233static void
1234sqlite_plugin_estimate_size (void *cls, unsigned long long *estimate)
1235{
1236 struct Plugin *plugin = cls;
1237 sqlite3_stmt *stmt;
1238 uint64_t pages;
1239 uint64_t page_size;
1240
1241#if ENULL_DEFINED
1242 char *e;
1243#endif
1244
1245 if (NULL == estimate)
1246 return;
1247 if (SQLITE_VERSION_NUMBER < 3006000)
1248 {
1249 GNUNET_log_from (
1250 GNUNET_ERROR_TYPE_WARNING,
1251 "datastore-sqlite",
1252 _ ("sqlite version to old to determine size, assuming zero\n"));
1253 *estimate = 0;
1254 return;
1255 }
1256 CHECK (SQLITE_OK == sqlite3_exec (plugin->dbh, "VACUUM", NULL, NULL, ENULL));
1257 CHECK (SQLITE_OK == sqlite3_exec (plugin->dbh,
1258 "PRAGMA auto_vacuum=INCREMENTAL",
1259 NULL,
1260 NULL,
1261 ENULL));
1262 if (SQLITE_OK != sq_prepare (plugin->dbh, "PRAGMA page_count", &stmt))
1263 {
1264 GNUNET_log_from (
1265 GNUNET_ERROR_TYPE_WARNING,
1266 "datastore-sqlite",
1267 _("error preparing statement\n"));
1268 return;
1269 }
1270 if (SQLITE_ROW == sqlite3_step (stmt))
1271 pages = sqlite3_column_int64 (stmt, 0);
1272 else
1273 pages = 0;
1274 sqlite3_finalize (stmt);
1275 if (SQLITE_OK != sq_prepare (plugin->dbh, "PRAGMA page_size", &stmt))
1276 {
1277 GNUNET_log_from (
1278 GNUNET_ERROR_TYPE_WARNING,
1279 "datastore-sqlite",
1280 _("error preparing statement\n"));
1281 return;
1282 }
1283 if (SQLITE_ROW != sqlite3_step (stmt))
1284 {
1285 GNUNET_log_from (
1286 GNUNET_ERROR_TYPE_WARNING,
1287 "datastore-sqlite",
1288 _("error stepping\n"));
1289 return;
1290 }
1291 page_size = sqlite3_column_int64 (stmt, 0);
1292 sqlite3_finalize (stmt);
1293 GNUNET_log (
1294 GNUNET_ERROR_TYPE_INFO,
1295 _ (
1296 "Using sqlite page utilization to estimate payload (%llu pages of size %llu bytes)\n"),
1297 (unsigned long long) pages,
1298 (unsigned long long) page_size);
1299 *estimate = pages * page_size;
1300}
1301
1302
1303/**
1304 * Entry point for the plugin.
1305 *
1306 * @param cls the `struct GNUNET_DATASTORE_PluginEnvironment *`
1307 * @return NULL on error, othrewise the plugin context
1308 */
1309void *
1310libgnunet_plugin_datastore_sqlite_init (void *cls)
1311{
1312 static struct Plugin plugin;
1313 struct GNUNET_DATASTORE_PluginEnvironment *env = cls;
1314 struct GNUNET_DATASTORE_PluginFunctions *api;
1315
1316 if (NULL != plugin.env)
1317 return NULL; /* can only initialize once! */
1318 memset (&plugin, 0, sizeof(struct Plugin));
1319 plugin.env = env;
1320 if (GNUNET_OK != database_setup (env->cfg, &plugin))
1321 {
1322 database_shutdown (&plugin);
1323 return NULL;
1324 }
1325 api = GNUNET_new (struct GNUNET_DATASTORE_PluginFunctions);
1326 api->cls = &plugin;
1327 api->estimate_size = &sqlite_plugin_estimate_size;
1328 api->put = &sqlite_plugin_put;
1329 api->get_key = &sqlite_plugin_get_key;
1330 api->get_replication = &sqlite_plugin_get_replication;
1331 api->get_expiration = &sqlite_plugin_get_expiration;
1332 api->get_zero_anonymity = &sqlite_plugin_get_zero_anonymity;
1333 api->get_keys = &sqlite_plugin_get_keys;
1334 api->drop = &sqlite_plugin_drop;
1335 api->remove_key = &sqlite_plugin_remove_key;
1336 GNUNET_log_from (GNUNET_ERROR_TYPE_INFO,
1337 "sqlite",
1338 _ ("Sqlite database running\n"));
1339 return api;
1340}
1341
1342
1343/**
1344 * Exit point from the plugin.
1345 *
1346 * @param cls the plugin context (as returned by "init")
1347 * @return always NULL
1348 */
1349void *
1350libgnunet_plugin_datastore_sqlite_done (void *cls)
1351{
1352 char *fn;
1353 struct GNUNET_DATASTORE_PluginFunctions *api = cls;
1354 struct Plugin *plugin = api->cls;
1355
1356 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
1357 "sqlite",
1358 "sqlite plugin is done\n");
1359 fn = NULL;
1360 if (plugin->drop_on_shutdown)
1361 fn = GNUNET_strdup (plugin->fn);
1362 database_shutdown (plugin);
1363 plugin->env = NULL;
1364 GNUNET_free (api);
1365 if (NULL != fn)
1366 {
1367 if (0 != unlink (fn))
1368 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", fn);
1369 GNUNET_free (fn);
1370 }
1371 return NULL;
1372}
1373
1374
1375/* end of plugin_datastore_sqlite.c */
diff --git a/src/datastore/plugin_datastore_template.c b/src/datastore/plugin_datastore_template.c
deleted file mode 100644
index 2b455f8cb..000000000
--- a/src/datastore/plugin_datastore_template.c
+++ /dev/null
@@ -1,274 +0,0 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2009, 2011 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file datastore/plugin_datastore_template.c
23 * @brief template-based datastore backend
24 * @author Christian Grothoff
25 */
26
27#include "platform.h"
28#include "gnunet_datastore_plugin.h"
29
30
31/**
32 * Context for all functions in this plugin.
33 */
34struct Plugin
35{
36 /**
37 * Our execution environment.
38 */
39 struct GNUNET_DATASTORE_PluginEnvironment *env;
40};
41
42
43/**
44 * Get an estimate of how much space the database is
45 * currently using.
46 *
47 * @param cls our "struct Plugin*"
48 * @return number of bytes used on disk
49 */
50static void
51template_plugin_estimate_size (void *cls, unsigned long long *estimate)
52{
53 if (NULL == estimate)
54 return;
55 GNUNET_break (0);
56 *estimate = 0;
57}
58
59
60/**
61 * Store an item in the datastore.
62 *
63 * @param cls closure
64 * @param key key for the item
65 * @param absent true if the key was not found in the bloom filter
66 * @param size number of bytes in data
67 * @param data content stored
68 * @param type type of the content
69 * @param priority priority of the content
70 * @param anonymity anonymity-level for the content
71 * @param replication replication-level for the content
72 * @param expiration expiration time for the content
73 * @param cont continuation called with success or failure status
74 * @param cont_cls continuation closure
75 */
76static void
77template_plugin_put (void *cls,
78 const struct GNUNET_HashCode *key,
79 bool absent,
80 uint32_t size,
81 const void *data,
82 enum GNUNET_BLOCK_Type type,
83 uint32_t priority,
84 uint32_t anonymity,
85 uint32_t replication,
86 struct GNUNET_TIME_Absolute expiration,
87 PluginPutCont cont,
88 void *cont_cls)
89{
90 GNUNET_break (0);
91 cont (cont_cls, key, size, GNUNET_SYSERR, "not implemented");
92}
93
94
95/**
96 * Get one of the results for a particular key in the datastore.
97 *
98 * @param cls closure
99 * @param next_uid return the result with lowest uid >= next_uid
100 * @param random if true, return a random result instead of using next_uid
101 * @param key maybe NULL (to match all entries)
102 * @param type entries of which type are relevant?
103 * Use 0 for any type.
104 * @param proc function to call on each matching value;
105 * will be called with NULL if nothing matches
106 * @param proc_cls closure for proc
107 */
108static void
109template_plugin_get_key (void *cls,
110 uint64_t next_uid,
111 bool random,
112 const struct GNUNET_HashCode *key,
113 enum GNUNET_BLOCK_Type type,
114 PluginDatumProcessor proc,
115 void *proc_cls)
116{
117 GNUNET_break (0);
118}
119
120
121/**
122 * Get a random item for replication. Returns a single, not expired,
123 * random item from those with the highest replication counters. The
124 * item's replication counter is decremented by one IF it was positive
125 * before. Call 'proc' with all values ZERO or NULL if the datastore
126 * is empty.
127 *
128 * @param cls closure
129 * @param proc function to call the value (once only).
130 * @param proc_cls closure for proc
131 */
132static void
133template_plugin_get_replication (void *cls, PluginDatumProcessor proc,
134 void *proc_cls)
135{
136 GNUNET_break (0);
137}
138
139
140/**
141 * Get a random item for expiration. Call 'proc' with all values ZERO
142 * or NULL if the datastore is empty.
143 *
144 * @param cls closure
145 * @param proc function to call the value (once only).
146 * @param proc_cls closure for proc
147 */
148static void
149template_plugin_get_expiration (void *cls, PluginDatumProcessor proc,
150 void *proc_cls)
151{
152 GNUNET_break (0);
153}
154
155
156/**
157 * Call the given processor on an item with zero anonymity.
158 *
159 * @param cls our "struct Plugin*"
160 * @param next_uid return the result with lowest uid >= next_uid
161 * @param type entries of which type should be considered?
162 * Must not be zero (ANY).
163 * @param proc function to call on the matching value;
164 * will be called with NULL if no value matches
165 * @param proc_cls closure for proc
166 */
167static void
168template_plugin_get_zero_anonymity (void *cls, uint64_t next_uid,
169 enum GNUNET_BLOCK_Type type,
170 PluginDatumProcessor proc, void *proc_cls)
171{
172 GNUNET_break (0);
173}
174
175
176/**
177 * Drop database.
178 */
179static void
180template_plugin_drop (void *cls)
181{
182 GNUNET_break (0);
183}
184
185
186/**
187 * Get all of the keys in the datastore.
188 *
189 * @param cls closure
190 * @param proc function to call on each key
191 * @param proc_cls closure for proc
192 */
193static void
194template_get_keys (void *cls,
195 PluginKeyProcessor proc,
196 void *proc_cls)
197{
198 proc (proc_cls, NULL, 0);
199}
200
201
202/**
203 * Remove a particular key in the datastore.
204 *
205 * @param cls closure
206 * @param key key for the content
207 * @param size number of bytes in data
208 * @param data content stored
209 * @param cont continuation called with success or failure status
210 * @param cont_cls continuation closure for @a cont
211 */
212static void
213template_plugin_remove_key (void *cls,
214 const struct GNUNET_HashCode *key,
215 uint32_t size,
216 const void *data,
217 PluginRemoveCont cont,
218 void *cont_cls)
219{
220 GNUNET_break (0);
221 cont (cont_cls, key, size, GNUNET_SYSERR, "not implemented");
222}
223
224
225/**
226 * Entry point for the plugin.
227 *
228 * @param cls the "struct GNUNET_DATASTORE_PluginEnvironment*"
229 * @return our "struct Plugin*"
230 */
231void *
232libgnunet_plugin_datastore_template_init (void *cls)
233{
234 struct GNUNET_DATASTORE_PluginEnvironment *env = cls;
235 struct GNUNET_DATASTORE_PluginFunctions *api;
236 struct Plugin *plugin;
237
238 plugin = GNUNET_new (struct Plugin);
239 plugin->env = env;
240 api = GNUNET_new (struct GNUNET_DATASTORE_PluginFunctions);
241 api->cls = plugin;
242 api->estimate_size = &template_plugin_estimate_size;
243 api->put = &template_plugin_put;
244 api->get_key = &template_plugin_get_key;
245 api->get_replication = &template_plugin_get_replication;
246 api->get_expiration = &template_plugin_get_expiration;
247 api->get_zero_anonymity = &template_plugin_get_zero_anonymity;
248 api->drop = &template_plugin_drop;
249 api->get_keys = &template_get_keys;
250 api->remove_key = &template_plugin_remove_key;
251 GNUNET_log_from (GNUNET_ERROR_TYPE_INFO, "template",
252 _ ("Template database running\n"));
253 return api;
254}
255
256
257/**
258 * Exit point from the plugin.
259 * @param cls our "struct Plugin*"
260 * @return always NULL
261 */
262void *
263libgnunet_plugin_datastore_template_done (void *cls)
264{
265 struct GNUNET_DATASTORE_PluginFunctions *api = cls;
266 struct Plugin *plugin = api->cls;
267
268 GNUNET_free (plugin);
269 GNUNET_free (api);
270 return NULL;
271}
272
273
274/* end of plugin_datastore_template.c */
diff --git a/src/datastore/selectrandom.sql b/src/datastore/selectrandom.sql
deleted file mode 100644
index 82830a13a..000000000
--- a/src/datastore/selectrandom.sql
+++ /dev/null
@@ -1,9 +0,0 @@
1select *
2from (select random() as v from (values(1))) t1,
3 (select max(repl) as m from data) t2,
4 (select * from data
5 where repl=t2.m and
6 rnd>=t.v
7 order by rnd
8 limit 1)
9
diff --git a/src/datastore/test_datastore_api.c b/src/datastore/test_datastore_api.c
deleted file mode 100644
index e72a6acd3..000000000
--- a/src/datastore/test_datastore_api.c
+++ /dev/null
@@ -1,732 +0,0 @@
1/*
2 This file is part of GNUnet.
3 Copyright (C) 2004, 2005, 2006, 2007, 2009, 2015 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20/*
21 * @file datastore/test_datastore_api.c
22 * @brief Test for the basic datastore API.
23 * @author Christian Grothoff
24 *
25 * TODO:
26 * - test reservation failure
27 */
28
29#include "platform.h"
30#include "gnunet_util_lib.h"
31#include "gnunet_protocols.h"
32#include "gnunet_datastore_service.h"
33#include "gnunet_datastore_plugin.h"
34#include "gnunet_testing_lib.h"
35
36
37/**
38 * How long until we give up on transmitting the message?
39 */
40#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 15)
41
42#define ITERATIONS 256
43
44/**
45 * Handle to the datastore.
46 */
47static struct GNUNET_DATASTORE_Handle *datastore;
48
49static struct GNUNET_TIME_Absolute now;
50
51/**
52 * Value we return from #main().
53 */
54static int ok;
55
56/**
57 * Name of plugin under test.
58 */
59static const char *plugin_name;
60
61
62static size_t
63get_size (int i)
64{
65 return 8 * i;
66}
67
68
69static const void *
70get_data (int i)
71{
72 static char buf[60000];
73
74 memset (buf, i, 8 * i);
75 return buf;
76}
77
78
79static int
80get_type (int i)
81{
82 return i + 1;
83}
84
85
86static int
87get_priority (int i)
88{
89 return i + 1;
90}
91
92
93static int
94get_anonymity (int i)
95{
96 return i;
97}
98
99
100static struct GNUNET_TIME_Absolute
101get_expiration (int i)
102{
103 struct GNUNET_TIME_Absolute av;
104
105 av.abs_value_us = now.abs_value_us + 20000000000LL - i * 1000 * 1000LL;
106 return av;
107}
108
109
110/**
111 * Which phase of the process are we in?
112 */
113enum RunPhase
114{
115 /**
116 * We are done (shutting down normally).
117 */
118 RP_DONE = 0,
119
120 /**
121 * We are adding new entries to the datastore.
122 */
123 RP_PUT = 1,
124 RP_GET = 2,
125 RP_DEL = 3,
126 RP_DO_DEL = 4,
127 RP_DELVALIDATE = 5,
128 RP_RESERVE = 6,
129 RP_PUT_MULTIPLE = 7,
130 RP_PUT_MULTIPLE_NEXT = 8,
131 RP_GET_MULTIPLE = 9,
132 RP_GET_MULTIPLE_NEXT = 10,
133
134 /**
135 * Execution failed with some kind of error.
136 */
137 RP_ERROR
138};
139
140
141/**
142 * Closure we give to all of the functions executing the
143 * benchmark. Could right now be global, but this allows
144 * us to theoretically run multiple clients "in parallel".
145 */
146struct CpsRunContext
147{
148 /**
149 * Execution phase we are in.
150 */
151 enum RunPhase phase;
152
153 struct GNUNET_HashCode key;
154 int i;
155 int rid;
156 void *data;
157 size_t size;
158
159 uint64_t first_uid;
160};
161
162
163/**
164 * Main state machine. Executes the next step of the test
165 * depending on the current state.
166 *
167 * @param cls the `struct CpsRunContext`
168 */
169static void
170run_continuation (void *cls);
171
172
173/**
174 * Continuation called to notify client about result of an
175 * operation. Checks for errors, updates our iteration counters and
176 * continues execution with #run_continuation().
177 *
178 * @param cls the `struct CpsRunContext`
179 * @param success #GNUNET_SYSERR on failure
180 * @param min_expiration minimum expiration time required for content to be stored
181 * by the datacache at this time, zero for unknown
182 * @param msg NULL on success, otherwise an error message
183 */
184static void
185check_success (void *cls,
186 int success,
187 struct GNUNET_TIME_Absolute min_expiration,
188 const char *msg)
189{
190 struct CpsRunContext *crc = cls;
191
192 if (GNUNET_OK != success)
193 {
194 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
195 "Operation %d/%d not successful: `%s'\n",
196 crc->phase,
197 crc->i,
198 msg);
199 crc->phase = RP_ERROR;
200 }
201 GNUNET_free (crc->data);
202 crc->data = NULL;
203 GNUNET_SCHEDULER_add_now (&run_continuation, crc);
204}
205
206
207static void
208get_reserved (void *cls,
209 int success,
210 struct GNUNET_TIME_Absolute min_expiration,
211 const char *msg)
212{
213 struct CpsRunContext *crc = cls;
214
215 if (0 >= success)
216 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
217 "Error obtaining reservation: `%s'\n",
218 msg);
219 GNUNET_assert (0 < success);
220 crc->rid = success;
221 GNUNET_SCHEDULER_add_now (&run_continuation,
222 crc);
223}
224
225
226static void
227check_value (void *cls,
228 const struct GNUNET_HashCode *key,
229 size_t size,
230 const void *data,
231 enum GNUNET_BLOCK_Type type,
232 uint32_t priority,
233 uint32_t anonymity,
234 uint32_t replication,
235 struct GNUNET_TIME_Absolute expiration,
236 uint64_t uid)
237{
238 struct CpsRunContext *crc = cls;
239 int i;
240
241 i = crc->i;
242 if (NULL == key)
243 {
244 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
245 "Value check failed (got NULL key) in %d/%d\n",
246 crc->phase,
247 crc->i);
248 crc->phase = RP_ERROR;
249 GNUNET_SCHEDULER_add_now (&run_continuation,
250 crc);
251 return;
252 }
253#if 0
254 fprintf (stderr,
255 "Check value got `%s' of size %u, type %d, expire %s\n",
256 GNUNET_h2s (key), (unsigned int) size, type,
257 GNUNET_STRINGS_absolute_time_to_string (expiration));
258 fprintf (stderr,
259 "Check value iteration %d wants size %u, type %d, expire %s\n", i,
260 (unsigned int) get_size (i), get_type (i),
261 GNUNET_STRINGS_absolute_time_to_string (get_expiration (i)));
262#endif
263 GNUNET_assert (size == get_size (i));
264 GNUNET_assert (0 == memcmp (data, get_data (i), size));
265 GNUNET_assert (type == get_type (i));
266 GNUNET_assert (priority == get_priority (i));
267 GNUNET_assert (anonymity == get_anonymity (i));
268 GNUNET_assert (expiration.abs_value_us == get_expiration (i).abs_value_us);
269 if (crc->i == 0)
270 {
271 crc->phase = RP_DEL;
272 crc->i = ITERATIONS;
273 }
274 GNUNET_SCHEDULER_add_now (&run_continuation,
275 crc);
276}
277
278
279static void
280delete_value (void *cls,
281 const struct GNUNET_HashCode *key,
282 size_t size,
283 const void *data,
284 enum GNUNET_BLOCK_Type type,
285 uint32_t priority,
286 uint32_t anonymity,
287 uint32_t replication,
288 struct GNUNET_TIME_Absolute expiration,
289 uint64_t uid)
290{
291 struct CpsRunContext *crc = cls;
292
293 GNUNET_assert (NULL == crc->data);
294 GNUNET_assert (NULL != key);
295 crc->size = size;
296 crc->key = *key;
297 crc->data = GNUNET_malloc (size);
298 GNUNET_memcpy (crc->data, data, size);
299 crc->phase = RP_DO_DEL;
300 GNUNET_SCHEDULER_add_now (&run_continuation,
301 crc);
302}
303
304
305static void
306check_nothing (void *cls,
307 const struct GNUNET_HashCode *key,
308 size_t size,
309 const void *data,
310 enum GNUNET_BLOCK_Type type,
311 uint32_t priority,
312 uint32_t anonymity,
313 uint32_t replication,
314 struct GNUNET_TIME_Absolute expiration,
315 uint64_t uid)
316{
317 struct CpsRunContext *crc = cls;
318
319 GNUNET_assert (key == NULL);
320 if (crc->i == 0)
321 crc->phase = RP_RESERVE;
322 GNUNET_SCHEDULER_add_now (&run_continuation,
323 crc);
324}
325
326
327static void
328check_multiple (void *cls,
329 const struct GNUNET_HashCode *key,
330 size_t size,
331 const void *data,
332 enum GNUNET_BLOCK_Type type,
333 uint32_t priority,
334 uint32_t anonymity,
335 uint32_t replication,
336 struct GNUNET_TIME_Absolute expiration,
337 uint64_t uid)
338{
339 struct CpsRunContext *crc = cls;
340
341 GNUNET_assert (key != NULL);
342 switch (crc->phase)
343 {
344 case RP_GET_MULTIPLE:
345 crc->phase = RP_GET_MULTIPLE_NEXT;
346 crc->first_uid = uid;
347 break;
348
349 case RP_GET_MULTIPLE_NEXT:
350 GNUNET_assert (uid != crc->first_uid);
351 crc->phase = RP_DONE;
352 break;
353
354 default:
355 GNUNET_break (0);
356 crc->phase = RP_ERROR;
357 break;
358 }
359 GNUNET_SCHEDULER_add_now (&run_continuation, crc);
360}
361
362
363/**
364 * Main state machine. Executes the next step of the test
365 * depending on the current state.
366 *
367 * @param cls the `struct CpsRunContext`
368 */
369static void
370run_continuation (void *cls)
371{
372 struct CpsRunContext *crc = cls;
373
374 ok = (int) crc->phase;
375 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
376 "Test in phase %u\n",
377 crc->phase);
378 switch (crc->phase)
379 {
380 case RP_PUT:
381 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
382 "Executing PUT number %u\n",
383 crc->i);
384 GNUNET_CRYPTO_hash (&crc->i, sizeof(int), &crc->key);
385 GNUNET_DATASTORE_put (datastore, 0, &crc->key, get_size (crc->i),
386 get_data (crc->i), get_type (crc->i),
387 get_priority (crc->i), get_anonymity (crc->i), 0,
388 get_expiration (crc->i), 1, 1,
389 &check_success, crc);
390 crc->i++;
391 if (crc->i == ITERATIONS)
392 crc->phase = RP_GET;
393 break;
394
395 case RP_GET:
396 crc->i--;
397 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
398 "Executing GET number %u\n",
399 crc->i);
400 GNUNET_CRYPTO_hash (&crc->i,
401 sizeof(int),
402 &crc->key);
403 GNUNET_DATASTORE_get_key (datastore,
404 0,
405 false,
406 &crc->key,
407 get_type (crc->i),
408 1,
409 1,
410 &check_value,
411 crc);
412 break;
413
414 case RP_DEL:
415 crc->i--;
416 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
417 "Executing DEL number %u\n",
418 crc->i);
419 crc->data = NULL;
420 GNUNET_CRYPTO_hash (&crc->i, sizeof(int), &crc->key);
421 GNUNET_assert (NULL !=
422 GNUNET_DATASTORE_get_key (datastore,
423 0,
424 false,
425 &crc->key,
426 get_type (crc->i),
427 1,
428 1,
429 &delete_value,
430 crc));
431 break;
432
433 case RP_DO_DEL:
434 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
435 "Executing DO_DEL number %u\n",
436 crc->i);
437 if (crc->i == 0)
438 {
439 crc->i = ITERATIONS;
440 crc->phase = RP_DELVALIDATE;
441 }
442 else
443 {
444 crc->phase = RP_DEL;
445 }
446 GNUNET_assert (NULL !=
447 GNUNET_DATASTORE_remove (datastore, &crc->key, crc->size,
448 crc->data, 1, 1,
449 &check_success, crc));
450 break;
451
452 case RP_DELVALIDATE:
453 crc->i--;
454 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
455 "Executing DELVALIDATE number %u\n",
456 crc->i);
457 GNUNET_CRYPTO_hash (&crc->i, sizeof(int), &crc->key);
458 GNUNET_assert (NULL !=
459 GNUNET_DATASTORE_get_key (datastore,
460 0,
461 false,
462 &crc->key,
463 get_type (crc->i),
464 1,
465 1,
466 &check_nothing,
467 crc));
468 break;
469
470 case RP_RESERVE:
471 crc->phase = RP_PUT_MULTIPLE;
472 GNUNET_DATASTORE_reserve (datastore, 128 * 1024, 2,
473 &get_reserved, crc);
474 break;
475
476 case RP_PUT_MULTIPLE:
477 crc->phase = RP_PUT_MULTIPLE_NEXT;
478 GNUNET_DATASTORE_put (datastore, crc->rid, &crc->key, get_size (42),
479 get_data (42), get_type (42), get_priority (42),
480 get_anonymity (42), 0, get_expiration (42), 1, 1,
481 &check_success, crc);
482 break;
483
484 case RP_PUT_MULTIPLE_NEXT:
485 crc->phase = RP_GET_MULTIPLE;
486 GNUNET_DATASTORE_put (datastore, crc->rid,
487 &crc->key,
488 get_size (43),
489 get_data (43),
490 get_type (42),
491 get_priority (43),
492 get_anonymity (43),
493 0,
494 get_expiration (43),
495 1, 1,
496 &check_success, crc);
497 break;
498
499 case RP_GET_MULTIPLE:
500 GNUNET_assert (NULL !=
501 GNUNET_DATASTORE_get_key (datastore,
502 0,
503 false,
504 &crc->key,
505 get_type (42),
506 1,
507 1,
508 &check_multiple,
509 crc));
510 break;
511
512 case RP_GET_MULTIPLE_NEXT:
513 GNUNET_assert (NULL !=
514 GNUNET_DATASTORE_get_key (datastore,
515 crc->first_uid + 1,
516 false,
517 &crc->key,
518 get_type (42),
519 1,
520 1,
521 &check_multiple,
522 crc));
523 break;
524
525 case RP_DONE:
526 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
527 "Finished, disconnecting\n");
528 GNUNET_DATASTORE_disconnect (datastore,
529 GNUNET_YES);
530 GNUNET_free (crc);
531 ok = 0;
532 break;
533
534 case RP_ERROR:
535 GNUNET_DATASTORE_disconnect (datastore,
536 GNUNET_YES);
537 GNUNET_free (crc);
538 ok = 43;
539 break;
540 }
541}
542
543
544/**
545 * Function called with the result of the initial PUT operation. If
546 * the PUT succeeded, we start the actual benchmark loop, otherwise we
547 * bail out with an error.
548 *
549 *
550 * @param cls closure
551 * @param success #GNUNET_SYSERR on failure
552 * @param min_expiration minimum expiration time required for content to be stored
553 * by the datacache at this time, zero for unknown
554 * @param msg NULL on success, otherwise an error message
555 */
556static void
557run_tests (void *cls,
558 int32_t success,
559 struct GNUNET_TIME_Absolute min_expiration,
560 const char *msg)
561{
562 struct CpsRunContext *crc = cls;
563
564 switch (success)
565 {
566 case GNUNET_YES:
567 GNUNET_SCHEDULER_add_now (&run_continuation,
568 crc);
569 return;
570
571 case GNUNET_NO:
572 fprintf (stderr,
573 "%s", "Test 'put' operation failed, key already exists (!?)\n");
574 GNUNET_DATASTORE_disconnect (datastore,
575 GNUNET_YES);
576 GNUNET_free (crc);
577 return;
578
579 case GNUNET_SYSERR:
580 fprintf (stderr,
581 "Test 'put' operation failed with error `%s' database likely not setup, skipping test.\n",
582 msg);
583 GNUNET_DATASTORE_disconnect (datastore,
584 GNUNET_YES);
585 GNUNET_free (crc);
586 return;
587
588 default:
589 GNUNET_assert (0);
590 }
591}
592
593
594/**
595 * Beginning of the actual execution of the benchmark.
596 * Performs a first test operation (PUT) to verify that
597 * the plugin works at all.
598 *
599 * @param cls NULL
600 * @param cfg configuration to use
601 * @param peer peer handle (unused)
602 */
603static void
604run (void *cls,
605 const struct GNUNET_CONFIGURATION_Handle *cfg,
606 struct GNUNET_TESTING_Peer *peer)
607{
608 struct CpsRunContext *crc;
609 static struct GNUNET_HashCode zkey;
610
611 crc = GNUNET_new (struct CpsRunContext);
612 crc->phase = RP_PUT;
613 now = GNUNET_TIME_absolute_get ();
614 datastore = GNUNET_DATASTORE_connect (cfg);
615 if (NULL ==
616 GNUNET_DATASTORE_put (datastore,
617 0,
618 &zkey,
619 4,
620 "TEST",
621 GNUNET_BLOCK_TYPE_TEST,
622 0, 0, 0,
623 GNUNET_TIME_relative_to_absolute
624 (GNUNET_TIME_UNIT_SECONDS),
625 0, 1,
626 &run_tests, crc))
627 {
628 fprintf (stderr,
629 "%s",
630 "Test 'put' operation failed.\n");
631 ok = 1;
632 GNUNET_free (crc);
633 }
634}
635
636
637/**
638 * Function invoked to notify service of disk utilization
639 * changes.
640 *
641 * @param cls closure
642 * @param delta change in disk utilization,
643 * 0 for "reset to empty"
644 */
645static void
646duc_dummy (void *cls,
647 int delta)
648{
649 /* intentionally empty */
650}
651
652
653/**
654 * check if plugin is actually working
655 */
656static int
657test_plugin (const char *cfg_name)
658{
659 char libname[128];
660 struct GNUNET_CONFIGURATION_Handle *cfg;
661 struct GNUNET_DATASTORE_PluginFunctions *api;
662 struct GNUNET_DATASTORE_PluginEnvironment env;
663
664 cfg = GNUNET_CONFIGURATION_create ();
665 if (GNUNET_OK !=
666 GNUNET_CONFIGURATION_load (cfg,
667 cfg_name))
668 {
669 GNUNET_CONFIGURATION_destroy (cfg);
670 fprintf (stderr,
671 "Failed to load configuration %s\n",
672 cfg_name);
673 return 1;
674 }
675 memset (&env, 0, sizeof(env));
676 env.cfg = cfg;
677 env.duc = &duc_dummy;
678 GNUNET_snprintf (libname,
679 sizeof(libname),
680 "libgnunet_plugin_datastore_%s",
681 plugin_name);
682 api = GNUNET_PLUGIN_load (libname, &env);
683 if (NULL == api)
684 {
685 GNUNET_CONFIGURATION_destroy (cfg);
686 fprintf (stderr,
687 "Failed to load plugin `%s'\n",
688 libname);
689 return 77;
690 }
691 GNUNET_PLUGIN_unload (libname, api);
692 GNUNET_CONFIGURATION_destroy (cfg);
693 return 0;
694}
695
696
697/**
698 * Entry point into the test. Determines which configuration / plugin
699 * we are running with based on the name of the binary and starts
700 * the peer.
701 *
702 * @param argc should be 1
703 * @param argv used to determine plugin / configuration name.
704 * @return 0 on success
705 */
706int
707main (int argc,
708 char *argv[])
709{
710 char cfg_name[PATH_MAX];
711 int ret;
712
713 plugin_name = GNUNET_TESTING_get_testname_from_underscore (argv[0]);
714 GNUNET_snprintf (cfg_name,
715 sizeof(cfg_name),
716 "test_datastore_api_data_%s.conf",
717 plugin_name);
718 ret = test_plugin (cfg_name);
719 if (0 != ret)
720 return ret;
721 /* run actual test */
722 if (0 !=
723 GNUNET_TESTING_peer_run ("test-gnunet-datastore",
724 cfg_name,
725 &run,
726 NULL))
727 return 1;
728 return ok;
729}
730
731
732/* end of test_datastore_api.c */
diff --git a/src/datastore/test_datastore_api_data_heap.conf b/src/datastore/test_datastore_api_data_heap.conf
deleted file mode 100644
index 6f11fb3f7..000000000
--- a/src/datastore/test_datastore_api_data_heap.conf
+++ /dev/null
@@ -1,19 +0,0 @@
1@INLINE@ test_defaults.conf
2[PATHS]
3GNUNET_TEST_HOME = $GNUNET_TMP/test-gnunet-datastore-heap/
4
5[TESTING]
6WEAKRANDOM = YES
7
8[arm]
9PORT = 42466
10
11[statistics]
12PORT = 22667
13
14[resolver]
15PORT = 42464
16
17[datastore]
18QUOTA = 10 MB
19DATABASE = heap
diff --git a/src/datastore/test_datastore_api_data_mysql.conf b/src/datastore/test_datastore_api_data_mysql.conf
deleted file mode 100644
index c0052c5ea..000000000
--- a/src/datastore/test_datastore_api_data_mysql.conf
+++ /dev/null
@@ -1,10 +0,0 @@
1@INLINE@ test_defaults.conf
2[PATHS]
3GNUNET_TEST_HOME = $GNUNET_TMP/test-gnunet-datastore-mysql/
4
5[datastore]
6QUOTA = 10 MB
7DATABASE = mysql
8
9[datastore-mysql]
10DATABASE = gnunetcheck
diff --git a/src/datastore/test_datastore_api_data_postgres.conf b/src/datastore/test_datastore_api_data_postgres.conf
deleted file mode 100644
index 65fe11806..000000000
--- a/src/datastore/test_datastore_api_data_postgres.conf
+++ /dev/null
@@ -1,10 +0,0 @@
1@INLINE@ test_defaults.conf
2[PATHS]
3GNUNET_TEST_HOME = $GNUNET_TMP/test-gnunet-datastore-postgres/
4
5[datastore]
6QUOTA = 10 MB
7DATABASE = postgres
8
9[datastore-postgres]
10CONFIG = dbname=gnunetcheck
diff --git a/src/datastore/test_datastore_api_data_sqlite.conf b/src/datastore/test_datastore_api_data_sqlite.conf
deleted file mode 100644
index ecdd0c6ee..000000000
--- a/src/datastore/test_datastore_api_data_sqlite.conf
+++ /dev/null
@@ -1,7 +0,0 @@
1@INLINE@ test_defaults.conf
2[PATHS]
3GNUNET_TEST_HOME = $GNUNET_TMP/test-gnunet-datastore-sqlite/
4
5[datastore]
6QUOTA = 10 MB
7DATABASE = sqlite
diff --git a/src/datastore/test_datastore_api_management.c b/src/datastore/test_datastore_api_management.c
deleted file mode 100644
index 175765267..000000000
--- a/src/datastore/test_datastore_api_management.c
+++ /dev/null
@@ -1,408 +0,0 @@
1/*
2 This file is part of GNUnet.
3 Copyright (C) 2004, 2005, 2006, 2007, 2009, 2011 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20/*
21 * @file datastore/test_datastore_api_management.c
22 * @brief Test for the space management functions of the datastore implementation.
23 * @author Christian Grothoff
24 */
25#include "platform.h"
26#include "gnunet_util_lib.h"
27#include "gnunet_protocols.h"
28#include "gnunet_datastore_service.h"
29#include "gnunet_datastore_plugin.h"
30#include "gnunet_testing_lib.h"
31
32
33/**
34 * How long until we give up on transmitting the message?
35 */
36#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 60)
37
38/**
39 * Number of iterations to run; must be large enough
40 * so that the quota will be exceeded!
41 */
42#define ITERATIONS 5000
43
44enum RunPhase
45{
46 RP_PUT,
47 RP_GET,
48 RP_DONE,
49 RP_GET_FAIL
50};
51
52
53struct CpsRunContext
54{
55 struct GNUNET_HashCode key;
56 int i;
57 int found;
58 const struct GNUNET_CONFIGURATION_Handle *cfg;
59 void *data;
60 enum RunPhase phase;
61};
62
63
64static struct GNUNET_DATASTORE_Handle *datastore;
65
66static struct GNUNET_TIME_Absolute now;
67
68static int ok;
69
70static const char *plugin_name;
71
72
73static size_t
74get_size (int i)
75{
76 return 8 + 8 * (i % 256);
77}
78
79
80static const void *
81get_data (int i)
82{
83 static char buf[60000];
84
85 memset (buf, i, 8 + 8 * (i % 256));
86 return buf;
87}
88
89
90static int
91get_type (int i)
92{
93 return 1;
94}
95
96
97static int
98get_priority (int i)
99{
100 return i + 1;
101}
102
103
104static int
105get_anonymity (int i)
106{
107 return i;
108}
109
110
111static struct GNUNET_TIME_Absolute
112get_expiration (int i)
113{
114 struct GNUNET_TIME_Absolute av;
115
116 av.abs_value_us = now.abs_value_us + i * 1000 * 1000LL;
117 return av;
118}
119
120
121static void
122run_continuation (void *cls);
123
124
125static void
126check_success (void *cls, int success, struct GNUNET_TIME_Absolute
127 min_expiration, const char *msg)
128{
129 struct CpsRunContext *crc = cls;
130
131 if (GNUNET_OK != success)
132 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "%s\n", msg);
133 GNUNET_assert (GNUNET_OK == success);
134 GNUNET_free (crc->data);
135 crc->data = NULL;
136 GNUNET_SCHEDULER_add_now (&run_continuation, crc);
137}
138
139
140static void
141check_value (void *cls,
142 const struct GNUNET_HashCode *key,
143 size_t size,
144 const void *data,
145 enum GNUNET_BLOCK_Type type,
146 uint32_t priority,
147 uint32_t anonymity,
148 uint32_t replication,
149 struct GNUNET_TIME_Absolute expiration,
150 uint64_t uid)
151{
152 struct CpsRunContext *crc = cls;
153 int i;
154
155 if (NULL == key)
156 {
157 crc->phase = RP_GET_FAIL;
158 GNUNET_SCHEDULER_add_now (&run_continuation, crc);
159 return;
160 }
161 i = crc->i;
162 GNUNET_assert (size == get_size (i));
163 GNUNET_assert (0 == memcmp (data, get_data (i), size));
164 GNUNET_assert (type == get_type (i));
165 GNUNET_assert (priority == get_priority (i));
166 GNUNET_assert (anonymity == get_anonymity (i));
167 GNUNET_assert (expiration.abs_value_us == get_expiration (i).abs_value_us);
168 crc->i--;
169 if (crc->i == 0)
170 crc->phase = RP_DONE;
171 GNUNET_SCHEDULER_add_now (&run_continuation, crc);
172}
173
174
175static void
176check_nothing (void *cls,
177 const struct GNUNET_HashCode *key,
178 size_t size,
179 const void *data,
180 enum GNUNET_BLOCK_Type type,
181 uint32_t priority,
182 uint32_t anonymity,
183 uint32_t replication,
184 struct GNUNET_TIME_Absolute expiration,
185 uint64_t uid)
186{
187 struct CpsRunContext *crc = cls;
188
189 GNUNET_assert (key == NULL);
190 if (0 == --crc->i)
191 crc->phase = RP_DONE;
192 GNUNET_SCHEDULER_add_now (&run_continuation, crc);
193}
194
195
196static void
197run_continuation (void *cls)
198{
199 struct CpsRunContext *crc = cls;
200
201 ok = (int) crc->phase;
202 switch (crc->phase)
203 {
204 case RP_PUT:
205 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Executing `%s' number %u\n", "PUT",
206 crc->i);
207 GNUNET_CRYPTO_hash (&crc->i, sizeof(int), &crc->key);
208 GNUNET_DATASTORE_put (datastore,
209 0,
210 &crc->key,
211 get_size (crc->i),
212 get_data (crc->i),
213 get_type (crc->i),
214 get_priority (crc->i),
215 get_anonymity (crc->i),
216 0,
217 get_expiration (crc->i),
218 1,
219 1,
220 &check_success, crc);
221 crc->i++;
222 if (crc->i == ITERATIONS)
223 {
224 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
225 "Sleeping to give datastore time to clean up\n");
226 sleep (1);
227 crc->phase = RP_GET;
228 crc->i--;
229 }
230 break;
231
232 case RP_GET:
233 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Executing `%s' number %u\n", "GET",
234 crc->i);
235 GNUNET_CRYPTO_hash (&crc->i, sizeof(int), &crc->key);
236 GNUNET_DATASTORE_get_key (datastore,
237 0,
238 false,
239 &crc->key,
240 get_type (crc->i),
241 1,
242 1,
243 &check_value,
244 crc);
245 break;
246
247 case RP_GET_FAIL:
248 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Executing `%s' number %u\n", "GET(f)",
249 crc->i);
250 GNUNET_CRYPTO_hash (&crc->i, sizeof(int), &crc->key);
251 GNUNET_DATASTORE_get_key (datastore,
252 0,
253 false,
254 &crc->key,
255 get_type (crc->i),
256 1,
257 1,
258 &check_nothing,
259 crc);
260 break;
261
262 case RP_DONE:
263 GNUNET_assert (0 == crc->i);
264 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Finished, disconnecting\n");
265 GNUNET_DATASTORE_disconnect (datastore, GNUNET_YES);
266 GNUNET_free (crc);
267 ok = 0;
268 }
269}
270
271
272static void
273run_tests (void *cls, int success, struct GNUNET_TIME_Absolute min_expiration,
274 const char *msg)
275{
276 struct CpsRunContext *crc = cls;
277
278 if (success != GNUNET_YES)
279 {
280 fprintf (stderr,
281 "Test 'put' operation failed with error `%s' database likely not setup, skipping test.\n",
282 msg);
283 GNUNET_DATASTORE_disconnect (datastore, GNUNET_YES);
284 GNUNET_free (crc);
285 return;
286 }
287 GNUNET_SCHEDULER_add_now (&run_continuation, crc);
288}
289
290
291static void
292run (void *cls,
293 const struct GNUNET_CONFIGURATION_Handle *cfg,
294 struct GNUNET_TESTING_Peer *peer)
295{
296 struct CpsRunContext *crc;
297 static struct GNUNET_HashCode zkey;
298
299 crc = GNUNET_new (struct CpsRunContext);
300 crc->cfg = cfg;
301 crc->phase = RP_PUT;
302 now = GNUNET_TIME_absolute_get ();
303 datastore = GNUNET_DATASTORE_connect (cfg);
304 if (NULL ==
305 GNUNET_DATASTORE_put (datastore,
306 0,
307 &zkey,
308 4,
309 "TEST",
310 GNUNET_BLOCK_TYPE_TEST,
311 0, 0, 0,
312 GNUNET_TIME_relative_to_absolute (
313 GNUNET_TIME_UNIT_SECONDS),
314 0,
315 1,
316 &run_tests,
317 crc))
318 {
319 fprintf (stderr, "%s", "Test 'put' operation failed.\n");
320 GNUNET_free (crc);
321 ok = 1;
322 }
323}
324
325
326/**
327 * Function called when disk utilization changes, does nothing.
328 *
329 * @param cls closure
330 * @param delta change in utilization
331 */
332static void
333ignore_payload_cb (void *cls,
334 int delta)
335{
336 /* do nothing */
337}
338
339
340/**
341 * check if plugin is actually working
342 */
343static int
344test_plugin (const char *cfg_name)
345{
346 char libname[PATH_MAX];
347 struct GNUNET_CONFIGURATION_Handle *cfg;
348 struct GNUNET_DATASTORE_PluginFunctions *api;
349 struct GNUNET_DATASTORE_PluginEnvironment env;
350
351 cfg = GNUNET_CONFIGURATION_create ();
352 if (GNUNET_OK !=
353 GNUNET_CONFIGURATION_load (cfg,
354 cfg_name))
355 {
356 GNUNET_CONFIGURATION_destroy (cfg);
357 fprintf (stderr,
358 "Failed to load configuration %s\n",
359 cfg_name);
360 return 1;
361 }
362 memset (&env, 0, sizeof(env));
363 env.cfg = cfg;
364 env.duc = &ignore_payload_cb;
365 GNUNET_snprintf (libname,
366 sizeof(libname),
367 "libgnunet_plugin_datastore_%s",
368 plugin_name);
369 api = GNUNET_PLUGIN_load (libname, &env);
370 if (NULL == api)
371 {
372 GNUNET_CONFIGURATION_destroy (cfg);
373 fprintf (stderr,
374 "Failed to load plugin `%s'\n",
375 libname);
376 return 77;
377 }
378 GNUNET_PLUGIN_unload (libname, api);
379 GNUNET_CONFIGURATION_destroy (cfg);
380 return 0;
381}
382
383
384int
385main (int argc, char *argv[])
386{
387 char cfg_name[PATH_MAX];
388 int ret;
389
390 plugin_name = GNUNET_TESTING_get_testname_from_underscore (argv[0]);
391 GNUNET_snprintf (cfg_name,
392 sizeof(cfg_name),
393 "test_datastore_api_data_%s.conf",
394 plugin_name);
395 ret = test_plugin (cfg_name);
396 if (0 != ret)
397 return ret;
398 if (0 !=
399 GNUNET_TESTING_peer_run ("test-gnunet-datastore-management",
400 cfg_name,
401 &run,
402 NULL))
403 return 1;
404 return ok;
405}
406
407
408/* end of test_datastore_api_management.c */
diff --git a/src/datastore/test_defaults.conf b/src/datastore/test_defaults.conf
deleted file mode 100644
index 1f971de8f..000000000
--- a/src/datastore/test_defaults.conf
+++ /dev/null
@@ -1,10 +0,0 @@
1@inline@ ../../contrib/conf/gnunet/no_autostart_above_core.conf
2@inline@ ../../contrib/conf/gnunet/no_forcestart.conf
3
4[datastore]
5PORT = 22654
6QUOTA = 1 MB
7START_ON_DEMAND = YES
8
9[nse]
10WORKBITS = 1
diff --git a/src/datastore/test_plugin_datastore.c b/src/datastore/test_plugin_datastore.c
deleted file mode 100644
index 9fe2462e7..000000000
--- a/src/datastore/test_plugin_datastore.c
+++ /dev/null
@@ -1,478 +0,0 @@
1/*
2 This file is part of GNUnet.
3 Copyright (C) 2011 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20/*
21 * @file test_plugin_datastore.c
22 * @brief Test database plugin directly, calling each API function once
23 * @author Christian Grothoff
24 */
25
26#include "platform.h"
27#include "gnunet_util_lib.h"
28#include "gnunet_protocols.h"
29#include "gnunet_datastore_plugin.h"
30#include "gnunet_testing_lib.h"
31
32/**
33 * Number of put operations to perform.
34 */
35#define PUT_10 10
36
37static unsigned long long stored_bytes;
38
39static unsigned long long stored_entries;
40
41static unsigned long long stored_ops;
42
43static const char *plugin_name;
44
45static int ok;
46
47enum RunPhase
48{
49 RP_ERROR = 0,
50 RP_PUT,
51 RP_GET,
52 RP_ITER_ZERO,
53 RP_REPL_GET,
54 RP_EXPI_GET,
55 RP_REMOVE,
56 RP_DROP
57};
58
59
60struct CpsRunContext
61{
62 const struct GNUNET_CONFIGURATION_Handle *cfg;
63 struct GNUNET_DATASTORE_PluginFunctions *api;
64 enum RunPhase phase;
65 unsigned int cnt;
66 unsigned int i;
67};
68
69
70/**
71 * Function called by plugins to notify us about a
72 * change in their disk utilization.
73 *
74 * @param cls closure (NULL)
75 * @param delta change in disk utilization,
76 * 0 for "reset to empty"
77 */
78static void
79disk_utilization_change_cb (void *cls, int delta)
80{
81 /* do nothing */
82}
83
84
85static void
86test (void *cls);
87
88
89/**
90 * Put continuation.
91 *
92 * @param cls closure
93 * @param key key for the item stored
94 * @param size size of the item stored
95 * @param status #GNUNET_OK or #GNUNET_SYSERROR
96 * @param msg error message on error
97 */
98static void
99put_continuation (void *cls,
100 const struct GNUNET_HashCode *key,
101 uint32_t size,
102 int status,
103 const char *msg)
104{
105 struct CpsRunContext *crc = cls;
106 static unsigned long long os;
107 unsigned long long cs;
108
109 if (GNUNET_OK != status)
110 {
111 fprintf (stderr,
112 "ERROR: `%s'\n",
113 msg);
114 }
115 else
116 {
117 crc->api->estimate_size (crc->api->cls,
118 &cs);
119 GNUNET_assert (os <= cs);
120 os = cs;
121 stored_bytes += size;
122 stored_ops++;
123 stored_entries++;
124 }
125 GNUNET_SCHEDULER_add_now (&test, crc);
126}
127
128
129static void
130gen_key (int i, struct GNUNET_HashCode *key)
131{
132 memset (key, 0, sizeof(struct GNUNET_HashCode));
133 key->bits[0] = (unsigned int) i;
134 GNUNET_CRYPTO_hash (key, sizeof(struct GNUNET_HashCode), key);
135}
136
137
138static void
139do_put (struct CpsRunContext *crc)
140{
141 char value[65536];
142 size_t size;
143 struct GNUNET_HashCode key;
144 unsigned int prio;
145 static int i;
146
147 if (PUT_10 == i)
148 {
149 i = 0;
150 crc->phase++;
151 GNUNET_SCHEDULER_add_now (&test, crc);
152 return;
153 }
154 /* most content is 32k */
155 size = 32 * 1024;
156
157 if ((0 != i) && (GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 16) ==
158 0) ) /* but some of it is less! */
159 size = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 32 * 1024);
160 size = size - (size & 7); /* always multiple of 8 */
161
162 /* generate random key */
163 gen_key (i, &key);
164 memset (value, i, size);
165 if (i > 255)
166 memset (value, i - 255, size / 2);
167 value[0] = crc->i;
168 prio = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 100);
169 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
170 "putting type %u, anon %u under key %s\n", i + 1, i,
171 GNUNET_h2s (&key));
172 crc->api->put (crc->api->cls,
173 &key,
174 false /* absent */,
175 size,
176 value, i + 1 /* type */,
177 prio,
178 i /* anonymity */,
179 0 /* replication */,
180 GNUNET_TIME_relative_to_absolute
181 (GNUNET_TIME_relative_multiply
182 (GNUNET_TIME_UNIT_MILLISECONDS,
183 60 * 60 * 60 * 1000
184 + GNUNET_CRYPTO_random_u32
185 (GNUNET_CRYPTO_QUALITY_WEAK, 1000))),
186 put_continuation,
187 crc);
188 i++;
189}
190
191
192static uint64_t guid;
193
194
195static int
196iterate_one_shot (void *cls,
197 const struct GNUNET_HashCode *key,
198 uint32_t size,
199 const void *data,
200 enum GNUNET_BLOCK_Type type,
201 uint32_t priority,
202 uint32_t anonymity,
203 uint32_t replication,
204 struct GNUNET_TIME_Absolute expiration,
205 uint64_t uid)
206{
207 struct CpsRunContext *crc = cls;
208
209 GNUNET_assert (NULL != key);
210 guid = uid;
211 crc->phase++;
212 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
213 "Found result type=%u, priority=%u, size=%u, expire=%s, key %s\n",
214 (unsigned int) type,
215 (unsigned int) priority,
216 (unsigned int) size,
217 GNUNET_STRINGS_absolute_time_to_string (expiration),
218 GNUNET_h2s (key));
219 GNUNET_SCHEDULER_add_now (&test,
220 crc);
221 return GNUNET_OK;
222}
223
224
225static void
226remove_continuation (void *cls,
227 const struct GNUNET_HashCode *key,
228 uint32_t size,
229 int status,
230 const char *msg)
231{
232 struct CpsRunContext *crc = cls;
233
234 GNUNET_assert (NULL != key);
235 GNUNET_assert (32768 == size);
236 GNUNET_assert (GNUNET_OK == status);
237 GNUNET_assert (NULL == msg);
238 crc->phase++;
239 GNUNET_SCHEDULER_add_now (&test,
240 crc);
241}
242
243
244/**
245 * Function called when the service shuts
246 * down. Unloads our datastore plugin.
247 *
248 * @param api api to unload
249 * @param cfg configuration to use
250 */
251static void
252unload_plugin (struct GNUNET_DATASTORE_PluginFunctions *api,
253 const struct GNUNET_CONFIGURATION_Handle *cfg)
254{
255 char *name;
256 char *libname;
257
258 if (GNUNET_OK !=
259 GNUNET_CONFIGURATION_get_value_string (cfg,
260 "DATASTORE",
261 "DATABASE",
262 &name))
263 {
264 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
265 _ ("No `%s' specified for `%s' in configuration!\n"),
266 "DATABASE",
267 "DATASTORE");
268 return;
269 }
270 GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", name);
271 GNUNET_break (NULL == GNUNET_PLUGIN_unload (libname, api));
272 GNUNET_free (libname);
273 GNUNET_free (name);
274}
275
276
277/**
278 * Last task run during shutdown. Disconnects us from
279 * the transport and core.
280 */
281static void
282cleaning_task (void *cls)
283{
284 struct CpsRunContext *crc = cls;
285
286 unload_plugin (crc->api, crc->cfg);
287 GNUNET_free (crc);
288}
289
290
291static void
292test (void *cls)
293{
294 struct CpsRunContext *crc = cls;
295 struct GNUNET_HashCode key;
296
297 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
298 "In phase %d, iteration %u\n", crc->phase, crc->cnt);
299 switch (crc->phase)
300 {
301 case RP_ERROR:
302 ok = 1;
303 GNUNET_break (0);
304 crc->api->drop (crc->api->cls);
305 GNUNET_SCHEDULER_add_now (&cleaning_task, crc);
306 break;
307
308 case RP_PUT:
309 do_put (crc);
310 break;
311
312 case RP_GET:
313 if (crc->cnt == 1)
314 {
315 crc->cnt = 0;
316 crc->phase++;
317 GNUNET_SCHEDULER_add_now (&test, crc);
318 break;
319 }
320 gen_key (5, &key);
321 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
322 "Looking for %s\n",
323 GNUNET_h2s (&key));
324 crc->api->get_key (crc->api->cls,
325 0,
326 false,
327 &key,
328 GNUNET_BLOCK_TYPE_ANY,
329 &iterate_one_shot,
330 crc);
331 break;
332
333 case RP_ITER_ZERO:
334 if (crc->cnt == 1)
335 {
336 crc->cnt = 0;
337 crc->phase++;
338 GNUNET_SCHEDULER_add_now (&test, crc);
339 break;
340 }
341 crc->api->get_zero_anonymity (crc->api->cls, 0, 1, &iterate_one_shot, crc);
342 break;
343
344 case RP_REPL_GET:
345 crc->api->get_replication (crc->api->cls, &iterate_one_shot, crc);
346 break;
347
348 case RP_EXPI_GET:
349 crc->api->get_expiration (crc->api->cls, &iterate_one_shot, crc);
350 break;
351
352 case RP_REMOVE:
353 {
354 struct GNUNET_HashCode key;
355 uint32_t size = 32768;
356 char value[size];
357
358 gen_key (0, &key);
359 memset (value, 0, size);
360 value[0] = crc->i;
361 crc->api->remove_key (crc->api->cls,
362 &key,
363 size,
364 value,
365 &remove_continuation,
366 crc);
367 break;
368 }
369
370 case RP_DROP:
371 crc->api->drop (crc->api->cls);
372 GNUNET_SCHEDULER_add_now (&cleaning_task, crc);
373 break;
374 }
375}
376
377
378/**
379 * Load the datastore plugin.
380 */
381static struct GNUNET_DATASTORE_PluginFunctions *
382load_plugin (const struct GNUNET_CONFIGURATION_Handle *cfg)
383{
384 static struct GNUNET_DATASTORE_PluginEnvironment env;
385 struct GNUNET_DATASTORE_PluginFunctions *ret;
386 char *name;
387 char *libname;
388
389 if (GNUNET_OK !=
390 GNUNET_CONFIGURATION_get_value_string (cfg,
391 "DATASTORE",
392 "DATABASE",
393 &name))
394 {
395 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
396 _ ("No `%s' specified for `%s' in configuration!\n"),
397 "DATABASE",
398 "DATASTORE");
399 return NULL;
400 }
401 env.cfg = cfg;
402 env.duc = &disk_utilization_change_cb;
403 env.cls = NULL;
404 GNUNET_log (GNUNET_ERROR_TYPE_INFO, _ ("Loading `%s' datastore plugin\n"),
405 name);
406 GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", name);
407 if (NULL == (ret = GNUNET_PLUGIN_load (libname, &env)))
408 {
409 fprintf (stderr, "Failed to load plugin `%s'!\n", name);
410 GNUNET_free (libname);
411 GNUNET_free (name);
412 ok = 77; /* mark test as skipped */
413 return NULL;
414 }
415 GNUNET_free (libname);
416 GNUNET_free (name);
417 return ret;
418}
419
420
421static void
422run (void *cls, char *const *args, const char *cfgfile,
423 const struct GNUNET_CONFIGURATION_Handle *c)
424{
425 struct GNUNET_DATASTORE_PluginFunctions *api;
426 struct CpsRunContext *crc;
427
428 api = load_plugin (c);
429 if (api == NULL)
430 {
431 fprintf (stderr,
432 "%s",
433 "Could not initialize plugin, assuming database not configured. Test not run!\n");
434 return;
435 }
436 crc = GNUNET_new (struct CpsRunContext);
437 crc->api = api;
438 crc->cfg = c;
439 crc->phase = RP_PUT;
440 GNUNET_SCHEDULER_add_now (&test, crc);
441}
442
443
444int
445main (int argc, char *argv[])
446{
447 char dir_name[PATH_MAX];
448 char cfg_name[PATH_MAX];
449 char *const xargv[] = {
450 "test-plugin-datastore",
451 "-c",
452 cfg_name,
453 NULL
454 };
455 static struct GNUNET_GETOPT_CommandLineOption options[] = {
456 GNUNET_GETOPT_OPTION_END
457 };
458
459 /* determine name of plugin to use */
460 plugin_name = GNUNET_TESTING_get_testname_from_underscore (argv[0]);
461 GNUNET_snprintf (dir_name, sizeof(dir_name),
462 "/tmp/test-gnunet-datastore-plugin-%s", plugin_name);
463 GNUNET_DISK_directory_remove (dir_name);
464 GNUNET_log_setup ("test-plugin-datastore",
465 "WARNING",
466 NULL);
467 GNUNET_snprintf (cfg_name, sizeof(cfg_name),
468 "test_plugin_datastore_data_%s.conf", plugin_name);
469 GNUNET_PROGRAM_run ((sizeof(xargv) / sizeof(char *)) - 1, xargv,
470 "test-plugin-datastore", "nohelp", options, &run, NULL);
471 if ((0 != ok) && (77 != ok))
472 fprintf (stderr, "Missed some testcases: %u\n", ok);
473 GNUNET_DISK_directory_remove (dir_name);
474 return ok;
475}
476
477
478/* end of test_plugin_datastore.c */
diff --git a/src/datastore/test_plugin_datastore_data_heap.conf b/src/datastore/test_plugin_datastore_data_heap.conf
deleted file mode 100644
index b1ea8ff67..000000000
--- a/src/datastore/test_plugin_datastore_data_heap.conf
+++ /dev/null
@@ -1,6 +0,0 @@
1@INLINE@ test_defaults.conf
2[PATHS]
3GNUNET_TEST_HOME = $GNUNET_TMP/test-gnunet-datastore-plugin-heap/
4
5[datastore]
6DATABASE = heap
diff --git a/src/datastore/test_plugin_datastore_data_mysql.conf b/src/datastore/test_plugin_datastore_data_mysql.conf
deleted file mode 100644
index 07d3ec58e..000000000
--- a/src/datastore/test_plugin_datastore_data_mysql.conf
+++ /dev/null
@@ -1,9 +0,0 @@
1@INLINE@ test_defaults.conf
2[PATHS]
3GNUNET_TEST_HOME = $GNUNET_TMP/test-gnunet-datastore-plugin-mysql/
4
5[datastore]
6DATABASE = mysql
7
8[datastore-mysql]
9DATABASE = gnunetcheck
diff --git a/src/datastore/test_plugin_datastore_data_postgres.conf b/src/datastore/test_plugin_datastore_data_postgres.conf
deleted file mode 100644
index d0e29437f..000000000
--- a/src/datastore/test_plugin_datastore_data_postgres.conf
+++ /dev/null
@@ -1,10 +0,0 @@
1@INLINE@ test_defaults.conf
2[PATHS]
3GNUNET_TEST_HOME = $GNUNET_TMP/test-gnunet-datastore-plugin-postgres/
4
5[datastore]
6DATABASE = postgres
7
8[datastore-postgres]
9CONFIG = dbname=gnunetcheck
10
diff --git a/src/datastore/test_plugin_datastore_data_sqlite.conf b/src/datastore/test_plugin_datastore_data_sqlite.conf
deleted file mode 100644
index ca837c77a..000000000
--- a/src/datastore/test_plugin_datastore_data_sqlite.conf
+++ /dev/null
@@ -1,4 +0,0 @@
1@INLINE@ test_defaults.conf
2[PATHS]
3GNUNET_TEST_HOME = $GNUNET_TMP/test-gnunet-datastore-plugin-sqlite/
4
generated by cgit v1.2.3 (git 2.39.1) at 2024-05-27 19:20:22 +0000