aboutsummaryrefslogtreecommitdiff
path: root/src/datastore
diff options
context:
space:
mode:
Diffstat (limited to 'src/datastore')
-rw-r--r--src/datastore/.gitignore16
-rw-r--r--src/datastore/Makefile.am327
-rw-r--r--src/datastore/datastore.conf.in32
-rw-r--r--src/datastore/datastore.h257
-rw-r--r--src/datastore/datastore_api.c1474
-rw-r--r--src/datastore/gnunet-datastore.c508
-rw-r--r--src/datastore/gnunet-service-datastore.c1647
-rw-r--r--src/datastore/perf_datastore_api.c629
-rw-r--r--src/datastore/perf_plugin_datastore.c573
-rw-r--r--src/datastore/perf_plugin_datastore_data_heap.conf7
-rw-r--r--src/datastore/perf_plugin_datastore_data_mysql.conf10
-rw-r--r--src/datastore/perf_plugin_datastore_data_postgres.conf10
-rw-r--r--src/datastore/perf_plugin_datastore_data_sqlite.conf4
-rw-r--r--src/datastore/plugin_datastore_heap.c944
-rw-r--r--src/datastore/plugin_datastore_mysql.c1203
-rw-r--r--src/datastore/plugin_datastore_postgres.c980
-rw-r--r--src/datastore/plugin_datastore_sqlite.c1374
-rw-r--r--src/datastore/plugin_datastore_template.c274
-rw-r--r--src/datastore/selectrandom.sql9
-rw-r--r--src/datastore/test_datastore_api.c732
-rw-r--r--src/datastore/test_datastore_api_data_heap.conf19
-rw-r--r--src/datastore/test_datastore_api_data_mysql.conf10
-rw-r--r--src/datastore/test_datastore_api_data_postgres.conf10
-rw-r--r--src/datastore/test_datastore_api_data_sqlite.conf7
-rw-r--r--src/datastore/test_datastore_api_management.c408
-rw-r--r--src/datastore/test_defaults.conf10
-rw-r--r--src/datastore/test_plugin_datastore.c478
-rw-r--r--src/datastore/test_plugin_datastore_data_heap.conf6
-rw-r--r--src/datastore/test_plugin_datastore_data_mysql.conf9
-rw-r--r--src/datastore/test_plugin_datastore_data_postgres.conf10
-rw-r--r--src/datastore/test_plugin_datastore_data_sqlite.conf4
31 files changed, 0 insertions, 11981 deletions
diff --git a/src/datastore/.gitignore b/src/datastore/.gitignore
deleted file mode 100644
index 51d3391b9..000000000
--- a/src/datastore/.gitignore
+++ /dev/null
@@ -1,16 +0,0 @@
1gnunet-service-datastore
2gnunet-datastore
3perf_datastore_api_heap
4perf_plugin_datastore_heap
5test_datastore_api_heap
6test_datastore_api_management_heap
7test_datastore_api_management_mysql
8test_datastore_api_management_postgres
9test_datastore_api_management_sqlite
10test_datastore_api_mysql
11test_datastore_api_postgres
12test_datastore_api_sqlite
13test_plugin_datastore_heap
14test_plugin_datastore_mysql
15test_plugin_datastore_postgres
16test_plugin_datastore_sqlite
diff --git a/src/datastore/Makefile.am b/src/datastore/Makefile.am
deleted file mode 100644
index cd1df4e5e..000000000
--- a/src/datastore/Makefile.am
+++ /dev/null
@@ -1,327 +0,0 @@
1# This Makefile.am is in the public domain
2AM_CPPFLAGS = -I$(top_srcdir)/src/include
3
4plugindir = $(libdir)/gnunet
5
6pkgcfgdir= $(pkgdatadir)/config.d/
7
8libexecdir= $(pkglibdir)/libexec/
9
10pkgcfg_DATA = \
11 datastore.conf
12
13if USE_COVERAGE
14 AM_CFLAGS = --coverage -O0
15 XLIBS = -lgcov
16endif
17
18
19lib_LTLIBRARIES = \
20 libgnunetdatastore.la
21
22libgnunetdatastore_la_SOURCES = \
23 datastore_api.c datastore.h
24libgnunetdatastore_la_LIBADD = \
25 $(top_builddir)/src/statistics/libgnunetstatistics.la \
26 $(top_builddir)/src/util/libgnunetutil.la \
27 $(GN_LIBINTL)
28libgnunetdatastore_la_LDFLAGS = \
29 $(GN_LIB_LDFLAGS) \
30 -version-info 1:0:0
31
32bin_PROGRAMS = \
33 gnunet-datastore
34
35libexec_PROGRAMS = \
36 gnunet-service-datastore
37
38gnunet_service_datastore_SOURCES = \
39 gnunet-service-datastore.c
40gnunet_service_datastore_LDADD = \
41 $(top_builddir)/src/statistics/libgnunetstatistics.la \
42 $(top_builddir)/src/util/libgnunetutil.la \
43 $(GN_LIBINTL)
44
45gnunet_datastore_SOURCES = \
46 gnunet-datastore.c
47gnunet_datastore_LDADD = \
48 libgnunetdatastore.la \
49 $(top_builddir)/src/util/libgnunetutil.la \
50 $(GN_LIBINTL)
51
52
53if HAVE_MYSQL
54 MYSQL_PLUGIN = libgnunet_plugin_datastore_mysql.la
55if HAVE_TESTING
56if HAVE_BENCHMARKS
57 MYSQL_BENCHMARKS = \
58 perf_datastore_api_mysql \
59 perf_plugin_datastore_mysql
60endif
61 MYSQL_TESTS = \
62 test_datastore_api_mysql \
63 test_datastore_api_management_mysql \
64 test_plugin_datastore_mysql \
65 $(MYSQL_BENCHMARKS)
66endif
67endif
68if HAVE_SQLITE
69 SQLITE_PLUGIN = libgnunet_plugin_datastore_sqlite.la
70if HAVE_TESTING
71if HAVE_BENCHMARKS
72 SQLITE_BENCHMARKS = \
73 perf_datastore_api_sqlite \
74 perf_plugin_datastore_sqlite
75endif
76 SQLITE_TESTS = \
77 test_datastore_api_sqlite \
78 test_datastore_api_management_sqlite \
79 test_plugin_datastore_sqlite \
80 $(SQLITE_BENCHMARKS)
81endif
82endif
83if HAVE_POSTGRESQL
84 POSTGRES_PLUGIN = libgnunet_plugin_datastore_postgres.la
85if HAVE_TESTING
86if HAVE_BENCHMARKS
87 POSTGRES_BENCHMARKS = \
88 perf_datastore_api_postgres \
89 perf_plugin_datastore_postgres
90endif
91 POSTGRES_TESTS = \
92 test_datastore_api_postgres \
93 test_datastore_api_management_postgres \
94 test_plugin_datastore_postgres \
95 $(POSTGRES_BENCHMARKS)
96endif
97endif
98
99plugin_LTLIBRARIES = \
100 $(SQLITE_PLUGIN) \
101 $(MYSQL_PLUGIN) \
102 $(POSTGRES_PLUGIN) \
103 libgnunet_plugin_datastore_heap.la
104
105# Real plugins should of course go into
106# plugin_LTLIBRARIES
107noinst_LTLIBRARIES = \
108 libgnunet_plugin_datastore_template.la
109
110
111libgnunet_plugin_datastore_sqlite_la_SOURCES = \
112 plugin_datastore_sqlite.c
113libgnunet_plugin_datastore_sqlite_la_LIBADD = \
114 $(top_builddir)/src/sq/libgnunetsq.la \
115 $(top_builddir)/src/statistics/libgnunetstatistics.la \
116 $(top_builddir)/src/util/libgnunetutil.la $(XLIBS) -lsqlite3 \
117 $(LTLIBINTL)
118libgnunet_plugin_datastore_sqlite_la_LDFLAGS = \
119 $(GN_PLUGIN_LDFLAGS)
120
121
122libgnunet_plugin_datastore_heap_la_SOURCES = \
123 plugin_datastore_heap.c
124libgnunet_plugin_datastore_heap_la_LIBADD = \
125 $(top_builddir)/src/util/libgnunetutil.la $(XLIBS) \
126 $(LTLIBINTL)
127libgnunet_plugin_datastore_heap_la_LDFLAGS = \
128 $(GN_PLUGIN_LDFLAGS)
129
130
131libgnunet_plugin_datastore_mysql_la_SOURCES = \
132 plugin_datastore_mysql.c
133libgnunet_plugin_datastore_mysql_la_LIBADD = \
134 $(top_builddir)/src/my/libgnunetmy.la \
135 $(top_builddir)/src/mysql/libgnunetmysql.la \
136 $(top_builddir)/src/statistics/libgnunetstatistics.la \
137 $(top_builddir)/src/util/libgnunetutil.la $(XLIBS) $(Z_LIBS) -lmysqlclient
138libgnunet_plugin_datastore_mysql_la_LDFLAGS = \
139 $(GN_PLUGIN_LDFLAGS) $(MYSQL_LDFLAGS) -lmysqlclient
140libgnunet_plugin_datastore_mysql_la_CPPFLAGS = \
141 $(MYSQL_CPPFLAGS) $(AM_CPPFLAGS)
142
143libgnunet_plugin_datastore_postgres_la_SOURCES = \
144 plugin_datastore_postgres.c
145libgnunet_plugin_datastore_postgres_la_LIBADD = \
146 $(top_builddir)/src/statistics/libgnunetstatistics.la \
147 $(top_builddir)/src/pq/libgnunetpq.la \
148 $(top_builddir)/src/util/libgnunetutil.la $(XLIBS) -lpq
149libgnunet_plugin_datastore_postgres_la_LDFLAGS = \
150 $(GN_PLUGIN_LDFLAGS) $(POSTGRESQL_LDFLAGS)
151libgnunet_plugin_datastore_postgres_la_CPPFLAGS = \
152 $(POSTGRESQL_CPPFLAGS) $(AM_CPPFLAGS)
153
154
155libgnunet_plugin_datastore_template_la_SOURCES = \
156 plugin_datastore_template.c
157libgnunet_plugin_datastore_template_la_LIBADD = \
158 $(top_builddir)/src/util/libgnunetutil.la $(XLIBS) \
159 $(LTLIBINTL)
160libgnunet_plugin_datastore_template_la_LDFLAGS = \
161 $(GN_PLUGIN_LDFLAGS)
162
163check_PROGRAMS = \
164 test_datastore_api_heap \
165 test_datastore_api_management_heap \
166 perf_datastore_api_heap \
167 perf_plugin_datastore_heap \
168 test_plugin_datastore_heap \
169 $(SQLITE_TESTS) \
170 $(MYSQL_TESTS) \
171 $(POSTGRES_TESTS)
172
173if ENABLE_TEST_RUN
174AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
175TESTS = $(check_PROGRAMS)
176endif
177
178test_datastore_api_heap_SOURCES = \
179 test_datastore_api.c
180test_datastore_api_heap_LDADD = \
181 $(top_builddir)/src/testing/libgnunettesting.la \
182 libgnunetdatastore.la \
183 $(top_builddir)/src/util/libgnunetutil.la
184
185test_datastore_api_management_heap_SOURCES = \
186 test_datastore_api_management.c
187test_datastore_api_management_heap_LDADD = \
188 $(top_builddir)/src/testing/libgnunettesting.la \
189 libgnunetdatastore.la \
190 $(top_builddir)/src/util/libgnunetutil.la
191
192perf_datastore_api_heap_SOURCES = \
193 perf_datastore_api.c
194perf_datastore_api_heap_LDADD = \
195 $(top_builddir)/src/testing/libgnunettesting.la \
196 libgnunetdatastore.la \
197 $(top_builddir)/src/util/libgnunetutil.la
198
199perf_plugin_datastore_heap_SOURCES = \
200 perf_plugin_datastore.c
201perf_plugin_datastore_heap_LDADD = \
202 $(top_builddir)/src/testing/libgnunettesting.la \
203 $(top_builddir)/src/util/libgnunetutil.la
204
205test_plugin_datastore_heap_SOURCES = \
206 test_plugin_datastore.c
207test_plugin_datastore_heap_LDADD = \
208 $(top_builddir)/src/testing/libgnunettesting.la \
209 $(top_builddir)/src/util/libgnunetutil.la
210
211
212test_datastore_api_sqlite_SOURCES = \
213 test_datastore_api.c
214test_datastore_api_sqlite_LDADD = \
215 $(top_builddir)/src/testing/libgnunettesting.la \
216 libgnunetdatastore.la \
217 $(top_builddir)/src/util/libgnunetutil.la
218
219test_datastore_api_management_sqlite_SOURCES = \
220 test_datastore_api_management.c
221test_datastore_api_management_sqlite_LDADD = \
222 $(top_builddir)/src/testing/libgnunettesting.la \
223 libgnunetdatastore.la \
224 $(top_builddir)/src/util/libgnunetutil.la
225
226perf_datastore_api_sqlite_SOURCES = \
227 perf_datastore_api.c
228perf_datastore_api_sqlite_LDADD = \
229 $(top_builddir)/src/testing/libgnunettesting.la \
230 libgnunetdatastore.la \
231 $(top_builddir)/src/util/libgnunetutil.la
232
233perf_plugin_datastore_sqlite_SOURCES = \
234 perf_plugin_datastore.c
235perf_plugin_datastore_sqlite_LDADD = \
236 $(top_builddir)/src/testing/libgnunettesting.la \
237 $(top_builddir)/src/util/libgnunetutil.la
238
239test_plugin_datastore_sqlite_SOURCES = \
240 test_plugin_datastore.c
241test_plugin_datastore_sqlite_LDADD = \
242 $(top_builddir)/src/testing/libgnunettesting.la \
243 $(top_builddir)/src/util/libgnunetutil.la
244
245
246test_datastore_api_mysql_SOURCES = \
247 test_datastore_api.c
248test_datastore_api_mysql_LDADD = \
249 $(top_builddir)/src/testing/libgnunettesting.la \
250 libgnunetdatastore.la \
251 $(top_builddir)/src/util/libgnunetutil.la
252
253test_datastore_api_management_mysql_SOURCES = \
254 test_datastore_api_management.c
255test_datastore_api_management_mysql_LDADD = \
256 $(top_builddir)/src/testing/libgnunettesting.la \
257 libgnunetdatastore.la \
258 $(top_builddir)/src/util/libgnunetutil.la
259
260perf_datastore_api_mysql_SOURCES = \
261 perf_datastore_api.c
262perf_datastore_api_mysql_LDADD = \
263 $(top_builddir)/src/testing/libgnunettesting.la \
264 libgnunetdatastore.la \
265 $(top_builddir)/src/util/libgnunetutil.la
266
267test_plugin_datastore_mysql_SOURCES = \
268 test_plugin_datastore.c
269test_plugin_datastore_mysql_LDADD = \
270 $(top_builddir)/src/testing/libgnunettesting.la \
271 $(top_builddir)/src/util/libgnunetutil.la
272
273perf_plugin_datastore_mysql_SOURCES = \
274 perf_plugin_datastore.c
275perf_plugin_datastore_mysql_LDADD = \
276 $(top_builddir)/src/testing/libgnunettesting.la \
277 $(top_builddir)/src/util/libgnunetutil.la
278
279
280test_datastore_api_postgres_SOURCES = \
281 test_datastore_api.c
282test_datastore_api_postgres_LDADD = \
283 $(top_builddir)/src/testing/libgnunettesting.la \
284 libgnunetdatastore.la \
285 $(top_builddir)/src/util/libgnunetutil.la
286
287test_datastore_api_management_postgres_SOURCES = \
288 test_datastore_api_management.c
289test_datastore_api_management_postgres_LDADD = \
290 $(top_builddir)/src/testing/libgnunettesting.la \
291 libgnunetdatastore.la \
292 $(top_builddir)/src/util/libgnunetutil.la
293
294perf_datastore_api_postgres_SOURCES = \
295 perf_datastore_api.c
296perf_datastore_api_postgres_LDADD = \
297 $(top_builddir)/src/testing/libgnunettesting.la \
298 libgnunetdatastore.la \
299 $(top_builddir)/src/util/libgnunetutil.la
300
301test_plugin_datastore_postgres_SOURCES = \
302 test_plugin_datastore.c
303test_plugin_datastore_postgres_LDADD = \
304 $(top_builddir)/src/testing/libgnunettesting.la \
305 $(top_builddir)/src/util/libgnunetutil.la
306
307perf_plugin_datastore_postgres_SOURCES = \
308 perf_plugin_datastore.c
309perf_plugin_datastore_postgres_LDADD = \
310 $(top_builddir)/src/testing/libgnunettesting.la \
311 $(top_builddir)/src/util/libgnunetutil.la
312
313
314EXTRA_DIST = \
315 test_defaults.conf \
316 test_datastore_api_data_sqlite.conf \
317 perf_plugin_datastore_data_sqlite.conf \
318 test_plugin_datastore_data_sqlite.conf \
319 test_datastore_api_data_heap.conf \
320 perf_plugin_datastore_data_heap.conf \
321 test_plugin_datastore_data_heap.conf \
322 test_datastore_api_data_mysql.conf \
323 perf_plugin_datastore_data_mysql.conf \
324 test_plugin_datastore_data_mysql.conf \
325 test_datastore_api_data_postgres.conf \
326 perf_plugin_datastore_data_postgres.conf \
327 test_plugin_datastore_data_postgres.conf
diff --git a/src/datastore/datastore.conf.in b/src/datastore/datastore.conf.in
deleted file mode 100644
index 21d24bb52..000000000
--- a/src/datastore/datastore.conf.in
+++ /dev/null
@@ -1,32 +0,0 @@
1[datastore]
2START_ON_DEMAND = @START_ON_DEMAND@
3UNIXPATH = $GNUNET_RUNTIME_DIR/gnunet-service-datastore.sock
4UNIX_MATCH_UID = NO
5UNIX_MATCH_GID = YES
6@UNIXONLY@ PORT = 2093
7HOSTNAME = localhost
8BINARY = gnunet-service-datastore
9ACCEPT_FROM = 127.0.0.1;
10ACCEPT_FROM6 = ::1;
11QUOTA = 5 GB
12BLOOMFILTER = $GNUNET_DATA_HOME/datastore/bloomfilter
13DATABASE = sqlite
14# DISABLE_SOCKET_FORWARDING = NO
15
16[datastore-sqlite]
17FILENAME = $GNUNET_DATA_HOME/datastore/sqlite.db
18
19[datastore-postgres]
20CONFIG = postgres:///gnunet
21
22[datastore-mysql]
23DATABASE = gnunet
24CONFIG = ~/.my.cnf
25# USER = gnunet
26# PASSWORD =
27# HOST = localhost
28# PORT = 3306
29
30
31[datastore-heap]
32HASHMAPSIZE = 1024
diff --git a/src/datastore/datastore.h b/src/datastore/datastore.h
deleted file mode 100644
index 7af926617..000000000
--- a/src/datastore/datastore.h
+++ /dev/null
@@ -1,257 +0,0 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2004, 2005, 2006, 2007, 2009 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file datastore/datastore.h
23 * @brief structs for communication between datastore service and API
24 * @author Christian Grothoff
25 */
26
27#ifndef DATASTORE_H
28#define DATASTORE_H
29
30
31#include "gnunet_util_lib.h"
32
33GNUNET_NETWORK_STRUCT_BEGIN
34
35/**
36 * Message from datastore service informing client about
37 * the current size of the datastore.
38 */
39struct ReserveMessage
40{
41 /**
42 * Type is GNUNET_MESSAGE_TYPE_DATASTORE_RESERVE.
43 */
44 struct GNUNET_MessageHeader header;
45
46 /**
47 * Number of items to reserve.
48 */
49 uint32_t entries GNUNET_PACKED;
50
51 /**
52 * Number of bytes to reserve.
53 */
54 uint64_t amount GNUNET_PACKED;
55};
56
57
58/**
59 * Message from datastore service informing client about
60 * the success or failure of a requested operation.
61 * This header is optionally followed by a variable-size,
62 * 0-terminated error message.
63 */
64struct StatusMessage
65{
66 /**
67 * Type is GNUNET_MESSAGE_TYPE_DATASTORE_STATUS.
68 */
69 struct GNUNET_MessageHeader header;
70
71 /**
72 * Status code, -1 for errors.
73 */
74 int32_t status GNUNET_PACKED;
75
76 /**
77 * Minimum expiration time required for content to be stored
78 * by the datacache at this time, zero for unknown or no limit.
79 */
80 struct GNUNET_TIME_AbsoluteNBO min_expiration;
81};
82
83
84/**
85 * Message from datastore client informing service that
86 * the remainder of the reserved bytes can now be released
87 * for other requests.
88 */
89struct ReleaseReserveMessage
90{
91 /**
92 * Type is GNUNET_MESSAGE_TYPE_DATASTORE_RELEASE_RESERVE.
93 */
94 struct GNUNET_MessageHeader header;
95
96 /**
97 * Reservation id.
98 */
99 int32_t rid GNUNET_PACKED;
100};
101
102
103/**
104 * Message to the datastore service asking about specific
105 * content.
106 */
107struct GetKeyMessage
108{
109 /**
110 * Type is #GNUNET_MESSAGE_TYPE_DATASTORE_GET_KEY.
111 */
112 struct GNUNET_MessageHeader header;
113
114 /**
115 * Desired content type. (actually an enum GNUNET_BLOCK_Type)
116 */
117 uint32_t type GNUNET_PACKED;
118
119 /**
120 * UID at which to start the search
121 */
122 uint64_t next_uid GNUNET_PACKED;
123
124 /**
125 * If true return a random result
126 */
127 uint32_t random GNUNET_PACKED;
128
129 /**
130 * Desired key.
131 */
132 struct GNUNET_HashCode key;
133};
134
135
136/**
137 * Message to the datastore service asking about specific
138 * content.
139 */
140struct GetMessage
141{
142 /**
143 * Type is #GNUNET_MESSAGE_TYPE_DATASTORE_GET.
144 */
145 struct GNUNET_MessageHeader header;
146
147 /**
148 * Desired content type. (actually an enum GNUNET_BLOCK_Type)
149 */
150 uint32_t type GNUNET_PACKED;
151
152 /**
153 * UID at which to start the search
154 */
155 uint64_t next_uid GNUNET_PACKED;
156
157 /**
158 * If true return a random result
159 */
160 uint32_t random GNUNET_PACKED;
161};
162
163
164/**
165 * Message to the datastore service asking about zero
166 * anonymity content.
167 */
168struct GetZeroAnonymityMessage
169{
170 /**
171 * Type is GNUNET_MESSAGE_TYPE_DATASTORE_GET_ZERO_ANONYMITY.
172 */
173 struct GNUNET_MessageHeader header;
174
175 /**
176 * Desired content type (actually an enum GNUNET_BLOCK_Type)
177 */
178 uint32_t type GNUNET_PACKED;
179
180 /**
181 * UID at which to start the search
182 */
183 uint64_t next_uid GNUNET_PACKED;
184};
185
186
187/**
188 * Message transmitting content from or to the datastore
189 * service.
190 */
191struct DataMessage
192{
193 /**
194 * Type is either GNUNET_MESSAGE_TYPE_DATASTORE_PUT,
195 * GNUNET_MESSAGE_TYPE_DATASTORE_REMOVE or
196 * GNUNET_MESSAGE_TYPE_DATASTORE_DATA. Depending on the message
197 * type, some fields may simply have values of zero.
198 */
199 struct GNUNET_MessageHeader header;
200
201 /**
202 * Reservation ID to use; use zero for none.
203 */
204 uint32_t rid GNUNET_PACKED;
205
206 /**
207 * Number of bytes in the item (NBO).
208 */
209 uint32_t size GNUNET_PACKED;
210
211 /**
212 * Type of the item (NBO), zero for remove, (actually an enum GNUNET_BLOCK_Type)
213 */
214 uint32_t type GNUNET_PACKED;
215
216 /**
217 * Priority of the item (NBO), zero for remove.
218 */
219 uint32_t priority GNUNET_PACKED;
220
221 /**
222 * Desired anonymity level (NBO), zero for remove.
223 */
224 uint32_t anonymity GNUNET_PACKED;
225
226 /**
227 * Desired replication level.
228 */
229 uint32_t replication GNUNET_PACKED;
230
231 /**
232 * For alignment.
233 */
234 uint32_t reserved GNUNET_PACKED;
235
236 /**
237 * Unique ID for the content (can be used for UPDATE);
238 * can be zero for remove (which indicates that
239 * the datastore should use whatever UID matches
240 * the key and content).
241 */
242 uint64_t uid;
243
244 /**
245 * Expiration time (NBO); zero for remove.
246 */
247 struct GNUNET_TIME_AbsoluteNBO expiration;
248
249 /**
250 * Key under which the item can be found.
251 */
252 struct GNUNET_HashCode key;
253};
254GNUNET_NETWORK_STRUCT_END
255
256
257#endif
diff --git a/src/datastore/datastore_api.c b/src/datastore/datastore_api.c
deleted file mode 100644
index a49bc8586..000000000
--- a/src/datastore/datastore_api.c
+++ /dev/null
@@ -1,1474 +0,0 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2004-2013, 2016 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file datastore/datastore_api.c
23 * @brief Management for the datastore for files stored on a GNUnet node. Implements
24 * a priority queue for requests
25 * @author Christian Grothoff
26 */
27#include "platform.h"
28#include "gnunet_arm_service.h"
29#include "gnunet_constants.h"
30#include "gnunet_datastore_service.h"
31#include "gnunet_statistics_service.h"
32#include "datastore.h"
33
34#define LOG(kind, ...) GNUNET_log_from (kind, "datastore-api", __VA_ARGS__)
35
36#define DELAY_WARN_TIMEOUT GNUNET_TIME_UNIT_MINUTES
37
38/**
39 * Collect an instance number of statistics? May cause excessive IPC.
40 */
41#define INSANE_STATISTICS GNUNET_NO
42
43/**
44 * If a client stopped asking for more results, how many more do
45 * we receive from the DB before killing the connection? Trade-off
46 * between re-doing TCP handshakes and (needlessly) receiving
47 * useless results.
48 */
49#define MAX_EXCESS_RESULTS 8
50
51/**
52 * Context for processing status messages.
53 */
54struct StatusContext
55{
56 /**
57 * Continuation to call with the status.
58 */
59 GNUNET_DATASTORE_ContinuationWithStatus cont;
60
61 /**
62 * Closure for @e cont.
63 */
64 void *cont_cls;
65};
66
67
68/**
69 * Context for processing result messages.
70 */
71struct ResultContext
72{
73 /**
74 * Function to call with the result.
75 */
76 GNUNET_DATASTORE_DatumProcessor proc;
77
78 /**
79 * Closure for @e proc.
80 */
81 void *proc_cls;
82};
83
84
85/**
86 * Context for a queue operation.
87 */
88union QueueContext
89{
90 struct StatusContext sc;
91
92 struct ResultContext rc;
93};
94
95
96/**
97 * Entry in our priority queue.
98 */
99struct GNUNET_DATASTORE_QueueEntry
100{
101 /**
102 * This is a linked list.
103 */
104 struct GNUNET_DATASTORE_QueueEntry *next;
105
106 /**
107 * This is a linked list.
108 */
109 struct GNUNET_DATASTORE_QueueEntry *prev;
110
111 /**
112 * Handle to the master context.
113 */
114 struct GNUNET_DATASTORE_Handle *h;
115
116 /**
117 * Function to call after transmission of the request.
118 */
119 GNUNET_DATASTORE_ContinuationWithStatus cont;
120
121 /**
122 * Closure for @e cont.
123 */
124 void *cont_cls;
125
126 /**
127 * Context for the operation.
128 */
129 union QueueContext qc;
130
131 /**
132 * Envelope of the request to transmit, NULL after
133 * transmission.
134 */
135 struct GNUNET_MQ_Envelope *env;
136
137 /**
138 * Task we run if this entry stalls the queue and we
139 * need to warn the user.
140 */
141 struct GNUNET_SCHEDULER_Task *delay_warn_task;
142
143 /**
144 * Priority in the queue.
145 */
146 unsigned int priority;
147
148 /**
149 * Maximum allowed length of queue (otherwise
150 * this request should be discarded).
151 */
152 unsigned int max_queue;
153
154 /**
155 * Expected response type.
156 */
157 uint16_t response_type;
158};
159
160
161/**
162 * Handle to the datastore service.
163 */
164struct GNUNET_DATASTORE_Handle
165{
166 /**
167 * Our configuration.
168 */
169 const struct GNUNET_CONFIGURATION_Handle *cfg;
170
171 /**
172 * Current connection to the datastore service.
173 */
174 struct GNUNET_MQ_Handle *mq;
175
176 /**
177 * Handle for statistics.
178 */
179 struct GNUNET_STATISTICS_Handle *stats;
180
181 /**
182 * Current head of priority queue.
183 */
184 struct GNUNET_DATASTORE_QueueEntry *queue_head;
185
186 /**
187 * Current tail of priority queue.
188 */
189 struct GNUNET_DATASTORE_QueueEntry *queue_tail;
190
191 /**
192 * Task for trying to reconnect.
193 */
194 struct GNUNET_SCHEDULER_Task *reconnect_task;
195
196 /**
197 * How quickly should we retry? Used for exponential back-off on
198 * connect-errors.
199 */
200 struct GNUNET_TIME_Relative retry_time;
201
202 /**
203 * Number of entries in the queue.
204 */
205 unsigned int queue_size;
206
207 /**
208 * Number of results we're receiving for the current query
209 * after application stopped to care. Used to determine when
210 * to reset the connection.
211 */
212 unsigned int result_count;
213
214 /**
215 * We should ignore the next message(s) from the service.
216 */
217 unsigned int skip_next_messages;
218};
219
220
221/**
222 * Try reconnecting to the datastore service.
223 *
224 * @param cls the `struct GNUNET_DATASTORE_Handle`
225 */
226static void
227try_reconnect (void *cls);
228
229
230/**
231 * Disconnect from the service and then try reconnecting to the datastore service
232 * after some delay.
233 *
234 * @param h handle to datastore to disconnect and reconnect
235 */
236static void
237do_disconnect (struct GNUNET_DATASTORE_Handle *h)
238{
239 if (NULL == h->mq)
240 {
241 GNUNET_break (0);
242 return;
243 }
244 GNUNET_MQ_destroy (h->mq);
245 h->mq = NULL;
246 h->skip_next_messages = 0;
247 h->reconnect_task
248 = GNUNET_SCHEDULER_add_delayed (h->retry_time,
249 &try_reconnect,
250 h);
251}
252
253
254/**
255 * Free a queue entry. Removes the given entry from the
256 * queue and releases associated resources. Does NOT
257 * call the callback.
258 *
259 * @param qe entry to free.
260 */
261static void
262free_queue_entry (struct GNUNET_DATASTORE_QueueEntry *qe)
263{
264 struct GNUNET_DATASTORE_Handle *h = qe->h;
265
266 GNUNET_CONTAINER_DLL_remove (h->queue_head,
267 h->queue_tail,
268 qe);
269 h->queue_size--;
270 if (NULL != qe->env)
271 GNUNET_MQ_discard (qe->env);
272 if (NULL != qe->delay_warn_task)
273 GNUNET_SCHEDULER_cancel (qe->delay_warn_task);
274 GNUNET_free (qe);
275}
276
277
278/**
279 * Task that logs an error after some time.
280 *
281 * @param qe `struct GNUNET_DATASTORE_QueueEntry` about which the error is
282 */
283static void
284delay_warning (void *cls)
285{
286 struct GNUNET_DATASTORE_QueueEntry *qe = cls;
287
288 qe->delay_warn_task = NULL;
289 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
290 "Request %p of type %u at head of datastore queue for more than %s\n",
291 qe,
292 (unsigned int) qe->response_type,
293 GNUNET_STRINGS_relative_time_to_string (DELAY_WARN_TIMEOUT,
294 GNUNET_YES));
295 qe->delay_warn_task = GNUNET_SCHEDULER_add_delayed (DELAY_WARN_TIMEOUT,
296 &delay_warning,
297 qe);
298}
299
300
301/**
302 * Handle error in sending drop request to datastore.
303 *
304 * @param cls closure with the datastore handle
305 * @param error error code
306 */
307static void
308mq_error_handler (void *cls,
309 enum GNUNET_MQ_Error error)
310{
311 struct GNUNET_DATASTORE_Handle *h = cls;
312 struct GNUNET_DATASTORE_QueueEntry *qe;
313
314 LOG (GNUNET_ERROR_TYPE_DEBUG,
315 "MQ error, reconnecting to DATASTORE\n");
316 do_disconnect (h);
317 qe = h->queue_head;
318 if (NULL == qe)
319 return;
320 if (NULL != qe->delay_warn_task)
321 {
322 GNUNET_SCHEDULER_cancel (qe->delay_warn_task);
323 qe->delay_warn_task = NULL;
324 }
325 if (NULL == qe->env)
326 {
327 union QueueContext qc = qe->qc;
328 uint16_t rt = qe->response_type;
329
330 LOG (GNUNET_ERROR_TYPE_DEBUG,
331 "Failed to receive response from database.\n");
332 free_queue_entry (qe);
333 switch (rt)
334 {
335 case GNUNET_MESSAGE_TYPE_DATASTORE_STATUS:
336 if (NULL != qc.sc.cont)
337 qc.sc.cont (qc.sc.cont_cls,
338 GNUNET_SYSERR,
339 GNUNET_TIME_UNIT_ZERO_ABS,
340 _ ("DATASTORE disconnected"));
341 break;
342
343 case GNUNET_MESSAGE_TYPE_DATASTORE_DATA:
344 if (NULL != qc.rc.proc)
345 qc.rc.proc (qc.rc.proc_cls,
346 NULL,
347 0,
348 NULL,
349 0,
350 0,
351 0,
352 0,
353 GNUNET_TIME_UNIT_ZERO_ABS,
354 0);
355 break;
356
357 default:
358 GNUNET_break (0);
359 }
360 }
361}
362
363
364/**
365 * Connect to the datastore service.
366 *
367 * @param cfg configuration to use
368 * @return handle to use to access the service
369 */
370struct GNUNET_DATASTORE_Handle *
371GNUNET_DATASTORE_connect (const struct GNUNET_CONFIGURATION_Handle *cfg)
372{
373 struct GNUNET_DATASTORE_Handle *h;
374
375 LOG (GNUNET_ERROR_TYPE_DEBUG,
376 "Establishing DATASTORE connection!\n");
377 h = GNUNET_new (struct GNUNET_DATASTORE_Handle);
378 h->cfg = cfg;
379 try_reconnect (h);
380 if (NULL == h->mq)
381 {
382 GNUNET_free (h);
383 return NULL;
384 }
385 h->stats = GNUNET_STATISTICS_create ("datastore-api",
386 cfg);
387 return h;
388}
389
390
391/**
392 * Task used by to disconnect from the datastore after
393 * we send the #GNUNET_MESSAGE_TYPE_DATASTORE_DROP message.
394 *
395 * @param cls the datastore handle
396 */
397static void
398disconnect_after_drop (void *cls)
399{
400 struct GNUNET_DATASTORE_Handle *h = cls;
401
402 LOG (GNUNET_ERROR_TYPE_DEBUG,
403 "Drop sent, disconnecting\n");
404 GNUNET_DATASTORE_disconnect (h,
405 GNUNET_NO);
406}
407
408
409/**
410 * Handle error in sending drop request to datastore.
411 *
412 * @param cls closure with the datastore handle
413 * @param error error code
414 */
415static void
416disconnect_on_mq_error (void *cls,
417 enum GNUNET_MQ_Error error)
418{
419 struct GNUNET_DATASTORE_Handle *h = cls;
420
421 LOG (GNUNET_ERROR_TYPE_ERROR,
422 "Failed to ask datastore to drop tables\n");
423 GNUNET_DATASTORE_disconnect (h,
424 GNUNET_NO);
425}
426
427
428/**
429 * Disconnect from the datastore service (and free
430 * associated resources).
431 *
432 * @param h handle to the datastore
433 * @param drop set to #GNUNET_YES to delete all data in datastore (!)
434 */
435void
436GNUNET_DATASTORE_disconnect (struct GNUNET_DATASTORE_Handle *h,
437 int drop)
438{
439 struct GNUNET_DATASTORE_QueueEntry *qe;
440
441 LOG (GNUNET_ERROR_TYPE_DEBUG,
442 "Datastore disconnect\n");
443 if (NULL != h->mq)
444 {
445 GNUNET_MQ_destroy (h->mq);
446 h->mq = NULL;
447 }
448 if (NULL != h->reconnect_task)
449 {
450 GNUNET_SCHEDULER_cancel (h->reconnect_task);
451 h->reconnect_task = NULL;
452 }
453 while (NULL != (qe = h->queue_head))
454 {
455 switch (qe->response_type)
456 {
457 case GNUNET_MESSAGE_TYPE_DATASTORE_STATUS:
458 if (NULL != qe->qc.sc.cont)
459 qe->qc.sc.cont (qe->qc.sc.cont_cls,
460 GNUNET_SYSERR,
461 GNUNET_TIME_UNIT_ZERO_ABS,
462 _ ("Disconnected from DATASTORE"));
463 break;
464
465 case GNUNET_MESSAGE_TYPE_DATASTORE_DATA:
466 if (NULL != qe->qc.rc.proc)
467 qe->qc.rc.proc (qe->qc.rc.proc_cls,
468 NULL,
469 0,
470 NULL,
471 0,
472 0,
473 0,
474 0,
475 GNUNET_TIME_UNIT_ZERO_ABS,
476 0);
477 break;
478
479 default:
480 GNUNET_break (0);
481 }
482 free_queue_entry (qe);
483 }
484 if (GNUNET_YES == drop)
485 {
486 LOG (GNUNET_ERROR_TYPE_DEBUG,
487 "Re-connecting to issue DROP!\n");
488 GNUNET_assert (NULL == h->mq);
489 h->mq = GNUNET_CLIENT_connect (h->cfg,
490 "datastore",
491 NULL,
492 &disconnect_on_mq_error,
493 h);
494 if (NULL != h->mq)
495 {
496 struct GNUNET_MessageHeader *hdr;
497 struct GNUNET_MQ_Envelope *env;
498
499 env = GNUNET_MQ_msg (hdr,
500 GNUNET_MESSAGE_TYPE_DATASTORE_DROP);
501 GNUNET_MQ_notify_sent (env,
502 &disconnect_after_drop,
503 h);
504 GNUNET_MQ_send (h->mq,
505 env);
506 return;
507 }
508 GNUNET_break (0);
509 }
510 GNUNET_STATISTICS_destroy (h->stats,
511 GNUNET_NO);
512 h->stats = NULL;
513 GNUNET_free (h);
514}
515
516
517/**
518 * Create a new entry for our priority queue (and possibly discard other entries if
519 * the queue is getting too long).
520 *
521 * @param h handle to the datastore
522 * @param env envelope with the message to queue
523 * @param queue_priority priority of the entry
524 * @param max_queue_size at what queue size should this request be dropped
525 * (if other requests of higher priority are in the queue)
526 * @param expected_type which type of response do we expect,
527 * #GNUNET_MESSAGE_TYPE_DATASTORE_STATUS or
528 * #GNUNET_MESSAGE_TYPE_DATASTORE_DATA
529 * @param qc client context (NOT a closure for @a response_proc)
530 * @return NULL if the queue is full
531 */
532static struct GNUNET_DATASTORE_QueueEntry *
533make_queue_entry (struct GNUNET_DATASTORE_Handle *h,
534 struct GNUNET_MQ_Envelope *env,
535 unsigned int queue_priority,
536 unsigned int max_queue_size,
537 uint16_t expected_type,
538 const union QueueContext *qc)
539{
540 struct GNUNET_DATASTORE_QueueEntry *qe;
541 struct GNUNET_DATASTORE_QueueEntry *pos;
542 unsigned int c;
543
544 if ((NULL != h->queue_tail) &&
545 (h->queue_tail->priority >= queue_priority))
546 {
547 c = h->queue_size;
548 pos = NULL;
549 }
550 else
551 {
552 c = 0;
553 pos = h->queue_head;
554 }
555 while ((NULL != pos) &&
556 (c < max_queue_size) &&
557 (pos->priority >= queue_priority))
558 {
559 c++;
560 pos = pos->next;
561 }
562 if (c >= max_queue_size)
563 {
564 GNUNET_STATISTICS_update (h->stats,
565 gettext_noop ("# queue overflows"),
566 1,
567 GNUNET_NO);
568 GNUNET_MQ_discard (env);
569 return NULL;
570 }
571 qe = GNUNET_new (struct GNUNET_DATASTORE_QueueEntry);
572 qe->h = h;
573 qe->env = env;
574 qe->response_type = expected_type;
575 qe->qc = *qc;
576 qe->priority = queue_priority;
577 qe->max_queue = max_queue_size;
578 if (NULL == pos)
579 {
580 /* append at the tail */
581 pos = h->queue_tail;
582 }
583 else
584 {
585 pos = pos->prev;
586 /* do not insert at HEAD if HEAD query was already
587 * transmitted and we are still receiving replies! */
588 if ((NULL == pos) &&
589 (NULL == h->queue_head->env))
590 pos = h->queue_head;
591 }
592 c++;
593#if INSANE_STATISTICS
594 GNUNET_STATISTICS_update (h->stats,
595 gettext_noop ("# queue entries created"),
596 1,
597 GNUNET_NO);
598#endif
599 GNUNET_CONTAINER_DLL_insert_after (h->queue_head,
600 h->queue_tail,
601 pos,
602 qe);
603 h->queue_size++;
604 return qe;
605}
606
607
608/**
609 * Process entries in the queue (or do nothing if we are already
610 * doing so).
611 *
612 * @param h handle to the datastore
613 */
614static void
615process_queue (struct GNUNET_DATASTORE_Handle *h)
616{
617 struct GNUNET_DATASTORE_QueueEntry *qe;
618
619 if (NULL == (qe = h->queue_head))
620 {
621 /* no entry in queue */
622 LOG (GNUNET_ERROR_TYPE_DEBUG,
623 "Queue empty\n");
624 return;
625 }
626 if (NULL == qe->env)
627 {
628 /* waiting for replies */
629 LOG (GNUNET_ERROR_TYPE_DEBUG,
630 "Head request already transmitted\n");
631 return;
632 }
633 if (NULL == h->mq)
634 {
635 /* waiting for reconnect */
636 LOG (GNUNET_ERROR_TYPE_DEBUG,
637 "Not connected\n");
638 return;
639 }
640 GNUNET_assert (NULL == qe->delay_warn_task);
641 qe->delay_warn_task = GNUNET_SCHEDULER_add_delayed (DELAY_WARN_TIMEOUT,
642 &delay_warning,
643 qe);
644 GNUNET_MQ_send (h->mq,
645 qe->env);
646 qe->env = NULL;
647}
648
649
650/**
651 * Get the entry at the head of the message queue.
652 *
653 * @param h handle to the datastore
654 * @param response_type the expected response type
655 * @return the queue entry
656 */
657static struct GNUNET_DATASTORE_QueueEntry *
658get_queue_head (struct GNUNET_DATASTORE_Handle *h,
659 uint16_t response_type)
660{
661 struct GNUNET_DATASTORE_QueueEntry *qe;
662
663 if (h->skip_next_messages > 0)
664 {
665 h->skip_next_messages--;
666 process_queue (h);
667 return NULL;
668 }
669 qe = h->queue_head;
670 if (NULL == qe)
671 {
672 GNUNET_break (0);
673 do_disconnect (h);
674 return NULL;
675 }
676 if (NULL != qe->env)
677 {
678 GNUNET_break (0);
679 do_disconnect (h);
680 return NULL;
681 }
682 if (response_type != qe->response_type)
683 {
684 GNUNET_break (0);
685 do_disconnect (h);
686 return NULL;
687 }
688 return qe;
689}
690
691
692/**
693 * Function called to check status message from the service.
694 *
695 * @param cls closure
696 * @param sm status message received
697 * @return #GNUNET_OK if the message is well-formed
698 */
699static int
700check_status (void *cls,
701 const struct StatusMessage *sm)
702{
703 uint16_t msize = ntohs (sm->header.size) - sizeof(*sm);
704 int32_t status = ntohl (sm->status);
705
706 if (msize > 0)
707 {
708 const char *emsg = (const char *) &sm[1];
709
710 if ('\0' != emsg[msize - 1])
711 {
712 GNUNET_break (0);
713 return GNUNET_SYSERR;
714 }
715 }
716 else if (GNUNET_SYSERR == status)
717 {
718 GNUNET_break (0);
719 return GNUNET_SYSERR;
720 }
721 return GNUNET_OK;
722}
723
724
725/**
726 * Function called to handle status message from the service.
727 *
728 * @param cls closure
729 * @param sm status message received
730 */
731static void
732handle_status (void *cls,
733 const struct StatusMessage *sm)
734{
735 struct GNUNET_DATASTORE_Handle *h = cls;
736 struct GNUNET_DATASTORE_QueueEntry *qe;
737 struct StatusContext rc;
738 const char *emsg;
739 int32_t status = ntohl (sm->status);
740
741 qe = get_queue_head (h,
742 GNUNET_MESSAGE_TYPE_DATASTORE_STATUS);
743 if (NULL == qe)
744 return;
745 rc = qe->qc.sc;
746 free_queue_entry (qe);
747 if (ntohs (sm->header.size) > sizeof(struct StatusMessage))
748 emsg = (const char *) &sm[1];
749 else
750 emsg = NULL;
751 LOG (GNUNET_ERROR_TYPE_DEBUG,
752 "Received status %d/%s\n",
753 (int) status,
754 emsg);
755 GNUNET_STATISTICS_update (h->stats,
756 gettext_noop ("# status messages received"),
757 1,
758 GNUNET_NO);
759 h->retry_time = GNUNET_TIME_UNIT_ZERO;
760 process_queue (h);
761 if (NULL != rc.cont)
762 rc.cont (rc.cont_cls,
763 status,
764 GNUNET_TIME_absolute_ntoh (sm->min_expiration),
765 emsg);
766}
767
768
769/**
770 * Check data message we received from the service.
771 *
772 * @param cls closure with the `struct GNUNET_DATASTORE_Handle *`
773 * @param dm message received
774 */
775static int
776check_data (void *cls,
777 const struct DataMessage *dm)
778{
779 uint16_t msize = ntohs (dm->header.size) - sizeof(*dm);
780
781 if (msize != ntohl (dm->size))
782 {
783 GNUNET_break (0);
784 return GNUNET_SYSERR;
785 }
786 return GNUNET_OK;
787}
788
789
790/**
791 * Handle data message we got from the service.
792 *
793 * @param cls closure with the `struct GNUNET_DATASTORE_Handle *`
794 * @param dm message received
795 */
796static void
797handle_data (void *cls,
798 const struct DataMessage *dm)
799{
800 struct GNUNET_DATASTORE_Handle *h = cls;
801 struct GNUNET_DATASTORE_QueueEntry *qe;
802 struct ResultContext rc;
803
804 qe = get_queue_head (h,
805 GNUNET_MESSAGE_TYPE_DATASTORE_DATA);
806 if (NULL == qe)
807 return;
808#if INSANE_STATISTICS
809 GNUNET_STATISTICS_update (h->stats,
810 gettext_noop ("# Results received"),
811 1,
812 GNUNET_NO);
813#endif
814 LOG (GNUNET_ERROR_TYPE_DEBUG,
815 "Received result %llu with type %u and size %u with key %s\n",
816 (unsigned long long) GNUNET_ntohll (dm->uid),
817 ntohl (dm->type),
818 ntohl (dm->size),
819 GNUNET_h2s (&dm->key));
820 rc = qe->qc.rc;
821 free_queue_entry (qe);
822 h->retry_time = GNUNET_TIME_UNIT_ZERO;
823 process_queue (h);
824 if (NULL != rc.proc)
825 rc.proc (rc.proc_cls,
826 &dm->key,
827 ntohl (dm->size),
828 &dm[1],
829 ntohl (dm->type),
830 ntohl (dm->priority),
831 ntohl (dm->anonymity),
832 ntohl (dm->replication),
833 GNUNET_TIME_absolute_ntoh (dm->expiration),
834 GNUNET_ntohll (dm->uid));
835}
836
837
838/**
839 * Type of a function to call when we receive a
840 * #GNUNET_MESSAGE_TYPE_DATASTORE_DATA_END message from the service.
841 *
842 * @param cls closure with the `struct GNUNET_DATASTORE_Handle *`
843 * @param msg message received
844 */
845static void
846handle_data_end (void *cls,
847 const struct GNUNET_MessageHeader *msg)
848{
849 struct GNUNET_DATASTORE_Handle *h = cls;
850 struct GNUNET_DATASTORE_QueueEntry *qe;
851 struct ResultContext rc;
852
853 qe = get_queue_head (h,
854 GNUNET_MESSAGE_TYPE_DATASTORE_DATA);
855 if (NULL == qe)
856 return;
857 rc = qe->qc.rc;
858 free_queue_entry (qe);
859 LOG (GNUNET_ERROR_TYPE_DEBUG,
860 "Received end of result set, new queue size is %u\n",
861 h->queue_size);
862 h->retry_time = GNUNET_TIME_UNIT_ZERO;
863 h->result_count = 0;
864 process_queue (h);
865 /* signal end of iteration */
866 if (NULL != rc.proc)
867 rc.proc (rc.proc_cls,
868 NULL,
869 0,
870 NULL,
871 0,
872 0,
873 0,
874 0,
875 GNUNET_TIME_UNIT_ZERO_ABS,
876 0);
877}
878
879
880/**
881 * Try reconnecting to the datastore service.
882 *
883 * @param cls the `struct GNUNET_DATASTORE_Handle`
884 */
885static void
886try_reconnect (void *cls)
887{
888 struct GNUNET_DATASTORE_Handle *h = cls;
889 struct GNUNET_MQ_MessageHandler handlers[] = {
890 GNUNET_MQ_hd_var_size (status,
891 GNUNET_MESSAGE_TYPE_DATASTORE_STATUS,
892 struct StatusMessage,
893 h),
894 GNUNET_MQ_hd_var_size (data,
895 GNUNET_MESSAGE_TYPE_DATASTORE_DATA,
896 struct DataMessage,
897 h),
898 GNUNET_MQ_hd_fixed_size (data_end,
899 GNUNET_MESSAGE_TYPE_DATASTORE_DATA_END,
900 struct GNUNET_MessageHeader,
901 h),
902 GNUNET_MQ_handler_end ()
903 };
904
905 h->retry_time = GNUNET_TIME_STD_BACKOFF (h->retry_time);
906 h->reconnect_task = NULL;
907 GNUNET_assert (NULL == h->mq);
908 h->mq = GNUNET_CLIENT_connect (h->cfg,
909 "datastore",
910 handlers,
911 &mq_error_handler,
912 h);
913 if (NULL == h->mq)
914 return;
915 GNUNET_STATISTICS_update (h->stats,
916 gettext_noop (
917 "# datastore connections (re)created"),
918 1,
919 GNUNET_NO);
920 LOG (GNUNET_ERROR_TYPE_DEBUG,
921 "Reconnected to DATASTORE\n");
922 process_queue (h);
923}
924
925
926/**
927 * Dummy continuation used to do nothing (but be non-zero).
928 *
929 * @param cls closure
930 * @param result result
931 * @param min_expiration expiration time
932 * @param emsg error message
933 */
934static void
935drop_status_cont (void *cls,
936 int32_t result,
937 struct GNUNET_TIME_Absolute min_expiration,
938 const char *emsg)
939{
940 /* do nothing */
941}
942
943
944/**
945 * Store an item in the datastore. If the item is already present,
946 * the priorities are summed up and the higher expiration time and
947 * lower anonymity level is used.
948 *
949 * @param h handle to the datastore
950 * @param rid reservation ID to use (from "reserve"); use 0 if no
951 * prior reservation was made
952 * @param key key for the value
953 * @param size number of bytes in data
954 * @param data content stored
955 * @param type type of the content
956 * @param priority priority of the content
957 * @param anonymity anonymity-level for the content
958 * @param replication how often should the content be replicated to other peers?
959 * @param expiration expiration time for the content
960 * @param queue_priority ranking of this request in the priority queue
961 * @param max_queue_size at what queue size should this request be dropped
962 * (if other requests of higher priority are in the queue)
963 * @param cont continuation to call when done
964 * @param cont_cls closure for @a cont
965 * @return NULL if the entry was not queued, otherwise a handle that can be used to
966 * cancel; note that even if NULL is returned, the callback will be invoked
967 * (or rather, will already have been invoked)
968 */
969struct GNUNET_DATASTORE_QueueEntry *
970GNUNET_DATASTORE_put (struct GNUNET_DATASTORE_Handle *h,
971 uint32_t rid,
972 const struct GNUNET_HashCode *key,
973 size_t size,
974 const void *data,
975 enum GNUNET_BLOCK_Type type,
976 uint32_t priority,
977 uint32_t anonymity,
978 uint32_t replication,
979 struct GNUNET_TIME_Absolute expiration,
980 unsigned int queue_priority,
981 unsigned int max_queue_size,
982 GNUNET_DATASTORE_ContinuationWithStatus cont,
983 void *cont_cls)
984{
985 struct GNUNET_DATASTORE_QueueEntry *qe;
986 struct GNUNET_MQ_Envelope *env;
987 struct DataMessage *dm;
988 union QueueContext qc;
989
990 if (size + sizeof(*dm) >= GNUNET_MAX_MESSAGE_SIZE)
991 {
992 GNUNET_break (0);
993 return NULL;
994 }
995
996 LOG (GNUNET_ERROR_TYPE_DEBUG,
997 "Asked to put %lu bytes of data under key `%s' for %s\n",
998 (unsigned long) size,
999 GNUNET_h2s (key),
1000 GNUNET_STRINGS_relative_time_to_string (
1001 GNUNET_TIME_absolute_get_remaining (expiration),
1002 GNUNET_YES));
1003 env = GNUNET_MQ_msg_extra (dm,
1004 size,
1005 GNUNET_MESSAGE_TYPE_DATASTORE_PUT);
1006 dm->rid = htonl (rid);
1007 dm->size = htonl ((uint32_t) size);
1008 dm->type = htonl (type);
1009 dm->priority = htonl (priority);
1010 dm->anonymity = htonl (anonymity);
1011 dm->replication = htonl (replication);
1012 dm->expiration = GNUNET_TIME_absolute_hton (expiration);
1013 dm->key = *key;
1014 GNUNET_memcpy (&dm[1],
1015 data,
1016 size);
1017 qc.sc.cont = cont;
1018 qc.sc.cont_cls = cont_cls;
1019 qe = make_queue_entry (h,
1020 env,
1021 queue_priority,
1022 max_queue_size,
1023 GNUNET_MESSAGE_TYPE_DATASTORE_STATUS,
1024 &qc);
1025 if (NULL == qe)
1026 {
1027 LOG (GNUNET_ERROR_TYPE_DEBUG,
1028 "Could not create queue entry for PUT\n");
1029 return NULL;
1030 }
1031 GNUNET_STATISTICS_update (h->stats,
1032 gettext_noop ("# PUT requests executed"),
1033 1,
1034 GNUNET_NO);
1035 process_queue (h);
1036 return qe;
1037}
1038
1039
1040/**
1041 * Reserve space in the datastore. This function should be used
1042 * to avoid "out of space" failures during a longer sequence of "put"
1043 * operations (for example, when a file is being inserted).
1044 *
1045 * @param h handle to the datastore
1046 * @param amount how much space (in bytes) should be reserved (for content only)
1047 * @param entries how many entries will be created (to calculate per-entry overhead)
1048 * @param cont continuation to call when done; "success" will be set to
1049 * a positive reservation value if space could be reserved.
1050 * @param cont_cls closure for @a cont
1051 * @return NULL if the entry was not queued, otherwise a handle that can be used to
1052 * cancel; note that even if NULL is returned, the callback will be invoked
1053 * (or rather, will already have been invoked)
1054 */
1055struct GNUNET_DATASTORE_QueueEntry *
1056GNUNET_DATASTORE_reserve (struct GNUNET_DATASTORE_Handle *h,
1057 uint64_t amount,
1058 uint32_t entries,
1059 GNUNET_DATASTORE_ContinuationWithStatus cont,
1060 void *cont_cls)
1061{
1062 struct GNUNET_DATASTORE_QueueEntry *qe;
1063 struct GNUNET_MQ_Envelope *env;
1064 struct ReserveMessage *rm;
1065 union QueueContext qc;
1066
1067 if (NULL == cont)
1068 cont = &drop_status_cont;
1069 LOG (GNUNET_ERROR_TYPE_DEBUG,
1070 "Asked to reserve %llu bytes of data and %u entries\n",
1071 (unsigned long long) amount,
1072 (unsigned int) entries);
1073 env = GNUNET_MQ_msg (rm,
1074 GNUNET_MESSAGE_TYPE_DATASTORE_RESERVE);
1075 rm->entries = htonl (entries);
1076 rm->amount = GNUNET_htonll (amount);
1077
1078 qc.sc.cont = cont;
1079 qc.sc.cont_cls = cont_cls;
1080 qe = make_queue_entry (h,
1081 env,
1082 UINT_MAX,
1083 UINT_MAX,
1084 GNUNET_MESSAGE_TYPE_DATASTORE_STATUS,
1085 &qc);
1086 if (NULL == qe)
1087 {
1088 LOG (GNUNET_ERROR_TYPE_DEBUG,
1089 "Could not create queue entry to reserve\n");
1090 return NULL;
1091 }
1092 GNUNET_STATISTICS_update (h->stats,
1093 gettext_noop ("# RESERVE requests executed"),
1094 1,
1095 GNUNET_NO);
1096 process_queue (h);
1097 return qe;
1098}
1099
1100
1101/**
1102 * Signal that all of the data for which a reservation was made has
1103 * been stored and that whatever excess space might have been reserved
1104 * can now be released.
1105 *
1106 * @param h handle to the datastore
1107 * @param rid reservation ID (value of "success" in original continuation
1108 * from the "reserve" function).
1109 * @param queue_priority ranking of this request in the priority queue
1110 * @param max_queue_size at what queue size should this request be dropped
1111 * (if other requests of higher priority are in the queue)
1112 * @param queue_priority ranking of this request in the priority queue
1113 * @param max_queue_size at what queue size should this request be dropped
1114 * (if other requests of higher priority are in the queue)
1115 * @param cont continuation to call when done
1116 * @param cont_cls closure for @a cont
1117 * @return NULL if the entry was not queued, otherwise a handle that can be used to
1118 * cancel; note that even if NULL is returned, the callback will be invoked
1119 * (or rather, will already have been invoked)
1120 */
1121struct GNUNET_DATASTORE_QueueEntry *
1122GNUNET_DATASTORE_release_reserve (struct GNUNET_DATASTORE_Handle *h,
1123 uint32_t rid,
1124 unsigned int queue_priority,
1125 unsigned int max_queue_size,
1126 GNUNET_DATASTORE_ContinuationWithStatus cont,
1127 void *cont_cls)
1128{
1129 struct GNUNET_DATASTORE_QueueEntry *qe;
1130 struct GNUNET_MQ_Envelope *env;
1131 struct ReleaseReserveMessage *rrm;
1132 union QueueContext qc;
1133
1134 if (NULL == cont)
1135 cont = &drop_status_cont;
1136 LOG (GNUNET_ERROR_TYPE_DEBUG,
1137 "Asked to release reserve %d\n",
1138 rid);
1139 env = GNUNET_MQ_msg (rrm,
1140 GNUNET_MESSAGE_TYPE_DATASTORE_RELEASE_RESERVE);
1141 rrm->rid = htonl (rid);
1142 qc.sc.cont = cont;
1143 qc.sc.cont_cls = cont_cls;
1144 qe = make_queue_entry (h,
1145 env,
1146 queue_priority,
1147 max_queue_size,
1148 GNUNET_MESSAGE_TYPE_DATASTORE_STATUS,
1149 &qc);
1150 if (NULL == qe)
1151 {
1152 LOG (GNUNET_ERROR_TYPE_DEBUG,
1153 "Could not create queue entry to release reserve\n");
1154 return NULL;
1155 }
1156 GNUNET_STATISTICS_update (h->stats,
1157 gettext_noop
1158 ("# RELEASE RESERVE requests executed"), 1,
1159 GNUNET_NO);
1160 process_queue (h);
1161 return qe;
1162}
1163
1164
1165/**
1166 * Explicitly remove some content from the database.
1167 * The @a cont continuation will be called with `status`
1168 * #GNUNET_OK" if content was removed, #GNUNET_NO
1169 * if no matching entry was found and #GNUNET_SYSERR
1170 * on all other types of errors.
1171 *
1172 * @param h handle to the datastore
1173 * @param key key for the value
1174 * @param size number of bytes in data
1175 * @param data content stored
1176 * @param queue_priority ranking of this request in the priority queue
1177 * @param max_queue_size at what queue size should this request be dropped
1178 * (if other requests of higher priority are in the queue)
1179 * @param cont continuation to call when done
1180 * @param cont_cls closure for @a cont
1181 * @return NULL if the entry was not queued, otherwise a handle that can be used to
1182 * cancel; note that even if NULL is returned, the callback will be invoked
1183 * (or rather, will already have been invoked)
1184 */
1185struct GNUNET_DATASTORE_QueueEntry *
1186GNUNET_DATASTORE_remove (struct GNUNET_DATASTORE_Handle *h,
1187 const struct GNUNET_HashCode *key,
1188 size_t size,
1189 const void *data,
1190 unsigned int queue_priority,
1191 unsigned int max_queue_size,
1192 GNUNET_DATASTORE_ContinuationWithStatus cont,
1193 void *cont_cls)
1194{
1195 struct GNUNET_DATASTORE_QueueEntry *qe;
1196 struct DataMessage *dm;
1197 struct GNUNET_MQ_Envelope *env;
1198 union QueueContext qc;
1199
1200 if (sizeof(*dm) + size >= GNUNET_MAX_MESSAGE_SIZE)
1201 {
1202 GNUNET_break (0);
1203 return NULL;
1204 }
1205 if (NULL == cont)
1206 cont = &drop_status_cont;
1207 LOG (GNUNET_ERROR_TYPE_DEBUG,
1208 "Asked to remove %lu bytes under key `%s'\n",
1209 (unsigned long) size,
1210 GNUNET_h2s (key));
1211 env = GNUNET_MQ_msg_extra (dm,
1212 size,
1213 GNUNET_MESSAGE_TYPE_DATASTORE_REMOVE);
1214 dm->size = htonl (size);
1215 dm->key = *key;
1216 GNUNET_memcpy (&dm[1],
1217 data,
1218 size);
1219
1220 qc.sc.cont = cont;
1221 qc.sc.cont_cls = cont_cls;
1222
1223 qe = make_queue_entry (h,
1224 env,
1225 queue_priority,
1226 max_queue_size,
1227 GNUNET_MESSAGE_TYPE_DATASTORE_STATUS,
1228 &qc);
1229 if (NULL == qe)
1230 {
1231 LOG (GNUNET_ERROR_TYPE_DEBUG,
1232 "Could not create queue entry for REMOVE\n");
1233 return NULL;
1234 }
1235 GNUNET_STATISTICS_update (h->stats,
1236 gettext_noop ("# REMOVE requests executed"),
1237 1,
1238 GNUNET_NO);
1239 process_queue (h);
1240 return qe;
1241}
1242
1243
1244/**
1245 * Get a random value from the datastore for content replication.
1246 * Returns a single, random value among those with the highest
1247 * replication score, lowering positive replication scores by one for
1248 * the chosen value (if only content with a replication score exists,
1249 * a random value is returned and replication scores are not changed).
1250 *
1251 * @param h handle to the datastore
1252 * @param queue_priority ranking of this request in the priority queue
1253 * @param max_queue_size at what queue size should this request be dropped
1254 * (if other requests of higher priority are in the queue)
1255 * @param proc function to call on a random value; it
1256 * will be called once with a value (if available)
1257 * and always once with a value of NULL.
1258 * @param proc_cls closure for @a proc
1259 * @return NULL if the entry was not queued, otherwise a handle that can be used to
1260 * cancel
1261 */
1262struct GNUNET_DATASTORE_QueueEntry *
1263GNUNET_DATASTORE_get_for_replication (struct GNUNET_DATASTORE_Handle *h,
1264 unsigned int queue_priority,
1265 unsigned int max_queue_size,
1266 GNUNET_DATASTORE_DatumProcessor proc,
1267 void *proc_cls)
1268{
1269 struct GNUNET_DATASTORE_QueueEntry *qe;
1270 struct GNUNET_MQ_Envelope *env;
1271 struct GNUNET_MessageHeader *m;
1272 union QueueContext qc;
1273
1274 GNUNET_assert (NULL != proc);
1275 LOG (GNUNET_ERROR_TYPE_DEBUG,
1276 "Asked to get replication entry\n");
1277 env = GNUNET_MQ_msg (m,
1278 GNUNET_MESSAGE_TYPE_DATASTORE_GET_REPLICATION);
1279 qc.rc.proc = proc;
1280 qc.rc.proc_cls = proc_cls;
1281 qe = make_queue_entry (h,
1282 env,
1283 queue_priority,
1284 max_queue_size,
1285 GNUNET_MESSAGE_TYPE_DATASTORE_DATA,
1286 &qc);
1287 if (NULL == qe)
1288 {
1289 LOG (GNUNET_ERROR_TYPE_DEBUG,
1290 "Could not create queue entry for GET REPLICATION\n");
1291 return NULL;
1292 }
1293 GNUNET_STATISTICS_update (h->stats,
1294 gettext_noop
1295 ("# GET REPLICATION requests executed"), 1,
1296 GNUNET_NO);
1297 process_queue (h);
1298 return qe;
1299}
1300
1301
1302/**
1303 * Get a single zero-anonymity value from the datastore.
1304 *
1305 * @param h handle to the datastore
1306 * @param next_uid return the result with lowest uid >= next_uid
1307 * @param queue_priority ranking of this request in the priority queue
1308 * @param max_queue_size at what queue size should this request be dropped
1309 * (if other requests of higher priority are in the queue)
1310 * @param type allowed type for the operation (never zero)
1311 * @param proc function to call on a random value; it
1312 * will be called once with a value (if available)
1313 * or with NULL if none value exists.
1314 * @param proc_cls closure for @a proc
1315 * @return NULL if the entry was not queued, otherwise a handle that can be used to
1316 * cancel
1317 */
1318struct GNUNET_DATASTORE_QueueEntry *
1319GNUNET_DATASTORE_get_zero_anonymity (struct GNUNET_DATASTORE_Handle *h,
1320 uint64_t next_uid,
1321 unsigned int queue_priority,
1322 unsigned int max_queue_size,
1323 enum GNUNET_BLOCK_Type type,
1324 GNUNET_DATASTORE_DatumProcessor proc,
1325 void *proc_cls)
1326{
1327 struct GNUNET_DATASTORE_QueueEntry *qe;
1328 struct GNUNET_MQ_Envelope *env;
1329 struct GetZeroAnonymityMessage *m;
1330 union QueueContext qc;
1331
1332 GNUNET_assert (NULL != proc);
1333 GNUNET_assert (type != GNUNET_BLOCK_TYPE_ANY);
1334 LOG (GNUNET_ERROR_TYPE_DEBUG,
1335 "Asked to get a zero-anonymity entry of type %d\n",
1336 type);
1337 env = GNUNET_MQ_msg (m,
1338 GNUNET_MESSAGE_TYPE_DATASTORE_GET_ZERO_ANONYMITY);
1339 m->type = htonl ((uint32_t) type);
1340 m->next_uid = GNUNET_htonll (next_uid);
1341 qc.rc.proc = proc;
1342 qc.rc.proc_cls = proc_cls;
1343 qe = make_queue_entry (h,
1344 env,
1345 queue_priority,
1346 max_queue_size,
1347 GNUNET_MESSAGE_TYPE_DATASTORE_DATA,
1348 &qc);
1349 if (NULL == qe)
1350 {
1351 LOG (GNUNET_ERROR_TYPE_DEBUG,
1352 "Could not create queue entry for zero-anonymity procation\n");
1353 return NULL;
1354 }
1355 GNUNET_STATISTICS_update (h->stats,
1356 gettext_noop
1357 ("# GET ZERO ANONYMITY requests executed"), 1,
1358 GNUNET_NO);
1359 process_queue (h);
1360 return qe;
1361}
1362
1363
1364/**
1365 * Get a result for a particular key from the datastore. The processor
1366 * will only be called once.
1367 *
1368 * @param h handle to the datastore
1369 * @param next_uid return the result with lowest uid >= next_uid
1370 * @param random if true, return a random result instead of using next_uid
1371 * @param key maybe NULL (to match all entries)
1372 * @param type desired type, 0 for any
1373 * @param queue_priority ranking of this request in the priority queue
1374 * @param max_queue_size at what queue size should this request be dropped
1375 * (if other requests of higher priority are in the queue)
1376 * @param proc function to call on each matching value;
1377 * will be called once with a NULL value at the end
1378 * @param proc_cls closure for @a proc
1379 * @return NULL if the entry was not queued, otherwise a handle that can be used to
1380 * cancel
1381 */
1382struct GNUNET_DATASTORE_QueueEntry *
1383GNUNET_DATASTORE_get_key (struct GNUNET_DATASTORE_Handle *h,
1384 uint64_t next_uid,
1385 bool random,
1386 const struct GNUNET_HashCode *key,
1387 enum GNUNET_BLOCK_Type type,
1388 unsigned int queue_priority,
1389 unsigned int max_queue_size,
1390 GNUNET_DATASTORE_DatumProcessor proc,
1391 void *proc_cls)
1392{
1393 struct GNUNET_DATASTORE_QueueEntry *qe;
1394 struct GNUNET_MQ_Envelope *env;
1395 struct GetKeyMessage *gkm;
1396 struct GetMessage *gm;
1397 union QueueContext qc;
1398
1399 GNUNET_assert (NULL != proc);
1400 LOG (GNUNET_ERROR_TYPE_DEBUG,
1401 "Asked to look for data of type %u under key `%s'\n",
1402 (unsigned int) type,
1403 (NULL != key) ? GNUNET_h2s (key) : "NULL");
1404 if (NULL == key)
1405 {
1406 env = GNUNET_MQ_msg (gm,
1407 GNUNET_MESSAGE_TYPE_DATASTORE_GET);
1408 gm->type = htonl (type);
1409 gm->next_uid = GNUNET_htonll (next_uid);
1410 gm->random = random;
1411 }
1412 else
1413 {
1414 env = GNUNET_MQ_msg (gkm,
1415 GNUNET_MESSAGE_TYPE_DATASTORE_GET_KEY);
1416 gkm->type = htonl (type);
1417 gkm->next_uid = GNUNET_htonll (next_uid);
1418 gkm->random = random;
1419 gkm->key = *key;
1420 }
1421 qc.rc.proc = proc;
1422 qc.rc.proc_cls = proc_cls;
1423 qe = make_queue_entry (h,
1424 env,
1425 queue_priority,
1426 max_queue_size,
1427 GNUNET_MESSAGE_TYPE_DATASTORE_DATA,
1428 &qc);
1429 if (NULL == qe)
1430 {
1431 LOG (GNUNET_ERROR_TYPE_DEBUG,
1432 "Could not queue request for `%s'\n",
1433 (NULL != key) ? GNUNET_h2s (key): "NULL");
1434 return NULL;
1435 }
1436#if INSANE_STATISTICS
1437 GNUNET_STATISTICS_update (h->stats,
1438 gettext_noop ("# GET requests executed"),
1439 1,
1440 GNUNET_NO);
1441#endif
1442 process_queue (h);
1443 return qe;
1444}
1445
1446
1447/**
1448 * Cancel a datastore operation. The final callback from the
1449 * operation must not have been done yet.
1450 *
1451 * @param qe operation to cancel
1452 */
1453void
1454GNUNET_DATASTORE_cancel (struct GNUNET_DATASTORE_QueueEntry *qe)
1455{
1456 struct GNUNET_DATASTORE_Handle *h = qe->h;
1457
1458 LOG (GNUNET_ERROR_TYPE_DEBUG,
1459 "Pending DATASTORE request %p cancelled (%d, %d)\n",
1460 qe,
1461 NULL == qe->env,
1462 h->queue_head == qe);
1463 if (NULL == qe->env)
1464 {
1465 free_queue_entry (qe);
1466 h->skip_next_messages++;
1467 return;
1468 }
1469 free_queue_entry (qe);
1470 process_queue (h);
1471}
1472
1473
1474/* end of datastore_api.c */
diff --git a/src/datastore/gnunet-datastore.c b/src/datastore/gnunet-datastore.c
deleted file mode 100644
index 5901cce54..000000000
--- a/src/datastore/gnunet-datastore.c
+++ /dev/null
@@ -1,508 +0,0 @@
1/*
2 This file is part of GNUnet.
3 Copyright (C) 2010, 2013 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file datastore/gnunet-datastore.c
23 * @brief tool to manipulate datastores
24 * @author Christian Grothoff
25 */
26#include <inttypes.h>
27#include "platform.h"
28#include "gnunet_util_lib.h"
29#include "gnunet_datastore_service.h"
30
31GNUNET_NETWORK_STRUCT_BEGIN
32
33struct DataRecord
34{
35 /**
36 * Number of bytes in the item (NBO).
37 */
38 uint32_t size GNUNET_PACKED;
39
40 /**
41 * Type of the item (NBO) (actually an enum GNUNET_BLOCK_Type)
42 */
43 uint32_t type GNUNET_PACKED;
44
45 /**
46 * Priority of the item (NBO).
47 */
48 uint32_t priority GNUNET_PACKED;
49
50 /**
51 * Desired anonymity level (NBO).
52 */
53 uint32_t anonymity GNUNET_PACKED;
54
55 /**
56 * Desired replication level (NBO).
57 */
58 uint32_t replication GNUNET_PACKED;
59
60 /**
61 * Expiration time (NBO).
62 */
63 struct GNUNET_TIME_AbsoluteNBO expiration;
64
65 /**
66 * Key under which the item can be found.
67 */
68 struct GNUNET_HashCode key;
69};
70GNUNET_NETWORK_STRUCT_END
71
72
73/**
74 * Length of our magic header.
75 */
76static const size_t MAGIC_LEN = 16;
77
78/**
79 * Magic header bytes.
80 */
81static const uint8_t MAGIC_BYTES[16] = "GNUNETDATASTORE1";
82
83/**
84 * Dump the database.
85 */
86static int dump;
87
88/**
89 * Insert into the database.
90 */
91static int insert;
92
93/**
94 * Dump file name.
95 */
96static char *file_name;
97
98/**
99 * Dump file handle.
100 */
101static struct GNUNET_DISK_FileHandle *file_handle;
102
103/**
104 * Global return value.
105 */
106static int ret;
107
108/**
109 * Handle for datastore.
110 */
111static struct GNUNET_DATASTORE_Handle *datastore;
112
113/**
114 * Current operation.
115 */
116static struct GNUNET_DATASTORE_QueueEntry *qe;
117
118/**
119 * Record count.
120 */
121static uint64_t record_count;
122
123
124static void
125do_shutdown (void *cls)
126{
127 if (NULL != qe)
128 GNUNET_DATASTORE_cancel (qe);
129 if (NULL != datastore)
130 GNUNET_DATASTORE_disconnect (datastore, GNUNET_NO);
131 if (NULL != file_handle)
132 GNUNET_DISK_file_close (file_handle);
133}
134
135
136/**
137 * Begin dumping the database.
138 */
139static void
140start_dump (void);
141
142
143/**
144 * Begin inserting into the database.
145 */
146static void
147start_insert (void);
148
149
150/**
151 * Perform next GET operation.
152 */
153static void
154do_get (const uint64_t next_uid);
155
156
157/**
158 * Process a datum that was stored in the datastore.
159 *
160 * @param cls closure
161 * @param key key for the content
162 * @param size number of bytes in data
163 * @param data content stored
164 * @param type type of the content
165 * @param priority priority of the content
166 * @param anonymity anonymity-level for the content
167 * @param replication replication-level for the content
168 * @param expiration expiration time for the content
169 * @param uid unique identifier for the datum;
170 * maybe 0 if no unique identifier is available
171 */
172static void
173get_cb (void *cls,
174 const struct GNUNET_HashCode *key,
175 size_t size,
176 const void *data,
177 enum GNUNET_BLOCK_Type type,
178 uint32_t priority,
179 uint32_t anonymity,
180 uint32_t replication,
181 struct GNUNET_TIME_Absolute expiration,
182 uint64_t uid)
183{
184 qe = NULL;
185 if (NULL == key)
186 {
187 fprintf (stderr, _ ("Dumped %" PRIu64 " records\n"), record_count);
188 GNUNET_DISK_file_close (file_handle);
189 file_handle = NULL;
190 if (insert)
191 start_insert ();
192 else
193 {
194 ret = 0;
195 GNUNET_SCHEDULER_shutdown ();
196 }
197 return;
198 }
199
200 struct DataRecord dr;
201 dr.size = htonl ((uint32_t) size);
202 dr.type = htonl (type);
203 dr.priority = htonl (priority);
204 dr.anonymity = htonl (anonymity);
205 dr.replication = htonl (replication);
206 dr.expiration = GNUNET_TIME_absolute_hton (expiration);
207 dr.key = *key;
208
209 ssize_t len;
210 len = GNUNET_DISK_file_write (file_handle, &dr, sizeof(dr));
211 if (sizeof(dr) != len)
212 {
213 fprintf (stderr,
214 _ ("Short write to file: %zd bytes expecting %zd\n"),
215 len,
216 sizeof(dr));
217 ret = 1;
218 GNUNET_SCHEDULER_shutdown ();
219 return;
220 }
221
222 len = GNUNET_DISK_file_write (file_handle, data, size);
223 if (size != len)
224 {
225 fprintf (stderr,
226 _ ("Short write to file: %zd bytes expecting %zd\n"),
227 len,
228 size);
229 ret = 1;
230 GNUNET_SCHEDULER_shutdown ();
231 return;
232 }
233
234 record_count++;
235 do_get (uid + 1);
236}
237
238
239/**
240 * Perform next GET operation.
241 */
242static void
243do_get (const uint64_t next_uid)
244{
245 GNUNET_assert (NULL == qe);
246 qe = GNUNET_DATASTORE_get_key (datastore,
247 next_uid,
248 false /* random */,
249 NULL /* key */,
250 GNUNET_BLOCK_TYPE_ANY,
251 0 /* queue_priority */,
252 1 /* max_queue_size */,
253 &get_cb,
254 NULL /* proc_cls */);
255 if (NULL == qe)
256 {
257 fprintf (stderr, _ ("Error queueing datastore GET operation\n"));
258 ret = 1;
259 GNUNET_SCHEDULER_shutdown ();
260 }
261}
262
263
264/**
265 * Begin dumping the database.
266 */
267static void
268start_dump ()
269{
270 record_count = 0;
271
272 if (NULL != file_name)
273 {
274 file_handle = GNUNET_DISK_file_open (file_name,
275 GNUNET_DISK_OPEN_WRITE
276 | GNUNET_DISK_OPEN_TRUNCATE
277 | GNUNET_DISK_OPEN_CREATE,
278 GNUNET_DISK_PERM_USER_READ
279 | GNUNET_DISK_PERM_USER_WRITE);
280 if (NULL == file_handle)
281 {
282 fprintf (stderr, _ ("Unable to open dump file: %s\n"), file_name);
283 ret = 1;
284 GNUNET_SCHEDULER_shutdown ();
285 return;
286 }
287 }
288 else
289 {
290 file_handle = GNUNET_DISK_get_handle_from_int_fd (STDOUT_FILENO);
291 }
292 GNUNET_DISK_file_write (file_handle, MAGIC_BYTES, MAGIC_LEN);
293 do_get (0);
294}
295
296
297/**
298 * Continuation called to notify client about result of the
299 * operation.
300 *
301 * @param cls closure
302 * @param success GNUNET_SYSERR on failure (including timeout/queue drop)
303 * GNUNET_NO if content was already there
304 * GNUNET_YES (or other positive value) on success
305 * @param min_expiration minimum expiration time required for 0-priority content to be stored
306 * by the datacache at this time, zero for unknown, forever if we have no
307 * space for 0-priority content
308 * @param msg NULL on success, otherwise an error message
309 */
310static void
311put_cb (void *cls,
312 int32_t success,
313 struct GNUNET_TIME_Absolute min_expiration,
314 const char *msg)
315{
316 qe = NULL;
317 if (GNUNET_SYSERR == success)
318 {
319 fprintf (stderr, _ ("Failed to store item: %s, aborting\n"), msg);
320 ret = 1;
321 GNUNET_SCHEDULER_shutdown ();
322 return;
323 }
324
325 struct DataRecord dr;
326 ssize_t len;
327
328 len = GNUNET_DISK_file_read (file_handle, &dr, sizeof(dr));
329 if (0 == len)
330 {
331 fprintf (stderr, _ ("Inserted %" PRIu64 " records\n"), record_count);
332 ret = 0;
333 GNUNET_SCHEDULER_shutdown ();
334 return;
335 }
336 else if (sizeof(dr) != len)
337 {
338 fprintf (stderr,
339 _ ("Short read from file: %zd bytes expecting %zd\n"),
340 len,
341 sizeof(dr));
342 ret = 1;
343 GNUNET_SCHEDULER_shutdown ();
344 return;
345 }
346
347 const size_t size = ntohl (dr.size);
348 uint8_t data[size];
349 len = GNUNET_DISK_file_read (file_handle, data, size);
350 if (size != len)
351 {
352 fprintf (stderr,
353 _ ("Short read from file: %zd bytes expecting %zd\n"),
354 len,
355 size);
356 ret = 1;
357 GNUNET_SCHEDULER_shutdown ();
358 return;
359 }
360
361 record_count++;
362 qe = GNUNET_DATASTORE_put (datastore,
363 0,
364 &dr.key,
365 size,
366 data,
367 ntohl (dr.type),
368 ntohl (dr.priority),
369 ntohl (dr.anonymity),
370 ntohl (dr.replication),
371 GNUNET_TIME_absolute_ntoh (dr.expiration),
372 0,
373 1,
374 &put_cb,
375 NULL);
376 if (NULL == qe)
377 {
378 fprintf (stderr, _ ("Error queueing datastore PUT operation\n"));
379 ret = 1;
380 GNUNET_SCHEDULER_shutdown ();
381 }
382}
383
384
385/**
386 * Begin inserting into the database.
387 */
388static void
389start_insert ()
390{
391 record_count = 0;
392
393 if (NULL != file_name)
394 {
395 file_handle = GNUNET_DISK_file_open (file_name,
396 GNUNET_DISK_OPEN_READ,
397 GNUNET_DISK_PERM_NONE);
398 if (NULL == file_handle)
399 {
400 fprintf (stderr, _ ("Unable to open dump file: %s\n"), file_name);
401 ret = 1;
402 GNUNET_SCHEDULER_shutdown ();
403 return;
404 }
405 }
406 else
407 {
408 file_handle = GNUNET_DISK_get_handle_from_int_fd (STDIN_FILENO);
409 }
410
411 uint8_t buf[MAGIC_LEN];
412 ssize_t len;
413
414 len = GNUNET_DISK_file_read (file_handle, buf, MAGIC_LEN);
415 if ((len != MAGIC_LEN) || (0 != memcmp (buf, MAGIC_BYTES, MAGIC_LEN)))
416 {
417 fprintf (stderr, _ ("Input file is not of a supported format\n"));
418 return;
419 }
420 put_cb (NULL, GNUNET_YES, GNUNET_TIME_UNIT_ZERO_ABS, NULL);
421}
422
423
424/**
425 * Main function that will be run by the scheduler.
426 *
427 * @param cls closure
428 * @param args remaining command-line arguments
429 * @param cfgfile name of the configuration file used
430 * @param cfg configuration
431 */
432static void
433run (void *cls,
434 char *const *args,
435 const char *cfgfile,
436 const struct GNUNET_CONFIGURATION_Handle *cfg)
437{
438 GNUNET_SCHEDULER_add_shutdown (&do_shutdown, NULL);
439 datastore = GNUNET_DATASTORE_connect (cfg);
440 if (NULL == datastore)
441 {
442 fprintf (stderr, _ ("Failed connecting to the datastore.\n"));
443 ret = 1;
444 GNUNET_SCHEDULER_shutdown ();
445 return;
446 }
447 if (dump)
448 start_dump ();
449 else if (insert)
450 start_insert ();
451 else
452 {
453 fprintf (stderr,
454 _ ("Please choose at least one operation: %s, %s\n"),
455 "dump",
456 "insert");
457 ret = 1;
458 GNUNET_SCHEDULER_shutdown ();
459 }
460}
461
462
463/**
464 * The main function to manipulate datastores.
465 *
466 * @param argc number of arguments from the command line
467 * @param argv command line arguments
468 * @return 0 ok, 1 on error
469 */
470int
471main (int argc, char *const *argv)
472{
473 struct GNUNET_GETOPT_CommandLineOption options[] =
474 { GNUNET_GETOPT_option_flag ('d',
475 "dump",
476 gettext_noop (
477 "Dump all records from the datastore"),
478 &dump),
479 GNUNET_GETOPT_option_flag ('i',
480 "insert",
481 gettext_noop (
482 "Insert records into the datastore"),
483 &insert),
484 GNUNET_GETOPT_option_filename ('f',
485 "file",
486 "FILENAME",
487 gettext_noop ("File to dump or insert"),
488 &file_name),
489 GNUNET_GETOPT_OPTION_END };
490
491 if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
492 return 2;
493
494 if (GNUNET_OK !=
495 GNUNET_PROGRAM_run (argc,
496 argv,
497 "gnunet-datastore",
498 gettext_noop ("Manipulate GNUnet datastore"),
499 options,
500 &run,
501 NULL))
502 ret = 1;
503 GNUNET_free_nz ((void *) argv);
504 return ret;
505}
506
507
508/* end of gnunet-datastore.c */
diff --git a/src/datastore/gnunet-service-datastore.c b/src/datastore/gnunet-service-datastore.c
deleted file mode 100644
index 97888ce03..000000000
--- a/src/datastore/gnunet-service-datastore.c
+++ /dev/null
@@ -1,1647 +0,0 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2004-2014, 2016 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file datastore/gnunet-service-datastore.c
23 * @brief Management for the datastore for files stored on a GNUnet node
24 * @author Christian Grothoff
25 */
26
27#include "platform.h"
28#include "gnunet_util_lib.h"
29#include "gnunet_protocols.h"
30#include "gnunet_statistics_service.h"
31#include "gnunet_datastore_plugin.h"
32#include "datastore.h"
33
34/**
35 * How many messages do we queue at most per client?
36 */
37#define MAX_PENDING 1024
38
39/**
40 * Limit size of bloom filter to 2 GB.
41 */
42#define MAX_BF_SIZE ((uint32_t) (1LL << 31))
43
44/**
45 * How long are we at most keeping "expired" content
46 * past the expiration date in the database?
47 */
48#define MAX_EXPIRE_DELAY \
49 GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15)
50
51/**
52 * How fast are we allowed to query the database for deleting
53 * expired content? (1 item per second).
54 */
55#define MIN_EXPIRE_DELAY \
56 GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 1)
57
58/**
59 * Name under which we store current space consumption.
60 */
61static char *quota_stat_name;
62
63/**
64 * Task to timeout stat GET.
65 */
66static struct GNUNET_SCHEDULER_Task *stat_timeout_task;
67
68/**
69 * After how many payload-changing operations
70 * do we sync our statistics?
71 */
72#define MAX_STAT_SYNC_LAG 50
73
74
75/**
76 * Our datastore plugin.
77 */
78struct DatastorePlugin
79{
80 /**
81 * API of the transport as returned by the plugin's
82 * initialization function.
83 */
84 struct GNUNET_DATASTORE_PluginFunctions *api;
85
86 /**
87 * Short name for the plugin (e.g. "sqlite").
88 */
89 char *short_name;
90
91 /**
92 * Name of the library (e.g. "gnunet_plugin_datastore_sqlite").
93 */
94 char *lib_name;
95
96 /**
97 * Environment this transport service is using
98 * for this plugin.
99 */
100 struct GNUNET_DATASTORE_PluginEnvironment env;
101};
102
103
104/**
105 * Linked list of active reservations.
106 */
107struct ReservationList
108{
109 /**
110 * This is a linked list.
111 */
112 struct ReservationList *next;
113
114 /**
115 * Client that made the reservation.
116 */
117 struct GNUNET_SERVICE_Client *client;
118
119 /**
120 * Number of bytes (still) reserved.
121 */
122 uint64_t amount;
123
124 /**
125 * Number of items (still) reserved.
126 */
127 uint64_t entries;
128
129 /**
130 * Reservation identifier.
131 */
132 int32_t rid;
133};
134
135
136/**
137 * Our datastore plugin (NULL if not available).
138 */
139static struct DatastorePlugin *plugin;
140
141/**
142 * Linked list of space reservations made by clients.
143 */
144static struct ReservationList *reservations;
145
146/**
147 * Bloomfilter to quickly tell if we don't have the content.
148 */
149static struct GNUNET_CONTAINER_BloomFilter *filter;
150
151/**
152 * Name of our plugin.
153 */
154static char *plugin_name;
155
156/**
157 * Our configuration.
158 */
159static const struct GNUNET_CONFIGURATION_Handle *cfg;
160
161/**
162 * Handle for reporting statistics.
163 */
164static struct GNUNET_STATISTICS_Handle *stats;
165
166/**
167 * How much space are we using for the cache? (space available for
168 * insertions that will be instantly reclaimed by discarding less
169 * important content --- or possibly whatever we just inserted into
170 * the "cache").
171 */
172static unsigned long long cache_size;
173
174/**
175 * How much space have we currently reserved?
176 */
177static unsigned long long reserved;
178
179/**
180 * How much data are we currently storing
181 * in the database?
182 */
183static unsigned long long payload;
184
185/**
186 * Identity of the task that is used to delete
187 * expired content.
188 */
189static struct GNUNET_SCHEDULER_Task *expired_kill_task;
190
191/**
192 * Minimum time that content should have to not be discarded instantly
193 * (time stamp of any content that we've been discarding recently to
194 * stay below the quota). FOREVER if we had to expire content with
195 * non-zero priority.
196 */
197static struct GNUNET_TIME_Absolute min_expiration;
198
199/**
200 * How much space are we allowed to use?
201 */
202static unsigned long long quota;
203
204/**
205 * Should the database be dropped on exit?
206 */
207static int do_drop;
208
209/**
210 * Should we refresh the BF when the DB is loaded?
211 */
212static int refresh_bf;
213
214/**
215 * Number of updates that were made to the
216 * payload value since we last synchronized
217 * it with the statistics service.
218 */
219static unsigned int last_sync;
220
221/**
222 * Did we get an answer from statistics?
223 */
224static int stats_worked;
225
226
227/**
228 * Synchronize our utilization statistics with the
229 * statistics service.
230 */
231static void
232sync_stats ()
233{
234 GNUNET_STATISTICS_set (stats, quota_stat_name, payload, GNUNET_YES);
235 GNUNET_STATISTICS_set (stats,
236 "# utilization by current datastore",
237 payload,
238 GNUNET_NO);
239 last_sync = 0;
240}
241
242
243/**
244 * Have we already cleaned up the TCCs and are hence no longer
245 * willing (or able) to transmit anything to anyone?
246 */
247static int cleaning_done;
248
249/**
250 * Handle for pending get request.
251 */
252static struct GNUNET_STATISTICS_GetHandle *stat_get;
253
254/**
255 * Handle to our server.
256 */
257static struct GNUNET_SERVICE_Handle *service;
258
259/**
260 * Task that is used to remove expired entries from
261 * the datastore. This task will schedule itself
262 * again automatically to always delete all expired
263 * content quickly.
264 *
265 * @param cls not used
266 */
267static void
268delete_expired (void *cls);
269
270
271/**
272 * Iterate over the expired items stored in the datastore.
273 * Delete all expired items; once we have processed all
274 * expired items, re-schedule the "delete_expired" task.
275 *
276 * @param cls not used
277 * @param key key for the content
278 * @param size number of bytes in data
279 * @param data content stored
280 * @param type type of the content
281 * @param priority priority of the content
282 * @param anonymity anonymity-level for the content
283 * @param replication replication-level for the content
284 * @param expiration expiration time for the content
285 * @param uid unique identifier for the datum;
286 * maybe 0 if no unique identifier is available
287 *
288 * @return #GNUNET_SYSERR to abort the iteration, #GNUNET_OK to continue
289 * (continue on call to "next", of course),
290 * #GNUNET_NO to delete the item and continue (if supported)
291 */
292static int
293expired_processor (void *cls,
294 const struct GNUNET_HashCode *key,
295 uint32_t size,
296 const void *data,
297 enum GNUNET_BLOCK_Type type,
298 uint32_t priority,
299 uint32_t anonymity,
300 uint32_t replication,
301 struct GNUNET_TIME_Absolute expiration,
302 uint64_t uid)
303{
304 struct GNUNET_TIME_Absolute now;
305
306 if (NULL == key)
307 {
308 expired_kill_task =
309 GNUNET_SCHEDULER_add_delayed_with_priority (MAX_EXPIRE_DELAY,
310 GNUNET_SCHEDULER_PRIORITY_IDLE,
311 &delete_expired,
312 NULL);
313 return GNUNET_SYSERR;
314 }
315 now = GNUNET_TIME_absolute_get ();
316 if (expiration.abs_value_us > now.abs_value_us)
317 {
318 /* finished processing */
319 expired_kill_task =
320 GNUNET_SCHEDULER_add_delayed_with_priority (MAX_EXPIRE_DELAY,
321 GNUNET_SCHEDULER_PRIORITY_IDLE,
322 &delete_expired,
323 NULL);
324 return GNUNET_SYSERR;
325 }
326 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
327 "Deleting content `%s' of type %u that expired %s ago\n",
328 GNUNET_h2s (key),
329 type,
330 GNUNET_STRINGS_relative_time_to_string (
331 GNUNET_TIME_absolute_get_difference (expiration, now),
332 GNUNET_YES));
333 min_expiration = now;
334 GNUNET_STATISTICS_update (stats,
335 gettext_noop ("# bytes expired"),
336 size,
337 GNUNET_YES);
338 GNUNET_CONTAINER_bloomfilter_remove (filter, key);
339 expired_kill_task =
340 GNUNET_SCHEDULER_add_delayed_with_priority (MIN_EXPIRE_DELAY,
341 GNUNET_SCHEDULER_PRIORITY_IDLE,
342 &delete_expired,
343 NULL);
344 return GNUNET_NO;
345}
346
347
348/**
349 * Task that is used to remove expired entries from
350 * the datastore. This task will schedule itself
351 * again automatically to always delete all expired
352 * content quickly.
353 *
354 * @param cls not used
355 */
356static void
357delete_expired (void *cls)
358{
359 expired_kill_task = NULL;
360 plugin->api->get_expiration (plugin->api->cls, &expired_processor, NULL);
361}
362
363
364/**
365 * An iterator over a set of items stored in the datastore
366 * that deletes until we're happy with respect to our quota.
367 *
368 * @param cls closure
369 * @param key key for the content
370 * @param size number of bytes in data
371 * @param data content stored
372 * @param type type of the content
373 * @param priority priority of the content
374 * @param anonymity anonymity-level for the content
375 * @param replication replication-level for the content
376 * @param expiration expiration time for the content
377 * @param uid unique identifier for the datum;
378 * maybe 0 if no unique identifier is available
379 * @return #GNUNET_SYSERR to abort the iteration, #GNUNET_OK to continue
380 * (continue on call to "next", of course),
381 * #GNUNET_NO to delete the item and continue (if supported)
382 */
383static int
384quota_processor (void *cls,
385 const struct GNUNET_HashCode *key,
386 uint32_t size,
387 const void *data,
388 enum GNUNET_BLOCK_Type type,
389 uint32_t priority,
390 uint32_t anonymity,
391 uint32_t replication,
392 struct GNUNET_TIME_Absolute expiration,
393 uint64_t uid)
394{
395 unsigned long long *need = cls;
396
397 if (NULL == key)
398 return GNUNET_SYSERR;
399 GNUNET_log (
400 GNUNET_ERROR_TYPE_DEBUG,
401 "Deleting %llu bytes of low-priority (%u) content `%s' of type %u at %s prior to expiration (still trying to free another %llu bytes)\n",
402 (unsigned long long) (size + GNUNET_DATASTORE_ENTRY_OVERHEAD),
403 (unsigned int) priority,
404 GNUNET_h2s (key),
405 type,
406 GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_remaining (
407 expiration),
408 GNUNET_YES),
409 *need);
410 if (size + GNUNET_DATASTORE_ENTRY_OVERHEAD > *need)
411 *need = 0;
412 else
413 *need -= size + GNUNET_DATASTORE_ENTRY_OVERHEAD;
414 if (priority > 0)
415 min_expiration = GNUNET_TIME_UNIT_FOREVER_ABS;
416 else
417 min_expiration = expiration;
418 GNUNET_STATISTICS_update (stats,
419 gettext_noop ("# bytes purged (low-priority)"),
420 size,
421 GNUNET_YES);
422 GNUNET_CONTAINER_bloomfilter_remove (filter, key);
423 return GNUNET_NO;
424}
425
426
427/**
428 * Manage available disk space by running tasks
429 * that will discard content if necessary. This
430 * function will be run whenever a request for
431 * "need" bytes of storage could only be satisfied
432 * by eating into the "cache" (and we want our cache
433 * space back).
434 *
435 * @param need number of bytes of content that were
436 * placed into the "cache" (and hence the
437 * number of bytes that should be removed).
438 */
439static void
440manage_space (unsigned long long need)
441{
442 unsigned long long last;
443
444 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
445 "Asked to free up %llu bytes of cache space\n",
446 need);
447 last = 0;
448 while ((need > 0) && (last != need))
449 {
450 last = need;
451 plugin->api->get_expiration (plugin->api->cls, &quota_processor, &need);
452 }
453}
454
455
456/**
457 * Transmit a status code to the client.
458 *
459 * @param client receiver of the response
460 * @param code status code
461 * @param msg optional error message (can be NULL)
462 */
463static void
464transmit_status (struct GNUNET_SERVICE_Client *client,
465 int code,
466 const char *msg)
467{
468 struct GNUNET_MQ_Envelope *env;
469 struct StatusMessage *sm;
470 size_t slen;
471
472 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
473 "Transmitting `%s' message with value %d and message `%s'\n",
474 "STATUS",
475 code,
476 msg != NULL ? msg : "(none)");
477 slen = (msg == NULL) ? 0 : strlen (msg) + 1;
478 env = GNUNET_MQ_msg_extra (sm, slen, GNUNET_MESSAGE_TYPE_DATASTORE_STATUS);
479 sm->status = htonl (code);
480 sm->min_expiration = GNUNET_TIME_absolute_hton (min_expiration);
481 GNUNET_memcpy (&sm[1], msg, slen);
482 GNUNET_MQ_send (GNUNET_SERVICE_client_get_mq (client), env);
483}
484
485
486/**
487 * Function that will transmit the given datastore entry
488 * to the client.
489 *
490 * @param cls closure, pointer to the client (of type `struct GNUNET_SERVICE_Client`).
491 * @param key key for the content
492 * @param size number of bytes in data
493 * @param data content stored
494 * @param type type of the content
495 * @param priority priority of the content
496 * @param anonymity anonymity-level for the content
497 * @param replication replication-level for the content
498 * @param expiration expiration time for the content
499 * @param uid unique identifier for the datum;
500 * maybe 0 if no unique identifier is available
501 * @return #GNUNET_SYSERR to abort the iteration, #GNUNET_OK to continue,
502 * #GNUNET_NO to delete the item and continue (if supported)
503 */
504static int
505transmit_item (void *cls,
506 const struct GNUNET_HashCode *key,
507 uint32_t size,
508 const void *data,
509 enum GNUNET_BLOCK_Type type,
510 uint32_t priority,
511 uint32_t anonymity,
512 uint32_t replication,
513 struct GNUNET_TIME_Absolute expiration,
514 uint64_t uid)
515{
516 struct GNUNET_SERVICE_Client *client = cls;
517 struct GNUNET_MQ_Envelope *env;
518 struct GNUNET_MessageHeader *end;
519 struct DataMessage *dm;
520
521 if (NULL == key)
522 {
523 /* transmit 'DATA_END' */
524 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Transmitting DATA_END message\n");
525 env = GNUNET_MQ_msg (end, GNUNET_MESSAGE_TYPE_DATASTORE_DATA_END);
526 GNUNET_MQ_send (GNUNET_SERVICE_client_get_mq (client), env);
527 return GNUNET_OK;
528 }
529 GNUNET_assert (sizeof(struct DataMessage) + size < GNUNET_MAX_MESSAGE_SIZE);
530 env = GNUNET_MQ_msg_extra (dm, size, GNUNET_MESSAGE_TYPE_DATASTORE_DATA);
531 dm->rid = htonl (0);
532 dm->size = htonl (size);
533 dm->type = htonl (type);
534 dm->priority = htonl (priority);
535 dm->anonymity = htonl (anonymity);
536 dm->replication = htonl (replication);
537 dm->expiration = GNUNET_TIME_absolute_hton (expiration);
538 dm->uid = GNUNET_htonll (uid);
539 dm->key = *key;
540 GNUNET_memcpy (&dm[1], data, size);
541 GNUNET_log (
542 GNUNET_ERROR_TYPE_DEBUG,
543 "Transmitting DATA message for `%s' of type %u with expiration %s (in: %s)\n",
544 GNUNET_h2s (key),
545 type,
546 GNUNET_STRINGS_absolute_time_to_string (expiration),
547 GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_remaining (
548 expiration),
549 GNUNET_YES));
550 GNUNET_STATISTICS_update (stats,
551 gettext_noop ("# results found"),
552 1,
553 GNUNET_NO);
554 GNUNET_MQ_send (GNUNET_SERVICE_client_get_mq (client), env);
555 return GNUNET_OK;
556}
557
558
559/**
560 * Handle RESERVE-message.
561 *
562 * @param cls identification of the client
563 * @param message the actual message
564 */
565static void
566handle_reserve (void *cls, const struct ReserveMessage *msg)
567{
568 /**
569 * Static counter to produce reservation identifiers.
570 */
571 static int reservation_gen;
572 struct GNUNET_SERVICE_Client *client = cls;
573 struct ReservationList *e;
574 unsigned long long used;
575 unsigned long long req;
576 uint64_t amount;
577 uint32_t entries;
578
579 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Processing RESERVE request\n");
580 amount = GNUNET_ntohll (msg->amount);
581 entries = ntohl (msg->entries);
582 used = payload + reserved;
583 req =
584 amount + ((unsigned long long) GNUNET_DATASTORE_ENTRY_OVERHEAD) * entries;
585 if (used + req > quota)
586 {
587 if (quota < used)
588 used =
589 quota; /* cheat a bit for error message (to avoid negative numbers) */
590 GNUNET_log (
591 GNUNET_ERROR_TYPE_WARNING,
592 _ (
593 "Insufficient space (%llu bytes are available) to satisfy RESERVE request for %llu bytes\n"),
594 quota - used,
595 req);
596 if (cache_size < req)
597 {
598 /* TODO: document this in the FAQ; essentially, if this
599 * message happens, the insertion request could be blocked
600 * by less-important content from migration because it is
601 * larger than 1/8th of the overall available space, and
602 * we only reserve 1/8th for "fresh" insertions */GNUNET_log (
603 GNUNET_ERROR_TYPE_WARNING,
604 _ (
605 "The requested amount (%llu bytes) is larger than the cache size (%llu bytes)\n"),
606 req,
607 cache_size);
608 transmit_status (client,
609 0,
610 gettext_noop (
611 "Insufficient space to satisfy request and "
612 "requested amount is larger than cache size"));
613 }
614 else
615 {
616 transmit_status (client,
617 0,
618 gettext_noop ("Insufficient space to satisfy request"));
619 }
620 GNUNET_SERVICE_client_continue (client);
621 return;
622 }
623 reserved += req;
624 GNUNET_STATISTICS_set (stats,
625 gettext_noop ("# reserved"),
626 reserved,
627 GNUNET_NO);
628 e = GNUNET_new (struct ReservationList);
629 e->next = reservations;
630 reservations = e;
631 e->client = client;
632 e->amount = amount;
633 e->entries = entries;
634 e->rid = ++reservation_gen;
635 if (reservation_gen < 0)
636 reservation_gen = 0; /* wrap around */
637 transmit_status (client, e->rid, NULL);
638 GNUNET_SERVICE_client_continue (client);
639}
640
641
642/**
643 * Handle RELEASE_RESERVE-message.
644 *
645 * @param cls identification of the client
646 * @param message the actual message
647 */
648static void
649handle_release_reserve (void *cls, const struct ReleaseReserveMessage *msg)
650{
651 struct GNUNET_SERVICE_Client *client = cls;
652 struct ReservationList *pos;
653 struct ReservationList *prev;
654 struct ReservationList *next;
655 int rid = ntohl (msg->rid);
656 unsigned long long rem;
657
658 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Processing RELEASE_RESERVE request\n");
659 next = reservations;
660 prev = NULL;
661 while (NULL != (pos = next))
662 {
663 next = pos->next;
664 if (rid == pos->rid)
665 {
666 if (prev == NULL)
667 reservations = next;
668 else
669 prev->next = next;
670 rem =
671 pos->amount
672 + ((unsigned long long) GNUNET_DATASTORE_ENTRY_OVERHEAD) * pos->entries;
673 GNUNET_assert (reserved >= rem);
674 reserved -= rem;
675 GNUNET_STATISTICS_set (stats,
676 gettext_noop ("# reserved"),
677 reserved,
678 GNUNET_NO);
679 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
680 "Returning %llu remaining reserved bytes to storage pool\n",
681 rem);
682 GNUNET_free (pos);
683 transmit_status (client, GNUNET_OK, NULL);
684 GNUNET_SERVICE_client_continue (client);
685 return;
686 }
687 prev = pos;
688 }
689 GNUNET_break (0);
690 transmit_status (client,
691 GNUNET_SYSERR,
692 gettext_noop ("Could not find matching reservation"));
693 GNUNET_SERVICE_client_continue (client);
694}
695
696
697/**
698 * Check that the given message is a valid data message.
699 *
700 * @param dm message to check
701 * @return #GNUNET_SYSERR is not well-formed, otherwise #GNUNET_OK
702 */
703static int
704check_data (const struct DataMessage *dm)
705{
706 uint16_t size;
707 uint32_t dsize;
708
709 size = ntohs (dm->header.size);
710 dsize = ntohl (dm->size);
711 if (size != dsize + sizeof(struct DataMessage))
712 {
713 GNUNET_break (0);
714 return GNUNET_SYSERR;
715 }
716 return GNUNET_OK;
717}
718
719
720/**
721 * Put continuation.
722 *
723 * @param cls closure
724 * @param key key for the item stored
725 * @param size size of the item stored
726 * @param status #GNUNET_OK if inserted, #GNUNET_NO if updated,
727 * or #GNUNET_SYSERROR if error
728 * @param msg error message on error
729 */
730static void
731put_continuation (void *cls,
732 const struct GNUNET_HashCode *key,
733 uint32_t size,
734 int status,
735 const char *msg)
736{
737 struct GNUNET_SERVICE_Client *client = cls;
738
739 if (GNUNET_OK == status)
740 {
741 GNUNET_STATISTICS_update (stats,
742 gettext_noop ("# bytes stored"),
743 size,
744 GNUNET_YES);
745 GNUNET_CONTAINER_bloomfilter_add (filter, key);
746 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
747 "Successfully stored %u bytes under key `%s'\n",
748 size,
749 GNUNET_h2s (key));
750 }
751 transmit_status (client,
752 GNUNET_SYSERR == status ? GNUNET_SYSERR : GNUNET_OK,
753 msg);
754 if (quota - reserved - cache_size < payload)
755 {
756 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
757 _ ("Need %llu bytes more space (%llu allowed, using %llu)\n"),
758 (unsigned long long) size + GNUNET_DATASTORE_ENTRY_OVERHEAD,
759 (unsigned long long) (quota - reserved - cache_size),
760 (unsigned long long) payload);
761 manage_space (size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
762 }
763}
764
765
766/**
767 * Verify PUT-message.
768 *
769 * @param cls identification of the client
770 * @param message the actual message
771 * @return #GNUNET_OK if @a dm is well-formed
772 */
773static int
774check_put (void *cls, const struct DataMessage *dm)
775{
776 if (GNUNET_OK != check_data (dm))
777 {
778 GNUNET_break (0);
779 return GNUNET_SYSERR;
780 }
781 return GNUNET_OK;
782}
783
784
785/**
786 * Handle PUT-message.
787 *
788 * @param cls identification of the client
789 * @param message the actual message
790 */
791static void
792handle_put (void *cls, const struct DataMessage *dm)
793{
794 struct GNUNET_SERVICE_Client *client = cls;
795 int rid;
796 struct ReservationList *pos;
797 uint32_t size;
798
799 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
800 "Processing PUT request for `%s' of type %u\n",
801 GNUNET_h2s (&dm->key),
802 (uint32_t) ntohl (dm->type));
803 rid = ntohl (dm->rid);
804 size = ntohl (dm->size);
805 if (rid > 0)
806 {
807 pos = reservations;
808 while ((NULL != pos) && (rid != pos->rid))
809 pos = pos->next;
810 GNUNET_break (pos != NULL);
811 if (NULL != pos)
812 {
813 GNUNET_break (pos->entries > 0);
814 GNUNET_break (pos->amount >= size);
815 pos->entries--;
816 pos->amount -= size;
817 reserved -= (size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
818 GNUNET_STATISTICS_set (stats,
819 gettext_noop ("# reserved"),
820 reserved,
821 GNUNET_NO);
822 }
823 }
824 bool absent =
825 GNUNET_NO == GNUNET_CONTAINER_bloomfilter_test (filter, &dm->key);
826 plugin->api->put (plugin->api->cls,
827 &dm->key,
828 absent,
829 ntohl (dm->size),
830 &dm[1],
831 ntohl (dm->type),
832 ntohl (dm->priority),
833 ntohl (dm->anonymity),
834 ntohl (dm->replication),
835 GNUNET_TIME_absolute_ntoh (dm->expiration),
836 &put_continuation,
837 client);
838 GNUNET_SERVICE_client_continue (client);
839}
840
841
842/**
843 * Handle #GNUNET_MESSAGE_TYPE_DATASTORE_GET-message.
844 *
845 * @param cls identification of the client
846 * @param msg the actual message
847 */
848static void
849handle_get (void *cls, const struct GetMessage *msg)
850{
851 struct GNUNET_SERVICE_Client *client = cls;
852
853 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
854 "Processing GET request of type %u\n",
855 (uint32_t) ntohl (msg->type));
856 GNUNET_STATISTICS_update (stats,
857 gettext_noop ("# GET requests received"),
858 1,
859 GNUNET_NO);
860 plugin->api->get_key (plugin->api->cls,
861 GNUNET_ntohll (msg->next_uid),
862 msg->random,
863 NULL,
864 ntohl (msg->type),
865 &transmit_item,
866 client);
867 GNUNET_SERVICE_client_continue (client);
868}
869
870
871/**
872 * Handle #GNUNET_MESSAGE_TYPE_DATASTORE_GET_KEY-message.
873 *
874 * @param cls closure
875 * @param msg the actual message
876 */
877static void
878handle_get_key (void *cls, const struct GetKeyMessage *msg)
879{
880 struct GNUNET_SERVICE_Client *client = cls;
881
882 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
883 "Processing GET request for `%s' of type %u\n",
884 GNUNET_h2s (&msg->key),
885 (uint32_t) ntohl (msg->type));
886 GNUNET_STATISTICS_update (stats,
887 gettext_noop ("# GET KEY requests received"),
888 1,
889 GNUNET_NO);
890 if (GNUNET_YES != GNUNET_CONTAINER_bloomfilter_test (filter, &msg->key))
891 {
892 /* don't bother database... */
893 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
894 "Empty result set for GET request for `%s' (bloomfilter).\n",
895 GNUNET_h2s (&msg->key));
896 GNUNET_STATISTICS_update (stats,
897 gettext_noop (
898 "# requests filtered by bloomfilter"),
899 1,
900 GNUNET_NO);
901 transmit_item (client,
902 NULL,
903 0,
904 NULL,
905 0,
906 0,
907 0,
908 0,
909 GNUNET_TIME_UNIT_ZERO_ABS,
910 0);
911 GNUNET_SERVICE_client_continue (client);
912 return;
913 }
914 plugin->api->get_key (plugin->api->cls,
915 GNUNET_ntohll (msg->next_uid),
916 msg->random,
917 &msg->key,
918 ntohl (msg->type),
919 &transmit_item,
920 client);
921 GNUNET_SERVICE_client_continue (client);
922}
923
924
925/**
926 * Handle GET_REPLICATION-message.
927 *
928 * @param cls identification of the client
929 * @param message the actual message
930 */
931static void
932handle_get_replication (void *cls, const struct GNUNET_MessageHeader *message)
933{
934 struct GNUNET_SERVICE_Client *client = cls;
935
936 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Processing GET_REPLICATION request\n");
937 GNUNET_STATISTICS_update (stats,
938 gettext_noop (
939 "# GET REPLICATION requests received"),
940 1,
941 GNUNET_NO);
942 plugin->api->get_replication (plugin->api->cls, &transmit_item, client);
943 GNUNET_SERVICE_client_continue (client);
944}
945
946
947/**
948 * Handle GET_ZERO_ANONYMITY-message.
949 *
950 * @param cls client identification of the client
951 * @param message the actual message
952 */
953static void
954handle_get_zero_anonymity (void *cls, const struct GetZeroAnonymityMessage *msg)
955{
956 struct GNUNET_SERVICE_Client *client = cls;
957 enum GNUNET_BLOCK_Type type;
958
959 type = (enum GNUNET_BLOCK_Type) ntohl (msg->type);
960 if (type == GNUNET_BLOCK_TYPE_ANY)
961 {
962 GNUNET_break (0);
963 GNUNET_SERVICE_client_drop (client);
964 return;
965 }
966 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
967 "Processing GET_ZERO_ANONYMITY request\n");
968 GNUNET_STATISTICS_update (stats,
969 gettext_noop (
970 "# GET ZERO ANONYMITY requests received"),
971 1,
972 GNUNET_NO);
973 plugin->api->get_zero_anonymity (plugin->api->cls,
974 GNUNET_ntohll (msg->next_uid),
975 type,
976 &transmit_item,
977 client);
978 GNUNET_SERVICE_client_continue (client);
979}
980
981
982/**
983 * Remove continuation.
984 *
985 * @param cls closure
986 * @param key key for the content
987 * @param size number of bytes in data
988 * @param status #GNUNET_OK if removed, #GNUNET_NO if not found,
989 * or #GNUNET_SYSERROR if error
990 * @param msg error message on error
991 */
992static void
993remove_continuation (void *cls,
994 const struct GNUNET_HashCode *key,
995 uint32_t size,
996 int status,
997 const char *msg)
998{
999 struct GNUNET_SERVICE_Client *client = cls;
1000
1001 if (GNUNET_SYSERR == status)
1002 {
1003 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "REMOVE request failed: %s.\n", msg);
1004 transmit_status (client, GNUNET_NO, msg);
1005 return;
1006 }
1007 if (GNUNET_NO == status)
1008 {
1009 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1010 "Content not found for REMOVE request.\n");
1011 transmit_status (client, GNUNET_NO, _ ("Content not found"));
1012 return;
1013 }
1014 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1015 "Item matches REMOVE request for key `%s'.\n",
1016 GNUNET_h2s (key));
1017 GNUNET_STATISTICS_update (stats,
1018 gettext_noop ("# bytes removed (explicit request)"),
1019 size,
1020 GNUNET_YES);
1021 GNUNET_CONTAINER_bloomfilter_remove (filter, key);
1022 transmit_status (client, GNUNET_OK, NULL);
1023}
1024
1025
1026/**
1027 * Verify REMOVE-message.
1028 *
1029 * @param cls identification of the client
1030 * @param message the actual message
1031 * @return #GNUNET_OK if @a dm is well-formed
1032 */
1033static int
1034check_remove (void *cls, const struct DataMessage *dm)
1035{
1036 if (GNUNET_OK != check_data (dm))
1037 {
1038 GNUNET_break (0);
1039 return GNUNET_SYSERR;
1040 }
1041 return GNUNET_OK;
1042}
1043
1044
1045/**
1046 * Handle REMOVE-message.
1047 *
1048 * @param cls closure
1049 * @param client identification of the client
1050 * @param message the actual message
1051 */
1052static void
1053handle_remove (void *cls, const struct DataMessage *dm)
1054{
1055 struct GNUNET_SERVICE_Client *client = cls;
1056
1057 GNUNET_STATISTICS_update (stats,
1058 gettext_noop ("# REMOVE requests received"),
1059 1,
1060 GNUNET_NO);
1061 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1062 "Processing REMOVE request for `%s'\n",
1063 GNUNET_h2s (&dm->key));
1064 plugin->api->remove_key (plugin->api->cls,
1065 &dm->key,
1066 ntohl (dm->size),
1067 &dm[1],
1068 &remove_continuation,
1069 client);
1070 GNUNET_SERVICE_client_continue (client);
1071}
1072
1073
1074/**
1075 * Handle DROP-message.
1076 *
1077 * @param cls identification of the client
1078 * @param message the actual message
1079 */
1080static void
1081handle_drop (void *cls, const struct GNUNET_MessageHeader *message)
1082{
1083 struct GNUNET_SERVICE_Client *client = cls;
1084
1085 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Processing DROP request\n");
1086 do_drop = GNUNET_YES;
1087 GNUNET_SERVICE_client_continue (client);
1088}
1089
1090
1091/**
1092 * Function called by plugins to notify us about a
1093 * change in their disk utilization.
1094 *
1095 * @param cls closure (NULL)
1096 * @param delta change in disk utilization,
1097 * 0 for "reset to empty"
1098 */
1099static void
1100disk_utilization_change_cb (void *cls, int delta)
1101{
1102 if ((delta < 0) && (payload < -delta))
1103 {
1104 GNUNET_log (
1105 GNUNET_ERROR_TYPE_WARNING,
1106 _ (
1107 "Datastore payload must have been inaccurate (%lld < %lld). Recomputing it.\n"),
1108 (long long) payload,
1109 (long long) -delta);
1110 plugin->api->estimate_size (plugin->api->cls, &payload);
1111 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1112 _ ("New payload: %lld\n"),
1113 (long long) payload);
1114 sync_stats ();
1115 return;
1116 }
1117 payload += delta;
1118 last_sync++;
1119 if (last_sync >= MAX_STAT_SYNC_LAG)
1120 sync_stats ();
1121}
1122
1123
1124/**
1125 * Callback function to process statistic values.
1126 *
1127 * @param cls closure (struct Plugin*)
1128 * @param subsystem name of subsystem that created the statistic
1129 * @param name the name of the datum
1130 * @param value the current value
1131 * @param is_persistent #GNUNET_YES if the value is persistent, #GNUNET_NO if not
1132 * @return #GNUNET_OK to continue, #GNUNET_SYSERR to abort iteration
1133 */
1134static int
1135process_stat_in (void *cls,
1136 const char *subsystem,
1137 const char *name,
1138 uint64_t value,
1139 int is_persistent)
1140{
1141 GNUNET_assert (GNUNET_NO == stats_worked);
1142 stats_worked = GNUNET_YES;
1143 payload += value;
1144 GNUNET_log (
1145 GNUNET_ERROR_TYPE_DEBUG,
1146 "Notification from statistics about existing payload (%llu), new payload is %llu\n",
1147 (unsigned long long) value,
1148 (unsigned long long) payload);
1149 return GNUNET_OK;
1150}
1151
1152
1153/**
1154 * Load the datastore plugin.
1155 */
1156static struct DatastorePlugin *
1157load_plugin ()
1158{
1159 struct DatastorePlugin *ret;
1160 char *libname;
1161
1162 ret = GNUNET_new (struct DatastorePlugin);
1163 ret->env.cfg = cfg;
1164 ret->env.duc = &disk_utilization_change_cb;
1165 ret->env.cls = NULL;
1166 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1167 _ ("Loading `%s' datastore plugin\n"),
1168 plugin_name);
1169 GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", plugin_name);
1170 ret->short_name = GNUNET_strdup (plugin_name);
1171 ret->lib_name = libname;
1172 ret->api = GNUNET_PLUGIN_load (libname, &ret->env);
1173 if (NULL == ret->api)
1174 {
1175 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1176 _ ("Failed to load datastore plugin for `%s'\n"),
1177 plugin_name);
1178 GNUNET_free (ret->short_name);
1179 GNUNET_free (libname);
1180 GNUNET_free (ret);
1181 return NULL;
1182 }
1183 return ret;
1184}
1185
1186
1187/**
1188 * Function called when the service shuts
1189 * down. Unloads our datastore plugin.
1190 *
1191 * @param plug plugin to unload
1192 */
1193static void
1194unload_plugin (struct DatastorePlugin *plug)
1195{
1196 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1197 "Datastore service is unloading plugin...\n");
1198 GNUNET_break (NULL == GNUNET_PLUGIN_unload (plug->lib_name, plug->api));
1199 GNUNET_free (plug->lib_name);
1200 GNUNET_free (plug->short_name);
1201 GNUNET_free (plug);
1202}
1203
1204
1205/**
1206 * Initialization complete, start operating the service.
1207 */
1208static void
1209begin_service ()
1210{
1211 GNUNET_SERVICE_resume (service);
1212 expired_kill_task =
1213 GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_IDLE,
1214 &delete_expired,
1215 NULL);
1216}
1217
1218
1219/**
1220 * Adds a given @a key to the bloomfilter in @a cls @a count times.
1221 *
1222 * @param cls the bloomfilter
1223 * @param key key to add
1224 * @param count number of times to add key
1225 */
1226static void
1227add_key_to_bloomfilter (void *cls,
1228 const struct GNUNET_HashCode *key,
1229 unsigned int count)
1230{
1231 struct GNUNET_CONTAINER_BloomFilter *bf = cls;
1232
1233 if (NULL == key)
1234 {
1235 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1236 _ ("Bloomfilter construction complete.\n"));
1237 begin_service ();
1238 return;
1239 }
1240
1241 while (0 < count--)
1242 GNUNET_CONTAINER_bloomfilter_add (bf, key);
1243}
1244
1245
1246/**
1247 * We finished receiving the statistic. Initialize the plugin; if
1248 * loading the statistic failed, run the estimator.
1249 *
1250 * @param cls NULL
1251 * @param success #GNUNET_NO if we failed to read the stat
1252 */
1253static void
1254process_stat_done (void *cls, int success)
1255{
1256 stat_get = NULL;
1257 if (NULL != stat_timeout_task)
1258 {
1259 GNUNET_SCHEDULER_cancel (stat_timeout_task);
1260 stat_timeout_task = NULL;
1261 }
1262 plugin = load_plugin ();
1263 if (NULL == plugin)
1264 {
1265 GNUNET_CONTAINER_bloomfilter_free (filter);
1266 filter = NULL;
1267 if (NULL != stats)
1268 {
1269 GNUNET_STATISTICS_destroy (stats, GNUNET_YES);
1270 stats = NULL;
1271 }
1272 return;
1273 }
1274
1275 if (GNUNET_NO == stats_worked)
1276 {
1277 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1278 "Failed to obtain value from statistics service, recomputing it\n");
1279 plugin->api->estimate_size (plugin->api->cls, &payload);
1280 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1281 _ ("New payload: %lld\n"),
1282 (long long) payload);
1283 }
1284
1285 if (GNUNET_YES == refresh_bf)
1286 {
1287 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1288 _ ("Rebuilding bloomfilter. Please be patient.\n"));
1289 if (NULL != plugin->api->get_keys)
1290 {
1291 plugin->api->get_keys (plugin->api->cls, &add_key_to_bloomfilter, filter);
1292 return;
1293 }
1294 else
1295 {
1296 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1297 _ (
1298 "Plugin does not support get_keys function. Please fix!\n"));
1299 }
1300 }
1301 begin_service ();
1302}
1303
1304
1305/**
1306 * Fetching stats took to long, run without.
1307 *
1308 * @param cls NULL
1309 */
1310static void
1311stat_timeout (void *cls)
1312{
1313 stat_timeout_task = NULL;
1314 GNUNET_STATISTICS_get_cancel (stat_get);
1315 process_stat_done (NULL, GNUNET_NO);
1316}
1317
1318
1319/**
1320 * Task run during shutdown.
1321 */
1322static void
1323cleaning_task (void *cls)
1324{
1325 cleaning_done = GNUNET_YES;
1326 if (NULL != expired_kill_task)
1327 {
1328 GNUNET_SCHEDULER_cancel (expired_kill_task);
1329 expired_kill_task = NULL;
1330 }
1331 if (GNUNET_YES == do_drop)
1332 {
1333 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Dropping database!\n");
1334 plugin->api->drop (plugin->api->cls);
1335 payload = 0;
1336 last_sync++;
1337 }
1338 if (NULL != plugin)
1339 {
1340 unload_plugin (plugin);
1341 plugin = NULL;
1342 }
1343 if (NULL != filter)
1344 {
1345 GNUNET_CONTAINER_bloomfilter_free (filter);
1346 filter = NULL;
1347 }
1348 if (NULL != stat_get)
1349 {
1350 GNUNET_STATISTICS_get_cancel (stat_get);
1351 stat_get = NULL;
1352 }
1353 if (NULL != stat_timeout_task)
1354 {
1355 GNUNET_SCHEDULER_cancel (stat_timeout_task);
1356 stat_timeout_task = NULL;
1357 }
1358 GNUNET_free (plugin_name);
1359 plugin_name = NULL;
1360 if (last_sync > 0)
1361 sync_stats ();
1362 if (NULL != stats)
1363 {
1364 GNUNET_STATISTICS_destroy (stats, GNUNET_YES);
1365 stats = NULL;
1366 }
1367 GNUNET_free (quota_stat_name);
1368 quota_stat_name = NULL;
1369}
1370
1371
1372/**
1373 * Add a client to our list of active clients.
1374 *
1375 * @param cls NULL
1376 * @param client client to add
1377 * @param mq message queue for @a client
1378 * @return @a client
1379 */
1380static void *
1381client_connect_cb (void *cls,
1382 struct GNUNET_SERVICE_Client *client,
1383 struct GNUNET_MQ_Handle *mq)
1384{
1385 return client;
1386}
1387
1388
1389/**
1390 * Called whenever a client is disconnected.
1391 * Frees our resources associated with that client.
1392 *
1393 * @param cls closure
1394 * @param client identification of the client
1395 * @param app_ctx must match @a client
1396 */
1397static void
1398client_disconnect_cb (void *cls,
1399 struct GNUNET_SERVICE_Client *client,
1400 void *app_ctx)
1401{
1402 struct ReservationList *pos;
1403 struct ReservationList *prev;
1404 struct ReservationList *next;
1405
1406 GNUNET_assert (app_ctx == client);
1407 prev = NULL;
1408 pos = reservations;
1409 while (NULL != pos)
1410 {
1411 next = pos->next;
1412 if (pos->client == client)
1413 {
1414 if (NULL == prev)
1415 reservations = next;
1416 else
1417 prev->next = next;
1418 reserved -= pos->amount + pos->entries * GNUNET_DATASTORE_ENTRY_OVERHEAD;
1419 GNUNET_free (pos);
1420 }
1421 else
1422 {
1423 prev = pos;
1424 }
1425 pos = next;
1426 }
1427 GNUNET_STATISTICS_set (stats,
1428 gettext_noop ("# reserved"),
1429 reserved,
1430 GNUNET_NO);
1431}
1432
1433
1434/**
1435 * Process datastore requests.
1436 *
1437 * @param cls closure
1438 * @param serv the initialized service
1439 * @param c configuration to use
1440 */
1441static void
1442run (void *cls,
1443 const struct GNUNET_CONFIGURATION_Handle *c,
1444 struct GNUNET_SERVICE_Handle *serv)
1445{
1446 char *fn;
1447 char *pfn;
1448 unsigned int bf_size;
1449
1450 service = serv;
1451 cfg = c;
1452 if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_string (cfg,
1453 "DATASTORE",
1454 "DATABASE",
1455 &plugin_name))
1456 {
1457 GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
1458 "DATABASE",
1459 "DATASTORE");
1460 return;
1461 }
1462 GNUNET_asprintf (&quota_stat_name,
1463 _ ("# bytes used in file-sharing datastore `%s'"),
1464 plugin_name);
1465 if (GNUNET_OK !=
1466 GNUNET_CONFIGURATION_get_value_size (cfg, "DATASTORE", "QUOTA", &quota))
1467 {
1468 GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR, "QUOTA", "DATASTORE");
1469 return;
1470 }
1471 stats = GNUNET_STATISTICS_create ("datastore", cfg);
1472 GNUNET_STATISTICS_set (stats, gettext_noop ("# quota"), quota, GNUNET_NO);
1473 cache_size = quota / 8; /* Or should we make this an option? */
1474 GNUNET_STATISTICS_set (stats,
1475 gettext_noop ("# cache size"),
1476 cache_size,
1477 GNUNET_NO);
1478 if (quota / (32 * 1024LL) > MAX_BF_SIZE)
1479 bf_size = MAX_BF_SIZE;
1480 else
1481 bf_size =
1482 quota / (32 * 1024LL); /* 8 bit per entry, 1 bit per 32 kb in DB */
1483 fn = NULL;
1484 if ((GNUNET_OK != GNUNET_CONFIGURATION_get_value_filename (cfg,
1485 "DATASTORE",
1486 "BLOOMFILTER",
1487 &fn)) ||
1488 (GNUNET_OK != GNUNET_DISK_directory_create_for_file (fn)))
1489 {
1490 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1491 _ ("Could not use specified filename `%s' for bloomfilter.\n"),
1492 NULL != fn ? fn : "");
1493 GNUNET_free (fn);
1494 fn = NULL;
1495 }
1496 if (NULL != fn)
1497 {
1498 GNUNET_asprintf (&pfn, "%s.%s", fn, plugin_name);
1499 if (GNUNET_YES == GNUNET_DISK_file_test (pfn))
1500 {
1501 filter =
1502 GNUNET_CONTAINER_bloomfilter_load (pfn,
1503 bf_size,
1504 5); /* approx. 3% false positives at max use */
1505 if (NULL == filter)
1506 {
1507 /* file exists but not valid, remove and try again, but refresh */
1508 if (0 != unlink (pfn))
1509 {
1510 /* failed to remove, run without file */
1511 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1512 _ ("Failed to remove bogus bloomfilter file `%s'\n"),
1513 pfn);
1514 GNUNET_free (pfn);
1515 pfn = NULL;
1516 filter = GNUNET_CONTAINER_bloomfilter_load (
1517 NULL,
1518 bf_size,
1519 5); /* approx. 3% false positives at max use */
1520 refresh_bf = GNUNET_YES;
1521 }
1522 else
1523 {
1524 /* try again after remove */
1525 filter = GNUNET_CONTAINER_bloomfilter_load (
1526 pfn,
1527 bf_size,
1528 5); /* approx. 3% false positives at max use */
1529 refresh_bf = GNUNET_YES;
1530 if (NULL == filter)
1531 {
1532 /* failed yet again, give up on using file */
1533 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1534 _ ("Failed to remove bogus bloomfilter file `%s'\n"),
1535 pfn);
1536 GNUNET_free (pfn);
1537 pfn = NULL;
1538 filter = GNUNET_CONTAINER_bloomfilter_init (
1539 NULL,
1540 bf_size,
1541 5); /* approx. 3% false positives at max use */
1542 }
1543 }
1544 }
1545 else
1546 {
1547 /* normal case: have an existing valid bf file, no need to refresh */
1548 refresh_bf = GNUNET_NO;
1549 }
1550 }
1551 else
1552 {
1553 filter =
1554 GNUNET_CONTAINER_bloomfilter_load (pfn,
1555 bf_size,
1556 5); /* approx. 3% false positives at max use */
1557 refresh_bf = GNUNET_YES;
1558 }
1559 GNUNET_free (pfn);
1560 }
1561 else
1562 {
1563 filter =
1564 GNUNET_CONTAINER_bloomfilter_init (NULL,
1565 bf_size,
1566 5); /* approx. 3% false positives at max use */
1567 refresh_bf = GNUNET_YES;
1568 }
1569 GNUNET_free (fn);
1570 if (NULL == filter)
1571 {
1572 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1573 _ ("Failed to initialize bloomfilter.\n"));
1574 if (NULL != stats)
1575 {
1576 GNUNET_STATISTICS_destroy (stats, GNUNET_YES);
1577 stats = NULL;
1578 }
1579 return;
1580 }
1581 GNUNET_SERVICE_suspend (service);
1582 stat_get = GNUNET_STATISTICS_get (stats,
1583 "datastore",
1584 quota_stat_name,
1585 &process_stat_done,
1586 &process_stat_in,
1587 NULL);
1588 if (NULL == stat_get)
1589 process_stat_done (NULL, GNUNET_SYSERR);
1590 else
1591 stat_timeout_task = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_SECONDS,
1592 &stat_timeout,
1593 NULL);
1594 GNUNET_SCHEDULER_add_shutdown (&cleaning_task, NULL);
1595}
1596
1597
1598/**
1599 * Define "main" method using service macro.
1600 */
1601GNUNET_SERVICE_MAIN (
1602 "datastore",
1603 GNUNET_SERVICE_OPTION_NONE,
1604 &run,
1605 &client_connect_cb,
1606 &client_disconnect_cb,
1607 NULL,
1608 GNUNET_MQ_hd_fixed_size (reserve,
1609 GNUNET_MESSAGE_TYPE_DATASTORE_RESERVE,
1610 struct ReserveMessage,
1611 NULL),
1612 GNUNET_MQ_hd_fixed_size (release_reserve,
1613 GNUNET_MESSAGE_TYPE_DATASTORE_RELEASE_RESERVE,
1614 struct ReleaseReserveMessage,
1615 NULL),
1616 GNUNET_MQ_hd_var_size (put,
1617 GNUNET_MESSAGE_TYPE_DATASTORE_PUT,
1618 struct DataMessage,
1619 NULL),
1620 GNUNET_MQ_hd_fixed_size (get,
1621 GNUNET_MESSAGE_TYPE_DATASTORE_GET,
1622 struct GetMessage,
1623 NULL),
1624 GNUNET_MQ_hd_fixed_size (get_key,
1625 GNUNET_MESSAGE_TYPE_DATASTORE_GET_KEY,
1626 struct GetKeyMessage,
1627 NULL),
1628 GNUNET_MQ_hd_fixed_size (get_replication,
1629 GNUNET_MESSAGE_TYPE_DATASTORE_GET_REPLICATION,
1630 struct GNUNET_MessageHeader,
1631 NULL),
1632 GNUNET_MQ_hd_fixed_size (get_zero_anonymity,
1633 GNUNET_MESSAGE_TYPE_DATASTORE_GET_ZERO_ANONYMITY,
1634 struct GetZeroAnonymityMessage,
1635 NULL),
1636 GNUNET_MQ_hd_var_size (remove,
1637 GNUNET_MESSAGE_TYPE_DATASTORE_REMOVE,
1638 struct DataMessage,
1639 NULL),
1640 GNUNET_MQ_hd_fixed_size (drop,
1641 GNUNET_MESSAGE_TYPE_DATASTORE_DROP,
1642 struct GNUNET_MessageHeader,
1643 NULL),
1644 GNUNET_MQ_handler_end ());
1645
1646
1647/* end of gnunet-service-datastore.c */
diff --git a/src/datastore/perf_datastore_api.c b/src/datastore/perf_datastore_api.c
deleted file mode 100644
index fef38891e..000000000
--- a/src/datastore/perf_datastore_api.c
+++ /dev/null
@@ -1,629 +0,0 @@
1/*
2 This file is part of GNUnet.
3 Copyright (C) 2004, 2005, 2006, 2007, 2009, 2011, 2015 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20/*
21 * @file datastore/perf_datastore_api.c
22 * @brief performance measurement for the datastore implementation
23 * @author Christian Grothoff
24 *
25 * This testcase inserts a bunch of (variable size) data and then
26 * deletes data until the (reported) database size drops below a given
27 * threshold. This is iterated 10 times, with the actual size of the
28 * content stored and the number of operations performed being printed
29 * for each iteration. The code also prints a "I" for every 40 blocks
30 * inserted and a "D" for every 40 blocks deleted. The deletion
31 * strategy uses the "random" iterator. Priorities and expiration
32 * dates are set using a pseudo-random value within a realistic range.
33 */
34#include "platform.h"
35#include "gnunet_util_lib.h"
36#include "gnunet_protocols.h"
37#include "gnunet_datastore_service.h"
38#include "gnunet_testing_lib.h"
39#include <gauger.h>
40
41/**
42 * How long until we give up on transmitting the message?
43 */
44#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 15)
45
46/**
47 * Target datastore size (in bytes).
48 */
49#define MAX_SIZE (1024LL * 1024 * 4)
50
51/**
52 * Report progress outside of major reports? Should probably be #GNUNET_YES if
53 * size is > 16 MB.
54 */
55#define REPORT_ID GNUNET_YES
56
57/**
58 * Number of put operations equivalent to 1/3rd of #MAX_SIZE
59 */
60#define PUT_10 MAX_SIZE / 32 / 1024 / 3
61
62/**
63 * Total number of iterations (each iteration doing
64 * PUT_10 put operations); we report full status every
65 * 10 iterations. Abort with CTRL-C.
66 */
67#define ITERATIONS 8
68
69/**
70 * Total number of iterations to do to go beyond the quota.
71 * The quota is set to 10 MB or 2.5 times #MAX_SIZE,
72 * so we got 16 times #MAX_SIZE to be sure to hit it a LOT.
73 */
74#define QUOTA_PUTS (MAX_SIZE / 32 / 1024 * 16LL)
75
76
77/**
78 * Number of bytes stored in the datastore in total.
79 */
80static unsigned long long stored_bytes;
81
82/**
83 * Number of entries stored in the datastore in total.
84 */
85static unsigned long long stored_entries;
86
87/**
88 * Number of database operations performed. Inserting
89 * counts as one operation, deleting as two (as deletion
90 * requires selecting a value for deletion first).
91 */
92static unsigned long long stored_ops;
93
94/**
95 * Start time of the benchmark.
96 */
97static struct GNUNET_TIME_Absolute start_time;
98
99/**
100 * Database backend we use.
101 */
102static const char *plugin_name;
103
104/**
105 * Handle to the datastore.
106 */
107static struct GNUNET_DATASTORE_Handle *datastore;
108
109/**
110 * Value we return from #main().
111 */
112static int ok;
113
114/**
115 * Which phase of the process are we in?
116 */
117enum RunPhase
118{
119 /**
120 * We are done (shutting down normally).
121 */
122 RP_DONE = 0,
123
124 /**
125 * We are adding new entries to the datastore.
126 */
127 RP_PUT,
128
129 /**
130 * We are deleting entries from the datastore.
131 */
132 RP_CUT,
133
134 /**
135 * We are putting as much as we can to see how the database performs
136 * when it reaches the quota and has to auto-delete (see #3903).
137 */
138 RP_PUT_QUOTA,
139
140 /**
141 * We are generating a report.
142 */
143 RP_REPORT,
144
145 /**
146 * Execution failed with some kind of error.
147 */
148 RP_ERROR
149};
150
151
152/**
153 * Closure we give to all of the functions executing the
154 * benchmark. Could right now be global, but this allows
155 * us to theoretically run multiple clients "in parallel".
156 */
157struct CpsRunContext
158{
159 /**
160 * Execution phase we are in.
161 */
162 enum RunPhase phase;
163
164 /**
165 * Size of the value we are currently storing (during #RP_PUT).
166 */
167 size_t size;
168
169 /**
170 * Current iteration counter, we are done with the benchmark
171 * once it hits #ITERATIONS.
172 */
173 unsigned int i;
174
175 /**
176 * Counts the number of items put in the current phase.
177 * Once it hits #PUT_10, we progress to the #RP_CUT phase
178 * or are done if @e i reaches #ITERATIONS.
179 */
180 unsigned int j;
181};
182
183
184/**
185 * Main state machine. Executes the next step of the benchmark
186 * depending on the current state.
187 *
188 * @param cls the `struct CpsRunContext`
189 */
190static void
191run_continuation (void *cls);
192
193
194/**
195 * Continuation called to notify client about result of the insertion
196 * operation. Checks for errors, updates our iteration counters and
197 * continues execution with #run_continuation().
198 *
199 * @param cls the `struct CpsRunContext`
200 * @param success #GNUNET_SYSERR on failure
201 * @param min_expiration minimum expiration time required for content to be stored
202 * by the datacache at this time, zero for unknown
203 * @param msg NULL on success, otherwise an error message
204 */
205static void
206check_success (void *cls,
207 int success,
208 struct GNUNET_TIME_Absolute min_expiration,
209 const char *msg)
210{
211 struct CpsRunContext *crc = cls;
212
213#if REPORT_ID
214 fprintf (stderr, "%s", (GNUNET_OK == success) ? "I" : "i");
215#endif
216 if (GNUNET_OK != success)
217 {
218 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
219 "Check success failed: `%s'\n",
220 msg);
221 crc->phase = RP_ERROR;
222 GNUNET_SCHEDULER_add_now (&run_continuation,
223 crc);
224 return;
225 }
226 stored_bytes += crc->size;
227 stored_ops++;
228 stored_entries++;
229 crc->j++;
230 switch (crc->phase)
231 {
232 case RP_PUT:
233 if (crc->j >= PUT_10)
234 {
235 crc->j = 0;
236 crc->i++;
237 if (crc->i == ITERATIONS)
238 crc->phase = RP_PUT_QUOTA;
239 else
240 crc->phase = RP_CUT;
241 }
242 break;
243
244 case RP_PUT_QUOTA:
245 if (crc->j >= QUOTA_PUTS)
246 {
247 crc->j = 0;
248 crc->phase = RP_DONE;
249 }
250 break;
251
252 default:
253 GNUNET_assert (0);
254 }
255 GNUNET_SCHEDULER_add_now (&run_continuation,
256 crc);
257}
258
259
260/**
261 * Continuation called to notify client about result of the
262 * deletion operation. Checks for errors and continues
263 * execution with #run_continuation().
264 *
265 * @param cls the `struct CpsRunContext`
266 * @param success #GNUNET_SYSERR on failure
267 * @param min_expiration minimum expiration time required for content to be stored
268 * by the datacache at this time, zero for unknown
269 * @param msg NULL on success, otherwise an error message
270 */
271static void
272remove_next (void *cls,
273 int success,
274 struct GNUNET_TIME_Absolute min_expiration,
275 const char *msg)
276{
277 struct CpsRunContext *crc = cls;
278
279 if (GNUNET_OK != success)
280 {
281 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
282 "remove_next failed: `%s'\n",
283 msg);
284 crc->phase = RP_ERROR;
285 GNUNET_SCHEDULER_add_now (&run_continuation,
286 crc);
287 return;
288 }
289#if REPORT_ID
290 fprintf (stderr, "%s", "D");
291#endif
292 GNUNET_assert (GNUNET_OK == success);
293 GNUNET_SCHEDULER_add_now (&run_continuation,
294 crc);
295}
296
297
298/**
299 * We have selected a value for deletion, trigger removal.
300 *
301 * @param cls the `struct CpsRunContext`
302 * @param key key for the content
303 * @param size number of bytes in data
304 * @param data content stored
305 * @param type type of the content
306 * @param priority priority of the content
307 * @param anonymity anonymity-level for the content
308 * @param replication replication-level for the content
309 * @param expiration expiration time for the content
310 * @param uid unique identifier for the datum;
311 * maybe 0 if no unique identifier is available
312 */
313static void
314delete_value (void *cls,
315 const struct GNUNET_HashCode *key,
316 size_t size,
317 const void *data,
318 enum GNUNET_BLOCK_Type type,
319 uint32_t priority,
320 uint32_t anonymity,
321 uint32_t replication,
322 struct GNUNET_TIME_Absolute expiration,
323 uint64_t uid)
324{
325 struct CpsRunContext *crc = cls;
326
327 GNUNET_assert (NULL != key);
328 stored_ops++;
329 stored_bytes -= size;
330 stored_entries--;
331 stored_ops++;
332 if (stored_bytes < MAX_SIZE)
333 crc->phase = RP_PUT;
334 GNUNET_assert (NULL !=
335 GNUNET_DATASTORE_remove (datastore,
336 key,
337 size,
338 data, 1, 1,
339 &remove_next, crc));
340}
341
342
343/**
344 * Main state machine. Executes the next step of the benchmark
345 * depending on the current state.
346 *
347 * @param cls the `struct CpsRunContext`
348 */
349static void
350run_continuation (void *cls)
351{
352 struct CpsRunContext *crc = cls;
353 size_t size;
354 static struct GNUNET_HashCode key;
355 static char data[65536];
356 char gstr[128];
357
358 ok = (int) crc->phase;
359 switch (crc->phase)
360 {
361 case RP_PUT:
362 memset (&key,
363 256 - crc->i,
364 sizeof(struct GNUNET_HashCode));
365 /* most content is 32k */
366 size = 32 * 1024;
367 if (0 ==
368 GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
369 16)) /* but some of it is less! */
370 size = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
371 32 * 1024);
372 crc->size = size = size - (size & 7); /* always multiple of 8 */
373 GNUNET_CRYPTO_hash (&key,
374 sizeof(struct GNUNET_HashCode),
375 &key);
376 memset (data,
377 (int) crc->j,
378 size);
379 if (crc->j > 255)
380 memset (data,
381 (int) (crc->j - 255),
382 size / 2);
383 data[0] = crc->i;
384 GNUNET_assert (NULL !=
385 GNUNET_DATASTORE_put (datastore,
386 0,
387 &key,
388 size,
389 data,
390 crc->j + 1,
391 GNUNET_CRYPTO_random_u32
392 (GNUNET_CRYPTO_QUALITY_WEAK, 100),
393 crc->j,
394 0,
395 GNUNET_TIME_relative_to_absolute
396 (GNUNET_TIME_relative_multiply
397 (GNUNET_TIME_UNIT_SECONDS,
398 GNUNET_CRYPTO_random_u32
399 (GNUNET_CRYPTO_QUALITY_WEAK,
400 1000))),
401 1,
402 1,
403 &check_success, crc));
404 break;
405
406 case RP_CUT:
407 /* trim down below MAX_SIZE again */
408 GNUNET_assert (NULL !=
409 GNUNET_DATASTORE_get_for_replication (datastore,
410 1, 1,
411 &delete_value,
412 crc));
413 break;
414
415 case RP_REPORT:
416 printf (
417#if REPORT_ID
418 "\n"
419#endif
420 "Stored %llu kB / %lluk ops / %llu ops/s\n",
421 stored_bytes / 1024, /* used size in k */
422 stored_ops / 1024, /* total operations (in k) */
423 1000LL * 1000LL * stored_ops / (1
424 + GNUNET_TIME_absolute_get_duration
425 (start_time).rel_value_us));
426 crc->phase = RP_PUT;
427 crc->j = 0;
428 GNUNET_SCHEDULER_add_now (&run_continuation,
429 crc);
430 break;
431
432 case RP_PUT_QUOTA:
433 memset (&key,
434 256 - crc->i,
435 sizeof(struct GNUNET_HashCode));
436 /* most content is 32k */
437 size = 32 * 1024;
438 if (0 ==
439 GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
440 16)) /* but some of it is less! */
441 size = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
442 32 * 1024);
443 crc->size = size = size - (size & 7); /* always multiple of 8 */
444 GNUNET_CRYPTO_hash (&key,
445 sizeof(struct GNUNET_HashCode),
446 &key);
447 memset (data,
448 (int) crc->j,
449 size);
450 if (crc->j > 255)
451 memset (data,
452 (int) (crc->j - 255),
453 size / 2);
454 data[0] = crc->i;
455 GNUNET_assert (NULL !=
456 GNUNET_DATASTORE_put (datastore,
457 0, /* reservation ID */
458 &key,
459 size,
460 data,
461 crc->j + 1, /* type */
462 GNUNET_CRYPTO_random_u32
463 (GNUNET_CRYPTO_QUALITY_WEAK,
464 100), /* priority */
465 crc->j, /* anonymity */
466 0, /* replication */
467 GNUNET_TIME_relative_to_absolute
468 (GNUNET_TIME_relative_multiply
469 (GNUNET_TIME_UNIT_SECONDS,
470 GNUNET_CRYPTO_random_u32
471 (GNUNET_CRYPTO_QUALITY_WEAK,
472 1000))),
473 1,
474 1,
475 &check_success, crc));
476 break;
477
478 case RP_DONE:
479 GNUNET_snprintf (gstr,
480 sizeof(gstr),
481 "DATASTORE-%s",
482 plugin_name);
483 if ((crc->i == ITERATIONS) && (stored_ops > 0))
484 {
485 GAUGER (gstr,
486 "PUT operation duration",
487 GNUNET_TIME_absolute_get_duration (start_time).rel_value_us
488 / 1000LL
489 / stored_ops,
490 "ms/operation");
491 fprintf (stdout,
492 "\nPUT performance: %s for %llu operations\n",
493 GNUNET_STRINGS_relative_time_to_string (
494 GNUNET_TIME_absolute_get_duration (start_time),
495 GNUNET_YES),
496 stored_ops);
497 fprintf (stdout,
498 "PUT performance: %llu ms/operation\n",
499 GNUNET_TIME_absolute_get_duration (start_time).rel_value_us
500 / 1000LL
501 / stored_ops);
502 }
503 GNUNET_DATASTORE_disconnect (datastore,
504 GNUNET_YES);
505 GNUNET_free (crc);
506 ok = 0;
507 break;
508
509 case RP_ERROR:
510 GNUNET_DATASTORE_disconnect (datastore, GNUNET_YES);
511 GNUNET_free (crc);
512 ok = 1;
513 break;
514
515 default:
516 GNUNET_assert (0);
517 }
518}
519
520
521/**
522 * Function called with the result of the initial PUT operation. If
523 * the PUT succeeded, we start the actual benchmark loop, otherwise we
524 * bail out with an error.
525 *
526 *
527 * @param cls closure
528 * @param success #GNUNET_SYSERR on failure
529 * @param min_expiration minimum expiration time required for content to be stored
530 * by the datacache at this time, zero for unknown
531 * @param msg NULL on success, otherwise an error message
532 */
533static void
534run_tests (void *cls,
535 int success,
536 struct GNUNET_TIME_Absolute min_expiration,
537 const char *msg)
538{
539 struct CpsRunContext *crc = cls;
540
541 if (success != GNUNET_YES)
542 {
543 fprintf (stderr,
544 "Test 'put' operation failed with error `%s' database likely not setup, skipping test.\n",
545 msg);
546 GNUNET_DATASTORE_disconnect (datastore,
547 GNUNET_YES);
548 GNUNET_free (crc);
549 return;
550 }
551 GNUNET_SCHEDULER_add_now (&run_continuation,
552 crc);
553}
554
555
556/**
557 * Beginning of the actual execution of the benchmark.
558 * Performs a first test operation (PUT) to verify that
559 * the plugin works at all.
560 *
561 * @param cls NULL
562 * @param cfg configuration to use
563 * @param peer peer handle (unused)
564 */
565static void
566run (void *cls,
567 const struct GNUNET_CONFIGURATION_Handle *cfg,
568 struct GNUNET_TESTING_Peer *peer)
569{
570 struct CpsRunContext *crc;
571 static struct GNUNET_HashCode zkey;
572
573 datastore = GNUNET_DATASTORE_connect (cfg);
574 start_time = GNUNET_TIME_absolute_get ();
575 crc = GNUNET_new (struct CpsRunContext);
576 crc->phase = RP_PUT;
577 if (NULL ==
578 GNUNET_DATASTORE_put (datastore,
579 0,
580 &zkey,
581 4, "TEST",
582 GNUNET_BLOCK_TYPE_TEST,
583 0, 0, 0,
584 GNUNET_TIME_relative_to_absolute (
585 GNUNET_TIME_UNIT_SECONDS),
586 0, 1,
587 &run_tests, crc))
588 {
589 fprintf (stderr,
590 "%s",
591 "Test 'put' operation failed.\n");
592 ok = 1;
593 GNUNET_free (crc);
594 }
595}
596
597
598/**
599 * Entry point into the test. Determines which configuration / plugin
600 * we are running with based on the name of the binary and starts
601 * the peer.
602 *
603 * @param argc should be 1
604 * @param argv used to determine plugin / configuration name.
605 * @return 0 on success
606 */
607int
608main (int argc,
609 char *argv[])
610{
611 char cfg_name[PATH_MAX];
612
613 plugin_name = GNUNET_TESTING_get_testname_from_underscore (argv[0]);
614 GNUNET_snprintf (cfg_name,
615 sizeof(cfg_name),
616 "test_datastore_api_data_%s.conf",
617 plugin_name);
618 if (0 !=
619 GNUNET_TESTING_peer_run ("perf-gnunet-datastore",
620 cfg_name,
621 &run,
622 NULL))
623 return 1;
624 fprintf (stderr, "%s", "\n");
625 return ok;
626}
627
628
629/* end of perf_datastore_api.c */
diff --git a/src/datastore/perf_plugin_datastore.c b/src/datastore/perf_plugin_datastore.c
deleted file mode 100644
index d7488d4e7..000000000
--- a/src/datastore/perf_plugin_datastore.c
+++ /dev/null
@@ -1,573 +0,0 @@
1/*
2 This file is part of GNUnet.
3 Copyright (C) 2004, 2005, 2006, 2007, 2009, 2011 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20/*
21 * @file perf_plugin_datastore.c
22 * @brief Profile database plugin directly, focusing on iterators.
23 * @author Christian Grothoff
24 */
25
26#include "platform.h"
27#include "gnunet_util_lib.h"
28#include "gnunet_protocols.h"
29#include "gnunet_datastore_plugin.h"
30#include "gnunet_testing_lib.h"
31#include <gauger.h>
32
33/**
34 * Target datastore size (in bytes). Realistic sizes are
35 * more like 16 GB (not the default of 16 MB); however,
36 * those take too long to run them in the usual "make check"
37 * sequence. Hence the value used for shipping is tiny.
38 */
39#define MAX_SIZE 1024LL * 1024 * 16 * 1
40
41#define ITERATIONS 2
42
43/**
44 * Number of put operations equivalent to 1/10th of MAX_SIZE
45 */
46#define PUT_10 (MAX_SIZE / 32 / 1024 / ITERATIONS)
47
48static char category[256];
49
50static unsigned int hits[PUT_10 / 8 + 1];
51
52static unsigned long long stored_bytes;
53
54static unsigned long long stored_entries;
55
56static unsigned long long stored_ops;
57
58static const char *plugin_name;
59
60static int ok;
61
62enum RunPhase
63{
64 RP_ERROR = 0,
65 RP_PUT,
66 RP_REP_GET,
67 RP_ZA_GET,
68 RP_EXP_GET,
69 RP_DONE
70};
71
72
73struct CpsRunContext
74{
75 unsigned int i;
76 struct GNUNET_TIME_Absolute start;
77 struct GNUNET_TIME_Absolute end;
78 const struct GNUNET_CONFIGURATION_Handle *cfg;
79 struct GNUNET_DATASTORE_PluginFunctions *api;
80 enum RunPhase phase;
81 unsigned int cnt;
82 unsigned int iter;
83 uint64_t offset;
84};
85
86
87/**
88 * Function called by plugins to notify us about a
89 * change in their disk utilization.
90 *
91 * @param cls closure (NULL)
92 * @param delta change in disk utilization,
93 * 0 for "reset to empty"
94 */
95static void
96disk_utilization_change_cb (void *cls, int delta)
97{
98}
99
100
101static void
102test (void *cls);
103
104
105/**
106 * Put continuation.
107 *
108 * @param cls closure
109 * @param key key for the item stored
110 * @param size size of the item stored
111 * @param status #GNUNET_OK or #GNUNET_SYSERROR
112 * @param msg error message on error
113 */
114static void
115put_continuation (void *cls,
116 const struct GNUNET_HashCode *key,
117 uint32_t size,
118 int status,
119 const char *msg)
120{
121 struct CpsRunContext *crc = cls;
122
123 if (GNUNET_OK != status)
124 {
125 fprintf (stderr, "ERROR: `%s'\n", msg);
126 }
127 else
128 {
129 stored_bytes += size;
130 stored_ops++;
131 stored_entries++;
132 }
133 GNUNET_SCHEDULER_add_now (&test, crc);
134}
135
136
137static void
138do_put (struct CpsRunContext *crc)
139{
140 char value[65536];
141 size_t size;
142 static struct GNUNET_HashCode key;
143 static int i;
144 unsigned int prio;
145
146 if (0 == i)
147 crc->start = GNUNET_TIME_absolute_get ();
148 if (PUT_10 == i)
149 {
150 i = 0;
151 crc->end = GNUNET_TIME_absolute_get ();
152 {
153 printf ("%s took %s for %llu items\n", "Storing an item",
154 GNUNET_STRINGS_relative_time_to_string (
155 GNUNET_TIME_absolute_get_difference (crc->start,
156 crc
157 ->end),
158 GNUNET_YES),
159 PUT_10);
160 if (PUT_10 > 0)
161 GAUGER (category, "Storing an item",
162 (crc->end.abs_value_us - crc->start.abs_value_us) / 1000LL
163 / PUT_10,
164 "ms/item");
165 }
166 crc->i++;
167 crc->start = GNUNET_TIME_absolute_get ();
168 crc->phase++;
169 GNUNET_SCHEDULER_add_now (&test, crc);
170 return;
171 }
172 /* most content is 32k */
173 size = 32 * 1024;
174 if (GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 16) == 0) /* but some of it is less! */
175 size = 8 + GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 32 * 1024);
176 size = size - (size & 7); /* always multiple of 8 */
177
178 /* generate random key */
179 key.bits[0] = (unsigned int) GNUNET_TIME_absolute_get ().abs_value_us;
180 GNUNET_CRYPTO_hash (&key, sizeof(struct GNUNET_HashCode), &key);
181 memset (value, i, size);
182 if (i > 255)
183 memset (value, i - 255, size / 2);
184 value[0] = crc->i;
185 GNUNET_memcpy (&value[4], &i, sizeof(i));
186 prio = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 100);
187 crc->api->put (crc->api->cls,
188 &key,
189 false /* absent */,
190 size,
191 value,
192 1 + i % 4 /* type */,
193 prio,
194 i % 4 /* anonymity */,
195 0 /* replication */,
196 GNUNET_TIME_relative_to_absolute
197 (GNUNET_TIME_relative_multiply
198 (GNUNET_TIME_UNIT_MILLISECONDS,
199 60 * 60 * 60 * 1000
200 + GNUNET_CRYPTO_random_u32
201 (GNUNET_CRYPTO_QUALITY_WEAK, 1000))),
202 put_continuation,
203 crc);
204 i++;
205}
206
207
208static int
209iterate_zeros (void *cls,
210 const struct GNUNET_HashCode *key,
211 uint32_t size,
212 const void *data,
213 enum GNUNET_BLOCK_Type type,
214 uint32_t priority,
215 uint32_t anonymity,
216 uint32_t replication,
217 struct GNUNET_TIME_Absolute expiration,
218 uint64_t uid)
219{
220 struct CpsRunContext *crc = cls;
221 int i;
222 const char *cdata = data;
223
224 GNUNET_assert (key != NULL);
225 GNUNET_assert (size >= 8);
226 GNUNET_memcpy (&i, &cdata[4], sizeof(i));
227 hits[i / 8] |= (1 << (i % 8));
228
229 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
230 "Found result %d type=%u, priority=%u, size=%u, expire=%s\n",
231 i,
232 type, priority, size,
233 GNUNET_STRINGS_absolute_time_to_string (expiration));
234 crc->cnt++;
235 if (crc->cnt == PUT_10 / 4 - 1)
236 {
237 unsigned int bc;
238
239 bc = 0;
240 for (i = 0; i < PUT_10; i++)
241 if (0 != (hits[i / 8] & (1 << (i % 8))))
242 bc++;
243
244 crc->end = GNUNET_TIME_absolute_get ();
245 printf ("%s took %s yielding %u/%u items\n",
246 "Select random zero-anonymity item",
247 GNUNET_STRINGS_relative_time_to_string (
248 GNUNET_TIME_absolute_get_difference (crc->start,
249 crc
250 ->end),
251 GNUNET_YES),
252 bc, crc->cnt);
253 if (crc->cnt > 0)
254 GAUGER (category, "Select random zero-anonymity item",
255 (crc->end.abs_value_us - crc->start.abs_value_us) / 1000LL
256 / crc->cnt,
257 "ms/item");
258 memset (hits, 0, sizeof(hits));
259 crc->phase++;
260 crc->cnt = 0;
261 crc->start = GNUNET_TIME_absolute_get ();
262 }
263 GNUNET_SCHEDULER_add_now (&test, crc);
264 return GNUNET_OK;
265}
266
267
268static int
269expiration_get (void *cls,
270 const struct GNUNET_HashCode *key,
271 uint32_t size,
272 const void *data,
273 enum GNUNET_BLOCK_Type type,
274 uint32_t priority,
275 uint32_t anonymity,
276 uint32_t replication,
277 struct GNUNET_TIME_Absolute expiration,
278 uint64_t uid)
279{
280 struct CpsRunContext *crc = cls;
281 int i;
282 const char *cdata = data;
283
284 GNUNET_assert (size >= 8);
285 GNUNET_memcpy (&i, &cdata[4], sizeof(i));
286 hits[i / 8] |= (1 << (i % 8));
287 crc->cnt++;
288 if (PUT_10 <= crc->cnt)
289 {
290 unsigned int bc;
291
292 bc = 0;
293 for (i = 0; i < PUT_10; i++)
294 if (0 != (hits[i / 8] & (1 << (i % 8))))
295 bc++;
296
297 crc->end = GNUNET_TIME_absolute_get ();
298 printf ("%s took %s yielding %u/%u items\n",
299 "Selecting and deleting by expiration",
300 GNUNET_STRINGS_relative_time_to_string (
301 GNUNET_TIME_absolute_get_difference (crc->start,
302 crc
303 ->end),
304 GNUNET_YES),
305 bc, (unsigned int) PUT_10);
306 if (crc->cnt > 0)
307 GAUGER (category, "Selecting and deleting by expiration",
308 (crc->end.abs_value_us - crc->start.abs_value_us) / 1000LL
309 / crc->cnt,
310 "ms/item");
311 memset (hits, 0, sizeof(hits));
312 if (++crc->iter == ITERATIONS)
313 crc->phase++;
314 else
315 crc->phase = RP_PUT;
316 crc->cnt = 0;
317 crc->start = GNUNET_TIME_absolute_get ();
318 }
319 GNUNET_SCHEDULER_add_now (&test, crc);
320 return GNUNET_NO;
321}
322
323
324static int
325replication_get (void *cls,
326 const struct GNUNET_HashCode *key,
327 uint32_t size,
328 const void *data,
329 enum GNUNET_BLOCK_Type type,
330 uint32_t priority,
331 uint32_t anonymity,
332 uint32_t replication,
333 struct GNUNET_TIME_Absolute expiration,
334 uint64_t uid)
335{
336 struct CpsRunContext *crc = cls;
337 int i;
338 const char *cdata = data;
339
340 GNUNET_assert (NULL != key);
341 GNUNET_assert (size >= 8);
342 GNUNET_memcpy (&i, &cdata[4], sizeof(i));
343 hits[i / 8] |= (1 << (i % 8));
344 crc->cnt++;
345 if (PUT_10 <= crc->cnt)
346 {
347 unsigned int bc;
348
349 bc = 0;
350 for (i = 0; i < PUT_10; i++)
351 if (0 != (hits[i / 8] & (1 << (i % 8))))
352 bc++;
353
354 crc->end = GNUNET_TIME_absolute_get ();
355 printf ("%s took %s yielding %u/%u items\n",
356 "Selecting random item for replication",
357 GNUNET_STRINGS_relative_time_to_string (
358 GNUNET_TIME_absolute_get_difference (crc->start,
359 crc
360 ->end),
361 GNUNET_YES),
362 bc, (unsigned int) PUT_10);
363 if (crc->cnt > 0)
364 GAUGER (category, "Selecting random item for replication",
365 (crc->end.abs_value_us - crc->start.abs_value_us) / 1000LL
366 / crc->cnt,
367 "ms/item");
368 memset (hits, 0, sizeof(hits));
369 crc->phase++;
370 crc->offset = 0;
371 crc->cnt = 0;
372 crc->start = GNUNET_TIME_absolute_get ();
373 }
374
375 GNUNET_SCHEDULER_add_now (&test, crc);
376 return GNUNET_OK;
377}
378
379
380/**
381 * Function called when the service shuts
382 * down. Unloads our datastore plugin.
383 *
384 * @param api api to unload
385 * @param cfg configuration to use
386 */
387static void
388unload_plugin (struct GNUNET_DATASTORE_PluginFunctions *api,
389 const struct GNUNET_CONFIGURATION_Handle *cfg)
390{
391 char *name;
392 char *libname;
393
394 if (GNUNET_OK !=
395 GNUNET_CONFIGURATION_get_value_string (cfg, "DATASTORE", "DATABASE",
396 &name))
397 {
398 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
399 _ ("No `%s' specified for `%s' in configuration!\n"),
400 "DATABASE",
401 "DATASTORE");
402 return;
403 }
404 GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", name);
405 GNUNET_break (NULL == GNUNET_PLUGIN_unload (libname, api));
406 GNUNET_free (libname);
407 GNUNET_free (name);
408}
409
410
411/**
412 * Last task run during shutdown. Disconnects us from
413 * the transport and core.
414 */
415static void
416cleaning_task (void *cls)
417{
418 struct CpsRunContext *crc = cls;
419
420 unload_plugin (crc->api, crc->cfg);
421 GNUNET_free (crc);
422}
423
424
425static void
426test (void *cls)
427{
428 struct CpsRunContext *crc = cls;
429
430 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
431 "In phase %d, iteration %u\n", crc->phase, crc->cnt);
432 switch (crc->phase)
433 {
434 case RP_ERROR:
435 GNUNET_break (0);
436 crc->api->drop (crc->api->cls);
437 ok = 1;
438 GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_IDLE,
439 &cleaning_task, crc);
440 break;
441
442 case RP_PUT:
443 do_put (crc);
444 break;
445
446 case RP_REP_GET:
447 crc->api->get_replication (crc->api->cls, &replication_get, crc);
448 break;
449
450 case RP_ZA_GET:
451 crc->api->get_zero_anonymity (crc->api->cls, crc->offset++, 1,
452 &iterate_zeros, crc);
453 break;
454
455 case RP_EXP_GET:
456 crc->api->get_expiration (crc->api->cls, &expiration_get, crc);
457 break;
458
459 case RP_DONE:
460 crc->api->drop (crc->api->cls);
461 ok = 0;
462 GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_IDLE,
463 &cleaning_task, crc);
464 break;
465 }
466}
467
468
469/**
470 * Load the datastore plugin.
471 */
472static struct GNUNET_DATASTORE_PluginFunctions *
473load_plugin (const struct GNUNET_CONFIGURATION_Handle *cfg)
474{
475 static struct GNUNET_DATASTORE_PluginEnvironment env;
476 struct GNUNET_DATASTORE_PluginFunctions *ret;
477 char *name;
478 char *libname;
479
480 if (GNUNET_OK !=
481 GNUNET_CONFIGURATION_get_value_string (cfg, "DATASTORE", "DATABASE",
482 &name))
483 {
484 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
485 _ ("No `%s' specified for `%s' in configuration!\n"),
486 "DATABASE",
487 "DATASTORE");
488 return NULL;
489 }
490 env.cfg = cfg;
491 env.duc = &disk_utilization_change_cb;
492 env.cls = NULL;
493 GNUNET_log (GNUNET_ERROR_TYPE_INFO, _ ("Loading `%s' datastore plugin\n"),
494 name);
495 GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", name);
496 if (NULL == (ret = GNUNET_PLUGIN_load (libname, &env)))
497 {
498 fprintf (stderr, "Failed to load plugin `%s'!\n", name);
499 GNUNET_free (name);
500 GNUNET_free (libname);
501 return NULL;
502 }
503 GNUNET_free (libname);
504 GNUNET_free (name);
505 return ret;
506}
507
508
509static void
510run (void *cls, char *const *args, const char *cfgfile,
511 const struct GNUNET_CONFIGURATION_Handle *c)
512{
513 struct GNUNET_DATASTORE_PluginFunctions *api;
514 struct CpsRunContext *crc;
515
516 if (NULL == c)
517 {
518 GNUNET_break (0);
519 return;
520 }
521 api = load_plugin (c);
522 if (api == NULL)
523 {
524 fprintf (stderr,
525 "%s",
526 "Could not initialize plugin, assuming database not configured. Test not run!\n");
527 return;
528 }
529 crc = GNUNET_new (struct CpsRunContext);
530 crc->api = api;
531 crc->cfg = c;
532 crc->phase = RP_PUT;
533 ok = 2;
534 GNUNET_SCHEDULER_add_now (&test, crc);
535}
536
537
538int
539main (int argc, char *argv[])
540{
541 char dir_name[PATH_MAX];
542 char cfg_name[PATH_MAX];
543 char *const xargv[] = {
544 "perf-plugin-datastore",
545 "-c",
546 cfg_name,
547 NULL
548 };
549 struct GNUNET_GETOPT_CommandLineOption options[] = {
550 GNUNET_GETOPT_OPTION_END
551 };
552
553 plugin_name = GNUNET_TESTING_get_testname_from_underscore (argv[0]);
554 GNUNET_snprintf (dir_name, sizeof(dir_name), "/tmp/perf-gnunet-datastore-%s",
555 plugin_name);
556 GNUNET_DISK_directory_remove (dir_name);
557 GNUNET_log_setup ("perf-plugin-datastore",
558 "WARNING",
559 NULL);
560 GNUNET_snprintf (category, sizeof(category), "DATASTORE-%s", plugin_name);
561 GNUNET_snprintf (cfg_name, sizeof(cfg_name),
562 "perf_plugin_datastore_data_%s.conf", plugin_name);
563 GNUNET_PROGRAM_run ((sizeof(xargv) / sizeof(char *)) - 1, xargv,
564 "perf-plugin-datastore", "nohelp", options, &run, NULL);
565 if (ok != 0)
566 fprintf (stderr, "Missed some testcases: %u\n", ok);
567 GNUNET_DISK_directory_remove (dir_name);
568
569 return ok;
570}
571
572
573/* end of perf_plugin_datastore.c */
diff --git a/src/datastore/perf_plugin_datastore_data_heap.conf b/src/datastore/perf_plugin_datastore_data_heap.conf
deleted file mode 100644
index 873cf9606..000000000
--- a/src/datastore/perf_plugin_datastore_data_heap.conf
+++ /dev/null
@@ -1,7 +0,0 @@
1@INLINE@ test_defaults.conf
2[PATHS]
3GNUNET_TEST_HOME = $GNUNET_TMP/perf-gnunet-datastore-heap/
4
5
6[datastore]
7DATABASE = heap
diff --git a/src/datastore/perf_plugin_datastore_data_mysql.conf b/src/datastore/perf_plugin_datastore_data_mysql.conf
deleted file mode 100644
index a32b830c3..000000000
--- a/src/datastore/perf_plugin_datastore_data_mysql.conf
+++ /dev/null
@@ -1,10 +0,0 @@
1@INLINE@ test_defaults.conf
2[PATHS]
3GNUNET_TEST_HOME = $GNUNET_TMP/perf-gnunet-datastore-mysql/
4
5[datastore]
6DATABASE = mysql
7
8[datastore-mysql]
9DATABASE = gnunetcheck
10
diff --git a/src/datastore/perf_plugin_datastore_data_postgres.conf b/src/datastore/perf_plugin_datastore_data_postgres.conf
deleted file mode 100644
index 7683887a8..000000000
--- a/src/datastore/perf_plugin_datastore_data_postgres.conf
+++ /dev/null
@@ -1,10 +0,0 @@
1@INLINE@ test_defaults.conf
2[PATHS]
3GNUNET_TEST_HOME = $GNUNET_TMP/perf-gnunet-datastore-postgres/
4
5[datastore]
6DATABASE = postgres
7
8[datastore-postgres]
9CONFIG = dbname=gnunetcheck
10
diff --git a/src/datastore/perf_plugin_datastore_data_sqlite.conf b/src/datastore/perf_plugin_datastore_data_sqlite.conf
deleted file mode 100644
index 888e020a6..000000000
--- a/src/datastore/perf_plugin_datastore_data_sqlite.conf
+++ /dev/null
@@ -1,4 +0,0 @@
1@INLINE@ test_defaults.conf
2[PATHS]
3GNUNET_TEST_HOME = $GNUNET_TMP/perf-gnunet-datastore-sqlite/
4
diff --git a/src/datastore/plugin_datastore_heap.c b/src/datastore/plugin_datastore_heap.c
deleted file mode 100644
index a827a2763..000000000
--- a/src/datastore/plugin_datastore_heap.c
+++ /dev/null
@@ -1,944 +0,0 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2012 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file datastore/plugin_datastore_heap.c
23 * @brief heap-based datastore backend; usually we want the datastore
24 * to be persistent, and storing data in the heap is obviously
25 * NOT going to be persistent; still, this plugin is useful for
26 * testing/benchmarking --- but never for production!
27 * @author Christian Grothoff
28 */
29
30#include "platform.h"
31#include "gnunet_datastore_plugin.h"
32
33
34/**
35 * A value that we are storing.
36 */
37struct Value
38{
39 /**
40 * Key for the value.
41 */
42 struct GNUNET_HashCode key;
43
44 /**
45 * Pointer to the value's data (allocated at the end of this struct).
46 */
47 const void *data;
48
49 /**
50 * Entry for this value in the 'expire' heap.
51 */
52 struct GNUNET_CONTAINER_HeapNode *expire_heap;
53
54 /**
55 * Entry for this value in the 'replication' heap.
56 */
57 struct GNUNET_CONTAINER_HeapNode *replication_heap;
58
59 /**
60 * Expiration time for this value.
61 */
62 struct GNUNET_TIME_Absolute expiration;
63
64 /**
65 * Offset of this value in the array of the 'struct ZeroAnonByType';
66 * only used if anonymity is zero.
67 */
68 unsigned int zero_anon_offset;
69
70 /**
71 * Number of bytes in 'data'.
72 */
73 uint32_t size;
74
75 /**
76 * Priority of the value.
77 */
78 uint32_t priority;
79
80 /**
81 * Anonymity level for the value.
82 */
83 uint32_t anonymity;
84
85 /**
86 * Replication level for the value.
87 */
88 uint32_t replication;
89
90 /**
91 * Type of 'data'.
92 */
93 enum GNUNET_BLOCK_Type type;
94};
95
96
97/**
98 * We organize 0-anonymity values in arrays "by type".
99 */
100struct ZeroAnonByType
101{
102 /**
103 * We keep these in a DLL.
104 */
105 struct ZeroAnonByType *next;
106
107 /**
108 * We keep these in a DLL.
109 */
110 struct ZeroAnonByType *prev;
111
112 /**
113 * Array of 0-anonymity items of the given type.
114 */
115 struct Value **array;
116
117 /**
118 * Allocated size of the array.
119 */
120 unsigned int array_size;
121
122 /**
123 * First unused offset in 'array'.
124 */
125 unsigned int array_pos;
126
127 /**
128 * Type of all of the values in 'array'.
129 */
130 enum GNUNET_BLOCK_Type type;
131};
132
133
134/**
135 * Context for all functions in this plugin.
136 */
137struct Plugin
138{
139 /**
140 * Our execution environment.
141 */
142 struct GNUNET_DATASTORE_PluginEnvironment *env;
143
144 /**
145 * Mapping from keys to 'struct Value's.
146 */
147 struct GNUNET_CONTAINER_MultiHashMap *keyvalue;
148
149 /**
150 * Heap organized by minimum expiration time.
151 */
152 struct GNUNET_CONTAINER_Heap *by_expiration;
153
154 /**
155 * Heap organized by maximum replication value.
156 */
157 struct GNUNET_CONTAINER_Heap *by_replication;
158
159 /**
160 * Head of list of arrays containing zero-anonymity values by type.
161 */
162 struct ZeroAnonByType *zero_head;
163
164 /**
165 * Tail of list of arrays containing zero-anonymity values by type.
166 */
167 struct ZeroAnonByType *zero_tail;
168
169 /**
170 * Size of all values we're storing.
171 */
172 unsigned long long size;
173};
174
175
176/**
177 * Get an estimate of how much space the database is
178 * currently using.
179 *
180 * @param cls our "struct Plugin*"
181 * @return number of bytes used on disk
182 */
183static void
184heap_plugin_estimate_size (void *cls, unsigned long long *estimate)
185{
186 struct Plugin *plugin = cls;
187
188 if (NULL != estimate)
189 *estimate = plugin->size;
190}
191
192
193/**
194 * Closure for iterator for updating.
195 */
196struct UpdateContext
197{
198 /**
199 * Number of bytes in 'data'.
200 */
201 uint32_t size;
202
203 /**
204 * Pointer to the data.
205 */
206 const void *data;
207
208 /**
209 * Priority of the value.
210 */
211 uint32_t priority;
212
213 /**
214 * Replication level for the value.
215 */
216 uint32_t replication;
217
218 /**
219 * Expiration time for this value.
220 */
221 struct GNUNET_TIME_Absolute expiration;
222
223 /**
224 * True if the value was found and updated.
225 */
226 bool updated;
227};
228
229
230/**
231 * Update the matching value.
232 *
233 * @param cls the 'struct UpdateContext'
234 * @param key unused
235 * @param val the 'struct Value'
236 * @return GNUNET_YES (continue iteration), GNUNET_NO if value was found
237 */
238static int
239update_iterator (void *cls,
240 const struct GNUNET_HashCode *key,
241 void *val)
242{
243 struct UpdateContext *uc = cls;
244 struct Value *value = val;
245
246 if (value->size != uc->size)
247 return GNUNET_YES;
248 if (0 != memcmp (value->data, uc->data, uc->size))
249 return GNUNET_YES;
250 uc->expiration = GNUNET_TIME_absolute_max (value->expiration,
251 uc->expiration);
252 if (value->expiration.abs_value_us != uc->expiration.abs_value_us)
253 {
254 value->expiration = uc->expiration;
255 GNUNET_CONTAINER_heap_update_cost (value->expire_heap,
256 value->expiration.abs_value_us);
257 }
258 /* Saturating adds, don't overflow */
259 if (value->priority > UINT32_MAX - uc->priority)
260 value->priority = UINT32_MAX;
261 else
262 value->priority += uc->priority;
263 if (value->replication > UINT32_MAX - uc->replication)
264 value->replication = UINT32_MAX;
265 else
266 value->replication += uc->replication;
267 uc->updated = true;
268 return GNUNET_NO;
269}
270
271
272/**
273 * Store an item in the datastore.
274 *
275 * @param cls closure
276 * @param key key for the item
277 * @param absent true if the key was not found in the bloom filter
278 * @param size number of bytes in data
279 * @param data content stored
280 * @param type type of the content
281 * @param priority priority of the content
282 * @param anonymity anonymity-level for the content
283 * @param replication replication-level for the content
284 * @param expiration expiration time for the content
285 * @param cont continuation called with success or failure status
286 * @param cont_cls continuation closure
287 */
288static void
289heap_plugin_put (void *cls,
290 const struct GNUNET_HashCode *key,
291 bool absent,
292 uint32_t size,
293 const void *data,
294 enum GNUNET_BLOCK_Type type,
295 uint32_t priority,
296 uint32_t anonymity,
297 uint32_t replication,
298 struct GNUNET_TIME_Absolute expiration,
299 PluginPutCont cont,
300 void *cont_cls)
301{
302 struct Plugin *plugin = cls;
303 struct Value *value;
304
305 if (! absent)
306 {
307 struct UpdateContext uc;
308
309 uc.size = size;
310 uc.data = data;
311 uc.priority = priority;
312 uc.replication = replication;
313 uc.expiration = expiration;
314 uc.updated = false;
315 GNUNET_CONTAINER_multihashmap_get_multiple (plugin->keyvalue,
316 key,
317 &update_iterator,
318 &uc);
319 if (uc.updated)
320 {
321 cont (cont_cls, key, size, GNUNET_NO, NULL);
322 return;
323 }
324 }
325 value = GNUNET_malloc (sizeof(struct Value) + size);
326 value->key = *key;
327 value->data = &value[1];
328 value->expire_heap = GNUNET_CONTAINER_heap_insert (plugin->by_expiration,
329 value,
330 expiration.abs_value_us);
331 value->replication_heap = GNUNET_CONTAINER_heap_insert (
332 plugin->by_replication,
333 value,
334 replication);
335 value->expiration = expiration;
336 if (0 == anonymity)
337 {
338 struct ZeroAnonByType *zabt;
339
340 for (zabt = plugin->zero_head; NULL != zabt; zabt = zabt->next)
341 if (zabt->type == type)
342 break;
343 if (NULL == zabt)
344 {
345 zabt = GNUNET_new (struct ZeroAnonByType);
346 zabt->type = type;
347 GNUNET_CONTAINER_DLL_insert (plugin->zero_head,
348 plugin->zero_tail,
349 zabt);
350 }
351 if (zabt->array_size == zabt->array_pos)
352 {
353 GNUNET_array_grow (zabt->array,
354 zabt->array_size,
355 zabt->array_size * 2 + 4);
356 }
357 value->zero_anon_offset = zabt->array_pos;
358 zabt->array[zabt->array_pos++] = value;
359 }
360 value->size = size;
361 value->priority = priority;
362 value->anonymity = anonymity;
363 value->replication = replication;
364 value->type = type;
365 GNUNET_memcpy (&value[1], data, size);
366 GNUNET_CONTAINER_multihashmap_put (plugin->keyvalue,
367 &value->key,
368 value,
369 GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
370 plugin->size += size;
371 cont (cont_cls, key, size, GNUNET_OK, NULL);
372}
373
374
375/**
376 * Delete the given value, removing it from the plugin's data
377 * structures.
378 *
379 * @param plugin the plugin
380 * @param value value to delete
381 */
382static void
383delete_value (struct Plugin *plugin,
384 struct Value *value)
385{
386 GNUNET_assert (GNUNET_YES ==
387 GNUNET_CONTAINER_multihashmap_remove (plugin->keyvalue,
388 &value->key,
389 value));
390 GNUNET_assert (value == GNUNET_CONTAINER_heap_remove_node (
391 value->expire_heap));
392 GNUNET_assert (value == GNUNET_CONTAINER_heap_remove_node (
393 value->replication_heap));
394 if (0 == value->anonymity)
395 {
396 struct ZeroAnonByType *zabt;
397
398 for (zabt = plugin->zero_head; NULL != zabt; zabt = zabt->next)
399 if (zabt->type == value->type)
400 break;
401 GNUNET_assert (NULL != zabt);
402 zabt->array[value->zero_anon_offset] = zabt->array[--zabt->array_pos];
403 zabt->array[value->zero_anon_offset]->zero_anon_offset =
404 value->zero_anon_offset;
405 if (0 == zabt->array_pos)
406 {
407 GNUNET_array_grow (zabt->array,
408 zabt->array_size,
409 0);
410 GNUNET_CONTAINER_DLL_remove (plugin->zero_head,
411 plugin->zero_tail,
412 zabt);
413 GNUNET_free (zabt);
414 }
415 }
416 plugin->size -= value->size;
417 GNUNET_free (value);
418}
419
420
421/**
422 * Closure for iterator called during 'get_key'.
423 */
424struct GetContext
425{
426 /**
427 * Lowest uid to consider.
428 */
429 uint64_t next_uid;
430
431 /**
432 * Value with lowest uid >= next_uid found so far.
433 */
434 struct Value *value;
435
436 /**
437 * Requested type.
438 */
439 enum GNUNET_BLOCK_Type type;
440
441 /**
442 * If true, return a random value
443 */
444 bool random;
445};
446
447
448/**
449 * Obtain the matching value with the lowest uid >= next_uid.
450 *
451 * @param cls the 'struct GetContext'
452 * @param key unused
453 * @param val the 'struct Value'
454 * @return GNUNET_YES (continue iteration), GNUNET_NO if result was found
455 */
456static int
457get_iterator (void *cls,
458 const struct GNUNET_HashCode *key,
459 void *val)
460{
461 struct GetContext *gc = cls;
462 struct Value *value = val;
463
464 if ((gc->type != GNUNET_BLOCK_TYPE_ANY) &&
465 (gc->type != value->type))
466 return GNUNET_OK;
467 if (gc->random)
468 {
469 gc->value = value;
470 return GNUNET_NO;
471 }
472 if ((uint64_t) (intptr_t) value < gc->next_uid)
473 return GNUNET_OK;
474 if ((NULL != gc->value) &&
475 (value > gc->value))
476 return GNUNET_OK;
477 gc->value = value;
478 return GNUNET_OK;
479}
480
481
482/**
483 * Get one of the results for a particular key in the datastore.
484 *
485 * @param cls closure
486 * @param next_uid return the result with lowest uid >= next_uid
487 * @param random if true, return a random result instead of using next_uid
488 * @param key maybe NULL (to match all entries)
489 * @param type entries of which type are relevant?
490 * Use 0 for any type.
491 * @param proc function to call on the matching value;
492 * will be called with NULL if nothing matches
493 * @param proc_cls closure for @a proc
494 */
495static void
496heap_plugin_get_key (void *cls,
497 uint64_t next_uid,
498 bool random,
499 const struct GNUNET_HashCode *key,
500 enum GNUNET_BLOCK_Type type,
501 PluginDatumProcessor proc,
502 void *proc_cls)
503{
504 struct Plugin *plugin = cls;
505 struct GetContext gc;
506
507 gc.value = NULL;
508 gc.next_uid = next_uid;
509 gc.random = random;
510 gc.type = type;
511 if (NULL == key)
512 {
513 GNUNET_CONTAINER_multihashmap_iterate (plugin->keyvalue,
514 &get_iterator,
515 &gc);
516 }
517 else
518 {
519 GNUNET_CONTAINER_multihashmap_get_multiple (plugin->keyvalue,
520 key,
521 &get_iterator,
522 &gc);
523 }
524 if (NULL == gc.value)
525 {
526 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
527 return;
528 }
529 GNUNET_assert (GNUNET_OK ==
530 proc (proc_cls,
531 &gc.value->key,
532 gc.value->size,
533 &gc.value[1],
534 gc.value->type,
535 gc.value->priority,
536 gc.value->anonymity,
537 gc.value->replication,
538 gc.value->expiration,
539 (uint64_t) (intptr_t) gc.value));
540}
541
542
543/**
544 * Get a random item for replication. Returns a single, not expired,
545 * random item from those with the highest replication counters. The
546 * item's replication counter is decremented by one IF it was positive
547 * before. Call 'proc' with all values ZERO or NULL if the datastore
548 * is empty.
549 *
550 * @param cls closure
551 * @param proc function to call the value (once only).
552 * @param proc_cls closure for proc
553 */
554static void
555heap_plugin_get_replication (void *cls,
556 PluginDatumProcessor proc,
557 void *proc_cls)
558{
559 struct Plugin *plugin = cls;
560 struct Value *value;
561
562 value = GNUNET_CONTAINER_heap_remove_root (plugin->by_replication);
563 if (NULL == value)
564 {
565 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
566 return;
567 }
568 if (value->replication > 0)
569 {
570 value->replication--;
571 value->replication_heap = GNUNET_CONTAINER_heap_insert (
572 plugin->by_replication,
573 value,
574 value->replication);
575 }
576 else
577 {
578 /* need a better way to pick a random item, replication level is always 0 */
579 value->replication_heap = GNUNET_CONTAINER_heap_insert (
580 plugin->by_replication,
581 value,
582 value->replication);
583 value = GNUNET_CONTAINER_heap_walk_get_next (plugin->by_replication);
584 }
585 GNUNET_assert (GNUNET_OK ==
586 proc (proc_cls,
587 &value->key,
588 value->size,
589 &value[1],
590 value->type,
591 value->priority,
592 value->anonymity,
593 value->replication,
594 value->expiration,
595 (uint64_t) (intptr_t) value));
596}
597
598
599/**
600 * Get a random item for expiration. Call 'proc' with all values ZERO
601 * or NULL if the datastore is empty.
602 *
603 * @param cls closure
604 * @param proc function to call the value (once only).
605 * @param proc_cls closure for proc
606 */
607static void
608heap_plugin_get_expiration (void *cls, PluginDatumProcessor proc,
609 void *proc_cls)
610{
611 struct Plugin *plugin = cls;
612 struct Value *value;
613
614 value = GNUNET_CONTAINER_heap_peek (plugin->by_expiration);
615 if (NULL == value)
616 {
617 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
618 return;
619 }
620 if (GNUNET_NO ==
621 proc (proc_cls,
622 &value->key,
623 value->size,
624 &value[1],
625 value->type,
626 value->priority,
627 value->anonymity,
628 value->replication,
629 value->expiration,
630 (uint64_t) (intptr_t) value))
631 delete_value (plugin, value);
632}
633
634
635/**
636 * Call the given processor on an item with zero anonymity.
637 *
638 * @param cls our "struct Plugin*"
639 * @param next_uid return the result with lowest uid >= next_uid
640 * @param type entries of which type should be considered?
641 * Must not be zero (ANY).
642 * @param proc function to call on each matching value;
643 * will be called with NULL if no value matches
644 * @param proc_cls closure for proc
645 */
646static void
647heap_plugin_get_zero_anonymity (void *cls, uint64_t next_uid,
648 enum GNUNET_BLOCK_Type type,
649 PluginDatumProcessor proc, void *proc_cls)
650{
651 struct Plugin *plugin = cls;
652 struct ZeroAnonByType *zabt;
653 struct Value *value = NULL;
654
655 for (zabt = plugin->zero_head; NULL != zabt; zabt = zabt->next)
656 {
657 if ((type != GNUNET_BLOCK_TYPE_ANY) &&
658 (type != zabt->type))
659 continue;
660 for (int i = 0; i < zabt->array_pos; ++i)
661 {
662 if ((uint64_t) (intptr_t) zabt->array[i] < next_uid)
663 continue;
664 if ((NULL != value) &&
665 (zabt->array[i] > value))
666 continue;
667 value = zabt->array[i];
668 }
669 }
670 if (NULL == value)
671 {
672 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
673 return;
674 }
675 GNUNET_assert (GNUNET_OK ==
676 proc (proc_cls,
677 &value->key,
678 value->size,
679 &value[1],
680 value->type,
681 value->priority,
682 value->anonymity,
683 value->replication,
684 value->expiration,
685 (uint64_t) (intptr_t) value));
686}
687
688
689/**
690 * Drop database.
691 */
692static void
693heap_plugin_drop (void *cls)
694{
695 /* nothing needs to be done */
696}
697
698
699/**
700 * Closure for the 'return_value' function.
701 */
702struct GetAllContext
703{
704 /**
705 * Function to call.
706 */
707 PluginKeyProcessor proc;
708
709 /**
710 * Closure for 'proc'.
711 */
712 void *proc_cls;
713};
714
715
716/**
717 * Callback invoked to call callback on each value.
718 *
719 * @param cls the plugin
720 * @param key unused
721 * @param val the value
722 * @return GNUNET_OK (continue to iterate)
723 */
724static int
725return_value (void *cls,
726 const struct GNUNET_HashCode *key,
727 void *val)
728{
729 struct GetAllContext *gac = cls;
730
731 gac->proc (gac->proc_cls,
732 key,
733 1);
734 return GNUNET_OK;
735}
736
737
738/**
739 * Get all of the keys in the datastore.
740 *
741 * @param cls closure
742 * @param proc function to call on each key
743 * @param proc_cls closure for proc
744 */
745static void
746heap_get_keys (void *cls,
747 PluginKeyProcessor proc,
748 void *proc_cls)
749{
750 struct Plugin *plugin = cls;
751 struct GetAllContext gac;
752
753 gac.proc = proc;
754 gac.proc_cls = proc_cls;
755 GNUNET_CONTAINER_multihashmap_iterate (plugin->keyvalue,
756 &return_value,
757 &gac);
758 proc (proc_cls, NULL, 0);
759}
760
761
762/**
763 * Closure for iterator called during 'remove_key'.
764 */
765struct RemoveContext
766{
767 /**
768 * Value found.
769 */
770 struct Value *value;
771
772 /**
773 * Size of data.
774 */
775 uint32_t size;
776
777 /**
778 * Data to remove.
779 */
780 const void *data;
781};
782
783
784/**
785 * Obtain the matching value with the lowest uid >= next_uid.
786 *
787 * @param cls the 'struct GetContext'
788 * @param key unused
789 * @param val the 'struct Value'
790 * @return GNUNET_YES (continue iteration), GNUNET_NO if result was found
791 */
792static int
793remove_iterator (void *cls,
794 const struct GNUNET_HashCode *key,
795 void *val)
796{
797 struct RemoveContext *rc = cls;
798 struct Value *value = val;
799
800 if (value->size != rc->size)
801 return GNUNET_YES;
802 if (0 != memcmp (value->data, rc->data, rc->size))
803 return GNUNET_YES;
804 rc->value = value;
805 return GNUNET_NO;
806}
807
808
809/**
810 * Remove a particular key in the datastore.
811 *
812 * @param cls closure
813 * @param key key for the content
814 * @param size number of bytes in data
815 * @param data content stored
816 * @param cont continuation called with success or failure status
817 * @param cont_cls continuation closure for @a cont
818 */
819static void
820heap_plugin_remove_key (void *cls,
821 const struct GNUNET_HashCode *key,
822 uint32_t size,
823 const void *data,
824 PluginRemoveCont cont,
825 void *cont_cls)
826{
827 struct Plugin *plugin = cls;
828 struct RemoveContext rc;
829
830 rc.value = NULL;
831 rc.size = size;
832 rc.data = data;
833 GNUNET_CONTAINER_multihashmap_get_multiple (plugin->keyvalue,
834 key,
835 &remove_iterator,
836 &rc);
837 if (NULL == rc.value)
838 {
839 cont (cont_cls,
840 key,
841 size,
842 GNUNET_NO,
843 NULL);
844 return;
845 }
846 delete_value (plugin,
847 rc.value);
848 cont (cont_cls,
849 key,
850 size,
851 GNUNET_OK,
852 NULL);
853}
854
855
856/**
857 * Entry point for the plugin.
858 *
859 * @param cls the "struct GNUNET_DATASTORE_PluginEnvironment*"
860 * @return our "struct Plugin*"
861 */
862void *
863libgnunet_plugin_datastore_heap_init (void *cls)
864{
865 struct GNUNET_DATASTORE_PluginEnvironment *env = cls;
866 struct GNUNET_DATASTORE_PluginFunctions *api;
867 struct Plugin *plugin;
868 unsigned long long esize;
869
870 if (GNUNET_OK !=
871 GNUNET_CONFIGURATION_get_value_number (env->cfg,
872 "datastore-heap",
873 "HASHMAPSIZE",
874 &esize))
875 esize = 128 * 1024;
876 plugin = GNUNET_new (struct Plugin);
877 plugin->env = env;
878 plugin->keyvalue = GNUNET_CONTAINER_multihashmap_create (esize, GNUNET_YES);
879 plugin->by_expiration = GNUNET_CONTAINER_heap_create (
880 GNUNET_CONTAINER_HEAP_ORDER_MIN);
881 plugin->by_replication = GNUNET_CONTAINER_heap_create (
882 GNUNET_CONTAINER_HEAP_ORDER_MAX);
883 api = GNUNET_new (struct GNUNET_DATASTORE_PluginFunctions);
884 api->cls = plugin;
885 api->estimate_size = &heap_plugin_estimate_size;
886 api->put = &heap_plugin_put;
887 api->get_key = &heap_plugin_get_key;
888 api->get_replication = &heap_plugin_get_replication;
889 api->get_expiration = &heap_plugin_get_expiration;
890 api->get_zero_anonymity = &heap_plugin_get_zero_anonymity;
891 api->drop = &heap_plugin_drop;
892 api->get_keys = &heap_get_keys;
893 api->remove_key = &heap_plugin_remove_key;
894 GNUNET_log_from (GNUNET_ERROR_TYPE_INFO, "heap",
895 _ ("Heap database running\n"));
896 return api;
897}
898
899
900/**
901 * Callback invoked to free all value.
902 *
903 * @param cls the plugin
904 * @param key unused
905 * @param val the value
906 * @return GNUNET_OK (continue to iterate)
907 */
908static int
909free_value (void *cls,
910 const struct GNUNET_HashCode *key,
911 void *val)
912{
913 struct Plugin *plugin = cls;
914 struct Value *value = val;
915
916 delete_value (plugin, value);
917 return GNUNET_OK;
918}
919
920
921/**
922 * Exit point from the plugin.
923 * @param cls our "struct Plugin*"
924 * @return always NULL
925 */
926void *
927libgnunet_plugin_datastore_heap_done (void *cls)
928{
929 struct GNUNET_DATASTORE_PluginFunctions *api = cls;
930 struct Plugin *plugin = api->cls;
931
932 GNUNET_CONTAINER_multihashmap_iterate (plugin->keyvalue,
933 &free_value,
934 plugin);
935 GNUNET_CONTAINER_multihashmap_destroy (plugin->keyvalue);
936 GNUNET_CONTAINER_heap_destroy (plugin->by_expiration);
937 GNUNET_CONTAINER_heap_destroy (plugin->by_replication);
938 GNUNET_free (plugin);
939 GNUNET_free (api);
940 return NULL;
941}
942
943
944/* end of plugin_datastore_heap.c */
diff --git a/src/datastore/plugin_datastore_mysql.c b/src/datastore/plugin_datastore_mysql.c
deleted file mode 100644
index 216a6faa4..000000000
--- a/src/datastore/plugin_datastore_mysql.c
+++ /dev/null
@@ -1,1203 +0,0 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2009, 2010, 2011 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file datastore/plugin_datastore_mysql.c
23 * @brief mysql-based datastore backend
24 * @author Igor Wronsky
25 * @author Christian Grothoff
26 * @author Christophe Genevey
27 *
28 * NOTE: This db module does NOT work with mysql prior to 4.1 since
29 * it uses prepared statements. MySQL 5.0.46 promises to fix a bug
30 * in MyISAM that is causing us grief. At the time of this writing,
31 * that version is yet to be released. In anticipation, the code
32 * will use MyISAM with 5.0.46 (and higher). If you run such a
33 * version, please run "make check" to verify that the MySQL bug
34 * was actually fixed in your version (and if not, change the
35 * code below to use MyISAM for gn071).
36 *
37 * HIGHLIGHTS
38 *
39 * Pros
40 * + On up-to-date hardware where mysql can be used comfortably, this
41 * module will have better performance than the other db choices
42 * (according to our tests).
43 * + Its often possible to recover the mysql database from internal
44 * inconsistencies. The other db choices do not support repair!
45 * Cons
46 * - Memory usage (Comment: "I have 1G and it never caused me trouble")
47 * - Manual setup
48 *
49 * MANUAL SETUP INSTRUCTIONS
50 *
51 * 1) in gnunet.conf, set
52 * @verbatim
53 [datastore]
54 DATABASE = "mysql"
55 @endverbatim
56 * 2) Then access mysql as root,
57 * @verbatim
58 $ mysql -u root -p
59 @endverbatim
60 * and do the following. [You should replace $USER with the username
61 * that will be running the gnunetd process].
62 * @verbatim
63 CREATE DATABASE gnunet;
64 GRANT select,insert,update,delete,create,alter,drop,create temporary tables
65 ON gnunet.* TO $USER@localhost;
66 SET PASSWORD FOR $USER@localhost=PASSWORD('$the_password_you_like');
67 FLUSH PRIVILEGES;
68 @endverbatim
69 * 3) In the $HOME directory of $USER, create a ".my.cnf" file
70 * with the following lines
71 * @verbatim
72 [client]
73 user=$USER
74 password=$the_password_you_like
75 @endverbatim
76 *
77 * That's it. Note that .my.cnf file is a security risk unless its on
78 * a safe partition etc. The $HOME/.my.cnf can of course be a symbolic
79 * link. Even greater security risk can be achieved by setting no
80 * password for $USER. Luckily $USER has only privileges to mess
81 * up GNUnet's tables, nothing else (unless you give them more,
82 * of course).<p>
83 *
84 * 4) Still, perhaps you should briefly try if the DB connection
85 * works. First, login as $USER. Then use,
86 *
87 * @verbatim
88 $ mysql -u $USER -p $the_password_you_like
89 mysql> use gnunet;
90 @endverbatim
91 *
92 * If you get the message &quot;Database changed&quot; it probably works.
93 *
94 * [If you get &quot;ERROR 2002: Can't connect to local MySQL server
95 * through socket '/tmp/mysql.sock' (2)&quot; it may be resolvable by
96 * &quot;ln -s /var/run/mysqld/mysqld.sock /tmp/mysql.sock&quot;
97 * so there may be some additional trouble depending on your mysql setup.]
98 *
99 * REPAIRING TABLES
100 *
101 * - Its probably healthy to check your tables for inconsistencies
102 * every now and then.
103 * - If you get odd SEGVs on gnunetd startup, it might be that the mysql
104 * databases have been corrupted.
105 * - The tables can be verified/fixed in two ways;
106 * 1) by running mysqlcheck -A, or
107 * 2) by executing (inside of mysql using the GNUnet database):
108 * @verbatim
109 mysql> REPAIR TABLE gn090;
110 @endverbatim
111 *
112 * PROBLEMS?
113 *
114 * If you have problems related to the mysql module, your best
115 * friend is probably the mysql manual. The first thing to check
116 * is that mysql is basically operational, that you can connect
117 * to it, create tables, issue queries etc.
118 */
119
120#include "platform.h"
121#include "gnunet_datastore_plugin.h"
122#include "gnunet_util_lib.h"
123#include "gnunet_mysql_lib.h"
124#include "gnunet_my_lib.h"
125
126#define MAX_DATUM_SIZE 65536
127
128
129/**
130 * Context for all functions in this plugin.
131 */
132struct Plugin
133{
134 /**
135 * Our execution environment.
136 */
137 struct GNUNET_DATASTORE_PluginEnvironment *env;
138
139 /**
140 * Handle to talk to MySQL.
141 */
142 struct GNUNET_MYSQL_Context *mc;
143
144 /**
145 * Prepared statements.
146 */
147#define INSERT_ENTRY \
148 "INSERT INTO gn090 (repl,type,prio,anonLevel,expire,rvalue,hash,vhash,value) VALUES (?,?,?,?,?,?,?,?,?)"
149 struct GNUNET_MYSQL_StatementHandle *insert_entry;
150
151#define DELETE_ENTRY_BY_UID "DELETE FROM gn090 WHERE uid=?"
152 struct GNUNET_MYSQL_StatementHandle *delete_entry_by_uid;
153
154#define DELETE_ENTRY_BY_HASH_VALUE "DELETE FROM gn090 " \
155 "WHERE hash = ? AND " \
156 "value = ? " \
157 "LIMIT 1"
158 struct GNUNET_MYSQL_StatementHandle *delete_entry_by_hash_value;
159
160#define RESULT_COLUMNS "repl, type, prio, anonLevel, expire, hash, value, uid"
161
162#define SELECT_ENTRY "SELECT " RESULT_COLUMNS " FROM gn090 " \
163 "WHERE uid >= ? AND " \
164 "(rvalue >= ? OR 0 = ?) " \
165 "ORDER BY uid LIMIT 1"
166 struct GNUNET_MYSQL_StatementHandle *select_entry;
167
168#define SELECT_ENTRY_BY_HASH "SELECT " RESULT_COLUMNS " FROM gn090 " \
169 "FORCE INDEX (idx_hash_type_uid) " \
170 "WHERE hash=? AND " \
171 "uid >= ? AND " \
172 "(rvalue >= ? OR 0 = ?) " \
173 "ORDER BY uid LIMIT 1"
174 struct GNUNET_MYSQL_StatementHandle *select_entry_by_hash;
175
176#define SELECT_ENTRY_BY_HASH_AND_TYPE "SELECT " RESULT_COLUMNS " FROM gn090 " \
177 "FORCE INDEX (idx_hash_type_uid) " \
178 "WHERE hash = ? AND " \
179 "type = ? AND " \
180 "uid >= ? AND " \
181 "(rvalue >= ? OR 0 = ?) " \
182 "ORDER BY uid LIMIT 1"
183 struct GNUNET_MYSQL_StatementHandle *select_entry_by_hash_and_type;
184
185#define UPDATE_ENTRY "UPDATE gn090 SET " \
186 "prio = prio + ?, " \
187 "repl = repl + ?, " \
188 "expire = GREATEST(expire, ?) " \
189 "WHERE hash = ? AND vhash = ?"
190 struct GNUNET_MYSQL_StatementHandle *update_entry;
191
192#define DEC_REPL "UPDATE gn090 SET repl=GREATEST (1, repl) - 1 WHERE uid=?"
193 struct GNUNET_MYSQL_StatementHandle *dec_repl;
194
195#define SELECT_SIZE "SELECT SUM(LENGTH(value)+256) FROM gn090"
196 struct GNUNET_MYSQL_StatementHandle *get_size;
197
198#define SELECT_IT_NON_ANONYMOUS "SELECT " RESULT_COLUMNS " FROM gn090 " \
199 "FORCE INDEX (idx_anonLevel_type_rvalue) " \
200 "WHERE anonLevel=0 AND " \
201 "type=? AND " \
202 "uid >= ? " \
203 "ORDER BY uid LIMIT 1"
204 struct GNUNET_MYSQL_StatementHandle *zero_iter;
205
206#define SELECT_IT_EXPIRATION "SELECT " RESULT_COLUMNS " FROM gn090 " \
207 "FORCE INDEX (idx_expire) " \
208 "WHERE expire < ? " \
209 "ORDER BY expire ASC LIMIT 1"
210 struct GNUNET_MYSQL_StatementHandle *select_expiration;
211
212#define SELECT_IT_PRIORITY "SELECT " RESULT_COLUMNS " FROM gn090 " \
213 "FORCE INDEX (idx_prio) " \
214 "ORDER BY prio ASC LIMIT 1"
215 struct GNUNET_MYSQL_StatementHandle *select_priority;
216
217#define SELECT_IT_REPLICATION "SELECT " RESULT_COLUMNS " FROM gn090 " \
218 "FORCE INDEX (idx_repl_rvalue) " \
219 "WHERE repl=? AND " \
220 " (rvalue>=? OR" \
221 " NOT EXISTS (SELECT 1 FROM gn090 FORCE INDEX (idx_repl_rvalue) WHERE repl=? AND rvalue>=?)) " \
222 "ORDER BY rvalue ASC " \
223 "LIMIT 1"
224 struct GNUNET_MYSQL_StatementHandle *select_replication;
225
226#define SELECT_MAX_REPL "SELECT MAX(repl) FROM gn090"
227 struct GNUNET_MYSQL_StatementHandle *max_repl;
228
229#define GET_ALL_KEYS "SELECT hash from gn090"
230 struct GNUNET_MYSQL_StatementHandle *get_all_keys;
231};
232
233#define MAX_PARAM 16
234
235/**
236 * Delete an entry from the gn090 table.
237 *
238 * @param plugin plugin context
239 * @param uid unique ID of the entry to delete
240 * @return #GNUNET_OK on success, #GNUNET_NO if no such value exists, #GNUNET_SYSERR on error
241 */
242static int
243do_delete_entry (struct Plugin *plugin,
244 unsigned long long uid)
245{
246 int ret;
247 uint64_t uid64 = (uint64_t) uid;
248 struct GNUNET_MY_QueryParam params_delete[] = {
249 GNUNET_MY_query_param_uint64 (&uid64),
250 GNUNET_MY_query_param_end
251 };
252
253 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
254 "Deleting value %llu from gn090 table\n",
255 uid);
256 ret = GNUNET_MY_exec_prepared (plugin->mc,
257 plugin->delete_entry_by_uid,
258 params_delete);
259 if (ret >= 0)
260 {
261 return GNUNET_OK;
262 }
263 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
264 "Deleting value %llu from gn090 table failed\n",
265 (unsigned long long) uid);
266 return ret;
267}
268
269
270/**
271 * Get an estimate of how much space the database is
272 * currently using.
273 *
274 * @param cls our `struct Plugin *`
275 * @return number of bytes used on disk
276 */
277static void
278mysql_plugin_estimate_size (void *cls,
279 unsigned long long *estimate)
280{
281 struct Plugin *plugin = cls;
282 uint64_t total;
283 int ret;
284 struct GNUNET_MY_QueryParam params_get[] = {
285 GNUNET_MY_query_param_end
286 };
287 struct GNUNET_MY_ResultSpec results_get[] = {
288 GNUNET_MY_result_spec_uint64 (&total),
289 GNUNET_MY_result_spec_end
290 };
291
292 ret = GNUNET_MY_exec_prepared (plugin->mc,
293 plugin->get_size,
294 params_get);
295 *estimate = 0;
296 total = UINT64_MAX;
297 if ((GNUNET_OK == ret) &&
298 (GNUNET_OK ==
299 GNUNET_MY_extract_result (plugin->get_size,
300 results_get)))
301 {
302 *estimate = (unsigned long long) total;
303 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
304 "Size estimate for MySQL payload is %lld\n",
305 (long long) total);
306 GNUNET_assert (UINT64_MAX != total);
307 GNUNET_break (GNUNET_NO ==
308 GNUNET_MY_extract_result (plugin->get_size,
309 NULL));
310 }
311}
312
313
314/**
315 * Store an item in the datastore.
316 *
317 * @param cls closure
318 * @param key key for the item
319 * @param absent true if the key was not found in the bloom filter
320 * @param size number of bytes in @a data
321 * @param data content stored
322 * @param type type of the content
323 * @param priority priority of the content
324 * @param anonymity anonymity-level for the content
325 * @param replication replication-level for the content
326 * @param expiration expiration time for the content
327 * @param cont continuation called with success or failure status
328 * @param cont_cls closure for @a cont
329 */
330static void
331mysql_plugin_put (void *cls,
332 const struct GNUNET_HashCode *key,
333 bool absent,
334 uint32_t size,
335 const void *data,
336 enum GNUNET_BLOCK_Type type,
337 uint32_t priority,
338 uint32_t anonymity,
339 uint32_t replication,
340 struct GNUNET_TIME_Absolute expiration,
341 PluginPutCont cont,
342 void *cont_cls)
343{
344 struct Plugin *plugin = cls;
345 uint64_t lexpiration = expiration.abs_value_us;
346 struct GNUNET_HashCode vhash;
347
348 GNUNET_CRYPTO_hash (data,
349 size,
350 &vhash);
351 if (! absent)
352 {
353 struct GNUNET_MY_QueryParam params_update[] = {
354 GNUNET_MY_query_param_uint32 (&priority),
355 GNUNET_MY_query_param_uint32 (&replication),
356 GNUNET_MY_query_param_uint64 (&lexpiration),
357 GNUNET_MY_query_param_auto_from_type (key),
358 GNUNET_MY_query_param_auto_from_type (&vhash),
359 GNUNET_MY_query_param_end
360 };
361
362 if (GNUNET_OK !=
363 GNUNET_MY_exec_prepared (plugin->mc,
364 plugin->update_entry,
365 params_update))
366 {
367 cont (cont_cls,
368 key,
369 size,
370 GNUNET_SYSERR,
371 _ ("MySQL statement run failure"));
372 return;
373 }
374
375 MYSQL_STMT *stmt = GNUNET_MYSQL_statement_get_stmt (plugin->update_entry);
376 my_ulonglong rows = mysql_stmt_affected_rows (stmt);
377
378 GNUNET_break (GNUNET_NO ==
379 GNUNET_MY_extract_result (plugin->update_entry,
380 NULL));
381 if (0 != rows)
382 {
383 cont (cont_cls,
384 key,
385 size,
386 GNUNET_NO,
387 NULL);
388 return;
389 }
390 }
391
392 uint64_t lrvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
393 UINT64_MAX);
394 struct GNUNET_MY_QueryParam params_insert[] = {
395 GNUNET_MY_query_param_uint32 (&replication),
396 GNUNET_MY_query_param_uint32 (&type),
397 GNUNET_MY_query_param_uint32 (&priority),
398 GNUNET_MY_query_param_uint32 (&anonymity),
399 GNUNET_MY_query_param_uint64 (&lexpiration),
400 GNUNET_MY_query_param_uint64 (&lrvalue),
401 GNUNET_MY_query_param_auto_from_type (key),
402 GNUNET_MY_query_param_auto_from_type (&vhash),
403 GNUNET_MY_query_param_fixed_size (data, size),
404 GNUNET_MY_query_param_end
405 };
406
407 if (size > MAX_DATUM_SIZE)
408 {
409 GNUNET_break (0);
410 cont (cont_cls, key, size, GNUNET_SYSERR, _ ("Data too large"));
411 return;
412 }
413
414 if (GNUNET_OK !=
415 GNUNET_MY_exec_prepared (plugin->mc,
416 plugin->insert_entry,
417 params_insert))
418 {
419 cont (cont_cls,
420 key,
421 size,
422 GNUNET_SYSERR,
423 _ ("MySQL statement run failure"));
424 return;
425 }
426 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
427 "Inserted value `%s' with size %u into gn090 table\n",
428 GNUNET_h2s (key),
429 (unsigned int) size);
430 if (size > 0)
431 plugin->env->duc (plugin->env->cls,
432 size);
433 GNUNET_break (GNUNET_NO ==
434 GNUNET_MY_extract_result (plugin->insert_entry,
435 NULL));
436 cont (cont_cls,
437 key,
438 size,
439 GNUNET_OK,
440 NULL);
441}
442
443
444/**
445 * Run the given select statement and call 'proc' on the resulting
446 * values (which must be in particular positions).
447 *
448 * @param plugin the plugin handle
449 * @param stmt select statement to run
450 * @param proc function to call on result
451 * @param proc_cls closure for @a proc
452 * @param params_select arguments to initialize stmt
453 */
454static void
455execute_select (struct Plugin *plugin,
456 struct GNUNET_MYSQL_StatementHandle *stmt,
457 PluginDatumProcessor proc,
458 void *proc_cls,
459 struct GNUNET_MY_QueryParam *params_select)
460{
461 int ret;
462 uint32_t replication;
463 uint32_t type;
464 uint32_t priority;
465 uint32_t anonymity;
466 uint64_t uid;
467 size_t value_size;
468 void *value;
469 struct GNUNET_HashCode key;
470 struct GNUNET_TIME_Absolute expiration;
471 struct GNUNET_MY_ResultSpec results_select[] = {
472 GNUNET_MY_result_spec_uint32 (&replication),
473 GNUNET_MY_result_spec_uint32 (&type),
474 GNUNET_MY_result_spec_uint32 (&priority),
475 GNUNET_MY_result_spec_uint32 (&anonymity),
476 GNUNET_MY_result_spec_absolute_time (&expiration),
477 GNUNET_MY_result_spec_auto_from_type (&key),
478 GNUNET_MY_result_spec_variable_size (&value, &value_size),
479 GNUNET_MY_result_spec_uint64 (&uid),
480 GNUNET_MY_result_spec_end
481 };
482
483 ret = GNUNET_MY_exec_prepared (plugin->mc,
484 stmt,
485 params_select);
486 if (GNUNET_OK != ret)
487 {
488 proc (proc_cls,
489 NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
490 return;
491 }
492
493 ret = GNUNET_MY_extract_result (stmt,
494 results_select);
495 if (GNUNET_OK != ret)
496 {
497 proc (proc_cls,
498 NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
499 return;
500 }
501
502 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
503 "Found %u-byte value under key `%s' with prio %u, anon %u, expire %s selecting from gn090 table\n",
504 (unsigned int) value_size,
505 GNUNET_h2s (&key),
506 (unsigned int) priority,
507 (unsigned int) anonymity,
508 GNUNET_STRINGS_absolute_time_to_string (expiration));
509 GNUNET_assert (value_size < MAX_DATUM_SIZE);
510 GNUNET_break (GNUNET_NO ==
511 GNUNET_MY_extract_result (stmt,
512 NULL));
513 ret = proc (proc_cls,
514 &key,
515 value_size,
516 value,
517 type,
518 priority,
519 anonymity,
520 replication,
521 expiration,
522 uid);
523 GNUNET_MY_cleanup_result (results_select);
524 if (GNUNET_NO == ret)
525 {
526 do_delete_entry (plugin, uid);
527 if (0 != value_size)
528 plugin->env->duc (plugin->env->cls,
529 -value_size);
530 }
531}
532
533
534/**
535 * Get one of the results for a particular key in the datastore.
536 *
537 * @param cls closure
538 * @param next_uid return the result with lowest uid >= next_uid
539 * @param random if true, return a random result instead of using next_uid
540 * @param key key to match, never NULL
541 * @param type entries of which type are relevant?
542 * Use 0 for any type.
543 * @param proc function to call on the matching value,
544 * with NULL for if no value matches
545 * @param proc_cls closure for @a proc
546 */
547static void
548mysql_plugin_get_key (void *cls,
549 uint64_t next_uid,
550 bool random,
551 const struct GNUNET_HashCode *key,
552 enum GNUNET_BLOCK_Type type,
553 PluginDatumProcessor proc,
554 void *proc_cls)
555{
556 struct Plugin *plugin = cls;
557 uint64_t rvalue;
558
559 if (random)
560 {
561 rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
562 UINT64_MAX);
563 next_uid = 0;
564 }
565 else
566 rvalue = 0;
567
568 if (NULL == key)
569 {
570 struct GNUNET_MY_QueryParam params_select[] = {
571 GNUNET_MY_query_param_uint64 (&next_uid),
572 GNUNET_MY_query_param_uint64 (&rvalue),
573 GNUNET_MY_query_param_uint64 (&rvalue),
574 GNUNET_MY_query_param_end
575 };
576
577 execute_select (plugin,
578 plugin->select_entry,
579 proc,
580 proc_cls,
581 params_select);
582 }
583 else if (type != GNUNET_BLOCK_TYPE_ANY)
584 {
585 struct GNUNET_MY_QueryParam params_select[] = {
586 GNUNET_MY_query_param_auto_from_type (key),
587 GNUNET_MY_query_param_uint32 (&type),
588 GNUNET_MY_query_param_uint64 (&next_uid),
589 GNUNET_MY_query_param_uint64 (&rvalue),
590 GNUNET_MY_query_param_uint64 (&rvalue),
591 GNUNET_MY_query_param_end
592 };
593
594 execute_select (plugin,
595 plugin->select_entry_by_hash_and_type,
596 proc,
597 proc_cls,
598 params_select);
599 }
600 else
601 {
602 struct GNUNET_MY_QueryParam params_select[] = {
603 GNUNET_MY_query_param_auto_from_type (key),
604 GNUNET_MY_query_param_uint64 (&next_uid),
605 GNUNET_MY_query_param_uint64 (&rvalue),
606 GNUNET_MY_query_param_uint64 (&rvalue),
607 GNUNET_MY_query_param_end
608 };
609
610 execute_select (plugin,
611 plugin->select_entry_by_hash,
612 proc,
613 proc_cls,
614 params_select);
615 }
616}
617
618
619/**
620 * Get a zero-anonymity datum from the datastore.
621 *
622 * @param cls our `struct Plugin *`
623 * @param next_uid return the result with lowest uid >= next_uid
624 * @param type entries of which type should be considered?
625 * Must not be zero (ANY).
626 * @param proc function to call on a matching value;
627 * will be called with NULL if no value matches
628 * @param proc_cls closure for @a proc
629 */
630static void
631mysql_plugin_get_zero_anonymity (void *cls,
632 uint64_t next_uid,
633 enum GNUNET_BLOCK_Type type,
634 PluginDatumProcessor proc,
635 void *proc_cls)
636{
637 struct Plugin *plugin = cls;
638 uint32_t typei = (uint32_t) type;
639
640 struct GNUNET_MY_QueryParam params_zero_iter[] = {
641 GNUNET_MY_query_param_uint32 (&typei),
642 GNUNET_MY_query_param_uint64 (&next_uid),
643 GNUNET_MY_query_param_end
644 };
645
646 execute_select (plugin,
647 plugin->zero_iter,
648 proc,
649 proc_cls,
650 params_zero_iter);
651}
652
653
654/**
655 * Context for #repl_proc() function.
656 */
657struct ReplCtx
658{
659 /**
660 * Plugin handle.
661 */
662 struct Plugin *plugin;
663
664 /**
665 * Function to call for the result (or the NULL).
666 */
667 PluginDatumProcessor proc;
668
669 /**
670 * Closure for @e proc.
671 */
672 void *proc_cls;
673};
674
675
676/**
677 * Wrapper for the processor for #mysql_plugin_get_replication().
678 * Decrements the replication counter and calls the original
679 * iterator.
680 *
681 * @param cls closure
682 * @param key key for the content
683 * @param size number of bytes in @a data
684 * @param data content stored
685 * @param type type of the content
686 * @param priority priority of the content
687 * @param anonymity anonymity-level for the content
688 * @param replication replication-level for the content
689 * @param expiration expiration time for the content
690 * @param uid unique identifier for the datum;
691 * maybe 0 if no unique identifier is available
692 * @return #GNUNET_SYSERR to abort the iteration, #GNUNET_OK to continue
693 * (continue on call to "next", of course),
694 * #GNUNET_NO to delete the item and continue (if supported)
695 */
696static int
697repl_proc (void *cls,
698 const struct GNUNET_HashCode *key,
699 uint32_t size,
700 const void *data,
701 enum GNUNET_BLOCK_Type type,
702 uint32_t priority,
703 uint32_t anonymity,
704 uint32_t replication,
705 struct GNUNET_TIME_Absolute expiration,
706 uint64_t uid)
707{
708 struct ReplCtx *rc = cls;
709 struct Plugin *plugin = rc->plugin;
710 int ret;
711 int iret;
712
713 ret = rc->proc (rc->proc_cls,
714 key,
715 size,
716 data,
717 type,
718 priority,
719 anonymity,
720 replication,
721 expiration,
722 uid);
723 if (NULL != key)
724 {
725 struct GNUNET_MY_QueryParam params_proc[] = {
726 GNUNET_MY_query_param_uint64 (&uid),
727 GNUNET_MY_query_param_end
728 };
729
730 iret = GNUNET_MY_exec_prepared (plugin->mc,
731 plugin->dec_repl,
732 params_proc);
733 if (GNUNET_SYSERR == iret)
734 {
735 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
736 "Failed to reduce replication counter\n");
737 return GNUNET_SYSERR;
738 }
739 }
740 return ret;
741}
742
743
744/**
745 * Get a random item for replication. Returns a single, not expired,
746 * random item from those with the highest replication counters. The
747 * item's replication counter is decremented by one IF it was positive
748 * before. Call @a proc with all values ZERO or NULL if the datastore
749 * is empty.
750 *
751 * @param cls closure
752 * @param proc function to call the value (once only).
753 * @param proc_cls closure for @a proc
754 */
755static void
756mysql_plugin_get_replication (void *cls,
757 PluginDatumProcessor proc,
758 void *proc_cls)
759{
760 struct Plugin *plugin = cls;
761 uint64_t rvalue;
762 uint32_t repl;
763 struct ReplCtx rc;
764 struct GNUNET_MY_QueryParam params_get[] = {
765 GNUNET_MY_query_param_end
766 };
767 struct GNUNET_MY_ResultSpec results_get[] = {
768 GNUNET_MY_result_spec_uint32 (&repl),
769 GNUNET_MY_result_spec_end
770 };
771 struct GNUNET_MY_QueryParam params_select[] = {
772 GNUNET_MY_query_param_uint32 (&repl),
773 GNUNET_MY_query_param_uint64 (&rvalue),
774 GNUNET_MY_query_param_uint32 (&repl),
775 GNUNET_MY_query_param_uint64 (&rvalue),
776 GNUNET_MY_query_param_end
777 };
778
779 rc.plugin = plugin;
780 rc.proc = proc;
781 rc.proc_cls = proc_cls;
782
783 if (1 !=
784 GNUNET_MY_exec_prepared (plugin->mc,
785 plugin->max_repl,
786 params_get))
787 {
788 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
789 return;
790 }
791
792 if (GNUNET_OK !=
793 GNUNET_MY_extract_result (plugin->max_repl,
794 results_get))
795 {
796 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
797 return;
798 }
799 GNUNET_break (GNUNET_NO ==
800 GNUNET_MY_extract_result (plugin->max_repl,
801 NULL));
802 rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
803 UINT64_MAX);
804
805 execute_select (plugin,
806 plugin->select_replication,
807 &repl_proc,
808 &rc,
809 params_select);
810}
811
812
813/**
814 * Get all of the keys in the datastore.
815 *
816 * @param cls closure
817 * @param proc function to call on each key
818 * @param proc_cls closure for @a proc
819 */
820static void
821mysql_plugin_get_keys (void *cls,
822 PluginKeyProcessor proc,
823 void *proc_cls)
824{
825 struct Plugin *plugin = cls;
826 int ret;
827 MYSQL_STMT *statement;
828 unsigned int cnt;
829 struct GNUNET_HashCode key;
830 struct GNUNET_HashCode last;
831 struct GNUNET_MY_QueryParam params_select[] = {
832 GNUNET_MY_query_param_end
833 };
834 struct GNUNET_MY_ResultSpec results_select[] = {
835 GNUNET_MY_result_spec_auto_from_type (&key),
836 GNUNET_MY_result_spec_end
837 };
838
839 GNUNET_assert (NULL != proc);
840 statement = GNUNET_MYSQL_statement_get_stmt (plugin->get_all_keys);
841 if (GNUNET_OK !=
842 GNUNET_MY_exec_prepared (plugin->mc,
843 plugin->get_all_keys,
844 params_select))
845 {
846 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
847 _ ("`%s' for `%s' failed at %s:%d with error: %s\n"),
848 "mysql_stmt_execute",
849 GET_ALL_KEYS,
850 __FILE__,
851 __LINE__,
852 mysql_stmt_error (statement));
853 GNUNET_MYSQL_statements_invalidate (plugin->mc);
854 proc (proc_cls, NULL, 0);
855 return;
856 }
857 memset (&last, 0, sizeof(last)); /* make static analysis happy */
858 ret = GNUNET_YES;
859 cnt = 0;
860 while (ret == GNUNET_YES)
861 {
862 ret = GNUNET_MY_extract_result (plugin->get_all_keys,
863 results_select);
864 if (0 != GNUNET_memcmp (&last,
865 &key))
866 {
867 if (0 != cnt)
868 proc (proc_cls,
869 &last,
870 cnt);
871 cnt = 1;
872 last = key;
873 }
874 else
875 {
876 cnt++;
877 }
878 }
879 if (0 != cnt)
880 proc (proc_cls,
881 &last,
882 cnt);
883 /* finally, let app know we are done */
884 proc (proc_cls,
885 NULL,
886 0);
887 if (GNUNET_SYSERR == ret)
888 {
889 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
890 _ ("`%s' failed at %s:%d with error: %s\n"),
891 "mysql_stmt_fetch",
892 __FILE__,
893 __LINE__,
894 mysql_stmt_error (statement));
895 GNUNET_MYSQL_statements_invalidate (plugin->mc);
896 return;
897 }
898}
899
900
901/**
902 * Context for #expi_proc() function.
903 */
904struct ExpiCtx
905{
906 /**
907 * Plugin handle.
908 */
909 struct Plugin *plugin;
910
911 /**
912 * Function to call for the result (or the NULL).
913 */
914 PluginDatumProcessor proc;
915
916 /**
917 * Closure for @e proc.
918 */
919 void *proc_cls;
920};
921
922
923/**
924 * Wrapper for the processor for #mysql_plugin_get_expiration().
925 * If no expired value was found, we do a second query for
926 * low-priority content.
927 *
928 * @param cls closure
929 * @param key key for the content
930 * @param size number of bytes in data
931 * @param data content stored
932 * @param type type of the content
933 * @param priority priority of the content
934 * @param anonymity anonymity-level for the content
935 * @param replication replication-level for the content
936 * @param expiration expiration time for the content
937 * @param uid unique identifier for the datum;
938 * maybe 0 if no unique identifier is available
939 * @return #GNUNET_SYSERR to abort the iteration, #GNUNET_OK to continue
940 * (continue on call to "next", of course),
941 * #GNUNET_NO to delete the item and continue (if supported)
942 */
943static int
944expi_proc (void *cls,
945 const struct GNUNET_HashCode *key,
946 uint32_t size,
947 const void *data,
948 enum GNUNET_BLOCK_Type type,
949 uint32_t priority,
950 uint32_t anonymity,
951 uint32_t replication,
952 struct GNUNET_TIME_Absolute expiration,
953 uint64_t uid)
954{
955 struct ExpiCtx *rc = cls;
956 struct Plugin *plugin = rc->plugin;
957 struct GNUNET_MY_QueryParam params_select[] = {
958 GNUNET_MY_query_param_end
959 };
960
961 if (NULL == key)
962 {
963 execute_select (plugin,
964 plugin->select_priority,
965 rc->proc,
966 rc->proc_cls,
967 params_select);
968 return GNUNET_SYSERR;
969 }
970 return rc->proc (rc->proc_cls,
971 key,
972 size,
973 data,
974 type,
975 priority,
976 anonymity,
977 replication,
978 expiration,
979 uid);
980}
981
982
983/**
984 * Get a random item for expiration.
985 * Call @a proc with all values ZERO or NULL if the datastore is empty.
986 *
987 * @param cls closure
988 * @param proc function to call the value (once only).
989 * @param proc_cls closure for @a proc
990 */
991static void
992mysql_plugin_get_expiration (void *cls,
993 PluginDatumProcessor proc,
994 void *proc_cls)
995{
996 struct Plugin *plugin = cls;
997 struct GNUNET_TIME_Absolute now = { 0 };
998 struct GNUNET_MY_QueryParam params_select[] = {
999 GNUNET_MY_query_param_absolute_time (&now),
1000 GNUNET_MY_query_param_end
1001 };
1002 struct ExpiCtx rc;
1003
1004 rc.plugin = plugin;
1005 rc.proc = proc;
1006 rc.proc_cls = proc_cls;
1007 now = GNUNET_TIME_absolute_get ();
1008 execute_select (plugin,
1009 plugin->select_expiration,
1010 expi_proc,
1011 &rc,
1012 params_select);
1013}
1014
1015
1016/**
1017 * Drop database.
1018 *
1019 * @param cls the `struct Plugin *`
1020 */
1021static void
1022mysql_plugin_drop (void *cls)
1023{
1024 struct Plugin *plugin = cls;
1025
1026 if (GNUNET_OK !=
1027 GNUNET_MYSQL_statement_run (plugin->mc,
1028 "DROP TABLE gn090"))
1029 return; /* error */
1030 plugin->env->duc (plugin->env->cls, 0);
1031}
1032
1033
1034/**
1035 * Remove a particular key in the datastore.
1036 *
1037 * @param cls closure
1038 * @param key key for the content
1039 * @param size number of bytes in data
1040 * @param data content stored
1041 * @param cont continuation called with success or failure status
1042 * @param cont_cls continuation closure for @a cont
1043 */
1044static void
1045mysql_plugin_remove_key (void *cls,
1046 const struct GNUNET_HashCode *key,
1047 uint32_t size,
1048 const void *data,
1049 PluginRemoveCont cont,
1050 void *cont_cls)
1051{
1052 struct Plugin *plugin = cls;
1053 struct GNUNET_MY_QueryParam params_delete[] = {
1054 GNUNET_MY_query_param_auto_from_type (key),
1055 GNUNET_MY_query_param_fixed_size (data, size),
1056 GNUNET_MY_query_param_end
1057 };
1058
1059 if (GNUNET_OK !=
1060 GNUNET_MY_exec_prepared (plugin->mc,
1061 plugin->delete_entry_by_hash_value,
1062 params_delete))
1063 {
1064 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1065 "Removing key `%s' from gn090 table failed\n",
1066 GNUNET_h2s (key));
1067 cont (cont_cls,
1068 key,
1069 size,
1070 GNUNET_SYSERR,
1071 _ ("MySQL statement run failure"));
1072 return;
1073 }
1074
1075 MYSQL_STMT *stmt = GNUNET_MYSQL_statement_get_stmt (
1076 plugin->delete_entry_by_hash_value);
1077 my_ulonglong rows = mysql_stmt_affected_rows (stmt);
1078
1079 if (0 == rows)
1080 {
1081 cont (cont_cls,
1082 key,
1083 size,
1084 GNUNET_NO,
1085 NULL);
1086 return;
1087 }
1088 plugin->env->duc (plugin->env->cls,
1089 -size);
1090 cont (cont_cls,
1091 key,
1092 size,
1093 GNUNET_OK,
1094 NULL);
1095}
1096
1097
1098/**
1099 * Entry point for the plugin.
1100 *
1101 * @param cls the `struct GNUNET_DATASTORE_PluginEnvironment *`
1102 * @return our `struct Plugin *`
1103 */
1104void *
1105libgnunet_plugin_datastore_mysql_init (void *cls)
1106{
1107 struct GNUNET_DATASTORE_PluginEnvironment *env = cls;
1108 struct GNUNET_DATASTORE_PluginFunctions *api;
1109 struct Plugin *plugin;
1110
1111 plugin = GNUNET_new (struct Plugin);
1112 plugin->env = env;
1113 plugin->mc = GNUNET_MYSQL_context_create (env->cfg,
1114 "datastore-mysql");
1115 if (NULL == plugin->mc)
1116 {
1117 GNUNET_free (plugin);
1118 return NULL;
1119 }
1120#define MRUNS(a) (GNUNET_OK != GNUNET_MYSQL_statement_run (plugin->mc, a))
1121#define PINIT(a, b) (NULL == (a = GNUNET_MYSQL_statement_prepare (plugin->mc, \
1122 b)))
1123 if (MRUNS
1124 ("CREATE TABLE IF NOT EXISTS gn090 ("
1125 " repl INT(11) UNSIGNED NOT NULL DEFAULT 0,"
1126 " type INT(11) UNSIGNED NOT NULL DEFAULT 0,"
1127 " prio INT(11) UNSIGNED NOT NULL DEFAULT 0,"
1128 " anonLevel INT(11) UNSIGNED NOT NULL DEFAULT 0,"
1129 " expire BIGINT UNSIGNED NOT NULL DEFAULT 0,"
1130 " rvalue BIGINT UNSIGNED NOT NULL,"
1131 " hash BINARY(64) NOT NULL DEFAULT '',"
1132 " vhash BINARY(64) NOT NULL DEFAULT '',"
1133 " value BLOB NOT NULL DEFAULT '',"
1134 " uid BIGINT NOT NULL AUTO_INCREMENT,"
1135 " PRIMARY KEY (uid),"
1136 " INDEX idx_hash_type_uid (hash(64),type,rvalue),"
1137 " INDEX idx_prio (prio),"
1138 " INDEX idx_repl_rvalue (repl,rvalue),"
1139 " INDEX idx_expire (expire),"
1140 " INDEX idx_anonLevel_type_rvalue (anonLevel,type,rvalue)"
1141 ") ENGINE=InnoDB") || MRUNS ("SET AUTOCOMMIT = 1") ||
1142 PINIT (plugin->insert_entry, INSERT_ENTRY) ||
1143 PINIT (plugin->delete_entry_by_uid, DELETE_ENTRY_BY_UID) ||
1144 PINIT (plugin->delete_entry_by_hash_value, DELETE_ENTRY_BY_HASH_VALUE) ||
1145 PINIT (plugin->select_entry, SELECT_ENTRY) ||
1146 PINIT (plugin->select_entry_by_hash, SELECT_ENTRY_BY_HASH) ||
1147 PINIT (plugin->select_entry_by_hash_and_type,
1148 SELECT_ENTRY_BY_HASH_AND_TYPE) ||
1149 PINIT (plugin->get_size, SELECT_SIZE) ||
1150 PINIT (plugin->update_entry, UPDATE_ENTRY) ||
1151 PINIT (plugin->dec_repl, DEC_REPL) ||
1152 PINIT (plugin->zero_iter, SELECT_IT_NON_ANONYMOUS) ||
1153 PINIT (plugin->select_expiration, SELECT_IT_EXPIRATION) ||
1154 PINIT (plugin->select_priority, SELECT_IT_PRIORITY) ||
1155 PINIT (plugin->max_repl, SELECT_MAX_REPL) ||
1156 PINIT (plugin->get_all_keys, GET_ALL_KEYS) ||
1157 PINIT (plugin->select_replication, SELECT_IT_REPLICATION) ||
1158 false)
1159 {
1160 GNUNET_MYSQL_context_destroy (plugin->mc);
1161 GNUNET_free (plugin);
1162 return NULL;
1163 }
1164#undef PINIT
1165#undef MRUNS
1166
1167 api = GNUNET_new (struct GNUNET_DATASTORE_PluginFunctions);
1168 api->cls = plugin;
1169 api->estimate_size = &mysql_plugin_estimate_size;
1170 api->put = &mysql_plugin_put;
1171 api->get_key = &mysql_plugin_get_key;
1172 api->get_replication = &mysql_plugin_get_replication;
1173 api->get_expiration = &mysql_plugin_get_expiration;
1174 api->get_zero_anonymity = &mysql_plugin_get_zero_anonymity;
1175 api->get_keys = &mysql_plugin_get_keys;
1176 api->drop = &mysql_plugin_drop;
1177 api->remove_key = &mysql_plugin_remove_key;
1178 GNUNET_log_from (GNUNET_ERROR_TYPE_INFO, "mysql",
1179 _ ("Mysql database running\n"));
1180 return api;
1181}
1182
1183
1184/**
1185 * Exit point from the plugin.
1186 *
1187 * @param cls our `struct Plugin *`
1188 * @return always NULL
1189 */
1190void *
1191libgnunet_plugin_datastore_mysql_done (void *cls)
1192{
1193 struct GNUNET_DATASTORE_PluginFunctions *api = cls;
1194 struct Plugin *plugin = api->cls;
1195
1196 GNUNET_MYSQL_context_destroy (plugin->mc);
1197 GNUNET_free (plugin);
1198 GNUNET_free (api);
1199 return NULL;
1200}
1201
1202
1203/* end of plugin_datastore_mysql.c */
diff --git a/src/datastore/plugin_datastore_postgres.c b/src/datastore/plugin_datastore_postgres.c
deleted file mode 100644
index 8fb0bf6ee..000000000
--- a/src/datastore/plugin_datastore_postgres.c
+++ /dev/null
@@ -1,980 +0,0 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2009-2017 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file datastore/plugin_datastore_postgres.c
23 * @brief postgres-based datastore backend
24 * @author Christian Grothoff
25 */
26#include "platform.h"
27#include "gnunet_datastore_plugin.h"
28#include "gnunet_pq_lib.h"
29
30
31/**
32 * After how many ms "busy" should a DB operation fail for good?
33 * A low value makes sure that we are more responsive to requests
34 * (especially PUTs). A high value guarantees a higher success
35 * rate (SELECTs in iterate can take several seconds despite LIMIT=1).
36 *
37 * The default value of 1s should ensure that users do not experience
38 * huge latencies while at the same time allowing operations to succeed
39 * with reasonable probability.
40 */
41#define BUSY_TIMEOUT GNUNET_TIME_UNIT_SECONDS
42
43
44/**
45 * Context for all functions in this plugin.
46 */
47struct Plugin
48{
49 /**
50 * Our execution environment.
51 */
52 struct GNUNET_DATASTORE_PluginEnvironment *env;
53
54 /**
55 * Native Postgres database handle.
56 */
57 struct GNUNET_PQ_Context *dbh;
58};
59
60
61/**
62 * @brief Get a database handle
63 *
64 * @param plugin global context
65 * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
66 */
67static int
68init_connection (struct Plugin *plugin)
69{
70 struct GNUNET_PQ_ExecuteStatement es[] = {
71 /* FIXME: PostgreSQL does not have unsigned integers! This is ok for the type column because
72 * we only test equality on it and can cast it to/from uint32_t. For repl, prio, and anonLevel
73 * we do math or inequality tests, so we can't handle the entire range of uint32_t.
74 * This will also cause problems for expiration times after 294247-01-10-04:00:54 UTC.
75 */
76 GNUNET_PQ_make_try_execute (
77 "CREATE SEQUENCE IF NOT EXISTS gn090_oid_seq"),
78 GNUNET_PQ_make_execute ("CREATE TABLE IF NOT EXISTS gn090 ("
79 " repl INTEGER NOT NULL DEFAULT 0,"
80 " type INTEGER NOT NULL DEFAULT 0,"
81 " prio INTEGER NOT NULL DEFAULT 0,"
82 " anonLevel INTEGER NOT NULL DEFAULT 0,"
83 " expire BIGINT NOT NULL DEFAULT 0,"
84 " rvalue BIGINT NOT NULL DEFAULT 0,"
85 " hash BYTEA NOT NULL DEFAULT '',"
86 " vhash BYTEA NOT NULL DEFAULT '',"
87 " value BYTEA NOT NULL DEFAULT '',"
88 " oid OID NOT NULL DEFAULT nextval('gn090_oid_seq'))"),
89 GNUNET_PQ_make_try_execute (
90 "ALTER SEQUENCE gn090_oid_seq OWNED BY gn090.oid"),
91 GNUNET_PQ_make_try_execute (
92 "CREATE INDEX IF NOT EXISTS oid_hash ON gn090 (oid)"),
93 GNUNET_PQ_make_try_execute (
94 "CREATE INDEX IF NOT EXISTS idx_hash ON gn090 (hash)"),
95 GNUNET_PQ_make_try_execute (
96 "CREATE INDEX IF NOT EXISTS idx_prio ON gn090 (prio)"),
97 GNUNET_PQ_make_try_execute (
98 "CREATE INDEX IF NOT EXISTS idx_expire ON gn090 (expire)"),
99 GNUNET_PQ_make_try_execute (
100 "CREATE INDEX IF NOT EXISTS idx_prio_anon ON gn090 (prio,anonLevel)"),
101 GNUNET_PQ_make_try_execute (
102 "CREATE INDEX IF NOT EXISTS idx_prio_hash_anon ON gn090 (prio,hash,anonLevel)"),
103 GNUNET_PQ_make_try_execute (
104 "CREATE INDEX IF NOT EXISTS idx_repl_rvalue ON gn090 (repl,rvalue)"),
105 GNUNET_PQ_make_try_execute (
106 "CREATE INDEX IF NOT EXISTS idx_expire_hash ON gn090 (expire,hash)"),
107 GNUNET_PQ_make_execute (
108 "ALTER TABLE gn090 ALTER value SET STORAGE EXTERNAL"),
109 GNUNET_PQ_make_execute ("ALTER TABLE gn090 ALTER hash SET STORAGE PLAIN"),
110 GNUNET_PQ_make_execute ("ALTER TABLE gn090 ALTER vhash SET STORAGE PLAIN"),
111 GNUNET_PQ_EXECUTE_STATEMENT_END
112 };
113
114#define RESULT_COLUMNS "repl, type, prio, anonLevel, expire, hash, value, oid"
115 struct GNUNET_PQ_PreparedStatement ps[] = {
116 GNUNET_PQ_make_prepare ("get",
117 "SELECT " RESULT_COLUMNS " FROM gn090"
118 " WHERE oid >= $1::bigint AND"
119 " (rvalue >= $2 OR 0 = $3::smallint) AND"
120 " (hash = $4 OR 0 = $5::smallint) AND"
121 " (type = $6 OR 0 = $7::smallint)"
122 " ORDER BY oid ASC LIMIT 1",
123 7),
124 GNUNET_PQ_make_prepare ("put",
125 "INSERT INTO gn090 (repl, type, prio, anonLevel, expire, rvalue, hash, vhash, value) "
126 "VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)",
127 9),
128 GNUNET_PQ_make_prepare ("update",
129 "UPDATE gn090"
130 " SET prio = prio + $1,"
131 " repl = repl + $2,"
132 " expire = GREATEST(expire, $3)"
133 " WHERE hash = $4 AND vhash = $5",
134 5),
135 GNUNET_PQ_make_prepare ("decrepl",
136 "UPDATE gn090 SET repl = GREATEST (repl - 1, 0) "
137 "WHERE oid = $1",
138 1),
139 GNUNET_PQ_make_prepare ("select_non_anonymous",
140 "SELECT " RESULT_COLUMNS " FROM gn090 "
141 "WHERE anonLevel = 0 AND type = $1 AND oid >= $2::bigint "
142 "ORDER BY oid ASC LIMIT 1",
143 2),
144 GNUNET_PQ_make_prepare ("select_expiration_order",
145 "(SELECT " RESULT_COLUMNS " FROM gn090 "
146 "WHERE expire < $1 ORDER BY prio ASC LIMIT 1) "
147 "UNION "
148 "(SELECT " RESULT_COLUMNS " FROM gn090 "
149 "ORDER BY prio ASC LIMIT 1) "
150 "ORDER BY expire ASC LIMIT 1",
151 1),
152 GNUNET_PQ_make_prepare ("select_replication_order",
153 "SELECT " RESULT_COLUMNS " FROM gn090 "
154 "ORDER BY repl DESC,RANDOM() LIMIT 1",
155 0),
156 GNUNET_PQ_make_prepare ("delrow",
157 "DELETE FROM gn090 "
158 "WHERE oid=$1",
159 1),
160 GNUNET_PQ_make_prepare ("remove",
161 "DELETE FROM gn090"
162 " WHERE hash = $1 AND"
163 " value = $2",
164 2),
165 GNUNET_PQ_make_prepare ("get_keys",
166 "SELECT hash FROM gn090",
167 0),
168 GNUNET_PQ_make_prepare ("estimate_size",
169 "SELECT CASE WHEN NOT EXISTS"
170 " (SELECT 1 FROM gn090)"
171 " THEN 0"
172 " ELSE (SELECT SUM(LENGTH(value))+256*COUNT(*) FROM gn090)"
173 "END AS total",
174 0),
175 GNUNET_PQ_PREPARED_STATEMENT_END
176 };
177#undef RESULT_COLUMNS
178
179 plugin->dbh = GNUNET_PQ_connect_with_cfg (plugin->env->cfg,
180 "datastore-postgres",
181 NULL,
182 es,
183 ps);
184 if (NULL == plugin->dbh)
185 return GNUNET_SYSERR;
186 return GNUNET_OK;
187}
188
189
190/**
191 * Get an estimate of how much space the database is
192 * currently using.
193 *
194 * @param cls our `struct Plugin *`
195 * @return number of bytes used on disk
196 */
197static void
198postgres_plugin_estimate_size (void *cls,
199 unsigned long long *estimate)
200{
201 struct Plugin *plugin = cls;
202 uint64_t total;
203 struct GNUNET_PQ_QueryParam params[] = {
204 GNUNET_PQ_query_param_end
205 };
206 struct GNUNET_PQ_ResultSpec rs[] = {
207 GNUNET_PQ_result_spec_uint64 ("total",
208 &total),
209 GNUNET_PQ_result_spec_end
210 };
211 enum GNUNET_DB_QueryStatus ret;
212
213 if (NULL == estimate)
214 return;
215 ret = GNUNET_PQ_eval_prepared_singleton_select (plugin->dbh,
216 "estimate_size",
217 params,
218 rs);
219 if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT != ret)
220 {
221 *estimate = 0LL;
222 return;
223 }
224 *estimate = total;
225}
226
227
228/**
229 * Store an item in the datastore.
230 *
231 * @param cls closure with the `struct Plugin`
232 * @param key key for the item
233 * @param absent true if the key was not found in the bloom filter
234 * @param size number of bytes in data
235 * @param data content stored
236 * @param type type of the content
237 * @param priority priority of the content
238 * @param anonymity anonymity-level for the content
239 * @param replication replication-level for the content
240 * @param expiration expiration time for the content
241 * @param cont continuation called with success or failure status
242 * @param cont_cls continuation closure
243 */
244static void
245postgres_plugin_put (void *cls,
246 const struct GNUNET_HashCode *key,
247 bool absent,
248 uint32_t size,
249 const void *data,
250 enum GNUNET_BLOCK_Type type,
251 uint32_t priority,
252 uint32_t anonymity,
253 uint32_t replication,
254 struct GNUNET_TIME_Absolute expiration,
255 PluginPutCont cont,
256 void *cont_cls)
257{
258 struct Plugin *plugin = cls;
259 struct GNUNET_HashCode vhash;
260 enum GNUNET_DB_QueryStatus ret;
261
262 GNUNET_CRYPTO_hash (data,
263 size,
264 &vhash);
265 if (! absent)
266 {
267 struct GNUNET_PQ_QueryParam params[] = {
268 GNUNET_PQ_query_param_uint32 (&priority),
269 GNUNET_PQ_query_param_uint32 (&replication),
270 GNUNET_PQ_query_param_absolute_time (&expiration),
271 GNUNET_PQ_query_param_auto_from_type (key),
272 GNUNET_PQ_query_param_auto_from_type (&vhash),
273 GNUNET_PQ_query_param_end
274 };
275 ret = GNUNET_PQ_eval_prepared_non_select (plugin->dbh,
276 "update",
277 params);
278 if (0 > ret)
279 {
280 cont (cont_cls,
281 key,
282 size,
283 GNUNET_SYSERR,
284 _ ("Postgresql exec failure"));
285 return;
286 }
287 bool affected = (0 != ret);
288 if (affected)
289 {
290 cont (cont_cls,
291 key,
292 size,
293 GNUNET_NO,
294 NULL);
295 return;
296 }
297 }
298
299 {
300 uint32_t utype = (uint32_t) type;
301 uint64_t rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
302 UINT64_MAX);
303 struct GNUNET_PQ_QueryParam params[] = {
304 GNUNET_PQ_query_param_uint32 (&replication),
305 GNUNET_PQ_query_param_uint32 (&utype),
306 GNUNET_PQ_query_param_uint32 (&priority),
307 GNUNET_PQ_query_param_uint32 (&anonymity),
308 GNUNET_PQ_query_param_absolute_time (&expiration),
309 GNUNET_PQ_query_param_uint64 (&rvalue),
310 GNUNET_PQ_query_param_auto_from_type (key),
311 GNUNET_PQ_query_param_auto_from_type (&vhash),
312 GNUNET_PQ_query_param_fixed_size (data, size),
313 GNUNET_PQ_query_param_end
314 };
315
316 ret = GNUNET_PQ_eval_prepared_non_select (plugin->dbh,
317 "put",
318 params);
319 if (0 > ret)
320 {
321 cont (cont_cls,
322 key,
323 size,
324 GNUNET_SYSERR,
325 "Postgresql exec failure");
326 return;
327 }
328 }
329 plugin->env->duc (plugin->env->cls,
330 size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
331 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
332 "datastore-postgres",
333 "Stored %u bytes in database\n",
334 (unsigned int) size);
335 cont (cont_cls,
336 key,
337 size,
338 GNUNET_OK,
339 NULL);
340}
341
342
343/**
344 * Closure for #process_result.
345 */
346struct ProcessResultContext
347{
348 /**
349 * The plugin handle.
350 */
351 struct Plugin *plugin;
352
353 /**
354 * Function to call on each result.
355 */
356 PluginDatumProcessor proc;
357
358 /**
359 * Closure for @e proc.
360 */
361 void *proc_cls;
362};
363
364
365/**
366 * Function invoked to process the result and call the processor of @a
367 * cls.
368 *
369 * @param cls our `struct ProcessResultContext`
370 * @param res result from exec
371 * @param num_results number of results in @a res
372 */
373static void
374process_result (void *cls,
375 PGresult *res,
376 unsigned int num_results)
377{
378 struct ProcessResultContext *prc = cls;
379 struct Plugin *plugin = prc->plugin;
380
381 if (0 == num_results)
382 {
383 /* no result */
384 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
385 "datastore-postgres",
386 "Ending iteration (no more results)\n");
387 prc->proc (prc->proc_cls, NULL, 0, NULL, 0, 0, 0, 0,
388 GNUNET_TIME_UNIT_ZERO_ABS, 0);
389 return;
390 }
391 if (1 != num_results)
392 {
393 GNUNET_break (0);
394 prc->proc (prc->proc_cls, NULL, 0, NULL, 0, 0, 0, 0,
395 GNUNET_TIME_UNIT_ZERO_ABS, 0);
396 return;
397 }
398 /* Technically we don't need the loop here, but nicer in case
399 we ever relax the condition above. */
400 for (unsigned int i = 0; i < num_results; i++)
401 {
402 int iret;
403 uint32_t rowid;
404 uint32_t utype;
405 uint32_t anonymity;
406 uint32_t replication;
407 uint32_t priority;
408 size_t size;
409 void *data;
410 struct GNUNET_TIME_Absolute expiration_time;
411 struct GNUNET_HashCode key;
412 struct GNUNET_PQ_ResultSpec rs[] = {
413 GNUNET_PQ_result_spec_uint32 ("repl", &replication),
414 GNUNET_PQ_result_spec_uint32 ("type", &utype),
415 GNUNET_PQ_result_spec_uint32 ("prio", &priority),
416 GNUNET_PQ_result_spec_uint32 ("anonLevel", &anonymity),
417 GNUNET_PQ_result_spec_absolute_time ("expire", &expiration_time),
418 GNUNET_PQ_result_spec_auto_from_type ("hash", &key),
419 GNUNET_PQ_result_spec_variable_size ("value", &data, &size),
420 GNUNET_PQ_result_spec_uint32 ("oid", &rowid),
421 GNUNET_PQ_result_spec_end
422 };
423
424 if (GNUNET_OK !=
425 GNUNET_PQ_extract_result (res,
426 rs,
427 i))
428 {
429 GNUNET_break (0);
430 prc->proc (prc->proc_cls, NULL, 0, NULL, 0, 0, 0, 0,
431 GNUNET_TIME_UNIT_ZERO_ABS, 0);
432 return;
433 }
434
435 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
436 "datastore-postgres",
437 "Found result of size %u bytes and type %u in database\n",
438 (unsigned int) size,
439 (unsigned int) utype);
440 iret = prc->proc (prc->proc_cls,
441 &key,
442 size,
443 data,
444 (enum GNUNET_BLOCK_Type) utype,
445 priority,
446 anonymity,
447 replication,
448 expiration_time,
449 rowid);
450 if (iret == GNUNET_NO)
451 {
452 struct GNUNET_PQ_QueryParam param[] = {
453 GNUNET_PQ_query_param_uint32 (&rowid),
454 GNUNET_PQ_query_param_end
455 };
456
457 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
458 "Processor asked for item %u to be removed.\n",
459 (unsigned int) rowid);
460 if (0 <
461 GNUNET_PQ_eval_prepared_non_select (plugin->dbh,
462 "delrow",
463 param))
464 {
465 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
466 "datastore-postgres",
467 "Deleting %u bytes from database\n",
468 (unsigned int) size);
469 plugin->env->duc (plugin->env->cls,
470 -(size + GNUNET_DATASTORE_ENTRY_OVERHEAD));
471 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
472 "datastore-postgres",
473 "Deleted %u bytes from database\n",
474 (unsigned int) size);
475 }
476 }
477 GNUNET_PQ_cleanup_result (rs);
478 } /* for (i) */
479}
480
481
482/**
483 * Get one of the results for a particular key in the datastore.
484 *
485 * @param cls closure with the `struct Plugin`
486 * @param next_uid return the result with lowest uid >= next_uid
487 * @param random if true, return a random result instead of using next_uid
488 * @param key maybe NULL (to match all entries)
489 * @param type entries of which type are relevant?
490 * Use 0 for any type.
491 * @param proc function to call on the matching value;
492 * will be called with NULL if nothing matches
493 * @param proc_cls closure for @a proc
494 */
495static void
496postgres_plugin_get_key (void *cls,
497 uint64_t next_uid,
498 bool random,
499 const struct GNUNET_HashCode *key,
500 enum GNUNET_BLOCK_Type type,
501 PluginDatumProcessor proc,
502 void *proc_cls)
503{
504 struct Plugin *plugin = cls;
505 uint32_t utype = type;
506 uint16_t use_rvalue = random;
507 uint16_t use_key = NULL != key;
508 uint16_t use_type = GNUNET_BLOCK_TYPE_ANY != type;
509 uint64_t rvalue;
510 struct GNUNET_PQ_QueryParam params[] = {
511 GNUNET_PQ_query_param_uint64 (&next_uid),
512 GNUNET_PQ_query_param_uint64 (&rvalue),
513 GNUNET_PQ_query_param_uint16 (&use_rvalue),
514 GNUNET_PQ_query_param_auto_from_type (key),
515 GNUNET_PQ_query_param_uint16 (&use_key),
516 GNUNET_PQ_query_param_uint32 (&utype),
517 GNUNET_PQ_query_param_uint16 (&use_type),
518 GNUNET_PQ_query_param_end
519 };
520 struct ProcessResultContext prc;
521 enum GNUNET_DB_QueryStatus res;
522
523 if (random)
524 {
525 rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
526 UINT64_MAX);
527 next_uid = 0;
528 }
529 else
530 {
531 rvalue = 0;
532 }
533 prc.plugin = plugin;
534 prc.proc = proc;
535 prc.proc_cls = proc_cls;
536
537 res = GNUNET_PQ_eval_prepared_multi_select (plugin->dbh,
538 "get",
539 params,
540 &process_result,
541 &prc);
542 if (0 > res)
543 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0,
544 GNUNET_TIME_UNIT_ZERO_ABS, 0);
545}
546
547
548/**
549 * Select a subset of the items in the datastore and call
550 * the given iterator for each of them.
551 *
552 * @param cls our `struct Plugin *`
553 * @param next_uid return the result with lowest uid >= next_uid
554 * @param type entries of which type should be considered?
555 * Must not be zero (ANY).
556 * @param proc function to call on the matching value;
557 * will be called with NULL if no value matches
558 * @param proc_cls closure for @a proc
559 */
560static void
561postgres_plugin_get_zero_anonymity (void *cls,
562 uint64_t next_uid,
563 enum GNUNET_BLOCK_Type type,
564 PluginDatumProcessor proc,
565 void *proc_cls)
566{
567 struct Plugin *plugin = cls;
568 uint32_t utype = type;
569 struct GNUNET_PQ_QueryParam params[] = {
570 GNUNET_PQ_query_param_uint32 (&utype),
571 GNUNET_PQ_query_param_uint64 (&next_uid),
572 GNUNET_PQ_query_param_end
573 };
574 struct ProcessResultContext prc;
575 enum GNUNET_DB_QueryStatus res;
576
577 prc.plugin = plugin;
578 prc.proc = proc;
579 prc.proc_cls = proc_cls;
580 res = GNUNET_PQ_eval_prepared_multi_select (plugin->dbh,
581 "select_non_anonymous",
582 params,
583 &process_result,
584 &prc);
585 if (0 > res)
586 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0,
587 GNUNET_TIME_UNIT_ZERO_ABS, 0);
588}
589
590
591/**
592 * Context for #repl_iter() function.
593 */
594struct ReplCtx
595{
596 /**
597 * Plugin handle.
598 */
599 struct Plugin *plugin;
600
601 /**
602 * Function to call for the result (or the NULL).
603 */
604 PluginDatumProcessor proc;
605
606 /**
607 * Closure for @e proc.
608 */
609 void *proc_cls;
610};
611
612
613/**
614 * Wrapper for the iterator for 'sqlite_plugin_replication_get'.
615 * Decrements the replication counter and calls the original
616 * iterator.
617 *
618 * @param cls closure with the `struct ReplCtx *`
619 * @param key key for the content
620 * @param size number of bytes in @a data
621 * @param data content stored
622 * @param type type of the content
623 * @param priority priority of the content
624 * @param anonymity anonymity-level for the content
625 * @param replication replication-level for the content
626 * @param expiration expiration time for the content
627 * @param uid unique identifier for the datum;
628 * maybe 0 if no unique identifier is available
629 * @return #GNUNET_SYSERR to abort the iteration,
630 * #GNUNET_OK to continue
631 * (continue on call to "next", of course),
632 * #GNUNET_NO to delete the item and continue (if supported)
633 */
634static int
635repl_proc (void *cls,
636 const struct GNUNET_HashCode *key,
637 uint32_t size,
638 const void *data,
639 enum GNUNET_BLOCK_Type type,
640 uint32_t priority,
641 uint32_t anonymity,
642 uint32_t replication,
643 struct GNUNET_TIME_Absolute expiration,
644 uint64_t uid)
645{
646 struct ReplCtx *rc = cls;
647 struct Plugin *plugin = rc->plugin;
648 int ret;
649 uint32_t oid = (uint32_t) uid;
650 struct GNUNET_PQ_QueryParam params[] = {
651 GNUNET_PQ_query_param_uint32 (&oid),
652 GNUNET_PQ_query_param_end
653 };
654 enum GNUNET_DB_QueryStatus qret;
655
656 ret = rc->proc (rc->proc_cls,
657 key,
658 size,
659 data,
660 type,
661 priority,
662 anonymity,
663 replication,
664 expiration,
665 uid);
666 if (NULL == key)
667 return ret;
668 qret = GNUNET_PQ_eval_prepared_non_select (plugin->dbh,
669 "decrepl",
670 params);
671 if (0 > qret)
672 return GNUNET_SYSERR;
673 return ret;
674}
675
676
677/**
678 * Get a random item for replication. Returns a single, not expired,
679 * random item from those with the highest replication counters. The
680 * item's replication counter is decremented by one IF it was positive
681 * before. Call @a proc with all values ZERO or NULL if the datastore
682 * is empty.
683 *
684 * @param cls closure with the `struct Plugin`
685 * @param proc function to call the value (once only).
686 * @param proc_cls closure for @a proc
687 */
688static void
689postgres_plugin_get_replication (void *cls,
690 PluginDatumProcessor proc,
691 void *proc_cls)
692{
693 struct Plugin *plugin = cls;
694 struct GNUNET_PQ_QueryParam params[] = {
695 GNUNET_PQ_query_param_end
696 };
697 struct ReplCtx rc;
698 struct ProcessResultContext prc;
699 enum GNUNET_DB_QueryStatus res;
700
701 rc.plugin = plugin;
702 rc.proc = proc;
703 rc.proc_cls = proc_cls;
704 prc.plugin = plugin;
705 prc.proc = &repl_proc;
706 prc.proc_cls = &rc;
707 res = GNUNET_PQ_eval_prepared_multi_select (plugin->dbh,
708 "select_replication_order",
709 params,
710 &process_result,
711 &prc);
712 if (0 > res)
713 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0,
714 GNUNET_TIME_UNIT_ZERO_ABS, 0);
715}
716
717
718/**
719 * Get a random item for expiration. Call @a proc with all values
720 * ZERO or NULL if the datastore is empty.
721 *
722 * @param cls closure with the `struct Plugin`
723 * @param proc function to call the value (once only).
724 * @param proc_cls closure for @a proc
725 */
726static void
727postgres_plugin_get_expiration (void *cls,
728 PluginDatumProcessor proc,
729 void *proc_cls)
730{
731 struct Plugin *plugin = cls;
732 struct GNUNET_TIME_Absolute now = { 0 };
733 struct GNUNET_PQ_QueryParam params[] = {
734 GNUNET_PQ_query_param_absolute_time (&now),
735 GNUNET_PQ_query_param_end
736 };
737 struct ProcessResultContext prc;
738
739 now = GNUNET_TIME_absolute_get ();
740 prc.plugin = plugin;
741 prc.proc = proc;
742 prc.proc_cls = proc_cls;
743 (void) GNUNET_PQ_eval_prepared_multi_select (plugin->dbh,
744 "select_expiration_order",
745 params,
746 &process_result,
747 &prc);
748}
749
750
751/**
752 * Closure for #process_keys.
753 */
754struct ProcessKeysContext
755{
756 /**
757 * Function to call for each key.
758 */
759 PluginKeyProcessor proc;
760
761 /**
762 * Closure for @e proc.
763 */
764 void *proc_cls;
765};
766
767
768/**
769 * Function to be called with the results of a SELECT statement
770 * that has returned @a num_results results.
771 *
772 * @param cls closure with a `struct ProcessKeysContext`
773 * @param result the postgres result
774 * @param num_result the number of results in @a result
775 */
776static void
777process_keys (void *cls,
778 PGresult *result,
779 unsigned int num_results)
780{
781 struct ProcessKeysContext *pkc = cls;
782
783 for (unsigned i = 0; i < num_results; i++)
784 {
785 struct GNUNET_HashCode key;
786 struct GNUNET_PQ_ResultSpec rs[] = {
787 GNUNET_PQ_result_spec_auto_from_type ("hash",
788 &key),
789 GNUNET_PQ_result_spec_end
790 };
791
792 if (GNUNET_OK !=
793 GNUNET_PQ_extract_result (result,
794 rs,
795 i))
796 {
797 GNUNET_break (0);
798 continue;
799 }
800 pkc->proc (pkc->proc_cls,
801 &key,
802 1);
803 GNUNET_PQ_cleanup_result (rs);
804 }
805}
806
807
808/**
809 * Get all of the keys in the datastore.
810 *
811 * @param cls closure with the `struct Plugin *`
812 * @param proc function to call on each key
813 * @param proc_cls closure for @a proc
814 */
815static void
816postgres_plugin_get_keys (void *cls,
817 PluginKeyProcessor proc,
818 void *proc_cls)
819{
820 struct Plugin *plugin = cls;
821 struct GNUNET_PQ_QueryParam params[] = {
822 GNUNET_PQ_query_param_end
823 };
824 struct ProcessKeysContext pkc;
825
826 pkc.proc = proc;
827 pkc.proc_cls = proc_cls;
828 (void) GNUNET_PQ_eval_prepared_multi_select (plugin->dbh,
829 "get_keys",
830 params,
831 &process_keys,
832 &pkc);
833 proc (proc_cls,
834 NULL,
835 0);
836}
837
838
839/**
840 * Drop database.
841 *
842 * @param cls closure with the `struct Plugin *`
843 */
844static void
845postgres_plugin_drop (void *cls)
846{
847 struct Plugin *plugin = cls;
848 struct GNUNET_PQ_ExecuteStatement es[] = {
849 GNUNET_PQ_make_execute ("DROP TABLE gn090"),
850 GNUNET_PQ_EXECUTE_STATEMENT_END
851 };
852
853 if (GNUNET_OK !=
854 GNUNET_PQ_exec_statements (plugin->dbh,
855 es))
856 GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING,
857 "postgres",
858 _ ("Failed to drop table from database.\n"));
859}
860
861
862/**
863 * Remove a particular key in the datastore.
864 *
865 * @param cls closure
866 * @param key key for the content
867 * @param size number of bytes in data
868 * @param data content stored
869 * @param cont continuation called with success or failure status
870 * @param cont_cls continuation closure for @a cont
871 */
872static void
873postgres_plugin_remove_key (void *cls,
874 const struct GNUNET_HashCode *key,
875 uint32_t size,
876 const void *data,
877 PluginRemoveCont cont,
878 void *cont_cls)
879{
880 struct Plugin *plugin = cls;
881 enum GNUNET_DB_QueryStatus ret;
882 struct GNUNET_PQ_QueryParam params[] = {
883 GNUNET_PQ_query_param_auto_from_type (key),
884 GNUNET_PQ_query_param_fixed_size (data, size),
885 GNUNET_PQ_query_param_end
886 };
887
888 ret = GNUNET_PQ_eval_prepared_non_select (plugin->dbh,
889 "remove",
890 params);
891 if (0 > ret)
892 {
893 cont (cont_cls,
894 key,
895 size,
896 GNUNET_SYSERR,
897 _ ("Postgresql exec failure"));
898 return;
899 }
900 if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == ret)
901 {
902 cont (cont_cls,
903 key,
904 size,
905 GNUNET_NO,
906 NULL);
907 return;
908 }
909 plugin->env->duc (plugin->env->cls,
910 -(size + GNUNET_DATASTORE_ENTRY_OVERHEAD));
911 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
912 "datastore-postgres",
913 "Deleted %u bytes from database\n",
914 (unsigned int) size);
915 cont (cont_cls,
916 key,
917 size,
918 GNUNET_OK,
919 NULL);
920}
921
922
923/**
924 * Entry point for the plugin.
925 *
926 * @param cls the `struct GNUNET_DATASTORE_PluginEnvironment*`
927 * @return our `struct Plugin *`
928 */
929void *
930libgnunet_plugin_datastore_postgres_init (void *cls)
931{
932 struct GNUNET_DATASTORE_PluginEnvironment *env = cls;
933 struct GNUNET_DATASTORE_PluginFunctions *api;
934 struct Plugin *plugin;
935
936 plugin = GNUNET_new (struct Plugin);
937 plugin->env = env;
938 if (GNUNET_OK != init_connection (plugin))
939 {
940 GNUNET_free (plugin);
941 return NULL;
942 }
943 api = GNUNET_new (struct GNUNET_DATASTORE_PluginFunctions);
944 api->cls = plugin;
945 api->estimate_size = &postgres_plugin_estimate_size;
946 api->put = &postgres_plugin_put;
947 api->get_key = &postgres_plugin_get_key;
948 api->get_replication = &postgres_plugin_get_replication;
949 api->get_expiration = &postgres_plugin_get_expiration;
950 api->get_zero_anonymity = &postgres_plugin_get_zero_anonymity;
951 api->get_keys = &postgres_plugin_get_keys;
952 api->drop = &postgres_plugin_drop;
953 api->remove_key = &postgres_plugin_remove_key;
954 GNUNET_log_from (GNUNET_ERROR_TYPE_INFO,
955 "datastore-postgres",
956 _ ("Postgres database running\n"));
957 return api;
958}
959
960
961/**
962 * Exit point from the plugin.
963 *
964 * @param cls our `struct Plugin *`
965 * @return always NULL
966 */
967void *
968libgnunet_plugin_datastore_postgres_done (void *cls)
969{
970 struct GNUNET_DATASTORE_PluginFunctions *api = cls;
971 struct Plugin *plugin = api->cls;
972
973 GNUNET_PQ_disconnect (plugin->dbh);
974 GNUNET_free (plugin);
975 GNUNET_free (api);
976 return NULL;
977}
978
979
980/* end of plugin_datastore_postgres.c */
diff --git a/src/datastore/plugin_datastore_sqlite.c b/src/datastore/plugin_datastore_sqlite.c
deleted file mode 100644
index 3c2d7f2d4..000000000
--- a/src/datastore/plugin_datastore_sqlite.c
+++ /dev/null
@@ -1,1374 +0,0 @@
1/*
2 * This file is part of GNUnet
3 * Copyright (C) 2009, 2011, 2017 GNUnet e.V.
4 *
5 * GNUnet is free software: you can redistribute it and/or modify it
6 * under the terms of the GNU Affero General Public License as published
7 * by the Free Software Foundation, either version 3 of the License,
8 * or (at your option) any later version.
9 *
10 * GNUnet is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Affero General Public License for more details.
14 *
15 * You should have received a copy of the GNU Affero General Public License
16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file datastore/plugin_datastore_sqlite.c
23 * @brief sqlite-based datastore backend
24 * @author Christian Grothoff
25 */
26
27#include "platform.h"
28#include "gnunet_datastore_plugin.h"
29#include "gnunet_sq_lib.h"
30#include <sqlite3.h>
31
32
33/**
34 * We allocate items on the stack at times. To prevent a stack
35 * overflow, we impose a limit on the maximum size for the data per
36 * item. 64k should be enough.
37 */
38#define MAX_ITEM_SIZE 65536
39
40/**
41 * After how many ms "busy" should a DB operation fail for good?
42 * A low value makes sure that we are more responsive to requests
43 * (especially PUTs). A high value guarantees a higher success
44 * rate (SELECTs in iterate can take several seconds despite LIMIT=1).
45 *
46 * The default value of 250ms should ensure that users do not experience
47 * huge latencies while at the same time allowing operations to succeed
48 * with reasonable probability.
49 */
50#define BUSY_TIMEOUT_MS 250
51
52
53/**
54 * Log an error message at log-level 'level' that indicates
55 * a failure of the command 'cmd' on file 'filename'
56 * with the message given by strerror(errno).
57 */
58#define LOG_SQLITE(db, level, cmd) \
59 do \
60 { \
61 GNUNET_log_from (level, \
62 "sqlite", \
63 _ ("`%s' failed at %s:%d with error: %s\n"), \
64 cmd, \
65 __FILE__, \
66 __LINE__, \
67 sqlite3_errmsg (db->dbh)); \
68 } while (0)
69
70
71/**
72 * Log an error message at log-level 'level' that indicates
73 * a failure of the command 'cmd' on file 'filename'
74 * with the message given by strerror(errno).
75 */
76#define LOG_SQLITE_MSG(db, msg, level, cmd) \
77 do \
78 { \
79 GNUNET_log_from (level, \
80 "sqlite", \
81 _ ("`%s' failed at %s:%d with error: %s\n"), \
82 cmd, \
83 __FILE__, \
84 __LINE__, \
85 sqlite3_errmsg (db->dbh)); \
86 GNUNET_asprintf (msg, \
87 _ ("`%s' failed at %s:%u with error: %s"), \
88 cmd, \
89 __FILE__, \
90 __LINE__, \
91 sqlite3_errmsg (db->dbh)); \
92 } while (0)
93
94
95/**
96 * Context for all functions in this plugin.
97 */
98struct Plugin
99{
100 /**
101 * Our execution environment.
102 */
103 struct GNUNET_DATASTORE_PluginEnvironment *env;
104
105 /**
106 * Database filename.
107 */
108 char *fn;
109
110 /**
111 * Native SQLite database handle.
112 */
113 sqlite3 *dbh;
114
115 /**
116 * Precompiled SQL for remove_key.
117 */
118 sqlite3_stmt *remove;
119
120 /**
121 * Precompiled SQL for deletion.
122 */
123 sqlite3_stmt *delRow;
124
125 /**
126 * Precompiled SQL for update.
127 */
128 sqlite3_stmt *update;
129
130 /**
131 * Get maximum repl value in database.
132 */
133 sqlite3_stmt *maxRepl;
134
135 /**
136 * Precompiled SQL for replication decrement.
137 */
138 sqlite3_stmt *updRepl;
139
140 /**
141 * Precompiled SQL for replication selection.
142 */
143 sqlite3_stmt *selRepl;
144
145 /**
146 * Precompiled SQL for expiration selection.
147 */
148 sqlite3_stmt *selExpi;
149
150 /**
151 * Precompiled SQL for expiration selection.
152 */
153 sqlite3_stmt *selZeroAnon;
154
155 /**
156 * Precompiled SQL for insertion.
157 */
158 sqlite3_stmt *insertContent;
159
160 /**
161 * Precompiled SQL for selection
162 */
163 sqlite3_stmt *get[8];
164
165 /**
166 * Should the database be dropped on shutdown?
167 */
168 int drop_on_shutdown;
169};
170
171
172/**
173 * @brief Prepare a SQL statement
174 *
175 * @param dbh handle to the database
176 * @param zSql SQL statement, UTF-8 encoded
177 * @param ppStmt set to the prepared statement
178 * @return 0 on success
179 */
180static int
181sq_prepare (sqlite3 *dbh, const char *zSql, sqlite3_stmt **ppStmt)
182{
183 char *dummy;
184 int result;
185
186 result = sqlite3_prepare_v2 (dbh,
187 zSql,
188 strlen (zSql),
189 ppStmt,
190 (const char **) &dummy);
191 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
192 "sqlite",
193 "Prepared `%s' / %p: %d\n",
194 zSql,
195 *ppStmt,
196 result);
197 return result;
198}
199
200
201/**
202 * Create our database indices.
203 *
204 * @param dbh handle to the database
205 */
206static void
207create_indices (sqlite3 *dbh)
208{
209 /* create indices */
210 if (
211 0 !=
212 (SQLITE_OK !=
213 sqlite3_exec (dbh,
214 "CREATE INDEX IF NOT EXISTS idx_hash ON gn091 (hash)",
215 NULL,
216 NULL,
217 NULL))
218 + (SQLITE_OK !=
219 sqlite3_exec (
220 dbh,
221 "CREATE INDEX IF NOT EXISTS idx_anon_type ON gn091 (anonLevel ASC,type)",
222 NULL,
223 NULL,
224 NULL))
225 + (SQLITE_OK !=
226 sqlite3_exec (dbh,
227 "CREATE INDEX IF NOT EXISTS idx_expire ON gn091 (expire ASC)",
228 NULL,
229 NULL,
230 NULL))
231 + (SQLITE_OK !=
232 sqlite3_exec (
233 dbh,
234 "CREATE INDEX IF NOT EXISTS idx_repl_rvalue ON gn091 (repl,rvalue)",
235 NULL,
236 NULL,
237 NULL)))
238 GNUNET_log_from (GNUNET_ERROR_TYPE_ERROR,
239 "sqlite",
240 "Failed to create indices: %s\n",
241 sqlite3_errmsg (dbh));
242}
243
244
245#if 0
246#define CHECK(a) GNUNET_break (a)
247#define ENULL NULL
248#else
249#define ENULL &e
250#define ENULL_DEFINED 1
251#define CHECK(a) \
252 if (! (a)) \
253 { \
254 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "%s\n", e); \
255 sqlite3_free (e); \
256 }
257#endif
258
259
260/**
261 * Initialize the database connections and associated
262 * data structures (create tables and indices
263 * as needed as well).
264 *
265 * @param cfg our configuration
266 * @param plugin the plugin context (state for this module)
267 * @return #GNUNET_OK on success
268 */
269static int
270database_setup (const struct GNUNET_CONFIGURATION_Handle *cfg,
271 struct Plugin *plugin)
272{
273 sqlite3_stmt *stmt;
274 char *afsdir;
275
276#if ENULL_DEFINED
277 char *e;
278#endif
279
280 if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_filename (cfg,
281 "datastore-sqlite",
282 "FILENAME",
283 &afsdir))
284 {
285 GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
286 "datastore-sqlite",
287 "FILENAME");
288 return GNUNET_SYSERR;
289 }
290 if (GNUNET_OK != GNUNET_DISK_file_test (afsdir))
291 {
292 if (GNUNET_OK != GNUNET_DISK_directory_create_for_file (afsdir))
293 {
294 GNUNET_break (0);
295 GNUNET_free (afsdir);
296 return GNUNET_SYSERR;
297 }
298 /* database is new or got deleted, reset payload to zero! */
299 if (NULL != plugin->env->duc)
300 plugin->env->duc (plugin->env->cls, 0);
301 }
302 /* afsdir should be UTF-8-encoded. If it isn't, it's a bug */
303 plugin->fn = afsdir;
304
305 /* Open database and precompile statements */
306 if (SQLITE_OK != sqlite3_open (plugin->fn, &plugin->dbh))
307 {
308 GNUNET_log_from (GNUNET_ERROR_TYPE_ERROR,
309 "sqlite",
310 _ ("Unable to initialize SQLite: %s.\n"),
311 sqlite3_errmsg (plugin->dbh));
312 return GNUNET_SYSERR;
313 }
314 CHECK (
315 SQLITE_OK ==
316 sqlite3_exec (plugin->dbh, "PRAGMA temp_store=MEMORY", NULL, NULL, ENULL));
317 CHECK (
318 SQLITE_OK ==
319 sqlite3_exec (plugin->dbh, "PRAGMA synchronous=OFF", NULL, NULL, ENULL));
320 CHECK (SQLITE_OK == sqlite3_exec (plugin->dbh,
321 "PRAGMA legacy_file_format=OFF",
322 NULL,
323 NULL,
324 ENULL));
325 CHECK (SQLITE_OK == sqlite3_exec (plugin->dbh,
326 "PRAGMA auto_vacuum=INCREMENTAL",
327 NULL,
328 NULL,
329 ENULL));
330 CHECK (SQLITE_OK == sqlite3_exec (plugin->dbh,
331 "PRAGMA locking_mode=EXCLUSIVE",
332 NULL,
333 NULL,
334 ENULL));
335 CHECK (
336 SQLITE_OK ==
337 sqlite3_exec (plugin->dbh, "PRAGMA page_size=4096", NULL, NULL, ENULL));
338
339 CHECK (SQLITE_OK == sqlite3_busy_timeout (plugin->dbh, BUSY_TIMEOUT_MS));
340
341
342 /* We have to do it here, because otherwise precompiling SQL might fail */
343 CHECK (SQLITE_OK ==
344 sq_prepare (plugin->dbh,
345 "SELECT 1 FROM sqlite_master WHERE tbl_name = 'gn091'",
346 &stmt));
347
348 /* FIXME: SQLite does not have unsigned integers! This is ok for the type column because
349 * we only test equality on it and can cast it to/from uint32_t. For repl, prio, and anonLevel
350 * we do math or inequality tests, so we can't handle the entire range of uint32_t.
351 * This will also cause problems for expiration times after 294247-01-10-04:00:54 UTC.
352 */if ((SQLITE_DONE == sqlite3_step (stmt)) &&
353 (SQLITE_OK != sqlite3_exec (plugin->dbh,
354 "CREATE TABLE gn091 ("
355 " repl INT4 NOT NULL DEFAULT 0,"
356 " type INT4 NOT NULL DEFAULT 0,"
357 " prio INT4 NOT NULL DEFAULT 0,"
358 " anonLevel INT4 NOT NULL DEFAULT 0,"
359 " expire INT8 NOT NULL DEFAULT 0,"
360 " rvalue INT8 NOT NULL,"
361 " hash TEXT NOT NULL DEFAULT '',"
362 " vhash TEXT NOT NULL DEFAULT '',"
363 " value BLOB NOT NULL DEFAULT '')",
364 NULL,
365 NULL,
366 NULL)))
367 {
368 LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "sqlite3_exec");
369 sqlite3_finalize (stmt);
370 return GNUNET_SYSERR;
371 }
372 sqlite3_finalize (stmt);
373 create_indices (plugin->dbh);
374
375#define RESULT_COLUMNS \
376 "repl, type, prio, anonLevel, expire, hash, value, _ROWID_"
377 if (
378 (SQLITE_OK != sq_prepare (plugin->dbh,
379 "UPDATE gn091 "
380 "SET prio = prio + ?, "
381 "repl = repl + ?, "
382 "expire = MAX(expire, ?) "
383 "WHERE hash = ? AND vhash = ?",
384 &plugin->update)) ||
385 (SQLITE_OK != sq_prepare (plugin->dbh,
386 "UPDATE gn091 "
387 "SET repl = MAX (0, repl - 1) WHERE _ROWID_ = ?",
388 &plugin->updRepl)) ||
389 (SQLITE_OK != sq_prepare (plugin->dbh,
390 "SELECT " RESULT_COLUMNS " FROM gn091 "
391 "WHERE repl=?2 AND "
392 " (rvalue>=?1 OR "
393 " NOT EXISTS (SELECT 1 FROM gn091 "
394 "WHERE repl=?2 AND rvalue>=?1 LIMIT 1) ) "
395 "ORDER BY rvalue ASC LIMIT 1",
396 &plugin->selRepl)) ||
397 (SQLITE_OK != sq_prepare (plugin->dbh,
398 "SELECT MAX(repl) FROM gn091",
399 &plugin->maxRepl)) ||
400 (SQLITE_OK !=
401 sq_prepare (plugin->dbh,
402 "SELECT " RESULT_COLUMNS " FROM gn091 "
403 "WHERE NOT EXISTS (SELECT 1 FROM gn091 WHERE expire < ?1 LIMIT 1) OR (expire < ?1) "
404 "ORDER BY expire ASC LIMIT 1",
405 &plugin->selExpi)) ||
406 (SQLITE_OK != sq_prepare (plugin->dbh,
407 "SELECT " RESULT_COLUMNS " FROM gn091 "
408 "WHERE _ROWID_ >= ? AND "
409 "anonLevel = 0 AND "
410 "type = ? "
411 "ORDER BY _ROWID_ ASC LIMIT 1",
412 &plugin->selZeroAnon)) ||
413 (SQLITE_OK !=
414 sq_prepare (plugin->dbh,
415 "INSERT INTO gn091 (repl, type, prio, anonLevel, expire, rvalue, hash, vhash, value) "
416 "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
417 &plugin->insertContent)) ||
418 (SQLITE_OK != sq_prepare (plugin->dbh,
419 "SELECT " RESULT_COLUMNS " FROM gn091 "
420 "WHERE _ROWID_ >= ?1 "
421 "ORDER BY _ROWID_ ASC LIMIT 1",
422 &plugin->get[0])) ||
423 (SQLITE_OK != sq_prepare (plugin->dbh,
424 "SELECT " RESULT_COLUMNS " FROM gn091 "
425 "WHERE _ROWID_ >= ?1 AND "
426 "type = ?4 "
427 "ORDER BY _ROWID_ ASC LIMIT 1",
428 &plugin->get[1])) ||
429 (SQLITE_OK != sq_prepare (plugin->dbh,
430 "SELECT " RESULT_COLUMNS " FROM gn091 "
431 "WHERE _ROWID_ >= ?1 AND "
432 "hash = ?3 "
433 "ORDER BY _ROWID_ ASC LIMIT 1",
434 &plugin->get[2])) ||
435 (SQLITE_OK != sq_prepare (plugin->dbh,
436 "SELECT " RESULT_COLUMNS " FROM gn091 "
437 "WHERE _ROWID_ >= ?1 AND "
438 "hash = ?3 AND "
439 "type = ?4 "
440 "ORDER BY _ROWID_ ASC LIMIT 1",
441 &plugin->get[3])) ||
442 (SQLITE_OK != sq_prepare (plugin->dbh,
443 "SELECT " RESULT_COLUMNS " FROM gn091 "
444 "WHERE _ROWID_ >= ?1 AND "
445 "rvalue >= ?2 "
446 "ORDER BY _ROWID_ ASC LIMIT 1",
447 &plugin->get[4])) ||
448 (SQLITE_OK != sq_prepare (plugin->dbh,
449 "SELECT " RESULT_COLUMNS " FROM gn091 "
450 "WHERE _ROWID_ >= ?1 AND "
451 "rvalue >= ?2 AND "
452 "type = ?4 "
453 "ORDER BY _ROWID_ ASC LIMIT 1",
454 &plugin->get[5])) ||
455 (SQLITE_OK != sq_prepare (plugin->dbh,
456 "SELECT " RESULT_COLUMNS " FROM gn091 "
457 "WHERE _ROWID_ >= ?1 AND "
458 "rvalue >= ?2 AND "
459 "hash = ?3 "
460 "ORDER BY _ROWID_ ASC LIMIT 1",
461 &plugin->get[6])) ||
462 (SQLITE_OK != sq_prepare (plugin->dbh,
463 "SELECT " RESULT_COLUMNS " FROM gn091 "
464 "WHERE _ROWID_ >= ?1 AND "
465 "rvalue >= ?2 AND "
466 "hash = ?3 AND "
467 "type = ?4 "
468 "ORDER BY _ROWID_ ASC LIMIT 1",
469 &plugin->get[7])) ||
470 (SQLITE_OK != sq_prepare (plugin->dbh,
471 "DELETE FROM gn091 WHERE _ROWID_ = ?",
472 &plugin->delRow)) ||
473 (SQLITE_OK != sq_prepare (plugin->dbh,
474 "DELETE FROM gn091 "
475 "WHERE hash = ? AND "
476 "value = ? ",
477 &plugin->remove)) ||
478 false)
479 {
480 LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "precompiling");
481 return GNUNET_SYSERR;
482 }
483 return GNUNET_OK;
484}
485
486
487/**
488 * Shutdown database connection and associate data
489 * structures.
490 *
491 * @param plugin the plugin context (state for this module)
492 */
493static void
494database_shutdown (struct Plugin *plugin)
495{
496 int result;
497
498#if SQLITE_VERSION_NUMBER >= 3007000
499 sqlite3_stmt *stmt;
500#endif
501
502 if (NULL != plugin->remove)
503 sqlite3_finalize (plugin->remove);
504 if (NULL != plugin->delRow)
505 sqlite3_finalize (plugin->delRow);
506 if (NULL != plugin->update)
507 sqlite3_finalize (plugin->update);
508 if (NULL != plugin->updRepl)
509 sqlite3_finalize (plugin->updRepl);
510 if (NULL != plugin->selRepl)
511 sqlite3_finalize (plugin->selRepl);
512 if (NULL != plugin->maxRepl)
513 sqlite3_finalize (plugin->maxRepl);
514 if (NULL != plugin->selExpi)
515 sqlite3_finalize (plugin->selExpi);
516 if (NULL != plugin->selZeroAnon)
517 sqlite3_finalize (plugin->selZeroAnon);
518 if (NULL != plugin->insertContent)
519 sqlite3_finalize (plugin->insertContent);
520 for (int i = 0; i < 8; ++i)
521 if (NULL != plugin->get[i])
522 sqlite3_finalize (plugin->get[i]);
523 result = sqlite3_close (plugin->dbh);
524#if SQLITE_VERSION_NUMBER >= 3007000
525 if (result == SQLITE_BUSY)
526 {
527 GNUNET_log_from (
528 GNUNET_ERROR_TYPE_WARNING,
529 "sqlite",
530 _ (
531 "Tried to close sqlite without finalizing all prepared statements.\n"));
532 stmt = sqlite3_next_stmt (plugin->dbh, NULL);
533 while (NULL != stmt)
534 {
535 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
536 "sqlite",
537 "Closing statement %p\n",
538 stmt);
539 result = sqlite3_finalize (stmt);
540 if (result != SQLITE_OK)
541 GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING,
542 "sqlite",
543 "Failed to close statement %p: %d\n",
544 stmt,
545 result);
546 stmt = sqlite3_next_stmt (plugin->dbh, NULL);
547 }
548 result = sqlite3_close (plugin->dbh);
549 }
550#endif
551 if (SQLITE_OK != result)
552 LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "sqlite3_close");
553 GNUNET_free (plugin->fn);
554}
555
556
557/**
558 * Delete the database entry with the given
559 * row identifier.
560 *
561 * @param plugin the plugin context (state for this module)
562 * @param rid the ID of the row to delete
563 */
564static int
565delete_by_rowid (struct Plugin *plugin, uint64_t rid)
566{
567 struct GNUNET_SQ_QueryParam params[] = { GNUNET_SQ_query_param_uint64 (&rid),
568 GNUNET_SQ_query_param_end };
569
570 if (GNUNET_OK != GNUNET_SQ_bind (plugin->delRow, params))
571 return GNUNET_SYSERR;
572 if (SQLITE_DONE != sqlite3_step (plugin->delRow))
573 {
574 LOG_SQLITE (plugin,
575 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
576 "sqlite3_step");
577 GNUNET_SQ_reset (plugin->dbh, plugin->delRow);
578 return GNUNET_SYSERR;
579 }
580 GNUNET_SQ_reset (plugin->dbh, plugin->delRow);
581 return GNUNET_OK;
582}
583
584
585/**
586 * Store an item in the datastore.
587 *
588 * @param cls closure
589 * @param key key for the item
590 * @param absent true if the key was not found in the bloom filter
591 * @param size number of bytes in @a data
592 * @param data content stored
593 * @param type type of the content
594 * @param priority priority of the content
595 * @param anonymity anonymity-level for the content
596 * @param replication replication-level for the content
597 * @param expiration expiration time for the content
598 * @param cont continuation called with success or failure status
599 * @param cont_cls continuation closure
600 */
601static void
602sqlite_plugin_put (void *cls,
603 const struct GNUNET_HashCode *key,
604 bool absent,
605 uint32_t size,
606 const void *data,
607 enum GNUNET_BLOCK_Type type,
608 uint32_t priority,
609 uint32_t anonymity,
610 uint32_t replication,
611 struct GNUNET_TIME_Absolute expiration,
612 PluginPutCont cont,
613 void *cont_cls)
614{
615 struct Plugin *plugin = cls;
616 struct GNUNET_HashCode vhash;
617 char *msg = NULL;
618
619 GNUNET_CRYPTO_hash (data, size, &vhash);
620
621 if (! absent)
622 {
623 struct GNUNET_SQ_QueryParam params[] =
624 { GNUNET_SQ_query_param_uint32 (&priority),
625 GNUNET_SQ_query_param_uint32 (&replication),
626 GNUNET_SQ_query_param_absolute_time (&expiration),
627 GNUNET_SQ_query_param_auto_from_type (key),
628 GNUNET_SQ_query_param_auto_from_type (&vhash),
629 GNUNET_SQ_query_param_end };
630
631 if (GNUNET_OK != GNUNET_SQ_bind (plugin->update, params))
632 {
633 cont (cont_cls, key, size, GNUNET_SYSERR, _ ("sqlite bind failure"));
634 return;
635 }
636 if (SQLITE_DONE != sqlite3_step (plugin->update))
637 {
638 LOG_SQLITE_MSG (plugin,
639 &msg,
640 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
641 "sqlite3_step");
642 cont (cont_cls, key, size, GNUNET_SYSERR, msg);
643 GNUNET_free (msg);
644 return;
645 }
646 int changes = sqlite3_changes (plugin->dbh);
647 GNUNET_SQ_reset (plugin->dbh, plugin->update);
648 if (0 != changes)
649 {
650 cont (cont_cls, key, size, GNUNET_NO, NULL);
651 return;
652 }
653 }
654
655 uint64_t rvalue;
656 uint32_t type32 = (uint32_t) type;
657 struct GNUNET_SQ_QueryParam params[] =
658 { GNUNET_SQ_query_param_uint32 (&replication),
659 GNUNET_SQ_query_param_uint32 (&type32),
660 GNUNET_SQ_query_param_uint32 (&priority),
661 GNUNET_SQ_query_param_uint32 (&anonymity),
662 GNUNET_SQ_query_param_absolute_time (&expiration),
663 GNUNET_SQ_query_param_uint64 (&rvalue),
664 GNUNET_SQ_query_param_auto_from_type (key),
665 GNUNET_SQ_query_param_auto_from_type (&vhash),
666 GNUNET_SQ_query_param_fixed_size (data, size),
667 GNUNET_SQ_query_param_end };
668 int n;
669 int ret;
670 sqlite3_stmt *stmt;
671
672 if (size > MAX_ITEM_SIZE)
673 {
674 cont (cont_cls, key, size, GNUNET_SYSERR, _ ("Data too large"));
675 return;
676 }
677 GNUNET_log_from (
678 GNUNET_ERROR_TYPE_DEBUG,
679 "sqlite",
680 "Storing in database block with type %u/key `%s'/priority %u/expiration in %s (%s).\n",
681 type,
682 GNUNET_h2s (key),
683 priority,
684 GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_remaining (
685 expiration),
686 GNUNET_YES),
687 GNUNET_STRINGS_absolute_time_to_string (expiration));
688 stmt = plugin->insertContent;
689 rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, UINT64_MAX);
690 if (GNUNET_OK != GNUNET_SQ_bind (stmt, params))
691 {
692 cont (cont_cls, key, size, GNUNET_SYSERR, NULL);
693 return;
694 }
695 n = sqlite3_step (stmt);
696 switch (n)
697 {
698 case SQLITE_DONE:
699 if (NULL != plugin->env->duc)
700 plugin->env->duc (plugin->env->cls,
701 size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
702 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
703 "sqlite",
704 "Stored new entry (%u bytes)\n",
705 size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
706 ret = GNUNET_OK;
707 break;
708
709 case SQLITE_BUSY:
710 GNUNET_break (0);
711 LOG_SQLITE_MSG (plugin,
712 &msg,
713 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
714 "sqlite3_step");
715 ret = GNUNET_SYSERR;
716 break;
717
718 default:
719 LOG_SQLITE_MSG (plugin,
720 &msg,
721 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
722 "sqlite3_step");
723 GNUNET_SQ_reset (plugin->dbh, stmt);
724 database_shutdown (plugin);
725 database_setup (plugin->env->cfg, plugin);
726 cont (cont_cls, key, size, GNUNET_SYSERR, msg);
727 GNUNET_free (msg);
728 return;
729 }
730 GNUNET_SQ_reset (plugin->dbh, stmt);
731 cont (cont_cls, key, size, ret, msg);
732 GNUNET_free (msg);
733}
734
735
736/**
737 * Execute statement that gets a row and call the callback
738 * with the result. Resets the statement afterwards.
739 *
740 * @param plugin the plugin
741 * @param stmt the statement
742 * @param proc processor to call
743 * @param proc_cls closure for @a proc
744 */
745static void
746execute_get (struct Plugin *plugin,
747 sqlite3_stmt *stmt,
748 PluginDatumProcessor proc,
749 void *proc_cls)
750{
751 int n;
752 struct GNUNET_TIME_Absolute expiration;
753 uint32_t replication;
754 uint32_t type;
755 uint32_t priority;
756 uint32_t anonymity;
757 uint64_t rowid;
758 void *value;
759 size_t value_size;
760 struct GNUNET_HashCode key;
761 int ret;
762 struct GNUNET_SQ_ResultSpec rs[] =
763 { GNUNET_SQ_result_spec_uint32 (&replication),
764 GNUNET_SQ_result_spec_uint32 (&type),
765 GNUNET_SQ_result_spec_uint32 (&priority),
766 GNUNET_SQ_result_spec_uint32 (&anonymity),
767 GNUNET_SQ_result_spec_absolute_time (&expiration),
768 GNUNET_SQ_result_spec_auto_from_type (&key),
769 GNUNET_SQ_result_spec_variable_size (&value, &value_size),
770 GNUNET_SQ_result_spec_uint64 (&rowid),
771 GNUNET_SQ_result_spec_end };
772
773 n = sqlite3_step (stmt);
774 switch (n)
775 {
776 case SQLITE_ROW:
777 if (GNUNET_OK != GNUNET_SQ_extract_result (stmt, rs))
778 {
779 GNUNET_break (0);
780 break;
781 }
782 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
783 "sqlite",
784 "Found reply in database with expiration %s\n",
785 GNUNET_STRINGS_absolute_time_to_string (expiration));
786 ret = proc (proc_cls,
787 &key,
788 value_size,
789 value,
790 type,
791 priority,
792 anonymity,
793 replication,
794 expiration,
795 rowid);
796 GNUNET_SQ_cleanup_result (rs);
797 GNUNET_SQ_reset (plugin->dbh, stmt);
798 if ((GNUNET_NO == ret) && (GNUNET_OK == delete_by_rowid (plugin, rowid)) &&
799 (NULL != plugin->env->duc))
800 plugin->env->duc (plugin->env->cls,
801 -(value_size + GNUNET_DATASTORE_ENTRY_OVERHEAD));
802 return;
803
804 case SQLITE_DONE:
805 /* database must be empty */
806 break;
807
808 case SQLITE_BUSY:
809 case SQLITE_ERROR:
810 case SQLITE_MISUSE:
811 default:
812 LOG_SQLITE (plugin,
813 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
814 "sqlite3_step");
815 if (SQLITE_OK != sqlite3_reset (stmt))
816 LOG_SQLITE (plugin,
817 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
818 "sqlite3_reset");
819 GNUNET_break (0);
820 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
821 database_shutdown (plugin);
822 database_setup (plugin->env->cfg, plugin);
823 return;
824 }
825 GNUNET_SQ_reset (plugin->dbh, stmt);
826 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
827}
828
829
830/**
831 * Select a subset of the items in the datastore and call
832 * the given processor for the item.
833 *
834 * @param cls our plugin context
835 * @param next_uid return the result with lowest uid >= next_uid
836 * @param type entries of which type should be considered?
837 * Must not be zero (ANY).
838 * @param proc function to call on the matching value;
839 * will be called with NULL if no value matches
840 * @param proc_cls closure for @a proc
841 */
842static void
843sqlite_plugin_get_zero_anonymity (void *cls,
844 uint64_t next_uid,
845 enum GNUNET_BLOCK_Type type,
846 PluginDatumProcessor proc,
847 void *proc_cls)
848{
849 struct Plugin *plugin = cls;
850 uint32_t type32 = type;
851 struct GNUNET_SQ_QueryParam params[] = { GNUNET_SQ_query_param_uint64 (
852 &next_uid),
853 GNUNET_SQ_query_param_uint32 (
854 &type32),
855 GNUNET_SQ_query_param_end };
856
857 GNUNET_assert (type != GNUNET_BLOCK_TYPE_ANY);
858 if (GNUNET_OK != GNUNET_SQ_bind (plugin->selZeroAnon, params))
859 {
860 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
861 return;
862 }
863 execute_get (plugin, plugin->selZeroAnon, proc, proc_cls);
864}
865
866
867/**
868 * Get results for a particular key in the datastore.
869 *
870 * @param cls closure
871 * @param next_uid return the result with lowest uid >= next_uid
872 * @param random if true, return a random result instead of using next_uid
873 * @param key maybe NULL (to match all entries)
874 * @param type entries of which type are relevant?
875 * Use 0 for any type.
876 * @param proc function to call on the matching value;
877 * will be called with NULL if nothing matches
878 * @param proc_cls closure for @a proc
879 */
880static void
881sqlite_plugin_get_key (void *cls,
882 uint64_t next_uid,
883 bool random,
884 const struct GNUNET_HashCode *key,
885 enum GNUNET_BLOCK_Type type,
886 PluginDatumProcessor proc,
887 void *proc_cls)
888{
889 struct Plugin *plugin = cls;
890 uint64_t rvalue;
891 int use_rvalue = random;
892 uint32_t type32 = (uint32_t) type;
893 int use_type = GNUNET_BLOCK_TYPE_ANY != type;
894 int use_key = NULL != key;
895 sqlite3_stmt *stmt = plugin->get[use_rvalue * 4 + use_key * 2 + use_type];
896 struct GNUNET_SQ_QueryParam params[] =
897 { GNUNET_SQ_query_param_uint64 (&next_uid),
898 GNUNET_SQ_query_param_uint64 (&rvalue),
899 GNUNET_SQ_query_param_auto_from_type (key),
900 GNUNET_SQ_query_param_uint32 (&type32),
901 GNUNET_SQ_query_param_end };
902
903 /* SQLite doesn't like it when you try to bind a parameter greater than the
904 * last numbered parameter, but unused parameters in the middle are OK.
905 */
906 if (! use_type)
907 {
908 params[3] = (struct GNUNET_SQ_QueryParam) GNUNET_SQ_query_param_end;
909 if (! use_key)
910 {
911 params[2] = (struct GNUNET_SQ_QueryParam) GNUNET_SQ_query_param_end;
912 if (! use_rvalue)
913 params[1] = (struct GNUNET_SQ_QueryParam) GNUNET_SQ_query_param_end;
914 }
915 }
916 if (random)
917 {
918 rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, UINT64_MAX);
919 next_uid = 0;
920 }
921 else
922 rvalue = 0;
923
924 if (GNUNET_OK != GNUNET_SQ_bind (stmt, params))
925 {
926 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
927 return;
928 }
929 execute_get (plugin, stmt, proc, proc_cls);
930}
931
932
933/**
934 * Context for #repl_proc() function.
935 */
936struct ReplCtx
937{
938 /**
939 * Function to call for the result (or the NULL).
940 */
941 PluginDatumProcessor proc;
942
943 /**
944 * Closure for @e proc.
945 */
946 void *proc_cls;
947
948 /**
949 * UID to use.
950 */
951 uint64_t uid;
952
953 /**
954 * Yes if UID was set.
955 */
956 int have_uid;
957};
958
959
960/**
961 * Wrapper for the processor for #sqlite_plugin_get_replication().
962 * Decrements the replication counter and calls the original
963 * processor.
964 *
965 * @param cls closure
966 * @param key key for the content
967 * @param size number of bytes in @a data
968 * @param data content stored
969 * @param type type of the content
970 * @param priority priority of the content
971 * @param anonymity anonymity-level for the content
972 * @param replication replication-level for the content
973 * @param expiration expiration time for the content
974 * @param uid unique identifier for the datum;
975 * maybe 0 if no unique identifier is available
976 * @return #GNUNET_OK for normal return,
977 * #GNUNET_NO to delete the item
978 */
979static int
980repl_proc (void *cls,
981 const struct GNUNET_HashCode *key,
982 uint32_t size,
983 const void *data,
984 enum GNUNET_BLOCK_Type type,
985 uint32_t priority,
986 uint32_t anonymity,
987 uint32_t replication,
988 struct GNUNET_TIME_Absolute expiration,
989 uint64_t uid)
990{
991 struct ReplCtx *rc = cls;
992 int ret;
993
994 if (GNUNET_SYSERR == rc->have_uid)
995 rc->have_uid = GNUNET_NO;
996 ret = rc->proc (rc->proc_cls,
997 key,
998 size,
999 data,
1000 type,
1001 priority,
1002 anonymity,
1003 replication,
1004 expiration,
1005 uid);
1006 if (NULL != key)
1007 {
1008 rc->uid = uid;
1009 rc->have_uid = GNUNET_YES;
1010 }
1011 return ret;
1012}
1013
1014
1015/**
1016 * Get a random item for replication. Returns a single random item
1017 * from those with the highest replication counters. The item's
1018 * replication counter is decremented by one IF it was positive before.
1019 * Call @a proc with all values ZERO or NULL if the datastore is empty.
1020 *
1021 * @param cls closure
1022 * @param proc function to call the value (once only).
1023 * @param proc_cls closure for @a proc
1024 */
1025static void
1026sqlite_plugin_get_replication (void *cls,
1027 PluginDatumProcessor proc,
1028 void *proc_cls)
1029{
1030 struct Plugin *plugin = cls;
1031 struct ReplCtx rc;
1032 uint64_t rvalue = 0;
1033 uint32_t repl;
1034 struct GNUNET_SQ_QueryParam params_sel_repl[] =
1035 { GNUNET_SQ_query_param_uint64 (&rvalue),
1036 GNUNET_SQ_query_param_uint32 (&repl),
1037 GNUNET_SQ_query_param_end };
1038 struct GNUNET_SQ_QueryParam params_upd_repl[] =
1039 { GNUNET_SQ_query_param_uint64 (&rc.uid), GNUNET_SQ_query_param_end };
1040
1041 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
1042 "datastore-sqlite",
1043 "Getting random block based on replication order.\n");
1044 if (SQLITE_ROW != sqlite3_step (plugin->maxRepl))
1045 {
1046 GNUNET_SQ_reset (plugin->dbh, plugin->maxRepl);
1047 /* DB empty */
1048 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
1049 return;
1050 }
1051 repl = sqlite3_column_int (plugin->maxRepl, 0);
1052 GNUNET_SQ_reset (plugin->dbh, plugin->maxRepl);
1053 rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, UINT64_MAX);
1054 if (GNUNET_OK != GNUNET_SQ_bind (plugin->selRepl, params_sel_repl))
1055 {
1056 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
1057 return;
1058 }
1059 rc.have_uid = GNUNET_SYSERR;
1060 rc.proc = proc;
1061 rc.proc_cls = proc_cls;
1062 execute_get (plugin, plugin->selRepl, &repl_proc, &rc);
1063 if (GNUNET_YES == rc.have_uid)
1064 {
1065 if (GNUNET_OK != GNUNET_SQ_bind (plugin->updRepl, params_upd_repl))
1066 {
1067 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
1068 return;
1069 }
1070 if (SQLITE_DONE != sqlite3_step (plugin->updRepl))
1071 LOG_SQLITE (plugin,
1072 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
1073 "sqlite3_step");
1074 GNUNET_SQ_reset (plugin->dbh, plugin->updRepl);
1075 }
1076 if (GNUNET_SYSERR == rc.have_uid)
1077 {
1078 /* proc was not called at all so far, do it now. */
1079 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
1080 }
1081}
1082
1083
1084/**
1085 * Get a random item that has expired or has low priority.
1086 * Call @a proc with all values ZERO or NULL if the datastore is empty.
1087 *
1088 * @param cls closure
1089 * @param proc function to call the value (once only).
1090 * @param proc_cls closure for @a proc
1091 */
1092static void
1093sqlite_plugin_get_expiration (void *cls,
1094 PluginDatumProcessor proc,
1095 void *proc_cls)
1096{
1097 struct Plugin *plugin = cls;
1098 sqlite3_stmt *stmt;
1099 struct GNUNET_TIME_Absolute now = { 0 };
1100 struct GNUNET_SQ_QueryParam params[] = { GNUNET_SQ_query_param_absolute_time (
1101 &now),
1102 GNUNET_SQ_query_param_end };
1103
1104 GNUNET_log_from (
1105 GNUNET_ERROR_TYPE_DEBUG,
1106 "sqlite",
1107 "Getting random block based on expiration and priority order.\n");
1108 now = GNUNET_TIME_absolute_get ();
1109 stmt = plugin->selExpi;
1110 if (GNUNET_OK != GNUNET_SQ_bind (stmt, params))
1111 {
1112 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
1113 return;
1114 }
1115 execute_get (plugin, stmt, proc, proc_cls);
1116}
1117
1118
1119/**
1120 * Get all of the keys in the datastore.
1121 *
1122 * @param cls closure
1123 * @param proc function to call on each key
1124 * @param proc_cls closure for @a proc
1125 */
1126static void
1127sqlite_plugin_get_keys (void *cls, PluginKeyProcessor proc, void *proc_cls)
1128{
1129 struct Plugin *plugin = cls;
1130 struct GNUNET_HashCode key;
1131 struct GNUNET_SQ_ResultSpec results[] =
1132 { GNUNET_SQ_result_spec_auto_from_type (&key), GNUNET_SQ_result_spec_end };
1133 sqlite3_stmt *stmt;
1134 int ret;
1135
1136 GNUNET_assert (NULL != proc);
1137 if (SQLITE_OK != sq_prepare (plugin->dbh, "SELECT hash FROM gn091", &stmt))
1138 {
1139 LOG_SQLITE (plugin,
1140 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
1141 "sqlite_prepare");
1142 proc (proc_cls, NULL, 0);
1143 return;
1144 }
1145 while (SQLITE_ROW == (ret = sqlite3_step (stmt)))
1146 {
1147 if (GNUNET_OK == GNUNET_SQ_extract_result (stmt, results))
1148 proc (proc_cls, &key, 1);
1149 else
1150 GNUNET_break (0);
1151 }
1152 if (SQLITE_DONE != ret)
1153 LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "sqlite_step");
1154 sqlite3_finalize (stmt);
1155 proc (proc_cls, NULL, 0);
1156}
1157
1158
1159/**
1160 * Drop database.
1161 *
1162 * @param cls our plugin context
1163 */
1164static void
1165sqlite_plugin_drop (void *cls)
1166{
1167 struct Plugin *plugin = cls;
1168
1169 plugin->drop_on_shutdown = GNUNET_YES;
1170}
1171
1172
1173/**
1174 * Remove a particular key in the datastore.
1175 *
1176 * @param cls closure
1177 * @param key key for the content
1178 * @param size number of bytes in data
1179 * @param data content stored
1180 * @param cont continuation called with success or failure status
1181 * @param cont_cls continuation closure for @a cont
1182 */
1183static void
1184sqlite_plugin_remove_key (void *cls,
1185 const struct GNUNET_HashCode *key,
1186 uint32_t size,
1187 const void *data,
1188 PluginRemoveCont cont,
1189 void *cont_cls)
1190{
1191 struct Plugin *plugin = cls;
1192 struct GNUNET_SQ_QueryParam params[] =
1193 { GNUNET_SQ_query_param_auto_from_type (key),
1194 GNUNET_SQ_query_param_fixed_size (data, size),
1195 GNUNET_SQ_query_param_end };
1196
1197 if (GNUNET_OK != GNUNET_SQ_bind (plugin->remove, params))
1198 {
1199 cont (cont_cls, key, size, GNUNET_SYSERR, "bind failed");
1200 return;
1201 }
1202 if (SQLITE_DONE != sqlite3_step (plugin->remove))
1203 {
1204 LOG_SQLITE (plugin,
1205 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
1206 "sqlite3_step");
1207 GNUNET_SQ_reset (plugin->dbh, plugin->remove);
1208 cont (cont_cls, key, size, GNUNET_SYSERR, "sqlite3_step failed");
1209 return;
1210 }
1211 int changes = sqlite3_changes (plugin->dbh);
1212 GNUNET_SQ_reset (plugin->dbh, plugin->remove);
1213 if (0 == changes)
1214 {
1215 cont (cont_cls, key, size, GNUNET_NO, NULL);
1216 return;
1217 }
1218 if (NULL != plugin->env->duc)
1219 plugin->env->duc (plugin->env->cls,
1220 -(size + GNUNET_DATASTORE_ENTRY_OVERHEAD));
1221 cont (cont_cls, key, size, GNUNET_OK, NULL);
1222}
1223
1224
1225/**
1226 * Get an estimate of how much space the database is
1227 * currently using.
1228 *
1229 * @param cls the `struct Plugin`
1230 * @return the size of the database on disk (estimate)
1231 */
1232static void
1233sqlite_plugin_estimate_size (void *cls, unsigned long long *estimate)
1234{
1235 struct Plugin *plugin = cls;
1236 sqlite3_stmt *stmt;
1237 uint64_t pages;
1238 uint64_t page_size;
1239
1240#if ENULL_DEFINED
1241 char *e;
1242#endif
1243
1244 if (NULL == estimate)
1245 return;
1246 if (SQLITE_VERSION_NUMBER < 3006000)
1247 {
1248 GNUNET_log_from (
1249 GNUNET_ERROR_TYPE_WARNING,
1250 "datastore-sqlite",
1251 _ ("sqlite version to old to determine size, assuming zero\n"));
1252 *estimate = 0;
1253 return;
1254 }
1255 CHECK (SQLITE_OK == sqlite3_exec (plugin->dbh, "VACUUM", NULL, NULL, ENULL));
1256 CHECK (SQLITE_OK == sqlite3_exec (plugin->dbh,
1257 "PRAGMA auto_vacuum=INCREMENTAL",
1258 NULL,
1259 NULL,
1260 ENULL));
1261 if (SQLITE_OK != sq_prepare (plugin->dbh, "PRAGMA page_count", &stmt))
1262 {
1263 GNUNET_log_from (
1264 GNUNET_ERROR_TYPE_WARNING,
1265 "datastore-sqlite",
1266 _("error preparing statement\n"));
1267 return;
1268 }
1269 if (SQLITE_ROW == sqlite3_step (stmt))
1270 pages = sqlite3_column_int64 (stmt, 0);
1271 else
1272 pages = 0;
1273 sqlite3_finalize (stmt);
1274 if (SQLITE_OK != sq_prepare (plugin->dbh, "PRAGMA page_size", &stmt))
1275 {
1276 GNUNET_log_from (
1277 GNUNET_ERROR_TYPE_WARNING,
1278 "datastore-sqlite",
1279 _("error preparing statement\n"));
1280 return;
1281 }
1282 if (SQLITE_ROW != sqlite3_step (stmt))
1283 {
1284 GNUNET_log_from (
1285 GNUNET_ERROR_TYPE_WARNING,
1286 "datastore-sqlite",
1287 _("error stepping\n"));
1288 return;
1289 }
1290 page_size = sqlite3_column_int64 (stmt, 0);
1291 sqlite3_finalize (stmt);
1292 GNUNET_log (
1293 GNUNET_ERROR_TYPE_INFO,
1294 _ (
1295 "Using sqlite page utilization to estimate payload (%llu pages of size %llu bytes)\n"),
1296 (unsigned long long) pages,
1297 (unsigned long long) page_size);
1298 *estimate = pages * page_size;
1299}
1300
1301
1302/**
1303 * Entry point for the plugin.
1304 *
1305 * @param cls the `struct GNUNET_DATASTORE_PluginEnvironment *`
1306 * @return NULL on error, othrewise the plugin context
1307 */
1308void *
1309libgnunet_plugin_datastore_sqlite_init (void *cls)
1310{
1311 static struct Plugin plugin;
1312 struct GNUNET_DATASTORE_PluginEnvironment *env = cls;
1313 struct GNUNET_DATASTORE_PluginFunctions *api;
1314
1315 if (NULL != plugin.env)
1316 return NULL; /* can only initialize once! */
1317 memset (&plugin, 0, sizeof(struct Plugin));
1318 plugin.env = env;
1319 if (GNUNET_OK != database_setup (env->cfg, &plugin))
1320 {
1321 database_shutdown (&plugin);
1322 return NULL;
1323 }
1324 api = GNUNET_new (struct GNUNET_DATASTORE_PluginFunctions);
1325 api->cls = &plugin;
1326 api->estimate_size = &sqlite_plugin_estimate_size;
1327 api->put = &sqlite_plugin_put;
1328 api->get_key = &sqlite_plugin_get_key;
1329 api->get_replication = &sqlite_plugin_get_replication;
1330 api->get_expiration = &sqlite_plugin_get_expiration;
1331 api->get_zero_anonymity = &sqlite_plugin_get_zero_anonymity;
1332 api->get_keys = &sqlite_plugin_get_keys;
1333 api->drop = &sqlite_plugin_drop;
1334 api->remove_key = &sqlite_plugin_remove_key;
1335 GNUNET_log_from (GNUNET_ERROR_TYPE_INFO,
1336 "sqlite",
1337 _ ("Sqlite database running\n"));
1338 return api;
1339}
1340
1341
1342/**
1343 * Exit point from the plugin.
1344 *
1345 * @param cls the plugin context (as returned by "init")
1346 * @return always NULL
1347 */
1348void *
1349libgnunet_plugin_datastore_sqlite_done (void *cls)
1350{
1351 char *fn;
1352 struct GNUNET_DATASTORE_PluginFunctions *api = cls;
1353 struct Plugin *plugin = api->cls;
1354
1355 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
1356 "sqlite",
1357 "sqlite plugin is done\n");
1358 fn = NULL;
1359 if (plugin->drop_on_shutdown)
1360 fn = GNUNET_strdup (plugin->fn);
1361 database_shutdown (plugin);
1362 plugin->env = NULL;
1363 GNUNET_free (api);
1364 if (NULL != fn)
1365 {
1366 if (0 != unlink (fn))
1367 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", fn);
1368 GNUNET_free (fn);
1369 }
1370 return NULL;
1371}
1372
1373
1374/* end of plugin_datastore_sqlite.c */
diff --git a/src/datastore/plugin_datastore_template.c b/src/datastore/plugin_datastore_template.c
deleted file mode 100644
index 2b455f8cb..000000000
--- a/src/datastore/plugin_datastore_template.c
+++ /dev/null
@@ -1,274 +0,0 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2009, 2011 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file datastore/plugin_datastore_template.c
23 * @brief template-based datastore backend
24 * @author Christian Grothoff
25 */
26
27#include "platform.h"
28#include "gnunet_datastore_plugin.h"
29
30
31/**
32 * Context for all functions in this plugin.
33 */
34struct Plugin
35{
36 /**
37 * Our execution environment.
38 */
39 struct GNUNET_DATASTORE_PluginEnvironment *env;
40};
41
42
43/**
44 * Get an estimate of how much space the database is
45 * currently using.
46 *
47 * @param cls our "struct Plugin*"
48 * @return number of bytes used on disk
49 */
50static void
51template_plugin_estimate_size (void *cls, unsigned long long *estimate)
52{
53 if (NULL == estimate)
54 return;
55 GNUNET_break (0);
56 *estimate = 0;
57}
58
59
60/**
61 * Store an item in the datastore.
62 *
63 * @param cls closure
64 * @param key key for the item
65 * @param absent true if the key was not found in the bloom filter
66 * @param size number of bytes in data
67 * @param data content stored
68 * @param type type of the content
69 * @param priority priority of the content
70 * @param anonymity anonymity-level for the content
71 * @param replication replication-level for the content
72 * @param expiration expiration time for the content
73 * @param cont continuation called with success or failure status
74 * @param cont_cls continuation closure
75 */
76static void
77template_plugin_put (void *cls,
78 const struct GNUNET_HashCode *key,
79 bool absent,
80 uint32_t size,
81 const void *data,
82 enum GNUNET_BLOCK_Type type,
83 uint32_t priority,
84 uint32_t anonymity,
85 uint32_t replication,
86 struct GNUNET_TIME_Absolute expiration,
87 PluginPutCont cont,
88 void *cont_cls)
89{
90 GNUNET_break (0);
91 cont (cont_cls, key, size, GNUNET_SYSERR, "not implemented");
92}
93
94
95/**
96 * Get one of the results for a particular key in the datastore.
97 *
98 * @param cls closure
99 * @param next_uid return the result with lowest uid >= next_uid
100 * @param random if true, return a random result instead of using next_uid
101 * @param key maybe NULL (to match all entries)
102 * @param type entries of which type are relevant?
103 * Use 0 for any type.
104 * @param proc function to call on each matching value;
105 * will be called with NULL if nothing matches
106 * @param proc_cls closure for proc
107 */
108static void
109template_plugin_get_key (void *cls,
110 uint64_t next_uid,
111 bool random,
112 const struct GNUNET_HashCode *key,
113 enum GNUNET_BLOCK_Type type,
114 PluginDatumProcessor proc,
115 void *proc_cls)
116{
117 GNUNET_break (0);
118}
119
120
121/**
122 * Get a random item for replication. Returns a single, not expired,
123 * random item from those with the highest replication counters. The
124 * item's replication counter is decremented by one IF it was positive
125 * before. Call 'proc' with all values ZERO or NULL if the datastore
126 * is empty.
127 *
128 * @param cls closure
129 * @param proc function to call the value (once only).
130 * @param proc_cls closure for proc
131 */
132static void
133template_plugin_get_replication (void *cls, PluginDatumProcessor proc,
134 void *proc_cls)
135{
136 GNUNET_break (0);
137}
138
139
140/**
141 * Get a random item for expiration. Call 'proc' with all values ZERO
142 * or NULL if the datastore is empty.
143 *
144 * @param cls closure
145 * @param proc function to call the value (once only).
146 * @param proc_cls closure for proc
147 */
148static void
149template_plugin_get_expiration (void *cls, PluginDatumProcessor proc,
150 void *proc_cls)
151{
152 GNUNET_break (0);
153}
154
155
156/**
157 * Call the given processor on an item with zero anonymity.
158 *
159 * @param cls our "struct Plugin*"
160 * @param next_uid return the result with lowest uid >= next_uid
161 * @param type entries of which type should be considered?
162 * Must not be zero (ANY).
163 * @param proc function to call on the matching value;
164 * will be called with NULL if no value matches
165 * @param proc_cls closure for proc
166 */
167static void
168template_plugin_get_zero_anonymity (void *cls, uint64_t next_uid,
169 enum GNUNET_BLOCK_Type type,
170 PluginDatumProcessor proc, void *proc_cls)
171{
172 GNUNET_break (0);
173}
174
175
176/**
177 * Drop database.
178 */
179static void
180template_plugin_drop (void *cls)
181{
182 GNUNET_break (0);
183}
184
185
186/**
187 * Get all of the keys in the datastore.
188 *
189 * @param cls closure
190 * @param proc function to call on each key
191 * @param proc_cls closure for proc
192 */
193static void
194template_get_keys (void *cls,
195 PluginKeyProcessor proc,
196 void *proc_cls)
197{
198 proc (proc_cls, NULL, 0);
199}
200
201
202/**
203 * Remove a particular key in the datastore.
204 *
205 * @param cls closure
206 * @param key key for the content
207 * @param size number of bytes in data
208 * @param data content stored
209 * @param cont continuation called with success or failure status
210 * @param cont_cls continuation closure for @a cont
211 */
212static void
213template_plugin_remove_key (void *cls,
214 const struct GNUNET_HashCode *key,
215 uint32_t size,
216 const void *data,
217 PluginRemoveCont cont,
218 void *cont_cls)
219{
220 GNUNET_break (0);
221 cont (cont_cls, key, size, GNUNET_SYSERR, "not implemented");
222}
223
224
225/**
226 * Entry point for the plugin.
227 *
228 * @param cls the "struct GNUNET_DATASTORE_PluginEnvironment*"
229 * @return our "struct Plugin*"
230 */
231void *
232libgnunet_plugin_datastore_template_init (void *cls)
233{
234 struct GNUNET_DATASTORE_PluginEnvironment *env = cls;
235 struct GNUNET_DATASTORE_PluginFunctions *api;
236 struct Plugin *plugin;
237
238 plugin = GNUNET_new (struct Plugin);
239 plugin->env = env;
240 api = GNUNET_new (struct GNUNET_DATASTORE_PluginFunctions);
241 api->cls = plugin;
242 api->estimate_size = &template_plugin_estimate_size;
243 api->put = &template_plugin_put;
244 api->get_key = &template_plugin_get_key;
245 api->get_replication = &template_plugin_get_replication;
246 api->get_expiration = &template_plugin_get_expiration;
247 api->get_zero_anonymity = &template_plugin_get_zero_anonymity;
248 api->drop = &template_plugin_drop;
249 api->get_keys = &template_get_keys;
250 api->remove_key = &template_plugin_remove_key;
251 GNUNET_log_from (GNUNET_ERROR_TYPE_INFO, "template",
252 _ ("Template database running\n"));
253 return api;
254}
255
256
257/**
258 * Exit point from the plugin.
259 * @param cls our "struct Plugin*"
260 * @return always NULL
261 */
262void *
263libgnunet_plugin_datastore_template_done (void *cls)
264{
265 struct GNUNET_DATASTORE_PluginFunctions *api = cls;
266 struct Plugin *plugin = api->cls;
267
268 GNUNET_free (plugin);
269 GNUNET_free (api);
270 return NULL;
271}
272
273
274/* end of plugin_datastore_template.c */
diff --git a/src/datastore/selectrandom.sql b/src/datastore/selectrandom.sql
deleted file mode 100644
index 82830a13a..000000000
--- a/src/datastore/selectrandom.sql
+++ /dev/null
@@ -1,9 +0,0 @@
1select *
2from (select random() as v from (values(1))) t1,
3 (select max(repl) as m from data) t2,
4 (select * from data
5 where repl=t2.m and
6 rnd>=t.v
7 order by rnd
8 limit 1)
9
diff --git a/src/datastore/test_datastore_api.c b/src/datastore/test_datastore_api.c
deleted file mode 100644
index e72a6acd3..000000000
--- a/src/datastore/test_datastore_api.c
+++ /dev/null
@@ -1,732 +0,0 @@
1/*
2 This file is part of GNUnet.
3 Copyright (C) 2004, 2005, 2006, 2007, 2009, 2015 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20/*
21 * @file datastore/test_datastore_api.c
22 * @brief Test for the basic datastore API.
23 * @author Christian Grothoff
24 *
25 * TODO:
26 * - test reservation failure
27 */
28
29#include "platform.h"
30#include "gnunet_util_lib.h"
31#include "gnunet_protocols.h"
32#include "gnunet_datastore_service.h"
33#include "gnunet_datastore_plugin.h"
34#include "gnunet_testing_lib.h"
35
36
37/**
38 * How long until we give up on transmitting the message?
39 */
40#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 15)
41
42#define ITERATIONS 256
43
44/**
45 * Handle to the datastore.
46 */
47static struct GNUNET_DATASTORE_Handle *datastore;
48
49static struct GNUNET_TIME_Absolute now;
50
51/**
52 * Value we return from #main().
53 */
54static int ok;
55
56/**
57 * Name of plugin under test.
58 */
59static const char *plugin_name;
60
61
62static size_t
63get_size (int i)
64{
65 return 8 * i;
66}
67
68
69static const void *
70get_data (int i)
71{
72 static char buf[60000];
73
74 memset (buf, i, 8 * i);
75 return buf;
76}
77
78
79static int
80get_type (int i)
81{
82 return i + 1;
83}
84
85
86static int
87get_priority (int i)
88{
89 return i + 1;
90}
91
92
93static int
94get_anonymity (int i)
95{
96 return i;
97}
98
99
100static struct GNUNET_TIME_Absolute
101get_expiration (int i)
102{
103 struct GNUNET_TIME_Absolute av;
104
105 av.abs_value_us = now.abs_value_us + 20000000000LL - i * 1000 * 1000LL;
106 return av;
107}
108
109
110/**
111 * Which phase of the process are we in?
112 */
113enum RunPhase
114{
115 /**
116 * We are done (shutting down normally).
117 */
118 RP_DONE = 0,
119
120 /**
121 * We are adding new entries to the datastore.
122 */
123 RP_PUT = 1,
124 RP_GET = 2,
125 RP_DEL = 3,
126 RP_DO_DEL = 4,
127 RP_DELVALIDATE = 5,
128 RP_RESERVE = 6,
129 RP_PUT_MULTIPLE = 7,
130 RP_PUT_MULTIPLE_NEXT = 8,
131 RP_GET_MULTIPLE = 9,
132 RP_GET_MULTIPLE_NEXT = 10,
133
134 /**
135 * Execution failed with some kind of error.
136 */
137 RP_ERROR
138};
139
140
141/**
142 * Closure we give to all of the functions executing the
143 * benchmark. Could right now be global, but this allows
144 * us to theoretically run multiple clients "in parallel".
145 */
146struct CpsRunContext
147{
148 /**
149 * Execution phase we are in.
150 */
151 enum RunPhase phase;
152
153 struct GNUNET_HashCode key;
154 int i;
155 int rid;
156 void *data;
157 size_t size;
158
159 uint64_t first_uid;
160};
161
162
163/**
164 * Main state machine. Executes the next step of the test
165 * depending on the current state.
166 *
167 * @param cls the `struct CpsRunContext`
168 */
169static void
170run_continuation (void *cls);
171
172
173/**
174 * Continuation called to notify client about result of an
175 * operation. Checks for errors, updates our iteration counters and
176 * continues execution with #run_continuation().
177 *
178 * @param cls the `struct CpsRunContext`
179 * @param success #GNUNET_SYSERR on failure
180 * @param min_expiration minimum expiration time required for content to be stored
181 * by the datacache at this time, zero for unknown
182 * @param msg NULL on success, otherwise an error message
183 */
184static void
185check_success (void *cls,
186 int success,
187 struct GNUNET_TIME_Absolute min_expiration,
188 const char *msg)
189{
190 struct CpsRunContext *crc = cls;
191
192 if (GNUNET_OK != success)
193 {
194 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
195 "Operation %d/%d not successful: `%s'\n",
196 crc->phase,
197 crc->i,
198 msg);
199 crc->phase = RP_ERROR;
200 }
201 GNUNET_free (crc->data);
202 crc->data = NULL;
203 GNUNET_SCHEDULER_add_now (&run_continuation, crc);
204}
205
206
207static void
208get_reserved (void *cls,
209 int success,
210 struct GNUNET_TIME_Absolute min_expiration,
211 const char *msg)
212{
213 struct CpsRunContext *crc = cls;
214
215 if (0 >= success)
216 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
217 "Error obtaining reservation: `%s'\n",
218 msg);
219 GNUNET_assert (0 < success);
220 crc->rid = success;
221 GNUNET_SCHEDULER_add_now (&run_continuation,
222 crc);
223}
224
225
226static void
227check_value (void *cls,
228 const struct GNUNET_HashCode *key,
229 size_t size,
230 const void *data,
231 enum GNUNET_BLOCK_Type type,
232 uint32_t priority,
233 uint32_t anonymity,
234 uint32_t replication,
235 struct GNUNET_TIME_Absolute expiration,
236 uint64_t uid)
237{
238 struct CpsRunContext *crc = cls;
239 int i;
240
241 i = crc->i;
242 if (NULL == key)
243 {
244 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
245 "Value check failed (got NULL key) in %d/%d\n",
246 crc->phase,
247 crc->i);
248 crc->phase = RP_ERROR;
249 GNUNET_SCHEDULER_add_now (&run_continuation,
250 crc);
251 return;
252 }
253#if 0
254 fprintf (stderr,
255 "Check value got `%s' of size %u, type %d, expire %s\n",
256 GNUNET_h2s (key), (unsigned int) size, type,
257 GNUNET_STRINGS_absolute_time_to_string (expiration));
258 fprintf (stderr,
259 "Check value iteration %d wants size %u, type %d, expire %s\n", i,
260 (unsigned int) get_size (i), get_type (i),
261 GNUNET_STRINGS_absolute_time_to_string (get_expiration (i)));
262#endif
263 GNUNET_assert (size == get_size (i));
264 GNUNET_assert (0 == memcmp (data, get_data (i), size));
265 GNUNET_assert (type == get_type (i));
266 GNUNET_assert (priority == get_priority (i));
267 GNUNET_assert (anonymity == get_anonymity (i));
268 GNUNET_assert (expiration.abs_value_us == get_expiration (i).abs_value_us);
269 if (crc->i == 0)
270 {
271 crc->phase = RP_DEL;
272 crc->i = ITERATIONS;
273 }
274 GNUNET_SCHEDULER_add_now (&run_continuation,
275 crc);
276}
277
278
279static void
280delete_value (void *cls,
281 const struct GNUNET_HashCode *key,
282 size_t size,
283 const void *data,
284 enum GNUNET_BLOCK_Type type,
285 uint32_t priority,
286 uint32_t anonymity,
287 uint32_t replication,
288 struct GNUNET_TIME_Absolute expiration,
289 uint64_t uid)
290{
291 struct CpsRunContext *crc = cls;
292
293 GNUNET_assert (NULL == crc->data);
294 GNUNET_assert (NULL != key);
295 crc->size = size;
296 crc->key = *key;
297 crc->data = GNUNET_malloc (size);
298 GNUNET_memcpy (crc->data, data, size);
299 crc->phase = RP_DO_DEL;
300 GNUNET_SCHEDULER_add_now (&run_continuation,
301 crc);
302}
303
304
305static void
306check_nothing (void *cls,
307 const struct GNUNET_HashCode *key,
308 size_t size,
309 const void *data,
310 enum GNUNET_BLOCK_Type type,
311 uint32_t priority,
312 uint32_t anonymity,
313 uint32_t replication,
314 struct GNUNET_TIME_Absolute expiration,
315 uint64_t uid)
316{
317 struct CpsRunContext *crc = cls;
318
319 GNUNET_assert (key == NULL);
320 if (crc->i == 0)
321 crc->phase = RP_RESERVE;
322 GNUNET_SCHEDULER_add_now (&run_continuation,
323 crc);
324}
325
326
327static void
328check_multiple (void *cls,
329 const struct GNUNET_HashCode *key,
330 size_t size,
331 const void *data,
332 enum GNUNET_BLOCK_Type type,
333 uint32_t priority,
334 uint32_t anonymity,
335 uint32_t replication,
336 struct GNUNET_TIME_Absolute expiration,
337 uint64_t uid)
338{
339 struct CpsRunContext *crc = cls;
340
341 GNUNET_assert (key != NULL);
342 switch (crc->phase)
343 {
344 case RP_GET_MULTIPLE:
345 crc->phase = RP_GET_MULTIPLE_NEXT;
346 crc->first_uid = uid;
347 break;
348
349 case RP_GET_MULTIPLE_NEXT:
350 GNUNET_assert (uid != crc->first_uid);
351 crc->phase = RP_DONE;
352 break;
353
354 default:
355 GNUNET_break (0);
356 crc->phase = RP_ERROR;
357 break;
358 }
359 GNUNET_SCHEDULER_add_now (&run_continuation, crc);
360}
361
362
363/**
364 * Main state machine. Executes the next step of the test
365 * depending on the current state.
366 *
367 * @param cls the `struct CpsRunContext`
368 */
369static void
370run_continuation (void *cls)
371{
372 struct CpsRunContext *crc = cls;
373
374 ok = (int) crc->phase;
375 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
376 "Test in phase %u\n",
377 crc->phase);
378 switch (crc->phase)
379 {
380 case RP_PUT:
381 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
382 "Executing PUT number %u\n",
383 crc->i);
384 GNUNET_CRYPTO_hash (&crc->i, sizeof(int), &crc->key);
385 GNUNET_DATASTORE_put (datastore, 0, &crc->key, get_size (crc->i),
386 get_data (crc->i), get_type (crc->i),
387 get_priority (crc->i), get_anonymity (crc->i), 0,
388 get_expiration (crc->i), 1, 1,
389 &check_success, crc);
390 crc->i++;
391 if (crc->i == ITERATIONS)
392 crc->phase = RP_GET;
393 break;
394
395 case RP_GET:
396 crc->i--;
397 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
398 "Executing GET number %u\n",
399 crc->i);
400 GNUNET_CRYPTO_hash (&crc->i,
401 sizeof(int),
402 &crc->key);
403 GNUNET_DATASTORE_get_key (datastore,
404 0,
405 false,
406 &crc->key,
407 get_type (crc->i),
408 1,
409 1,
410 &check_value,
411 crc);
412 break;
413
414 case RP_DEL:
415 crc->i--;
416 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
417 "Executing DEL number %u\n",
418 crc->i);
419 crc->data = NULL;
420 GNUNET_CRYPTO_hash (&crc->i, sizeof(int), &crc->key);
421 GNUNET_assert (NULL !=
422 GNUNET_DATASTORE_get_key (datastore,
423 0,
424 false,
425 &crc->key,
426 get_type (crc->i),
427 1,
428 1,
429 &delete_value,
430 crc));
431 break;
432
433 case RP_DO_DEL:
434 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
435 "Executing DO_DEL number %u\n",
436 crc->i);
437 if (crc->i == 0)
438 {
439 crc->i = ITERATIONS;
440 crc->phase = RP_DELVALIDATE;
441 }
442 else
443 {
444 crc->phase = RP_DEL;
445 }
446 GNUNET_assert (NULL !=
447 GNUNET_DATASTORE_remove (datastore, &crc->key, crc->size,
448 crc->data, 1, 1,
449 &check_success, crc));
450 break;
451
452 case RP_DELVALIDATE:
453 crc->i--;
454 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
455 "Executing DELVALIDATE number %u\n",
456 crc->i);
457 GNUNET_CRYPTO_hash (&crc->i, sizeof(int), &crc->key);
458 GNUNET_assert (NULL !=
459 GNUNET_DATASTORE_get_key (datastore,
460 0,
461 false,
462 &crc->key,
463 get_type (crc->i),
464 1,
465 1,
466 &check_nothing,
467 crc));
468 break;
469
470 case RP_RESERVE:
471 crc->phase = RP_PUT_MULTIPLE;
472 GNUNET_DATASTORE_reserve (datastore, 128 * 1024, 2,
473 &get_reserved, crc);
474 break;
475
476 case RP_PUT_MULTIPLE:
477 crc->phase = RP_PUT_MULTIPLE_NEXT;
478 GNUNET_DATASTORE_put (datastore, crc->rid, &crc->key, get_size (42),
479 get_data (42), get_type (42), get_priority (42),
480 get_anonymity (42), 0, get_expiration (42), 1, 1,
481 &check_success, crc);
482 break;
483
484 case RP_PUT_MULTIPLE_NEXT:
485 crc->phase = RP_GET_MULTIPLE;
486 GNUNET_DATASTORE_put (datastore, crc->rid,
487 &crc->key,
488 get_size (43),
489 get_data (43),
490 get_type (42),
491 get_priority (43),
492 get_anonymity (43),
493 0,
494 get_expiration (43),
495 1, 1,
496 &check_success, crc);
497 break;
498
499 case RP_GET_MULTIPLE:
500 GNUNET_assert (NULL !=
501 GNUNET_DATASTORE_get_key (datastore,
502 0,
503 false,
504 &crc->key,
505 get_type (42),
506 1,
507 1,
508 &check_multiple,
509 crc));
510 break;
511
512 case RP_GET_MULTIPLE_NEXT:
513 GNUNET_assert (NULL !=
514 GNUNET_DATASTORE_get_key (datastore,
515 crc->first_uid + 1,
516 false,
517 &crc->key,
518 get_type (42),
519 1,
520 1,
521 &check_multiple,
522 crc));
523 break;
524
525 case RP_DONE:
526 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
527 "Finished, disconnecting\n");
528 GNUNET_DATASTORE_disconnect (datastore,
529 GNUNET_YES);
530 GNUNET_free (crc);
531 ok = 0;
532 break;
533
534 case RP_ERROR:
535 GNUNET_DATASTORE_disconnect (datastore,
536 GNUNET_YES);
537 GNUNET_free (crc);
538 ok = 43;
539 break;
540 }
541}
542
543
544/**
545 * Function called with the result of the initial PUT operation. If
546 * the PUT succeeded, we start the actual benchmark loop, otherwise we
547 * bail out with an error.
548 *
549 *
550 * @param cls closure
551 * @param success #GNUNET_SYSERR on failure
552 * @param min_expiration minimum expiration time required for content to be stored
553 * by the datacache at this time, zero for unknown
554 * @param msg NULL on success, otherwise an error message
555 */
556static void
557run_tests (void *cls,
558 int32_t success,
559 struct GNUNET_TIME_Absolute min_expiration,
560 const char *msg)
561{
562 struct CpsRunContext *crc = cls;
563
564 switch (success)
565 {
566 case GNUNET_YES:
567 GNUNET_SCHEDULER_add_now (&run_continuation,
568 crc);
569 return;
570
571 case GNUNET_NO:
572 fprintf (stderr,
573 "%s", "Test 'put' operation failed, key already exists (!?)\n");
574 GNUNET_DATASTORE_disconnect (datastore,
575 GNUNET_YES);
576 GNUNET_free (crc);
577 return;
578
579 case GNUNET_SYSERR:
580 fprintf (stderr,
581 "Test 'put' operation failed with error `%s' database likely not setup, skipping test.\n",
582 msg);
583 GNUNET_DATASTORE_disconnect (datastore,
584 GNUNET_YES);
585 GNUNET_free (crc);
586 return;
587
588 default:
589 GNUNET_assert (0);
590 }
591}
592
593
594/**
595 * Beginning of the actual execution of the benchmark.
596 * Performs a first test operation (PUT) to verify that
597 * the plugin works at all.
598 *
599 * @param cls NULL
600 * @param cfg configuration to use
601 * @param peer peer handle (unused)
602 */
603static void
604run (void *cls,
605 const struct GNUNET_CONFIGURATION_Handle *cfg,
606 struct GNUNET_TESTING_Peer *peer)
607{
608 struct CpsRunContext *crc;
609 static struct GNUNET_HashCode zkey;
610
611 crc = GNUNET_new (struct CpsRunContext);
612 crc->phase = RP_PUT;
613 now = GNUNET_TIME_absolute_get ();
614 datastore = GNUNET_DATASTORE_connect (cfg);
615 if (NULL ==
616 GNUNET_DATASTORE_put (datastore,
617 0,
618 &zkey,
619 4,
620 "TEST",
621 GNUNET_BLOCK_TYPE_TEST,
622 0, 0, 0,
623 GNUNET_TIME_relative_to_absolute
624 (GNUNET_TIME_UNIT_SECONDS),
625 0, 1,
626 &run_tests, crc))
627 {
628 fprintf (stderr,
629 "%s",
630 "Test 'put' operation failed.\n");
631 ok = 1;
632 GNUNET_free (crc);
633 }
634}
635
636
637/**
638 * Function invoked to notify service of disk utilization
639 * changes.
640 *
641 * @param cls closure
642 * @param delta change in disk utilization,
643 * 0 for "reset to empty"
644 */
645static void
646duc_dummy (void *cls,
647 int delta)
648{
649 /* intentionally empty */
650}
651
652
653/**
654 * check if plugin is actually working
655 */
656static int
657test_plugin (const char *cfg_name)
658{
659 char libname[128];
660 struct GNUNET_CONFIGURATION_Handle *cfg;
661 struct GNUNET_DATASTORE_PluginFunctions *api;
662 struct GNUNET_DATASTORE_PluginEnvironment env;
663
664 cfg = GNUNET_CONFIGURATION_create ();
665 if (GNUNET_OK !=
666 GNUNET_CONFIGURATION_load (cfg,
667 cfg_name))
668 {
669 GNUNET_CONFIGURATION_destroy (cfg);
670 fprintf (stderr,
671 "Failed to load configuration %s\n",
672 cfg_name);
673 return 1;
674 }
675 memset (&env, 0, sizeof(env));
676 env.cfg = cfg;
677 env.duc = &duc_dummy;
678 GNUNET_snprintf (libname,
679 sizeof(libname),
680 "libgnunet_plugin_datastore_%s",
681 plugin_name);
682 api = GNUNET_PLUGIN_load (libname, &env);
683 if (NULL == api)
684 {
685 GNUNET_CONFIGURATION_destroy (cfg);
686 fprintf (stderr,
687 "Failed to load plugin `%s'\n",
688 libname);
689 return 77;
690 }
691 GNUNET_PLUGIN_unload (libname, api);
692 GNUNET_CONFIGURATION_destroy (cfg);
693 return 0;
694}
695
696
697/**
698 * Entry point into the test. Determines which configuration / plugin
699 * we are running with based on the name of the binary and starts
700 * the peer.
701 *
702 * @param argc should be 1
703 * @param argv used to determine plugin / configuration name.
704 * @return 0 on success
705 */
706int
707main (int argc,
708 char *argv[])
709{
710 char cfg_name[PATH_MAX];
711 int ret;
712
713 plugin_name = GNUNET_TESTING_get_testname_from_underscore (argv[0]);
714 GNUNET_snprintf (cfg_name,
715 sizeof(cfg_name),
716 "test_datastore_api_data_%s.conf",
717 plugin_name);
718 ret = test_plugin (cfg_name);
719 if (0 != ret)
720 return ret;
721 /* run actual test */
722 if (0 !=
723 GNUNET_TESTING_peer_run ("test-gnunet-datastore",
724 cfg_name,
725 &run,
726 NULL))
727 return 1;
728 return ok;
729}
730
731
732/* end of test_datastore_api.c */
diff --git a/src/datastore/test_datastore_api_data_heap.conf b/src/datastore/test_datastore_api_data_heap.conf
deleted file mode 100644
index 6f11fb3f7..000000000
--- a/src/datastore/test_datastore_api_data_heap.conf
+++ /dev/null
@@ -1,19 +0,0 @@
1@INLINE@ test_defaults.conf
2[PATHS]
3GNUNET_TEST_HOME = $GNUNET_TMP/test-gnunet-datastore-heap/
4
5[TESTING]
6WEAKRANDOM = YES
7
8[arm]
9PORT = 42466
10
11[statistics]
12PORT = 22667
13
14[resolver]
15PORT = 42464
16
17[datastore]
18QUOTA = 10 MB
19DATABASE = heap
diff --git a/src/datastore/test_datastore_api_data_mysql.conf b/src/datastore/test_datastore_api_data_mysql.conf
deleted file mode 100644
index c0052c5ea..000000000
--- a/src/datastore/test_datastore_api_data_mysql.conf
+++ /dev/null
@@ -1,10 +0,0 @@
1@INLINE@ test_defaults.conf
2[PATHS]
3GNUNET_TEST_HOME = $GNUNET_TMP/test-gnunet-datastore-mysql/
4
5[datastore]
6QUOTA = 10 MB
7DATABASE = mysql
8
9[datastore-mysql]
10DATABASE = gnunetcheck
diff --git a/src/datastore/test_datastore_api_data_postgres.conf b/src/datastore/test_datastore_api_data_postgres.conf
deleted file mode 100644
index 65fe11806..000000000
--- a/src/datastore/test_datastore_api_data_postgres.conf
+++ /dev/null
@@ -1,10 +0,0 @@
1@INLINE@ test_defaults.conf
2[PATHS]
3GNUNET_TEST_HOME = $GNUNET_TMP/test-gnunet-datastore-postgres/
4
5[datastore]
6QUOTA = 10 MB
7DATABASE = postgres
8
9[datastore-postgres]
10CONFIG = dbname=gnunetcheck
diff --git a/src/datastore/test_datastore_api_data_sqlite.conf b/src/datastore/test_datastore_api_data_sqlite.conf
deleted file mode 100644
index ecdd0c6ee..000000000
--- a/src/datastore/test_datastore_api_data_sqlite.conf
+++ /dev/null
@@ -1,7 +0,0 @@
1@INLINE@ test_defaults.conf
2[PATHS]
3GNUNET_TEST_HOME = $GNUNET_TMP/test-gnunet-datastore-sqlite/
4
5[datastore]
6QUOTA = 10 MB
7DATABASE = sqlite
diff --git a/src/datastore/test_datastore_api_management.c b/src/datastore/test_datastore_api_management.c
deleted file mode 100644
index 175765267..000000000
--- a/src/datastore/test_datastore_api_management.c
+++ /dev/null
@@ -1,408 +0,0 @@
1/*
2 This file is part of GNUnet.
3 Copyright (C) 2004, 2005, 2006, 2007, 2009, 2011 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20/*
21 * @file datastore/test_datastore_api_management.c
22 * @brief Test for the space management functions of the datastore implementation.
23 * @author Christian Grothoff
24 */
25#include "platform.h"
26#include "gnunet_util_lib.h"
27#include "gnunet_protocols.h"
28#include "gnunet_datastore_service.h"
29#include "gnunet_datastore_plugin.h"
30#include "gnunet_testing_lib.h"
31
32
33/**
34 * How long until we give up on transmitting the message?
35 */
36#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 60)
37
38/**
39 * Number of iterations to run; must be large enough
40 * so that the quota will be exceeded!
41 */
42#define ITERATIONS 5000
43
44enum RunPhase
45{
46 RP_PUT,
47 RP_GET,
48 RP_DONE,
49 RP_GET_FAIL
50};
51
52
53struct CpsRunContext
54{
55 struct GNUNET_HashCode key;
56 int i;
57 int found;
58 const struct GNUNET_CONFIGURATION_Handle *cfg;
59 void *data;
60 enum RunPhase phase;
61};
62
63
64static struct GNUNET_DATASTORE_Handle *datastore;
65
66static struct GNUNET_TIME_Absolute now;
67
68static int ok;
69
70static const char *plugin_name;
71
72
73static size_t
74get_size (int i)
75{
76 return 8 + 8 * (i % 256);
77}
78
79
80static const void *
81get_data (int i)
82{
83 static char buf[60000];
84
85 memset (buf, i, 8 + 8 * (i % 256));
86 return buf;
87}
88
89
90static int
91get_type (int i)
92{
93 return 1;
94}
95
96
97static int
98get_priority (int i)
99{
100 return i + 1;
101}
102
103
104static int
105get_anonymity (int i)
106{
107 return i;
108}
109
110
111static struct GNUNET_TIME_Absolute
112get_expiration (int i)
113{
114 struct GNUNET_TIME_Absolute av;
115
116 av.abs_value_us = now.abs_value_us + i * 1000 * 1000LL;
117 return av;
118}
119
120
121static void
122run_continuation (void *cls);
123
124
125static void
126check_success (void *cls, int success, struct GNUNET_TIME_Absolute
127 min_expiration, const char *msg)
128{
129 struct CpsRunContext *crc = cls;
130
131 if (GNUNET_OK != success)
132 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "%s\n", msg);
133 GNUNET_assert (GNUNET_OK == success);
134 GNUNET_free (crc->data);
135 crc->data = NULL;
136 GNUNET_SCHEDULER_add_now (&run_continuation, crc);
137}
138
139
140static void
141check_value (void *cls,
142 const struct GNUNET_HashCode *key,
143 size_t size,
144 const void *data,
145 enum GNUNET_BLOCK_Type type,
146 uint32_t priority,
147 uint32_t anonymity,
148 uint32_t replication,
149 struct GNUNET_TIME_Absolute expiration,
150 uint64_t uid)
151{
152 struct CpsRunContext *crc = cls;
153 int i;
154
155 if (NULL == key)
156 {
157 crc->phase = RP_GET_FAIL;
158 GNUNET_SCHEDULER_add_now (&run_continuation, crc);
159 return;
160 }
161 i = crc->i;
162 GNUNET_assert (size == get_size (i));
163 GNUNET_assert (0 == memcmp (data, get_data (i), size));
164 GNUNET_assert (type == get_type (i));
165 GNUNET_assert (priority == get_priority (i));
166 GNUNET_assert (anonymity == get_anonymity (i));
167 GNUNET_assert (expiration.abs_value_us == get_expiration (i).abs_value_us);
168 crc->i--;
169 if (crc->i == 0)
170 crc->phase = RP_DONE;
171 GNUNET_SCHEDULER_add_now (&run_continuation, crc);
172}
173
174
175static void
176check_nothing (void *cls,
177 const struct GNUNET_HashCode *key,
178 size_t size,
179 const void *data,
180 enum GNUNET_BLOCK_Type type,
181 uint32_t priority,
182 uint32_t anonymity,
183 uint32_t replication,
184 struct GNUNET_TIME_Absolute expiration,
185 uint64_t uid)
186{
187 struct CpsRunContext *crc = cls;
188
189 GNUNET_assert (key == NULL);
190 if (0 == --crc->i)
191 crc->phase = RP_DONE;
192 GNUNET_SCHEDULER_add_now (&run_continuation, crc);
193}
194
195
196static void
197run_continuation (void *cls)
198{
199 struct CpsRunContext *crc = cls;
200
201 ok = (int) crc->phase;
202 switch (crc->phase)
203 {
204 case RP_PUT:
205 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Executing `%s' number %u\n", "PUT",
206 crc->i);
207 GNUNET_CRYPTO_hash (&crc->i, sizeof(int), &crc->key);
208 GNUNET_DATASTORE_put (datastore,
209 0,
210 &crc->key,
211 get_size (crc->i),
212 get_data (crc->i),
213 get_type (crc->i),
214 get_priority (crc->i),
215 get_anonymity (crc->i),
216 0,
217 get_expiration (crc->i),
218 1,
219 1,
220 &check_success, crc);
221 crc->i++;
222 if (crc->i == ITERATIONS)
223 {
224 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
225 "Sleeping to give datastore time to clean up\n");
226 sleep (1);
227 crc->phase = RP_GET;
228 crc->i--;
229 }
230 break;
231
232 case RP_GET:
233 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Executing `%s' number %u\n", "GET",
234 crc->i);
235 GNUNET_CRYPTO_hash (&crc->i, sizeof(int), &crc->key);
236 GNUNET_DATASTORE_get_key (datastore,
237 0,
238 false,
239 &crc->key,
240 get_type (crc->i),
241 1,
242 1,
243 &check_value,
244 crc);
245 break;
246
247 case RP_GET_FAIL:
248 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Executing `%s' number %u\n", "GET(f)",
249 crc->i);
250 GNUNET_CRYPTO_hash (&crc->i, sizeof(int), &crc->key);
251 GNUNET_DATASTORE_get_key (datastore,
252 0,
253 false,
254 &crc->key,
255 get_type (crc->i),
256 1,
257 1,
258 &check_nothing,
259 crc);
260 break;
261
262 case RP_DONE:
263 GNUNET_assert (0 == crc->i);
264 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Finished, disconnecting\n");
265 GNUNET_DATASTORE_disconnect (datastore, GNUNET_YES);
266 GNUNET_free (crc);
267 ok = 0;
268 }
269}
270
271
272static void
273run_tests (void *cls, int success, struct GNUNET_TIME_Absolute min_expiration,
274 const char *msg)
275{
276 struct CpsRunContext *crc = cls;
277
278 if (success != GNUNET_YES)
279 {
280 fprintf (stderr,
281 "Test 'put' operation failed with error `%s' database likely not setup, skipping test.\n",
282 msg);
283 GNUNET_DATASTORE_disconnect (datastore, GNUNET_YES);
284 GNUNET_free (crc);
285 return;
286 }
287 GNUNET_SCHEDULER_add_now (&run_continuation, crc);
288}
289
290
291static void
292run (void *cls,
293 const struct GNUNET_CONFIGURATION_Handle *cfg,
294 struct GNUNET_TESTING_Peer *peer)
295{
296 struct CpsRunContext *crc;
297 static struct GNUNET_HashCode zkey;
298
299 crc = GNUNET_new (struct CpsRunContext);
300 crc->cfg = cfg;
301 crc->phase = RP_PUT;
302 now = GNUNET_TIME_absolute_get ();
303 datastore = GNUNET_DATASTORE_connect (cfg);
304 if (NULL ==
305 GNUNET_DATASTORE_put (datastore,
306 0,
307 &zkey,
308 4,
309 "TEST",
310 GNUNET_BLOCK_TYPE_TEST,
311 0, 0, 0,
312 GNUNET_TIME_relative_to_absolute (
313 GNUNET_TIME_UNIT_SECONDS),
314 0,
315 1,
316 &run_tests,
317 crc))
318 {
319 fprintf (stderr, "%s", "Test 'put' operation failed.\n");
320 GNUNET_free (crc);
321 ok = 1;
322 }
323}
324
325
326/**
327 * Function called when disk utilization changes, does nothing.
328 *
329 * @param cls closure
330 * @param delta change in utilization
331 */
332static void
333ignore_payload_cb (void *cls,
334 int delta)
335{
336 /* do nothing */
337}
338
339
340/**
341 * check if plugin is actually working
342 */
343static int
344test_plugin (const char *cfg_name)
345{
346 char libname[PATH_MAX];
347 struct GNUNET_CONFIGURATION_Handle *cfg;
348 struct GNUNET_DATASTORE_PluginFunctions *api;
349 struct GNUNET_DATASTORE_PluginEnvironment env;
350
351 cfg = GNUNET_CONFIGURATION_create ();
352 if (GNUNET_OK !=
353 GNUNET_CONFIGURATION_load (cfg,
354 cfg_name))
355 {
356 GNUNET_CONFIGURATION_destroy (cfg);
357 fprintf (stderr,
358 "Failed to load configuration %s\n",
359 cfg_name);
360 return 1;
361 }
362 memset (&env, 0, sizeof(env));
363 env.cfg = cfg;
364 env.duc = &ignore_payload_cb;
365 GNUNET_snprintf (libname,
366 sizeof(libname),
367 "libgnunet_plugin_datastore_%s",
368 plugin_name);
369 api = GNUNET_PLUGIN_load (libname, &env);
370 if (NULL == api)
371 {
372 GNUNET_CONFIGURATION_destroy (cfg);
373 fprintf (stderr,
374 "Failed to load plugin `%s'\n",
375 libname);
376 return 77;
377 }
378 GNUNET_PLUGIN_unload (libname, api);
379 GNUNET_CONFIGURATION_destroy (cfg);
380 return 0;
381}
382
383
384int
385main (int argc, char *argv[])
386{
387 char cfg_name[PATH_MAX];
388 int ret;
389
390 plugin_name = GNUNET_TESTING_get_testname_from_underscore (argv[0]);
391 GNUNET_snprintf (cfg_name,
392 sizeof(cfg_name),
393 "test_datastore_api_data_%s.conf",
394 plugin_name);
395 ret = test_plugin (cfg_name);
396 if (0 != ret)
397 return ret;
398 if (0 !=
399 GNUNET_TESTING_peer_run ("test-gnunet-datastore-management",
400 cfg_name,
401 &run,
402 NULL))
403 return 1;
404 return ok;
405}
406
407
408/* end of test_datastore_api_management.c */
diff --git a/src/datastore/test_defaults.conf b/src/datastore/test_defaults.conf
deleted file mode 100644
index 1f971de8f..000000000
--- a/src/datastore/test_defaults.conf
+++ /dev/null
@@ -1,10 +0,0 @@
1@inline@ ../../contrib/conf/gnunet/no_autostart_above_core.conf
2@inline@ ../../contrib/conf/gnunet/no_forcestart.conf
3
4[datastore]
5PORT = 22654
6QUOTA = 1 MB
7START_ON_DEMAND = YES
8
9[nse]
10WORKBITS = 1
diff --git a/src/datastore/test_plugin_datastore.c b/src/datastore/test_plugin_datastore.c
deleted file mode 100644
index 9fe2462e7..000000000
--- a/src/datastore/test_plugin_datastore.c
+++ /dev/null
@@ -1,478 +0,0 @@
1/*
2 This file is part of GNUnet.
3 Copyright (C) 2011 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20/*
21 * @file test_plugin_datastore.c
22 * @brief Test database plugin directly, calling each API function once
23 * @author Christian Grothoff
24 */
25
26#include "platform.h"
27#include "gnunet_util_lib.h"
28#include "gnunet_protocols.h"
29#include "gnunet_datastore_plugin.h"
30#include "gnunet_testing_lib.h"
31
32/**
33 * Number of put operations to perform.
34 */
35#define PUT_10 10
36
37static unsigned long long stored_bytes;
38
39static unsigned long long stored_entries;
40
41static unsigned long long stored_ops;
42
43static const char *plugin_name;
44
45static int ok;
46
47enum RunPhase
48{
49 RP_ERROR = 0,
50 RP_PUT,
51 RP_GET,
52 RP_ITER_ZERO,
53 RP_REPL_GET,
54 RP_EXPI_GET,
55 RP_REMOVE,
56 RP_DROP
57};
58
59
60struct CpsRunContext
61{
62 const struct GNUNET_CONFIGURATION_Handle *cfg;
63 struct GNUNET_DATASTORE_PluginFunctions *api;
64 enum RunPhase phase;
65 unsigned int cnt;
66 unsigned int i;
67};
68
69
70/**
71 * Function called by plugins to notify us about a
72 * change in their disk utilization.
73 *
74 * @param cls closure (NULL)
75 * @param delta change in disk utilization,
76 * 0 for "reset to empty"
77 */
78static void
79disk_utilization_change_cb (void *cls, int delta)
80{
81 /* do nothing */
82}
83
84
85static void
86test (void *cls);
87
88
89/**
90 * Put continuation.
91 *
92 * @param cls closure
93 * @param key key for the item stored
94 * @param size size of the item stored
95 * @param status #GNUNET_OK or #GNUNET_SYSERROR
96 * @param msg error message on error
97 */
98static void
99put_continuation (void *cls,
100 const struct GNUNET_HashCode *key,
101 uint32_t size,
102 int status,
103 const char *msg)
104{
105 struct CpsRunContext *crc = cls;
106 static unsigned long long os;
107 unsigned long long cs;
108
109 if (GNUNET_OK != status)
110 {
111 fprintf (stderr,
112 "ERROR: `%s'\n",
113 msg);
114 }
115 else
116 {
117 crc->api->estimate_size (crc->api->cls,
118 &cs);
119 GNUNET_assert (os <= cs);
120 os = cs;
121 stored_bytes += size;
122 stored_ops++;
123 stored_entries++;
124 }
125 GNUNET_SCHEDULER_add_now (&test, crc);
126}
127
128
129static void
130gen_key (int i, struct GNUNET_HashCode *key)
131{
132 memset (key, 0, sizeof(struct GNUNET_HashCode));
133 key->bits[0] = (unsigned int) i;
134 GNUNET_CRYPTO_hash (key, sizeof(struct GNUNET_HashCode), key);
135}
136
137
138static void
139do_put (struct CpsRunContext *crc)
140{
141 char value[65536];
142 size_t size;
143 struct GNUNET_HashCode key;
144 unsigned int prio;
145 static int i;
146
147 if (PUT_10 == i)
148 {
149 i = 0;
150 crc->phase++;
151 GNUNET_SCHEDULER_add_now (&test, crc);
152 return;
153 }
154 /* most content is 32k */
155 size = 32 * 1024;
156
157 if ((0 != i) && (GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 16) ==
158 0) ) /* but some of it is less! */
159 size = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 32 * 1024);
160 size = size - (size & 7); /* always multiple of 8 */
161
162 /* generate random key */
163 gen_key (i, &key);
164 memset (value, i, size);
165 if (i > 255)
166 memset (value, i - 255, size / 2);
167 value[0] = crc->i;
168 prio = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 100);
169 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
170 "putting type %u, anon %u under key %s\n", i + 1, i,
171 GNUNET_h2s (&key));
172 crc->api->put (crc->api->cls,
173 &key,
174 false /* absent */,
175 size,
176 value, i + 1 /* type */,
177 prio,
178 i /* anonymity */,
179 0 /* replication */,
180 GNUNET_TIME_relative_to_absolute
181 (GNUNET_TIME_relative_multiply
182 (GNUNET_TIME_UNIT_MILLISECONDS,
183 60 * 60 * 60 * 1000
184 + GNUNET_CRYPTO_random_u32
185 (GNUNET_CRYPTO_QUALITY_WEAK, 1000))),
186 put_continuation,
187 crc);
188 i++;
189}
190
191
192static uint64_t guid;
193
194
195static int
196iterate_one_shot (void *cls,
197 const struct GNUNET_HashCode *key,
198 uint32_t size,
199 const void *data,
200 enum GNUNET_BLOCK_Type type,
201 uint32_t priority,
202 uint32_t anonymity,
203 uint32_t replication,
204 struct GNUNET_TIME_Absolute expiration,
205 uint64_t uid)
206{
207 struct CpsRunContext *crc = cls;
208
209 GNUNET_assert (NULL != key);
210 guid = uid;
211 crc->phase++;
212 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
213 "Found result type=%u, priority=%u, size=%u, expire=%s, key %s\n",
214 (unsigned int) type,
215 (unsigned int) priority,
216 (unsigned int) size,
217 GNUNET_STRINGS_absolute_time_to_string (expiration),
218 GNUNET_h2s (key));
219 GNUNET_SCHEDULER_add_now (&test,
220 crc);
221 return GNUNET_OK;
222}
223
224
225static void
226remove_continuation (void *cls,
227 const struct GNUNET_HashCode *key,
228 uint32_t size,
229 int status,
230 const char *msg)
231{
232 struct CpsRunContext *crc = cls;
233
234 GNUNET_assert (NULL != key);
235 GNUNET_assert (32768 == size);
236 GNUNET_assert (GNUNET_OK == status);
237 GNUNET_assert (NULL == msg);
238 crc->phase++;
239 GNUNET_SCHEDULER_add_now (&test,
240 crc);
241}
242
243
244/**
245 * Function called when the service shuts
246 * down. Unloads our datastore plugin.
247 *
248 * @param api api to unload
249 * @param cfg configuration to use
250 */
251static void
252unload_plugin (struct GNUNET_DATASTORE_PluginFunctions *api,
253 const struct GNUNET_CONFIGURATION_Handle *cfg)
254{
255 char *name;
256 char *libname;
257
258 if (GNUNET_OK !=
259 GNUNET_CONFIGURATION_get_value_string (cfg,
260 "DATASTORE",
261 "DATABASE",
262 &name))
263 {
264 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
265 _ ("No `%s' specified for `%s' in configuration!\n"),
266 "DATABASE",
267 "DATASTORE");
268 return;
269 }
270 GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", name);
271 GNUNET_break (NULL == GNUNET_PLUGIN_unload (libname, api));
272 GNUNET_free (libname);
273 GNUNET_free (name);
274}
275
276
277/**
278 * Last task run during shutdown. Disconnects us from
279 * the transport and core.
280 */
281static void
282cleaning_task (void *cls)
283{
284 struct CpsRunContext *crc = cls;
285
286 unload_plugin (crc->api, crc->cfg);
287 GNUNET_free (crc);
288}
289
290
291static void
292test (void *cls)
293{
294 struct CpsRunContext *crc = cls;
295 struct GNUNET_HashCode key;
296
297 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
298 "In phase %d, iteration %u\n", crc->phase, crc->cnt);
299 switch (crc->phase)
300 {
301 case RP_ERROR:
302 ok = 1;
303 GNUNET_break (0);
304 crc->api->drop (crc->api->cls);
305 GNUNET_SCHEDULER_add_now (&cleaning_task, crc);
306 break;
307
308 case RP_PUT:
309 do_put (crc);
310 break;
311
312 case RP_GET:
313 if (crc->cnt == 1)
314 {
315 crc->cnt = 0;
316 crc->phase++;
317 GNUNET_SCHEDULER_add_now (&test, crc);
318 break;
319 }
320 gen_key (5, &key);
321 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
322 "Looking for %s\n",
323 GNUNET_h2s (&key));
324 crc->api->get_key (crc->api->cls,
325 0,
326 false,
327 &key,
328 GNUNET_BLOCK_TYPE_ANY,
329 &iterate_one_shot,
330 crc);
331 break;
332
333 case RP_ITER_ZERO:
334 if (crc->cnt == 1)
335 {
336 crc->cnt = 0;
337 crc->phase++;
338 GNUNET_SCHEDULER_add_now (&test, crc);
339 break;
340 }
341 crc->api->get_zero_anonymity (crc->api->cls, 0, 1, &iterate_one_shot, crc);
342 break;
343
344 case RP_REPL_GET:
345 crc->api->get_replication (crc->api->cls, &iterate_one_shot, crc);
346 break;
347
348 case RP_EXPI_GET:
349 crc->api->get_expiration (crc->api->cls, &iterate_one_shot, crc);
350 break;
351
352 case RP_REMOVE:
353 {
354 struct GNUNET_HashCode key;
355 uint32_t size = 32768;
356 char value[size];
357
358 gen_key (0, &key);
359 memset (value, 0, size);
360 value[0] = crc->i;
361 crc->api->remove_key (crc->api->cls,
362 &key,
363 size,
364 value,
365 &remove_continuation,
366 crc);
367 break;
368 }
369
370 case RP_DROP:
371 crc->api->drop (crc->api->cls);
372 GNUNET_SCHEDULER_add_now (&cleaning_task, crc);
373 break;
374 }
375}
376
377
378/**
379 * Load the datastore plugin.
380 */
381static struct GNUNET_DATASTORE_PluginFunctions *
382load_plugin (const struct GNUNET_CONFIGURATION_Handle *cfg)
383{
384 static struct GNUNET_DATASTORE_PluginEnvironment env;
385 struct GNUNET_DATASTORE_PluginFunctions *ret;
386 char *name;
387 char *libname;
388
389 if (GNUNET_OK !=
390 GNUNET_CONFIGURATION_get_value_string (cfg,
391 "DATASTORE",
392 "DATABASE",
393 &name))
394 {
395 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
396 _ ("No `%s' specified for `%s' in configuration!\n"),
397 "DATABASE",
398 "DATASTORE");
399 return NULL;
400 }
401 env.cfg = cfg;
402 env.duc = &disk_utilization_change_cb;
403 env.cls = NULL;
404 GNUNET_log (GNUNET_ERROR_TYPE_INFO, _ ("Loading `%s' datastore plugin\n"),
405 name);
406 GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", name);
407 if (NULL == (ret = GNUNET_PLUGIN_load (libname, &env)))
408 {
409 fprintf (stderr, "Failed to load plugin `%s'!\n", name);
410 GNUNET_free (libname);
411 GNUNET_free (name);
412 ok = 77; /* mark test as skipped */
413 return NULL;
414 }
415 GNUNET_free (libname);
416 GNUNET_free (name);
417 return ret;
418}
419
420
421static void
422run (void *cls, char *const *args, const char *cfgfile,
423 const struct GNUNET_CONFIGURATION_Handle *c)
424{
425 struct GNUNET_DATASTORE_PluginFunctions *api;
426 struct CpsRunContext *crc;
427
428 api = load_plugin (c);
429 if (api == NULL)
430 {
431 fprintf (stderr,
432 "%s",
433 "Could not initialize plugin, assuming database not configured. Test not run!\n");
434 return;
435 }
436 crc = GNUNET_new (struct CpsRunContext);
437 crc->api = api;
438 crc->cfg = c;
439 crc->phase = RP_PUT;
440 GNUNET_SCHEDULER_add_now (&test, crc);
441}
442
443
444int
445main (int argc, char *argv[])
446{
447 char dir_name[PATH_MAX];
448 char cfg_name[PATH_MAX];
449 char *const xargv[] = {
450 "test-plugin-datastore",
451 "-c",
452 cfg_name,
453 NULL
454 };
455 static struct GNUNET_GETOPT_CommandLineOption options[] = {
456 GNUNET_GETOPT_OPTION_END
457 };
458
459 /* determine name of plugin to use */
460 plugin_name = GNUNET_TESTING_get_testname_from_underscore (argv[0]);
461 GNUNET_snprintf (dir_name, sizeof(dir_name),
462 "/tmp/test-gnunet-datastore-plugin-%s", plugin_name);
463 GNUNET_DISK_directory_remove (dir_name);
464 GNUNET_log_setup ("test-plugin-datastore",
465 "WARNING",
466 NULL);
467 GNUNET_snprintf (cfg_name, sizeof(cfg_name),
468 "test_plugin_datastore_data_%s.conf", plugin_name);
469 GNUNET_PROGRAM_run ((sizeof(xargv) / sizeof(char *)) - 1, xargv,
470 "test-plugin-datastore", "nohelp", options, &run, NULL);
471 if ((0 != ok) && (77 != ok))
472 fprintf (stderr, "Missed some testcases: %u\n", ok);
473 GNUNET_DISK_directory_remove (dir_name);
474 return ok;
475}
476
477
478/* end of test_plugin_datastore.c */
diff --git a/src/datastore/test_plugin_datastore_data_heap.conf b/src/datastore/test_plugin_datastore_data_heap.conf
deleted file mode 100644
index b1ea8ff67..000000000
--- a/src/datastore/test_plugin_datastore_data_heap.conf
+++ /dev/null
@@ -1,6 +0,0 @@
1@INLINE@ test_defaults.conf
2[PATHS]
3GNUNET_TEST_HOME = $GNUNET_TMP/test-gnunet-datastore-plugin-heap/
4
5[datastore]
6DATABASE = heap
diff --git a/src/datastore/test_plugin_datastore_data_mysql.conf b/src/datastore/test_plugin_datastore_data_mysql.conf
deleted file mode 100644
index 07d3ec58e..000000000
--- a/src/datastore/test_plugin_datastore_data_mysql.conf
+++ /dev/null
@@ -1,9 +0,0 @@
1@INLINE@ test_defaults.conf
2[PATHS]
3GNUNET_TEST_HOME = $GNUNET_TMP/test-gnunet-datastore-plugin-mysql/
4
5[datastore]
6DATABASE = mysql
7
8[datastore-mysql]
9DATABASE = gnunetcheck
diff --git a/src/datastore/test_plugin_datastore_data_postgres.conf b/src/datastore/test_plugin_datastore_data_postgres.conf
deleted file mode 100644
index d0e29437f..000000000
--- a/src/datastore/test_plugin_datastore_data_postgres.conf
+++ /dev/null
@@ -1,10 +0,0 @@
1@INLINE@ test_defaults.conf
2[PATHS]
3GNUNET_TEST_HOME = $GNUNET_TMP/test-gnunet-datastore-plugin-postgres/
4
5[datastore]
6DATABASE = postgres
7
8[datastore-postgres]
9CONFIG = dbname=gnunetcheck
10
diff --git a/src/datastore/test_plugin_datastore_data_sqlite.conf b/src/datastore/test_plugin_datastore_data_sqlite.conf
deleted file mode 100644
index ca837c77a..000000000
--- a/src/datastore/test_plugin_datastore_data_sqlite.conf
+++ /dev/null
@@ -1,4 +0,0 @@
1@INLINE@ test_defaults.conf
2[PATHS]
3GNUNET_TEST_HOME = $GNUNET_TMP/test-gnunet-datastore-plugin-sqlite/
4
generated by cgit v1.2.3 (git 2.39.1) at 2024-04-27 20:02:31 +0000