aboutsummaryrefslogtreecommitdiff
path: root/src/service/datastore
diff options
context:
space:
mode:
Diffstat (limited to 'src/service/datastore')
-rw-r--r--src/service/datastore/.gitignore15
-rw-r--r--src/service/datastore/Makefile.am142
-rw-r--r--src/service/datastore/datastore.conf.in33
-rw-r--r--src/service/datastore/datastore.h257
-rw-r--r--src/service/datastore/datastore_api.c1393
-rw-r--r--src/service/datastore/gnunet-service-datastore.c1650
-rw-r--r--src/service/datastore/meson.build132
-rw-r--r--src/service/datastore/perf_datastore_api.c630
-rw-r--r--src/service/datastore/selectrandom.sql9
-rw-r--r--src/service/datastore/test_datastore_api.c732
-rw-r--r--src/service/datastore/test_datastore_api_data_heap.conf19
-rw-r--r--src/service/datastore/test_datastore_api_data_postgres.conf10
-rw-r--r--src/service/datastore/test_datastore_api_data_sqlite.conf7
-rw-r--r--src/service/datastore/test_datastore_api_management.c413
-rw-r--r--src/service/datastore/test_defaults.conf10
15 files changed, 5452 insertions, 0 deletions
diff --git a/src/service/datastore/.gitignore b/src/service/datastore/.gitignore
new file mode 100644
index 000000000..bd5c170ca
--- /dev/null
+++ b/src/service/datastore/.gitignore
@@ -0,0 +1,15 @@
1gnunet-service-datastore
2perf_datastore_api_heap
3perf_plugin_datastore_heap
4test_datastore_api_heap
5test_datastore_api_management_heap
6test_datastore_api_management_mysql
7test_datastore_api_management_postgres
8test_datastore_api_management_sqlite
9test_datastore_api_mysql
10test_datastore_api_postgres
11test_datastore_api_sqlite
12test_plugin_datastore_heap
13test_plugin_datastore_mysql
14test_plugin_datastore_postgres
15test_plugin_datastore_sqlite
diff --git a/src/service/datastore/Makefile.am b/src/service/datastore/Makefile.am
new file mode 100644
index 000000000..22f98482a
--- /dev/null
+++ b/src/service/datastore/Makefile.am
@@ -0,0 +1,142 @@
1# This Makefile.am is in the public domain
2AM_CPPFLAGS = -I$(top_srcdir)/src/include
3
4plugindir = $(libdir)/gnunet
5
6pkgcfgdir= $(pkgdatadir)/config.d/
7
8libexecdir= $(pkglibdir)/libexec/
9
10pkgcfg_DATA = \
11 datastore.conf
12
13if USE_COVERAGE
14 AM_CFLAGS = --coverage -O0
15 XLIBS = -lgcov
16endif
17
18
19lib_LTLIBRARIES = \
20 libgnunetdatastore.la
21
22libgnunetdatastore_la_SOURCES = \
23 datastore_api.c datastore.h
24libgnunetdatastore_la_LIBADD = \
25 $(top_builddir)/src/service/statistics/libgnunetstatistics.la \
26 $(top_builddir)/src/lib/util/libgnunetutil.la \
27 $(GN_LIBINTL)
28libgnunetdatastore_la_LDFLAGS = \
29 $(GN_LIB_LDFLAGS) \
30 -version-info 1:0:0
31
32libexec_PROGRAMS = \
33 gnunet-service-datastore
34
35gnunet_service_datastore_SOURCES = \
36 gnunet-service-datastore.c
37gnunet_service_datastore_LDADD = \
38 $(top_builddir)/src/service/statistics/libgnunetstatistics.la \
39 $(top_builddir)/src/lib/util/libgnunetutil.la \
40 $(GN_LIBINTL)
41
42if HAVE_SQLITE
43if HAVE_BENCHMARKS
44 SQLITE_BENCHMARKS = \
45 perf_datastore_api_sqlite
46endif
47 SQLITE_TESTS = \
48 test_datastore_api_sqlite \
49 test_datastore_api_management_sqlite \
50 $(SQLITE_BENCHMARKS)
51endif
52if HAVE_POSTGRESQL
53if HAVE_BENCHMARKS
54 POSTGRES_BENCHMARKS = \
55 perf_datastore_api_postgres
56endif
57 POSTGRES_TESTS = \
58 test_datastore_api_postgres \
59 test_datastore_api_management_postgres \
60 $(POSTGRES_BENCHMARKS)
61endif
62
63check_PROGRAMS = \
64 test_datastore_api_heap \
65 test_datastore_api_management_heap \
66 perf_datastore_api_heap \
67 $(SQLITE_TESTS) \
68 $(POSTGRES_TESTS)
69
70if ENABLE_TEST_RUN
71AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
72TESTS = $(check_PROGRAMS)
73endif
74
75test_datastore_api_heap_SOURCES = \
76 test_datastore_api.c
77test_datastore_api_heap_LDADD = \
78 $(top_builddir)/src/service/testing/libgnunettesting.la \
79 libgnunetdatastore.la \
80 $(top_builddir)/src/lib/util/libgnunetutil.la
81
82test_datastore_api_management_heap_SOURCES = \
83 test_datastore_api_management.c
84test_datastore_api_management_heap_LDADD = \
85 $(top_builddir)/src/service/testing/libgnunettesting.la \
86 libgnunetdatastore.la \
87 $(top_builddir)/src/lib/util/libgnunetutil.la
88
89perf_datastore_api_heap_SOURCES = \
90 perf_datastore_api.c
91perf_datastore_api_heap_LDADD = \
92 $(top_builddir)/src/service/testing/libgnunettesting.la \
93 libgnunetdatastore.la \
94 $(top_builddir)/src/lib/util/libgnunetutil.la
95
96test_datastore_api_sqlite_SOURCES = \
97 test_datastore_api.c
98test_datastore_api_sqlite_LDADD = \
99 $(top_builddir)/src/service/testing/libgnunettesting.la \
100 libgnunetdatastore.la \
101 $(top_builddir)/src/lib/util/libgnunetutil.la
102
103test_datastore_api_management_sqlite_SOURCES = \
104 test_datastore_api_management.c
105test_datastore_api_management_sqlite_LDADD = \
106 $(top_builddir)/src/service/testing/libgnunettesting.la \
107 libgnunetdatastore.la \
108 $(top_builddir)/src/lib/util/libgnunetutil.la
109
110perf_datastore_api_sqlite_SOURCES = \
111 perf_datastore_api.c
112perf_datastore_api_sqlite_LDADD = \
113 $(top_builddir)/src/service/testing/libgnunettesting.la \
114 libgnunetdatastore.la \
115 $(top_builddir)/src/lib/util/libgnunetutil.la
116
117test_datastore_api_postgres_SOURCES = \
118 test_datastore_api.c
119test_datastore_api_postgres_LDADD = \
120 $(top_builddir)/src/service/testing/libgnunettesting.la \
121 libgnunetdatastore.la \
122 $(top_builddir)/src/lib/util/libgnunetutil.la
123
124test_datastore_api_management_postgres_SOURCES = \
125 test_datastore_api_management.c
126test_datastore_api_management_postgres_LDADD = \
127 $(top_builddir)/src/service/testing/libgnunettesting.la \
128 libgnunetdatastore.la \
129 $(top_builddir)/src/lib/util/libgnunetutil.la
130
131perf_datastore_api_postgres_SOURCES = \
132 perf_datastore_api.c
133perf_datastore_api_postgres_LDADD = \
134 $(top_builddir)/src/service/testing/libgnunettesting.la \
135 libgnunetdatastore.la \
136 $(top_builddir)/src/lib/util/libgnunetutil.la
137
138EXTRA_DIST = \
139 test_defaults.conf \
140 test_datastore_api_data_sqlite.conf \
141 test_datastore_api_data_heap.conf \
142 test_datastore_api_data_postgres.conf
diff --git a/src/service/datastore/datastore.conf.in b/src/service/datastore/datastore.conf.in
new file mode 100644
index 000000000..bcd495c8f
--- /dev/null
+++ b/src/service/datastore/datastore.conf.in
@@ -0,0 +1,33 @@
1[datastore]
2START_ON_DEMAND = @START_ON_DEMAND@
3UNIXPATH = $GNUNET_RUNTIME_DIR/gnunet-service-datastore.sock
4UNIX_MATCH_UID = NO
5UNIX_MATCH_GID = YES
6@UNIXONLY@ PORT = 2093
7HOSTNAME = localhost
8BINARY = gnunet-service-datastore
9ACCEPT_FROM = 127.0.0.1;
10ACCEPT_FROM6 = ::1;
11QUOTA = 5 GB
12BLOOMFILTER = $GNUNET_DATA_HOME/datastore/bloomfilter
13DATABASE = sqlite
14# DISABLE_SOCKET_FORWARDING = NO
15
16[datastore-sqlite]
17FILENAME = $GNUNET_DATA_HOME/datastore/sqlite.db
18
19[datastore-postgres]
20CONFIG = postgres:///gnunet
21SQL_DIR = ${DATADIR}/sql/
22
23[datastore-mysql]
24DATABASE = gnunet
25CONFIG = ~/.my.cnf
26# USER = gnunet
27# PASSWORD =
28# HOST = localhost
29# PORT = 3306
30
31
32[datastore-heap]
33HASHMAPSIZE = 1024
diff --git a/src/service/datastore/datastore.h b/src/service/datastore/datastore.h
new file mode 100644
index 000000000..7af926617
--- /dev/null
+++ b/src/service/datastore/datastore.h
@@ -0,0 +1,257 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2004, 2005, 2006, 2007, 2009 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file datastore/datastore.h
23 * @brief structs for communication between datastore service and API
24 * @author Christian Grothoff
25 */
26
27#ifndef DATASTORE_H
28#define DATASTORE_H
29
30
31#include "gnunet_util_lib.h"
32
33GNUNET_NETWORK_STRUCT_BEGIN
34
35/**
36 * Message from datastore service informing client about
37 * the current size of the datastore.
38 */
39struct ReserveMessage
40{
41 /**
42 * Type is GNUNET_MESSAGE_TYPE_DATASTORE_RESERVE.
43 */
44 struct GNUNET_MessageHeader header;
45
46 /**
47 * Number of items to reserve.
48 */
49 uint32_t entries GNUNET_PACKED;
50
51 /**
52 * Number of bytes to reserve.
53 */
54 uint64_t amount GNUNET_PACKED;
55};
56
57
58/**
59 * Message from datastore service informing client about
60 * the success or failure of a requested operation.
61 * This header is optionally followed by a variable-size,
62 * 0-terminated error message.
63 */
64struct StatusMessage
65{
66 /**
67 * Type is GNUNET_MESSAGE_TYPE_DATASTORE_STATUS.
68 */
69 struct GNUNET_MessageHeader header;
70
71 /**
72 * Status code, -1 for errors.
73 */
74 int32_t status GNUNET_PACKED;
75
76 /**
77 * Minimum expiration time required for content to be stored
78 * by the datacache at this time, zero for unknown or no limit.
79 */
80 struct GNUNET_TIME_AbsoluteNBO min_expiration;
81};
82
83
84/**
85 * Message from datastore client informing service that
86 * the remainder of the reserved bytes can now be released
87 * for other requests.
88 */
89struct ReleaseReserveMessage
90{
91 /**
92 * Type is GNUNET_MESSAGE_TYPE_DATASTORE_RELEASE_RESERVE.
93 */
94 struct GNUNET_MessageHeader header;
95
96 /**
97 * Reservation id.
98 */
99 int32_t rid GNUNET_PACKED;
100};
101
102
103/**
104 * Message to the datastore service asking about specific
105 * content.
106 */
107struct GetKeyMessage
108{
109 /**
110 * Type is #GNUNET_MESSAGE_TYPE_DATASTORE_GET_KEY.
111 */
112 struct GNUNET_MessageHeader header;
113
114 /**
115 * Desired content type. (actually an enum GNUNET_BLOCK_Type)
116 */
117 uint32_t type GNUNET_PACKED;
118
119 /**
120 * UID at which to start the search
121 */
122 uint64_t next_uid GNUNET_PACKED;
123
124 /**
125 * If true return a random result
126 */
127 uint32_t random GNUNET_PACKED;
128
129 /**
130 * Desired key.
131 */
132 struct GNUNET_HashCode key;
133};
134
135
136/**
137 * Message to the datastore service asking about specific
138 * content.
139 */
140struct GetMessage
141{
142 /**
143 * Type is #GNUNET_MESSAGE_TYPE_DATASTORE_GET.
144 */
145 struct GNUNET_MessageHeader header;
146
147 /**
148 * Desired content type. (actually an enum GNUNET_BLOCK_Type)
149 */
150 uint32_t type GNUNET_PACKED;
151
152 /**
153 * UID at which to start the search
154 */
155 uint64_t next_uid GNUNET_PACKED;
156
157 /**
158 * If true return a random result
159 */
160 uint32_t random GNUNET_PACKED;
161};
162
163
164/**
165 * Message to the datastore service asking about zero
166 * anonymity content.
167 */
168struct GetZeroAnonymityMessage
169{
170 /**
171 * Type is GNUNET_MESSAGE_TYPE_DATASTORE_GET_ZERO_ANONYMITY.
172 */
173 struct GNUNET_MessageHeader header;
174
175 /**
176 * Desired content type (actually an enum GNUNET_BLOCK_Type)
177 */
178 uint32_t type GNUNET_PACKED;
179
180 /**
181 * UID at which to start the search
182 */
183 uint64_t next_uid GNUNET_PACKED;
184};
185
186
187/**
188 * Message transmitting content from or to the datastore
189 * service.
190 */
191struct DataMessage
192{
193 /**
194 * Type is either GNUNET_MESSAGE_TYPE_DATASTORE_PUT,
195 * GNUNET_MESSAGE_TYPE_DATASTORE_REMOVE or
196 * GNUNET_MESSAGE_TYPE_DATASTORE_DATA. Depending on the message
197 * type, some fields may simply have values of zero.
198 */
199 struct GNUNET_MessageHeader header;
200
201 /**
202 * Reservation ID to use; use zero for none.
203 */
204 uint32_t rid GNUNET_PACKED;
205
206 /**
207 * Number of bytes in the item (NBO).
208 */
209 uint32_t size GNUNET_PACKED;
210
211 /**
212 * Type of the item (NBO), zero for remove, (actually an enum GNUNET_BLOCK_Type)
213 */
214 uint32_t type GNUNET_PACKED;
215
216 /**
217 * Priority of the item (NBO), zero for remove.
218 */
219 uint32_t priority GNUNET_PACKED;
220
221 /**
222 * Desired anonymity level (NBO), zero for remove.
223 */
224 uint32_t anonymity GNUNET_PACKED;
225
226 /**
227 * Desired replication level.
228 */
229 uint32_t replication GNUNET_PACKED;
230
231 /**
232 * For alignment.
233 */
234 uint32_t reserved GNUNET_PACKED;
235
236 /**
237 * Unique ID for the content (can be used for UPDATE);
238 * can be zero for remove (which indicates that
239 * the datastore should use whatever UID matches
240 * the key and content).
241 */
242 uint64_t uid;
243
244 /**
245 * Expiration time (NBO); zero for remove.
246 */
247 struct GNUNET_TIME_AbsoluteNBO expiration;
248
249 /**
250 * Key under which the item can be found.
251 */
252 struct GNUNET_HashCode key;
253};
254GNUNET_NETWORK_STRUCT_END
255
256
257#endif
diff --git a/src/service/datastore/datastore_api.c b/src/service/datastore/datastore_api.c
new file mode 100644
index 000000000..4d27efb4e
--- /dev/null
+++ b/src/service/datastore/datastore_api.c
@@ -0,0 +1,1393 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2004-2013, 2016 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file datastore/datastore_api.c
23 * @brief Management for the datastore for files stored on a GNUnet node. Implements
24 * a priority queue for requests
25 * @author Christian Grothoff
26 */
27#include "platform.h"
28#include "gnunet_arm_service.h"
29#include "gnunet_constants.h"
30#include "gnunet_datastore_service.h"
31#include "gnunet_statistics_service.h"
32#include "datastore.h"
33
34#define LOG(kind, ...) GNUNET_log_from (kind, "datastore-api", __VA_ARGS__)
35
36#define DELAY_WARN_TIMEOUT GNUNET_TIME_UNIT_MINUTES
37
38/**
39 * Collect an instance number of statistics? May cause excessive IPC.
40 */
41#define INSANE_STATISTICS GNUNET_NO
42
43/**
44 * If a client stopped asking for more results, how many more do
45 * we receive from the DB before killing the connection? Trade-off
46 * between re-doing TCP handshakes and (needlessly) receiving
47 * useless results.
48 */
49#define MAX_EXCESS_RESULTS 8
50
51/**
52 * Context for processing status messages.
53 */
54struct StatusContext
55{
56 /**
57 * Continuation to call with the status.
58 */
59 GNUNET_DATASTORE_ContinuationWithStatus cont;
60
61 /**
62 * Closure for @e cont.
63 */
64 void *cont_cls;
65};
66
67
68/**
69 * Context for processing result messages.
70 */
71struct ResultContext
72{
73 /**
74 * Function to call with the result.
75 */
76 GNUNET_DATASTORE_DatumProcessor proc;
77
78 /**
79 * Closure for @e proc.
80 */
81 void *proc_cls;
82};
83
84
85/**
86 * Context for a queue operation.
87 */
88union QueueContext
89{
90 struct StatusContext sc;
91
92 struct ResultContext rc;
93};
94
95
96/**
97 * Entry in our priority queue.
98 */
99struct GNUNET_DATASTORE_QueueEntry
100{
101 /**
102 * This is a linked list.
103 */
104 struct GNUNET_DATASTORE_QueueEntry *next;
105
106 /**
107 * This is a linked list.
108 */
109 struct GNUNET_DATASTORE_QueueEntry *prev;
110
111 /**
112 * Handle to the master context.
113 */
114 struct GNUNET_DATASTORE_Handle *h;
115
116 /**
117 * Function to call after transmission of the request.
118 */
119 GNUNET_DATASTORE_ContinuationWithStatus cont;
120
121 /**
122 * Closure for @e cont.
123 */
124 void *cont_cls;
125
126 /**
127 * Context for the operation.
128 */
129 union QueueContext qc;
130
131 /**
132 * Envelope of the request to transmit, NULL after
133 * transmission.
134 */
135 struct GNUNET_MQ_Envelope *env;
136
137 /**
138 * Task we run if this entry stalls the queue and we
139 * need to warn the user.
140 */
141 struct GNUNET_SCHEDULER_Task *delay_warn_task;
142
143 /**
144 * Priority in the queue.
145 */
146 unsigned int priority;
147
148 /**
149 * Maximum allowed length of queue (otherwise
150 * this request should be discarded).
151 */
152 unsigned int max_queue;
153
154 /**
155 * Expected response type.
156 */
157 uint16_t response_type;
158};
159
160
161/**
162 * Handle to the datastore service.
163 */
164struct GNUNET_DATASTORE_Handle
165{
166 /**
167 * Our configuration.
168 */
169 const struct GNUNET_CONFIGURATION_Handle *cfg;
170
171 /**
172 * Current connection to the datastore service.
173 */
174 struct GNUNET_MQ_Handle *mq;
175
176 /**
177 * Handle for statistics.
178 */
179 struct GNUNET_STATISTICS_Handle *stats;
180
181 /**
182 * Current head of priority queue.
183 */
184 struct GNUNET_DATASTORE_QueueEntry *queue_head;
185
186 /**
187 * Current tail of priority queue.
188 */
189 struct GNUNET_DATASTORE_QueueEntry *queue_tail;
190
191 /**
192 * Task for trying to reconnect.
193 */
194 struct GNUNET_SCHEDULER_Task *reconnect_task;
195
196 /**
197 * How quickly should we retry? Used for exponential back-off on
198 * connect-errors.
199 */
200 struct GNUNET_TIME_Relative retry_time;
201
202 /**
203 * Number of entries in the queue.
204 */
205 unsigned int queue_size;
206
207 /**
208 * Number of results we're receiving for the current query
209 * after application stopped to care. Used to determine when
210 * to reset the connection.
211 */
212 unsigned int result_count;
213
214 /**
215 * We should ignore the next message(s) from the service.
216 */
217 unsigned int skip_next_messages;
218};
219
220
221/**
222 * Try reconnecting to the datastore service.
223 *
224 * @param cls the `struct GNUNET_DATASTORE_Handle`
225 */
226static void
227try_reconnect (void *cls);
228
229
230/**
231 * Disconnect from the service and then try reconnecting to the datastore service
232 * after some delay.
233 *
234 * @param h handle to datastore to disconnect and reconnect
235 */
236static void
237do_disconnect (struct GNUNET_DATASTORE_Handle *h)
238{
239 if (NULL == h->mq)
240 {
241 GNUNET_break (0);
242 return;
243 }
244 GNUNET_MQ_destroy (h->mq);
245 h->mq = NULL;
246 h->skip_next_messages = 0;
247 h->reconnect_task
248 = GNUNET_SCHEDULER_add_delayed (h->retry_time,
249 &try_reconnect,
250 h);
251}
252
253
254/**
255 * Free a queue entry. Removes the given entry from the
256 * queue and releases associated resources. Does NOT
257 * call the callback.
258 *
259 * @param qe entry to free.
260 */
261static void
262free_queue_entry (struct GNUNET_DATASTORE_QueueEntry *qe)
263{
264 struct GNUNET_DATASTORE_Handle *h = qe->h;
265
266 GNUNET_CONTAINER_DLL_remove (h->queue_head,
267 h->queue_tail,
268 qe);
269 h->queue_size--;
270 if (NULL != qe->env)
271 GNUNET_MQ_discard (qe->env);
272 if (NULL != qe->delay_warn_task)
273 GNUNET_SCHEDULER_cancel (qe->delay_warn_task);
274 GNUNET_free (qe);
275}
276
277
278/**
279 * Task that logs an error after some time.
280 *
281 * @param cls `struct GNUNET_DATASTORE_QueueEntry` about which the error is
282 */
283static void
284delay_warning (void *cls)
285{
286 struct GNUNET_DATASTORE_QueueEntry *qe = cls;
287
288 qe->delay_warn_task = NULL;
289 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
290 "Request %p of type %u at head of datastore queue for more than %s\n",
291 qe,
292 (unsigned int) qe->response_type,
293 GNUNET_STRINGS_relative_time_to_string (DELAY_WARN_TIMEOUT,
294 GNUNET_YES));
295 qe->delay_warn_task = GNUNET_SCHEDULER_add_delayed (DELAY_WARN_TIMEOUT,
296 &delay_warning,
297 qe);
298}
299
300
301/**
302 * Handle error in sending drop request to datastore.
303 *
304 * @param cls closure with the datastore handle
305 * @param error error code
306 */
307static void
308mq_error_handler (void *cls,
309 enum GNUNET_MQ_Error error)
310{
311 struct GNUNET_DATASTORE_Handle *h = cls;
312 struct GNUNET_DATASTORE_QueueEntry *qe;
313
314 LOG (GNUNET_ERROR_TYPE_DEBUG,
315 "MQ error, reconnecting to DATASTORE\n");
316 do_disconnect (h);
317 qe = h->queue_head;
318 if (NULL == qe)
319 return;
320 if (NULL != qe->delay_warn_task)
321 {
322 GNUNET_SCHEDULER_cancel (qe->delay_warn_task);
323 qe->delay_warn_task = NULL;
324 }
325 if (NULL == qe->env)
326 {
327 union QueueContext qc = qe->qc;
328 uint16_t rt = qe->response_type;
329
330 LOG (GNUNET_ERROR_TYPE_DEBUG,
331 "Failed to receive response from database.\n");
332 free_queue_entry (qe);
333 switch (rt)
334 {
335 case GNUNET_MESSAGE_TYPE_DATASTORE_STATUS:
336 if (NULL != qc.sc.cont)
337 qc.sc.cont (qc.sc.cont_cls,
338 GNUNET_SYSERR,
339 GNUNET_TIME_UNIT_ZERO_ABS,
340 _ ("DATASTORE disconnected"));
341 break;
342
343 case GNUNET_MESSAGE_TYPE_DATASTORE_DATA:
344 if (NULL != qc.rc.proc)
345 qc.rc.proc (qc.rc.proc_cls,
346 NULL,
347 0,
348 NULL,
349 0,
350 0,
351 0,
352 0,
353 GNUNET_TIME_UNIT_ZERO_ABS,
354 0);
355 break;
356
357 default:
358 GNUNET_break (0);
359 }
360 }
361}
362
363
364/**
365 * Connect to the datastore service.
366 *
367 * @param cfg configuration to use
368 * @return handle to use to access the service
369 */
370struct GNUNET_DATASTORE_Handle *
371GNUNET_DATASTORE_connect (const struct GNUNET_CONFIGURATION_Handle *cfg)
372{
373 struct GNUNET_DATASTORE_Handle *h;
374
375 LOG (GNUNET_ERROR_TYPE_DEBUG,
376 "Establishing DATASTORE connection!\n");
377 h = GNUNET_new (struct GNUNET_DATASTORE_Handle);
378 h->cfg = cfg;
379 try_reconnect (h);
380 if (NULL == h->mq)
381 {
382 GNUNET_free (h);
383 return NULL;
384 }
385 h->stats = GNUNET_STATISTICS_create ("datastore-api",
386 cfg);
387 return h;
388}
389
390
391/**
392 * Task used by to disconnect from the datastore after
393 * we send the #GNUNET_MESSAGE_TYPE_DATASTORE_DROP message.
394 *
395 * @param cls the datastore handle
396 */
397static void
398disconnect_after_drop (void *cls)
399{
400 struct GNUNET_DATASTORE_Handle *h = cls;
401
402 LOG (GNUNET_ERROR_TYPE_DEBUG,
403 "Drop sent, disconnecting\n");
404 GNUNET_DATASTORE_disconnect (h,
405 GNUNET_NO);
406}
407
408
409/**
410 * Handle error in sending drop request to datastore.
411 *
412 * @param cls closure with the datastore handle
413 * @param error error code
414 */
415static void
416disconnect_on_mq_error (void *cls,
417 enum GNUNET_MQ_Error error)
418{
419 struct GNUNET_DATASTORE_Handle *h = cls;
420
421 LOG (GNUNET_ERROR_TYPE_ERROR,
422 "Failed to ask datastore to drop tables\n");
423 GNUNET_DATASTORE_disconnect (h,
424 GNUNET_NO);
425}
426
427
428/**
429 * Disconnect from the datastore service (and free
430 * associated resources).
431 *
432 * @param h handle to the datastore
433 * @param drop set to #GNUNET_YES to delete all data in datastore (!)
434 */
435void
436GNUNET_DATASTORE_disconnect (struct GNUNET_DATASTORE_Handle *h,
437 int drop)
438{
439 struct GNUNET_DATASTORE_QueueEntry *qe;
440
441 LOG (GNUNET_ERROR_TYPE_DEBUG,
442 "Datastore disconnect\n");
443 if (NULL != h->mq)
444 {
445 GNUNET_MQ_destroy (h->mq);
446 h->mq = NULL;
447 }
448 if (NULL != h->reconnect_task)
449 {
450 GNUNET_SCHEDULER_cancel (h->reconnect_task);
451 h->reconnect_task = NULL;
452 }
453 while (NULL != (qe = h->queue_head))
454 {
455 switch (qe->response_type)
456 {
457 case GNUNET_MESSAGE_TYPE_DATASTORE_STATUS:
458 if (NULL != qe->qc.sc.cont)
459 qe->qc.sc.cont (qe->qc.sc.cont_cls,
460 GNUNET_SYSERR,
461 GNUNET_TIME_UNIT_ZERO_ABS,
462 _ ("Disconnected from DATASTORE"));
463 break;
464
465 case GNUNET_MESSAGE_TYPE_DATASTORE_DATA:
466 if (NULL != qe->qc.rc.proc)
467 qe->qc.rc.proc (qe->qc.rc.proc_cls,
468 NULL,
469 0,
470 NULL,
471 0,
472 0,
473 0,
474 0,
475 GNUNET_TIME_UNIT_ZERO_ABS,
476 0);
477 break;
478
479 default:
480 GNUNET_break (0);
481 }
482 free_queue_entry (qe);
483 }
484 if (GNUNET_YES == drop)
485 {
486 LOG (GNUNET_ERROR_TYPE_DEBUG,
487 "Re-connecting to issue DROP!\n");
488 GNUNET_assert (NULL == h->mq);
489 h->mq = GNUNET_CLIENT_connect (h->cfg,
490 "datastore",
491 NULL,
492 &disconnect_on_mq_error,
493 h);
494 if (NULL != h->mq)
495 {
496 struct GNUNET_MessageHeader *hdr;
497 struct GNUNET_MQ_Envelope *env;
498
499 env = GNUNET_MQ_msg (hdr,
500 GNUNET_MESSAGE_TYPE_DATASTORE_DROP);
501 GNUNET_MQ_notify_sent (env,
502 &disconnect_after_drop,
503 h);
504 GNUNET_MQ_send (h->mq,
505 env);
506 return;
507 }
508 GNUNET_break (0);
509 }
510 GNUNET_STATISTICS_destroy (h->stats,
511 GNUNET_NO);
512 h->stats = NULL;
513 GNUNET_free (h);
514}
515
516
517/**
518 * Create a new entry for our priority queue (and possibly discard other entries if
519 * the queue is getting too long).
520 *
521 * @param h handle to the datastore
522 * @param env envelope with the message to queue
523 * @param queue_priority priority of the entry
524 * @param max_queue_size at what queue size should this request be dropped
525 * (if other requests of higher priority are in the queue)
526 * @param expected_type which type of response do we expect,
527 * #GNUNET_MESSAGE_TYPE_DATASTORE_STATUS or
528 * #GNUNET_MESSAGE_TYPE_DATASTORE_DATA
529 * @param qc client context (NOT a closure for @a response_proc)
530 * @return NULL if the queue is full
531 */
532static struct GNUNET_DATASTORE_QueueEntry *
533make_queue_entry (struct GNUNET_DATASTORE_Handle *h,
534 struct GNUNET_MQ_Envelope *env,
535 unsigned int queue_priority,
536 unsigned int max_queue_size,
537 uint16_t expected_type,
538 const union QueueContext *qc)
539{
540 struct GNUNET_DATASTORE_QueueEntry *qe;
541 struct GNUNET_DATASTORE_QueueEntry *pos;
542 unsigned int c;
543
544 if ((NULL != h->queue_tail) &&
545 (h->queue_tail->priority >= queue_priority))
546 {
547 c = h->queue_size;
548 pos = NULL;
549 }
550 else
551 {
552 c = 0;
553 pos = h->queue_head;
554 }
555 while ((NULL != pos) &&
556 (c < max_queue_size) &&
557 (pos->priority >= queue_priority))
558 {
559 c++;
560 pos = pos->next;
561 }
562 if (c >= max_queue_size)
563 {
564 GNUNET_STATISTICS_update (h->stats,
565 gettext_noop ("# queue overflows"),
566 1,
567 GNUNET_NO);
568 GNUNET_MQ_discard (env);
569 return NULL;
570 }
571 qe = GNUNET_new (struct GNUNET_DATASTORE_QueueEntry);
572 qe->h = h;
573 qe->env = env;
574 qe->response_type = expected_type;
575 qe->qc = *qc;
576 qe->priority = queue_priority;
577 qe->max_queue = max_queue_size;
578 if (NULL == pos)
579 {
580 /* append at the tail */
581 pos = h->queue_tail;
582 }
583 else
584 {
585 pos = pos->prev;
586 /* do not insert at HEAD if HEAD query was already
587 * transmitted and we are still receiving replies! */
588 if ((NULL == pos) &&
589 (NULL == h->queue_head->env))
590 pos = h->queue_head;
591 }
592 c++;
593#if INSANE_STATISTICS
594 GNUNET_STATISTICS_update (h->stats,
595 gettext_noop ("# queue entries created"),
596 1,
597 GNUNET_NO);
598#endif
599 GNUNET_CONTAINER_DLL_insert_after (h->queue_head,
600 h->queue_tail,
601 pos,
602 qe);
603 h->queue_size++;
604 return qe;
605}
606
607
608/**
609 * Process entries in the queue (or do nothing if we are already
610 * doing so).
611 *
612 * @param h handle to the datastore
613 */
614static void
615process_queue (struct GNUNET_DATASTORE_Handle *h)
616{
617 struct GNUNET_DATASTORE_QueueEntry *qe;
618
619 if (NULL == (qe = h->queue_head))
620 {
621 /* no entry in queue */
622 LOG (GNUNET_ERROR_TYPE_DEBUG,
623 "Queue empty\n");
624 return;
625 }
626 if (NULL == qe->env)
627 {
628 /* waiting for replies */
629 LOG (GNUNET_ERROR_TYPE_DEBUG,
630 "Head request already transmitted\n");
631 return;
632 }
633 if (NULL == h->mq)
634 {
635 /* waiting for reconnect */
636 LOG (GNUNET_ERROR_TYPE_DEBUG,
637 "Not connected\n");
638 return;
639 }
640 GNUNET_assert (NULL == qe->delay_warn_task);
641 qe->delay_warn_task = GNUNET_SCHEDULER_add_delayed (DELAY_WARN_TIMEOUT,
642 &delay_warning,
643 qe);
644 GNUNET_MQ_send (h->mq,
645 qe->env);
646 qe->env = NULL;
647}
648
649
650/**
651 * Get the entry at the head of the message queue.
652 *
653 * @param h handle to the datastore
654 * @param response_type the expected response type
655 * @return the queue entry
656 */
657static struct GNUNET_DATASTORE_QueueEntry *
658get_queue_head (struct GNUNET_DATASTORE_Handle *h,
659 uint16_t response_type)
660{
661 struct GNUNET_DATASTORE_QueueEntry *qe;
662
663 if (h->skip_next_messages > 0)
664 {
665 h->skip_next_messages--;
666 process_queue (h);
667 return NULL;
668 }
669 qe = h->queue_head;
670 if (NULL == qe)
671 {
672 GNUNET_break (0);
673 do_disconnect (h);
674 return NULL;
675 }
676 if (NULL != qe->env)
677 {
678 GNUNET_break (0);
679 do_disconnect (h);
680 return NULL;
681 }
682 if (response_type != qe->response_type)
683 {
684 GNUNET_break (0);
685 do_disconnect (h);
686 return NULL;
687 }
688 return qe;
689}
690
691
692/**
693 * Function called to check status message from the service.
694 *
695 * @param cls closure
696 * @param sm status message received
697 * @return #GNUNET_OK if the message is well-formed
698 */
699static int
700check_status (void *cls,
701 const struct StatusMessage *sm)
702{
703 uint16_t msize = ntohs (sm->header.size) - sizeof(*sm);
704 int32_t status = ntohl (sm->status);
705
706 if (msize > 0)
707 {
708 const char *emsg = (const char *) &sm[1];
709
710 if ('\0' != emsg[msize - 1])
711 {
712 GNUNET_break (0);
713 return GNUNET_SYSERR;
714 }
715 }
716 else if (GNUNET_SYSERR == status)
717 {
718 GNUNET_break (0);
719 return GNUNET_SYSERR;
720 }
721 return GNUNET_OK;
722}
723
724
725/**
726 * Function called to handle status message from the service.
727 *
728 * @param cls closure
729 * @param sm status message received
730 */
731static void
732handle_status (void *cls,
733 const struct StatusMessage *sm)
734{
735 struct GNUNET_DATASTORE_Handle *h = cls;
736 struct GNUNET_DATASTORE_QueueEntry *qe;
737 struct StatusContext rc;
738 const char *emsg;
739 int32_t status = ntohl (sm->status);
740
741 qe = get_queue_head (h,
742 GNUNET_MESSAGE_TYPE_DATASTORE_STATUS);
743 if (NULL == qe)
744 return;
745 rc = qe->qc.sc;
746 free_queue_entry (qe);
747 if (ntohs (sm->header.size) > sizeof(struct StatusMessage))
748 emsg = (const char *) &sm[1];
749 else
750 emsg = NULL;
751 LOG (GNUNET_ERROR_TYPE_DEBUG,
752 "Received status %d/%s\n",
753 (int) status,
754 emsg);
755 GNUNET_STATISTICS_update (h->stats,
756 gettext_noop ("# status messages received"),
757 1,
758 GNUNET_NO);
759 h->retry_time = GNUNET_TIME_UNIT_ZERO;
760 process_queue (h);
761 if (NULL != rc.cont)
762 rc.cont (rc.cont_cls,
763 status,
764 GNUNET_TIME_absolute_ntoh (sm->min_expiration),
765 emsg);
766}
767
768
769/**
770 * Check data message we received from the service.
771 *
772 * @param cls closure with the `struct GNUNET_DATASTORE_Handle *`
773 * @param dm message received
774 */
775static int
776check_data (void *cls,
777 const struct DataMessage *dm)
778{
779 uint16_t msize = ntohs (dm->header.size) - sizeof(*dm);
780
781 if (msize != ntohl (dm->size))
782 {
783 GNUNET_break (0);
784 return GNUNET_SYSERR;
785 }
786 return GNUNET_OK;
787}
788
789
790/**
791 * Handle data message we got from the service.
792 *
793 * @param cls closure with the `struct GNUNET_DATASTORE_Handle *`
794 * @param dm message received
795 */
796static void
797handle_data (void *cls,
798 const struct DataMessage *dm)
799{
800 struct GNUNET_DATASTORE_Handle *h = cls;
801 struct GNUNET_DATASTORE_QueueEntry *qe;
802 struct ResultContext rc;
803
804 qe = get_queue_head (h,
805 GNUNET_MESSAGE_TYPE_DATASTORE_DATA);
806 if (NULL == qe)
807 return;
808#if INSANE_STATISTICS
809 GNUNET_STATISTICS_update (h->stats,
810 gettext_noop ("# Results received"),
811 1,
812 GNUNET_NO);
813#endif
814 LOG (GNUNET_ERROR_TYPE_DEBUG,
815 "Received result %llu with type %u and size %u with key %s\n",
816 (unsigned long long) GNUNET_ntohll (dm->uid),
817 ntohl (dm->type),
818 ntohl (dm->size),
819 GNUNET_h2s (&dm->key));
820 rc = qe->qc.rc;
821 free_queue_entry (qe);
822 h->retry_time = GNUNET_TIME_UNIT_ZERO;
823 process_queue (h);
824 if (NULL != rc.proc)
825 rc.proc (rc.proc_cls,
826 &dm->key,
827 ntohl (dm->size),
828 &dm[1],
829 ntohl (dm->type),
830 ntohl (dm->priority),
831 ntohl (dm->anonymity),
832 ntohl (dm->replication),
833 GNUNET_TIME_absolute_ntoh (dm->expiration),
834 GNUNET_ntohll (dm->uid));
835}
836
837
838/**
839 * Type of a function to call when we receive a
840 * #GNUNET_MESSAGE_TYPE_DATASTORE_DATA_END message from the service.
841 *
842 * @param cls closure with the `struct GNUNET_DATASTORE_Handle *`
843 * @param msg message received
844 */
845static void
846handle_data_end (void *cls,
847 const struct GNUNET_MessageHeader *msg)
848{
849 struct GNUNET_DATASTORE_Handle *h = cls;
850 struct GNUNET_DATASTORE_QueueEntry *qe;
851 struct ResultContext rc;
852
853 qe = get_queue_head (h,
854 GNUNET_MESSAGE_TYPE_DATASTORE_DATA);
855 if (NULL == qe)
856 return;
857 rc = qe->qc.rc;
858 free_queue_entry (qe);
859 LOG (GNUNET_ERROR_TYPE_DEBUG,
860 "Received end of result set, new queue size is %u\n",
861 h->queue_size);
862 h->retry_time = GNUNET_TIME_UNIT_ZERO;
863 h->result_count = 0;
864 process_queue (h);
865 /* signal end of iteration */
866 if (NULL != rc.proc)
867 rc.proc (rc.proc_cls,
868 NULL,
869 0,
870 NULL,
871 0,
872 0,
873 0,
874 0,
875 GNUNET_TIME_UNIT_ZERO_ABS,
876 0);
877}
878
879
880/**
881 * Try reconnecting to the datastore service.
882 *
883 * @param cls the `struct GNUNET_DATASTORE_Handle`
884 */
885static void
886try_reconnect (void *cls)
887{
888 struct GNUNET_DATASTORE_Handle *h = cls;
889 struct GNUNET_MQ_MessageHandler handlers[] = {
890 GNUNET_MQ_hd_var_size (status,
891 GNUNET_MESSAGE_TYPE_DATASTORE_STATUS,
892 struct StatusMessage,
893 h),
894 GNUNET_MQ_hd_var_size (data,
895 GNUNET_MESSAGE_TYPE_DATASTORE_DATA,
896 struct DataMessage,
897 h),
898 GNUNET_MQ_hd_fixed_size (data_end,
899 GNUNET_MESSAGE_TYPE_DATASTORE_DATA_END,
900 struct GNUNET_MessageHeader,
901 h),
902 GNUNET_MQ_handler_end ()
903 };
904
905 h->retry_time = GNUNET_TIME_STD_BACKOFF (h->retry_time);
906 h->reconnect_task = NULL;
907 GNUNET_assert (NULL == h->mq);
908 h->mq = GNUNET_CLIENT_connect (h->cfg,
909 "datastore",
910 handlers,
911 &mq_error_handler,
912 h);
913 if (NULL == h->mq)
914 return;
915 GNUNET_STATISTICS_update (h->stats,
916 gettext_noop (
917 "# datastore connections (re)created"),
918 1,
919 GNUNET_NO);
920 LOG (GNUNET_ERROR_TYPE_DEBUG,
921 "Reconnected to DATASTORE\n");
922 process_queue (h);
923}
924
925
926/**
927 * Dummy continuation used to do nothing (but be non-zero).
928 *
929 * @param cls closure
930 * @param result result
931 * @param min_expiration expiration time
932 * @param emsg error message
933 */
934static void
935drop_status_cont (void *cls,
936 int32_t result,
937 struct GNUNET_TIME_Absolute min_expiration,
938 const char *emsg)
939{
940 /* do nothing */
941}
942
943
944struct GNUNET_DATASTORE_QueueEntry *
945GNUNET_DATASTORE_put (struct GNUNET_DATASTORE_Handle *h,
946 uint32_t rid,
947 const struct GNUNET_HashCode *key,
948 size_t size,
949 const void *data,
950 enum GNUNET_BLOCK_Type type,
951 uint32_t priority,
952 uint32_t anonymity,
953 uint32_t replication,
954 struct GNUNET_TIME_Absolute expiration,
955 unsigned int queue_priority,
956 unsigned int max_queue_size,
957 GNUNET_DATASTORE_ContinuationWithStatus cont,
958 void *cont_cls)
959{
960 struct GNUNET_DATASTORE_QueueEntry *qe;
961 struct GNUNET_MQ_Envelope *env;
962 struct DataMessage *dm;
963 union QueueContext qc;
964
965 if (size + sizeof(*dm) >= GNUNET_MAX_MESSAGE_SIZE)
966 {
967 GNUNET_break (0);
968 return NULL;
969 }
970
971 LOG (GNUNET_ERROR_TYPE_DEBUG,
972 "Asked to put %lu bytes of data under key `%s' for %s\n",
973 (unsigned long) size,
974 GNUNET_h2s (key),
975 GNUNET_STRINGS_relative_time_to_string (
976 GNUNET_TIME_absolute_get_remaining (expiration),
977 GNUNET_YES));
978 env = GNUNET_MQ_msg_extra (dm,
979 size,
980 GNUNET_MESSAGE_TYPE_DATASTORE_PUT);
981 dm->rid = htonl (rid);
982 dm->size = htonl ((uint32_t) size);
983 dm->type = htonl (type);
984 dm->priority = htonl (priority);
985 dm->anonymity = htonl (anonymity);
986 dm->replication = htonl (replication);
987 dm->expiration = GNUNET_TIME_absolute_hton (expiration);
988 dm->key = *key;
989 GNUNET_memcpy (&dm[1],
990 data,
991 size);
992 qc.sc.cont = cont;
993 qc.sc.cont_cls = cont_cls;
994 qe = make_queue_entry (h,
995 env,
996 queue_priority,
997 max_queue_size,
998 GNUNET_MESSAGE_TYPE_DATASTORE_STATUS,
999 &qc);
1000 if (NULL == qe)
1001 {
1002 LOG (GNUNET_ERROR_TYPE_DEBUG,
1003 "Could not create queue entry for PUT\n");
1004 return NULL;
1005 }
1006 GNUNET_STATISTICS_update (h->stats,
1007 gettext_noop ("# PUT requests executed"),
1008 1,
1009 GNUNET_NO);
1010 process_queue (h);
1011 return qe;
1012}
1013
1014
1015/**
1016 * Reserve space in the datastore. This function should be used
1017 * to avoid "out of space" failures during a longer sequence of "put"
1018 * operations (for example, when a file is being inserted).
1019 *
1020 * @param h handle to the datastore
1021 * @param amount how much space (in bytes) should be reserved (for content only)
1022 * @param entries how many entries will be created (to calculate per-entry overhead)
1023 * @param cont continuation to call when done; "success" will be set to
1024 * a positive reservation value if space could be reserved.
1025 * @param cont_cls closure for @a cont
1026 * @return NULL if the entry was not queued, otherwise a handle that can be used to
1027 * cancel; note that even if NULL is returned, the callback will be invoked
1028 * (or rather, will already have been invoked)
1029 */
1030struct GNUNET_DATASTORE_QueueEntry *
1031GNUNET_DATASTORE_reserve (struct GNUNET_DATASTORE_Handle *h,
1032 uint64_t amount,
1033 uint32_t entries,
1034 GNUNET_DATASTORE_ContinuationWithStatus cont,
1035 void *cont_cls)
1036{
1037 struct GNUNET_DATASTORE_QueueEntry *qe;
1038 struct GNUNET_MQ_Envelope *env;
1039 struct ReserveMessage *rm;
1040 union QueueContext qc;
1041
1042 if (NULL == cont)
1043 cont = &drop_status_cont;
1044 LOG (GNUNET_ERROR_TYPE_DEBUG,
1045 "Asked to reserve %llu bytes of data and %u entries\n",
1046 (unsigned long long) amount,
1047 (unsigned int) entries);
1048 env = GNUNET_MQ_msg (rm,
1049 GNUNET_MESSAGE_TYPE_DATASTORE_RESERVE);
1050 rm->entries = htonl (entries);
1051 rm->amount = GNUNET_htonll (amount);
1052
1053 qc.sc.cont = cont;
1054 qc.sc.cont_cls = cont_cls;
1055 qe = make_queue_entry (h,
1056 env,
1057 UINT_MAX,
1058 UINT_MAX,
1059 GNUNET_MESSAGE_TYPE_DATASTORE_STATUS,
1060 &qc);
1061 if (NULL == qe)
1062 {
1063 LOG (GNUNET_ERROR_TYPE_DEBUG,
1064 "Could not create queue entry to reserve\n");
1065 return NULL;
1066 }
1067 GNUNET_STATISTICS_update (h->stats,
1068 gettext_noop ("# RESERVE requests executed"),
1069 1,
1070 GNUNET_NO);
1071 process_queue (h);
1072 return qe;
1073}
1074
1075
1076struct GNUNET_DATASTORE_QueueEntry *
1077GNUNET_DATASTORE_release_reserve (struct GNUNET_DATASTORE_Handle *h,
1078 uint32_t rid,
1079 unsigned int queue_priority,
1080 unsigned int max_queue_size,
1081 GNUNET_DATASTORE_ContinuationWithStatus cont,
1082 void *cont_cls)
1083{
1084 struct GNUNET_DATASTORE_QueueEntry *qe;
1085 struct GNUNET_MQ_Envelope *env;
1086 struct ReleaseReserveMessage *rrm;
1087 union QueueContext qc;
1088
1089 if (NULL == cont)
1090 cont = &drop_status_cont;
1091 LOG (GNUNET_ERROR_TYPE_DEBUG,
1092 "Asked to release reserve %d\n",
1093 rid);
1094 env = GNUNET_MQ_msg (rrm,
1095 GNUNET_MESSAGE_TYPE_DATASTORE_RELEASE_RESERVE);
1096 rrm->rid = htonl (rid);
1097 qc.sc.cont = cont;
1098 qc.sc.cont_cls = cont_cls;
1099 qe = make_queue_entry (h,
1100 env,
1101 queue_priority,
1102 max_queue_size,
1103 GNUNET_MESSAGE_TYPE_DATASTORE_STATUS,
1104 &qc);
1105 if (NULL == qe)
1106 {
1107 LOG (GNUNET_ERROR_TYPE_DEBUG,
1108 "Could not create queue entry to release reserve\n");
1109 return NULL;
1110 }
1111 GNUNET_STATISTICS_update (h->stats,
1112 gettext_noop
1113 ("# RELEASE RESERVE requests executed"), 1,
1114 GNUNET_NO);
1115 process_queue (h);
1116 return qe;
1117}
1118
1119
1120struct GNUNET_DATASTORE_QueueEntry *
1121GNUNET_DATASTORE_remove (struct GNUNET_DATASTORE_Handle *h,
1122 const struct GNUNET_HashCode *key,
1123 size_t size,
1124 const void *data,
1125 unsigned int queue_priority,
1126 unsigned int max_queue_size,
1127 GNUNET_DATASTORE_ContinuationWithStatus cont,
1128 void *cont_cls)
1129{
1130 struct GNUNET_DATASTORE_QueueEntry *qe;
1131 struct DataMessage *dm;
1132 struct GNUNET_MQ_Envelope *env;
1133 union QueueContext qc;
1134
1135 if (sizeof(*dm) + size >= GNUNET_MAX_MESSAGE_SIZE)
1136 {
1137 GNUNET_break (0);
1138 return NULL;
1139 }
1140 if (NULL == cont)
1141 cont = &drop_status_cont;
1142 LOG (GNUNET_ERROR_TYPE_DEBUG,
1143 "Asked to remove %lu bytes under key `%s'\n",
1144 (unsigned long) size,
1145 GNUNET_h2s (key));
1146 env = GNUNET_MQ_msg_extra (dm,
1147 size,
1148 GNUNET_MESSAGE_TYPE_DATASTORE_REMOVE);
1149 dm->size = htonl (size);
1150 dm->key = *key;
1151 GNUNET_memcpy (&dm[1],
1152 data,
1153 size);
1154
1155 qc.sc.cont = cont;
1156 qc.sc.cont_cls = cont_cls;
1157
1158 qe = make_queue_entry (h,
1159 env,
1160 queue_priority,
1161 max_queue_size,
1162 GNUNET_MESSAGE_TYPE_DATASTORE_STATUS,
1163 &qc);
1164 if (NULL == qe)
1165 {
1166 LOG (GNUNET_ERROR_TYPE_DEBUG,
1167 "Could not create queue entry for REMOVE\n");
1168 return NULL;
1169 }
1170 GNUNET_STATISTICS_update (h->stats,
1171 gettext_noop ("# REMOVE requests executed"),
1172 1,
1173 GNUNET_NO);
1174 process_queue (h);
1175 return qe;
1176}
1177
1178
1179/**
1180 * Get a random value from the datastore for content replication.
1181 * Returns a single, random value among those with the highest
1182 * replication score, lowering positive replication scores by one for
1183 * the chosen value (if only content with a replication score exists,
1184 * a random value is returned and replication scores are not changed).
1185 *
1186 * @param h handle to the datastore
1187 * @param queue_priority ranking of this request in the priority queue
1188 * @param max_queue_size at what queue size should this request be dropped
1189 * (if other requests of higher priority are in the queue)
1190 * @param proc function to call on a random value; it
1191 * will be called once with a value (if available)
1192 * and always once with a value of NULL.
1193 * @param proc_cls closure for @a proc
1194 * @return NULL if the entry was not queued, otherwise a handle that can be used to
1195 * cancel
1196 */
1197struct GNUNET_DATASTORE_QueueEntry *
1198GNUNET_DATASTORE_get_for_replication (struct GNUNET_DATASTORE_Handle *h,
1199 unsigned int queue_priority,
1200 unsigned int max_queue_size,
1201 GNUNET_DATASTORE_DatumProcessor proc,
1202 void *proc_cls)
1203{
1204 struct GNUNET_DATASTORE_QueueEntry *qe;
1205 struct GNUNET_MQ_Envelope *env;
1206 struct GNUNET_MessageHeader *m;
1207 union QueueContext qc;
1208
1209 GNUNET_assert (NULL != proc);
1210 LOG (GNUNET_ERROR_TYPE_DEBUG,
1211 "Asked to get replication entry\n");
1212 env = GNUNET_MQ_msg (m,
1213 GNUNET_MESSAGE_TYPE_DATASTORE_GET_REPLICATION);
1214 qc.rc.proc = proc;
1215 qc.rc.proc_cls = proc_cls;
1216 qe = make_queue_entry (h,
1217 env,
1218 queue_priority,
1219 max_queue_size,
1220 GNUNET_MESSAGE_TYPE_DATASTORE_DATA,
1221 &qc);
1222 if (NULL == qe)
1223 {
1224 LOG (GNUNET_ERROR_TYPE_DEBUG,
1225 "Could not create queue entry for GET REPLICATION\n");
1226 return NULL;
1227 }
1228 GNUNET_STATISTICS_update (h->stats,
1229 gettext_noop
1230 ("# GET REPLICATION requests executed"), 1,
1231 GNUNET_NO);
1232 process_queue (h);
1233 return qe;
1234}
1235
1236
1237struct GNUNET_DATASTORE_QueueEntry *
1238GNUNET_DATASTORE_get_zero_anonymity (struct GNUNET_DATASTORE_Handle *h,
1239 uint64_t next_uid,
1240 unsigned int queue_priority,
1241 unsigned int max_queue_size,
1242 enum GNUNET_BLOCK_Type type,
1243 GNUNET_DATASTORE_DatumProcessor proc,
1244 void *proc_cls)
1245{
1246 struct GNUNET_DATASTORE_QueueEntry *qe;
1247 struct GNUNET_MQ_Envelope *env;
1248 struct GetZeroAnonymityMessage *m;
1249 union QueueContext qc;
1250
1251 GNUNET_assert (NULL != proc);
1252 GNUNET_assert (type != GNUNET_BLOCK_TYPE_ANY);
1253 LOG (GNUNET_ERROR_TYPE_DEBUG,
1254 "Asked to get a zero-anonymity entry of type %d\n",
1255 type);
1256 env = GNUNET_MQ_msg (m,
1257 GNUNET_MESSAGE_TYPE_DATASTORE_GET_ZERO_ANONYMITY);
1258 m->type = htonl ((uint32_t) type);
1259 m->next_uid = GNUNET_htonll (next_uid);
1260 qc.rc.proc = proc;
1261 qc.rc.proc_cls = proc_cls;
1262 qe = make_queue_entry (h,
1263 env,
1264 queue_priority,
1265 max_queue_size,
1266 GNUNET_MESSAGE_TYPE_DATASTORE_DATA,
1267 &qc);
1268 if (NULL == qe)
1269 {
1270 LOG (GNUNET_ERROR_TYPE_DEBUG,
1271 "Could not create queue entry for zero-anonymity procation\n");
1272 return NULL;
1273 }
1274 GNUNET_STATISTICS_update (h->stats,
1275 gettext_noop
1276 ("# GET ZERO ANONYMITY requests executed"), 1,
1277 GNUNET_NO);
1278 process_queue (h);
1279 return qe;
1280}
1281
1282
1283/**
1284 * Get a result for a particular key from the datastore. The processor
1285 * will only be called once.
1286 *
1287 * @param h handle to the datastore
1288 * @param next_uid return the result with lowest uid >= next_uid
1289 * @param random if true, return a random result instead of using next_uid
1290 * @param key maybe NULL (to match all entries)
1291 * @param type desired type, 0 for any
1292 * @param queue_priority ranking of this request in the priority queue
1293 * @param max_queue_size at what queue size should this request be dropped
1294 * (if other requests of higher priority are in the queue)
1295 * @param proc function to call on each matching value;
1296 * will be called once with a NULL value at the end
1297 * @param proc_cls closure for @a proc
1298 * @return NULL if the entry was not queued, otherwise a handle that can be used to
1299 * cancel
1300 */
1301struct GNUNET_DATASTORE_QueueEntry *
1302GNUNET_DATASTORE_get_key (struct GNUNET_DATASTORE_Handle *h,
1303 uint64_t next_uid,
1304 bool random,
1305 const struct GNUNET_HashCode *key,
1306 enum GNUNET_BLOCK_Type type,
1307 unsigned int queue_priority,
1308 unsigned int max_queue_size,
1309 GNUNET_DATASTORE_DatumProcessor proc,
1310 void *proc_cls)
1311{
1312 struct GNUNET_DATASTORE_QueueEntry *qe;
1313 struct GNUNET_MQ_Envelope *env;
1314 struct GetKeyMessage *gkm;
1315 struct GetMessage *gm;
1316 union QueueContext qc;
1317
1318 GNUNET_assert (NULL != proc);
1319 LOG (GNUNET_ERROR_TYPE_DEBUG,
1320 "Asked to look for data of type %u under key `%s'\n",
1321 (unsigned int) type,
1322 (NULL != key) ? GNUNET_h2s (key) : "NULL");
1323 if (NULL == key)
1324 {
1325 env = GNUNET_MQ_msg (gm,
1326 GNUNET_MESSAGE_TYPE_DATASTORE_GET);
1327 gm->type = htonl (type);
1328 gm->next_uid = GNUNET_htonll (next_uid);
1329 gm->random = random;
1330 }
1331 else
1332 {
1333 env = GNUNET_MQ_msg (gkm,
1334 GNUNET_MESSAGE_TYPE_DATASTORE_GET_KEY);
1335 gkm->type = htonl (type);
1336 gkm->next_uid = GNUNET_htonll (next_uid);
1337 gkm->random = random;
1338 gkm->key = *key;
1339 }
1340 qc.rc.proc = proc;
1341 qc.rc.proc_cls = proc_cls;
1342 qe = make_queue_entry (h,
1343 env,
1344 queue_priority,
1345 max_queue_size,
1346 GNUNET_MESSAGE_TYPE_DATASTORE_DATA,
1347 &qc);
1348 if (NULL == qe)
1349 {
1350 LOG (GNUNET_ERROR_TYPE_DEBUG,
1351 "Could not queue request for `%s'\n",
1352 (NULL != key) ? GNUNET_h2s (key): "NULL");
1353 return NULL;
1354 }
1355#if INSANE_STATISTICS
1356 GNUNET_STATISTICS_update (h->stats,
1357 gettext_noop ("# GET requests executed"),
1358 1,
1359 GNUNET_NO);
1360#endif
1361 process_queue (h);
1362 return qe;
1363}
1364
1365
1366/**
1367 * Cancel a datastore operation. The final callback from the
1368 * operation must not have been done yet.
1369 *
1370 * @param qe operation to cancel
1371 */
1372void
1373GNUNET_DATASTORE_cancel (struct GNUNET_DATASTORE_QueueEntry *qe)
1374{
1375 struct GNUNET_DATASTORE_Handle *h = qe->h;
1376
1377 LOG (GNUNET_ERROR_TYPE_DEBUG,
1378 "Pending DATASTORE request %p cancelled (%d, %d)\n",
1379 qe,
1380 NULL == qe->env,
1381 h->queue_head == qe);
1382 if (NULL == qe->env)
1383 {
1384 free_queue_entry (qe);
1385 h->skip_next_messages++;
1386 return;
1387 }
1388 free_queue_entry (qe);
1389 process_queue (h);
1390}
1391
1392
1393/* end of datastore_api.c */
diff --git a/src/service/datastore/gnunet-service-datastore.c b/src/service/datastore/gnunet-service-datastore.c
new file mode 100644
index 000000000..d42e16fe2
--- /dev/null
+++ b/src/service/datastore/gnunet-service-datastore.c
@@ -0,0 +1,1650 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2004-2014, 2016 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file datastore/gnunet-service-datastore.c
23 * @brief Management for the datastore for files stored on a GNUnet node
24 * @author Christian Grothoff
25 */
26
27#include "platform.h"
28#include "gnunet_util_lib.h"
29#include "gnunet_protocols.h"
30#include "gnunet_statistics_service.h"
31#include "gnunet_datastore_plugin.h"
32#include "datastore.h"
33
34/**
35 * How many messages do we queue at most per client?
36 */
37#define MAX_PENDING 1024
38
39/**
40 * Limit size of bloom filter to 2 GB.
41 */
42#define MAX_BF_SIZE ((uint32_t) (1LL << 31))
43
44/**
45 * How long are we at most keeping "expired" content
46 * past the expiration date in the database?
47 */
48#define MAX_EXPIRE_DELAY \
49 GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15)
50
51/**
52 * How fast are we allowed to query the database for deleting
53 * expired content? (1 item per second).
54 */
55#define MIN_EXPIRE_DELAY \
56 GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 1)
57
58/**
59 * Name under which we store current space consumption.
60 */
61static char *quota_stat_name;
62
63/**
64 * Task to timeout stat GET.
65 */
66static struct GNUNET_SCHEDULER_Task *stat_timeout_task;
67
68/**
69 * After how many payload-changing operations
70 * do we sync our statistics?
71 */
72#define MAX_STAT_SYNC_LAG 50
73
74
75/**
76 * Our datastore plugin.
77 */
78struct DatastorePlugin
79{
80 /**
81 * API of the transport as returned by the plugin's
82 * initialization function.
83 */
84 struct GNUNET_DATASTORE_PluginFunctions *api;
85
86 /**
87 * Short name for the plugin (e.g. "sqlite").
88 */
89 char *short_name;
90
91 /**
92 * Name of the library (e.g. "gnunet_plugin_datastore_sqlite").
93 */
94 char *lib_name;
95
96 /**
97 * Environment this transport service is using
98 * for this plugin.
99 */
100 struct GNUNET_DATASTORE_PluginEnvironment env;
101};
102
103
104/**
105 * Linked list of active reservations.
106 */
107struct ReservationList
108{
109 /**
110 * This is a linked list.
111 */
112 struct ReservationList *next;
113
114 /**
115 * Client that made the reservation.
116 */
117 struct GNUNET_SERVICE_Client *client;
118
119 /**
120 * Number of bytes (still) reserved.
121 */
122 uint64_t amount;
123
124 /**
125 * Number of items (still) reserved.
126 */
127 uint64_t entries;
128
129 /**
130 * Reservation identifier.
131 */
132 int32_t rid;
133};
134
135
136/**
137 * Our datastore plugin (NULL if not available).
138 */
139static struct DatastorePlugin *plugin;
140
141/**
142 * Linked list of space reservations made by clients.
143 */
144static struct ReservationList *reservations;
145
146/**
147 * Bloomfilter to quickly tell if we don't have the content.
148 */
149static struct GNUNET_CONTAINER_BloomFilter *filter;
150
151/**
152 * Name of our plugin.
153 */
154static char *plugin_name;
155
156/**
157 * Our configuration.
158 */
159static const struct GNUNET_CONFIGURATION_Handle *cfg;
160
161/**
162 * Handle for reporting statistics.
163 */
164static struct GNUNET_STATISTICS_Handle *stats;
165
166/**
167 * How much space are we using for the cache? (space available for
168 * insertions that will be instantly reclaimed by discarding less
169 * important content --- or possibly whatever we just inserted into
170 * the "cache").
171 */
172static unsigned long long cache_size;
173
174/**
175 * How much space have we currently reserved?
176 */
177static unsigned long long reserved;
178
179/**
180 * How much data are we currently storing
181 * in the database?
182 */
183static unsigned long long payload;
184
185/**
186 * Identity of the task that is used to delete
187 * expired content.
188 */
189static struct GNUNET_SCHEDULER_Task *expired_kill_task;
190
191/**
192 * Minimum time that content should have to not be discarded instantly
193 * (time stamp of any content that we've been discarding recently to
194 * stay below the quota). FOREVER if we had to expire content with
195 * non-zero priority.
196 */
197static struct GNUNET_TIME_Absolute min_expiration;
198
199/**
200 * How much space are we allowed to use?
201 */
202static unsigned long long quota;
203
204/**
205 * Should the database be dropped on exit?
206 */
207static int do_drop;
208
209/**
210 * Should we refresh the BF when the DB is loaded?
211 */
212static int refresh_bf;
213
214/**
215 * Number of updates that were made to the
216 * payload value since we last synchronized
217 * it with the statistics service.
218 */
219static unsigned int last_sync;
220
221/**
222 * Did we get an answer from statistics?
223 */
224static int stats_worked;
225
226
227/**
228 * Synchronize our utilization statistics with the
229 * statistics service.
230 */
231static void
232sync_stats ()
233{
234 GNUNET_STATISTICS_set (stats, quota_stat_name, payload, GNUNET_YES);
235 GNUNET_STATISTICS_set (stats,
236 "# utilization by current datastore",
237 payload,
238 GNUNET_NO);
239 last_sync = 0;
240}
241
242
243/**
244 * Have we already cleaned up the TCCs and are hence no longer
245 * willing (or able) to transmit anything to anyone?
246 */
247static int cleaning_done;
248
249/**
250 * Handle for pending get request.
251 */
252static struct GNUNET_STATISTICS_GetHandle *stat_get;
253
254/**
255 * Handle to our server.
256 */
257static struct GNUNET_SERVICE_Handle *service;
258
259/**
260 * Task that is used to remove expired entries from
261 * the datastore. This task will schedule itself
262 * again automatically to always delete all expired
263 * content quickly.
264 *
265 * @param cls not used
266 */
267static void
268delete_expired (void *cls);
269
270
271/**
272 * Iterate over the expired items stored in the datastore.
273 * Delete all expired items; once we have processed all
274 * expired items, re-schedule the "delete_expired" task.
275 *
276 * @param cls not used
277 * @param key key for the content
278 * @param size number of bytes in data
279 * @param data content stored
280 * @param type type of the content
281 * @param priority priority of the content
282 * @param anonymity anonymity-level for the content
283 * @param replication replication-level for the content
284 * @param expiration expiration time for the content
285 * @param uid unique identifier for the datum;
286 * maybe 0 if no unique identifier is available
287 *
288 * @return #GNUNET_SYSERR to abort the iteration, #GNUNET_OK to continue
289 * (continue on call to "next", of course),
290 * #GNUNET_NO to delete the item and continue (if supported)
291 */
292static int
293expired_processor (void *cls,
294 const struct GNUNET_HashCode *key,
295 uint32_t size,
296 const void *data,
297 enum GNUNET_BLOCK_Type type,
298 uint32_t priority,
299 uint32_t anonymity,
300 uint32_t replication,
301 struct GNUNET_TIME_Absolute expiration,
302 uint64_t uid)
303{
304 struct GNUNET_TIME_Absolute now;
305
306 if (NULL == key)
307 {
308 expired_kill_task =
309 GNUNET_SCHEDULER_add_delayed_with_priority (MAX_EXPIRE_DELAY,
310 GNUNET_SCHEDULER_PRIORITY_IDLE,
311 &delete_expired,
312 NULL);
313 return GNUNET_SYSERR;
314 }
315 now = GNUNET_TIME_absolute_get ();
316 if (expiration.abs_value_us > now.abs_value_us)
317 {
318 /* finished processing */
319 expired_kill_task =
320 GNUNET_SCHEDULER_add_delayed_with_priority (MAX_EXPIRE_DELAY,
321 GNUNET_SCHEDULER_PRIORITY_IDLE,
322 &delete_expired,
323 NULL);
324 return GNUNET_SYSERR;
325 }
326 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
327 "Deleting content `%s' of type %u that expired %s ago\n",
328 GNUNET_h2s (key),
329 type,
330 GNUNET_STRINGS_relative_time_to_string (
331 GNUNET_TIME_absolute_get_difference (expiration, now),
332 GNUNET_YES));
333 min_expiration = now;
334 GNUNET_STATISTICS_update (stats,
335 gettext_noop ("# bytes expired"),
336 size,
337 GNUNET_YES);
338 GNUNET_CONTAINER_bloomfilter_remove (filter, key);
339 expired_kill_task =
340 GNUNET_SCHEDULER_add_delayed_with_priority (MIN_EXPIRE_DELAY,
341 GNUNET_SCHEDULER_PRIORITY_IDLE,
342 &delete_expired,
343 NULL);
344 return GNUNET_NO;
345}
346
347
348/**
349 * Task that is used to remove expired entries from
350 * the datastore. This task will schedule itself
351 * again automatically to always delete all expired
352 * content quickly.
353 *
354 * @param cls not used
355 */
356static void
357delete_expired (void *cls)
358{
359 expired_kill_task = NULL;
360 plugin->api->get_expiration (plugin->api->cls, &expired_processor, NULL);
361}
362
363
364/**
365 * An iterator over a set of items stored in the datastore
366 * that deletes until we're happy with respect to our quota.
367 *
368 * @param cls closure
369 * @param key key for the content
370 * @param size number of bytes in data
371 * @param data content stored
372 * @param type type of the content
373 * @param priority priority of the content
374 * @param anonymity anonymity-level for the content
375 * @param replication replication-level for the content
376 * @param expiration expiration time for the content
377 * @param uid unique identifier for the datum;
378 * maybe 0 if no unique identifier is available
379 * @return #GNUNET_SYSERR to abort the iteration, #GNUNET_OK to continue
380 * (continue on call to "next", of course),
381 * #GNUNET_NO to delete the item and continue (if supported)
382 */
383static int
384quota_processor (void *cls,
385 const struct GNUNET_HashCode *key,
386 uint32_t size,
387 const void *data,
388 enum GNUNET_BLOCK_Type type,
389 uint32_t priority,
390 uint32_t anonymity,
391 uint32_t replication,
392 struct GNUNET_TIME_Absolute expiration,
393 uint64_t uid)
394{
395 unsigned long long *need = cls;
396
397 if (NULL == key)
398 return GNUNET_SYSERR;
399 GNUNET_log (
400 GNUNET_ERROR_TYPE_DEBUG,
401 "Deleting %llu bytes of low-priority (%u) content `%s' of type %u at %s prior to expiration (still trying to free another %llu bytes)\n",
402 (unsigned long long) (size + GNUNET_DATASTORE_ENTRY_OVERHEAD),
403 (unsigned int) priority,
404 GNUNET_h2s (key),
405 type,
406 GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_remaining (
407 expiration),
408 GNUNET_YES),
409 *need);
410 if (size + GNUNET_DATASTORE_ENTRY_OVERHEAD > *need)
411 *need = 0;
412 else
413 *need -= size + GNUNET_DATASTORE_ENTRY_OVERHEAD;
414 if (priority > 0)
415 min_expiration = GNUNET_TIME_UNIT_FOREVER_ABS;
416 else
417 min_expiration = expiration;
418 GNUNET_STATISTICS_update (stats,
419 gettext_noop ("# bytes purged (low-priority)"),
420 size,
421 GNUNET_YES);
422 GNUNET_CONTAINER_bloomfilter_remove (filter, key);
423 return GNUNET_NO;
424}
425
426
427/**
428 * Manage available disk space by running tasks
429 * that will discard content if necessary. This
430 * function will be run whenever a request for
431 * "need" bytes of storage could only be satisfied
432 * by eating into the "cache" (and we want our cache
433 * space back).
434 *
435 * @param need number of bytes of content that were
436 * placed into the "cache" (and hence the
437 * number of bytes that should be removed).
438 */
439static void
440manage_space (unsigned long long need)
441{
442 unsigned long long last;
443
444 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
445 "Asked to free up %llu bytes of cache space\n",
446 need);
447 last = 0;
448 while ((need > 0) && (last != need))
449 {
450 last = need;
451 plugin->api->get_expiration (plugin->api->cls, &quota_processor, &need);
452 }
453}
454
455
456/**
457 * Transmit a status code to the client.
458 *
459 * @param client receiver of the response
460 * @param code status code
461 * @param msg optional error message (can be NULL)
462 */
463static void
464transmit_status (struct GNUNET_SERVICE_Client *client,
465 int code,
466 const char *msg)
467{
468 struct GNUNET_MQ_Envelope *env;
469 struct StatusMessage *sm;
470 size_t slen;
471
472 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
473 "Transmitting `%s' message with value %d and message `%s'\n",
474 "STATUS",
475 code,
476 msg != NULL ? msg : "(none)");
477 slen = (msg == NULL) ? 0 : strlen (msg) + 1;
478 env = GNUNET_MQ_msg_extra (sm, slen, GNUNET_MESSAGE_TYPE_DATASTORE_STATUS);
479 sm->status = htonl (code);
480 sm->min_expiration = GNUNET_TIME_absolute_hton (min_expiration);
481 GNUNET_memcpy (&sm[1], msg, slen);
482 GNUNET_MQ_send (GNUNET_SERVICE_client_get_mq (client), env);
483}
484
485
486/**
487 * Function that will transmit the given datastore entry
488 * to the client.
489 *
490 * @param cls closure, pointer to the client (of type `struct GNUNET_SERVICE_Client`).
491 * @param key key for the content
492 * @param size number of bytes in data
493 * @param data content stored
494 * @param type type of the content
495 * @param priority priority of the content
496 * @param anonymity anonymity-level for the content
497 * @param replication replication-level for the content
498 * @param expiration expiration time for the content
499 * @param uid unique identifier for the datum;
500 * maybe 0 if no unique identifier is available
501 * @return #GNUNET_SYSERR to abort the iteration, #GNUNET_OK to continue,
502 * #GNUNET_NO to delete the item and continue (if supported)
503 */
504static int
505transmit_item (void *cls,
506 const struct GNUNET_HashCode *key,
507 uint32_t size,
508 const void *data,
509 enum GNUNET_BLOCK_Type type,
510 uint32_t priority,
511 uint32_t anonymity,
512 uint32_t replication,
513 struct GNUNET_TIME_Absolute expiration,
514 uint64_t uid)
515{
516 struct GNUNET_SERVICE_Client *client = cls;
517 struct GNUNET_MQ_Envelope *env;
518 struct GNUNET_MessageHeader *end;
519 struct DataMessage *dm;
520
521 if (NULL == key)
522 {
523 /* transmit 'DATA_END' */
524 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Transmitting DATA_END message\n");
525 env = GNUNET_MQ_msg (end, GNUNET_MESSAGE_TYPE_DATASTORE_DATA_END);
526 GNUNET_MQ_send (GNUNET_SERVICE_client_get_mq (client), env);
527 return GNUNET_OK;
528 }
529 GNUNET_assert (sizeof(struct DataMessage) + size < GNUNET_MAX_MESSAGE_SIZE);
530 env = GNUNET_MQ_msg_extra (dm, size, GNUNET_MESSAGE_TYPE_DATASTORE_DATA);
531 dm->rid = htonl (0);
532 dm->size = htonl (size);
533 dm->type = htonl (type);
534 dm->priority = htonl (priority);
535 dm->anonymity = htonl (anonymity);
536 dm->replication = htonl (replication);
537 dm->expiration = GNUNET_TIME_absolute_hton (expiration);
538 dm->uid = GNUNET_htonll (uid);
539 dm->key = *key;
540 GNUNET_memcpy (&dm[1], data, size);
541 GNUNET_log (
542 GNUNET_ERROR_TYPE_DEBUG,
543 "Transmitting DATA message for `%s' of type %u with expiration %s (in: %s)\n",
544 GNUNET_h2s (key),
545 type,
546 GNUNET_STRINGS_absolute_time_to_string (expiration),
547 GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_remaining (
548 expiration),
549 GNUNET_YES));
550 GNUNET_STATISTICS_update (stats,
551 gettext_noop ("# results found"),
552 1,
553 GNUNET_NO);
554 GNUNET_MQ_send (GNUNET_SERVICE_client_get_mq (client), env);
555 return GNUNET_OK;
556}
557
558
559/**
560 * Handle RESERVE-message.
561 *
562 * @param cls identification of the client
563 * @param msg the actual message
564 */
565static void
566handle_reserve (void *cls, const struct ReserveMessage *msg)
567{
568 /**
569 * Static counter to produce reservation identifiers.
570 */
571 static int reservation_gen;
572 struct GNUNET_SERVICE_Client *client = cls;
573 struct ReservationList *e;
574 unsigned long long used;
575 unsigned long long req;
576 uint64_t amount;
577 uint32_t entries;
578
579 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Processing RESERVE request\n");
580 amount = GNUNET_ntohll (msg->amount);
581 entries = ntohl (msg->entries);
582 used = payload + reserved;
583 req =
584 amount + ((unsigned long long) GNUNET_DATASTORE_ENTRY_OVERHEAD) * entries;
585 if (used + req > quota)
586 {
587 if (quota < used)
588 used =
589 quota; /* cheat a bit for error message (to avoid negative numbers) */
590 GNUNET_log (
591 GNUNET_ERROR_TYPE_WARNING,
592 _ (
593 "Insufficient space (%llu bytes are available) to satisfy RESERVE request for %llu bytes\n"),
594 quota - used,
595 req);
596 if (cache_size < req)
597 {
598 /* TODO: document this in the FAQ; essentially, if this
599 * message happens, the insertion request could be blocked
600 * by less-important content from migration because it is
601 * larger than 1/8th of the overall available space, and
602 * we only reserve 1/8th for "fresh" insertions */
603 GNUNET_log (
604 GNUNET_ERROR_TYPE_WARNING,
605 _ (
606 "The requested amount (%llu bytes) is larger than the cache size (%llu bytes)\n"),
607 req,
608 cache_size);
609 transmit_status (client,
610 0,
611 gettext_noop (
612 "Insufficient space to satisfy request and "
613 "requested amount is larger than cache size"));
614 }
615 else
616 {
617 transmit_status (client,
618 0,
619 gettext_noop ("Insufficient space to satisfy request"));
620 }
621 GNUNET_SERVICE_client_continue (client);
622 return;
623 }
624 reserved += req;
625 GNUNET_STATISTICS_set (stats,
626 gettext_noop ("# reserved"),
627 reserved,
628 GNUNET_NO);
629 e = GNUNET_new (struct ReservationList);
630 e->next = reservations;
631 reservations = e;
632 e->client = client;
633 e->amount = amount;
634 e->entries = entries;
635 e->rid = ++reservation_gen;
636 if (reservation_gen < 0)
637 reservation_gen = 0; /* wrap around */
638 transmit_status (client, e->rid, NULL);
639 GNUNET_SERVICE_client_continue (client);
640}
641
642
643/**
644 * Handle RELEASE_RESERVE-message.
645 *
646 * @param cls identification of the client
647 * @param msg the actual message
648 */
649static void
650handle_release_reserve (void *cls, const struct ReleaseReserveMessage *msg)
651{
652 struct GNUNET_SERVICE_Client *client = cls;
653 struct ReservationList *pos;
654 struct ReservationList *prev;
655 struct ReservationList *next;
656 int rid = ntohl (msg->rid);
657 unsigned long long rem;
658
659 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Processing RELEASE_RESERVE request\n");
660 next = reservations;
661 prev = NULL;
662 while (NULL != (pos = next))
663 {
664 next = pos->next;
665 if (rid == pos->rid)
666 {
667 if (prev == NULL)
668 reservations = next;
669 else
670 prev->next = next;
671 rem =
672 pos->amount
673 + ((unsigned long long) GNUNET_DATASTORE_ENTRY_OVERHEAD) * pos->entries;
674 GNUNET_assert (reserved >= rem);
675 reserved -= rem;
676 GNUNET_STATISTICS_set (stats,
677 gettext_noop ("# reserved"),
678 reserved,
679 GNUNET_NO);
680 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
681 "Returning %llu remaining reserved bytes to storage pool\n",
682 rem);
683 GNUNET_free (pos);
684 transmit_status (client, GNUNET_OK, NULL);
685 GNUNET_SERVICE_client_continue (client);
686 return;
687 }
688 prev = pos;
689 }
690 GNUNET_break (0);
691 transmit_status (client,
692 GNUNET_SYSERR,
693 gettext_noop ("Could not find matching reservation"));
694 GNUNET_SERVICE_client_continue (client);
695}
696
697
698/**
699 * Check that the given message is a valid data message.
700 *
701 * @param dm message to check
702 * @return #GNUNET_SYSERR is not well-formed, otherwise #GNUNET_OK
703 */
704static int
705check_data (const struct DataMessage *dm)
706{
707 uint16_t size;
708 uint32_t dsize;
709
710 size = ntohs (dm->header.size);
711 dsize = ntohl (dm->size);
712 if (size != dsize + sizeof(struct DataMessage))
713 {
714 GNUNET_break (0);
715 return GNUNET_SYSERR;
716 }
717 return GNUNET_OK;
718}
719
720
721/**
722 * Put continuation.
723 *
724 * @param cls closure
725 * @param key key for the item stored
726 * @param size size of the item stored
727 * @param status #GNUNET_OK if inserted, #GNUNET_NO if updated,
728 * or #GNUNET_SYSERROR if error
729 * @param msg error message on error
730 */
731static void
732put_continuation (void *cls,
733 const struct GNUNET_HashCode *key,
734 uint32_t size,
735 int status,
736 const char *msg)
737{
738 struct GNUNET_SERVICE_Client *client = cls;
739
740 if (GNUNET_OK == status)
741 {
742 GNUNET_STATISTICS_update (stats,
743 gettext_noop ("# bytes stored"),
744 size,
745 GNUNET_YES);
746 GNUNET_CONTAINER_bloomfilter_add (filter, key);
747 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
748 "Successfully stored %u bytes under key `%s'\n",
749 size,
750 GNUNET_h2s (key));
751 }
752 transmit_status (client,
753 GNUNET_SYSERR == status ? GNUNET_SYSERR : GNUNET_OK,
754 msg);
755 if (quota - reserved - cache_size < payload)
756 {
757 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
758 _ ("Need %llu bytes more space (%llu allowed, using %llu)\n"),
759 (unsigned long long) size + GNUNET_DATASTORE_ENTRY_OVERHEAD,
760 (unsigned long long) (quota - reserved - cache_size),
761 (unsigned long long) payload);
762 manage_space (size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
763 }
764}
765
766
767/**
768 * Verify PUT-message.
769 *
770 * @param cls identification of the client
771 * @param dm the actual message
772 * @return #GNUNET_OK if @a dm is well-formed
773 */
774static int
775check_put (void *cls, const struct DataMessage *dm)
776{
777 if (GNUNET_OK != check_data (dm))
778 {
779 GNUNET_break (0);
780 return GNUNET_SYSERR;
781 }
782 return GNUNET_OK;
783}
784
785
786/**
787 * Handle PUT-message.
788 *
789 * @param cls identification of the client
790 * @param dm the actual message
791 */
792static void
793handle_put (void *cls, const struct DataMessage *dm)
794{
795 struct GNUNET_SERVICE_Client *client = cls;
796 int rid;
797 struct ReservationList *pos;
798 uint32_t size;
799
800 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
801 "Processing PUT request for `%s' of type %u\n",
802 GNUNET_h2s (&dm->key),
803 (uint32_t) ntohl (dm->type));
804 rid = ntohl (dm->rid);
805 size = ntohl (dm->size);
806 if (rid > 0)
807 {
808 pos = reservations;
809 while ((NULL != pos) && (rid != pos->rid))
810 pos = pos->next;
811 GNUNET_break (pos != NULL);
812 if (NULL != pos)
813 {
814 GNUNET_break (pos->entries > 0);
815 GNUNET_break (pos->amount >= size);
816 pos->entries--;
817 pos->amount -= size;
818 reserved -= (size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
819 GNUNET_STATISTICS_set (stats,
820 gettext_noop ("# reserved"),
821 reserved,
822 GNUNET_NO);
823 }
824 }
825 bool absent =
826 GNUNET_NO == GNUNET_CONTAINER_bloomfilter_test (filter, &dm->key);
827 plugin->api->put (plugin->api->cls,
828 &dm->key,
829 absent,
830 ntohl (dm->size),
831 &dm[1],
832 ntohl (dm->type),
833 ntohl (dm->priority),
834 ntohl (dm->anonymity),
835 ntohl (dm->replication),
836 GNUNET_TIME_absolute_ntoh (dm->expiration),
837 &put_continuation,
838 client);
839 GNUNET_SERVICE_client_continue (client);
840}
841
842
843/**
844 * Handle #GNUNET_MESSAGE_TYPE_DATASTORE_GET-message.
845 *
846 * @param cls identification of the client
847 * @param msg the actual message
848 */
849static void
850handle_get (void *cls, const struct GetMessage *msg)
851{
852 struct GNUNET_SERVICE_Client *client = cls;
853
854 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
855 "Processing GET request of type %u\n",
856 (uint32_t) ntohl (msg->type));
857 GNUNET_STATISTICS_update (stats,
858 gettext_noop ("# GET requests received"),
859 1,
860 GNUNET_NO);
861 plugin->api->get_key (plugin->api->cls,
862 GNUNET_ntohll (msg->next_uid),
863 msg->random,
864 NULL,
865 ntohl (msg->type),
866 &transmit_item,
867 client);
868 GNUNET_SERVICE_client_continue (client);
869}
870
871
872/**
873 * Handle #GNUNET_MESSAGE_TYPE_DATASTORE_GET_KEY-message.
874 *
875 * @param cls closure
876 * @param msg the actual message
877 */
878static void
879handle_get_key (void *cls, const struct GetKeyMessage *msg)
880{
881 struct GNUNET_SERVICE_Client *client = cls;
882
883 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
884 "Processing GET request for `%s' of type %u\n",
885 GNUNET_h2s (&msg->key),
886 (uint32_t) ntohl (msg->type));
887 GNUNET_STATISTICS_update (stats,
888 gettext_noop ("# GET KEY requests received"),
889 1,
890 GNUNET_NO);
891 if (GNUNET_YES != GNUNET_CONTAINER_bloomfilter_test (filter, &msg->key))
892 {
893 /* don't bother database... */
894 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
895 "Empty result set for GET request for `%s' (bloomfilter).\n",
896 GNUNET_h2s (&msg->key));
897 GNUNET_STATISTICS_update (stats,
898 gettext_noop (
899 "# requests filtered by bloomfilter"),
900 1,
901 GNUNET_NO);
902 transmit_item (client,
903 NULL,
904 0,
905 NULL,
906 0,
907 0,
908 0,
909 0,
910 GNUNET_TIME_UNIT_ZERO_ABS,
911 0);
912 GNUNET_SERVICE_client_continue (client);
913 return;
914 }
915 plugin->api->get_key (plugin->api->cls,
916 GNUNET_ntohll (msg->next_uid),
917 msg->random,
918 &msg->key,
919 ntohl (msg->type),
920 &transmit_item,
921 client);
922 GNUNET_SERVICE_client_continue (client);
923}
924
925
926/**
927 * Handle GET_REPLICATION-message.
928 *
929 * @param cls identification of the client
930 * @param message the actual message
931 */
932static void
933handle_get_replication (void *cls, const struct GNUNET_MessageHeader *message)
934{
935 struct GNUNET_SERVICE_Client *client = cls;
936
937 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Processing GET_REPLICATION request\n");
938 GNUNET_STATISTICS_update (stats,
939 gettext_noop (
940 "# GET REPLICATION requests received"),
941 1,
942 GNUNET_NO);
943 plugin->api->get_replication (plugin->api->cls, &transmit_item, client);
944 GNUNET_SERVICE_client_continue (client);
945}
946
947
948/**
949 * Handle GET_ZERO_ANONYMITY-message.
950 *
951 * @param cls client identification of the client
952 * @param msg the actual message
953 */
954static void
955handle_get_zero_anonymity (void *cls, const struct GetZeroAnonymityMessage *msg)
956{
957 struct GNUNET_SERVICE_Client *client = cls;
958 enum GNUNET_BLOCK_Type type;
959
960 type = (enum GNUNET_BLOCK_Type) ntohl (msg->type);
961 if (type == GNUNET_BLOCK_TYPE_ANY)
962 {
963 GNUNET_break (0);
964 GNUNET_SERVICE_client_drop (client);
965 return;
966 }
967 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
968 "Processing GET_ZERO_ANONYMITY request\n");
969 GNUNET_STATISTICS_update (stats,
970 gettext_noop (
971 "# GET ZERO ANONYMITY requests received"),
972 1,
973 GNUNET_NO);
974 plugin->api->get_zero_anonymity (plugin->api->cls,
975 GNUNET_ntohll (msg->next_uid),
976 type,
977 &transmit_item,
978 client);
979 GNUNET_SERVICE_client_continue (client);
980}
981
982
983/**
984 * Remove continuation.
985 *
986 * @param cls closure
987 * @param key key for the content
988 * @param size number of bytes in data
989 * @param status #GNUNET_OK if removed, #GNUNET_NO if not found,
990 * or #GNUNET_SYSERROR if error
991 * @param msg error message on error
992 */
993static void
994remove_continuation (void *cls,
995 const struct GNUNET_HashCode *key,
996 uint32_t size,
997 int status,
998 const char *msg)
999{
1000 struct GNUNET_SERVICE_Client *client = cls;
1001
1002 if (GNUNET_SYSERR == status)
1003 {
1004 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "REMOVE request failed: %s.\n", msg);
1005 transmit_status (client, GNUNET_NO, msg);
1006 return;
1007 }
1008 if (GNUNET_NO == status)
1009 {
1010 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1011 "Content not found for REMOVE request.\n");
1012 transmit_status (client, GNUNET_NO, _ ("Content not found"));
1013 return;
1014 }
1015 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1016 "Item matches REMOVE request for key `%s'.\n",
1017 GNUNET_h2s (key));
1018 GNUNET_STATISTICS_update (stats,
1019 gettext_noop ("# bytes removed (explicit request)"),
1020 size,
1021 GNUNET_YES);
1022 GNUNET_CONTAINER_bloomfilter_remove (filter, key);
1023 transmit_status (client, GNUNET_OK, NULL);
1024}
1025
1026
1027/**
1028 * Verify REMOVE-message.
1029 *
1030 * @param cls identification of the client
1031 * @param dm the actual message
1032 * @return #GNUNET_OK if @a dm is well-formed
1033 */
1034static int
1035check_remove (void *cls, const struct DataMessage *dm)
1036{
1037 if (GNUNET_OK != check_data (dm))
1038 {
1039 GNUNET_break (0);
1040 return GNUNET_SYSERR;
1041 }
1042 return GNUNET_OK;
1043}
1044
1045
1046/**
1047 * Handle REMOVE-message.
1048 *
1049 * @param cls closure
1050 * @param dm the actual message
1051 */
1052static void
1053handle_remove (void *cls, const struct DataMessage *dm)
1054{
1055 struct GNUNET_SERVICE_Client *client = cls;
1056
1057 GNUNET_STATISTICS_update (stats,
1058 gettext_noop ("# REMOVE requests received"),
1059 1,
1060 GNUNET_NO);
1061 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1062 "Processing REMOVE request for `%s'\n",
1063 GNUNET_h2s (&dm->key));
1064 plugin->api->remove_key (plugin->api->cls,
1065 &dm->key,
1066 ntohl (dm->size),
1067 &dm[1],
1068 &remove_continuation,
1069 client);
1070 GNUNET_SERVICE_client_continue (client);
1071}
1072
1073
1074/**
1075 * Handle DROP-message.
1076 *
1077 * @param cls identification of the client
1078 * @param message the actual message
1079 */
1080static void
1081handle_drop (void *cls,
1082 const struct GNUNET_MessageHeader *message)
1083{
1084 struct GNUNET_SERVICE_Client *client = cls;
1085
1086 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1087 "Processing DROP request\n");
1088 do_drop = GNUNET_YES;
1089 GNUNET_SERVICE_client_continue (client);
1090}
1091
1092
1093/**
1094 * Function called by plugins to notify us about a
1095 * change in their disk utilization.
1096 *
1097 * @param cls closure (NULL)
1098 * @param delta change in disk utilization,
1099 * 0 for "reset to empty"
1100 */
1101static void
1102disk_utilization_change_cb (void *cls, int delta)
1103{
1104 if ((delta < 0) && (payload < -delta))
1105 {
1106 GNUNET_log (
1107 GNUNET_ERROR_TYPE_WARNING,
1108 _ (
1109 "Datastore payload must have been inaccurate (%lld < %lld). Recomputing it.\n"),
1110 (long long) payload,
1111 (long long) -delta);
1112 plugin->api->estimate_size (plugin->api->cls, &payload);
1113 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1114 _ ("New payload: %lld\n"),
1115 (long long) payload);
1116 sync_stats ();
1117 return;
1118 }
1119 payload += delta;
1120 last_sync++;
1121 if (last_sync >= MAX_STAT_SYNC_LAG)
1122 sync_stats ();
1123}
1124
1125
1126/**
1127 * Callback function to process statistic values.
1128 *
1129 * @param cls closure (struct Plugin*)
1130 * @param subsystem name of subsystem that created the statistic
1131 * @param name the name of the datum
1132 * @param value the current value
1133 * @param is_persistent #GNUNET_YES if the value is persistent, #GNUNET_NO if not
1134 * @return #GNUNET_OK to continue, #GNUNET_SYSERR to abort iteration
1135 */
1136static int
1137process_stat_in (void *cls,
1138 const char *subsystem,
1139 const char *name,
1140 uint64_t value,
1141 int is_persistent)
1142{
1143 GNUNET_assert (GNUNET_NO == stats_worked);
1144 stats_worked = GNUNET_YES;
1145 payload += value;
1146 GNUNET_log (
1147 GNUNET_ERROR_TYPE_DEBUG,
1148 "Notification from statistics about existing payload (%llu), new payload is %llu\n",
1149 (unsigned long long) value,
1150 (unsigned long long) payload);
1151 return GNUNET_OK;
1152}
1153
1154
1155/**
1156 * Load the datastore plugin.
1157 */
1158static struct DatastorePlugin *
1159load_plugin ()
1160{
1161 struct DatastorePlugin *ret;
1162 char *libname;
1163
1164 ret = GNUNET_new (struct DatastorePlugin);
1165 ret->env.cfg = cfg;
1166 ret->env.duc = &disk_utilization_change_cb;
1167 ret->env.cls = NULL;
1168 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1169 _ ("Loading `%s' datastore plugin\n"),
1170 plugin_name);
1171 GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", plugin_name);
1172 ret->short_name = GNUNET_strdup (plugin_name);
1173 ret->lib_name = libname;
1174 ret->api = GNUNET_PLUGIN_load (libname, &ret->env);
1175 if (NULL == ret->api)
1176 {
1177 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1178 _ ("Failed to load datastore plugin for `%s'\n"),
1179 plugin_name);
1180 GNUNET_free (ret->short_name);
1181 GNUNET_free (libname);
1182 GNUNET_free (ret);
1183 return NULL;
1184 }
1185 return ret;
1186}
1187
1188
1189/**
1190 * Function called when the service shuts
1191 * down. Unloads our datastore plugin.
1192 *
1193 * @param plug plugin to unload
1194 */
1195static void
1196unload_plugin (struct DatastorePlugin *plug)
1197{
1198 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1199 "Datastore service is unloading plugin...\n");
1200 GNUNET_break (NULL == GNUNET_PLUGIN_unload (plug->lib_name, plug->api));
1201 GNUNET_free (plug->lib_name);
1202 GNUNET_free (plug->short_name);
1203 GNUNET_free (plug);
1204}
1205
1206
1207/**
1208 * Initialization complete, start operating the service.
1209 */
1210static void
1211begin_service ()
1212{
1213 GNUNET_SERVICE_resume (service);
1214 expired_kill_task =
1215 GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_IDLE,
1216 &delete_expired,
1217 NULL);
1218}
1219
1220
1221/**
1222 * Adds a given @a key to the bloomfilter in @a cls @a count times.
1223 *
1224 * @param cls the bloomfilter
1225 * @param key key to add
1226 * @param count number of times to add key
1227 */
1228static void
1229add_key_to_bloomfilter (void *cls,
1230 const struct GNUNET_HashCode *key,
1231 unsigned int count)
1232{
1233 struct GNUNET_CONTAINER_BloomFilter *bf = cls;
1234
1235 if (NULL == key)
1236 {
1237 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1238 _ ("Bloomfilter construction complete.\n"));
1239 begin_service ();
1240 return;
1241 }
1242
1243 while (0 < count--)
1244 GNUNET_CONTAINER_bloomfilter_add (bf, key);
1245}
1246
1247
1248/**
1249 * We finished receiving the statistic. Initialize the plugin; if
1250 * loading the statistic failed, run the estimator.
1251 *
1252 * @param cls NULL
1253 * @param success #GNUNET_NO if we failed to read the stat
1254 */
1255static void
1256process_stat_done (void *cls, int success)
1257{
1258 stat_get = NULL;
1259 if (NULL != stat_timeout_task)
1260 {
1261 GNUNET_SCHEDULER_cancel (stat_timeout_task);
1262 stat_timeout_task = NULL;
1263 }
1264 plugin = load_plugin ();
1265 if (NULL == plugin)
1266 {
1267 GNUNET_CONTAINER_bloomfilter_free (filter);
1268 filter = NULL;
1269 if (NULL != stats)
1270 {
1271 GNUNET_STATISTICS_destroy (stats, GNUNET_YES);
1272 stats = NULL;
1273 }
1274 return;
1275 }
1276
1277 if (GNUNET_NO == stats_worked)
1278 {
1279 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1280 "Failed to obtain value from statistics service, recomputing it\n");
1281 plugin->api->estimate_size (plugin->api->cls, &payload);
1282 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1283 _ ("New payload: %lld\n"),
1284 (long long) payload);
1285 }
1286
1287 if (GNUNET_YES == refresh_bf)
1288 {
1289 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1290 _ ("Rebuilding bloomfilter. Please be patient.\n"));
1291 if (NULL != plugin->api->get_keys)
1292 {
1293 plugin->api->get_keys (plugin->api->cls, &add_key_to_bloomfilter, filter);
1294 return;
1295 }
1296 else
1297 {
1298 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1299 _ (
1300 "Plugin does not support get_keys function. Please fix!\n"));
1301 }
1302 }
1303 begin_service ();
1304}
1305
1306
1307/**
1308 * Fetching stats took to long, run without.
1309 *
1310 * @param cls NULL
1311 */
1312static void
1313stat_timeout (void *cls)
1314{
1315 stat_timeout_task = NULL;
1316 GNUNET_STATISTICS_get_cancel (stat_get);
1317 process_stat_done (NULL, GNUNET_NO);
1318}
1319
1320
1321/**
1322 * Task run during shutdown.
1323 */
1324static void
1325cleaning_task (void *cls)
1326{
1327 cleaning_done = GNUNET_YES;
1328 if (NULL != expired_kill_task)
1329 {
1330 GNUNET_SCHEDULER_cancel (expired_kill_task);
1331 expired_kill_task = NULL;
1332 }
1333 if (GNUNET_YES == do_drop)
1334 {
1335 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1336 "Dropping database!\n");
1337 plugin->api->drop (plugin->api->cls);
1338 payload = 0;
1339 last_sync++;
1340 }
1341 if (NULL != plugin)
1342 {
1343 unload_plugin (plugin);
1344 plugin = NULL;
1345 }
1346 if (NULL != filter)
1347 {
1348 GNUNET_CONTAINER_bloomfilter_free (filter);
1349 filter = NULL;
1350 }
1351 if (NULL != stat_get)
1352 {
1353 GNUNET_STATISTICS_get_cancel (stat_get);
1354 stat_get = NULL;
1355 }
1356 if (NULL != stat_timeout_task)
1357 {
1358 GNUNET_SCHEDULER_cancel (stat_timeout_task);
1359 stat_timeout_task = NULL;
1360 }
1361 GNUNET_free (plugin_name);
1362 plugin_name = NULL;
1363 if (last_sync > 0)
1364 sync_stats ();
1365 if (NULL != stats)
1366 {
1367 GNUNET_STATISTICS_destroy (stats, GNUNET_YES);
1368 stats = NULL;
1369 }
1370 GNUNET_free (quota_stat_name);
1371 quota_stat_name = NULL;
1372}
1373
1374
1375/**
1376 * Add a client to our list of active clients.
1377 *
1378 * @param cls NULL
1379 * @param client client to add
1380 * @param mq message queue for @a client
1381 * @return @a client
1382 */
1383static void *
1384client_connect_cb (void *cls,
1385 struct GNUNET_SERVICE_Client *client,
1386 struct GNUNET_MQ_Handle *mq)
1387{
1388 return client;
1389}
1390
1391
1392/**
1393 * Called whenever a client is disconnected.
1394 * Frees our resources associated with that client.
1395 *
1396 * @param cls closure
1397 * @param client identification of the client
1398 * @param app_ctx must match @a client
1399 */
1400static void
1401client_disconnect_cb (void *cls,
1402 struct GNUNET_SERVICE_Client *client,
1403 void *app_ctx)
1404{
1405 struct ReservationList *pos;
1406 struct ReservationList *prev;
1407 struct ReservationList *next;
1408
1409 GNUNET_assert (app_ctx == client);
1410 prev = NULL;
1411 pos = reservations;
1412 while (NULL != pos)
1413 {
1414 next = pos->next;
1415 if (pos->client == client)
1416 {
1417 if (NULL == prev)
1418 reservations = next;
1419 else
1420 prev->next = next;
1421 reserved -= pos->amount + pos->entries * GNUNET_DATASTORE_ENTRY_OVERHEAD;
1422 GNUNET_free (pos);
1423 }
1424 else
1425 {
1426 prev = pos;
1427 }
1428 pos = next;
1429 }
1430 GNUNET_STATISTICS_set (stats,
1431 gettext_noop ("# reserved"),
1432 reserved,
1433 GNUNET_NO);
1434}
1435
1436
1437/**
1438 * Process datastore requests.
1439 *
1440 * @param cls closure
1441 * @param serv the initialized service
1442 * @param c configuration to use
1443 */
1444static void
1445run (void *cls,
1446 const struct GNUNET_CONFIGURATION_Handle *c,
1447 struct GNUNET_SERVICE_Handle *serv)
1448{
1449 char *fn;
1450 char *pfn;
1451 unsigned int bf_size;
1452
1453 service = serv;
1454 cfg = c;
1455 if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_string (cfg,
1456 "DATASTORE",
1457 "DATABASE",
1458 &plugin_name))
1459 {
1460 GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
1461 "DATABASE",
1462 "DATASTORE");
1463 return;
1464 }
1465 GNUNET_asprintf (&quota_stat_name,
1466 _ ("# bytes used in file-sharing datastore `%s'"),
1467 plugin_name);
1468 if (GNUNET_OK !=
1469 GNUNET_CONFIGURATION_get_value_size (cfg, "DATASTORE", "QUOTA", &quota))
1470 {
1471 GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR, "QUOTA", "DATASTORE");
1472 return;
1473 }
1474 stats = GNUNET_STATISTICS_create ("datastore", cfg);
1475 GNUNET_STATISTICS_set (stats, gettext_noop ("# quota"), quota, GNUNET_NO);
1476 cache_size = quota / 8; /* Or should we make this an option? */
1477 GNUNET_STATISTICS_set (stats,
1478 gettext_noop ("# cache size"),
1479 cache_size,
1480 GNUNET_NO);
1481 if (quota / (32 * 1024LL) > MAX_BF_SIZE)
1482 bf_size = MAX_BF_SIZE;
1483 else
1484 bf_size =
1485 quota / (32 * 1024LL); /* 8 bit per entry, 1 bit per 32 kb in DB */
1486 fn = NULL;
1487 if ((GNUNET_OK != GNUNET_CONFIGURATION_get_value_filename (cfg,
1488 "DATASTORE",
1489 "BLOOMFILTER",
1490 &fn)) ||
1491 (GNUNET_OK != GNUNET_DISK_directory_create_for_file (fn)))
1492 {
1493 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1494 _ ("Could not use specified filename `%s' for bloomfilter.\n"),
1495 NULL != fn ? fn : "");
1496 GNUNET_free (fn);
1497 fn = NULL;
1498 }
1499 if (NULL != fn)
1500 {
1501 GNUNET_asprintf (&pfn, "%s.%s", fn, plugin_name);
1502 if (GNUNET_YES == GNUNET_DISK_file_test (pfn))
1503 {
1504 filter =
1505 GNUNET_CONTAINER_bloomfilter_load (pfn,
1506 bf_size,
1507 5); /* approx. 3% false positives at max use */
1508 if (NULL == filter)
1509 {
1510 /* file exists but not valid, remove and try again, but refresh */
1511 if (0 != unlink (pfn))
1512 {
1513 /* failed to remove, run without file */
1514 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1515 _ ("Failed to remove bogus bloomfilter file `%s'\n"),
1516 pfn);
1517 GNUNET_free (pfn);
1518 pfn = NULL;
1519 filter = GNUNET_CONTAINER_bloomfilter_load (
1520 NULL,
1521 bf_size,
1522 5); /* approx. 3% false positives at max use */
1523 refresh_bf = GNUNET_YES;
1524 }
1525 else
1526 {
1527 /* try again after remove */
1528 filter = GNUNET_CONTAINER_bloomfilter_load (
1529 pfn,
1530 bf_size,
1531 5); /* approx. 3% false positives at max use */
1532 refresh_bf = GNUNET_YES;
1533 if (NULL == filter)
1534 {
1535 /* failed yet again, give up on using file */
1536 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1537 _ ("Failed to remove bogus bloomfilter file `%s'\n"),
1538 pfn);
1539 GNUNET_free (pfn);
1540 pfn = NULL;
1541 filter = GNUNET_CONTAINER_bloomfilter_init (
1542 NULL,
1543 bf_size,
1544 5); /* approx. 3% false positives at max use */
1545 }
1546 }
1547 }
1548 else
1549 {
1550 /* normal case: have an existing valid bf file, no need to refresh */
1551 refresh_bf = GNUNET_NO;
1552 }
1553 }
1554 else
1555 {
1556 filter =
1557 GNUNET_CONTAINER_bloomfilter_load (pfn,
1558 bf_size,
1559 5); /* approx. 3% false positives at max use */
1560 refresh_bf = GNUNET_YES;
1561 }
1562 GNUNET_free (pfn);
1563 }
1564 else
1565 {
1566 filter =
1567 GNUNET_CONTAINER_bloomfilter_init (NULL,
1568 bf_size,
1569 5); /* approx. 3% false positives at max use */
1570 refresh_bf = GNUNET_YES;
1571 }
1572 GNUNET_free (fn);
1573 if (NULL == filter)
1574 {
1575 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1576 _ ("Failed to initialize bloomfilter.\n"));
1577 if (NULL != stats)
1578 {
1579 GNUNET_STATISTICS_destroy (stats, GNUNET_YES);
1580 stats = NULL;
1581 }
1582 return;
1583 }
1584 GNUNET_SERVICE_suspend (service);
1585 stat_get = GNUNET_STATISTICS_get (stats,
1586 "datastore",
1587 quota_stat_name,
1588 &process_stat_done,
1589 &process_stat_in,
1590 NULL);
1591 if (NULL == stat_get)
1592 process_stat_done (NULL, GNUNET_SYSERR);
1593 else
1594 stat_timeout_task = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_SECONDS,
1595 &stat_timeout,
1596 NULL);
1597 GNUNET_SCHEDULER_add_shutdown (&cleaning_task, NULL);
1598}
1599
1600
1601/**
1602 * Define "main" method using service macro.
1603 */
1604GNUNET_SERVICE_MAIN (
1605 "datastore",
1606 GNUNET_SERVICE_OPTION_NONE,
1607 &run,
1608 &client_connect_cb,
1609 &client_disconnect_cb,
1610 NULL,
1611 GNUNET_MQ_hd_fixed_size (reserve,
1612 GNUNET_MESSAGE_TYPE_DATASTORE_RESERVE,
1613 struct ReserveMessage,
1614 NULL),
1615 GNUNET_MQ_hd_fixed_size (release_reserve,
1616 GNUNET_MESSAGE_TYPE_DATASTORE_RELEASE_RESERVE,
1617 struct ReleaseReserveMessage,
1618 NULL),
1619 GNUNET_MQ_hd_var_size (put,
1620 GNUNET_MESSAGE_TYPE_DATASTORE_PUT,
1621 struct DataMessage,
1622 NULL),
1623 GNUNET_MQ_hd_fixed_size (get,
1624 GNUNET_MESSAGE_TYPE_DATASTORE_GET,
1625 struct GetMessage,
1626 NULL),
1627 GNUNET_MQ_hd_fixed_size (get_key,
1628 GNUNET_MESSAGE_TYPE_DATASTORE_GET_KEY,
1629 struct GetKeyMessage,
1630 NULL),
1631 GNUNET_MQ_hd_fixed_size (get_replication,
1632 GNUNET_MESSAGE_TYPE_DATASTORE_GET_REPLICATION,
1633 struct GNUNET_MessageHeader,
1634 NULL),
1635 GNUNET_MQ_hd_fixed_size (get_zero_anonymity,
1636 GNUNET_MESSAGE_TYPE_DATASTORE_GET_ZERO_ANONYMITY,
1637 struct GetZeroAnonymityMessage,
1638 NULL),
1639 GNUNET_MQ_hd_var_size (remove,
1640 GNUNET_MESSAGE_TYPE_DATASTORE_REMOVE,
1641 struct DataMessage,
1642 NULL),
1643 GNUNET_MQ_hd_fixed_size (drop,
1644 GNUNET_MESSAGE_TYPE_DATASTORE_DROP,
1645 struct GNUNET_MessageHeader,
1646 NULL),
1647 GNUNET_MQ_handler_end ());
1648
1649
1650/* end of gnunet-service-datastore.c */
diff --git a/src/service/datastore/meson.build b/src/service/datastore/meson.build
new file mode 100644
index 000000000..802ebb0a0
--- /dev/null
+++ b/src/service/datastore/meson.build
@@ -0,0 +1,132 @@
1libgnunetdatastore_src = ['datastore_api.c']
2
3gnunetservicedatastore_src = ['gnunet-service-datastore.c']
4
5configure_file(input : 'datastore.conf.in',
6 output : 'datastore.conf',
7 configuration : cdata,
8 install: true,
9 install_dir: pkgcfgdir)
10
11if get_option('monolith')
12 foreach p : libgnunetdatastore_src + gnunetservicedatastore_src
13 gnunet_src += 'datastore/' + p
14 endforeach
15endif
16
17libgnunetdatastore = library('gnunetdatastore',
18 libgnunetdatastore_src,
19 soversion: '1',
20 version: '1.0.0',
21 dependencies: [libgnunetutil_dep,
22 libgnunetstatistics_dep,
23 libgnunetdatacache_dep],
24 include_directories: [incdir, configuration_inc],
25 install: true,
26 install_dir: get_option('libdir'))
27libgnunetdatastore_dep = declare_dependency(link_with : libgnunetdatastore)
28pkg.generate(libgnunetdatastore, url: 'https://www.gnunet.org',
29 description : 'Management API for the datastore for persistent storage to disk')
30
31executable ('gnunet-service-datastore',
32 gnunetservicedatastore_src,
33 dependencies: [libgnunetdatastore_dep,
34 libgnunetutil_dep,
35 libgnunetstatistics_dep,
36 libgnunetdatacache_dep],
37 include_directories: [incdir, configuration_inc],
38 install: true,
39 install_dir: get_option('libdir')/'gnunet'/'libexec')
40
41testds_sqlite = executable ('test_store_api_sqlite',
42 ['test_datastore_api.c'],
43 dependencies: [
44 libgnunetdatastore_dep,
45 libgnunetutil_dep,
46 libgnunettesting_dep
47 ],
48 include_directories: [incdir, configuration_inc],
49 install: false)
50
51testds_mgmt_sqlite = executable ('test_datastore_api_management_sqlite',
52 ['test_datastore_api_management.c'],
53 dependencies: [
54 libgnunetdatastore_dep,
55 libgnunetutil_dep,
56 libgnunettesting_dep
57 ],
58 include_directories: [incdir, configuration_inc],
59 install: false)
60
61testds_heap = executable ('test_datastore_api_heap',
62 ['test_datastore_api.c'],
63 dependencies: [
64 libgnunetdatastore_dep,
65 libgnunetutil_dep,
66 libgnunettesting_dep
67 ],
68 include_directories: [incdir, configuration_inc],
69 install: false)
70
71testds_mgmt_heap = executable ('test_datastore_api_management_heap',
72 ['test_datastore_api_management.c'],
73 dependencies: [
74 libgnunetdatastore_dep,
75 libgnunetutil_dep,
76 libgnunettesting_dep
77 ],
78 include_directories: [incdir, configuration_inc],
79 install: false)
80
81testds_pq = executable ('test_datastore_api_postgres',
82 ['test_datastore_api.c'],
83 dependencies: [
84 libgnunetdatastore_dep,
85 libgnunetutil_dep,
86 libgnunettesting_dep
87 ],
88 include_directories: [incdir, configuration_inc],
89 install: false)
90
91testds_mgmt_pq = executable ('test_datastore_api_management_postgres',
92 ['test_datastore_api_management.c'],
93 dependencies: [
94 libgnunetdatastore_dep,
95 libgnunetutil_dep,
96 libgnunettesting_dep
97 ],
98 include_directories: [incdir, configuration_inc],
99 install: false)
100
101configure_file(input : 'test_defaults.conf',
102 output : 'test_defaults.conf',
103 copy: true)
104configure_file(input : 'test_datastore_api_data_sqlite.conf',
105 output : 'test_datastore_api_data_sqlite.conf',
106 copy: true)
107configure_file(input : 'test_datastore_api_data_heap.conf',
108 output : 'test_datastore_api_data_heap.conf',
109 copy: true)
110configure_file(input : 'test_datastore_api_data_postgres.conf',
111 output : 'test_datastore_api_data_postgres.conf',
112 copy: true)
113
114test('test_datastore_api_sqlite', testds_sqlite,
115 is_parallel: false,
116 suite: 'datastore', workdir: meson.current_build_dir())
117test('test_datastore_api_management_sqlite', testds_mgmt_sqlite,
118 is_parallel: false,
119 suite: 'datastore', workdir: meson.current_build_dir())
120test('test_datastore_api_heap', testds_heap,
121 is_parallel: false,
122 suite: 'datastore', workdir: meson.current_build_dir())
123test('test_datastore_api_management_heap', testds_mgmt_heap,
124 is_parallel: false,
125 suite: 'datastore', workdir: meson.current_build_dir())
126test('test_datastore_api_postgres', testds_pq,
127 is_parallel: false,
128 suite: 'datastore', workdir: meson.current_build_dir())
129test('test_datastore_api_management_postgres', testds_mgmt_pq,
130 is_parallel: false,
131 suite: 'datastore', workdir: meson.current_build_dir())
132
diff --git a/src/service/datastore/perf_datastore_api.c b/src/service/datastore/perf_datastore_api.c
new file mode 100644
index 000000000..3b5488168
--- /dev/null
+++ b/src/service/datastore/perf_datastore_api.c
@@ -0,0 +1,630 @@
1/*
2 This file is part of GNUnet.
3 Copyright (C) 2004, 2005, 2006, 2007, 2009, 2011, 2015 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20/*
21 * @file datastore/perf_datastore_api.c
22 * @brief performance measurement for the datastore implementation
23 * @author Christian Grothoff
24 *
25 * This testcase inserts a bunch of (variable size) data and then
26 * deletes data until the (reported) database size drops below a given
27 * threshold. This is iterated 10 times, with the actual size of the
28 * content stored and the number of operations performed being printed
29 * for each iteration. The code also prints a "I" for every 40 blocks
30 * inserted and a "D" for every 40 blocks deleted. The deletion
31 * strategy uses the "random" iterator. Priorities and expiration
32 * dates are set using a pseudo-random value within a realistic range.
33 */
34#include "platform.h"
35#include "gnunet_util_lib.h"
36#include "gnunet_protocols.h"
37#include "gnunet_datastore_service.h"
38#include "gnunet_testing_lib.h"
39#include <gauger.h>
40
41/**
42 * How long until we give up on transmitting the message?
43 */
44#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 15)
45
46/**
47 * Target datastore size (in bytes).
48 */
49#define MAX_SIZE (1024LL * 1024 * 4)
50
51/**
52 * Report progress outside of major reports? Should probably be #GNUNET_YES if
53 * size is > 16 MB.
54 */
55#define REPORT_ID GNUNET_YES
56
57/**
58 * Number of put operations equivalent to 1/3rd of #MAX_SIZE
59 */
60#define PUT_10 MAX_SIZE / 32 / 1024 / 3
61
62/**
63 * Total number of iterations (each iteration doing
64 * PUT_10 put operations); we report full status every
65 * 10 iterations. Abort with CTRL-C.
66 */
67#define ITERATIONS 8
68
69/**
70 * Total number of iterations to do to go beyond the quota.
71 * The quota is set to 10 MB or 2.5 times #MAX_SIZE,
72 * so we got 16 times #MAX_SIZE to be sure to hit it a LOT.
73 */
74#define QUOTA_PUTS (MAX_SIZE / 32 / 1024 * 16LL)
75
76
77/**
78 * Number of bytes stored in the datastore in total.
79 */
80static unsigned long long stored_bytes;
81
82/**
83 * Number of entries stored in the datastore in total.
84 */
85static unsigned long long stored_entries;
86
87/**
88 * Number of database operations performed. Inserting
89 * counts as one operation, deleting as two (as deletion
90 * requires selecting a value for deletion first).
91 */
92static unsigned long long stored_ops;
93
94/**
95 * Start time of the benchmark.
96 */
97static struct GNUNET_TIME_Absolute start_time;
98
99/**
100 * Database backend we use.
101 */
102static const char *plugin_name;
103
104/**
105 * Handle to the datastore.
106 */
107static struct GNUNET_DATASTORE_Handle *datastore;
108
109/**
110 * Value we return from #main().
111 */
112static int ok;
113
114/**
115 * Which phase of the process are we in?
116 */
117enum RunPhase
118{
119 /**
120 * We are done (shutting down normally).
121 */
122 RP_DONE = 0,
123
124 /**
125 * We are adding new entries to the datastore.
126 */
127 RP_PUT,
128
129 /**
130 * We are deleting entries from the datastore.
131 */
132 RP_CUT,
133
134 /**
135 * We are putting as much as we can to see how the database performs
136 * when it reaches the quota and has to auto-delete (see #3903).
137 */
138 RP_PUT_QUOTA,
139
140 /**
141 * We are generating a report.
142 */
143 RP_REPORT,
144
145 /**
146 * Execution failed with some kind of error.
147 */
148 RP_ERROR
149};
150
151
152/**
153 * Closure we give to all of the functions executing the
154 * benchmark. Could right now be global, but this allows
155 * us to theoretically run multiple clients "in parallel".
156 */
157struct CpsRunContext
158{
159 /**
160 * Execution phase we are in.
161 */
162 enum RunPhase phase;
163
164 /**
165 * Size of the value we are currently storing (during #RP_PUT).
166 */
167 size_t size;
168
169 /**
170 * Current iteration counter, we are done with the benchmark
171 * once it hits #ITERATIONS.
172 */
173 unsigned int i;
174
175 /**
176 * Counts the number of items put in the current phase.
177 * Once it hits #PUT_10, we progress to the #RP_CUT phase
178 * or are done if @e i reaches #ITERATIONS.
179 */
180 unsigned int j;
181};
182
183
184/**
185 * Main state machine. Executes the next step of the benchmark
186 * depending on the current state.
187 *
188 * @param cls the `struct CpsRunContext`
189 */
190static void
191run_continuation (void *cls);
192
193
194/**
195 * Continuation called to notify client about result of the insertion
196 * operation. Checks for errors, updates our iteration counters and
197 * continues execution with #run_continuation().
198 *
199 * @param cls the `struct CpsRunContext`
200 * @param success #GNUNET_SYSERR on failure
201 * @param min_expiration minimum expiration time required for content to be stored
202 * by the datacache at this time, zero for unknown
203 * @param msg NULL on success, otherwise an error message
204 */
205static void
206check_success (void *cls,
207 int success,
208 struct GNUNET_TIME_Absolute min_expiration,
209 const char *msg)
210{
211 struct CpsRunContext *crc = cls;
212
213#if REPORT_ID
214 fprintf (stderr, "%s", (GNUNET_OK == success) ? "I" : "i");
215#endif
216 if (GNUNET_OK != success)
217 {
218 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
219 "Check success failed: `%s'\n",
220 msg);
221 crc->phase = RP_ERROR;
222 GNUNET_SCHEDULER_add_now (&run_continuation,
223 crc);
224 return;
225 }
226 stored_bytes += crc->size;
227 stored_ops++;
228 stored_entries++;
229 crc->j++;
230 switch (crc->phase)
231 {
232 case RP_PUT:
233 if (crc->j >= PUT_10)
234 {
235 crc->j = 0;
236 crc->i++;
237 if (crc->i == ITERATIONS)
238 crc->phase = RP_PUT_QUOTA;
239 else
240 crc->phase = RP_CUT;
241 }
242 break;
243
244 case RP_PUT_QUOTA:
245 if (crc->j >= QUOTA_PUTS)
246 {
247 crc->j = 0;
248 crc->phase = RP_DONE;
249 }
250 break;
251
252 default:
253 GNUNET_assert (0);
254 }
255 GNUNET_SCHEDULER_add_now (&run_continuation,
256 crc);
257}
258
259
260/**
261 * Continuation called to notify client about result of the
262 * deletion operation. Checks for errors and continues
263 * execution with #run_continuation().
264 *
265 * @param cls the `struct CpsRunContext`
266 * @param success #GNUNET_SYSERR on failure
267 * @param min_expiration minimum expiration time required for content to be stored
268 * by the datacache at this time, zero for unknown
269 * @param msg NULL on success, otherwise an error message
270 */
271static void
272remove_next (void *cls,
273 int success,
274 struct GNUNET_TIME_Absolute min_expiration,
275 const char *msg)
276{
277 struct CpsRunContext *crc = cls;
278
279 if (GNUNET_OK != success)
280 {
281 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
282 "remove_next failed: `%s'\n",
283 msg);
284 crc->phase = RP_ERROR;
285 GNUNET_SCHEDULER_add_now (&run_continuation,
286 crc);
287 return;
288 }
289#if REPORT_ID
290 fprintf (stderr, "%s", "D");
291#endif
292 GNUNET_assert (GNUNET_OK == success);
293 GNUNET_SCHEDULER_add_now (&run_continuation,
294 crc);
295}
296
297
298/**
299 * We have selected a value for deletion, trigger removal.
300 *
301 * @param cls the `struct CpsRunContext`
302 * @param key key for the content
303 * @param size number of bytes in data
304 * @param data content stored
305 * @param type type of the content
306 * @param priority priority of the content
307 * @param anonymity anonymity-level for the content
308 * @param replication replication-level for the content
309 * @param expiration expiration time for the content
310 * @param uid unique identifier for the datum;
311 * maybe 0 if no unique identifier is available
312 */
313static void
314delete_value (void *cls,
315 const struct GNUNET_HashCode *key,
316 size_t size,
317 const void *data,
318 enum GNUNET_BLOCK_Type type,
319 uint32_t priority,
320 uint32_t anonymity,
321 uint32_t replication,
322 struct GNUNET_TIME_Absolute expiration,
323 uint64_t uid)
324{
325 struct CpsRunContext *crc = cls;
326
327 GNUNET_assert (NULL != key);
328 stored_ops++;
329 stored_bytes -= size;
330 stored_entries--;
331 stored_ops++;
332 if (stored_bytes < MAX_SIZE)
333 crc->phase = RP_PUT;
334 GNUNET_assert (NULL !=
335 GNUNET_DATASTORE_remove (datastore,
336 key,
337 size,
338 data, 1, 1,
339 &remove_next, crc));
340}
341
342
343/**
344 * Main state machine. Executes the next step of the benchmark
345 * depending on the current state.
346 *
347 * @param cls the `struct CpsRunContext`
348 */
349static void
350run_continuation (void *cls)
351{
352 struct CpsRunContext *crc = cls;
353 size_t size;
354 static struct GNUNET_HashCode key;
355 static char data[65536];
356 char gstr[128];
357
358 ok = (int) crc->phase;
359 switch (crc->phase)
360 {
361 case RP_PUT:
362 memset (&key,
363 256 - crc->i,
364 sizeof(struct GNUNET_HashCode));
365 /* most content is 32k */
366 size = 32 * 1024;
367 if (0 ==
368 GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
369 16)) /* but some of it is less! */
370 size = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
371 32 * 1024);
372 crc->size = size = size - (size & 7); /* always multiple of 8 */
373 GNUNET_CRYPTO_hash (&key,
374 sizeof(struct GNUNET_HashCode),
375 &key);
376 memset (data,
377 (int) crc->j,
378 size);
379 if (crc->j > 255)
380 memset (data,
381 (int) (crc->j - 255),
382 size / 2);
383 data[0] = crc->i;
384 GNUNET_assert (NULL !=
385 GNUNET_DATASTORE_put (datastore,
386 0,
387 &key,
388 size,
389 data,
390 crc->j + 1,
391 GNUNET_CRYPTO_random_u32
392 (GNUNET_CRYPTO_QUALITY_WEAK, 100),
393 crc->j,
394 0,
395 GNUNET_TIME_relative_to_absolute
396 (GNUNET_TIME_relative_multiply
397 (GNUNET_TIME_UNIT_SECONDS,
398 GNUNET_CRYPTO_random_u32
399 (GNUNET_CRYPTO_QUALITY_WEAK,
400 1000))),
401 1,
402 1,
403 &check_success, crc));
404 break;
405
406 case RP_CUT:
407 /* trim down below MAX_SIZE again */
408 GNUNET_assert (NULL !=
409 GNUNET_DATASTORE_get_for_replication (datastore,
410 1, 1,
411 &delete_value,
412 crc));
413 break;
414
415 case RP_REPORT:
416 printf (
417#if REPORT_ID
418 "\n"
419#endif
420 "Stored %llu kB / %lluk ops / %llu ops/s\n",
421 stored_bytes / 1024, /* used size in k */
422 stored_ops / 1024, /* total operations (in k) */
423 1000LL * 1000LL * stored_ops / (1
424 + GNUNET_TIME_absolute_get_duration
425 (start_time).rel_value_us));
426 crc->phase = RP_PUT;
427 crc->j = 0;
428 GNUNET_SCHEDULER_add_now (&run_continuation,
429 crc);
430 break;
431
432 case RP_PUT_QUOTA:
433 memset (&key,
434 256 - crc->i,
435 sizeof(struct GNUNET_HashCode));
436 /* most content is 32k */
437 size = 32 * 1024;
438 if (0 ==
439 GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
440 16)) /* but some of it is less! */
441 size = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
442 32 * 1024);
443 crc->size = size = size - (size & 7); /* always multiple of 8 */
444 GNUNET_CRYPTO_hash (&key,
445 sizeof(struct GNUNET_HashCode),
446 &key);
447 memset (data,
448 (int) crc->j,
449 size);
450 if (crc->j > 255)
451 memset (data,
452 (int) (crc->j - 255),
453 size / 2);
454 data[0] = crc->i;
455 GNUNET_assert (NULL !=
456 GNUNET_DATASTORE_put (datastore,
457 0, /* reservation ID */
458 &key,
459 size,
460 data,
461 crc->j + 1, /* type */
462 GNUNET_CRYPTO_random_u32
463 (GNUNET_CRYPTO_QUALITY_WEAK,
464 100), /* priority */
465 crc->j, /* anonymity */
466 0, /* replication */
467 GNUNET_TIME_relative_to_absolute
468 (GNUNET_TIME_relative_multiply
469 (GNUNET_TIME_UNIT_SECONDS,
470 GNUNET_CRYPTO_random_u32
471 (GNUNET_CRYPTO_QUALITY_WEAK,
472 1000))),
473 1,
474 1,
475 &check_success, crc));
476 break;
477
478 case RP_DONE:
479 GNUNET_snprintf (gstr,
480 sizeof(gstr),
481 "DATASTORE-%s",
482 plugin_name);
483 if ((crc->i == ITERATIONS) && (stored_ops > 0))
484 {
485 GAUGER (gstr,
486 "PUT operation duration",
487 GNUNET_TIME_absolute_get_duration (start_time).rel_value_us
488 / 1000LL
489 / stored_ops,
490 "ms/operation");
491 fprintf (stdout,
492 "\nPUT performance: %s for %llu operations\n",
493 GNUNET_STRINGS_relative_time_to_string (
494 GNUNET_TIME_absolute_get_duration (start_time),
495 GNUNET_YES),
496 stored_ops);
497 fprintf (stdout,
498 "PUT performance: %llu ms/operation\n",
499 GNUNET_TIME_absolute_get_duration (start_time).rel_value_us
500 / 1000LL
501 / stored_ops);
502 }
503 GNUNET_DATASTORE_disconnect (datastore,
504 GNUNET_YES);
505 GNUNET_free (crc);
506 ok = 0;
507 break;
508
509 case RP_ERROR:
510 GNUNET_DATASTORE_disconnect (datastore,
511 GNUNET_YES);
512 GNUNET_free (crc);
513 ok = 1;
514 break;
515
516 default:
517 GNUNET_assert (0);
518 }
519}
520
521
522/**
523 * Function called with the result of the initial PUT operation. If
524 * the PUT succeeded, we start the actual benchmark loop, otherwise we
525 * bail out with an error.
526 *
527 *
528 * @param cls closure
529 * @param success #GNUNET_SYSERR on failure
530 * @param min_expiration minimum expiration time required for content to be stored
531 * by the datacache at this time, zero for unknown
532 * @param msg NULL on success, otherwise an error message
533 */
534static void
535run_tests (void *cls,
536 int success,
537 struct GNUNET_TIME_Absolute min_expiration,
538 const char *msg)
539{
540 struct CpsRunContext *crc = cls;
541
542 if (success != GNUNET_YES)
543 {
544 fprintf (stderr,
545 "Test 'put' operation failed with error `%s' database likely not setup, skipping test.\n",
546 msg);
547 GNUNET_DATASTORE_disconnect (datastore,
548 GNUNET_YES);
549 GNUNET_free (crc);
550 return;
551 }
552 GNUNET_SCHEDULER_add_now (&run_continuation,
553 crc);
554}
555
556
557/**
558 * Beginning of the actual execution of the benchmark.
559 * Performs a first test operation (PUT) to verify that
560 * the plugin works at all.
561 *
562 * @param cls NULL
563 * @param cfg configuration to use
564 * @param peer peer handle (unused)
565 */
566static void
567run (void *cls,
568 const struct GNUNET_CONFIGURATION_Handle *cfg,
569 struct GNUNET_TESTING_Peer *peer)
570{
571 struct CpsRunContext *crc;
572 static struct GNUNET_HashCode zkey;
573
574 datastore = GNUNET_DATASTORE_connect (cfg);
575 start_time = GNUNET_TIME_absolute_get ();
576 crc = GNUNET_new (struct CpsRunContext);
577 crc->phase = RP_PUT;
578 if (NULL ==
579 GNUNET_DATASTORE_put (datastore,
580 0,
581 &zkey,
582 4, "TEST",
583 GNUNET_BLOCK_TYPE_TEST,
584 0, 0, 0,
585 GNUNET_TIME_relative_to_absolute (
586 GNUNET_TIME_UNIT_SECONDS),
587 0, 1,
588 &run_tests, crc))
589 {
590 fprintf (stderr,
591 "%s",
592 "Test 'put' operation failed.\n");
593 ok = 1;
594 GNUNET_free (crc);
595 }
596}
597
598
599/**
600 * Entry point into the test. Determines which configuration / plugin
601 * we are running with based on the name of the binary and starts
602 * the peer.
603 *
604 * @param argc should be 1
605 * @param argv used to determine plugin / configuration name.
606 * @return 0 on success
607 */
608int
609main (int argc,
610 char *argv[])
611{
612 char cfg_name[PATH_MAX];
613
614 plugin_name = GNUNET_STRINGS_get_suffix_from_binary_name (argv[0]);
615 GNUNET_snprintf (cfg_name,
616 sizeof(cfg_name),
617 "test_datastore_api_data_%s.conf",
618 plugin_name);
619 if (0 !=
620 GNUNET_TESTING_peer_run ("perf-gnunet-datastore",
621 cfg_name,
622 &run,
623 NULL))
624 return 1;
625 fprintf (stderr, "%s", "\n");
626 return ok;
627}
628
629
630/* end of perf_datastore_api.c */
diff --git a/src/service/datastore/selectrandom.sql b/src/service/datastore/selectrandom.sql
new file mode 100644
index 000000000..82830a13a
--- /dev/null
+++ b/src/service/datastore/selectrandom.sql
@@ -0,0 +1,9 @@
1select *
2from (select random() as v from (values(1))) t1,
3 (select max(repl) as m from data) t2,
4 (select * from data
5 where repl=t2.m and
6 rnd>=t.v
7 order by rnd
8 limit 1)
9
diff --git a/src/service/datastore/test_datastore_api.c b/src/service/datastore/test_datastore_api.c
new file mode 100644
index 000000000..58a6b7a28
--- /dev/null
+++ b/src/service/datastore/test_datastore_api.c
@@ -0,0 +1,732 @@
1/*
2 This file is part of GNUnet.
3 Copyright (C) 2004, 2005, 2006, 2007, 2009, 2015 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20/*
21 * @file datastore/test_datastore_api.c
22 * @brief Test for the basic datastore API.
23 * @author Christian Grothoff
24 *
25 * TODO:
26 * - test reservation failure
27 */
28
29#include "platform.h"
30#include "gnunet_util_lib.h"
31#include "gnunet_protocols.h"
32#include "gnunet_datastore_service.h"
33#include "gnunet_datastore_plugin.h"
34#include "gnunet_testing_lib.h"
35
36
37/**
38 * How long until we give up on transmitting the message?
39 */
40#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 15)
41
42#define ITERATIONS 256
43
44/**
45 * Handle to the datastore.
46 */
47static struct GNUNET_DATASTORE_Handle *datastore;
48
49static struct GNUNET_TIME_Absolute now;
50
51/**
52 * Value we return from #main().
53 */
54static int ok;
55
56/**
57 * Name of plugin under test.
58 */
59static const char *plugin_name;
60
61
62static size_t
63get_size (int i)
64{
65 return 8 * i;
66}
67
68
69static const void *
70get_data (int i)
71{
72 static char buf[60000];
73
74 memset (buf, i, 8 * i);
75 return buf;
76}
77
78
79static int
80get_type (int i)
81{
82 return i + 1;
83}
84
85
86static int
87get_priority (int i)
88{
89 return i + 1;
90}
91
92
93static int
94get_anonymity (int i)
95{
96 return i;
97}
98
99
100static struct GNUNET_TIME_Absolute
101get_expiration (int i)
102{
103 struct GNUNET_TIME_Absolute av;
104
105 av.abs_value_us = now.abs_value_us + 20000000000LL - i * 1000 * 1000LL;
106 return av;
107}
108
109
110/**
111 * Which phase of the process are we in?
112 */
113enum RunPhase
114{
115 /**
116 * We are done (shutting down normally).
117 */
118 RP_DONE = 0,
119
120 /**
121 * We are adding new entries to the datastore.
122 */
123 RP_PUT = 1,
124 RP_GET = 2,
125 RP_DEL = 3,
126 RP_DO_DEL = 4,
127 RP_DELVALIDATE = 5,
128 RP_RESERVE = 6,
129 RP_PUT_MULTIPLE = 7,
130 RP_PUT_MULTIPLE_NEXT = 8,
131 RP_GET_MULTIPLE = 9,
132 RP_GET_MULTIPLE_NEXT = 10,
133
134 /**
135 * Execution failed with some kind of error.
136 */
137 RP_ERROR
138};
139
140
141/**
142 * Closure we give to all of the functions executing the
143 * benchmark. Could right now be global, but this allows
144 * us to theoretically run multiple clients "in parallel".
145 */
146struct CpsRunContext
147{
148 /**
149 * Execution phase we are in.
150 */
151 enum RunPhase phase;
152
153 struct GNUNET_HashCode key;
154 int i;
155 int rid;
156 void *data;
157 size_t size;
158
159 uint64_t first_uid;
160};
161
162
163/**
164 * Main state machine. Executes the next step of the test
165 * depending on the current state.
166 *
167 * @param cls the `struct CpsRunContext`
168 */
169static void
170run_continuation (void *cls);
171
172
173/**
174 * Continuation called to notify client about result of an
175 * operation. Checks for errors, updates our iteration counters and
176 * continues execution with #run_continuation().
177 *
178 * @param cls the `struct CpsRunContext`
179 * @param success #GNUNET_SYSERR on failure
180 * @param min_expiration minimum expiration time required for content to be stored
181 * by the datacache at this time, zero for unknown
182 * @param msg NULL on success, otherwise an error message
183 */
184static void
185check_success (void *cls,
186 int success,
187 struct GNUNET_TIME_Absolute min_expiration,
188 const char *msg)
189{
190 struct CpsRunContext *crc = cls;
191
192 if (GNUNET_OK != success)
193 {
194 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
195 "Operation %d/%d not successful: `%s'\n",
196 crc->phase,
197 crc->i,
198 msg);
199 crc->phase = RP_ERROR;
200 }
201 GNUNET_free (crc->data);
202 crc->data = NULL;
203 GNUNET_SCHEDULER_add_now (&run_continuation, crc);
204}
205
206
207static void
208get_reserved (void *cls,
209 int success,
210 struct GNUNET_TIME_Absolute min_expiration,
211 const char *msg)
212{
213 struct CpsRunContext *crc = cls;
214
215 if (0 >= success)
216 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
217 "Error obtaining reservation: `%s'\n",
218 msg);
219 GNUNET_assert (0 < success);
220 crc->rid = success;
221 GNUNET_SCHEDULER_add_now (&run_continuation,
222 crc);
223}
224
225
226static void
227check_value (void *cls,
228 const struct GNUNET_HashCode *key,
229 size_t size,
230 const void *data,
231 enum GNUNET_BLOCK_Type type,
232 uint32_t priority,
233 uint32_t anonymity,
234 uint32_t replication,
235 struct GNUNET_TIME_Absolute expiration,
236 uint64_t uid)
237{
238 struct CpsRunContext *crc = cls;
239 int i;
240
241 i = crc->i;
242 if (NULL == key)
243 {
244 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
245 "Value check failed (got NULL key) in %d/%d\n",
246 crc->phase,
247 crc->i);
248 crc->phase = RP_ERROR;
249 GNUNET_SCHEDULER_add_now (&run_continuation,
250 crc);
251 return;
252 }
253#if 0
254 fprintf (stderr,
255 "Check value got `%s' of size %u, type %d, expire %s\n",
256 GNUNET_h2s (key), (unsigned int) size, type,
257 GNUNET_STRINGS_absolute_time_to_string (expiration));
258 fprintf (stderr,
259 "Check value iteration %d wants size %u, type %d, expire %s\n", i,
260 (unsigned int) get_size (i), get_type (i),
261 GNUNET_STRINGS_absolute_time_to_string (get_expiration (i)));
262#endif
263 GNUNET_assert (size == get_size (i));
264 GNUNET_assert (0 == memcmp (data, get_data (i), size));
265 GNUNET_assert (type == get_type (i));
266 GNUNET_assert (priority == get_priority (i));
267 GNUNET_assert (anonymity == get_anonymity (i));
268 GNUNET_assert (expiration.abs_value_us == get_expiration (i).abs_value_us);
269 if (crc->i == 0)
270 {
271 crc->phase = RP_DEL;
272 crc->i = ITERATIONS;
273 }
274 GNUNET_SCHEDULER_add_now (&run_continuation,
275 crc);
276}
277
278
279static void
280delete_value (void *cls,
281 const struct GNUNET_HashCode *key,
282 size_t size,
283 const void *data,
284 enum GNUNET_BLOCK_Type type,
285 uint32_t priority,
286 uint32_t anonymity,
287 uint32_t replication,
288 struct GNUNET_TIME_Absolute expiration,
289 uint64_t uid)
290{
291 struct CpsRunContext *crc = cls;
292
293 GNUNET_assert (NULL == crc->data);
294 GNUNET_assert (NULL != key);
295 crc->size = size;
296 crc->key = *key;
297 crc->data = GNUNET_malloc (size);
298 GNUNET_memcpy (crc->data, data, size);
299 crc->phase = RP_DO_DEL;
300 GNUNET_SCHEDULER_add_now (&run_continuation,
301 crc);
302}
303
304
305static void
306check_nothing (void *cls,
307 const struct GNUNET_HashCode *key,
308 size_t size,
309 const void *data,
310 enum GNUNET_BLOCK_Type type,
311 uint32_t priority,
312 uint32_t anonymity,
313 uint32_t replication,
314 struct GNUNET_TIME_Absolute expiration,
315 uint64_t uid)
316{
317 struct CpsRunContext *crc = cls;
318
319 GNUNET_assert (key == NULL);
320 if (crc->i == 0)
321 crc->phase = RP_RESERVE;
322 GNUNET_SCHEDULER_add_now (&run_continuation,
323 crc);
324}
325
326
327static void
328check_multiple (void *cls,
329 const struct GNUNET_HashCode *key,
330 size_t size,
331 const void *data,
332 enum GNUNET_BLOCK_Type type,
333 uint32_t priority,
334 uint32_t anonymity,
335 uint32_t replication,
336 struct GNUNET_TIME_Absolute expiration,
337 uint64_t uid)
338{
339 struct CpsRunContext *crc = cls;
340
341 GNUNET_assert (key != NULL);
342 switch (crc->phase)
343 {
344 case RP_GET_MULTIPLE:
345 crc->phase = RP_GET_MULTIPLE_NEXT;
346 crc->first_uid = uid;
347 break;
348
349 case RP_GET_MULTIPLE_NEXT:
350 GNUNET_assert (uid != crc->first_uid);
351 crc->phase = RP_DONE;
352 break;
353
354 default:
355 GNUNET_break (0);
356 crc->phase = RP_ERROR;
357 break;
358 }
359 GNUNET_SCHEDULER_add_now (&run_continuation, crc);
360}
361
362
363/**
364 * Main state machine. Executes the next step of the test
365 * depending on the current state.
366 *
367 * @param cls the `struct CpsRunContext`
368 */
369static void
370run_continuation (void *cls)
371{
372 struct CpsRunContext *crc = cls;
373
374 ok = (int) crc->phase;
375 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
376 "Test in phase %u\n",
377 crc->phase);
378 switch (crc->phase)
379 {
380 case RP_PUT:
381 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
382 "Executing PUT number %u\n",
383 crc->i);
384 GNUNET_CRYPTO_hash (&crc->i, sizeof(int), &crc->key);
385 GNUNET_DATASTORE_put (datastore, 0, &crc->key, get_size (crc->i),
386 get_data (crc->i), get_type (crc->i),
387 get_priority (crc->i), get_anonymity (crc->i), 0,
388 get_expiration (crc->i), 1, 1,
389 &check_success, crc);
390 crc->i++;
391 if (crc->i == ITERATIONS)
392 crc->phase = RP_GET;
393 break;
394
395 case RP_GET:
396 crc->i--;
397 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
398 "Executing GET number %u\n",
399 crc->i);
400 GNUNET_CRYPTO_hash (&crc->i,
401 sizeof(int),
402 &crc->key);
403 GNUNET_DATASTORE_get_key (datastore,
404 0,
405 false,
406 &crc->key,
407 get_type (crc->i),
408 1,
409 1,
410 &check_value,
411 crc);
412 break;
413
414 case RP_DEL:
415 crc->i--;
416 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
417 "Executing DEL number %u\n",
418 crc->i);
419 crc->data = NULL;
420 GNUNET_CRYPTO_hash (&crc->i, sizeof(int), &crc->key);
421 GNUNET_assert (NULL !=
422 GNUNET_DATASTORE_get_key (datastore,
423 0,
424 false,
425 &crc->key,
426 get_type (crc->i),
427 1,
428 1,
429 &delete_value,
430 crc));
431 break;
432
433 case RP_DO_DEL:
434 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
435 "Executing DO_DEL number %u\n",
436 crc->i);
437 if (crc->i == 0)
438 {
439 crc->i = ITERATIONS;
440 crc->phase = RP_DELVALIDATE;
441 }
442 else
443 {
444 crc->phase = RP_DEL;
445 }
446 GNUNET_assert (NULL !=
447 GNUNET_DATASTORE_remove (datastore, &crc->key, crc->size,
448 crc->data, 1, 1,
449 &check_success, crc));
450 break;
451
452 case RP_DELVALIDATE:
453 crc->i--;
454 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
455 "Executing DELVALIDATE number %u\n",
456 crc->i);
457 GNUNET_CRYPTO_hash (&crc->i, sizeof(int), &crc->key);
458 GNUNET_assert (NULL !=
459 GNUNET_DATASTORE_get_key (datastore,
460 0,
461 false,
462 &crc->key,
463 get_type (crc->i),
464 1,
465 1,
466 &check_nothing,
467 crc));
468 break;
469
470 case RP_RESERVE:
471 crc->phase = RP_PUT_MULTIPLE;
472 GNUNET_DATASTORE_reserve (datastore, 128 * 1024, 2,
473 &get_reserved, crc);
474 break;
475
476 case RP_PUT_MULTIPLE:
477 crc->phase = RP_PUT_MULTIPLE_NEXT;
478 GNUNET_DATASTORE_put (datastore, crc->rid, &crc->key, get_size (42),
479 get_data (42), get_type (42), get_priority (42),
480 get_anonymity (42), 0, get_expiration (42), 1, 1,
481 &check_success, crc);
482 break;
483
484 case RP_PUT_MULTIPLE_NEXT:
485 crc->phase = RP_GET_MULTIPLE;
486 GNUNET_DATASTORE_put (datastore, crc->rid,
487 &crc->key,
488 get_size (43),
489 get_data (43),
490 get_type (42),
491 get_priority (43),
492 get_anonymity (43),
493 0,
494 get_expiration (43),
495 1, 1,
496 &check_success, crc);
497 break;
498
499 case RP_GET_MULTIPLE:
500 GNUNET_assert (NULL !=
501 GNUNET_DATASTORE_get_key (datastore,
502 0,
503 false,
504 &crc->key,
505 get_type (42),
506 1,
507 1,
508 &check_multiple,
509 crc));
510 break;
511
512 case RP_GET_MULTIPLE_NEXT:
513 GNUNET_assert (NULL !=
514 GNUNET_DATASTORE_get_key (datastore,
515 crc->first_uid + 1,
516 false,
517 &crc->key,
518 get_type (42),
519 1,
520 1,
521 &check_multiple,
522 crc));
523 break;
524
525 case RP_DONE:
526 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
527 "Finished, disconnecting\n");
528 GNUNET_DATASTORE_disconnect (datastore,
529 GNUNET_YES);
530 GNUNET_free (crc);
531 ok = 0;
532 break;
533
534 case RP_ERROR:
535 GNUNET_DATASTORE_disconnect (datastore,
536 GNUNET_YES);
537 GNUNET_free (crc);
538 ok = 43;
539 break;
540 }
541}
542
543
544/**
545 * Function called with the result of the initial PUT operation. If
546 * the PUT succeeded, we start the actual benchmark loop, otherwise we
547 * bail out with an error.
548 *
549 *
550 * @param cls closure
551 * @param success #GNUNET_SYSERR on failure
552 * @param min_expiration minimum expiration time required for content to be stored
553 * by the datacache at this time, zero for unknown
554 * @param msg NULL on success, otherwise an error message
555 */
556static void
557run_tests (void *cls,
558 int32_t success,
559 struct GNUNET_TIME_Absolute min_expiration,
560 const char *msg)
561{
562 struct CpsRunContext *crc = cls;
563
564 switch (success)
565 {
566 case GNUNET_YES:
567 GNUNET_SCHEDULER_add_now (&run_continuation,
568 crc);
569 return;
570
571 case GNUNET_NO:
572 fprintf (stderr,
573 "%s", "Test 'put' operation failed, key already exists (!?)\n");
574 GNUNET_DATASTORE_disconnect (datastore,
575 GNUNET_YES);
576 GNUNET_free (crc);
577 return;
578
579 case GNUNET_SYSERR:
580 fprintf (stderr,
581 "Test 'put' operation failed with error `%s' database likely not setup, skipping test.\n",
582 msg);
583 GNUNET_DATASTORE_disconnect (datastore,
584 GNUNET_YES);
585 GNUNET_free (crc);
586 return;
587
588 default:
589 GNUNET_assert (0);
590 }
591}
592
593
594/**
595 * Beginning of the actual execution of the benchmark.
596 * Performs a first test operation (PUT) to verify that
597 * the plugin works at all.
598 *
599 * @param cls NULL
600 * @param cfg configuration to use
601 * @param peer peer handle (unused)
602 */
603static void
604run (void *cls,
605 const struct GNUNET_CONFIGURATION_Handle *cfg,
606 struct GNUNET_TESTING_Peer *peer)
607{
608 struct CpsRunContext *crc;
609 static struct GNUNET_HashCode zkey;
610
611 crc = GNUNET_new (struct CpsRunContext);
612 crc->phase = RP_PUT;
613 now = GNUNET_TIME_absolute_get ();
614 datastore = GNUNET_DATASTORE_connect (cfg);
615 if (NULL ==
616 GNUNET_DATASTORE_put (datastore,
617 0,
618 &zkey,
619 4,
620 "TEST",
621 GNUNET_BLOCK_TYPE_TEST,
622 0, 0, 0,
623 GNUNET_TIME_relative_to_absolute
624 (GNUNET_TIME_UNIT_SECONDS),
625 0, 1,
626 &run_tests, crc))
627 {
628 fprintf (stderr,
629 "%s",
630 "Test 'put' operation failed.\n");
631 ok = 1;
632 GNUNET_free (crc);
633 }
634}
635
636
637/**
638 * Function invoked to notify service of disk utilization
639 * changes.
640 *
641 * @param cls closure
642 * @param delta change in disk utilization,
643 * 0 for "reset to empty"
644 */
645static void
646duc_dummy (void *cls,
647 int delta)
648{
649 /* intentionally empty */
650}
651
652
653/**
654 * check if plugin is actually working
655 */
656static int
657test_plugin (const char *cfg_name)
658{
659 char libname[128];
660 struct GNUNET_CONFIGURATION_Handle *cfg;
661 struct GNUNET_DATASTORE_PluginFunctions *api;
662 struct GNUNET_DATASTORE_PluginEnvironment env;
663
664 cfg = GNUNET_CONFIGURATION_create ();
665 if (GNUNET_OK !=
666 GNUNET_CONFIGURATION_load (cfg,
667 cfg_name))
668 {
669 GNUNET_CONFIGURATION_destroy (cfg);
670 fprintf (stderr,
671 "Failed to load configuration %s\n",
672 cfg_name);
673 return 1;
674 }
675 memset (&env, 0, sizeof(env));
676 env.cfg = cfg;
677 env.duc = &duc_dummy;
678 GNUNET_snprintf (libname,
679 sizeof(libname),
680 "libgnunet_plugin_datastore_%s",
681 plugin_name);
682 api = GNUNET_PLUGIN_load (libname, &env);
683 if (NULL == api)
684 {
685 GNUNET_CONFIGURATION_destroy (cfg);
686 fprintf (stderr,
687 "Failed to load plugin `%s'\n",
688 libname);
689 return 77;
690 }
691 GNUNET_PLUGIN_unload (libname, api);
692 GNUNET_CONFIGURATION_destroy (cfg);
693 return 0;
694}
695
696
697/**
698 * Entry point into the test. Determines which configuration / plugin
699 * we are running with based on the name of the binary and starts
700 * the peer.
701 *
702 * @param argc should be 1
703 * @param argv used to determine plugin / configuration name.
704 * @return 0 on success
705 */
706int
707main (int argc,
708 char *argv[])
709{
710 char cfg_name[PATH_MAX];
711 int ret;
712
713 plugin_name = GNUNET_STRINGS_get_suffix_from_binary_name (argv[0]);
714 GNUNET_snprintf (cfg_name,
715 sizeof(cfg_name),
716 "test_datastore_api_data_%s.conf",
717 plugin_name);
718 ret = test_plugin (cfg_name);
719 if (0 != ret)
720 return ret;
721 /* run actual test */
722 if (0 !=
723 GNUNET_TESTING_peer_run ("test-gnunet-datastore",
724 cfg_name,
725 &run,
726 NULL))
727 return 1;
728 return ok;
729}
730
731
732/* end of test_datastore_api.c */
diff --git a/src/service/datastore/test_datastore_api_data_heap.conf b/src/service/datastore/test_datastore_api_data_heap.conf
new file mode 100644
index 000000000..6f11fb3f7
--- /dev/null
+++ b/src/service/datastore/test_datastore_api_data_heap.conf
@@ -0,0 +1,19 @@
1@INLINE@ test_defaults.conf
2[PATHS]
3GNUNET_TEST_HOME = $GNUNET_TMP/test-gnunet-datastore-heap/
4
5[TESTING]
6WEAKRANDOM = YES
7
8[arm]
9PORT = 42466
10
11[statistics]
12PORT = 22667
13
14[resolver]
15PORT = 42464
16
17[datastore]
18QUOTA = 10 MB
19DATABASE = heap
diff --git a/src/service/datastore/test_datastore_api_data_postgres.conf b/src/service/datastore/test_datastore_api_data_postgres.conf
new file mode 100644
index 000000000..65fe11806
--- /dev/null
+++ b/src/service/datastore/test_datastore_api_data_postgres.conf
@@ -0,0 +1,10 @@
1@INLINE@ test_defaults.conf
2[PATHS]
3GNUNET_TEST_HOME = $GNUNET_TMP/test-gnunet-datastore-postgres/
4
5[datastore]
6QUOTA = 10 MB
7DATABASE = postgres
8
9[datastore-postgres]
10CONFIG = dbname=gnunetcheck
diff --git a/src/service/datastore/test_datastore_api_data_sqlite.conf b/src/service/datastore/test_datastore_api_data_sqlite.conf
new file mode 100644
index 000000000..ecdd0c6ee
--- /dev/null
+++ b/src/service/datastore/test_datastore_api_data_sqlite.conf
@@ -0,0 +1,7 @@
1@INLINE@ test_defaults.conf
2[PATHS]
3GNUNET_TEST_HOME = $GNUNET_TMP/test-gnunet-datastore-sqlite/
4
5[datastore]
6QUOTA = 10 MB
7DATABASE = sqlite
diff --git a/src/service/datastore/test_datastore_api_management.c b/src/service/datastore/test_datastore_api_management.c
new file mode 100644
index 000000000..de99d757d
--- /dev/null
+++ b/src/service/datastore/test_datastore_api_management.c
@@ -0,0 +1,413 @@
1/*
2 This file is part of GNUnet.
3 Copyright (C) 2004, 2005, 2006, 2007, 2009, 2011 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20/*
21 * @file datastore/test_datastore_api_management.c
22 * @brief Test for the space management functions of the datastore implementation.
23 * @author Christian Grothoff
24 */
25#include "platform.h"
26#include "gnunet_util_lib.h"
27#include "gnunet_protocols.h"
28#include "gnunet_datastore_service.h"
29#include "gnunet_datastore_plugin.h"
30#include "gnunet_testing_lib.h"
31
32
33/**
34 * How long until we give up on transmitting the message?
35 */
36#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 60)
37
38/**
39 * Number of iterations to run; must be large enough
40 * so that the quota will be exceeded!
41 */
42#define ITERATIONS 5000
43
44enum RunPhase
45{
46 RP_PUT,
47 RP_GET,
48 RP_DONE,
49 RP_GET_FAIL
50};
51
52
53struct CpsRunContext
54{
55 struct GNUNET_HashCode key;
56 int i;
57 int found;
58 const struct GNUNET_CONFIGURATION_Handle *cfg;
59 void *data;
60 enum RunPhase phase;
61};
62
63
64static struct GNUNET_DATASTORE_Handle *datastore;
65
66static struct GNUNET_TIME_Absolute now;
67
68static int ok;
69
70static const char *plugin_name;
71
72
73static size_t
74get_size (int i)
75{
76 return 8 + 8 * (i % 256);
77}
78
79
80static const void *
81get_data (int i)
82{
83 static char buf[60000];
84
85 memset (buf, i, 8 + 8 * (i % 256));
86 return buf;
87}
88
89
90static int
91get_type (int i)
92{
93 return 1;
94}
95
96
97static int
98get_priority (int i)
99{
100 return i + 1;
101}
102
103
104static int
105get_anonymity (int i)
106{
107 return i;
108}
109
110
111static struct GNUNET_TIME_Absolute
112get_expiration (int i)
113{
114 struct GNUNET_TIME_Absolute av;
115
116 av.abs_value_us = now.abs_value_us + i * 1000 * 1000LL;
117 return av;
118}
119
120
121static void
122run_continuation (void *cls);
123
124
125static void
126check_success (void *cls, int success, struct GNUNET_TIME_Absolute
127 min_expiration, const char *msg)
128{
129 struct CpsRunContext *crc = cls;
130
131 if (GNUNET_OK != success)
132 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "%s\n", msg);
133 GNUNET_assert (GNUNET_OK == success);
134 GNUNET_free (crc->data);
135 crc->data = NULL;
136 GNUNET_SCHEDULER_add_now (&run_continuation, crc);
137}
138
139
140static void
141check_value (void *cls,
142 const struct GNUNET_HashCode *key,
143 size_t size,
144 const void *data,
145 enum GNUNET_BLOCK_Type type,
146 uint32_t priority,
147 uint32_t anonymity,
148 uint32_t replication,
149 struct GNUNET_TIME_Absolute expiration,
150 uint64_t uid)
151{
152 struct CpsRunContext *crc = cls;
153 int i;
154
155 if (NULL == key)
156 {
157 crc->phase = RP_GET_FAIL;
158 GNUNET_SCHEDULER_add_now (&run_continuation, crc);
159 return;
160 }
161 i = crc->i;
162 GNUNET_assert (size == get_size (i));
163 GNUNET_assert (0 == memcmp (data, get_data (i), size));
164 GNUNET_assert (type == get_type (i));
165 GNUNET_assert (priority == get_priority (i));
166 GNUNET_assert (anonymity == get_anonymity (i));
167 GNUNET_assert (expiration.abs_value_us == get_expiration (i).abs_value_us);
168 crc->i--;
169 if (crc->i == 0)
170 crc->phase = RP_DONE;
171 GNUNET_SCHEDULER_add_now (&run_continuation, crc);
172}
173
174
175static void
176check_nothing (void *cls,
177 const struct GNUNET_HashCode *key,
178 size_t size,
179 const void *data,
180 enum GNUNET_BLOCK_Type type,
181 uint32_t priority,
182 uint32_t anonymity,
183 uint32_t replication,
184 struct GNUNET_TIME_Absolute expiration,
185 uint64_t uid)
186{
187 struct CpsRunContext *crc = cls;
188
189 GNUNET_assert (key == NULL);
190 if (0 == --crc->i)
191 crc->phase = RP_DONE;
192 GNUNET_SCHEDULER_add_now (&run_continuation, crc);
193}
194
195
196static void
197run_continuation (void *cls)
198{
199 struct CpsRunContext *crc = cls;
200
201 ok = (int) crc->phase;
202 switch (crc->phase)
203 {
204 case RP_PUT:
205 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Executing `%s' number %u\n", "PUT",
206 crc->i);
207 GNUNET_CRYPTO_hash (&crc->i, sizeof(int), &crc->key);
208 GNUNET_DATASTORE_put (datastore,
209 0,
210 &crc->key,
211 get_size (crc->i),
212 get_data (crc->i),
213 get_type (crc->i),
214 get_priority (crc->i),
215 get_anonymity (crc->i),
216 0,
217 get_expiration (crc->i),
218 1,
219 1,
220 &check_success, crc);
221 crc->i++;
222 if (crc->i == ITERATIONS)
223 {
224 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
225 "Sleeping to give datastore time to clean up\n");
226 sleep (1);
227 crc->phase = RP_GET;
228 crc->i--;
229 }
230 break;
231
232 case RP_GET:
233 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Executing `%s' number %u\n", "GET",
234 crc->i);
235 GNUNET_CRYPTO_hash (&crc->i, sizeof(int), &crc->key);
236 GNUNET_DATASTORE_get_key (datastore,
237 0,
238 false,
239 &crc->key,
240 get_type (crc->i),
241 1,
242 1,
243 &check_value,
244 crc);
245 break;
246
247 case RP_GET_FAIL:
248 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Executing `%s' number %u\n", "GET(f)",
249 crc->i);
250 GNUNET_CRYPTO_hash (&crc->i, sizeof(int), &crc->key);
251 GNUNET_DATASTORE_get_key (datastore,
252 0,
253 false,
254 &crc->key,
255 get_type (crc->i),
256 1,
257 1,
258 &check_nothing,
259 crc);
260 break;
261
262 case RP_DONE:
263 GNUNET_assert (0 == crc->i);
264 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Finished, disconnecting\n");
265 GNUNET_DATASTORE_disconnect (datastore,
266 GNUNET_YES);
267 GNUNET_free (crc);
268 ok = 0;
269 }
270}
271
272
273static void
274run_tests (void *cls,
275 int success,
276 struct GNUNET_TIME_Absolute min_expiration,
277 const char *msg)
278{
279 struct CpsRunContext *crc = cls;
280
281 if (success != GNUNET_YES)
282 {
283 fprintf (stderr,
284 "Test 'put' operation failed with error `%s' database likely not setup, skipping test.\n",
285 msg);
286 GNUNET_DATASTORE_disconnect (datastore,
287 GNUNET_YES);
288 GNUNET_free (crc);
289 return;
290 }
291 GNUNET_SCHEDULER_add_now (&run_continuation,
292 crc);
293}
294
295
296static void
297run (void *cls,
298 const struct GNUNET_CONFIGURATION_Handle *cfg,
299 struct GNUNET_TESTING_Peer *peer)
300{
301 struct CpsRunContext *crc;
302 static struct GNUNET_HashCode zkey;
303
304 crc = GNUNET_new (struct CpsRunContext);
305 crc->cfg = cfg;
306 crc->phase = RP_PUT;
307 now = GNUNET_TIME_absolute_get ();
308 datastore = GNUNET_DATASTORE_connect (cfg);
309 if (NULL ==
310 GNUNET_DATASTORE_put (datastore,
311 0,
312 &zkey,
313 4,
314 "TEST",
315 GNUNET_BLOCK_TYPE_TEST,
316 0, 0, 0,
317 GNUNET_TIME_relative_to_absolute (
318 GNUNET_TIME_UNIT_SECONDS),
319 0,
320 1,
321 &run_tests,
322 crc))
323 {
324 fprintf (stderr, "%s", "Test 'put' operation failed.\n");
325 GNUNET_free (crc);
326 ok = 1;
327 }
328}
329
330
331/**
332 * Function called when disk utilization changes, does nothing.
333 *
334 * @param cls closure
335 * @param delta change in utilization
336 */
337static void
338ignore_payload_cb (void *cls,
339 int delta)
340{
341 /* do nothing */
342}
343
344
345/**
346 * check if plugin is actually working
347 */
348static int
349test_plugin (const char *cfg_name)
350{
351 char libname[PATH_MAX];
352 struct GNUNET_CONFIGURATION_Handle *cfg;
353 struct GNUNET_DATASTORE_PluginFunctions *api;
354 struct GNUNET_DATASTORE_PluginEnvironment env;
355
356 cfg = GNUNET_CONFIGURATION_create ();
357 if (GNUNET_OK !=
358 GNUNET_CONFIGURATION_load (cfg,
359 cfg_name))
360 {
361 GNUNET_CONFIGURATION_destroy (cfg);
362 fprintf (stderr,
363 "Failed to load configuration %s\n",
364 cfg_name);
365 return 1;
366 }
367 memset (&env, 0, sizeof(env));
368 env.cfg = cfg;
369 env.duc = &ignore_payload_cb;
370 GNUNET_snprintf (libname,
371 sizeof(libname),
372 "libgnunet_plugin_datastore_%s",
373 plugin_name);
374 api = GNUNET_PLUGIN_load (libname, &env);
375 if (NULL == api)
376 {
377 GNUNET_CONFIGURATION_destroy (cfg);
378 fprintf (stderr,
379 "Failed to load plugin `%s'\n",
380 libname);
381 return 77;
382 }
383 GNUNET_PLUGIN_unload (libname, api);
384 GNUNET_CONFIGURATION_destroy (cfg);
385 return 0;
386}
387
388
389int
390main (int argc, char *argv[])
391{
392 char cfg_name[PATH_MAX];
393 int ret;
394
395 plugin_name = GNUNET_STRINGS_get_suffix_from_binary_name (argv[0]);
396 GNUNET_snprintf (cfg_name,
397 sizeof(cfg_name),
398 "test_datastore_api_data_%s.conf",
399 plugin_name);
400 ret = test_plugin (cfg_name);
401 if (0 != ret)
402 return ret;
403 if (0 !=
404 GNUNET_TESTING_peer_run ("test-gnunet-datastore-management",
405 cfg_name,
406 &run,
407 NULL))
408 return 1;
409 return ok;
410}
411
412
413/* end of test_datastore_api_management.c */
diff --git a/src/service/datastore/test_defaults.conf b/src/service/datastore/test_defaults.conf
new file mode 100644
index 000000000..1e6bbeeaf
--- /dev/null
+++ b/src/service/datastore/test_defaults.conf
@@ -0,0 +1,10 @@
1@inline@ ../../../contrib/conf/gnunet/no_autostart_above_core.conf
2@inline@ ../../../contrib/conf/gnunet/no_forcestart.conf
3
4[datastore]
5PORT = 22654
6QUOTA = 1 MB
7START_ON_DEMAND = YES
8
9[nse]
10WORKBITS = 1
generated by cgit v1.2.3 (git 2.39.1) at 2024-05-19 16:43:37 +0000