aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--TODO2
-rw-r--r--src/datastore/gnunet-service-datastore.c143
-rw-r--r--src/datastore/perf_plugin_datastore.c16
-rw-r--r--src/datastore/plugin_datastore.h28
-rw-r--r--src/datastore/plugin_datastore_mysql.c46
-rw-r--r--src/datastore/plugin_datastore_postgres.c158
-rw-r--r--src/datastore/plugin_datastore_sqlite.c230
-rw-r--r--src/include/gnunet_scheduler_lib.h2
-rw-r--r--src/peerinfo/Makefile.am10
-rw-r--r--src/peerinfo/gnunet-service-peerinfo.c195
-rwxr-xr-xsrc/peerinfo/perf_peerinfo_api.c217
-rw-r--r--src/peerinfo/test_peerinfo_api.c2
-rw-r--r--src/peerinfo/test_peerinfo_api_data.conf3
-rw-r--r--src/util/network.c1
-rw-r--r--src/util/scheduler.c214
15 files changed, 920 insertions, 347 deletions
diff --git a/TODO b/TODO
index c51b4a6c3..8ac9a592c 100644
--- a/TODO
+++ b/TODO
@@ -115,7 +115,7 @@
115* DV: [Nate] 115* DV: [Nate]
116 - proper bandwidth allocation 116 - proper bandwidth allocation
117 - performance tests 117 - performance tests
118* PEERINFO: 118* PEERINFO:
119 - merge multiple HELLOs of the same peer in the transmission queue 119 - merge multiple HELLOs of the same peer in the transmission queue
120 (theoretically reduces overhead; bounds message queue size) 120 (theoretically reduces overhead; bounds message queue size)
121 - merge multiple iteration requests over "all" peers in the queue 121 - merge multiple iteration requests over "all" peers in the queue
diff --git a/src/datastore/gnunet-service-datastore.c b/src/datastore/gnunet-service-datastore.c
index 40ea153de..01778b1b7 100644
--- a/src/datastore/gnunet-service-datastore.c
+++ b/src/datastore/gnunet-service-datastore.c
@@ -42,6 +42,13 @@
42 */ 42 */
43#define MAX_EXPIRE_DELAY GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15) 43#define MAX_EXPIRE_DELAY GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15)
44 44
45#define QUOTA_STAT_NAME gettext_noop ("# bytes used in file-sharing datastore")
46
47/**
48 * After how many payload-changing operations
49 * do we sync our statistics?
50 */
51#define MAX_STAT_SYNC_LAG 50
45 52
46 53
47/** 54/**
@@ -109,6 +116,7 @@ struct ReservationList
109}; 116};
110 117
111 118
119
112/** 120/**
113 * Our datastore plugin (NULL if not available). 121 * Our datastore plugin (NULL if not available).
114 */ 122 */
@@ -141,6 +149,24 @@ static unsigned long long cache_size;
141 * How much space have we currently reserved? 149 * How much space have we currently reserved?
142 */ 150 */
143static unsigned long long reserved; 151static unsigned long long reserved;
152
153/**
154 * How much data are we currently storing
155 * in the database?
156 */
157static unsigned long long payload;
158
159/**
160 * Number of updates that were made to the
161 * payload value since we last synchronized
162 * it with the statistics service.
163 */
164static unsigned int lastSync;
165
166/**
167 * Did we get an answer from statistics?
168 */
169static int stats_worked;
144 170
145/** 171/**
146 * Identity of the task that is used to delete 172 * Identity of the task that is used to delete
@@ -165,6 +191,23 @@ static struct GNUNET_STATISTICS_Handle *stats;
165 191
166 192
167/** 193/**
194 * Synchronize our utilization statistics with the
195 * statistics service.
196 */
197static void
198sync_stats ()
199{
200 GNUNET_STATISTICS_set (stats,
201 QUOTA_STAT_NAME,
202 payload,
203 GNUNET_YES);
204 lastSync = 0;
205}
206
207
208
209
210/**
168 * Function called once the transmit operation has 211 * Function called once the transmit operation has
169 * either failed or succeeded. 212 * either failed or succeeded.
170 * 213 *
@@ -242,6 +285,12 @@ static struct TransmitCallbackContext *tcc_tail;
242static int cleaning_done; 285static int cleaning_done;
243 286
244/** 287/**
288 * Handle for pending get request.
289 */
290static struct GNUNET_STATISTICS_GetHandle *stat_get;
291
292
293/**
245 * Task that is used to remove expired entries from 294 * Task that is used to remove expired entries from
246 * the datastore. This task will schedule itself 295 * the datastore. This task will schedule itself
247 * again automatically to always delete all expired 296 * again automatically to always delete all expired
@@ -731,7 +780,7 @@ handle_reserve (void *cls,
731#endif 780#endif
732 amount = GNUNET_ntohll(msg->amount); 781 amount = GNUNET_ntohll(msg->amount);
733 entries = ntohl(msg->entries); 782 entries = ntohl(msg->entries);
734 used = plugin->api->get_size (plugin->api->cls) + reserved; 783 used = payload + reserved;
735 req = amount + ((unsigned long long) GNUNET_DATASTORE_ENTRY_OVERHEAD) * entries; 784 req = amount + ((unsigned long long) GNUNET_DATASTORE_ENTRY_OVERHEAD) * entries;
736 if (used + req > quota) 785 if (used + req > quota)
737 { 786 {
@@ -931,13 +980,13 @@ execute_put (struct GNUNET_SERVER_Client *client,
931 (GNUNET_SYSERR == ret) ? GNUNET_SYSERR : GNUNET_OK, 980 (GNUNET_SYSERR == ret) ? GNUNET_SYSERR : GNUNET_OK,
932 msg); 981 msg);
933 GNUNET_free_non_null (msg); 982 GNUNET_free_non_null (msg);
934 if (quota - reserved - cache_size < plugin->api->get_size (plugin->api->cls)) 983 if (quota - reserved - cache_size < payload)
935 { 984 {
936 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 985 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
937 _("Need %llu bytes more space (%llu allowed, using %llu)\n"), 986 _("Need %llu bytes more space (%llu allowed, using %llu)\n"),
938 (unsigned long long) size + GNUNET_DATASTORE_ENTRY_OVERHEAD, 987 (unsigned long long) size + GNUNET_DATASTORE_ENTRY_OVERHEAD,
939 (unsigned long long) (quota - reserved - cache_size), 988 (unsigned long long) (quota - reserved - cache_size),
940 (unsigned long long) plugin->api->get_size (plugin->api->cls)); 989 (unsigned long long) payload);
941 manage_space (size + GNUNET_DATASTORE_ENTRY_OVERHEAD); 990 manage_space (size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
942 } 991 }
943} 992}
@@ -1351,6 +1400,78 @@ handle_drop (void *cls,
1351 1400
1352 1401
1353/** 1402/**
1403 * Function called by plugins to notify us about a
1404 * change in their disk utilization.
1405 *
1406 * @param cls closure (NULL)
1407 * @param delta change in disk utilization,
1408 * 0 for "reset to empty"
1409 */
1410static void
1411disk_utilization_change_cb (void *cls,
1412 int delta)
1413{
1414 if ( (delta < 0) &&
1415 (payload < -delta) )
1416 {
1417 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1418 _("Datastore payload inaccurate (%lld < %lld). Trying to fix.\n"),
1419 (long long) payload,
1420 (long long) -delta);
1421 payload = plugin->api->get_size (plugin->api->cls);
1422 sync_stats ();
1423 return;
1424 }
1425 payload += delta;
1426 lastSync++;
1427 if (lastSync >= MAX_STAT_SYNC_LAG)
1428 sync_stats ();
1429}
1430
1431
1432/**
1433 * Callback function to process statistic values.
1434 *
1435 * @param cls closure (struct Plugin*)
1436 * @param subsystem name of subsystem that created the statistic
1437 * @param name the name of the datum
1438 * @param value the current value
1439 * @param is_persistent GNUNET_YES if the value is persistent, GNUNET_NO if not
1440 * @return GNUNET_OK to continue, GNUNET_SYSERR to abort iteration
1441 */
1442static int
1443process_stat_in (void *cls,
1444 const char *subsystem,
1445 const char *name,
1446 uint64_t value,
1447 int is_persistent)
1448{
1449 GNUNET_assert (stats_worked == GNUNET_NO);
1450 stats_worked = GNUNET_YES;
1451 payload += value;
1452#if DEBUG_SQLITE
1453 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1454 "Notification from statistics about existing payload (%llu), new payload is %llu\n",
1455 value,
1456 payload);
1457#endif
1458 return GNUNET_OK;
1459}
1460
1461
1462static void
1463process_stat_done (void *cls,
1464 int success)
1465{
1466 struct DatastorePlugin *plugin = cls;
1467
1468 stat_get = NULL;
1469 if (stats_worked == GNUNET_NO)
1470 payload = plugin->api->get_size (plugin->api->cls);
1471}
1472
1473
1474/**
1354 * Load the datastore plugin. 1475 * Load the datastore plugin.
1355 */ 1476 */
1356static struct DatastorePlugin * 1477static struct DatastorePlugin *
@@ -1373,6 +1494,8 @@ load_plugin ()
1373 ret = GNUNET_malloc (sizeof(struct DatastorePlugin)); 1494 ret = GNUNET_malloc (sizeof(struct DatastorePlugin));
1374 ret->env.cfg = cfg; 1495 ret->env.cfg = cfg;
1375 ret->env.sched = sched; 1496 ret->env.sched = sched;
1497 ret->env.duc = &disk_utilization_change_cb;
1498 ret->env.cls = NULL;
1376 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1499 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1377 _("Loading `%s' datastore plugin\n"), name); 1500 _("Loading `%s' datastore plugin\n"), name);
1378 GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", name); 1501 GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", name);
@@ -1426,6 +1549,13 @@ unload_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
1426 GNUNET_CONTAINER_bloomfilter_free (filter); 1549 GNUNET_CONTAINER_bloomfilter_free (filter);
1427 filter = NULL; 1550 filter = NULL;
1428 } 1551 }
1552 if (lastSync > 0)
1553 sync_stats ();
1554 if (stat_get != NULL)
1555 {
1556 GNUNET_STATISTICS_get_cancel (stat_get);
1557 stat_get = NULL;
1558 }
1429 if (stats != NULL) 1559 if (stats != NULL)
1430 { 1560 {
1431 GNUNET_STATISTICS_destroy (stats, GNUNET_YES); 1561 GNUNET_STATISTICS_destroy (stats, GNUNET_YES);
@@ -1614,6 +1744,13 @@ run (void *cls,
1614 } 1744 }
1615 return; 1745 return;
1616 } 1746 }
1747 stat_get = GNUNET_STATISTICS_get (stats,
1748 "datastore",
1749 QUOTA_STAT_NAME,
1750 GNUNET_TIME_UNIT_SECONDS,
1751 &process_stat_done,
1752 &process_stat_in,
1753 plugin);
1617 GNUNET_SERVER_disconnect_notify (server, &cleanup_reservations, NULL); 1754 GNUNET_SERVER_disconnect_notify (server, &cleanup_reservations, NULL);
1618 GNUNET_SERVER_add_handlers (server, handlers); 1755 GNUNET_SERVER_add_handlers (server, handlers);
1619 expired_kill_task 1756 expired_kill_task
diff --git a/src/datastore/perf_plugin_datastore.c b/src/datastore/perf_plugin_datastore.c
index 17cd009bc..4d39d85ac 100644
--- a/src/datastore/perf_plugin_datastore.c
+++ b/src/datastore/perf_plugin_datastore.c
@@ -81,6 +81,20 @@ struct CpsRunContext
81}; 81};
82 82
83 83
84/**
85 * Function called by plugins to notify us about a
86 * change in their disk utilization.
87 *
88 * @param cls closure (NULL)
89 * @param delta change in disk utilization,
90 * 0 for "reset to empty"
91 */
92static void
93disk_utilization_change_cb (void *cls,
94 int delta)
95{
96}
97
84 98
85static void 99static void
86putValue (struct GNUNET_DATASTORE_PluginFunctions * api, int i, int k) 100putValue (struct GNUNET_DATASTORE_PluginFunctions * api, int i, int k)
@@ -331,6 +345,8 @@ load_plugin (const struct GNUNET_CONFIGURATION_Handle *cfg,
331 } 345 }
332 env.cfg = cfg; 346 env.cfg = cfg;
333 env.sched = sched; 347 env.sched = sched;
348 env.duc = &disk_utilization_change_cb;
349 env.cls = NULL;
334 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 350 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
335 _("Loading `%s' datastore plugin\n"), name); 351 _("Loading `%s' datastore plugin\n"), name);
336 GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", name); 352 GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", name);
diff --git a/src/datastore/plugin_datastore.h b/src/datastore/plugin_datastore.h
index e8f433671..fa08501cc 100644
--- a/src/datastore/plugin_datastore.h
+++ b/src/datastore/plugin_datastore.h
@@ -22,10 +22,6 @@
22 * @file datastore/plugin_datastore.h 22 * @file datastore/plugin_datastore.h
23 * @brief API for the database backend plugins. 23 * @brief API for the database backend plugins.
24 * @author Christian Grothoff 24 * @author Christian Grothoff
25 *
26 * TODO:
27 * - consider defining enumeration or at least typedef
28 * for the type of "type" (instead of using uint32_t)
29 */ 25 */
30#ifndef PLUGIN_DATASTORE_H 26#ifndef PLUGIN_DATASTORE_H
31#define PLUGIN_DATASTORE_H 27#define PLUGIN_DATASTORE_H
@@ -39,12 +35,24 @@
39 35
40/** 36/**
41 * How many bytes of overhead will we assume per entry 37 * How many bytes of overhead will we assume per entry
42 * in the SQlite DB? 38 * in any DB (for reservations)?
43 */ 39 */
44#define GNUNET_DATASTORE_ENTRY_OVERHEAD 256 40#define GNUNET_DATASTORE_ENTRY_OVERHEAD 256
45 41
46 42
47/** 43/**
44 * Function invoked to notify service of disk utilization
45 * changes.
46 *
47 * @param cls closure
48 * @param delta change in disk utilization,
49 * 0 for "reset to empty"
50 */
51typedef void (*DiskUtilizationChange)(void *cls,
52 int delta);
53
54
55/**
48 * The datastore service will pass a pointer to a struct 56 * The datastore service will pass a pointer to a struct
49 * of this type as the first and only argument to the 57 * of this type as the first and only argument to the
50 * entry point of each datastore plugin. 58 * entry point of each datastore plugin.
@@ -61,6 +69,16 @@ struct GNUNET_DATASTORE_PluginEnvironment
61 */ 69 */
62 struct GNUNET_SCHEDULER_Handle *sched; 70 struct GNUNET_SCHEDULER_Handle *sched;
63 71
72 /**
73 * Function to call on disk utilization change.
74 */
75 DiskUtilizationChange duc;
76
77 /**
78 * Closure.
79 */
80 void *cls;
81
64}; 82};
65 83
66 84
diff --git a/src/datastore/plugin_datastore_mysql.c b/src/datastore/plugin_datastore_mysql.c
index c216e989c..ea6cc2322 100644
--- a/src/datastore/plugin_datastore_mysql.c
+++ b/src/datastore/plugin_datastore_mysql.c
@@ -191,7 +191,6 @@
191 " ORDER BY expire DESC,vkey DESC LIMIT 1)"\ 191 " ORDER BY expire DESC,vkey DESC LIMIT 1)"\
192 "ORDER BY expire DESC,vkey DESC LIMIT 1" 192 "ORDER BY expire DESC,vkey DESC LIMIT 1"
193 193
194// #define SELECT_SIZE "SELECT SUM(BIT_LENGTH(value) DIV 8) FROM gn072"
195 194
196 195
197struct GNUNET_MysqlStatementHandle 196struct GNUNET_MysqlStatementHandle
@@ -344,12 +343,10 @@ struct Plugin
344#define UPDATE_ENTRY "UPDATE gn090 SET prio=prio+?,expire=IF(expire>=?,expire,?) WHERE vkey=?" 343#define UPDATE_ENTRY "UPDATE gn090 SET prio=prio+?,expire=IF(expire>=?,expire,?) WHERE vkey=?"
345 struct GNUNET_MysqlStatementHandle *update_entry; 344 struct GNUNET_MysqlStatementHandle *update_entry;
346 345
347 struct GNUNET_MysqlStatementHandle *iter[4]; 346#define SELECT_SIZE "SELECT SUM(BIT_LENGTH(value) DIV 8) FROM gn072"
347 struct GNUNET_MysqlStatementHandle *get_size;
348 348
349 /** 349 struct GNUNET_MysqlStatementHandle *iter[4];
350 * Size of the mysql database on disk.
351 */
352 unsigned long long content_size;
353 350
354}; 351};
355 352
@@ -957,8 +954,11 @@ do_delete_entry_by_vkey (struct Plugin *plugin,
957 return ret; 954 return ret;
958} 955}
959 956
957
960static int 958static int
961return_ok (void *cls, unsigned int num_values, MYSQL_BIND * values) 959return_ok (void *cls,
960 unsigned int num_values,
961 MYSQL_BIND * values)
962{ 962{
963 return GNUNET_OK; 963 return GNUNET_OK;
964} 964}
@@ -1189,7 +1189,9 @@ mysql_next_request_cont (void *next_cls,
1189 { 1189 {
1190 do_delete_value (plugin, vkey); 1190 do_delete_value (plugin, vkey);
1191 do_delete_entry_by_vkey (plugin, vkey); 1191 do_delete_entry_by_vkey (plugin, vkey);
1192 plugin->content_size -= length; 1192 if (length != 0)
1193 plugin->env->duc (plugin->env->cls,
1194 - length);
1193 } 1195 }
1194 return; 1196 return;
1195 END_SET: 1197 END_SET:
@@ -1279,14 +1281,29 @@ iterateHelper (struct Plugin *plugin,
1279 * Get an estimate of how much space the database is 1281 * Get an estimate of how much space the database is
1280 * currently using. 1282 * currently using.
1281 * 1283 *
1282 * @param cls our "struct Plugin*" 1284 * @param cls our "struct Plugin *"
1283 * @return number of bytes used on disk 1285 * @return number of bytes used on disk
1284 */ 1286 */
1285static unsigned long long 1287static unsigned long long
1286mysql_plugin_get_size (void *cls) 1288mysql_plugin_get_size (void *cls)
1287{ 1289{
1288 struct Plugin *plugin = cls; 1290 struct Plugin *plugin = cls;
1289 return plugin->content_size; 1291 MYSQL_BIND cbind[1];
1292 long long total;
1293
1294 memset (cbind, 0, sizeof (cbind));
1295 total = 0;
1296 cbind[0].buffer_type = MYSQL_TYPE_LONGLONG;
1297 cbind[0].buffer = &total;
1298 cbind[0].is_unsigned = GNUNET_NO;
1299 if (GNUNET_OK !=
1300 prepared_statement_run_select (plugin,
1301 plugin->get_size,
1302 1, cbind,
1303 &return_ok, NULL,
1304 -1))
1305 return 0;
1306 return total;
1290} 1307}
1291 1308
1292 1309
@@ -1373,7 +1390,9 @@ mysql_plugin_put (void *cls,
1373 vkey, 1390 vkey,
1374 (unsigned int) size); 1391 (unsigned int) size);
1375#endif 1392#endif
1376 plugin->content_size += size; 1393 if (size > 0)
1394 plugin->env->duc (plugin->env->cls,
1395 size);
1377 return GNUNET_OK; 1396 return GNUNET_OK;
1378} 1397}
1379 1398
@@ -1804,8 +1823,8 @@ mysql_plugin_drop (void *cls)
1804 "DROP TABLE gn090")) || 1823 "DROP TABLE gn090")) ||
1805 (GNUNET_OK != run_statement (plugin, 1824 (GNUNET_OK != run_statement (plugin,
1806 "DROP TABLE gn072"))) 1825 "DROP TABLE gn072")))
1807 return; /* error */ 1826 return; /* error */
1808 plugin->content_size = 0; 1827 plugin->env->duc (plugin->env->cls, 0);
1809} 1828}
1810 1829
1811 1830
@@ -1865,6 +1884,7 @@ libgnunet_plugin_datastore_mysql_init (void *cls)
1865 || PINIT (plugin->select_entry_by_hash_vhash_and_type, 1884 || PINIT (plugin->select_entry_by_hash_vhash_and_type,
1866 SELECT_ENTRY_BY_HASH_VHASH_AND_TYPE) 1885 SELECT_ENTRY_BY_HASH_VHASH_AND_TYPE)
1867 || PINIT (plugin->count_entry_by_hash, COUNT_ENTRY_BY_HASH) 1886 || PINIT (plugin->count_entry_by_hash, COUNT_ENTRY_BY_HASH)
1887 || PINIT (plugin->get_size, SELECT_SIZE)
1868 || PINIT (plugin->count_entry_by_hash_and_vhash, COUNT_ENTRY_BY_HASH_AND_VHASH) 1888 || PINIT (plugin->count_entry_by_hash_and_vhash, COUNT_ENTRY_BY_HASH_AND_VHASH)
1869 || PINIT (plugin->count_entry_by_hash_and_type, COUNT_ENTRY_BY_HASH_AND_TYPE) 1889 || PINIT (plugin->count_entry_by_hash_and_type, COUNT_ENTRY_BY_HASH_AND_TYPE)
1870 || PINIT (plugin->count_entry_by_hash_vhash_and_type, 1890 || PINIT (plugin->count_entry_by_hash_vhash_and_type,
diff --git a/src/datastore/plugin_datastore_postgres.c b/src/datastore/plugin_datastore_postgres.c
index 889309784..54bdde542 100644
--- a/src/datastore/plugin_datastore_postgres.c
+++ b/src/datastore/plugin_datastore_postgres.c
@@ -82,26 +82,104 @@
82#define BUSY_TIMEOUT GNUNET_TIME_UNIT_SECONDS 82#define BUSY_TIMEOUT GNUNET_TIME_UNIT_SECONDS
83 83
84 84
85/**
86 * Closure for 'postgres_next_request_cont'.
87 */
85struct NextRequestClosure 88struct NextRequestClosure
86{ 89{
90 /**
91 * Global plugin data.
92 */
87 struct Plugin *plugin; 93 struct Plugin *plugin;
94
95 /**
96 * Function to call for each matching entry.
97 */
88 PluginIterator iter; 98 PluginIterator iter;
99
100 /**
101 * Closure for 'iter'.
102 */
89 void *iter_cls; 103 void *iter_cls;
104
105 /**
106 * Parameters for the prepared statement.
107 */
90 const char *paramValues[5]; 108 const char *paramValues[5];
109
110 /**
111 * Name of the prepared statement to run.
112 */
91 const char *pname; 113 const char *pname;
114
115 /**
116 * Size of values pointed to by paramValues.
117 */
92 int paramLengths[5]; 118 int paramLengths[5];
119
120 /**
121 * Number of paramters in paramValues/paramLengths.
122 */
93 int nparams; 123 int nparams;
124
125 /**
126 * Current time (possible parameter), big-endian.
127 */
94 uint64_t bnow; 128 uint64_t bnow;
129
130 /**
131 * Key (possible parameter)
132 */
95 GNUNET_HashCode key; 133 GNUNET_HashCode key;
134
135 /**
136 * Hash of value (possible parameter)
137 */
96 GNUNET_HashCode vhash; 138 GNUNET_HashCode vhash;
139
140 /**
141 * Number of entries found so far
142 */
97 long long count; 143 long long count;
144
145 /**
146 * Offset this iteration starts at.
147 */
98 uint64_t off; 148 uint64_t off;
149
150 /**
151 * Current offset to use in query, big-endian.
152 */
99 uint64_t blimit_off; 153 uint64_t blimit_off;
154
155 /**
156 * Overall number of matching entries.
157 */
100 unsigned long long total; 158 unsigned long long total;
159
160 /**
161 * Expiration value of previous result (possible parameter), big-endian.
162 */
101 uint64_t blast_expire; 163 uint64_t blast_expire;
164
165 /**
166 * Row ID of last result (possible paramter), big-endian.
167 */
102 uint32_t blast_rowid; 168 uint32_t blast_rowid;
169
170 /**
171 * Priority of last result (possible parameter), big-endian.
172 */
103 uint32_t blast_prio; 173 uint32_t blast_prio;
174
175 /**
176 * Type of block (possible paramter), big-endian.
177 */
104 uint32_t btype; 178 uint32_t btype;
179
180 /**
181 * Flag set to GNUNET_YES to stop iteration.
182 */
105 int end_it; 183 int end_it;
106}; 184};
107 185
@@ -131,10 +209,6 @@ struct Plugin
131 */ 209 */
132 GNUNET_SCHEDULER_TaskIdentifier next_task; 210 GNUNET_SCHEDULER_TaskIdentifier next_task;
133 211
134 unsigned long long payload;
135
136 unsigned int lastSync;
137
138}; 212};
139 213
140 214
@@ -143,6 +217,12 @@ struct Plugin
143 * the desired status code. If not, log an error, clear the 217 * the desired status code. If not, log an error, clear the
144 * result and return GNUNET_SYSERR. 218 * result and return GNUNET_SYSERR.
145 * 219 *
220 * @param plugin global context
221 * @param ret result to check
222 * @param expected_status expected return value
223 * @param command name of SQL command that was run
224 * @param args arguments to SQL command
225 * @param line line number for error reporting
146 * @return GNUNET_OK if the result is acceptable 226 * @return GNUNET_OK if the result is acceptable
147 */ 227 */
148static int 228static int
@@ -173,6 +253,10 @@ check_result (struct Plugin *plugin,
173 253
174/** 254/**
175 * Run simple SQL statement (without results). 255 * Run simple SQL statement (without results).
256 *
257 * @param plugin global context
258 * @param sql statement to run
259 * @param line code line for error reporting
176 */ 260 */
177static int 261static int
178pq_exec (struct Plugin *plugin, 262pq_exec (struct Plugin *plugin,
@@ -190,6 +274,12 @@ pq_exec (struct Plugin *plugin,
190 274
191/** 275/**
192 * Prepare SQL statement. 276 * Prepare SQL statement.
277 *
278 * @param plugin global context
279 * @param sql SQL code to prepare
280 * @param nparams number of parameters in sql
281 * @param line code line for error reporting
282 * @return GNUNET_OK on success
193 */ 283 */
194static int 284static int
195pq_prepare (struct Plugin *plugin, 285pq_prepare (struct Plugin *plugin,
@@ -207,6 +297,8 @@ pq_prepare (struct Plugin *plugin,
207 297
208/** 298/**
209 * @brief Get a database handle 299 * @brief Get a database handle
300 *
301 * @param plugin global context
210 * @return GNUNET_OK on success, GNUNET_SYSERR on error 302 * @return GNUNET_OK on success, GNUNET_SYSERR on error
211 */ 303 */
212static int 304static int
@@ -413,6 +505,8 @@ init_connection (struct Plugin *plugin)
413 * Delete the row identified by the given rowid (qid 505 * Delete the row identified by the given rowid (qid
414 * in postgres). 506 * in postgres).
415 * 507 *
508 * @param plugin global context
509 * @param rowid which row to delete
416 * @return GNUNET_OK on success 510 * @return GNUNET_OK on success
417 */ 511 */
418static int 512static int
@@ -450,11 +544,32 @@ static unsigned long long
450postgres_plugin_get_size (void *cls) 544postgres_plugin_get_size (void *cls)
451{ 545{
452 struct Plugin *plugin = cls; 546 struct Plugin *plugin = cls;
453 double ret; 547 unsigned long long total;
548 PGresult *ret;
454 549
455 ret = plugin->payload; 550 ret = PQexecParams (plugin->dbh,
456 return (unsigned long long) (ret * 1.00); 551 "SELECT SUM(LENGTH(value))+256*COUNT(*) FROM gn090",
457 /* benchmarking shows XX% overhead */ 552 0, NULL, NULL, NULL, NULL, 1);
553 if (GNUNET_OK != check_result (plugin,
554 ret,
555 PGRES_TUPLES_OK,
556 "PQexecParams",
557 "get_size",
558 __LINE__))
559 {
560 return 0;
561 }
562 if ((PQntuples (ret) != 1) ||
563 (PQnfields (ret) != 1) ||
564 (PQgetlength (ret, 0, 0) != sizeof (unsigned long long)))
565 {
566 GNUNET_break (0);
567 PQclear (ret);
568 return 0;
569 }
570 total = GNUNET_ntohll (*(const unsigned long long *) PQgetvalue (ret, 0, 0));
571 PQclear (ret);
572 return total;
458} 573}
459 574
460 575
@@ -518,13 +633,12 @@ postgres_plugin_put (void *cls,
518 "PQexecPrepared", "put", __LINE__)) 633 "PQexecPrepared", "put", __LINE__))
519 return GNUNET_SYSERR; 634 return GNUNET_SYSERR;
520 PQclear (ret); 635 PQclear (ret);
521 plugin->payload += size; 636 plugin->env->duc (plugin->env->cls, size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
522#if DEBUG_POSTGRES 637#if DEBUG_POSTGRES
523 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, 638 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
524 "datastore-postgres", 639 "datastore-postgres",
525 "Stored %u bytes in database, new payload is %llu\n", 640 "Stored %u bytes in database\n",
526 (unsigned int) size, 641 (unsigned int) size);
527 (unsigned long long) plugin->payload);
528#endif 642#endif
529 return GNUNET_OK; 643 return GNUNET_OK;
530} 644}
@@ -695,18 +809,16 @@ postgres_next_request_cont (void *next_cls,
695#if DEBUG_POSTGRES 809#if DEBUG_POSTGRES
696 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, 810 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
697 "datastore-postgres", 811 "datastore-postgres",
698 "Deleting %u bytes from database, current payload is %llu\n", 812 "Deleting %u bytes from database\n",
699 (unsigned int) size, 813 (unsigned int) size);
700 (unsigned long long) plugin->payload);
701#endif 814#endif
702 GNUNET_assert (plugin->payload >= size); 815 plugin->env->duc (plugin->env->cls,
703 plugin->payload -= size; 816 - (size + GNUNET_DATASTORE_ENTRY_OVERHEAD));
704#if DEBUG_POSTGRES 817#if DEBUG_POSTGRES
705 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, 818 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
706 "datastore-postgres", 819 "datastore-postgres",
707 "Deleted %u bytes from database, new payload is %llu\n", 820 "Deleted %u bytes from database\n",
708 (unsigned int) size, 821 (unsigned int) size);
709 (unsigned long long) plugin->payload);
710#endif 822#endif
711 } 823 }
712 } 824 }
@@ -803,11 +915,15 @@ postgres_plugin_update (void *cls,
803 * Call a method for each key in the database and 915 * Call a method for each key in the database and
804 * call the callback method on it. 916 * call the callback method on it.
805 * 917 *
918 * @param plugin global context
806 * @param type entries of which type should be considered? 919 * @param type entries of which type should be considered?
920 * @param is_asc ascending or descending iteration?
921 * @param iter_select which SELECT method should be used?
807 * @param iter maybe NULL (to just count); iter 922 * @param iter maybe NULL (to just count); iter
808 * should return GNUNET_SYSERR to abort the 923 * should return GNUNET_SYSERR to abort the
809 * iteration, GNUNET_NO to delete the entry and 924 * iteration, GNUNET_NO to delete the entry and
810 * continue and GNUNET_OK to continue iterating 925 * continue and GNUNET_OK to continue iterating
926 * @param iter_cls closure for 'iter'
811 */ 927 */
812static void 928static void
813postgres_iterate (struct Plugin *plugin, 929postgres_iterate (struct Plugin *plugin,
@@ -1123,7 +1239,6 @@ postgres_plugin_iter_ascending_expiration (void *cls,
1123} 1239}
1124 1240
1125 1241
1126
1127/** 1242/**
1128 * Select a subset of the items in the datastore and call 1243 * Select a subset of the items in the datastore and call
1129 * the given iterator for each of them. 1244 * the given iterator for each of them.
@@ -1148,7 +1263,6 @@ postgres_plugin_iter_migration_order (void *cls,
1148} 1263}
1149 1264
1150 1265
1151
1152/** 1266/**
1153 * Select a subset of the items in the datastore and call 1267 * Select a subset of the items in the datastore and call
1154 * the given iterator for each of them. 1268 * the given iterator for each of them.
diff --git a/src/datastore/plugin_datastore_sqlite.c b/src/datastore/plugin_datastore_sqlite.c
index ca1f4e4ae..076d468ee 100644
--- a/src/datastore/plugin_datastore_sqlite.c
+++ b/src/datastore/plugin_datastore_sqlite.c
@@ -25,19 +25,11 @@
25 */ 25 */
26 26
27#include "platform.h" 27#include "platform.h"
28#include "gnunet_statistics_service.h"
29#include "plugin_datastore.h" 28#include "plugin_datastore.h"
30#include <sqlite3.h> 29#include <sqlite3.h>
31 30
32#define DEBUG_SQLITE GNUNET_NO 31#define DEBUG_SQLITE GNUNET_NO
33 32
34/**
35 * After how many payload-changing operations
36 * do we sync our statistics?
37 */
38#define MAX_STAT_SYNC_LAG 50
39
40#define QUOTA_STAT_NAME gettext_noop ("# bytes used in file-sharing datastore")
41 33
42/** 34/**
43 * Log an error message at log-level 'level' that indicates 35 * Log an error message at log-level 'level' that indicates
@@ -123,16 +115,6 @@ struct Plugin
123 sqlite3_stmt *insertContent; 115 sqlite3_stmt *insertContent;
124 116
125 /** 117 /**
126 * Handle to the statistics service.
127 */
128 struct GNUNET_STATISTICS_Handle *statistics;
129
130 /**
131 * Handle for pending get request.
132 */
133 struct GNUNET_STATISTICS_GetHandle *stat_get;
134
135 /**
136 * Closure of the 'next_task' (must be freed if 'next_task' is cancelled). 118 * Closure of the 'next_task' (must be freed if 'next_task' is cancelled).
137 */ 119 */
138 struct NextContext *next_task_nc; 120 struct NextContext *next_task_nc;
@@ -141,29 +123,12 @@ struct Plugin
141 * Pending task with scheduler for running the next request. 123 * Pending task with scheduler for running the next request.
142 */ 124 */
143 GNUNET_SCHEDULER_TaskIdentifier next_task; 125 GNUNET_SCHEDULER_TaskIdentifier next_task;
144
145 /**
146 * How much data are we currently storing
147 * in the database?
148 */
149 unsigned long long payload;
150
151 /**
152 * Number of updates that were made to the
153 * payload value since we last synchronized
154 * it with the statistics service.
155 */
156 unsigned int lastSync;
157 126
158 /** 127 /**
159 * Should the database be dropped on shutdown? 128 * Should the database be dropped on shutdown?
160 */ 129 */
161 int drop_on_shutdown; 130 int drop_on_shutdown;
162 131
163 /**
164 * Did we get an answer from statistics?
165 */
166 int stats_worked;
167}; 132};
168 133
169 134
@@ -267,12 +232,7 @@ database_setup (const struct GNUNET_CONFIGURATION_Handle *cfg,
267 return GNUNET_SYSERR; 232 return GNUNET_SYSERR;
268 } 233 }
269 /* database is new or got deleted, reset payload to zero! */ 234 /* database is new or got deleted, reset payload to zero! */
270 if (plugin->stat_get != NULL) 235 plugin->env->duc (plugin->env->cls, 0);
271 {
272 GNUNET_STATISTICS_get_cancel (plugin->stat_get);
273 plugin->stat_get = NULL;
274 }
275 plugin->payload = 0;
276 } 236 }
277 plugin->fn = GNUNET_STRINGS_to_utf8 (afsdir, strlen (afsdir), 237 plugin->fn = GNUNET_STRINGS_to_utf8 (afsdir, strlen (afsdir),
278#ifdef ENABLE_NLS 238#ifdef ENABLE_NLS
@@ -375,22 +335,6 @@ database_setup (const struct GNUNET_CONFIGURATION_Handle *cfg,
375 335
376 336
377/** 337/**
378 * Synchronize our utilization statistics with the
379 * statistics service.
380 * @param plugin the plugin context (state for this module)
381 */
382static void
383sync_stats (struct Plugin *plugin)
384{
385 GNUNET_STATISTICS_set (plugin->statistics,
386 QUOTA_STAT_NAME,
387 plugin->payload,
388 GNUNET_YES);
389 plugin->lastSync = 0;
390}
391
392
393/**
394 * Shutdown database connection and associate data 338 * Shutdown database connection and associate data
395 * structures. 339 * structures.
396 * @param plugin the plugin context (state for this module) 340 * @param plugin the plugin context (state for this module)
@@ -398,8 +342,6 @@ sync_stats (struct Plugin *plugin)
398static void 342static void
399database_shutdown (struct Plugin *plugin) 343database_shutdown (struct Plugin *plugin)
400{ 344{
401 if (plugin->lastSync > 0)
402 sync_stats (plugin);
403 if (plugin->updPrio != NULL) 345 if (plugin->updPrio != NULL)
404 sqlite3_finalize (plugin->updPrio); 346 sqlite3_finalize (plugin->updPrio);
405 if (plugin->insertContent != NULL) 347 if (plugin->insertContent != NULL)
@@ -410,20 +352,6 @@ database_shutdown (struct Plugin *plugin)
410 352
411 353
412/** 354/**
413 * Get an estimate of how much space the database is
414 * currently using.
415 *
416 * @param cls our plugin context
417 * @return number of bytes used on disk
418 */
419static unsigned long long sqlite_plugin_get_size (void *cls)
420{
421 struct Plugin *plugin = cls;
422 return plugin->payload;
423}
424
425
426/**
427 * Delete the database entry with the given 355 * Delete the database entry with the given
428 * row identifier. 356 * row identifier.
429 * 357 *
@@ -661,23 +589,15 @@ sqlite_next_request_cont (void *cls,
661 if ( (ret == GNUNET_NO) && 589 if ( (ret == GNUNET_NO) &&
662 (GNUNET_OK == delete_by_rowid (plugin, rowid)) ) 590 (GNUNET_OK == delete_by_rowid (plugin, rowid)) )
663 { 591 {
664 if (plugin->payload >= size + GNUNET_DATASTORE_ENTRY_OVERHEAD) 592 plugin->env->duc (plugin->env->cls,
665 plugin->payload -= (size + GNUNET_DATASTORE_ENTRY_OVERHEAD); 593 - (size + GNUNET_DATASTORE_ENTRY_OVERHEAD));
666 else
667 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
668 _("Datastore payload inaccurate, please fix and restart!\n"));
669 plugin->lastSync++;
670#if DEBUG_SQLITE 594#if DEBUG_SQLITE
671 if (ret == GNUNET_NO) 595 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
672 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, 596 "sqlite",
673 "sqlite", 597 "Removed entry %llu (%u bytes)\n",
674 "Removed entry %llu (%u bytes), new payload is %llu\n", 598 (unsigned long long) rowid,
675 (unsigned long long) rowid, 599 size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
676 size + GNUNET_DATASTORE_ENTRY_OVERHEAD,
677 plugin->payload);
678#endif 600#endif
679 if (plugin->lastSync >= MAX_STAT_SYNC_LAG)
680 sync_stats (plugin);
681 } 601 }
682} 602}
683 603
@@ -798,17 +718,14 @@ sqlite_plugin_put (void *cls,
798 LOG_SQLITE (plugin, NULL, 718 LOG_SQLITE (plugin, NULL,
799 GNUNET_ERROR_TYPE_ERROR | 719 GNUNET_ERROR_TYPE_ERROR |
800 GNUNET_ERROR_TYPE_BULK, "sqlite3_reset"); 720 GNUNET_ERROR_TYPE_BULK, "sqlite3_reset");
801 plugin->lastSync++; 721 plugin->env->duc (plugin->env->cls,
802 plugin->payload += size + GNUNET_DATASTORE_ENTRY_OVERHEAD; 722 size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
803#if DEBUG_SQLITE 723#if DEBUG_SQLITE
804 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, 724 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
805 "sqlite", 725 "sqlite",
806 "Stored new entry (%u bytes), new payload is %llu\n", 726 "Stored new entry (%u bytes)\n",
807 size + GNUNET_DATASTORE_ENTRY_OVERHEAD, 727 size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
808 plugin->payload);
809#endif 728#endif
810 if (plugin->lastSync >= MAX_STAT_SYNC_LAG)
811 sync_stats (plugin);
812 return GNUNET_OK; 729 return GNUNET_OK;
813} 730}
814 731
@@ -1574,81 +1491,50 @@ sqlite_plugin_drop (void *cls)
1574} 1491}
1575 1492
1576 1493
1577/** 1494static unsigned long long
1578 * Callback function to process statistic values. 1495sqlite_plugin_get_size (void *cls)
1579 *
1580 * @param cls closure
1581 * @param subsystem name of subsystem that created the statistic
1582 * @param name the name of the datum
1583 * @param value the current value
1584 * @param is_persistent GNUNET_YES if the value is persistent, GNUNET_NO if not
1585 * @return GNUNET_OK to continue, GNUNET_SYSERR to abort iteration
1586 */
1587static int
1588process_stat_in (void *cls,
1589 const char *subsystem,
1590 const char *name,
1591 uint64_t value,
1592 int is_persistent)
1593{
1594 struct Plugin *plugin = cls;
1595
1596 plugin->stats_worked = GNUNET_YES;
1597 plugin->payload += value;
1598#if DEBUG_SQLITE
1599 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
1600 "sqlite",
1601 "Notification from statistics about existing payload (%llu), new payload is %llu\n",
1602 value,
1603 plugin->payload);
1604#endif
1605 return GNUNET_OK;
1606}
1607
1608
1609static void
1610process_stat_done (void *cls,
1611 int success)
1612{ 1496{
1613 struct Plugin *plugin = cls; 1497 struct Plugin *plugin = cls;
1614 sqlite3_stmt *stmt; 1498 sqlite3_stmt *stmt;
1615 uint64_t pages; 1499 uint64_t pages;
1616 uint64_t page_size; 1500 uint64_t page_size;
1617 1501
1618 plugin->stat_get = NULL; 1502 if (SQLITE_VERSION_NUMBER < 3006000)
1619 if ( (plugin->stats_worked == GNUNET_NO) && 1503 {
1620 (SQLITE_VERSION_NUMBER >= 3006000) ) 1504 GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING,
1621 { 1505 "datastore-sqlite",
1622 CHECK (SQLITE_OK == 1506 _("sqlite version to old to determine size, assuming zero\n"));
1623 sqlite3_exec (plugin->dbh, 1507 return 0;
1624 "VACUUM", NULL, NULL, ENULL));
1625 CHECK (SQLITE_OK ==
1626 sqlite3_exec (plugin->dbh,
1627 "PRAGMA auto_vacuum=INCREMENTAL", NULL, NULL, ENULL));
1628 CHECK (SQLITE_OK ==
1629 sq_prepare (plugin->dbh,
1630 "PRAGMA page_count",
1631 &stmt));
1632 if (SQLITE_ROW ==
1633 sqlite3_step (stmt))
1634 pages = sqlite3_column_int64 (stmt, 0);
1635 else
1636 pages = 0;
1637 sqlite3_finalize (stmt);
1638 CHECK (SQLITE_OK ==
1639 sq_prepare (plugin->dbh,
1640 "PRAGMA page_size",
1641 &stmt));
1642 CHECK (SQLITE_ROW ==
1643 sqlite3_step (stmt));
1644 page_size = sqlite3_column_int64 (stmt, 0);
1645 sqlite3_finalize (stmt);
1646 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1647 _("Using sqlite page utilization to estimate payload (%llu pages of size %llu bytes)\n"),
1648 (unsigned long long) pages,
1649 (unsigned long long) page_size);
1650 plugin->payload = pages * page_size;
1651 } 1508 }
1509 CHECK (SQLITE_OK ==
1510 sqlite3_exec (plugin->dbh,
1511 "VACUUM", NULL, NULL, ENULL));
1512 CHECK (SQLITE_OK ==
1513 sqlite3_exec (plugin->dbh,
1514 "PRAGMA auto_vacuum=INCREMENTAL", NULL, NULL, ENULL));
1515 CHECK (SQLITE_OK ==
1516 sq_prepare (plugin->dbh,
1517 "PRAGMA page_count",
1518 &stmt));
1519 if (SQLITE_ROW ==
1520 sqlite3_step (stmt))
1521 pages = sqlite3_column_int64 (stmt, 0);
1522 else
1523 pages = 0;
1524 sqlite3_finalize (stmt);
1525 CHECK (SQLITE_OK ==
1526 sq_prepare (plugin->dbh,
1527 "PRAGMA page_size",
1528 &stmt));
1529 CHECK (SQLITE_ROW ==
1530 sqlite3_step (stmt));
1531 page_size = sqlite3_column_int64 (stmt, 0);
1532 sqlite3_finalize (stmt);
1533 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1534 _("Using sqlite page utilization to estimate payload (%llu pages of size %llu bytes)\n"),
1535 (unsigned long long) pages,
1536 (unsigned long long) page_size);
1537 return pages * page_size;
1652} 1538}
1653 1539
1654 1540
@@ -1669,16 +1555,6 @@ libgnunet_plugin_datastore_sqlite_init (void *cls)
1669 return NULL; /* can only initialize once! */ 1555 return NULL; /* can only initialize once! */
1670 memset (&plugin, 0, sizeof(struct Plugin)); 1556 memset (&plugin, 0, sizeof(struct Plugin));
1671 plugin.env = env; 1557 plugin.env = env;
1672 plugin.statistics = GNUNET_STATISTICS_create (env->sched,
1673 "ds-sqlite",
1674 env->cfg);
1675 plugin.stat_get = GNUNET_STATISTICS_get (plugin.statistics,
1676 "ds-sqlite",
1677 QUOTA_STAT_NAME,
1678 GNUNET_TIME_UNIT_SECONDS,
1679 &process_stat_done,
1680 &process_stat_in,
1681 &plugin);
1682 if (GNUNET_OK != 1558 if (GNUNET_OK !=
1683 database_setup (env->cfg, &plugin)) 1559 database_setup (env->cfg, &plugin))
1684 { 1560 {
@@ -1717,11 +1593,6 @@ libgnunet_plugin_datastore_sqlite_done (void *cls)
1717 struct GNUNET_DATASTORE_PluginFunctions *api = cls; 1593 struct GNUNET_DATASTORE_PluginFunctions *api = cls;
1718 struct Plugin *plugin = api->cls; 1594 struct Plugin *plugin = api->cls;
1719 1595
1720 if (plugin->stat_get != NULL)
1721 {
1722 GNUNET_STATISTICS_get_cancel (plugin->stat_get);
1723 plugin->stat_get = NULL;
1724 }
1725 if (plugin->next_task != GNUNET_SCHEDULER_NO_TASK) 1596 if (plugin->next_task != GNUNET_SCHEDULER_NO_TASK)
1726 { 1597 {
1727 GNUNET_SCHEDULER_cancel (plugin->env->sched, 1598 GNUNET_SCHEDULER_cancel (plugin->env->sched,
@@ -1735,10 +1606,7 @@ libgnunet_plugin_datastore_sqlite_done (void *cls)
1735 if (plugin->drop_on_shutdown) 1606 if (plugin->drop_on_shutdown)
1736 fn = GNUNET_strdup (plugin->fn); 1607 fn = GNUNET_strdup (plugin->fn);
1737 database_shutdown (plugin); 1608 database_shutdown (plugin);
1738 GNUNET_STATISTICS_destroy (plugin->statistics,
1739 GNUNET_NO);
1740 plugin->env = NULL; 1609 plugin->env = NULL;
1741 plugin->payload = 0;
1742 GNUNET_free (api); 1610 GNUNET_free (api);
1743 if (fn != NULL) 1611 if (fn != NULL)
1744 { 1612 {
diff --git a/src/include/gnunet_scheduler_lib.h b/src/include/gnunet_scheduler_lib.h
index 79739ef16..b39feb926 100644
--- a/src/include/gnunet_scheduler_lib.h
+++ b/src/include/gnunet_scheduler_lib.h
@@ -483,7 +483,7 @@ GNUNET_SCHEDULER_add_write_file (struct GNUNET_SCHEDULER_Handle *sched,
483 * && (delay-ready 483 * && (delay-ready
484 * || any-rs-ready 484 * || any-rs-ready
485 * || any-ws-ready 485 * || any-ws-ready
486 * || (shutdown-active && run-on-shutdown) ) 486 * || shutdown-active)
487 * </code> 487 * </code>
488 * 488 *
489 * @param sched scheduler to use 489 * @param sched scheduler to use
diff --git a/src/peerinfo/Makefile.am b/src/peerinfo/Makefile.am
index 31c7ab92d..116095f2a 100644
--- a/src/peerinfo/Makefile.am
+++ b/src/peerinfo/Makefile.am
@@ -35,7 +35,8 @@ gnunet_service_peerinfo_LDADD = \
35 35
36 36
37check_PROGRAMS = \ 37check_PROGRAMS = \
38 test_peerinfo_api 38 test_peerinfo_api \
39 perf_peerinfo_api
39 40
40if !DISABLE_TEST_RUN 41if !DISABLE_TEST_RUN
41TESTS = $(check_PROGRAMS) 42TESTS = $(check_PROGRAMS)
@@ -48,5 +49,12 @@ test_peerinfo_api_LDADD = \
48 $(top_builddir)/src/peerinfo/libgnunetpeerinfo.la \ 49 $(top_builddir)/src/peerinfo/libgnunetpeerinfo.la \
49 $(top_builddir)/src/util/libgnunetutil.la 50 $(top_builddir)/src/util/libgnunetutil.la
50 51
52perf_peerinfo_api_SOURCES = \
53 perf_peerinfo_api.c
54perf_peerinfo_api_LDADD = \
55 $(top_builddir)/src/hello/libgnunethello.la \
56 $(top_builddir)/src/peerinfo/libgnunetpeerinfo.la \
57 $(top_builddir)/src/util/libgnunetutil.la
58
51EXTRA_DIST = \ 59EXTRA_DIST = \
52 test_peerinfo_api_data.conf 60 test_peerinfo_api_data.conf
diff --git a/src/peerinfo/gnunet-service-peerinfo.c b/src/peerinfo/gnunet-service-peerinfo.c
index ee8749955..4879eea34 100644
--- a/src/peerinfo/gnunet-service-peerinfo.c
+++ b/src/peerinfo/gnunet-service-peerinfo.c
@@ -33,6 +33,7 @@
33 33
34#include "platform.h" 34#include "platform.h"
35#include "gnunet_crypto_lib.h" 35#include "gnunet_crypto_lib.h"
36#include "gnunet_container_lib.h"
36#include "gnunet_disk_lib.h" 37#include "gnunet_disk_lib.h"
37#include "gnunet_hello_lib.h" 38#include "gnunet_hello_lib.h"
38#include "gnunet_protocols.h" 39#include "gnunet_protocols.h"
@@ -57,11 +58,6 @@ struct HostEntry
57{ 58{
58 59
59 /** 60 /**
60 * This is a linked list.
61 */
62 struct HostEntry *next;
63
64 /**
65 * Identity of the peer. 61 * Identity of the peer.
66 */ 62 */
67 struct GNUNET_PeerIdentity identity; 63 struct GNUNET_PeerIdentity identity;
@@ -75,9 +71,10 @@ struct HostEntry
75 71
76 72
77/** 73/**
78 * The in-memory list of known hosts. 74 * The in-memory list of known hosts, mapping of
75 * host IDs to 'struct HostEntry*' values.
79 */ 76 */
80static struct HostEntry *hosts; 77static struct GNUNET_CONTAINER_MultiHashMap *hostmap;
81 78
82/** 79/**
83 * Clients to immediately notify about all changes. 80 * Clients to immediately notify about all changes.
@@ -163,24 +160,6 @@ get_host_filename (const struct GNUNET_PeerIdentity *id)
163 160
164 161
165/** 162/**
166 * Find the host entry for the given peer. FIXME: replace by hash map!
167 * @return NULL if not found
168 */
169static struct HostEntry *
170lookup_host_entry (const struct GNUNET_PeerIdentity *id)
171{
172 struct HostEntry *pos;
173
174 pos = hosts;
175 while ((pos != NULL) &&
176 (0 !=
177 memcmp (id, &pos->identity, sizeof (struct GNUNET_PeerIdentity))))
178 pos = pos->next;
179 return pos;
180}
181
182
183/**
184 * Broadcast information about the given entry to all 163 * Broadcast information about the given entry to all
185 * clients that care. 164 * clients that care.
186 * 165 *
@@ -215,7 +194,8 @@ add_host_to_known_hosts (const struct GNUNET_PeerIdentity *identity)
215 struct GNUNET_TIME_Absolute now; 194 struct GNUNET_TIME_Absolute now;
216 char *fn; 195 char *fn;
217 196
218 entry = lookup_host_entry (identity); 197 entry = GNUNET_CONTAINER_multihashmap_get (hostmap,
198 &identity->hashPubKey);
219 if (entry != NULL) 199 if (entry != NULL)
220 return; 200 return;
221 GNUNET_STATISTICS_update (stats, 201 GNUNET_STATISTICS_update (stats,
@@ -250,8 +230,10 @@ add_host_to_known_hosts (const struct GNUNET_PeerIdentity *identity)
250 } 230 }
251 } 231 }
252 GNUNET_free (fn); 232 GNUNET_free (fn);
253 entry->next = hosts; 233 GNUNET_CONTAINER_multihashmap_put (hostmap,
254 hosts = entry; 234 &identity->hashPubKey,
235 entry,
236 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY);
255 notify_all (entry); 237 notify_all (entry);
256} 238}
257 239
@@ -275,7 +257,8 @@ remove_garbage (const char *fullname)
275 257
276 258
277static int 259static int
278hosts_directory_scan_callback (void *cls, const char *fullname) 260hosts_directory_scan_callback (void *cls,
261 const char *fullname)
279{ 262{
280 unsigned int *matched = cls; 263 unsigned int *matched = cls;
281 struct GNUNET_PeerIdentity identity; 264 struct GNUNET_PeerIdentity identity;
@@ -350,7 +333,8 @@ bind_address (const struct GNUNET_PeerIdentity *peer,
350 struct GNUNET_TIME_Absolute delta; 333 struct GNUNET_TIME_Absolute delta;
351 334
352 add_host_to_known_hosts (peer); 335 add_host_to_known_hosts (peer);
353 host = lookup_host_entry (peer); 336 host = GNUNET_CONTAINER_multihashmap_get (hostmap,
337 &peer->hashPubKey);
354 GNUNET_assert (host != NULL); 338 GNUNET_assert (host != NULL);
355 if (host->hello == NULL) 339 if (host->hello == NULL)
356 { 340 {
@@ -383,63 +367,43 @@ bind_address (const struct GNUNET_PeerIdentity *peer,
383} 367}
384 368
385 369
370
386/** 371/**
387 * Do transmit info either for only the host matching the given 372 * Do transmit info about peer to given host.
388 * argument or for all known hosts.
389 * 373 *
390 * @param only NULL to hit all hosts, otherwise specifies a particular target 374 * @param cls NULL to hit all hosts, otherwise specifies a particular target
391 * @param client who is making the request (and will thus receive our confirmation) 375 * @param key hostID
376 * @param value information to transmit
377 * @return GNUNET_YES (continue to iterate)
392 */ 378 */
393static void 379static int
394send_to_each_host (const struct GNUNET_PeerIdentity *only, 380add_to_tc (void *cls,
395 struct GNUNET_SERVER_Client *client) 381 const GNUNET_HashCode *key,
382 void *value)
396{ 383{
397 struct HostEntry *pos; 384 struct GNUNET_SERVER_TransmitContext *tc = cls;
385 struct HostEntry *pos = value;
398 struct InfoMessage *im; 386 struct InfoMessage *im;
399 uint16_t hs; 387 uint16_t hs;
400 char buf[GNUNET_SERVER_MAX_MESSAGE_SIZE - 1]; 388 char buf[GNUNET_SERVER_MAX_MESSAGE_SIZE - 1];
401 struct GNUNET_SERVER_TransmitContext *tc;
402 int match;
403 389
404 tc = GNUNET_SERVER_transmit_context_create (client); 390 hs = 0;
405 match = GNUNET_NO; 391 im = (struct InfoMessage *) buf;
406 pos = hosts; 392 if (pos->hello != NULL)
407 while (pos != NULL)
408 { 393 {
409 if ((only == NULL) || 394 hs = GNUNET_HELLO_size (pos->hello);
410 (0 == 395 GNUNET_assert (hs <
411 memcmp (only, &pos->identity, 396 GNUNET_SERVER_MAX_MESSAGE_SIZE -
412 sizeof (struct GNUNET_PeerIdentity)))) 397 sizeof (struct InfoMessage));
413 { 398 memcpy (&im[1], pos->hello, hs);
414 hs = 0;
415 im = (struct InfoMessage *) buf;
416 if (pos->hello != NULL)
417 {
418 hs = GNUNET_HELLO_size (pos->hello);
419 GNUNET_assert (hs <
420 GNUNET_SERVER_MAX_MESSAGE_SIZE -
421 sizeof (struct InfoMessage));
422 memcpy (&im[1], pos->hello, hs);
423 match = GNUNET_YES;
424 }
425 im->header.type = htons (GNUNET_MESSAGE_TYPE_PEERINFO_INFO);
426 im->header.size = htons (sizeof (struct InfoMessage) + hs);
427 im->reserved = htonl (0);
428 im->peer = pos->identity;
429 GNUNET_SERVER_transmit_context_append_message (tc,
430 &im->header);
431 }
432 pos = pos->next;
433 } 399 }
434 if ( (only != NULL) && 400 im->header.type = htons (GNUNET_MESSAGE_TYPE_PEERINFO_INFO);
435 (match == GNUNET_NO) ) 401 im->header.size = htons (sizeof (struct InfoMessage) + hs);
436 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 402 im->reserved = htonl (0);
437 "No `%s' message was found for peer `%4s'\n", 403 im->peer = pos->identity;
438 "HELLO", 404 GNUNET_SERVER_transmit_context_append_message (tc,
439 GNUNET_i2s (only)); 405 &im->header);
440 GNUNET_SERVER_transmit_context_append_data (tc, NULL, 0, 406 return GNUNET_YES;
441 GNUNET_MESSAGE_TYPE_PEERINFO_INFO_END);
442 GNUNET_SERVER_transmit_context_run (tc, GNUNET_TIME_UNIT_FOREVER_REL);
443} 407}
444 408
445 409
@@ -500,7 +464,6 @@ cron_clean_data_hosts (void *cls,
500 now = GNUNET_TIME_absolute_get (); 464 now = GNUNET_TIME_absolute_get ();
501 GNUNET_DISK_directory_scan (networkIdDirectory, 465 GNUNET_DISK_directory_scan (networkIdDirectory,
502 &discard_hosts_helper, &now); 466 &discard_hosts_helper, &now);
503
504 GNUNET_SCHEDULER_add_delayed (tc->sched, 467 GNUNET_SCHEDULER_add_delayed (tc->sched,
505 DATA_HOST_CLEAN_FREQ, 468 DATA_HOST_CLEAN_FREQ,
506 &cron_clean_data_hosts, NULL); 469 &cron_clean_data_hosts, NULL);
@@ -553,6 +516,7 @@ handle_get (void *cls,
553 const struct GNUNET_MessageHeader *message) 516 const struct GNUNET_MessageHeader *message)
554{ 517{
555 const struct ListPeerMessage *lpm; 518 const struct ListPeerMessage *lpm;
519 struct GNUNET_SERVER_TransmitContext *tc;
556 520
557 lpm = (const struct ListPeerMessage *) message; 521 lpm = (const struct ListPeerMessage *) message;
558#if DEBUG_PEERINFO 522#if DEBUG_PEERINFO
@@ -561,7 +525,14 @@ handle_get (void *cls,
561 "GET", 525 "GET",
562 GNUNET_i2s (&lpm->peer)); 526 GNUNET_i2s (&lpm->peer));
563#endif 527#endif
564 send_to_each_host (&lpm->peer, client); 528 tc = GNUNET_SERVER_transmit_context_create (client);
529 GNUNET_CONTAINER_multihashmap_get_multiple (hostmap,
530 &lpm->peer.hashPubKey,
531 &add_to_tc,
532 tc);
533 GNUNET_SERVER_transmit_context_append_data (tc, NULL, 0,
534 GNUNET_MESSAGE_TYPE_PEERINFO_INFO_END);
535 GNUNET_SERVER_transmit_context_run (tc, GNUNET_TIME_UNIT_FOREVER_REL);
565} 536}
566 537
567 538
@@ -577,12 +548,39 @@ handle_get_all (void *cls,
577 struct GNUNET_SERVER_Client *client, 548 struct GNUNET_SERVER_Client *client,
578 const struct GNUNET_MessageHeader *message) 549 const struct GNUNET_MessageHeader *message)
579{ 550{
551 struct GNUNET_SERVER_TransmitContext *tc;
552
580#if DEBUG_PEERINFO 553#if DEBUG_PEERINFO
581 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 554 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
582 "`%s' message received\n", 555 "`%s' message received\n",
583 "GET_ALL"); 556 "GET_ALL");
584#endif 557#endif
585 send_to_each_host (NULL, client); 558 tc = GNUNET_SERVER_transmit_context_create (client);
559 GNUNET_CONTAINER_multihashmap_iterate (hostmap,
560 &add_to_tc,
561 tc);
562 GNUNET_SERVER_transmit_context_append_data (tc, NULL, 0,
563 GNUNET_MESSAGE_TYPE_PEERINFO_INFO_END);
564 GNUNET_SERVER_transmit_context_run (tc, GNUNET_TIME_UNIT_FOREVER_REL);
565}
566
567
568static int
569do_notify_entry (void *cls,
570 const GNUNET_HashCode *key,
571 void *value)
572{
573 struct GNUNET_SERVER_Client *client = cls;
574 struct HostEntry *he = value;
575 struct InfoMessage *msg;
576
577 msg = make_info_message (he);
578 GNUNET_SERVER_notification_context_unicast (notify_list,
579 client,
580 &msg->header,
581 GNUNET_NO);
582 GNUNET_free (msg);
583 return GNUNET_YES;
586} 584}
587 585
588 586
@@ -595,12 +593,9 @@ handle_get_all (void *cls,
595 */ 593 */
596static void 594static void
597handle_notify (void *cls, 595handle_notify (void *cls,
598 struct GNUNET_SERVER_Client *client, 596 struct GNUNET_SERVER_Client *client,
599 const struct GNUNET_MessageHeader *message) 597 const struct GNUNET_MessageHeader *message)
600{ 598{
601 struct InfoMessage *msg;
602 struct HostEntry *pos;
603
604#if DEBUG_PEERINFO 599#if DEBUG_PEERINFO
605 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 600 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
606 "`%s' message received\n", 601 "`%s' message received\n",
@@ -608,20 +603,22 @@ handle_notify (void *cls,
608#endif 603#endif
609 GNUNET_SERVER_notification_context_add (notify_list, 604 GNUNET_SERVER_notification_context_add (notify_list,
610 client); 605 client);
611 pos = hosts; 606 GNUNET_CONTAINER_multihashmap_iterate (hostmap,
612 while (NULL != pos) 607 &do_notify_entry,
613 { 608 client);
614 msg = make_info_message (pos);
615 GNUNET_SERVER_notification_context_unicast (notify_list,
616 client,
617 &msg->header,
618 GNUNET_NO);
619 GNUNET_free (msg);
620 pos = pos->next;
621 }
622} 609}
623 610
624 611
612static int
613free_host_entry (void *cls,
614 const GNUNET_HashCode *key,
615 void *value)
616{
617 struct HostEntry *he = value;
618 GNUNET_free (he);
619 return GNUNET_YES;
620}
621
625/** 622/**
626 * Clean up our state. Called during shutdown. 623 * Clean up our state. Called during shutdown.
627 * 624 *
@@ -634,6 +631,10 @@ shutdown_task (void *cls,
634{ 631{
635 GNUNET_SERVER_notification_context_destroy (notify_list); 632 GNUNET_SERVER_notification_context_destroy (notify_list);
636 notify_list = NULL; 633 notify_list = NULL;
634 GNUNET_CONTAINER_multihashmap_iterate (hostmap,
635 &free_host_entry,
636 NULL);
637 GNUNET_CONTAINER_multihashmap_destroy (hostmap);
637 if (stats != NULL) 638 if (stats != NULL)
638 { 639 {
639 GNUNET_STATISTICS_destroy (stats, GNUNET_NO); 640 GNUNET_STATISTICS_destroy (stats, GNUNET_NO);
@@ -666,6 +667,8 @@ run (void *cls,
666 sizeof (struct GNUNET_MessageHeader)}, 667 sizeof (struct GNUNET_MessageHeader)},
667 {NULL, NULL, 0, 0} 668 {NULL, NULL, 0, 0}
668 }; 669 };
670
671 hostmap = GNUNET_CONTAINER_multihashmap_create (1024);
669 stats = GNUNET_STATISTICS_create (sched, "peerinfo", cfg); 672 stats = GNUNET_STATISTICS_create (sched, "peerinfo", cfg);
670 notify_list = GNUNET_SERVER_notification_context_create (server, 0); 673 notify_list = GNUNET_SERVER_notification_context_create (server, 0);
671 GNUNET_assert (GNUNET_OK == 674 GNUNET_assert (GNUNET_OK ==
diff --git a/src/peerinfo/perf_peerinfo_api.c b/src/peerinfo/perf_peerinfo_api.c
new file mode 100755
index 000000000..1c0df0332
--- /dev/null
+++ b/src/peerinfo/perf_peerinfo_api.c
@@ -0,0 +1,217 @@
1/*
2 This file is part of GNUnet.
3 (C) 2004, 2009 Christian Grothoff (and other contributing authors)
4
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 3, or (at your
8 option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
14
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 Boston, MA 02111-1307, USA.
19*/
20
21/**
22 * @file peerinfo/test_peerinfo_hammer.c
23 * @brief testcase for peerinfo_api.c, hopefully hammer the peerinfo service
24 * @author Nathan Evans
25 */
26
27#include "platform.h"
28#include "gnunet_hello_lib.h"
29#include "gnunet_getopt_lib.h"
30#include "gnunet_os_lib.h"
31#include "gnunet_peerinfo_service.h"
32#include "gnunet_program_lib.h"
33#include "gnunet_time_lib.h"
34#include "peerinfo.h"
35
36#define START_SERVICE 1
37
38#define NUM_REQUESTS 5000
39
40static struct GNUNET_SCHEDULER_Handle *sched;
41
42static const struct GNUNET_CONFIGURATION_Handle *cfg;
43
44static struct GNUNET_PEERINFO_IteratorContext *ic[NUM_REQUESTS];
45
46static struct GNUNET_PEERINFO_Handle *h;
47
48static unsigned int numpeers;
49
50static int
51check_it (void *cls,
52 const char *tname,
53 struct GNUNET_TIME_Absolute expiration,
54 const void *addr, uint16_t addrlen)
55{
56 if (addrlen > 0)
57 {
58#if DEBUG
59 fprintf (stderr,
60 "name: %s, addr: %s\n",
61 tname,
62 (const char*) addr);
63#endif
64 }
65 return GNUNET_OK;
66}
67
68
69static size_t
70address_generator (void *cls, size_t max, void *buf)
71{
72 size_t *agc = cls;
73 size_t ret;
74 char *address;
75
76 if (*agc == 0)
77 return 0;
78
79 GNUNET_asprintf(&address, "Address%d", *agc);
80
81 ret = GNUNET_HELLO_add_address ("peerinfotest",
82 GNUNET_TIME_relative_to_absolute
83 (GNUNET_TIME_UNIT_HOURS), address, strlen(address) + 1,
84 buf, max);
85 *agc = 0;
86 return ret;
87}
88
89
90static void
91add_peer (size_t i)
92{
93 struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded pkey;
94 struct GNUNET_PeerIdentity pid;
95 struct GNUNET_HELLO_Message *h2;
96 size_t agc;
97
98 agc = 2;
99 memset (&pkey, i, sizeof (pkey));
100 GNUNET_CRYPTO_hash (&pkey, sizeof (pkey), &pid.hashPubKey);
101 h2 = GNUNET_HELLO_create (&pkey, &address_generator, &i);
102 GNUNET_PEERINFO_add_peer (h, h2);
103 GNUNET_free (h2);
104}
105
106
107static void
108process (void *cls,
109 const struct GNUNET_PeerIdentity *peer,
110 const struct GNUNET_HELLO_Message *hello)
111{
112 if (peer == NULL)
113 {
114#if DEBUG
115 fprintf(stderr, "Process received NULL response\n");
116#endif
117 }
118 else
119 {
120#if DEBUG
121 fprintf(stderr, "Processed a peer\n");
122#endif
123 numpeers++;
124 if (0 && (hello != NULL))
125 GNUNET_HELLO_iterate_addresses (hello, GNUNET_NO, &check_it, NULL);
126
127 }
128}
129
130
131static void
132run (void *cls,
133 struct GNUNET_SCHEDULER_Handle *s,
134 char *const *args,
135 const char *cfgfile,
136 const struct GNUNET_CONFIGURATION_Handle *c)
137{
138 size_t i;
139 sched = s;
140 cfg = c;
141 h = GNUNET_PEERINFO_connect (sched, cfg);
142
143 for (i = 0; i < NUM_REQUESTS; i++)
144 {
145 add_peer (i);
146 ic[i] = GNUNET_PEERINFO_iterate (h,
147 NULL,
148 GNUNET_TIME_relative_multiply
149 (GNUNET_TIME_UNIT_SECONDS, 30),
150 &process, cls);
151 }
152 fprintf (stderr,
153 "Issued %u requests\n",
154 NUM_REQUESTS);
155}
156
157static int
158check ()
159{
160 int ok = 3;
161 char *const argv[] = { "test-peerinfo-hammer",
162 "-c",
163 "test_peerinfo_api_data.conf",
164#if DEBUG_PEERINFO
165 "-L", "DEBUG",
166#endif
167 NULL
168 };
169#if START_SERVICE
170 pid_t pid;
171 struct GNUNET_GETOPT_CommandLineOption options[] = {
172 GNUNET_GETOPT_OPTION_END
173 };
174 pid = GNUNET_OS_start_process (NULL, NULL, "gnunet-service-peerinfo",
175 "gnunet-service-peerinfo",
176#if DEBUG_PEERINFO
177 "-L", "DEBUG",
178#endif
179 "-c", "test_peerinfo_api_data.conf", NULL);
180#endif
181 GNUNET_PROGRAM_run ((sizeof (argv) / sizeof (char *)) - 1,
182 argv, "test-peerinfo-api", "nohelp",
183 options, &run, &ok);
184 fprintf (stderr,
185 "Processed %u/%u peers\n",
186 numpeers,
187 NUM_REQUESTS);
188#if START_SERVICE
189 if (0 != PLIBC_KILL (pid, SIGTERM))
190 {
191 GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "kill");
192 ok = 1;
193 }
194 GNUNET_OS_process_wait(pid);
195#endif
196 return ok;
197}
198
199
200int
201main (int argc, char *argv[])
202{
203 int ret = 0;
204
205 GNUNET_log_setup ("test_peerinfo_api",
206#if DEBUG_PEERINFO
207 "DEBUG",
208#else
209 "WARNING",
210#endif
211 NULL);
212 ret = check ();
213 GNUNET_DISK_directory_remove ("/tmp/test-gnunet-peerinfo");
214 return ret;
215}
216
217/* end of test_peerinfo_hammer.c */
diff --git a/src/peerinfo/test_peerinfo_api.c b/src/peerinfo/test_peerinfo_api.c
index 47304d549..a3c2c99eb 100644
--- a/src/peerinfo/test_peerinfo_api.c
+++ b/src/peerinfo/test_peerinfo_api.c
@@ -208,7 +208,7 @@ main (int argc, char *argv[])
208#endif 208#endif
209 NULL); 209 NULL);
210 ret = check (); 210 ret = check ();
211 GNUNET_DISK_directory_remove ("/tmp/test-gnunetd-peerinfo"); 211 GNUNET_DISK_directory_remove ("/tmp/test-gnunet-peerinfo");
212 return ret; 212 return ret;
213} 213}
214 214
diff --git a/src/peerinfo/test_peerinfo_api_data.conf b/src/peerinfo/test_peerinfo_api_data.conf
index a81ffccb9..998df571a 100644
--- a/src/peerinfo/test_peerinfo_api_data.conf
+++ b/src/peerinfo/test_peerinfo_api_data.conf
@@ -1,5 +1,6 @@
1[PATHS] 1[PATHS]
2SERVICEHOME = /tmp/test-gnunetd-peerinfo/ 2SERVICEHOME = /tmp/test-gnunet-peerinfo/
3 3
4[peerinfo] 4[peerinfo]
5PORT = 22354 5PORT = 22354
6DEBUG = NO
diff --git a/src/util/network.c b/src/util/network.c
index 886813426..9b96436db 100644
--- a/src/util/network.c
+++ b/src/util/network.c
@@ -836,7 +836,6 @@ void
836GNUNET_NETWORK_fdset_handle_set (struct GNUNET_NETWORK_FDSet *fds, 836GNUNET_NETWORK_fdset_handle_set (struct GNUNET_NETWORK_FDSet *fds,
837 const struct GNUNET_DISK_FileHandle *h) 837 const struct GNUNET_DISK_FileHandle *h)
838{ 838{
839
840#ifdef MINGW 839#ifdef MINGW
841 HANDLE hw; 840 HANDLE hw;
842 GNUNET_DISK_internal_file_handle_ (h, &hw, sizeof (HANDLE)); 841 GNUNET_DISK_internal_file_handle_ (h, &hw, sizeof (HANDLE));
diff --git a/src/util/scheduler.c b/src/util/scheduler.c
index 1352fe0d8..0ff6f9612 100644
--- a/src/util/scheduler.c
+++ b/src/util/scheduler.c
@@ -174,6 +174,21 @@ struct GNUNET_SCHEDULER_Handle
174 struct Task *pending; 174 struct Task *pending;
175 175
176 /** 176 /**
177 * List of tasks waiting ONLY for a timeout event.
178 * Sorted by timeout (earliest first). Used so that
179 * we do not traverse the list of these tasks when
180 * building select sets (we just look at the head
181 * to determine the respective timeout ONCE).
182 */
183 struct Task *pending_timeout;
184
185 /**
186 * Last inserted task waiting ONLY for a timeout event.
187 * Used to (heuristically) speed up insertion.
188 */
189 struct Task *pending_timeout_last;
190
191 /**
177 * ID of the task that is running right now. 192 * ID of the task that is running right now.
178 */ 193 */
179 struct Task *active_task; 194 struct Task *active_task;
@@ -274,6 +289,15 @@ is_pending (struct GNUNET_SCHEDULER_Handle *sched,
274 min = pos->id; 289 min = pos->id;
275 pos = pos->next; 290 pos = pos->next;
276 } 291 }
292 pos = sched->pending_timeout;
293 while (pos != NULL)
294 {
295 if (pos->id == id)
296 return GNUNET_YES;
297 if (pos->id < min)
298 min = pos->id;
299 pos = pos->next;
300 }
277 for (p = 0; p < GNUNET_SCHEDULER_PRIORITY_COUNT; p++) 301 for (p = 0; p < GNUNET_SCHEDULER_PRIORITY_COUNT; p++)
278 { 302 {
279 pos = sched->ready[p]; 303 pos = sched->ready[p];
@@ -306,7 +330,19 @@ update_sets (struct GNUNET_SCHEDULER_Handle *sched,
306 struct GNUNET_TIME_Relative *timeout) 330 struct GNUNET_TIME_Relative *timeout)
307{ 331{
308 struct Task *pos; 332 struct Task *pos;
333 struct GNUNET_TIME_Absolute now;
334 struct GNUNET_TIME_Relative to;
309 335
336 now = GNUNET_TIME_absolute_get ();
337 pos = sched->pending_timeout;
338 if (pos != NULL)
339 {
340 to = GNUNET_TIME_absolute_get_difference (now, pos->timeout);
341 if (timeout->value > to.value)
342 *timeout = to;
343 if (pos->reason != 0)
344 *timeout = GNUNET_TIME_UNIT_ZERO;
345 }
310 pos = sched->pending; 346 pos = sched->pending;
311 while (pos != NULL) 347 while (pos != NULL)
312 { 348 {
@@ -316,12 +352,9 @@ update_sets (struct GNUNET_SCHEDULER_Handle *sched,
316 pos = pos->next; 352 pos = pos->next;
317 continue; 353 continue;
318 } 354 }
319
320 if (pos->timeout.value != GNUNET_TIME_UNIT_FOREVER_ABS.value) 355 if (pos->timeout.value != GNUNET_TIME_UNIT_FOREVER_ABS.value)
321 { 356 {
322 struct GNUNET_TIME_Relative to; 357 to = GNUNET_TIME_absolute_get_difference (now, pos->timeout);
323
324 to = GNUNET_TIME_absolute_get_remaining (pos->timeout);
325 if (timeout->value > to.value) 358 if (timeout->value > to.value)
326 *timeout = to; 359 *timeout = to;
327 } 360 }
@@ -384,24 +417,33 @@ is_ready (struct GNUNET_SCHEDULER_Handle *sched,
384 const struct GNUNET_NETWORK_FDSet *rs, 417 const struct GNUNET_NETWORK_FDSet *rs,
385 const struct GNUNET_NETWORK_FDSet *ws) 418 const struct GNUNET_NETWORK_FDSet *ws)
386{ 419{
420 enum GNUNET_SCHEDULER_Reason reason;
421
422 reason = task->reason;
387 if (now.value >= task->timeout.value) 423 if (now.value >= task->timeout.value)
388 task->reason |= GNUNET_SCHEDULER_REASON_TIMEOUT; 424 reason |= GNUNET_SCHEDULER_REASON_TIMEOUT;
389 if ( (0 == (task->reason & GNUNET_SCHEDULER_REASON_READ_READY)) && 425 if ( (0 == (reason & GNUNET_SCHEDULER_REASON_READ_READY)) &&
390 ( (GNUNET_YES == GNUNET_NETWORK_fdset_test_native (rs, task->read_fd)) || 426 ( ( (task->read_fd != -1) &&
427 (GNUNET_YES == GNUNET_NETWORK_fdset_test_native (rs, task->read_fd)) ) ||
391 (set_overlaps (rs, task->read_set) ) ) ) 428 (set_overlaps (rs, task->read_set) ) ) )
392 task->reason |= GNUNET_SCHEDULER_REASON_READ_READY; 429 reason |= GNUNET_SCHEDULER_REASON_READ_READY;
393 if ((0 == (task->reason & GNUNET_SCHEDULER_REASON_WRITE_READY)) && 430 if ((0 == (reason & GNUNET_SCHEDULER_REASON_WRITE_READY)) &&
394 ( (GNUNET_YES == GNUNET_NETWORK_fdset_test_native (ws, task->write_fd)) || 431 ( ( (task->write_fd != -1) &&
432 (GNUNET_YES == GNUNET_NETWORK_fdset_test_native (ws, task->write_fd)) ) ||
395 (set_overlaps (ws, task->write_set) ) ) ) 433 (set_overlaps (ws, task->write_set) ) ) )
396 task->reason |= GNUNET_SCHEDULER_REASON_WRITE_READY; 434 reason |= GNUNET_SCHEDULER_REASON_WRITE_READY;
397 if (task->reason == 0) 435 if (reason == 0)
398 return GNUNET_NO; /* not ready */ 436 return GNUNET_NO; /* not ready */
399 if (task->prereq_id != GNUNET_SCHEDULER_NO_TASK) 437 if (task->prereq_id != GNUNET_SCHEDULER_NO_TASK)
400 { 438 {
401 if (GNUNET_YES == is_pending (sched, task->prereq_id)) 439 if (GNUNET_YES == is_pending (sched, task->prereq_id))
402 return GNUNET_NO; /* prereq waiting */ 440 {
403 task->reason |= GNUNET_SCHEDULER_REASON_PREREQ_DONE; 441 task->reason = reason;
442 return GNUNET_NO; /* prereq waiting */
443 }
444 reason |= GNUNET_SCHEDULER_REASON_PREREQ_DONE;
404 } 445 }
446 task->reason = reason;
405 return GNUNET_YES; 447 return GNUNET_YES;
406} 448}
407 449
@@ -413,7 +455,8 @@ is_ready (struct GNUNET_SCHEDULER_Handle *sched,
413 * @param task task ready for execution 455 * @param task task ready for execution
414 */ 456 */
415static void 457static void
416queue_ready_task (struct GNUNET_SCHEDULER_Handle *handle, struct Task *task) 458queue_ready_task (struct GNUNET_SCHEDULER_Handle *handle,
459 struct Task *task)
417{ 460{
418 enum GNUNET_SCHEDULER_Priority p = task->priority; 461 enum GNUNET_SCHEDULER_Priority p = task->priority;
419 if (0 != (task->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN)) 462 if (0 != (task->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
@@ -444,6 +487,20 @@ check_ready (struct GNUNET_SCHEDULER_Handle *handle,
444 487
445 now = GNUNET_TIME_absolute_get (); 488 now = GNUNET_TIME_absolute_get ();
446 prev = NULL; 489 prev = NULL;
490 pos = handle->pending_timeout;
491 while (pos != NULL)
492 {
493 next = pos->next;
494 if (now.value >= pos->timeout.value)
495 pos->reason |= GNUNET_SCHEDULER_REASON_TIMEOUT;
496 if (0 == pos->reason)
497 break;
498 handle->pending_timeout = next;
499 if (handle->pending_timeout_last == pos)
500 handle->pending_timeout_last = NULL;
501 queue_ready_task (handle, pos);
502 pos = next;
503 }
447 pos = handle->pending; 504 pos = handle->pending;
448 while (pos != NULL) 505 while (pos != NULL)
449 { 506 {
@@ -484,6 +541,15 @@ GNUNET_SCHEDULER_shutdown (struct GNUNET_SCHEDULER_Handle *sched)
484 struct Task *pos; 541 struct Task *pos;
485 int i; 542 int i;
486 543
544 pos = sched->pending_timeout;
545 while (pos != NULL)
546 {
547 pos->reason |= GNUNET_SCHEDULER_REASON_SHUTDOWN;
548 /* we don't move the task into the ready queue yet; check_ready
549 will do that later, possibly adding additional
550 readiness-factors */
551 pos = pos->next;
552 }
487 pos = sched->pending; 553 pos = sched->pending;
488 while (pos != NULL) 554 while (pos != NULL)
489 { 555 {
@@ -615,7 +681,7 @@ run_ready (struct GNUNET_SCHEDULER_Handle *sched,
615 destroy_task (pos); 681 destroy_task (pos);
616 sched->tasks_run++; 682 sched->tasks_run++;
617 } 683 }
618 while ((sched->pending == NULL) || (p >= sched->max_priority_added)); 684 while ( (sched->pending == NULL) || (p >= sched->max_priority_added) );
619} 685}
620 686
621/** 687/**
@@ -700,7 +766,9 @@ GNUNET_SCHEDULER_run (GNUNET_SCHEDULER_Task task, void *task_cls)
700 GNUNET_SCHEDULER_REASON_STARTUP); 766 GNUNET_SCHEDULER_REASON_STARTUP);
701 last_tr = 0; 767 last_tr = 0;
702 busy_wait_warning = 0; 768 busy_wait_warning = 0;
703 while ((sched.pending != NULL) || (sched.ready_count > 0)) 769 while ((sched.pending != NULL) ||
770 (sched.pending_timeout != NULL) ||
771 (sched.ready_count > 0))
704 { 772 {
705 GNUNET_NETWORK_fdset_zero (rs); 773 GNUNET_NETWORK_fdset_zero (rs);
706 GNUNET_NETWORK_fdset_zero (ws); 774 GNUNET_NETWORK_fdset_zero (ws);
@@ -832,8 +900,10 @@ GNUNET_SCHEDULER_cancel (struct GNUNET_SCHEDULER_Handle *sched,
832 struct Task *t; 900 struct Task *t;
833 struct Task *prev; 901 struct Task *prev;
834 enum GNUNET_SCHEDULER_Priority p; 902 enum GNUNET_SCHEDULER_Priority p;
903 int to;
835 void *ret; 904 void *ret;
836 905
906 to = 0;
837 prev = NULL; 907 prev = NULL;
838 t = sched->pending; 908 t = sched->pending;
839 while (t != NULL) 909 while (t != NULL)
@@ -843,6 +913,21 @@ GNUNET_SCHEDULER_cancel (struct GNUNET_SCHEDULER_Handle *sched,
843 prev = t; 913 prev = t;
844 t = t->next; 914 t = t->next;
845 } 915 }
916 if (t == NULL)
917 {
918 prev = NULL;
919 to = 1;
920 t = sched->pending_timeout;
921 while (t != NULL)
922 {
923 if (t->id == task)
924 break;
925 prev = t;
926 t = t->next;
927 }
928 if (sched->pending_timeout_last == t)
929 sched->pending_timeout_last = NULL;
930 }
846 p = 0; 931 p = 0;
847 while (t == NULL) 932 while (t == NULL)
848 { 933 {
@@ -864,12 +949,25 @@ GNUNET_SCHEDULER_cancel (struct GNUNET_SCHEDULER_Handle *sched,
864 if (prev == NULL) 949 if (prev == NULL)
865 { 950 {
866 if (p == 0) 951 if (p == 0)
867 sched->pending = t->next; 952 {
953 if (to == 0)
954 {
955 sched->pending = t->next;
956 }
957 else
958 {
959 sched->pending_timeout = t->next;
960 }
961 }
868 else 962 else
869 sched->ready[p] = t->next; 963 {
964 sched->ready[p] = t->next;
965 }
870 } 966 }
871 else 967 else
872 prev->next = t->next; 968 {
969 prev->next = t->next;
970 }
873 ret = t->callback_cls; 971 ret = t->callback_cls;
874#if DEBUG_TASKS 972#if DEBUG_TASKS
875 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 973 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
@@ -998,10 +1096,84 @@ GNUNET_SCHEDULER_add_delayed (struct GNUNET_SCHEDULER_Handle * sched,
998 struct GNUNET_TIME_Relative delay, 1096 struct GNUNET_TIME_Relative delay,
999 GNUNET_SCHEDULER_Task task, void *task_cls) 1097 GNUNET_SCHEDULER_Task task, void *task_cls)
1000{ 1098{
1099#if 1
1100 /* new, optimized version */
1101 struct Task *t;
1102 struct Task *pos;
1103 struct Task *prev;
1104#if EXECINFO
1105 void *backtrace_array[MAX_TRACE_DEPTH];
1106#endif
1107
1108 GNUNET_assert (NULL != task);
1109 t = GNUNET_malloc (sizeof (struct Task));
1110 t->callback = task;
1111 t->callback_cls = task_cls;
1112#if EXECINFO
1113 t->num_backtrace_strings = backtrace(backtrace_array, MAX_TRACE_DEPTH);
1114 t->backtrace_strings = backtrace_symbols(backtrace_array, t->num_backtrace_strings);
1115#endif
1116 t->read_fd = -1;
1117 t->write_fd = -1;
1118 t->id = ++sched->last_id;
1119#if PROFILE_DELAYS
1120 t->start_time = GNUNET_TIME_absolute_get ();
1121#endif
1122 t->timeout = GNUNET_TIME_relative_to_absolute (delay);
1123 t->priority = sched->current_priority;
1124 /* try tail first (optimization in case we are
1125 appending to a long list of tasks with timeouts) */
1126 prev = sched->pending_timeout_last;
1127 if (prev != NULL)
1128 {
1129 if (prev->timeout.value > t->timeout.value)
1130 prev = NULL;
1131 else
1132 pos = prev->next; /* heuristic success! */
1133 }
1134 if (prev == NULL)
1135 {
1136 /* heuristic failed, do traversal of timeout list */
1137 pos = sched->pending_timeout;
1138 }
1139 while ( (pos != NULL) &&
1140 ( (pos->timeout.value <= t->timeout.value) ||
1141 (pos->reason != 0) ) )
1142 {
1143 prev = pos;
1144 pos = pos->next;
1145 }
1146 if (prev == NULL)
1147 sched->pending_timeout = t;
1148 else
1149 prev->next = t;
1150 t->next = pos;
1151 /* hyper-optimization... */
1152 sched->pending_timeout_last = t;
1153
1154#if DEBUG_TASKS
1155 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1156 "Adding task: %llu / %p\n", t->id, t->callback_cls);
1157#endif
1158#if EXECINFO
1159 int i;
1160
1161 for (i=0;i<t->num_backtrace_strings;i++)
1162 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1163 "Task %u trace %d: %s\n",
1164 t->id,
1165 i,
1166 t->backtrace_strings[i]);
1167#endif
1168 return t->id;
1169
1170#else
1171 /* unoptimized version */
1001 return GNUNET_SCHEDULER_add_select (sched, 1172 return GNUNET_SCHEDULER_add_select (sched,
1002 GNUNET_SCHEDULER_PRIORITY_KEEP, 1173 GNUNET_SCHEDULER_PRIORITY_KEEP,
1003 GNUNET_SCHEDULER_NO_TASK, delay, 1174 GNUNET_SCHEDULER_NO_TASK, delay,
1004 NULL, NULL, task, task_cls); 1175 NULL, NULL, task, task_cls);
1176#endif
1005} 1177}
1006 1178
1007 1179
@@ -1045,7 +1217,7 @@ GNUNET_SCHEDULER_add_now (struct GNUNET_SCHEDULER_Handle *sched,
1045 * && (delay-ready 1217 * && (delay-ready
1046 * || any-rs-ready 1218 * || any-rs-ready
1047 * || any-ws-ready 1219 * || any-ws-ready
1048 * || (shutdown-active && run-on-shutdown) ) 1220 * || shutdown-active )
1049 * </code> 1221 * </code>
1050 * 1222 *
1051 * @param sched scheduler to use 1223 * @param sched scheduler to use