diff options
25 files changed, 437 insertions, 21 deletions
diff --git a/src/datastore/datastore.h b/src/datastore/datastore.h index f827f8766..55ca7c8e5 100644 --- a/src/datastore/datastore.h +++ b/src/datastore/datastore.h | |||
@@ -109,7 +109,7 @@ struct GetMessage | |||
109 | struct GNUNET_MessageHeader header; | 109 | struct GNUNET_MessageHeader header; |
110 | 110 | ||
111 | /** | 111 | /** |
112 | * Desired content type. | 112 | * Desired content type. (actually an enum GNUNET_BLOCK_Type) |
113 | */ | 113 | */ |
114 | uint32_t type GNUNET_PACKED; | 114 | uint32_t type GNUNET_PACKED; |
115 | 115 | ||
@@ -123,6 +123,25 @@ struct GetMessage | |||
123 | 123 | ||
124 | 124 | ||
125 | /** | 125 | /** |
126 | * Message to the datastore service asking about zero | ||
127 | * anonymity content. | ||
128 | */ | ||
129 | struct GetZeroAnonymityMessage | ||
130 | { | ||
131 | /** | ||
132 | * Type is GNUNET_MESSAGE_TYPE_DATASTORE_GET_ZERO_ANONYMITY. | ||
133 | */ | ||
134 | struct GNUNET_MessageHeader header; | ||
135 | |||
136 | /** | ||
137 | * Desired content type (actually an enum GNUNET_BLOCK_Type) | ||
138 | */ | ||
139 | uint32_t type GNUNET_PACKED; | ||
140 | |||
141 | }; | ||
142 | |||
143 | |||
144 | /** | ||
126 | * Message to the datastore service requesting an update | 145 | * Message to the datastore service requesting an update |
127 | * to the priority or expiration for some content. | 146 | * to the priority or expiration for some content. |
128 | */ | 147 | */ |
@@ -176,7 +195,7 @@ struct DataMessage | |||
176 | uint32_t size GNUNET_PACKED; | 195 | uint32_t size GNUNET_PACKED; |
177 | 196 | ||
178 | /** | 197 | /** |
179 | * Type of the item (NBO), zero for remove. | 198 | * Type of the item (NBO), zero for remove, (actually an enum GNUNET_BLOCK_Type) |
180 | */ | 199 | */ |
181 | uint32_t type GNUNET_PACKED; | 200 | uint32_t type GNUNET_PACKED; |
182 | 201 | ||
diff --git a/src/datastore/datastore_api.c b/src/datastore/datastore_api.c index 18aa0bcdc..a15bc183a 100644 --- a/src/datastore/datastore_api.c +++ b/src/datastore/datastore_api.c | |||
@@ -1205,6 +1205,57 @@ GNUNET_DATASTORE_get_random (struct GNUNET_DATASTORE_Handle *h, | |||
1205 | } | 1205 | } |
1206 | 1206 | ||
1207 | 1207 | ||
1208 | /** | ||
1209 | * Get a zero-anonymity value from the datastore. | ||
1210 | * | ||
1211 | * @param h handle to the datastore | ||
1212 | * @param queue_priority ranking of this request in the priority queue | ||
1213 | * @param max_queue_size at what queue size should this request be dropped | ||
1214 | * (if other requests of higher priority are in the queue) | ||
1215 | * @param timeout how long to wait at most for a response | ||
1216 | * @param type allowed type for the operation | ||
1217 | * @param iter function to call on a random value; it | ||
1218 | * will be called once with a value (if available) | ||
1219 | * and always once with a value of NULL. | ||
1220 | * @param iter_cls closure for iter | ||
1221 | * @return NULL if the entry was not queued, otherwise a handle that can be used to | ||
1222 | * cancel; note that even if NULL is returned, the callback will be invoked | ||
1223 | * (or rather, will already have been invoked) | ||
1224 | */ | ||
1225 | struct GNUNET_DATASTORE_QueueEntry * | ||
1226 | GNUNET_DATASTORE_get_zero_anonymity (struct GNUNET_DATASTORE_Handle *h, | ||
1227 | unsigned int queue_priority, | ||
1228 | unsigned int max_queue_size, | ||
1229 | struct GNUNET_TIME_Relative timeout, | ||
1230 | enum GNUNET_BLOCK_Type type, | ||
1231 | GNUNET_DATASTORE_Iterator iter, | ||
1232 | void *iter_cls) | ||
1233 | { | ||
1234 | struct GNUNET_DATASTORE_QueueEntry *qe; | ||
1235 | struct GetZeroAnonymityMessage *m; | ||
1236 | union QueueContext qc; | ||
1237 | |||
1238 | #if DEBUG_DATASTORE | ||
1239 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
1240 | "Asked to get zero-anonymity entry in %llu ms\n", | ||
1241 | (unsigned long long) timeout.value); | ||
1242 | #endif | ||
1243 | qc.rc.iter = iter; | ||
1244 | qc.rc.iter_cls = iter_cls; | ||
1245 | qe = make_queue_entry (h, sizeof(struct GetZeroAnonymityMessage), | ||
1246 | queue_priority, max_queue_size, timeout, | ||
1247 | &process_result_message, &qc); | ||
1248 | if (qe == NULL) | ||
1249 | return NULL; | ||
1250 | m = (struct GetZeroAnonymityMessage*) &qe[1]; | ||
1251 | m->header.type = htons(GNUNET_MESSAGE_TYPE_DATASTORE_GET_RANDOM); | ||
1252 | m->header.size = htons(sizeof (struct GetZeroAnonymityMessage)); | ||
1253 | m->type = htonl ((uint32_t) type); | ||
1254 | process_queue (h); | ||
1255 | return qe; | ||
1256 | } | ||
1257 | |||
1258 | |||
1208 | 1259 | ||
1209 | /** | 1260 | /** |
1210 | * Iterate over the results for a particular key | 1261 | * Iterate over the results for a particular key |
diff --git a/src/datastore/gnunet-service-datastore.c b/src/datastore/gnunet-service-datastore.c index 8a896a7fb..2f505891c 100644 --- a/src/datastore/gnunet-service-datastore.c +++ b/src/datastore/gnunet-service-datastore.c | |||
@@ -1255,11 +1255,43 @@ handle_get_random (void *cls, | |||
1255 | GNUNET_NO); | 1255 | GNUNET_NO); |
1256 | GNUNET_SERVER_client_keep (client); | 1256 | GNUNET_SERVER_client_keep (client); |
1257 | plugin->api->iter_migration_order (plugin->api->cls, | 1257 | plugin->api->iter_migration_order (plugin->api->cls, |
1258 | 0, | 1258 | GNUNET_BLOCK_TYPE_ANY, |
1259 | &transmit_item, | 1259 | &transmit_item, |
1260 | client); | 1260 | client); |
1261 | } | 1261 | } |
1262 | 1262 | ||
1263 | /** | ||
1264 | * Handle GET_ZERO_ANONYMITY-message. | ||
1265 | * | ||
1266 | * @param cls closure | ||
1267 | * @param client identification of the client | ||
1268 | * @param message the actual message | ||
1269 | */ | ||
1270 | static void | ||
1271 | handle_get_zero_anonymity (void *cls, | ||
1272 | struct GNUNET_SERVER_Client *client, | ||
1273 | const struct GNUNET_MessageHeader *message) | ||
1274 | { | ||
1275 | const struct GetZeroAnonymityMessage * msg = (const struct GetZeroAnonymityMessage*) message; | ||
1276 | enum GNUNET_BLOCK_Type type; | ||
1277 | |||
1278 | type = (enum GNUNET_BLOCK_Type) ntohl (msg->type); | ||
1279 | #if DEBUG_DATASTORE | ||
1280 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
1281 | "Processing `%s' request\n", | ||
1282 | "GET_ZERO_ANONYMITY"); | ||
1283 | #endif | ||
1284 | GNUNET_STATISTICS_update (stats, | ||
1285 | gettext_noop ("# GET ZERO ANONYMITY requests received"), | ||
1286 | 1, | ||
1287 | GNUNET_NO); | ||
1288 | GNUNET_SERVER_client_keep (client); | ||
1289 | plugin->api->iter_zero_anonymity (plugin->api->cls, | ||
1290 | type, | ||
1291 | &transmit_item, | ||
1292 | client); | ||
1293 | } | ||
1294 | |||
1263 | 1295 | ||
1264 | /** | 1296 | /** |
1265 | * Context for the 'remove_callback'. | 1297 | * Context for the 'remove_callback'. |
@@ -1373,7 +1405,7 @@ handle_remove (void *cls, | |||
1373 | plugin->api->get (plugin->api->cls, | 1405 | plugin->api->get (plugin->api->cls, |
1374 | &dm->key, | 1406 | &dm->key, |
1375 | &vhash, | 1407 | &vhash, |
1376 | ntohl(dm->type), | 1408 | (enum GNUNET_BLOCK_Type) ntohl(dm->type), |
1377 | &remove_callback, | 1409 | &remove_callback, |
1378 | rc); | 1410 | rc); |
1379 | } | 1411 | } |
@@ -1675,6 +1707,8 @@ run (void *cls, | |||
1675 | {&handle_get, NULL, GNUNET_MESSAGE_TYPE_DATASTORE_GET, 0 }, | 1707 | {&handle_get, NULL, GNUNET_MESSAGE_TYPE_DATASTORE_GET, 0 }, |
1676 | {&handle_get_random, NULL, GNUNET_MESSAGE_TYPE_DATASTORE_GET_RANDOM, | 1708 | {&handle_get_random, NULL, GNUNET_MESSAGE_TYPE_DATASTORE_GET_RANDOM, |
1677 | sizeof(struct GNUNET_MessageHeader) }, | 1709 | sizeof(struct GNUNET_MessageHeader) }, |
1710 | {&handle_get_zero_anonymity, NULL, GNUNET_MESSAGE_TYPE_DATASTORE_GET_ZERO_ANONYMITY, | ||
1711 | sizeof(struct GetZeroAnonymityMessage) }, | ||
1678 | {&handle_remove, NULL, GNUNET_MESSAGE_TYPE_DATASTORE_REMOVE, 0 }, | 1712 | {&handle_remove, NULL, GNUNET_MESSAGE_TYPE_DATASTORE_REMOVE, 0 }, |
1679 | {&handle_drop, NULL, GNUNET_MESSAGE_TYPE_DATASTORE_DROP, | 1713 | {&handle_drop, NULL, GNUNET_MESSAGE_TYPE_DATASTORE_DROP, |
1680 | sizeof(struct GNUNET_MessageHeader) }, | 1714 | sizeof(struct GNUNET_MessageHeader) }, |
diff --git a/src/dht/dht.h b/src/dht/dht.h index 0f9d28fe4..bcfc5abd4 100644 --- a/src/dht/dht.h +++ b/src/dht/dht.h | |||
@@ -57,11 +57,6 @@ | |||
57 | /** | 57 | /** |
58 | * FIXME: document. | 58 | * FIXME: document. |
59 | */ | 59 | */ |
60 | #define DEFAULT_DHT_REPUBLISH_FREQUENCY GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MINUTES, 60) | ||
61 | |||
62 | /** | ||
63 | * FIXME: document. | ||
64 | */ | ||
65 | #define DHT_SEND_PRIORITY 4 | 60 | #define DHT_SEND_PRIORITY 4 |
66 | 61 | ||
67 | /** | 62 | /** |
diff --git a/src/dht/gnunet-service-dht.c b/src/dht/gnunet-service-dht.c index 5c49114ae..d216be67d 100644 --- a/src/dht/gnunet-service-dht.c +++ b/src/dht/gnunet-service-dht.c | |||
@@ -4668,7 +4668,7 @@ run (void *cls, | |||
4668 | malicious_put_frequency = DEFAULT_MALICIOUS_PUT_FREQUENCY; | 4668 | malicious_put_frequency = DEFAULT_MALICIOUS_PUT_FREQUENCY; |
4669 | } | 4669 | } |
4670 | 4670 | ||
4671 | dht_republish_frequency = DEFAULT_DHT_REPUBLISH_FREQUENCY; | 4671 | dht_republish_frequency = GNUNET_DHT_DEFAULT_REPUBLISH_FREQUENCY; |
4672 | if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number(cfg, "DHT", "REPLICATION_FREQUENCY", &temp_config_num)) | 4672 | if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number(cfg, "DHT", "REPLICATION_FREQUENCY", &temp_config_num)) |
4673 | { | 4673 | { |
4674 | dht_republish_frequency = GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MINUTES, temp_config_num); | 4674 | dht_republish_frequency = GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MINUTES, temp_config_num); |
diff --git a/src/fs/fs_test_lib_data.conf b/src/fs/fs_test_lib_data.conf index 7c8b867ae..c9dafa748 100644 --- a/src/fs/fs_test_lib_data.conf +++ b/src/fs/fs_test_lib_data.conf | |||
@@ -58,3 +58,9 @@ ACTIVEMIGRATION = NO | |||
58 | 58 | ||
59 | [testing] | 59 | [testing] |
60 | WEAKRANDOM = YES | 60 | WEAKRANDOM = YES |
61 | |||
62 | |||
63 | [dhtcache] | ||
64 | QUOTA=65536 | ||
65 | |||
66 | DATABASE=sqlite | ||
diff --git a/src/fs/gnunet-service-fs.c b/src/fs/gnunet-service-fs.c index 610c6e6dd..2e1c574c8 100644 --- a/src/fs/gnunet-service-fs.c +++ b/src/fs/gnunet-service-fs.c | |||
@@ -65,6 +65,11 @@ | |||
65 | #define TRUST_FLUSH_FREQ GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 5) | 65 | #define TRUST_FLUSH_FREQ GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 5) |
66 | 66 | ||
67 | /** | 67 | /** |
68 | * How often do we at most PUT content into the DHT? | ||
69 | */ | ||
70 | #define MAX_DHT_PUT_FREQ GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 5) | ||
71 | |||
72 | /** | ||
68 | * Inverse of the probability that we will submit the same query | 73 | * Inverse of the probability that we will submit the same query |
69 | * to the same peer again. If the same peer already got the query | 74 | * to the same peer again. If the same peer already got the query |
70 | * repeatedly recently, the probability is multiplied by the inverse | 75 | * repeatedly recently, the probability is multiplied by the inverse |
@@ -718,6 +723,16 @@ static struct MigrationReadyBlock *mig_tail; | |||
718 | static struct GNUNET_DATASTORE_QueueEntry *mig_qe; | 723 | static struct GNUNET_DATASTORE_QueueEntry *mig_qe; |
719 | 724 | ||
720 | /** | 725 | /** |
726 | * Request to datastore for DHT PUTs (or NULL). | ||
727 | */ | ||
728 | static struct GNUNET_DATASTORE_QueueEntry *dht_qe; | ||
729 | |||
730 | /** | ||
731 | * Type we will request for the next DHT PUT round from the datastore. | ||
732 | */ | ||
733 | static enum GNUNET_BLOCK_Type dht_put_type = GNUNET_BLOCK_TYPE_FS_KBLOCK; | ||
734 | |||
735 | /** | ||
721 | * Where do we store trust information? | 736 | * Where do we store trust information? |
722 | */ | 737 | */ |
723 | static char *trustDirectory; | 738 | static char *trustDirectory; |
@@ -728,6 +743,11 @@ static char *trustDirectory; | |||
728 | static GNUNET_SCHEDULER_TaskIdentifier mig_task; | 743 | static GNUNET_SCHEDULER_TaskIdentifier mig_task; |
729 | 744 | ||
730 | /** | 745 | /** |
746 | * ID of task that collects blocks for DHT PUTs. | ||
747 | */ | ||
748 | static GNUNET_SCHEDULER_TaskIdentifier dht_task; | ||
749 | |||
750 | /** | ||
731 | * What is the maximum frequency at which we are allowed to | 751 | * What is the maximum frequency at which we are allowed to |
732 | * poll the datastore for migration content? | 752 | * poll the datastore for migration content? |
733 | */ | 753 | */ |
@@ -749,6 +769,12 @@ static unsigned int mig_size; | |||
749 | static int active_migration; | 769 | static int active_migration; |
750 | 770 | ||
751 | /** | 771 | /** |
772 | * How many entires with zero anonymity do we currently estimate | ||
773 | * to have in the database? | ||
774 | */ | ||
775 | static unsigned int zero_anonymity_count_estimate; | ||
776 | |||
777 | /** | ||
752 | * Typical priorities we're seeing from other peers right now. Since | 778 | * Typical priorities we're seeing from other peers right now. Since |
753 | * most priorities will be zero, this value is the weighted average of | 779 | * most priorities will be zero, this value is the weighted average of |
754 | * non-zero priorities seen "recently". In order to ensure that new | 780 | * non-zero priorities seen "recently". In order to ensure that new |
@@ -983,6 +1009,19 @@ gather_migration_blocks (void *cls, | |||
983 | const struct GNUNET_SCHEDULER_TaskContext *tc); | 1009 | const struct GNUNET_SCHEDULER_TaskContext *tc); |
984 | 1010 | ||
985 | 1011 | ||
1012 | |||
1013 | |||
1014 | /** | ||
1015 | * Task that is run periodically to obtain blocks for DHT PUTs. | ||
1016 | * | ||
1017 | * @param cls type of blocks to gather | ||
1018 | * @param tc scheduler context (unused) | ||
1019 | */ | ||
1020 | static void | ||
1021 | gather_dht_put_blocks (void *cls, | ||
1022 | const struct GNUNET_SCHEDULER_TaskContext *tc); | ||
1023 | |||
1024 | |||
986 | /** | 1025 | /** |
987 | * If the migration task is not currently running, consider | 1026 | * If the migration task is not currently running, consider |
988 | * (re)scheduling it with the appropriate delay. | 1027 | * (re)scheduling it with the appropriate delay. |
@@ -1012,6 +1051,41 @@ consider_migration_gathering () | |||
1012 | 1051 | ||
1013 | 1052 | ||
1014 | /** | 1053 | /** |
1054 | * If the DHT PUT gathering task is not currently running, consider | ||
1055 | * (re)scheduling it with the appropriate delay. | ||
1056 | */ | ||
1057 | static void | ||
1058 | consider_dht_put_gathering (void *cls) | ||
1059 | { | ||
1060 | struct GNUNET_TIME_Relative delay; | ||
1061 | |||
1062 | if (dsh == NULL) | ||
1063 | return; | ||
1064 | if (dht_qe != NULL) | ||
1065 | return; | ||
1066 | if (dht_task != GNUNET_SCHEDULER_NO_TASK) | ||
1067 | return; | ||
1068 | if (zero_anonymity_count_estimate > 0) | ||
1069 | { | ||
1070 | delay = GNUNET_TIME_relative_divide (GNUNET_DHT_DEFAULT_REPUBLISH_FREQUENCY, | ||
1071 | zero_anonymity_count_estimate); | ||
1072 | delay = GNUNET_TIME_relative_min (delay, | ||
1073 | MAX_DHT_PUT_FREQ); | ||
1074 | } | ||
1075 | else | ||
1076 | { | ||
1077 | /* if we have NO zero-anonymity content yet, wait 5 minutes for some to | ||
1078 | (hopefully) appear */ | ||
1079 | delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 5); | ||
1080 | } | ||
1081 | dht_task = GNUNET_SCHEDULER_add_delayed (sched, | ||
1082 | delay, | ||
1083 | &gather_dht_put_blocks, | ||
1084 | cls); | ||
1085 | } | ||
1086 | |||
1087 | |||
1088 | /** | ||
1015 | * Process content offered for migration. | 1089 | * Process content offered for migration. |
1016 | * | 1090 | * |
1017 | * @param cls closure | 1091 | * @param cls closure |
@@ -1083,6 +1157,86 @@ process_migration_content (void *cls, | |||
1083 | 1157 | ||
1084 | 1158 | ||
1085 | /** | 1159 | /** |
1160 | * Function called upon completion of the DHT PUT operation. | ||
1161 | */ | ||
1162 | static void | ||
1163 | dht_put_continuation (void *cls, | ||
1164 | const struct GNUNET_SCHEDULER_TaskContext *tc) | ||
1165 | { | ||
1166 | GNUNET_DATASTORE_get_next (dsh, GNUNET_YES); | ||
1167 | } | ||
1168 | |||
1169 | |||
1170 | /** | ||
1171 | * Store content in DHT. | ||
1172 | * | ||
1173 | * @param cls closure | ||
1174 | * @param key key for the content | ||
1175 | * @param size number of bytes in data | ||
1176 | * @param data content stored | ||
1177 | * @param type type of the content | ||
1178 | * @param priority priority of the content | ||
1179 | * @param anonymity anonymity-level for the content | ||
1180 | * @param expiration expiration time for the content | ||
1181 | * @param uid unique identifier for the datum; | ||
1182 | * maybe 0 if no unique identifier is available | ||
1183 | */ | ||
1184 | static void | ||
1185 | process_dht_put_content (void *cls, | ||
1186 | const GNUNET_HashCode * key, | ||
1187 | size_t size, | ||
1188 | const void *data, | ||
1189 | enum GNUNET_BLOCK_Type type, | ||
1190 | uint32_t priority, | ||
1191 | uint32_t anonymity, | ||
1192 | struct GNUNET_TIME_Absolute | ||
1193 | expiration, uint64_t uid) | ||
1194 | { | ||
1195 | static unsigned int counter; | ||
1196 | static GNUNET_HashCode last_vhash; | ||
1197 | static GNUNET_HashCode vhash; | ||
1198 | |||
1199 | if (key == NULL) | ||
1200 | { | ||
1201 | dht_qe = NULL; | ||
1202 | consider_dht_put_gathering (cls); | ||
1203 | return; | ||
1204 | } | ||
1205 | /* slightly funky code to estimate the total number of values with zero | ||
1206 | anonymity from the maximum observed length of a monotonically increasing | ||
1207 | sequence of hashes over the contents */ | ||
1208 | GNUNET_CRYPTO_hash (data, size, &vhash); | ||
1209 | if (GNUNET_CRYPTO_hash_cmp (&vhash, &last_vhash) <= 0) | ||
1210 | { | ||
1211 | if (zero_anonymity_count_estimate > 0) | ||
1212 | zero_anonymity_count_estimate /= 2; | ||
1213 | counter = 0; | ||
1214 | } | ||
1215 | last_vhash = vhash; | ||
1216 | if (counter < 31) | ||
1217 | counter++; | ||
1218 | if (zero_anonymity_count_estimate < (1 << counter)) | ||
1219 | zero_anonymity_count_estimate = (1 << counter); | ||
1220 | #if DEBUG_FS | ||
1221 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | ||
1222 | "Retrieved block `%s' of type %u for DHT PUT\n", | ||
1223 | GNUNET_h2s (key), | ||
1224 | type); | ||
1225 | #endif | ||
1226 | GNUNET_DHT_put (dht_handle, | ||
1227 | key, | ||
1228 | GNUNET_DHT_RO_NONE, | ||
1229 | type, | ||
1230 | size, | ||
1231 | data, | ||
1232 | expiration, | ||
1233 | GNUNET_TIME_UNIT_FOREVER_REL, | ||
1234 | &dht_put_continuation, | ||
1235 | cls); | ||
1236 | } | ||
1237 | |||
1238 | |||
1239 | /** | ||
1086 | * Task that is run periodically to obtain blocks for content | 1240 | * Task that is run periodically to obtain blocks for content |
1087 | * migration | 1241 | * migration |
1088 | * | 1242 | * |
@@ -1096,7 +1250,7 @@ gather_migration_blocks (void *cls, | |||
1096 | mig_task = GNUNET_SCHEDULER_NO_TASK; | 1250 | mig_task = GNUNET_SCHEDULER_NO_TASK; |
1097 | if (dsh != NULL) | 1251 | if (dsh != NULL) |
1098 | { | 1252 | { |
1099 | mig_qe = GNUNET_DATASTORE_get_random (dsh, 0, -1, | 1253 | mig_qe = GNUNET_DATASTORE_get_random (dsh, 0, UINT_MAX, |
1100 | GNUNET_TIME_UNIT_FOREVER_REL, | 1254 | GNUNET_TIME_UNIT_FOREVER_REL, |
1101 | &process_migration_content, NULL); | 1255 | &process_migration_content, NULL); |
1102 | GNUNET_assert (mig_qe != NULL); | 1256 | GNUNET_assert (mig_qe != NULL); |
@@ -1105,6 +1259,30 @@ gather_migration_blocks (void *cls, | |||
1105 | 1259 | ||
1106 | 1260 | ||
1107 | /** | 1261 | /** |
1262 | * Task that is run periodically to obtain blocks for DHT PUTs. | ||
1263 | * | ||
1264 | * @param cls type of blocks to gather | ||
1265 | * @param tc scheduler context (unused) | ||
1266 | */ | ||
1267 | static void | ||
1268 | gather_dht_put_blocks (void *cls, | ||
1269 | const struct GNUNET_SCHEDULER_TaskContext *tc) | ||
1270 | { | ||
1271 | dht_task = GNUNET_SCHEDULER_NO_TASK; | ||
1272 | if (dsh != NULL) | ||
1273 | { | ||
1274 | if (dht_put_type == GNUNET_BLOCK_TYPE_FS_ONDEMAND) | ||
1275 | dht_put_type = GNUNET_BLOCK_TYPE_FS_KBLOCK; | ||
1276 | dht_qe = GNUNET_DATASTORE_get_zero_anonymity (dsh, 0, UINT_MAX, | ||
1277 | GNUNET_TIME_UNIT_FOREVER_REL, | ||
1278 | dht_put_type++, | ||
1279 | &process_dht_put_content, NULL); | ||
1280 | GNUNET_assert (dht_qe != NULL); | ||
1281 | } | ||
1282 | } | ||
1283 | |||
1284 | |||
1285 | /** | ||
1108 | * We're done with a particular message list entry. | 1286 | * We're done with a particular message list entry. |
1109 | * Free all associated resources. | 1287 | * Free all associated resources. |
1110 | * | 1288 | * |
@@ -1621,11 +1799,21 @@ shutdown_task (void *cls, | |||
1621 | GNUNET_DATASTORE_cancel (mig_qe); | 1799 | GNUNET_DATASTORE_cancel (mig_qe); |
1622 | mig_qe = NULL; | 1800 | mig_qe = NULL; |
1623 | } | 1801 | } |
1802 | if (dht_qe != NULL) | ||
1803 | { | ||
1804 | GNUNET_DATASTORE_cancel (dht_qe); | ||
1805 | dht_qe = NULL; | ||
1806 | } | ||
1624 | if (GNUNET_SCHEDULER_NO_TASK != mig_task) | 1807 | if (GNUNET_SCHEDULER_NO_TASK != mig_task) |
1625 | { | 1808 | { |
1626 | GNUNET_SCHEDULER_cancel (sched, mig_task); | 1809 | GNUNET_SCHEDULER_cancel (sched, mig_task); |
1627 | mig_task = GNUNET_SCHEDULER_NO_TASK; | 1810 | mig_task = GNUNET_SCHEDULER_NO_TASK; |
1628 | } | 1811 | } |
1812 | if (GNUNET_SCHEDULER_NO_TASK != dht_task) | ||
1813 | { | ||
1814 | GNUNET_SCHEDULER_cancel (sched, dht_task); | ||
1815 | dht_task = GNUNET_SCHEDULER_NO_TASK; | ||
1816 | } | ||
1629 | while (client_list != NULL) | 1817 | while (client_list != NULL) |
1630 | handle_client_disconnect (NULL, | 1818 | handle_client_disconnect (NULL, |
1631 | client_list->client); | 1819 | client_list->client); |
@@ -3917,6 +4105,7 @@ main_init (struct GNUNET_SCHEDULER_Handle *s, | |||
3917 | _("Content migration is enabled, will start to gather data\n")); | 4105 | _("Content migration is enabled, will start to gather data\n")); |
3918 | consider_migration_gathering (); | 4106 | consider_migration_gathering (); |
3919 | } | 4107 | } |
4108 | consider_dht_put_gathering (NULL); | ||
3920 | GNUNET_SERVER_disconnect_notify (server, | 4109 | GNUNET_SERVER_disconnect_notify (server, |
3921 | &handle_client_disconnect, | 4110 | &handle_client_disconnect, |
3922 | NULL); | 4111 | NULL); |
diff --git a/src/fs/test_fs_data.conf b/src/fs/test_fs_data.conf index 53fc6b500..0796be2df 100644 --- a/src/fs/test_fs_data.conf +++ b/src/fs/test_fs_data.conf | |||
@@ -42,3 +42,9 @@ ACTIVEMIGRATION = NO | |||
42 | 42 | ||
43 | [testing] | 43 | [testing] |
44 | WEAKRANDOM = YES | 44 | WEAKRANDOM = YES |
45 | |||
46 | |||
47 | [dhtcache] | ||
48 | QUOTA=65536 | ||
49 | |||
50 | DATABASE=sqlite | ||
diff --git a/src/fs/test_fs_download_data.conf b/src/fs/test_fs_download_data.conf index d647d448d..0bc6cc2b3 100644 --- a/src/fs/test_fs_download_data.conf +++ b/src/fs/test_fs_download_data.conf | |||
@@ -43,3 +43,9 @@ ACTIVEMIGRATION = NO | |||
43 | 43 | ||
44 | [testing] | 44 | [testing] |
45 | WEAKRANDOM = YES | 45 | WEAKRANDOM = YES |
46 | |||
47 | |||
48 | [dhtcache] | ||
49 | QUOTA=65536 | ||
50 | |||
51 | DATABASE=sqlite | ||
diff --git a/src/fs/test_fs_file_information_data.conf b/src/fs/test_fs_file_information_data.conf index 55cfddff4..e7ccf7b4d 100644 --- a/src/fs/test_fs_file_information_data.conf +++ b/src/fs/test_fs_file_information_data.conf | |||
@@ -40,3 +40,9 @@ HOSTNAME = localhost | |||
40 | 40 | ||
41 | [testing] | 41 | [testing] |
42 | WEAKRANDOM = YES | 42 | WEAKRANDOM = YES |
43 | |||
44 | |||
45 | [dhtcache] | ||
46 | QUOTA=65536 | ||
47 | |||
48 | DATABASE=sqlite | ||
diff --git a/src/fs/test_fs_list_indexed_data.conf b/src/fs/test_fs_list_indexed_data.conf index 06317822a..45013672e 100644 --- a/src/fs/test_fs_list_indexed_data.conf +++ b/src/fs/test_fs_list_indexed_data.conf | |||
@@ -41,3 +41,9 @@ ACTIVEMIGRATION = NO | |||
41 | 41 | ||
42 | [testing] | 42 | [testing] |
43 | WEAKRANDOM = YES | 43 | WEAKRANDOM = YES |
44 | |||
45 | |||
46 | [dhtcache] | ||
47 | QUOTA=65536 | ||
48 | |||
49 | DATABASE=sqlite | ||
diff --git a/src/fs/test_fs_namespace_data.conf b/src/fs/test_fs_namespace_data.conf index fd0b14261..d1ed94a8b 100644 --- a/src/fs/test_fs_namespace_data.conf +++ b/src/fs/test_fs_namespace_data.conf | |||
@@ -41,3 +41,9 @@ IDENTITY_DIR = $SERVICEHOME/pseudos/ | |||
41 | 41 | ||
42 | [testing] | 42 | [testing] |
43 | WEAKRANDOM = YES | 43 | WEAKRANDOM = YES |
44 | |||
45 | |||
46 | [dhtcache] | ||
47 | QUOTA=65536 | ||
48 | |||
49 | DATABASE=sqlite | ||
diff --git a/src/fs/test_fs_publish_data.conf b/src/fs/test_fs_publish_data.conf index 124f925b1..9ac58150f 100644 --- a/src/fs/test_fs_publish_data.conf +++ b/src/fs/test_fs_publish_data.conf | |||
@@ -41,3 +41,9 @@ ACTIVEMIGRATION = NO | |||
41 | 41 | ||
42 | [testing] | 42 | [testing] |
43 | WEAKRANDOM = YES | 43 | WEAKRANDOM = YES |
44 | |||
45 | |||
46 | [dhtcache] | ||
47 | QUOTA=65536 | ||
48 | |||
49 | DATABASE=sqlite | ||
diff --git a/src/fs/test_fs_search_data.conf b/src/fs/test_fs_search_data.conf index d8b34d91d..33c649dac 100644 --- a/src/fs/test_fs_search_data.conf +++ b/src/fs/test_fs_search_data.conf | |||
@@ -40,3 +40,9 @@ HOSTNAME = localhost | |||
40 | 40 | ||
41 | [testing] | 41 | [testing] |
42 | WEAKRANDOM = YES | 42 | WEAKRANDOM = YES |
43 | |||
44 | |||
45 | [dhtcache] | ||
46 | QUOTA=65536 | ||
47 | |||
48 | DATABASE=sqlite | ||
diff --git a/src/fs/test_fs_unindex_data.conf b/src/fs/test_fs_unindex_data.conf index b54d16f85..e25a3407e 100644 --- a/src/fs/test_fs_unindex_data.conf +++ b/src/fs/test_fs_unindex_data.conf | |||
@@ -40,3 +40,9 @@ HOSTNAME = localhost | |||
40 | 40 | ||
41 | [testing] | 41 | [testing] |
42 | WEAKRANDOM = YES | 42 | WEAKRANDOM = YES |
43 | |||
44 | |||
45 | [dhtcache] | ||
46 | QUOTA=65536 | ||
47 | |||
48 | DATABASE=sqlite | ||
diff --git a/src/fs/test_fs_uri_data.conf b/src/fs/test_fs_uri_data.conf index 12cf53b93..29c5cb202 100644 --- a/src/fs/test_fs_uri_data.conf +++ b/src/fs/test_fs_uri_data.conf | |||
@@ -7,3 +7,9 @@ HOSTKEY = $SERVICEHOME/.hostkey | |||
7 | 7 | ||
8 | [TESTING] | 8 | [TESTING] |
9 | WEAKRANDOM = YES | 9 | WEAKRANDOM = YES |
10 | |||
11 | |||
12 | [dhtcache] | ||
13 | QUOTA=65536 | ||
14 | |||
15 | DATABASE=sqlite | ||
diff --git a/src/fs/test_gnunet_fs_idx_data.conf b/src/fs/test_gnunet_fs_idx_data.conf index 43fcfc963..b3d9bc250 100644 --- a/src/fs/test_gnunet_fs_idx_data.conf +++ b/src/fs/test_gnunet_fs_idx_data.conf | |||
@@ -42,3 +42,9 @@ HOSTNAME = localhost | |||
42 | 42 | ||
43 | [testing] | 43 | [testing] |
44 | WEAKRANDOM = YES | 44 | WEAKRANDOM = YES |
45 | |||
46 | |||
47 | [dhtcache] | ||
48 | QUOTA=65536 | ||
49 | |||
50 | DATABASE=sqlite | ||
diff --git a/src/fs/test_gnunet_fs_ns_data.conf b/src/fs/test_gnunet_fs_ns_data.conf index e45cea44a..65bac0a15 100644 --- a/src/fs/test_gnunet_fs_ns_data.conf +++ b/src/fs/test_gnunet_fs_ns_data.conf | |||
@@ -42,3 +42,9 @@ HOSTNAME = localhost | |||
42 | 42 | ||
43 | [testing] | 43 | [testing] |
44 | WEAKRANDOM = YES | 44 | WEAKRANDOM = YES |
45 | |||
46 | |||
47 | [dhtcache] | ||
48 | QUOTA=65536 | ||
49 | |||
50 | DATABASE=sqlite | ||
diff --git a/src/fs/test_gnunet_fs_psd_data.conf b/src/fs/test_gnunet_fs_psd_data.conf index 75a96a1ae..1620c910a 100644 --- a/src/fs/test_gnunet_fs_psd_data.conf +++ b/src/fs/test_gnunet_fs_psd_data.conf | |||
@@ -42,3 +42,9 @@ HOSTNAME = localhost | |||
42 | 42 | ||
43 | [testing] | 43 | [testing] |
44 | WEAKRANDOM = YES | 44 | WEAKRANDOM = YES |
45 | |||
46 | |||
47 | [dhtcache] | ||
48 | QUOTA=65536 | ||
49 | |||
50 | DATABASE=sqlite | ||
diff --git a/src/fs/test_gnunet_fs_rec_data.conf b/src/fs/test_gnunet_fs_rec_data.conf index ad4b8cbcb..1679df036 100644 --- a/src/fs/test_gnunet_fs_rec_data.conf +++ b/src/fs/test_gnunet_fs_rec_data.conf | |||
@@ -43,3 +43,9 @@ HOSTNAME = localhost | |||
43 | 43 | ||
44 | [testing] | 44 | [testing] |
45 | WEAKRANDOM = YES | 45 | WEAKRANDOM = YES |
46 | |||
47 | |||
48 | [dhtcache] | ||
49 | QUOTA=65536 | ||
50 | |||
51 | DATABASE=sqlite | ||
diff --git a/src/fs/test_gnunet_service_fs_migration_data.conf b/src/fs/test_gnunet_service_fs_migration_data.conf index 4c7f95ee9..db1f5e87c 100644 --- a/src/fs/test_gnunet_service_fs_migration_data.conf +++ b/src/fs/test_gnunet_service_fs_migration_data.conf | |||
@@ -57,3 +57,9 @@ ACTIVEMIGRATION = YES | |||
57 | 57 | ||
58 | [testing] | 58 | [testing] |
59 | WEAKRANDOM = YES | 59 | WEAKRANDOM = YES |
60 | |||
61 | |||
62 | [dhtcache] | ||
63 | QUOTA=65536 | ||
64 | |||
65 | DATABASE=sqlite | ||
diff --git a/src/include/gnunet_block_lib.h b/src/include/gnunet_block_lib.h index 15119f445..d05cab2b6 100644 --- a/src/include/gnunet_block_lib.h +++ b/src/include/gnunet_block_lib.h | |||
@@ -58,7 +58,8 @@ enum GNUNET_BLOCK_Type | |||
58 | GNUNET_BLOCK_TYPE_FS_IBLOCK = 2, | 58 | GNUNET_BLOCK_TYPE_FS_IBLOCK = 2, |
59 | 59 | ||
60 | /** | 60 | /** |
61 | * Type of a block representing a keyword search result. | 61 | * Type of a block representing a keyword search result. Note that |
62 | * the values for KBLOCK, SBLOCK and NBLOCK must be consecutive. | ||
62 | */ | 63 | */ |
63 | GNUNET_BLOCK_TYPE_FS_KBLOCK = 3, | 64 | GNUNET_BLOCK_TYPE_FS_KBLOCK = 3, |
64 | 65 | ||
@@ -68,15 +69,15 @@ enum GNUNET_BLOCK_Type | |||
68 | GNUNET_BLOCK_TYPE_FS_SBLOCK = 4, | 69 | GNUNET_BLOCK_TYPE_FS_SBLOCK = 4, |
69 | 70 | ||
70 | /** | 71 | /** |
71 | * Type of a block representing a block to be encoded on demand from disk. | 72 | * Type of a block that is used to advertise a namespace. |
72 | * Should never appear on the network directly. | ||
73 | */ | 73 | */ |
74 | GNUNET_BLOCK_TYPE_FS_ONDEMAND = 5, | 74 | GNUNET_BLOCK_TYPE_FS_NBLOCK = 5, |
75 | 75 | ||
76 | /** | 76 | /** |
77 | * Type of a block that is used to advertise a namespace. | 77 | * Type of a block representing a block to be encoded on demand from disk. |
78 | * Should never appear on the network directly. | ||
78 | */ | 79 | */ |
79 | GNUNET_BLOCK_TYPE_FS_NBLOCK = 6, | 80 | GNUNET_BLOCK_TYPE_FS_ONDEMAND = 6, |
80 | 81 | ||
81 | /** | 82 | /** |
82 | * Type of a block that contains a HELLO for a peer (for | 83 | * Type of a block that contains a HELLO for a peer (for |
diff --git a/src/include/gnunet_datastore_service.h b/src/include/gnunet_datastore_service.h index c41d21058..8d35f7900 100644 --- a/src/include/gnunet_datastore_service.h +++ b/src/include/gnunet_datastore_service.h | |||
@@ -355,6 +355,33 @@ GNUNET_DATASTORE_get_random (struct GNUNET_DATASTORE_Handle *h, | |||
355 | 355 | ||
356 | 356 | ||
357 | /** | 357 | /** |
358 | * Get a zero-anonymity value from the datastore. | ||
359 | * | ||
360 | * @param h handle to the datastore | ||
361 | * @param queue_priority ranking of this request in the priority queue | ||
362 | * @param max_queue_size at what queue size should this request be dropped | ||
363 | * (if other requests of higher priority are in the queue) | ||
364 | * @param timeout how long to wait at most for a response | ||
365 | * @param type allowed type for the operation | ||
366 | * @param iter function to call on a random value; it | ||
367 | * will be called once with a value (if available) | ||
368 | * and always once with a value of NULL. | ||
369 | * @param iter_cls closure for iter | ||
370 | * @return NULL if the entry was not queued, otherwise a handle that can be used to | ||
371 | * cancel; note that even if NULL is returned, the callback will be invoked | ||
372 | * (or rather, will already have been invoked) | ||
373 | */ | ||
374 | struct GNUNET_DATASTORE_QueueEntry * | ||
375 | GNUNET_DATASTORE_get_zero_anonymity (struct GNUNET_DATASTORE_Handle *h, | ||
376 | unsigned int queue_priority, | ||
377 | unsigned int max_queue_size, | ||
378 | struct GNUNET_TIME_Relative timeout, | ||
379 | enum GNUNET_BLOCK_Type type, | ||
380 | GNUNET_DATASTORE_Iterator iter, | ||
381 | void *iter_cls); | ||
382 | |||
383 | |||
384 | /** | ||
358 | * Cancel a datastore operation. The final callback from the | 385 | * Cancel a datastore operation. The final callback from the |
359 | * operation must not have been done yet. | 386 | * operation must not have been done yet. |
360 | * | 387 | * |
diff --git a/src/include/gnunet_dht_service.h b/src/include/gnunet_dht_service.h index 9ad4cc809..b3bf2b84b 100644 --- a/src/include/gnunet_dht_service.h +++ b/src/include/gnunet_dht_service.h | |||
@@ -41,6 +41,11 @@ extern "C" | |||
41 | 41 | ||
42 | 42 | ||
43 | /** | 43 | /** |
44 | * FIXME: document. | ||
45 | */ | ||
46 | #define GNUNET_DHT_DEFAULT_REPUBLISH_FREQUENCY GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MINUTES, 60) | ||
47 | |||
48 | /** | ||
44 | * K-value that must be used for the bloom filter 'GET' | 49 | * K-value that must be used for the bloom filter 'GET' |
45 | * queries. | 50 | * queries. |
46 | */ | 51 | */ |
diff --git a/src/include/gnunet_protocols.h b/src/include/gnunet_protocols.h index 3de18f8d1..bf696c743 100644 --- a/src/include/gnunet_protocols.h +++ b/src/include/gnunet_protocols.h | |||
@@ -434,27 +434,32 @@ extern "C" | |||
434 | #define GNUNET_MESSAGE_TYPE_DATASTORE_GET_RANDOM 98 | 434 | #define GNUNET_MESSAGE_TYPE_DATASTORE_GET_RANDOM 98 |
435 | 435 | ||
436 | /** | 436 | /** |
437 | * Message sent by datastore client to get random data. | ||
438 | */ | ||
439 | #define GNUNET_MESSAGE_TYPE_DATASTORE_GET_ZERO_ANONYMITY 99 | ||
440 | |||
441 | /** | ||
437 | * Message sent by datastore to client providing requested data | 442 | * Message sent by datastore to client providing requested data |
438 | * (in response to GET or GET_RANDOM request). | 443 | * (in response to GET or GET_RANDOM request). |
439 | */ | 444 | */ |
440 | #define GNUNET_MESSAGE_TYPE_DATASTORE_DATA 99 | 445 | #define GNUNET_MESSAGE_TYPE_DATASTORE_DATA 100 |
441 | 446 | ||
442 | /** | 447 | /** |
443 | * Message sent by datastore to client signaling end of matching data. | 448 | * Message sent by datastore to client signaling end of matching data. |
444 | * This message will also be sent for "GET_RANDOM", even though | 449 | * This message will also be sent for "GET_RANDOM", even though |
445 | * "GET_RANDOM" returns at most one data item. | 450 | * "GET_RANDOM" returns at most one data item. |
446 | */ | 451 | */ |
447 | #define GNUNET_MESSAGE_TYPE_DATASTORE_DATA_END 100 | 452 | #define GNUNET_MESSAGE_TYPE_DATASTORE_DATA_END 101 |
448 | 453 | ||
449 | /** | 454 | /** |
450 | * Message sent by datastore client to remove data. | 455 | * Message sent by datastore client to remove data. |
451 | */ | 456 | */ |
452 | #define GNUNET_MESSAGE_TYPE_DATASTORE_REMOVE 101 | 457 | #define GNUNET_MESSAGE_TYPE_DATASTORE_REMOVE 102 |
453 | 458 | ||
454 | /** | 459 | /** |
455 | * Message sent by datastore client to drop the database. | 460 | * Message sent by datastore client to drop the database. |
456 | */ | 461 | */ |
457 | #define GNUNET_MESSAGE_TYPE_DATASTORE_DROP 102 | 462 | #define GNUNET_MESSAGE_TYPE_DATASTORE_DROP 103 |
458 | 463 | ||
459 | 464 | ||
460 | /** | 465 | /** |