diff options
author | Christian Grothoff <christian@grothoff.org> | 2013-06-27 13:37:33 +0000 |
---|---|---|
committer | Christian Grothoff <christian@grothoff.org> | 2013-06-27 13:37:33 +0000 |
commit | 64cd7cbca2bb45fbb31b19b587c7ae4118855a65 (patch) | |
tree | 124db1535340c0d91ef2e6d2adef1dd2ef006523 /src/regex | |
parent | d1b1c834fbb65d70fca837e1ab742e71e16adf50 (diff) | |
download | gnunet-64cd7cbca2bb45fbb31b19b587c7ae4118855a65.tar.gz gnunet-64cd7cbca2bb45fbb31b19b587c7ae4118855a65.zip |
-fixing #2585: optimized layout for regex blocks
Diffstat (limited to 'src/regex')
-rw-r--r-- | src/regex/regex_block_lib.c | 263 | ||||
-rw-r--r-- | src/regex/test_regex_iterate_api.c | 2 |
2 files changed, 155 insertions, 110 deletions
diff --git a/src/regex/regex_block_lib.c b/src/regex/regex_block_lib.c index 842c9f366..7fcb1a7f1 100644 --- a/src/regex/regex_block_lib.c +++ b/src/regex/regex_block_lib.c | |||
@@ -25,12 +25,32 @@ | |||
25 | */ | 25 | */ |
26 | #include "platform.h" | 26 | #include "platform.h" |
27 | #include "regex_block_lib.h" | 27 | #include "regex_block_lib.h" |
28 | #include "gnunet_constants.h" | ||
28 | 29 | ||
29 | #define LOG(kind,...) GNUNET_log_from (kind,"regex-bck",__VA_ARGS__) | 30 | #define LOG(kind,...) GNUNET_log_from (kind,"regex-bck",__VA_ARGS__) |
30 | 31 | ||
31 | GNUNET_NETWORK_STRUCT_BEGIN | 32 | GNUNET_NETWORK_STRUCT_BEGIN |
32 | 33 | ||
33 | /** | 34 | /** |
35 | * Information for each edge. | ||
36 | */ | ||
37 | struct EdgeInfo | ||
38 | { | ||
39 | /** | ||
40 | * Index of the destination of this edge in the | ||
41 | * unique destinations array. | ||
42 | */ | ||
43 | uint16_t destination_index GNUNET_PACKED; | ||
44 | |||
45 | /** | ||
46 | * Number of bytes the token for this edge takes in the | ||
47 | * token area. | ||
48 | */ | ||
49 | uint16_t token_length GNUNET_PACKED; | ||
50 | }; | ||
51 | |||
52 | |||
53 | /** | ||
34 | * @brief Block to announce a regex state. | 54 | * @brief Block to announce a regex state. |
35 | */ | 55 | */ |
36 | struct RegexBlock | 56 | struct RegexBlock |
@@ -47,31 +67,25 @@ struct RegexBlock | |||
47 | int16_t is_accepting GNUNET_PACKED; | 67 | int16_t is_accepting GNUNET_PACKED; |
48 | 68 | ||
49 | /** | 69 | /** |
50 | * Numer of edges parting from this state. | 70 | * Number of edges parting from this state. |
51 | */ | 71 | */ |
52 | uint32_t n_edges GNUNET_PACKED; | 72 | uint16_t num_edges GNUNET_PACKED; |
53 | 73 | ||
54 | /* char proof[n_proof] */ | ||
55 | /* struct RegexEdge edges[n_edges] */ | ||
56 | }; | ||
57 | |||
58 | |||
59 | /** | ||
60 | * @brief A RegexBlock contains one or more of this struct in the payload. | ||
61 | */ | ||
62 | struct RegexEdge | ||
63 | { | ||
64 | /** | ||
65 | * Destination of this edge. | ||
66 | */ | ||
67 | struct GNUNET_HashCode key; | ||
68 | |||
69 | /** | 74 | /** |
70 | * Length of the token towards the new state. | 75 | * Nubmer of unique destinations reachable from this state. |
71 | */ | 76 | */ |
72 | uint32_t n_token GNUNET_PACKED; | 77 | uint16_t num_destinations GNUNET_PACKED; |
78 | |||
79 | /* followed by 'struct GNUNET_HashCode[num_destinations]' */ | ||
80 | |||
81 | /* followed by 'struct EdgeInfo[edge_destination_indices]' */ | ||
82 | |||
83 | /* followed by 'char proof[n_proof]', NOT 0-terminated */ | ||
84 | |||
85 | /* followed by 'char tokens[num_edges][edge_info[k].token_length]'; | ||
86 | essentially all of the tokens one after the other in the | ||
87 | order of the edges; tokens are NOT 0-terminated */ | ||
73 | 88 | ||
74 | /* char token[n_token] */ | ||
75 | }; | 89 | }; |
76 | 90 | ||
77 | 91 | ||
@@ -186,20 +200,20 @@ REGEX_BLOCK_check (const struct RegexBlock *block, | |||
186 | const struct GNUNET_HashCode *query, | 200 | const struct GNUNET_HashCode *query, |
187 | const char *xquery) | 201 | const char *xquery) |
188 | { | 202 | { |
203 | struct GNUNET_HashCode key; | ||
189 | struct CheckEdgeContext ctx; | 204 | struct CheckEdgeContext ctx; |
190 | int res; | 205 | int res; |
191 | uint16_t len; | ||
192 | 206 | ||
193 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 207 | if (GNUNET_OK != |
194 | "Checking block with xquery `%s'\n", | 208 | REGEX_BLOCK_get_key (block, size, |
195 | NULL != xquery ? xquery : "NULL"); | 209 | &key)) |
196 | len = ntohs (block->proof_len); | ||
197 | if (size < sizeof (struct RegexBlock) + len) | ||
198 | { | 210 | { |
199 | GNUNET_break_op (0); | 211 | GNUNET_break_op (0); |
200 | return GNUNET_SYSERR; | 212 | return GNUNET_SYSERR; |
201 | } | 213 | } |
202 | if (GNUNET_OK != REGEX_BLOCK_check_proof ((const char *) &block[1], len, query)) | 214 | if (0 != memcmp (&key, |
215 | query, | ||
216 | sizeof (struct GNUNET_HashCode))) | ||
203 | { | 217 | { |
204 | GNUNET_break_op (0); | 218 | GNUNET_break_op (0); |
205 | return GNUNET_SYSERR; | 219 | return GNUNET_SYSERR; |
@@ -232,14 +246,29 @@ REGEX_BLOCK_get_key (const struct RegexBlock *block, | |||
232 | struct GNUNET_HashCode *key) | 246 | struct GNUNET_HashCode *key) |
233 | { | 247 | { |
234 | uint16_t len; | 248 | uint16_t len; |
249 | const struct GNUNET_HashCode *destinations; | ||
250 | const struct EdgeInfo *edges; | ||
251 | uint16_t num_destinations; | ||
252 | uint16_t num_edges; | ||
253 | size_t total; | ||
235 | 254 | ||
255 | if (block_len < sizeof (struct RegexBlock)) | ||
256 | { | ||
257 | GNUNET_break_op (0); | ||
258 | return GNUNET_SYSERR; | ||
259 | } | ||
260 | num_destinations = ntohs (block->num_destinations); | ||
261 | num_edges = ntohs (block->num_edges); | ||
236 | len = ntohs (block->proof_len); | 262 | len = ntohs (block->proof_len); |
237 | if (block_len < sizeof (struct RegexBlock) + len) | 263 | destinations = (const struct GNUNET_HashCode *) &block[1]; |
264 | edges = (const struct EdgeInfo *) &destinations[num_destinations]; | ||
265 | total = sizeof (struct RegexBlock) + num_destinations * sizeof (struct GNUNET_HashCode) + num_edges + sizeof (struct EdgeInfo) + len; | ||
266 | if (block_len < total) | ||
238 | { | 267 | { |
239 | GNUNET_break_op (0); | 268 | GNUNET_break_op (0); |
240 | return GNUNET_SYSERR; | 269 | return GNUNET_SYSERR; |
241 | } | 270 | } |
242 | GNUNET_CRYPTO_hash (&block[1], len, key); | 271 | GNUNET_CRYPTO_hash (&edges[num_edges], len, key); |
243 | return GNUNET_OK; | 272 | return GNUNET_OK; |
244 | } | 273 | } |
245 | 274 | ||
@@ -266,69 +295,57 @@ REGEX_BLOCK_iterate (const struct RegexBlock *block, | |||
266 | REGEX_INTERNAL_EgdeIterator iterator, | 295 | REGEX_INTERNAL_EgdeIterator iterator, |
267 | void *iter_cls) | 296 | void *iter_cls) |
268 | { | 297 | { |
269 | struct RegexEdge *edge; | 298 | uint16_t len; |
299 | const struct GNUNET_HashCode *destinations; | ||
300 | const struct EdgeInfo *edges; | ||
301 | const char *aux; | ||
302 | uint16_t num_destinations; | ||
303 | uint16_t num_edges; | ||
304 | size_t total; | ||
270 | unsigned int n; | 305 | unsigned int n; |
271 | unsigned int n_token; | 306 | size_t off; |
272 | unsigned int i; | ||
273 | size_t offset; | ||
274 | char *aux; | ||
275 | 307 | ||
276 | offset = sizeof (struct RegexBlock); | 308 | if (size < sizeof (struct RegexBlock)) |
277 | if (offset >= size) /* Is it safe to access the regex block? */ | 309 | { |
310 | GNUNET_break_op (0); | ||
311 | return GNUNET_SYSERR; | ||
312 | } | ||
313 | num_destinations = ntohs (block->num_destinations); | ||
314 | num_edges = ntohs (block->num_edges); | ||
315 | len = ntohs (block->proof_len); | ||
316 | destinations = (const struct GNUNET_HashCode *) &block[1]; | ||
317 | edges = (const struct EdgeInfo *) &destinations[num_destinations]; | ||
318 | aux = (const char *) &edges[num_edges]; | ||
319 | total = sizeof (struct RegexBlock) + num_destinations * sizeof (struct GNUNET_HashCode) + num_edges + sizeof (struct EdgeInfo) + len; | ||
320 | if (size < total) | ||
278 | { | 321 | { |
279 | GNUNET_break_op (0); | 322 | GNUNET_break_op (0); |
280 | return GNUNET_SYSERR; | 323 | return GNUNET_SYSERR; |
281 | } | 324 | } |
282 | n = ntohs (block->proof_len); | 325 | for (n=0;n<num_edges;n++) |
283 | offset += n; | 326 | total += ntohs (edges[n].token_length); |
284 | if (offset >= size) /* Is it safe to access the regex proof? */ | 327 | if (size != total) |
285 | { | 328 | { |
286 | GNUNET_break_op (0); | 329 | GNUNET_break_op (0); |
287 | return GNUNET_SYSERR; | 330 | return GNUNET_SYSERR; |
288 | } | 331 | } |
289 | aux = (char *) &block[1]; /* Skip regex block */ | 332 | off = len; |
290 | aux = &aux[n]; /* Skip regex proof */ | ||
291 | n = ntohl (block->n_edges); | ||
292 | LOG (GNUNET_ERROR_TYPE_DEBUG, | 333 | LOG (GNUNET_ERROR_TYPE_DEBUG, |
293 | "Start iterating block of size %u, proof %u, off %u edges %u\n", | 334 | "Start iterating block of size %u, proof %u, off %u edges %u\n", |
294 | size, ntohs (block->proof_len), offset, n); | 335 | size, len, off, n); |
295 | /* aux always points at the end of the previous block */ | 336 | /* &aux[off] always points to our token */ |
296 | for (i = 0; i < n; i++) | 337 | for (n=0;n<num_edges;n++) |
297 | { | 338 | { |
298 | offset += sizeof (struct RegexEdge); | 339 | LOG (GNUNET_ERROR_TYPE_DEBUG, |
299 | LOG (GNUNET_ERROR_TYPE_DEBUG, "* Edge %u, off %u\n", i, offset); | 340 | "Edge %u, off %u tokenlen %u\n", n, off, |
300 | if (offset >= size) /* Is it safe to access the next edge block? */ | 341 | ntohs (edges[n].token_length)); |
301 | { | ||
302 | LOG (GNUNET_ERROR_TYPE_WARNING, | ||
303 | "* Size not enough for RegexEdge, END\n"); | ||
304 | GNUNET_break_op (0); | ||
305 | return GNUNET_SYSERR; | ||
306 | } | ||
307 | edge = (struct RegexEdge *) aux; | ||
308 | n_token = ntohl (edge->n_token); | ||
309 | offset += n_token; | ||
310 | LOG (GNUNET_ERROR_TYPE_DEBUG, | ||
311 | "* Token length %u, off %u\n", n_token, offset); | ||
312 | if (offset > size) /* Is it safe to access the edge token? */ | ||
313 | { | ||
314 | LOG (GNUNET_ERROR_TYPE_WARNING, | ||
315 | "* Size not enough for edge token, END\n"); | ||
316 | GNUNET_break_op (0); | ||
317 | return GNUNET_SYSERR; | ||
318 | } | ||
319 | aux = (char *) &edge[1]; /* Skip edge block */ | ||
320 | if (NULL != iterator) | 342 | if (NULL != iterator) |
321 | if (GNUNET_NO == iterator (iter_cls, aux, n_token, &edge->key)) | 343 | if (GNUNET_NO == iterator (iter_cls, |
322 | return GNUNET_OK; | 344 | &aux[off], |
323 | aux = &aux[n_token]; /* Skip edge token */ | 345 | ntohs (edges[n].token_length), |
324 | } | 346 | &destinations[ntohs (edges[n].destination_index)])) |
325 | /* The total size should be exactly the size of (regex + all edges) blocks | 347 | return GNUNET_OK; |
326 | * If size == -1, block is from cache and therefore previously checked and | 348 | off += ntohs (edges[n].token_length); |
327 | * assumed correct. */ | ||
328 | if ( (offset != size) && (SIZE_MAX != size) ) | ||
329 | { | ||
330 | GNUNET_break_op (0); | ||
331 | return GNUNET_SYSERR; | ||
332 | } | 349 | } |
333 | return GNUNET_OK; | 350 | return GNUNET_OK; |
334 | } | 351 | } |
@@ -352,50 +369,78 @@ REGEX_BLOCK_create (const char *proof, | |||
352 | size_t *rsize) | 369 | size_t *rsize) |
353 | { | 370 | { |
354 | struct RegexBlock *block; | 371 | struct RegexBlock *block; |
355 | struct RegexEdge *block_edge; | 372 | struct GNUNET_HashCode destinations[1024]; /* 1024 = 64k/64 bytes/key == absolute MAX */ |
356 | size_t size; | 373 | uint16_t destination_indices[num_edges]; |
374 | struct GNUNET_HashCode *dests; | ||
375 | struct EdgeInfo *edgeinfos; | ||
376 | size_t off; | ||
357 | size_t len; | 377 | size_t len; |
378 | size_t total; | ||
379 | size_t slen; | ||
380 | unsigned int unique_destinations; | ||
381 | unsigned int j; | ||
358 | unsigned int i; | 382 | unsigned int i; |
359 | unsigned int offset; | ||
360 | char *aux; | 383 | char *aux; |
361 | 384 | ||
362 | len = strlen (proof); | 385 | len = strlen (proof); |
363 | if (len > UINT16_MAX) | 386 | if (len > UINT16_MAX) |
387 | { | ||
388 | GNUNET_break (0); | ||
389 | return NULL; | ||
390 | } | ||
391 | unique_destinations = 0; | ||
392 | total = sizeof (struct RegexBlock) + len; | ||
393 | for (i=0;i<num_edges;i++) | ||
394 | { | ||
395 | slen = strlen (edges[i].label); | ||
396 | if (len > UINT16_MAX) | ||
397 | { | ||
398 | GNUNET_break (0); | ||
399 | return NULL; | ||
400 | } | ||
401 | total += slen; | ||
402 | for (j=0;j<unique_destinations;j++) | ||
403 | if (0 == memcmp (&destinations[j], | ||
404 | &edges[i].destination, | ||
405 | sizeof (struct GNUNET_HashCode))) | ||
406 | break; | ||
407 | if (j >= 1024) | ||
364 | { | 408 | { |
365 | GNUNET_break (0); | 409 | GNUNET_break (0); |
366 | return NULL; | 410 | return NULL; |
367 | } | 411 | } |
368 | size = sizeof (struct RegexBlock) + len; | 412 | destination_indices[i] = j; |
369 | block = GNUNET_malloc (size); | 413 | if (j == unique_destinations) |
414 | destinations[unique_destinations++] = edges[i].destination; | ||
415 | } | ||
416 | total += num_edges * sizeof (struct EdgeInfo) + unique_destinations * sizeof (struct GNUNET_HashCode); | ||
417 | if (total >= GNUNET_CONSTANTS_MAX_BLOCK_SIZE) | ||
418 | { | ||
419 | GNUNET_break (0); | ||
420 | return NULL; | ||
421 | } | ||
422 | block = GNUNET_malloc (total); | ||
370 | block->proof_len = htons (len); | 423 | block->proof_len = htons (len); |
371 | block->n_edges = htonl (num_edges); | ||
372 | block->is_accepting = htons (accepting); | 424 | block->is_accepting = htons (accepting); |
373 | 425 | block->num_edges = htons (num_edges); | |
374 | /* Store the proof at the end of the block. */ | 426 | block->num_destinations = htons (unique_destinations); |
375 | aux = (char *) &block[1]; | 427 | dests = (struct GNUNET_HashCode *) &block[1]; |
376 | memcpy (aux, proof, len); | 428 | memcpy (dests, destinations, sizeof (struct GNUNET_HashCode) * unique_destinations); |
377 | aux = &aux[len]; | 429 | edgeinfos = (struct EdgeInfo *) &dests[unique_destinations]; |
378 | 430 | aux = (char *) &edgeinfos[num_edges]; | |
379 | /* Store each edge in a variable length MeshEdge struct at the | 431 | off = len; |
380 | * very end of the MeshRegexBlock structure. | 432 | memcpy (aux, proof, len); |
381 | */ | 433 | for (i=0;i<num_edges;i++) |
382 | for (i = 0; i < num_edges; i++) | ||
383 | { | 434 | { |
384 | /* aux points at the end of the last block */ | 435 | slen = strlen (edges[i].label); |
385 | len = strlen (edges[i].label); | 436 | edgeinfos[i].token_length = htons ((uint16_t) slen); |
386 | size += sizeof (struct RegexEdge) + len; | 437 | edgeinfos[i].destination_index = htons (destination_indices[i]); |
387 | // Calculate offset FIXME is this ok? use size instead? | 438 | memcpy (&aux[off], |
388 | offset = aux - (char *) block; | 439 | edges[i].label, |
389 | block = GNUNET_realloc (block, size); | 440 | slen); |
390 | aux = &((char *) block)[offset]; | 441 | off += slen; |
391 | block_edge = (struct RegexEdge *) aux; | ||
392 | block_edge->key = edges[i].destination; | ||
393 | block_edge->n_token = htonl (len); | ||
394 | aux = (char *) &block_edge[1]; | ||
395 | memcpy (aux, edges[i].label, len); | ||
396 | aux = &aux[len]; | ||
397 | } | 442 | } |
398 | *rsize = size; | 443 | *rsize = total; |
399 | return block; | 444 | return block; |
400 | } | 445 | } |
401 | 446 | ||
diff --git a/src/regex/test_regex_iterate_api.c b/src/regex/test_regex_iterate_api.c index acbdf9f44..69badb5d8 100644 --- a/src/regex/test_regex_iterate_api.c +++ b/src/regex/test_regex_iterate_api.c | |||
@@ -109,10 +109,10 @@ key_iterator (void *cls, const struct GNUNET_HashCode *key, | |||
109 | GNUNET_log (GNUNET_ERROR_TYPE_ERROR, | 109 | GNUNET_log (GNUNET_ERROR_TYPE_ERROR, |
110 | "Proof check failed: proof: %s key: %s\n", proof, state_id); | 110 | "Proof check failed: proof: %s key: %s\n", proof, state_id); |
111 | } | 111 | } |
112 | |||
113 | GNUNET_free (state_id); | 112 | GNUNET_free (state_id); |
114 | } | 113 | } |
115 | 114 | ||
115 | |||
116 | int | 116 | int |
117 | main (int argc, char *argv[]) | 117 | main (int argc, char *argv[]) |
118 | { | 118 | { |