aboutsummaryrefslogtreecommitdiff
path: root/src/regex/gnunet-regex-simulation-profiler.c
diff options
context:
space:
mode:
Diffstat (limited to 'src/regex/gnunet-regex-simulation-profiler.c')
-rw-r--r--src/regex/gnunet-regex-simulation-profiler.c723
1 files changed, 0 insertions, 723 deletions
diff --git a/src/regex/gnunet-regex-simulation-profiler.c b/src/regex/gnunet-regex-simulation-profiler.c
deleted file mode 100644
index b7e256f48..000000000
--- a/src/regex/gnunet-regex-simulation-profiler.c
+++ /dev/null
@@ -1,723 +0,0 @@
1/*
2 This file is part of GNUnet.
3 Copyright (C) 2011, 2012 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17*/
18
19
20/**
21 * @file regex/gnunet-regex-simulation-profiler.c
22 * @brief Regex profiler that dumps all DFAs into a database instead of
23 * using the DHT (with cadet).
24 * @author Maximilian Szengel
25 * @author Christophe Genevey
26 *
27 */
28
29#include "platform.h"
30#include "gnunet_util_lib.h"
31#include "regex_internal_lib.h"
32#include "gnunet_mysql_lib.h"
33#include "gnunet_my_lib.h"
34#include <mysql/mysql.h>
35
36/**
37 * MySQL statement to insert an edge.
38 */
39#define INSERT_EDGE_STMT "INSERT IGNORE INTO `%s` "\
40 "(`key`, `label`, `to_key`, `accepting`) "\
41 "VALUES (?, ?, ?, ?);"
42
43/**
44 * MySQL statement to select a key count.
45 */
46#define SELECT_KEY_STMT "SELECT COUNT(*) FROM `%s` "\
47 "WHERE `key` = ? AND `label` = ?;"
48
49/**
50 * Simple struct to keep track of progress, and print a
51 * nice little percentage meter for long running tasks.
52 */
53struct ProgressMeter
54{
55 /**
56 * Total number of elements.
57 */
58 unsigned int total;
59
60 /**
61 * Intervall for printing percentage.
62 */
63 unsigned int modnum;
64
65 /**
66 * Number of dots to print.
67 */
68 unsigned int dotnum;
69
70 /**
71 * Completed number.
72 */
73 unsigned int completed;
74
75 /**
76 * Should the meter be printed?
77 */
78 int print;
79
80 /**
81 * String to print on startup.
82 */
83 char *startup_string;
84};
85
86
87/**
88 * Handle for the progress meter
89 */
90static struct ProgressMeter *meter;
91
92/**
93 * Scan task identifier;
94 */
95static struct GNUNET_SCHEDULER_Task *scan_task;
96
97/**
98 * Global testing status.
99 */
100static int result;
101
102/**
103 * MySQL context.
104 */
105static struct GNUNET_MYSQL_Context *mysql_ctx;
106
107/**
108 * MySQL prepared statement handle.
109 */
110static struct GNUNET_MYSQL_StatementHandle *stmt_handle;
111
112/**
113 * MySQL prepared statement handle for `key` select.
114 */
115static struct GNUNET_MYSQL_StatementHandle *select_stmt_handle;
116
117/**
118 * MySQL table name.
119 */
120static char *table_name;
121
122/**
123 * Policy dir containing files that contain policies.
124 */
125static char *policy_dir;
126
127/**
128 * Number of policy files.
129 */
130static unsigned int num_policy_files;
131
132/**
133 * Number of policies.
134 */
135static unsigned int num_policies;
136
137/**
138 * Maximal path compression length.
139 */
140static unsigned int max_path_compression;
141
142/**
143 * Number of merged transitions.
144 */
145static unsigned long long num_merged_transitions;
146
147/**
148 * Number of merged states from different policies.
149 */
150static unsigned long long num_merged_states;
151
152/**
153 * Prefix to add before every regex we're announcing.
154 */
155static char *regex_prefix;
156
157
158/**
159 * Create a meter to keep track of the progress of some task.
160 *
161 * @param total the total number of items to complete
162 * @param start_string a string to prefix the meter with (if printing)
163 * @param print GNUNET_YES to print the meter, GNUNET_NO to count
164 * internally only
165 *
166 * @return the progress meter
167 */
168static struct ProgressMeter *
169create_meter (unsigned int total, char *start_string, int print)
170{
171 struct ProgressMeter *ret;
172
173 ret = GNUNET_new (struct ProgressMeter);
174 ret->print = print;
175 ret->total = total;
176 ret->modnum = total / 4;
177 if (ret->modnum == 0) /* Divide by zero check */
178 ret->modnum = 1;
179 ret->dotnum = (total / 50) + 1;
180 if (start_string != NULL)
181 ret->startup_string = GNUNET_strdup (start_string);
182 else
183 ret->startup_string = GNUNET_strdup ("");
184
185 return ret;
186}
187
188
189/**
190 * Update progress meter (increment by one).
191 *
192 * @param meter the meter to update and print info for
193 *
194 * @return GNUNET_YES if called the total requested,
195 * GNUNET_NO if more items expected
196 */
197static int
198update_meter (struct ProgressMeter *meter)
199{
200 if (meter->print == GNUNET_YES)
201 {
202 if (meter->completed % meter->modnum == 0)
203 {
204 if (meter->completed == 0)
205 {
206 FPRINTF (stdout, "%sProgress: [0%%", meter->startup_string);
207 }
208 else
209 FPRINTF (stdout, "%d%%",
210 (int) (((float) meter->completed / meter->total) * 100));
211 }
212 else if (meter->completed % meter->dotnum == 0)
213 FPRINTF (stdout, "%s", ".");
214
215 if (meter->completed + 1 == meter->total)
216 FPRINTF (stdout, "%d%%]\n", 100);
217 fflush (stdout);
218 }
219 meter->completed++;
220
221 if (meter->completed == meter->total)
222 return GNUNET_YES;
223 if (meter->completed > meter->total)
224 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Progress meter overflow!!\n");
225 return GNUNET_NO;
226}
227
228
229/**
230 * Reset progress meter.
231 *
232 * @param meter the meter to reset
233 *
234 * @return #GNUNET_YES if meter reset,
235 * #GNUNET_SYSERR on error
236 */
237static int
238reset_meter (struct ProgressMeter *meter)
239{
240 if (meter == NULL)
241 return GNUNET_SYSERR;
242
243 meter->completed = 0;
244 return GNUNET_YES;
245}
246
247
248/**
249 * Release resources for meter
250 *
251 * @param meter the meter to free
252 */
253static void
254free_meter (struct ProgressMeter *meter)
255{
256 GNUNET_free_non_null (meter->startup_string);
257 GNUNET_free (meter);
258}
259
260
261/**
262 * Shutdown task.
263 *
264 * @param cls NULL
265 */
266static void
267do_shutdown (void *cls)
268{
269 if (NULL != mysql_ctx)
270 {
271 GNUNET_MYSQL_context_destroy (mysql_ctx);
272 mysql_ctx = NULL;
273 }
274 if (NULL != meter)
275 {
276 free_meter (meter);
277 meter = NULL;
278 }
279}
280
281
282/**
283 * Abort task to run on test timed out.
284 *
285 * FIXME: this doesn't actually work, it used to cancel
286 * the already running 'scan_task', but now that should
287 * always be NULL and do nothing. We instead need to set
288 * a global variable and abort scan_task internally, not
289 * via scheduler.
290 *
291 * @param cls NULL
292 */
293static void
294do_abort (void *cls)
295{
296 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Aborting\n");
297 if (NULL != scan_task)
298 {
299 GNUNET_SCHEDULER_cancel (scan_task);
300 scan_task = NULL;
301 }
302 result = GNUNET_SYSERR;
303 GNUNET_SCHEDULER_shutdown ();
304}
305
306/**
307 * Iterator over all states that inserts each state into the MySQL db.
308 *
309 * @param cls closure.
310 * @param key hash for current state.
311 * @param proof proof for current state.
312 * @param accepting #GNUNET_YES if this is an accepting state, #GNUNET_NO if not.
313 * @param num_edges number of edges leaving current state.
314 * @param edges edges leaving current state.
315 */
316static void
317regex_iterator (void *cls,
318 const struct GNUNET_HashCode *key,
319 const char *proof,
320 int accepting,
321 unsigned int num_edges,
322 const struct REGEX_BLOCK_Edge *edges)
323{
324 unsigned int i;
325 int result;
326
327 uint32_t iaccepting = (uint32_t)accepting;
328 uint64_t total;
329
330 GNUNET_assert (NULL != mysql_ctx);
331
332 for (i = 0; i < num_edges; i++)
333 {
334 struct GNUNET_MY_QueryParam params_select[] = {
335 GNUNET_MY_query_param_auto_from_type (key),
336 GNUNET_MY_query_param_string (edges[i].label),
337 GNUNET_MY_query_param_end
338 };
339
340 struct GNUNET_MY_ResultSpec results_select[] = {
341 GNUNET_MY_result_spec_uint64 (&total),
342 GNUNET_MY_result_spec_end
343 };
344
345 result =
346 GNUNET_MY_exec_prepared (mysql_ctx,
347 select_stmt_handle,
348 params_select);
349
350 if (GNUNET_SYSERR == result)
351 {
352 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
353 "Error executing prepared mysql select statement\n");
354 GNUNET_SCHEDULER_add_now (&do_abort, NULL);
355 return;
356 }
357
358 result =
359 GNUNET_MY_extract_result (select_stmt_handle,
360 results_select);
361
362 if (GNUNET_SYSERR == result)
363 {
364 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
365 "Error extracting result mysql select statement\n");
366 GNUNET_SCHEDULER_add_now (&do_abort, NULL);
367 return;
368 }
369
370 if (-1 != total && total > 0)
371 {
372 GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Total: %llu (%s, %s)\n",
373 (unsigned long long)total,
374 GNUNET_h2s (key), edges[i].label);
375 }
376
377 struct GNUNET_MY_QueryParam params_stmt[] = {
378 GNUNET_MY_query_param_auto_from_type (&key),
379 GNUNET_MY_query_param_string (edges[i].label),
380 GNUNET_MY_query_param_auto_from_type (&edges[i].destination),
381 GNUNET_MY_query_param_uint32 (&iaccepting),
382 GNUNET_MY_query_param_end
383 };
384
385 result =
386 GNUNET_MY_exec_prepared (mysql_ctx,
387 stmt_handle,
388 params_stmt);
389
390 if (0 == result)
391 {
392 char *key_str = GNUNET_strdup (GNUNET_h2s (key));
393 char *to_key_str = GNUNET_strdup (GNUNET_h2s (&edges[i].destination));
394
395 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Merged (%s, %s, %s, %i)\n",
396 key_str,
397 edges[i].label,
398 to_key_str,
399 accepting);
400
401 GNUNET_free (key_str);
402 GNUNET_free (to_key_str);
403 num_merged_transitions++;
404 }
405 else if (-1 != total)
406 {
407 num_merged_states++;
408 }
409
410 if (GNUNET_SYSERR == result || (1 != result && 0 != result))
411 {
412 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
413 "Error executing prepared mysql statement for edge: Affected rows: %i, expected 0 or 1!\n",
414 result);
415 GNUNET_SCHEDULER_add_now (&do_abort, NULL);
416 }
417 }
418
419 if (0 == num_edges)
420 {
421 struct GNUNET_MY_QueryParam params_stmt[] = {
422 GNUNET_MY_query_param_auto_from_type (key),
423 GNUNET_MY_query_param_string (""),
424 GNUNET_MY_query_param_fixed_size (NULL, 0),
425 GNUNET_MY_query_param_uint32 (&iaccepting),
426 GNUNET_MY_query_param_end
427 };
428
429 result =
430 GNUNET_MY_exec_prepared (mysql_ctx,
431 stmt_handle,
432 params_stmt);
433
434 if (1 != result && 0 != result)
435 {
436 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
437 "Error executing prepared mysql statement for edge: Affected rows: %i, expected 0 or 1!\n",
438 result);
439 GNUNET_SCHEDULER_add_now (&do_abort, NULL);
440 }
441 }
442}
443
444
445/**
446 * Announce a regex by creating the DFA and iterating over each state, inserting
447 * each state into a MySQL database.
448 *
449 * @param regex regular expression.
450 * @return #GNUNET_OK on success, #GNUNET_SYSERR on failure.
451 */
452static int
453announce_regex (const char *regex)
454{
455 struct REGEX_INTERNAL_Automaton *dfa;
456
457 dfa =
458 REGEX_INTERNAL_construct_dfa (regex,
459 strlen (regex),
460 max_path_compression);
461
462 if (NULL == dfa)
463 {
464 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
465 "Failed to create DFA for regex %s\n",
466 regex);
467 GNUNET_SCHEDULER_add_now (&do_abort, NULL);
468 return GNUNET_SYSERR;
469 }
470 REGEX_INTERNAL_iterate_all_edges (dfa,
471 &regex_iterator, NULL);
472 REGEX_INTERNAL_automaton_destroy (dfa);
473
474 return GNUNET_OK;
475}
476
477
478/**
479 * Function called with a filename.
480 *
481 * @param cls closure
482 * @param filename complete filename (absolute path)
483 * @return #GNUNET_OK to continue to iterate,
484 * #GNUNET_SYSERR to abort iteration with error!
485 */
486static int
487policy_filename_cb (void *cls, const char *filename)
488{
489 char *regex;
490 char *data;
491 char *buf;
492 uint64_t filesize;
493 unsigned int offset;
494
495 GNUNET_assert (NULL != filename);
496
497 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
498 "Announcing regexes from file %s\n",
499 filename);
500
501 if (GNUNET_YES != GNUNET_DISK_file_test (filename))
502 {
503 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
504 "Could not find policy file %s\n",
505 filename);
506 return GNUNET_OK;
507 }
508 if (GNUNET_OK !=
509 GNUNET_DISK_file_size (filename, &filesize,
510 GNUNET_YES, GNUNET_YES))
511 filesize = 0;
512 if (0 == filesize)
513 {
514 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Policy file %s is empty.\n",
515 filename);
516 return GNUNET_OK;
517 }
518 data = GNUNET_malloc (filesize);
519 if (filesize != GNUNET_DISK_fn_read (filename, data, filesize))
520 {
521 GNUNET_free (data);
522 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
523 "Could not read policy file %s.\n",
524 filename);
525 return GNUNET_OK;
526 }
527
528 update_meter (meter);
529
530 buf = data;
531 offset = 0;
532 regex = NULL;
533 while (offset < (filesize - 1))
534 {
535 offset++;
536 if (((data[offset] == '\n')) && (buf != &data[offset]))
537 {
538 data[offset] = '|';
539 num_policies++;
540 buf = &data[offset + 1];
541 }
542 else if ((data[offset] == '\n') || (data[offset] == '\0'))
543 buf = &data[offset + 1];
544 }
545 data[offset] = '\0';
546 GNUNET_asprintf (&regex, "%s(%s)", regex_prefix, data);
547 GNUNET_assert (NULL != regex);
548 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
549 "Announcing regex: %s\n", regex);
550
551 if (GNUNET_OK != announce_regex (regex))
552 {
553 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
554 "Could not announce regex %s\n",
555 regex);
556 }
557 GNUNET_free (regex);
558 GNUNET_free (data);
559 return GNUNET_OK;
560}
561
562
563/**
564 * Iterate over files contained in policy_dir.
565 *
566 * @param cls NULL
567 */
568static void
569do_directory_scan (void *cls)
570{
571 struct GNUNET_TIME_Absolute start_time;
572 struct GNUNET_TIME_Relative duration;
573 char *stmt;
574
575 /* Create an MySQL prepared statement for the inserts */
576 scan_task = NULL;
577 GNUNET_asprintf (&stmt, INSERT_EDGE_STMT, table_name);
578 stmt_handle = GNUNET_MYSQL_statement_prepare (mysql_ctx, stmt);
579 GNUNET_free (stmt);
580
581 GNUNET_asprintf (&stmt, SELECT_KEY_STMT, table_name);
582 select_stmt_handle = GNUNET_MYSQL_statement_prepare (mysql_ctx, stmt);
583 GNUNET_free (stmt);
584
585 GNUNET_assert (NULL != stmt_handle);
586
587 meter = create_meter (num_policy_files,
588 "Announcing policy files\n",
589 GNUNET_YES);
590 start_time = GNUNET_TIME_absolute_get ();
591 GNUNET_DISK_directory_scan (policy_dir,
592 &policy_filename_cb,
593 stmt_handle);
594 duration = GNUNET_TIME_absolute_get_duration (start_time);
595 reset_meter (meter);
596 free_meter (meter);
597 meter = NULL;
598
599 printf ("Announced %u files containing %u policies in %s\n"
600 "Duplicate transitions: %llu\nMerged states: %llu\n",
601 num_policy_files,
602 num_policies,
603 GNUNET_STRINGS_relative_time_to_string (duration, GNUNET_NO),
604 num_merged_transitions,
605 num_merged_states);
606 result = GNUNET_OK;
607 GNUNET_SCHEDULER_shutdown ();
608}
609
610
611/**
612 * Main function that will be run by the scheduler.
613 *
614 * @param cls closure
615 * @param args remaining command-line arguments
616 * @param cfgfile name of the configuration file used (for saving, can be NULL!)
617 * @param config configuration
618 */
619static void
620run (void *cls,
621 char *const *args,
622 const char *cfgfile,
623 const struct GNUNET_CONFIGURATION_Handle *config)
624{
625 if (NULL == args[0])
626 {
627 fprintf (stderr,
628 _("No policy directory specified on command line. Exiting.\n"));
629 result = GNUNET_SYSERR;
630 return;
631 }
632 if (GNUNET_YES !=
633 GNUNET_DISK_directory_test (args[0], GNUNET_YES))
634 {
635 fprintf (stderr,
636 _("Specified policies directory does not exist. Exiting.\n"));
637 result = GNUNET_SYSERR;
638 return;
639 }
640 policy_dir = args[0];
641
642 num_policy_files = GNUNET_DISK_directory_scan (policy_dir,
643 NULL, NULL);
644 meter = NULL;
645
646 if (NULL == table_name)
647 {
648 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
649 "No table name specified, using default \"NFA\".\n");
650 table_name = "NFA";
651 }
652
653 mysql_ctx = GNUNET_MYSQL_context_create (config, "regex-mysql");
654 if (NULL == mysql_ctx)
655 {
656 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
657 "Failed to create mysql context\n");
658 result = GNUNET_SYSERR;
659 return;
660 }
661
662 if (GNUNET_OK !=
663 GNUNET_CONFIGURATION_get_value_string (config,
664 "regex-mysql",
665 "REGEX_PREFIX",
666 &regex_prefix))
667 {
668 GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
669 "regex-mysql",
670 "REGEX_PREFIX");
671 result = GNUNET_SYSERR;
672 return;
673 }
674
675 result = GNUNET_OK;
676 GNUNET_SCHEDULER_add_shutdown (&do_shutdown,
677 NULL);
678 scan_task = GNUNET_SCHEDULER_add_now (&do_directory_scan, NULL);
679}
680
681
682/**
683 * Main function.
684 *
685 * @param argc argument count
686 * @param argv argument values
687 * @return 0 on success
688 */
689int
690main (int argc, char *const *argv)
691{
692 struct GNUNET_GETOPT_CommandLineOption options[] = {
693
694 GNUNET_GETOPT_option_string ('t',
695 "table",
696 "TABLENAME",
697 gettext_noop ("name of the table to write DFAs"),
698 &table_name),
699
700 GNUNET_GETOPT_option_uint ('p',
701 "max-path-compression",
702 "MAX_PATH_COMPRESSION",
703 gettext_noop ("maximum path compression length"),
704 &max_path_compression),
705
706 GNUNET_GETOPT_OPTION_END
707 };
708 int ret;
709
710 if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
711 return 2;
712
713 result = GNUNET_SYSERR;
714 ret =
715 GNUNET_PROGRAM_run (argc, argv,
716 "gnunet-regex-simulationprofiler [OPTIONS] policy-dir",
717 _("Profiler for regex library"), options, &run, NULL);
718 if (GNUNET_OK != ret)
719 return ret;
720 if (GNUNET_OK != result)
721 return 1;
722 return 0;
723}
generated by cgit v1.2.3 (git 2.39.1) at 2024-04-20 05:58:17 +0000