aboutsummaryrefslogtreecommitdiff
path: root/src/regex/gnunet-regex-simulation-profiler.c
diff options
context:
space:
mode:
Diffstat (limited to 'src/regex/gnunet-regex-simulation-profiler.c')
-rw-r--r--src/regex/gnunet-regex-simulation-profiler.c726
1 files changed, 0 insertions, 726 deletions
diff --git a/src/regex/gnunet-regex-simulation-profiler.c b/src/regex/gnunet-regex-simulation-profiler.c
deleted file mode 100644
index ab6d65d58..000000000
--- a/src/regex/gnunet-regex-simulation-profiler.c
+++ /dev/null
@@ -1,726 +0,0 @@
1/*
2 This file is part of GNUnet.
3 Copyright (C) 2011, 2012 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21
22/**
23 * @file regex/gnunet-regex-simulation-profiler.c
24 * @brief Regex profiler that dumps all DFAs into a database instead of
25 * using the DHT (with cadet).
26 * @author Maximilian Szengel
27 * @author Christophe Genevey
28 *
29 */
30
31#include "platform.h"
32#include "gnunet_util_lib.h"
33#include "regex_internal_lib.h"
34#include "gnunet_mysql_lib.h"
35#include "gnunet_my_lib.h"
36#include <mysql/mysql.h>
37
38/**
39 * MySQL statement to insert an edge.
40 */
41#define INSERT_EDGE_STMT "INSERT IGNORE INTO `%s` " \
42 "(`key`, `label`, `to_key`, `accepting`) " \
43 "VALUES (?, ?, ?, ?);"
44
45/**
46 * MySQL statement to select a key count.
47 */
48#define SELECT_KEY_STMT "SELECT COUNT(*) FROM `%s` " \
49 "WHERE `key` = ? AND `label` = ?;"
50
51/**
52 * Simple struct to keep track of progress, and print a
53 * nice little percentage meter for long running tasks.
54 */
55struct ProgressMeter
56{
57 /**
58 * Total number of elements.
59 */
60 unsigned int total;
61
62 /**
63 * Interval for printing percentage.
64 */
65 unsigned int modnum;
66
67 /**
68 * Number of dots to print.
69 */
70 unsigned int dotnum;
71
72 /**
73 * Completed number.
74 */
75 unsigned int completed;
76
77 /**
78 * Should the meter be printed?
79 */
80 int print;
81
82 /**
83 * String to print on startup.
84 */
85 char *startup_string;
86};
87
88
89/**
90 * Handle for the progress meter
91 */
92static struct ProgressMeter *meter;
93
94/**
95 * Scan task identifier;
96 */
97static struct GNUNET_SCHEDULER_Task *scan_task;
98
99/**
100 * Global testing status.
101 */
102static int result;
103
104/**
105 * MySQL context.
106 */
107static struct GNUNET_MYSQL_Context *mysql_ctx;
108
109/**
110 * MySQL prepared statement handle.
111 */
112static struct GNUNET_MYSQL_StatementHandle *stmt_handle;
113
114/**
115 * MySQL prepared statement handle for `key` select.
116 */
117static struct GNUNET_MYSQL_StatementHandle *select_stmt_handle;
118
119/**
120 * MySQL table name.
121 */
122static char *table_name;
123
124/**
125 * Policy dir containing files that contain policies.
126 */
127static char *policy_dir;
128
129/**
130 * Number of policy files.
131 */
132static unsigned int num_policy_files;
133
134/**
135 * Number of policies.
136 */
137static unsigned int num_policies;
138
139/**
140 * Maximal path compression length.
141 */
142static unsigned int max_path_compression;
143
144/**
145 * Number of merged transitions.
146 */
147static unsigned long long num_merged_transitions;
148
149/**
150 * Number of merged states from different policies.
151 */
152static unsigned long long num_merged_states;
153
154/**
155 * Prefix to add before every regex we're announcing.
156 */
157static char *regex_prefix;
158
159
160/**
161 * Create a meter to keep track of the progress of some task.
162 *
163 * @param total the total number of items to complete
164 * @param start_string a string to prefix the meter with (if printing)
165 * @param print GNUNET_YES to print the meter, GNUNET_NO to count
166 * internally only
167 *
168 * @return the progress meter
169 */
170static struct ProgressMeter *
171create_meter (unsigned int total, char *start_string, int print)
172{
173 struct ProgressMeter *ret;
174
175 ret = GNUNET_new (struct ProgressMeter);
176 ret->print = print;
177 ret->total = total;
178 ret->modnum = total / 4;
179 if (ret->modnum == 0) /* Divide by zero check */
180 ret->modnum = 1;
181 ret->dotnum = (total / 50) + 1;
182 if (start_string != NULL)
183 ret->startup_string = GNUNET_strdup (start_string);
184 else
185 ret->startup_string = GNUNET_strdup ("");
186
187 return ret;
188}
189
190
191/**
192 * Update progress meter (increment by one).
193 *
194 * @param meter the meter to update and print info for
195 *
196 * @return GNUNET_YES if called the total requested,
197 * GNUNET_NO if more items expected
198 */
199static int
200update_meter (struct ProgressMeter *meter)
201{
202 if (meter->print == GNUNET_YES)
203 {
204 if (meter->completed % meter->modnum == 0)
205 {
206 if (meter->completed == 0)
207 {
208 fprintf (stdout, "%sProgress: [0%%", meter->startup_string);
209 }
210 else
211 fprintf (stdout, "%d%%",
212 (int) (((float) meter->completed / meter->total) * 100));
213 }
214 else if (meter->completed % meter->dotnum == 0)
215 fprintf (stdout, "%s", ".");
216
217 if (meter->completed + 1 == meter->total)
218 fprintf (stdout, "%d%%]\n", 100);
219 fflush (stdout);
220 }
221 meter->completed++;
222
223 if (meter->completed == meter->total)
224 return GNUNET_YES;
225 if (meter->completed > meter->total)
226 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Progress meter overflow!!\n");
227 return GNUNET_NO;
228}
229
230
231/**
232 * Reset progress meter.
233 *
234 * @param meter the meter to reset
235 *
236 * @return #GNUNET_YES if meter reset,
237 * #GNUNET_SYSERR on error
238 */
239static int
240reset_meter (struct ProgressMeter *meter)
241{
242 if (meter == NULL)
243 return GNUNET_SYSERR;
244
245 meter->completed = 0;
246 return GNUNET_YES;
247}
248
249
250/**
251 * Release resources for meter
252 *
253 * @param meter the meter to free
254 */
255static void
256free_meter (struct ProgressMeter *meter)
257{
258 GNUNET_free (meter->startup_string);
259 GNUNET_free (meter);
260}
261
262
263/**
264 * Shutdown task.
265 *
266 * @param cls NULL
267 */
268static void
269do_shutdown (void *cls)
270{
271 if (NULL != mysql_ctx)
272 {
273 GNUNET_MYSQL_context_destroy (mysql_ctx);
274 mysql_ctx = NULL;
275 }
276 if (NULL != meter)
277 {
278 free_meter (meter);
279 meter = NULL;
280 }
281}
282
283
284/**
285 * Abort task to run on test timed out.
286 *
287 * FIXME: this doesn't actually work, it used to cancel
288 * the already running 'scan_task', but now that should
289 * always be NULL and do nothing. We instead need to set
290 * a global variable and abort scan_task internally, not
291 * via scheduler.
292 *
293 * @param cls NULL
294 */
295static void
296do_abort (void *cls)
297{
298 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Aborting\n");
299 if (NULL != scan_task)
300 {
301 GNUNET_SCHEDULER_cancel (scan_task);
302 scan_task = NULL;
303 }
304 result = GNUNET_SYSERR;
305 GNUNET_SCHEDULER_shutdown ();
306}
307
308
309/**
310 * Iterator over all states that inserts each state into the MySQL db.
311 *
312 * @param cls closure.
313 * @param key hash for current state.
314 * @param proof proof for current state.
315 * @param accepting #GNUNET_YES if this is an accepting state, #GNUNET_NO if not.
316 * @param num_edges number of edges leaving current state.
317 * @param edges edges leaving current state.
318 */
319static void
320regex_iterator (void *cls,
321 const struct GNUNET_HashCode *key,
322 const char *proof,
323 int accepting,
324 unsigned int num_edges,
325 const struct REGEX_BLOCK_Edge *edges)
326{
327 unsigned int i;
328 int result;
329
330 uint32_t iaccepting = (uint32_t) accepting;
331 uint64_t total;
332
333 GNUNET_assert (NULL != mysql_ctx);
334
335 for (i = 0; i < num_edges; i++)
336 {
337 struct GNUNET_MY_QueryParam params_select[] = {
338 GNUNET_MY_query_param_auto_from_type (key),
339 GNUNET_MY_query_param_string (edges[i].label),
340 GNUNET_MY_query_param_end
341 };
342
343 struct GNUNET_MY_ResultSpec results_select[] = {
344 GNUNET_MY_result_spec_uint64 (&total),
345 GNUNET_MY_result_spec_end
346 };
347
348 result =
349 GNUNET_MY_exec_prepared (mysql_ctx,
350 select_stmt_handle,
351 params_select);
352
353 if (GNUNET_SYSERR == result)
354 {
355 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
356 "Error executing prepared mysql select statement\n");
357 GNUNET_SCHEDULER_add_now (&do_abort, NULL);
358 return;
359 }
360
361 result =
362 GNUNET_MY_extract_result (select_stmt_handle,
363 results_select);
364
365 if (GNUNET_SYSERR == result)
366 {
367 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
368 "Error extracting result mysql select statement\n");
369 GNUNET_SCHEDULER_add_now (&do_abort, NULL);
370 return;
371 }
372
373 if ((-1 != total) && (total > 0) )
374 {
375 GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Total: %llu (%s, %s)\n",
376 (unsigned long long) total,
377 GNUNET_h2s (key), edges[i].label);
378 }
379
380 struct GNUNET_MY_QueryParam params_stmt[] = {
381 GNUNET_MY_query_param_auto_from_type (&key),
382 GNUNET_MY_query_param_string (edges[i].label),
383 GNUNET_MY_query_param_auto_from_type (&edges[i].destination),
384 GNUNET_MY_query_param_uint32 (&iaccepting),
385 GNUNET_MY_query_param_end
386 };
387
388 result =
389 GNUNET_MY_exec_prepared (mysql_ctx,
390 stmt_handle,
391 params_stmt);
392
393 if (0 == result)
394 {
395 char *key_str = GNUNET_strdup (GNUNET_h2s (key));
396 char *to_key_str = GNUNET_strdup (GNUNET_h2s (&edges[i].destination));
397
398 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Merged (%s, %s, %s, %i)\n",
399 key_str,
400 edges[i].label,
401 to_key_str,
402 accepting);
403
404 GNUNET_free (key_str);
405 GNUNET_free (to_key_str);
406 num_merged_transitions++;
407 }
408 else if (-1 != total)
409 {
410 num_merged_states++;
411 }
412
413 if ((GNUNET_SYSERR == result) || ((1 != result) && (0 != result) ))
414 {
415 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
416 "Error executing prepared mysql statement for edge: Affected rows: %i, expected 0 or 1!\n",
417 result);
418 GNUNET_SCHEDULER_add_now (&do_abort, NULL);
419 }
420 }
421
422 if (0 == num_edges)
423 {
424 struct GNUNET_MY_QueryParam params_stmt[] = {
425 GNUNET_MY_query_param_auto_from_type (key),
426 GNUNET_MY_query_param_string (""),
427 GNUNET_MY_query_param_fixed_size (NULL, 0),
428 GNUNET_MY_query_param_uint32 (&iaccepting),
429 GNUNET_MY_query_param_end
430 };
431
432 result =
433 GNUNET_MY_exec_prepared (mysql_ctx,
434 stmt_handle,
435 params_stmt);
436
437 if ((1 != result) && (0 != result) )
438 {
439 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
440 "Error executing prepared mysql statement for edge: Affected rows: %i, expected 0 or 1!\n",
441 result);
442 GNUNET_SCHEDULER_add_now (&do_abort, NULL);
443 }
444 }
445}
446
447
448/**
449 * Announce a regex by creating the DFA and iterating over each state, inserting
450 * each state into a MySQL database.
451 *
452 * @param regex regular expression.
453 * @return #GNUNET_OK on success, #GNUNET_SYSERR on failure.
454 */
455static int
456announce_regex (const char *regex)
457{
458 struct REGEX_INTERNAL_Automaton *dfa;
459
460 dfa =
461 REGEX_INTERNAL_construct_dfa (regex,
462 strlen (regex),
463 max_path_compression);
464
465 if (NULL == dfa)
466 {
467 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
468 "Failed to create DFA for regex %s\n",
469 regex);
470 GNUNET_SCHEDULER_add_now (&do_abort, NULL);
471 return GNUNET_SYSERR;
472 }
473 REGEX_INTERNAL_iterate_all_edges (dfa,
474 &regex_iterator, NULL);
475 REGEX_INTERNAL_automaton_destroy (dfa);
476
477 return GNUNET_OK;
478}
479
480
481/**
482 * Function called with a filename.
483 *
484 * @param cls closure
485 * @param filename complete filename (absolute path)
486 * @return #GNUNET_OK to continue to iterate,
487 * #GNUNET_SYSERR to abort iteration with error!
488 */
489static int
490policy_filename_cb (void *cls, const char *filename)
491{
492 char *regex;
493 char *data;
494 char *buf;
495 uint64_t filesize;
496 unsigned int offset;
497
498 GNUNET_assert (NULL != filename);
499
500 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
501 "Announcing regexes from file %s\n",
502 filename);
503
504 if (GNUNET_YES != GNUNET_DISK_file_test (filename))
505 {
506 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
507 "Could not find policy file %s\n",
508 filename);
509 return GNUNET_OK;
510 }
511 if (GNUNET_OK !=
512 GNUNET_DISK_file_size (filename, &filesize,
513 GNUNET_YES, GNUNET_YES))
514 filesize = 0;
515 if (0 == filesize)
516 {
517 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Policy file %s is empty.\n",
518 filename);
519 return GNUNET_OK;
520 }
521 data = GNUNET_malloc (filesize);
522 if (filesize != GNUNET_DISK_fn_read (filename, data, filesize))
523 {
524 GNUNET_free (data);
525 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
526 "Could not read policy file %s.\n",
527 filename);
528 return GNUNET_OK;
529 }
530
531 update_meter (meter);
532
533 buf = data;
534 offset = 0;
535 regex = NULL;
536 while (offset < (filesize - 1))
537 {
538 offset++;
539 if (((data[offset] == '\n')) && (buf != &data[offset]))
540 {
541 data[offset] = '|';
542 num_policies++;
543 buf = &data[offset + 1];
544 }
545 else if ((data[offset] == '\n') || (data[offset] == '\0'))
546 buf = &data[offset + 1];
547 }
548 data[offset] = '\0';
549 GNUNET_asprintf (&regex, "%s(%s)", regex_prefix, data);
550 GNUNET_assert (NULL != regex);
551 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
552 "Announcing regex: %s\n", regex);
553
554 if (GNUNET_OK != announce_regex (regex))
555 {
556 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
557 "Could not announce regex %s\n",
558 regex);
559 }
560 GNUNET_free (regex);
561 GNUNET_free (data);
562 return GNUNET_OK;
563}
564
565
566/**
567 * Iterate over files contained in policy_dir.
568 *
569 * @param cls NULL
570 */
571static void
572do_directory_scan (void *cls)
573{
574 struct GNUNET_TIME_Absolute start_time;
575 struct GNUNET_TIME_Relative duration;
576 char *stmt;
577
578 /* Create an MySQL prepared statement for the inserts */
579 scan_task = NULL;
580 GNUNET_asprintf (&stmt, INSERT_EDGE_STMT, table_name);
581 stmt_handle = GNUNET_MYSQL_statement_prepare (mysql_ctx, stmt);
582 GNUNET_free (stmt);
583
584 GNUNET_asprintf (&stmt, SELECT_KEY_STMT, table_name);
585 select_stmt_handle = GNUNET_MYSQL_statement_prepare (mysql_ctx, stmt);
586 GNUNET_free (stmt);
587
588 GNUNET_assert (NULL != stmt_handle);
589
590 meter = create_meter (num_policy_files,
591 "Announcing policy files\n",
592 GNUNET_YES);
593 start_time = GNUNET_TIME_absolute_get ();
594 GNUNET_DISK_directory_scan (policy_dir,
595 &policy_filename_cb,
596 stmt_handle);
597 duration = GNUNET_TIME_absolute_get_duration (start_time);
598 reset_meter (meter);
599 free_meter (meter);
600 meter = NULL;
601
602 printf ("Announced %u files containing %u policies in %s\n"
603 "Duplicate transitions: %llu\nMerged states: %llu\n",
604 num_policy_files,
605 num_policies,
606 GNUNET_STRINGS_relative_time_to_string (duration, GNUNET_NO),
607 num_merged_transitions,
608 num_merged_states);
609 result = GNUNET_OK;
610 GNUNET_SCHEDULER_shutdown ();
611}
612
613
614/**
615 * Main function that will be run by the scheduler.
616 *
617 * @param cls closure
618 * @param args remaining command-line arguments
619 * @param cfgfile name of the configuration file used (for saving, can be NULL!)
620 * @param config configuration
621 */
622static void
623run (void *cls,
624 char *const *args,
625 const char *cfgfile,
626 const struct GNUNET_CONFIGURATION_Handle *config)
627{
628 if (NULL == args[0])
629 {
630 fprintf (stderr,
631 _ ("No policy directory specified on command line. Exiting.\n"));
632 result = GNUNET_SYSERR;
633 return;
634 }
635 if (GNUNET_YES !=
636 GNUNET_DISK_directory_test (args[0], GNUNET_YES))
637 {
638 fprintf (stderr,
639 _ ("Specified policies directory does not exist. Exiting.\n"));
640 result = GNUNET_SYSERR;
641 return;
642 }
643 policy_dir = args[0];
644
645 num_policy_files = GNUNET_DISK_directory_scan (policy_dir,
646 NULL, NULL);
647 meter = NULL;
648
649 if (NULL == table_name)
650 {
651 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
652 "No table name specified, using default \"NFA\".\n");
653 table_name = "NFA";
654 }
655
656 mysql_ctx = GNUNET_MYSQL_context_create (config, "regex-mysql");
657 if (NULL == mysql_ctx)
658 {
659 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
660 "Failed to create mysql context\n");
661 result = GNUNET_SYSERR;
662 return;
663 }
664
665 if (GNUNET_OK !=
666 GNUNET_CONFIGURATION_get_value_string (config,
667 "regex-mysql",
668 "REGEX_PREFIX",
669 &regex_prefix))
670 {
671 GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
672 "regex-mysql",
673 "REGEX_PREFIX");
674 result = GNUNET_SYSERR;
675 return;
676 }
677
678 result = GNUNET_OK;
679 GNUNET_SCHEDULER_add_shutdown (&do_shutdown,
680 NULL);
681 scan_task = GNUNET_SCHEDULER_add_now (&do_directory_scan, NULL);
682}
683
684
685/**
686 * Main function.
687 *
688 * @param argc argument count
689 * @param argv argument values
690 * @return 0 on success
691 */
692int
693main (int argc, char *const *argv)
694{
695 struct GNUNET_GETOPT_CommandLineOption options[] = {
696 GNUNET_GETOPT_option_string ('t',
697 "table",
698 "TABLENAME",
699 gettext_noop (
700 "name of the table to write DFAs"),
701 &table_name),
702
703 GNUNET_GETOPT_option_uint ('p',
704 "max-path-compression",
705 "MAX_PATH_COMPRESSION",
706 gettext_noop ("maximum path compression length"),
707 &max_path_compression),
708
709 GNUNET_GETOPT_OPTION_END
710 };
711 int ret;
712
713 if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
714 return 2;
715
716 result = GNUNET_SYSERR;
717 ret =
718 GNUNET_PROGRAM_run (argc, argv,
719 "gnunet-regex-simulationprofiler [OPTIONS] policy-dir",
720 _ ("Profiler for regex library"), options, &run, NULL);
721 if (GNUNET_OK != ret)
722 return ret;
723 if (GNUNET_OK != result)
724 return 1;
725 return 0;
726}