-basic santiy for testing API, breaks transport/ build

7 files changed, 376 insertions, 332 deletions

diff --git a/src/include/gnunet_testing_ng_lib.h b/src/include/gnunet_testing_ng_lib.h
index 363c7ff0c..4ef9aac18 100644
--- a/src/include/gnunet_testing_ng_lib.h
+++ b/src/include/gnunet_testing_ng_lib.h
@@ -17,7 +17,6 @@
 
      SPDX-License-Identifier: AGPL3.0-or-later
  */
-
 /**
  * @brief API for writing an interpreter to test GNUnet components
  * @author Christian Grothoff <christian@grothoff.org>
@@ -49,6 +48,7 @@
 
 /**
  * Router of a network namespace.
+ * // FIXME: this does not belong here!
  */
 struct GNUNET_TESTING_NetjailRouter
 {
@@ -63,8 +63,10 @@ struct GNUNET_TESTING_NetjailRouter
   unsigned int udp_port;
 };
 
+
 /**
  * Enum for the different types of nodes.
+ * // FIXME: this does not belong here!
  */
 enum GNUNET_TESTING_NODE_TYPE
 {
@@ -79,6 +81,8 @@ enum GNUNET_TESTING_NODE_TYPE
   GNUNET_TESTING_GLOBAL_NODE
 };
 
+
+ // FIXME: this does not belong here!
 struct GNUNET_TESTING_ADDRESS_PREFIX
 {
   /**
@@ -97,10 +101,13 @@ struct GNUNET_TESTING_ADDRESS_PREFIX
   char *address_prefix;
 };
 
+
+// FIXME: this does not belong here!
 struct GNUNET_TESTING_NetjailNode;
 
 /**
  * Connection to another node.
+ * // FIXME: this does not belong here!
  */
 struct GNUNET_TESTING_NodeConnection
 {
@@ -148,6 +155,7 @@ struct GNUNET_TESTING_NodeConnection
 
 /**
  * Node in the netjail topology.
+ * // FIXME: this does not belong here!
  */
 struct GNUNET_TESTING_NetjailNode
 {
@@ -185,6 +193,7 @@ struct GNUNET_TESTING_NetjailNode
 
 /**
  * Namespace in a topology.
+ * // FIXME: this does not belong here!
  */
 struct GNUNET_TESTING_NetjailNamespace
 {
@@ -206,6 +215,7 @@ struct GNUNET_TESTING_NetjailNamespace
 
 /**
  * Toplogy of our netjail setup.
+ * // FIXME: this does not belong here!
  */
 struct GNUNET_TESTING_NetjailTopology
 {
@@ -318,6 +328,9 @@ struct GNUNET_TESTING_Command
   (*cleanup)(void *cls);
 
   /**
+   * FIXME: logic is often the same!
+   * => Think about refactoring API to reduce duplication!
+   *
    * Extract information from a command that is useful for other
    * commands.
    *
@@ -356,13 +369,6 @@ struct GNUNET_TESTING_Command
   struct GNUNET_TIME_Absolute last_req_time;
 
   /**
-   * How often did we try to execute this command? (In case it is a request
-   * that is repated.)  Note that a command must have some built-in retry
-   * mechanism for this value to be useful.
-   */
-  unsigned int num_tries;
-
-  /**
    * In case @e asynchronous_finish is true, how long should we wait for this
    * command to complete? If @e finish did not complete after this amount of
    * time, the interpreter will fail.  Should be set generously to ensure
@@ -371,6 +377,13 @@ struct GNUNET_TESTING_Command
   struct GNUNET_TIME_Relative default_timeout;
 
   /**
+   * How often did we try to execute this command? (In case it is a request
+   * that is repated.)  Note that a command must have some built-in retry
+   * mechanism for this value to be useful.
+   */
+  unsigned int num_tries;
+
+  /**
    * If "true", the interpreter should not immediately call
    * @e finish, even if @e finish is non-NULL.  Otherwise,
    * #TALER_TESTING_cmd_finish() must be used
@@ -466,22 +479,32 @@ GNUNET_TESTING_cmd_rewind_ip (const char *label,
 
 
 /**
+ * Function called with the final result of the test.
+ *
+ * @param cls closure
+ * @param rv #GNUNET_OK if the test passed
+ */
+typedef void
+(*GNUNET_TESTING_ResultCallback)(void *cls,
+                                 enum GNUNET_GenericReturnValue rv);
+
+
+/**
  * Run the testsuite.  Note, CMDs are copied into
  * the interpreter state because they are _usually_
  * defined into the "run" method that returns after
  * having scheduled the test interpreter.
  *
- * @param cfg_name name of configuration file to use
  * @param commands the list of command to execute
  * @param timeout how long to wait for each command to execute
- * @return #GNUNET_OK if all is okay, != #GNUNET_OK otherwise.
- *         non-GNUNET_OK codes are #GNUNET_SYSERR most of the
- *         times.
+ * @param rc function to call with the final result
+ * @param rc_cls closure for @a rc
  */
-enum GNUNET_GenericReturnValue
-GNUNET_TESTING_run (const char *cfg_filename,
-                    struct GNUNET_TESTING_Command *commands,
-                    struct GNUNET_TIME_Relative timeout);
+void
+GNUNET_TESTING_run (struct GNUNET_TESTING_Command *commands,
+                    struct GNUNET_TIME_Relative timeout,
+                    GNUNET_TESTING_ResultCallback rc,
+                    void *rc_cls);
 
 
 /**
@@ -489,14 +512,12 @@ GNUNET_TESTING_run (const char *cfg_filename,
  * run the testsuite.  Return 0 upon success.
  * Expected to be called directly from main().
  *
- * @param cfg_name name of configuration file to use
  * @param commands the list of command to execute
  * @param timeout how long to wait for each command to execute
  * @return EXIT_SUCCESS on success, EXIT_FAILURE on failure
  */
 int
-GNUNET_TESTING_main (const char *cfg_filename,
-                     struct GNUNET_TESTING_Command *commands,
+GNUNET_TESTING_main (struct GNUNET_TESTING_Command *commands,
                      struct GNUNET_TIME_Relative timeout);
 
 
@@ -505,6 +526,8 @@ GNUNET_TESTING_main (const char *cfg_filename,
  *
  * @param prog program's name to look into
  * @param marker chunk to find in @a prog
+ * // FIXME: this does not belong here! => libgnunetutil, maybe?
+ * // FIXME: return bool? document return value!
  */
 int
 GNUNET_TESTING_has_in_name (const char *prog,
@@ -518,7 +541,7 @@ GNUNET_TESTING_has_in_name (const char *prog,
  *
  * @param label command label.
  * @param process_label label of a command that has a process trait
- * @param process_index index of the process trait at @a process_label
+ * @param process_index index of the process trait at @a process_label // FIXME: enum? needed?
  * @param signal signal to send to @a process.
  * @return the command.
  */
@@ -557,49 +580,8 @@ GNUNET_TESTING_cmd_batch (const char *label,
 
 
 /**
- * Test if this command is a batch command.
- *
- * @return false if not, true if it is a batch command
- */
-// TODO: figure out if this needs to be exposed in the public API.
-int
-GNUNET_TESTING_cmd_is_batch (const struct GNUNET_TESTING_Command *cmd);
-
-
-/**
- * Advance internal pointer to next command.
- *
- * @param is interpreter state.
- */
-// TODO: figure out if this needs to be exposed in the public API.
-void
-GNUNET_TESTING_cmd_batch_next (struct GNUNET_TESTING_Interpreter *is);
-
-
-/**
- * Obtain what command the batch is at.
- *
- * @return cmd current batch command
- */
-// TODO: figure out if this needs to be exposed in the public API.
-struct GNUNET_TESTING_Command *
-GNUNET_TESTING_cmd_batch_get_current (const struct GNUNET_TESTING_Command *cmd);
-
-
-/**
- * Set what command the batch should be at.
- *
- * @param cmd current batch command
- * @param new_ip where to move the IP
- */
-// TODO: figure out if this needs to be exposed in the public API.
-void
-GNUNET_TESTING_cmd_batch_set_current (const struct GNUNET_TESTING_Command *cmd,
-                                      unsigned int new_ip);
-
-
-/**
  * Performance counter.
+ * // FIXME: this might not belong here!
  */
 struct GNUNET_TESTING_Timer
 {
@@ -695,7 +677,7 @@ GNUNET_TESTING_trait_end (void);
  * @param index index number of the trait to extract.
  * @return #GNUNET_OK when the trait is found.
  */
-int
+enum GNUNET_GenericReturnValue
 GNUNET_TESTING_get_trait (const struct GNUNET_TESTING_Trait *traits,
                           const void **ret,
                           const char *trait,
@@ -709,12 +691,12 @@ GNUNET_TESTING_get_trait (const struct GNUNET_TESTING_Trait *traits,
  *
  * @param cmd command to extract trait from.
  * @param index which process to pick if @a cmd
- *        has multiple on offer.
+ *        has multiple on offer. -- FIXME: remove?
  * @param[out] processp set to the address of the pointer to the
  *        process.
  * @return #GNUNET_OK on success.
  */
-int
+enum GNUNET_GenericReturnValue
 GNUNET_TESTING_get_trait_process (const struct GNUNET_TESTING_Command *cmd,
                                   unsigned int index,
                                   struct GNUNET_OS_Process ***processp);
@@ -724,7 +706,7 @@ GNUNET_TESTING_get_trait_process (const struct GNUNET_TESTING_Command *cmd,
  * Offer location where a command stores a pointer to a process.
  *
  * @param index offered location index number, in case there are
- *        multiple on offer.
+ *        multiple on offer. // FIXME: remove?
  * @param processp process location to offer.
  * @return the trait.
  */
@@ -736,7 +718,7 @@ GNUNET_TESTING_make_trait_process (unsigned int index,
 /**
  * Offer number trait, 32-bit version.
  *
- * @param index the number's index number.
+ * @param index the number's index number. // FIXME: introduce enum?
  * @param n number to offer.
  */
 struct GNUNET_TESTING_Trait
@@ -748,11 +730,11 @@ GNUNET_TESTING_make_trait_uint32 (unsigned int index,
  * Obtain a "number" value from @a cmd, 32-bit version.
  *
  * @param cmd command to extract the number from.
- * @param index the number's index number.
+ * @param index the number's index number. // FIXME: introduce enum?
  * @param[out] n set to the number coming from @a cmd.
  * @return #GNUNET_OK on success.
  */
-int
+enum GNUNET_GenericReturnValue
 GNUNET_TESTING_get_trait_uint32 (const struct GNUNET_TESTING_Command *cmd,
                                  unsigned int index,
                                  const uint32_t **n);
@@ -761,7 +743,7 @@ GNUNET_TESTING_get_trait_uint32 (const struct GNUNET_TESTING_Command *cmd,
 /**
  * Offer number trait, 64-bit version.
  *
- * @param index the number's index number.
+ * @param index the number's index number. // FIXME: introduce enum?
  * @param n number to offer.
  */
 struct GNUNET_TESTING_Trait
@@ -773,11 +755,11 @@ GNUNET_TESTING_make_trait_uint64 (unsigned int index,
  * Obtain a "number" value from @a cmd, 64-bit version.
  *
  * @param cmd command to extract the number from.
- * @param index the number's index number.
+ * @param index the number's index number. // FIXME: introduce enum?
  * @param[out] n set to the number coming from @a cmd.
  * @return #GNUNET_OK on success.
  */
-int
+enum GNUNET_GenericReturnValue
 GNUNET_TESTING_get_trait_uint64 (const struct GNUNET_TESTING_Command *cmd,
                                  unsigned int index,
                                  const uint64_t **n);
@@ -786,7 +768,7 @@ GNUNET_TESTING_get_trait_uint64 (const struct GNUNET_TESTING_Command *cmd,
 /**
  * Offer number trait, 64-bit signed version.
  *
- * @param index the number's index number.
+ * @param index the number's index number. // FIXME: introduce enum?
  * @param n number to offer.
  */
 struct GNUNET_TESTING_Trait
@@ -798,11 +780,11 @@ GNUNET_TESTING_make_trait_int64 (unsigned int index,
  * Obtain a "number" value from @a cmd, 64-bit signed version.
  *
  * @param cmd command to extract the number from.
- * @param index the number's index number.
+ * @param index the number's index number. // FIXME: introduce enum?
  * @param[out] n set to the number coming from @a cmd.
  * @return #GNUNET_OK on success.
  */
-int
+enum GNUNET_GenericReturnValue
 GNUNET_TESTING_get_trait_int64 (const struct GNUNET_TESTING_Command *cmd,
                                 unsigned int index,
                                 const int64_t **n);
@@ -812,7 +794,7 @@ GNUNET_TESTING_get_trait_int64 (const struct GNUNET_TESTING_Command *cmd,
  * Offer a number.
  *
  * @param index the number's index number.
- * @param n the number to offer.
+ * @param n the number to offer. // FIXME: introduce enum?
  * @return #GNUNET_OK on success.
  */
 struct GNUNET_TESTING_Trait
@@ -824,11 +806,11 @@ GNUNET_TESTING_make_trait_uint (unsigned int index,
  * Obtain a number from @a cmd.
  *
  * @param cmd command to extract the number from.
- * @param index the number's index number.
+ * @param index the number's index number. // FIXME: introduce enum?
  * @param[out] n set to the number coming from @a cmd.
  * @return #GNUNET_OK on success.
  */
-int
+enum GNUNET_GenericReturnValue
 GNUNET_TESTING_get_trait_uint (const struct GNUNET_TESTING_Command *cmd,
                                unsigned int index,
                                const unsigned int **n);
@@ -838,12 +820,12 @@ GNUNET_TESTING_get_trait_uint (const struct GNUNET_TESTING_Command *cmd,
  *
  * @param cmd command to extract the subject from.
  * @param index index number associated with the transfer
- *        subject to offer.
+ *        subject to offer. // FIXME: introduce enum?
  * @param[out] s where to write the offered
  *        string.
  * @return #GNUNET_OK on success.
  */
-int
+enum GNUNET_GenericReturnValue
 GNUNET_TESTING_get_trait_string (
   const struct GNUNET_TESTING_Command *cmd,
   unsigned int index,
@@ -854,7 +836,7 @@ GNUNET_TESTING_get_trait_string (
  * Offer string subject.
  *
  * @param index index number associated with the transfer
- *        subject being offered.
+ *        subject being offered. // FIXME: introduce enum?
  * @param s string to offer.
  * @return the trait.
  */
@@ -868,9 +850,8 @@ GNUNET_TESTING_make_trait_string (unsigned int index,
  * @param index always zero.  Commands offering this
  *        kind of traits do not need this index.  For
  *        example, a "meta" CMD returns always the
- *        CMD currently being executed.
+ *        CMD currently being executed. FIXME: remove!
  * @param cmd wire details to offer.
- *
  * @return the trait.
  */
 struct GNUNET_TESTING_Trait
@@ -885,11 +866,11 @@ GNUNET_TESTING_make_trait_cmd (unsigned int index,
  * @param index always zero.  Commands offering this
  *        kind of traits do not need this index.  For
  *        example, a "meta" CMD returns always the
- *        CMD currently being executed.
+ *        CMD currently being executed. FIXME: remove!
  * @param[out] _cmd where to write the wire details.
  * @return #GNUNET_OK on success.
  */
-int
+enum GNUNET_GenericReturnValue
 GNUNET_TESTING_get_trait_cmd (const struct GNUNET_TESTING_Command *cmd,
                               unsigned int index,
                               struct GNUNET_TESTING_Command **_cmd);
@@ -900,11 +881,11 @@ GNUNET_TESTING_get_trait_cmd (const struct GNUNET_TESTING_Command *cmd,
  *
  * @param cmd command to extract the uuid from.
  * @param index which amount to pick if @a cmd has multiple
- *        on offer
+ *        on offer   // FIXME: introduce enum?
  * @param[out] uuid where to write the uuid.
  * @return #GNUNET_OK on success.
  */
-int
+enum GNUNET_GenericReturnValue
 GNUNET_TESTING_get_trait_uuid (const struct GNUNET_TESTING_Command *cmd,
                                unsigned int index,
                                struct GNUNET_Uuid **uuid);
@@ -914,9 +895,8 @@ GNUNET_TESTING_get_trait_uuid (const struct GNUNET_TESTING_Command *cmd,
  * Offer a uuid in a trait.
  *
  * @param index which uuid to offer, in case there are
- *        multiple available.
+ *        multiple available.  // FIXME: introduce enum?
  * @param uuid the uuid to offer.
- *
  * @return the trait.
  */
 struct GNUNET_TESTING_Trait
@@ -929,11 +909,11 @@ GNUNET_TESTING_make_trait_uuid (unsigned int index,
  *
  * @param cmd command to extract trait from
  * @param index which time stamp to pick if
- *        @a cmd has multiple on offer.
+ *        @a cmd has multiple on offer // FIXME: introduce enum?
  * @param[out] time set to the wanted WTID.
  * @return #GNUNET_OK on success
  */
-int
+enum GNUNET_GenericReturnValue
 GNUNET_TESTING_get_trait_absolute_time (
   const struct GNUNET_TESTING_Command *cmd,
   unsigned int index,
@@ -982,6 +962,9 @@ GNUNET_TESTING_make_trait_relative_time (
   const struct GNUNET_TIME_Relative *time);
 
 
+// FIXME: move these commands into a separate libgnunetestingnetjail lib or so!
+
+
 /**
  * Create command.
  *
@@ -1133,7 +1116,9 @@ GNUNET_TESTING_cmd_stop_testing_system_v2 (const char *label,
                                            const char *topology_config);
 
 
-int
+
+// FIXME: document!
+enum GNUNET_GenericReturnValue
 GNUNET_TESTING_get_trait_helper_handles (const struct
                                          GNUNET_TESTING_Command *cmd,
                                          struct GNUNET_HELPER_Handle ***helper);
@@ -1146,31 +1131,30 @@ GNUNET_TESTING_get_trait_helper_handles (const struct
  * @param pt pointer to message.
  * @return #GNUNET_OK on success.
  */
-int
-GNUNET_TESTING_get_trait_helper_handles_v2 (const struct
-                                            GNUNET_TESTING_Command *cmd,
-                                            struct GNUNET_HELPER_Handle ***
-                                            helper);
+enum GNUNET_GenericReturnValue
+GNUNET_TESTING_get_trait_helper_handles_v2 (
+  const struct GNUNET_TESTING_Command *cmd,
+  struct GNUNET_HELPER_Handle ***helper);
 
 
 struct GNUNET_TESTING_Command
-GNUNET_TESTING_cmd_block_until_all_peers_started (const char *label,
-                                                  unsigned int *
-                                                  all_peers_started);
+GNUNET_TESTING_cmd_block_until_all_peers_started (
+  const char *label,
+  unsigned int *all_peers_started);
 
 
 struct GNUNET_TESTING_Command
-GNUNET_TESTING_cmd_block_until_external_trigger (const char *label,
-                                                 unsigned int *
-                                                 stop_blocking);
+GNUNET_TESTING_cmd_block_until_external_trigger (
+  const char *label,
+  unsigned int *stop_blocking);
 
 struct GNUNET_TESTING_Command
 GNUNET_TESTING_cmd_send_peer_ready (const char *label,
                                     TESTING_CMD_HELPER_write_cb write_message);
 
 struct GNUNET_TESTING_Command
-GNUNET_TESTING_cmd_local_test_finished (const char *label,
-                                        TESTING_CMD_HELPER_write_cb
-                                        write_message);
+GNUNET_TESTING_cmd_local_test_finished (
+  const char *label,
+  TESTING_CMD_HELPER_write_cb write_message);
 
 #endif
diff --git a/src/testing/Makefile.am b/src/testing/Makefile.am
index 3e0d4d443..2bb03d157 100644
--- a/src/testing/Makefile.am
+++ b/src/testing/Makefile.am
@@ -42,9 +42,11 @@ libgnunet_test_testing_plugin_testcmd_la_LIBADD = \
 libgnunet_test_testing_plugin_testcmd_la_LDFLAGS = \
  $(GN_PLUGIN_LDFLAGS)
 
+#  testing_api_cmd_finish.c 
+
 libgnunettesting_la_SOURCES = \
+  testing_api_cmd_end.c \
   testing_api_cmd_local_test_finished.c \
-  testing_api_cmd_finish.c \
   testing_api_cmd_send_peer_ready.c \
   testing_api_cmd_block_until_all_peers_started.c \
   testing_api_cmd_block_until_external_trigger.c \
@@ -56,7 +58,7 @@ libgnunettesting_la_SOURCES = \
   testing_api_cmd_netjail_stop_testsystem_v2.c \
   testing_api_cmd_netjail_stop.c \
   testing_api_cmd_netjail_stop_v2.c \
-  testing.c testing.h \
+  testing.c \
   testing_api_cmd_system_create.c \
   testing_api_cmd_system_destroy.c \
   testing_api_cmd_batch.c \
diff --git a/src/testing/test_testing_plugin_testcmd.c b/src/testing/test_testing_plugin_testcmd.c
index 444272fcd..32e2b38a7 100644
--- a/src/testing/test_testing_plugin_testcmd.c
+++ b/src/testing/test_testing_plugin_testcmd.c
@@ -17,11 +17,12 @@
 
      SPDX-License-Identifier: AGPL3.0-or-later
  */
-
 /**
  * @file testbed/plugin_testcmd.c
  * @brief a plugin to provide the API for running test cases.
  * @author t3sserakt
+ *
+ * // FIXME: too verbose, no logic to return final status, will segv!
  */
 #include "platform.h"
 #include "gnunet_testing_ng_lib.h"
@@ -33,8 +34,11 @@
  */
 #define LOG(kind, ...) GNUNET_log (kind, __VA_ARGS__)
 
+
+// FIXME: bad global!
 unsigned int are_all_peers_started;
 
+
 static void
 all_peers_started ()
 {
@@ -44,8 +48,10 @@ all_peers_started ()
        are_all_peers_started);
 }
 
+
 static void
-start_testcase (TESTING_CMD_HELPER_write_cb write_message, char *router_ip,
+start_testcase (TESTING_CMD_HELPER_write_cb write_message,
+                char *router_ip,
                 char *node_ip,
                 char *n,
                 char *m,
@@ -70,9 +76,10 @@ start_testcase (TESTING_CMD_HELPER_write_cb write_message, char *router_ip,
                                             write_message)
   };
 
-  GNUNET_TESTING_run (NULL,
-                      commands,
-                      GNUNET_TIME_UNIT_FOREVER_REL);
+  GNUNET_TESTING_run (commands,
+                      GNUNET_TIME_UNIT_FOREVER_REL,
+                      NULL, /* FIXME: pass continuation! */
+                      NULL);
   LOG (GNUNET_ERROR_TYPE_ERROR,
        "We got here 7!\n");
 
@@ -113,4 +120,4 @@ libgnunet_plugin_testcmd_done (void *cls)
 }
 
 
-/* end of plugin_testcmd.c */
+
diff --git a/src/testing/testing.h b/src/testing/testing.h
index b12466530..8aba09e4b 100644
--- a/src/testing/testing.h
+++ b/src/testing/testing.h
@@ -21,54 +21,54 @@
 /**
  * @author t3sserakt
  */
-
+#ifndef TESTING_H
+#define TESTING_H
 #include "gnunet_util_lib.h"
 
+
+/**
+ * Advance internal pointer to next command.
+ *
+ * @param cls batch internal state
+ * @return true if we could advance, false if the batch
+ *         has completed and cannot advance anymore
+ */
+bool
+GNUNET_TESTING_cmd_batch_next_ (void *cls);
+
+
+/**
+ * Test if this command is a batch command.
+ *
+ * @return false if not, true if it is a batch command
+ */
+bool
+GNUNET_TESTING_cmd_is_batch_ (const struct GNUNET_TESTING_Command *cmd);
+
+
+/**
+ * Obtain what command the batch is at.
+ *
+ * @return cmd current batch command
+ */
+struct GNUNET_TESTING_Command *
+GNUNET_TESTING_cmd_batch_get_current_ (const struct GNUNET_TESTING_Command *cmd);
+
+
 /**
- * Global state of the interpreter, used by a command
- * to access information about other commands.
+ * Set what command the batch should be at.  Needed for
+ * loops. We may want to change this to take a label
+ * and/or expose it in the public API in the future.
+ * Not used for now.
+ *
+ * @param cmd current batch command
+ * @param new_ip where to move the IP
  */
-// SUGGESTION: consider making this struct opaque (only known inside of libgnunettesting,
-// say main loop and a few select commands, like next/fail/batch); + helper
-// function to access 'cfg'?
-struct GNUNET_TESTING_Interpreter
-{
-
-  /**
-   * Commands the interpreter will run.
-   */
-  struct GNUNET_TESTING_Command *commands;
-
-  /**
-   * Interpreter task (if one is scheduled).
-   */
-  struct GNUNET_SCHEDULER_Task *task;
-
-  /**
-   * Finish task of a blocking call to a commands finish method.
-   */
-  struct GNUNET_SCHEDULER_Task *finish_task;
-
-  /**
-   * Our configuration.
-   */
-  const struct GNUNET_CONFIGURATION_Handle *cfg;
-
-  /**
-   * Task run on timeout.
-   */
-  struct GNUNET_SCHEDULER_Task *timeout_task;
-
-  /**
-   * Instruction pointer.  Tells #interpreter_run() which instruction to run
-   * next.  Need (signed) int because it gets -1 when rewinding the
-   * interpreter to the first CMD.
-   */
-  int ip;
-
-  /**
-   * Result of the testcases, #GNUNET_OK on success
-   */
-  int result;
-
-};
+void
+GNUNET_TESTING_cmd_batch_set_current_ (const struct GNUNET_TESTING_Command *cmd,
+                                       unsigned int new_ip);
+
+
+
+
+#endif
diff --git a/src/testing/testing_api_cmd_batch.c b/src/testing/testing_api_cmd_batch.c
index e39489616..080a4880d 100644
--- a/src/testing/testing_api_cmd_batch.c
+++ b/src/testing/testing_api_cmd_batch.c
@@ -119,16 +119,15 @@ batch_traits (void *cls,
               const char *trait,
               unsigned int index)
 {
+  struct BatchState *bs = cls;
+  // FIXME: these constants should be more global!
 #define CURRENT_CMD_INDEX 0
 #define BATCH_INDEX 1
-
-  struct BatchState *bs = cls;
-
   struct GNUNET_TESTING_Trait traits[] = {
-    GNUNET_TESTING_make_trait_cmd
-      (CURRENT_CMD_INDEX, &bs->batch[bs->batch_ip]),
-    GNUNET_TESTING_make_trait_cmd
-      (BATCH_INDEX, bs->batch),
+    GNUNET_TESTING_make_trait_cmd (CURRENT_CMD_INDEX,
+                                   &bs->batch[bs->batch_ip]),
+    GNUNET_TESTING_make_trait_cmd (BATCH_INDEX,
+                                   bs->batch),
     GNUNET_TESTING_trait_end ()
   };
 
@@ -185,68 +184,45 @@ GNUNET_TESTING_cmd_batch (const char *label,
 }
 
 
-/**
- * Advance internal pointer to next command.
- *
- * @param is interpreter state.
- */
-void
-GNUNET_TESTING_cmd_batch_next (struct GNUNET_TESTING_Interpreter *is)
+bool
+GNUNET_TESTING_cmd_batch_next_ (void *cls)
 {
-  struct BatchState *bs = is->commands[is->ip].cls;
+  struct BatchState *bs = cls;
 
   if (NULL == bs->batch[bs->batch_ip].label)
-  {
-    is->commands[is->ip].finish_time = GNUNET_TIME_absolute_get ();
-    is->ip++;
-    return;
-  }
-  bs->batch[bs->batch_ip].finish_time = GNUNET_TIME_absolute_get ();
+    return false;
+  bs->batch[bs->batch_ip].finish_time
+    = GNUNET_TIME_absolute_get ();
   bs->batch_ip++;
+  return true;
 }
 
 
-/**
- * Test if this command is a batch command.
- *
- * @return false if not, true if it is a batch command
- */
-int
-GNUNET_TESTING_cmd_is_batch (const struct GNUNET_TESTING_Command *cmd)
+bool
+GNUNET_TESTING_cmd_is_batch_ (const struct GNUNET_TESTING_Command *cmd)
 {
   return cmd->run == &batch_run;
 }
 
 
-/**
- * Obtain what command the batch is at.
- *
- * @return cmd current batch command
- */
 struct GNUNET_TESTING_Command *
-GNUNET_TESTING_cmd_batch_get_current (const struct GNUNET_TESTING_Command *cmd)
+GNUNET_TESTING_cmd_batch_get_current_ (const struct GNUNET_TESTING_Command *cmd)
 {
   struct BatchState *bs = cmd->cls;
 
-  GNUNET_assert (cmd->run == &batch_run);
+  GNUNET_assert (GNUNET_TESTING_cmd_is_batch_ (cmd));
   return &bs->batch[bs->batch_ip];
 }
 
 
-/**
- * Set what command the batch should be at.
- *
- * @param cmd current batch command
- * @param new_ip where to move the IP
- */
 void
-GNUNET_TESTING_cmd_batch_set_current (const struct GNUNET_TESTING_Command *cmd,
-                                      unsigned int new_ip)
+GNUNET_TESTING_cmd_batch_set_current_ (const struct GNUNET_TESTING_Command *cmd,
+                                       unsigned int new_ip)
 {
   struct BatchState *bs = cmd->cls;
 
   /* sanity checks */
-  GNUNET_assert (cmd->run == &batch_run);
+  GNUNET_assert (GNUNET_TESTING_cmd_is_batch_ (cmd));
   for (unsigned int i = 0; i < new_ip; i++)
     GNUNET_assert (NULL != bs->batch[i].label);
   /* actual logic */
diff --git a/src/testing/testing_api_cmd_end.c b/src/testing/testing_api_cmd_end.c
new file mode 100644
index 000000000..f0f036429
--- /dev/null
+++ b/src/testing/testing_api_cmd_end.c
@@ -0,0 +1,39 @@
+/*
+      This file is part of GNUnet
+      Copyright (C) 2021 GNUnet e.V.
+
+      GNUnet is free software: you can redistribute it and/or modify it
+      under the terms of the GNU Affero General Public License as published
+      by the Free Software Foundation, either version 3 of the License,
+      or (at your option) any later version.
+
+      GNUnet is distributed in the hope that it will be useful, but
+      WITHOUT ANY WARRANTY; without even the implied warranty of
+      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+      Affero General Public License for more details.
+
+      You should have received a copy of the GNU Affero General Public License
+      along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+/**
+ * @file testing/testing_api_cmd_end.c
+ * @brief command to end a command array
+ */
+#include "platform.h"
+#include "gnunet_util_lib.h"
+#include "gnunet_testing_ng_lib.h"
+
+
+struct GNUNET_TESTING_Command
+GNUNET_TESTING_cmd_end (void)
+{
+  static struct GNUNET_TESTING_Command cmd = {
+    .label = NULL
+  };
+
+  return cmd;
+}
+
+
diff --git a/src/testing/testing_api_loop.c b/src/testing/testing_api_loop.c
index 1c8eb1db6..b21e01fcc 100644
--- a/src/testing/testing_api_loop.c
+++ b/src/testing/testing_api_loop.c
@@ -24,25 +24,64 @@
  * @author Christian Grothoff (GNU Taler testing)
  * @author Marcello Stanisci (GNU Taler testing)
  * @author t3sserakt
- *
- * FIXME:
- * - interpreter failure is NOT returned properly yet!
- * - abuse of shutdown logic for interpreter termination
- *   => API design flaw to be fixed!
  */
 #include "platform.h"
 #include "gnunet_util_lib.h"
 #include "gnunet_testing_ng_lib.h"
 #include "testing.h"
 
-
 /**
- * Lookup command by label.
- *
- * @param is interpreter to lookup command in
- * @param label label to look for
- * @return NULL if command was not found
+ * Global state of the interpreter, used by a command
+ * to access information about other commands.
  */
+struct GNUNET_TESTING_Interpreter
+{
+
+  /**
+   * Function to call with the test result.
+   */
+  GNUNET_TESTING_ResultCallback rc;
+
+  /**
+   * Closure for @e rc.
+   */
+  void *rc_cls;
+  
+  /**
+   * Commands the interpreter will run.
+   */
+  struct GNUNET_TESTING_Command *commands;
+
+  /**
+   * Interpreter task (if one is scheduled).
+   */
+  struct GNUNET_SCHEDULER_Task *task;
+
+  /**
+   * Final task that returns the result.
+   */
+  struct GNUNET_SCHEDULER_Task *final_task;
+
+  /**
+   * Task run on timeout.
+   */
+  struct GNUNET_SCHEDULER_Task *timeout_task;
+
+  /**
+   * Instruction pointer.  Tells #interpreter_run() which instruction to run
+   * next.  Need (signed) int because it gets -1 when rewinding the
+   * interpreter to the first CMD.
+   */
+  int ip;
+
+  /**
+   * Result of the testcases, #GNUNET_OK on success
+   */
+  enum GNUNET_GenericReturnValue result;
+
+};
+
+
 const struct GNUNET_TESTING_Command *
 GNUNET_TESTING_interpreter_lookup_command (
   struct GNUNET_TESTING_Interpreter *is,
@@ -65,7 +104,7 @@ GNUNET_TESTING_interpreter_lookup_command (
                        label)) )
       return cmd;
 
-    if (GNUNET_TESTING_cmd_is_batch (cmd))
+    if (GNUNET_TESTING_cmd_is_batch_ (cmd))
     {
 #define BATCH_INDEX 1
       struct GNUNET_TESTING_Command *batch;
@@ -73,7 +112,7 @@ GNUNET_TESTING_interpreter_lookup_command (
       struct GNUNET_TESTING_Command *icmd;
       const struct GNUNET_TESTING_Command *match;
 
-      current = GNUNET_TESTING_cmd_batch_get_current (cmd);
+      current = GNUNET_TESTING_cmd_batch_get_current_ (cmd);
       GNUNET_assert (GNUNET_OK ==
                      GNUNET_TESTING_get_trait_cmd (cmd,
                                                    BATCH_INDEX,
@@ -96,10 +135,58 @@ GNUNET_TESTING_interpreter_lookup_command (
     }
   }
   GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-              "Command not found: %s\n",
+              "Command `%s' not found\n",
               label);
   return NULL;
+}
+
+
+/**
+ * Finish the test run, return the final result.
+ *
+ * @param cls the `struct GNUNET_TESTING_Interpreter`
+ */
+static void
+finish_test (void *cls)
+{
+  struct GNUNET_TESTING_Interpreter *is = cls;
+  struct GNUNET_TESTING_Command *cmd;
+  const char *label;
 
+  is->final_task = NULL;
+  label = is->commands[is->ip].label;
+  if (NULL == label)
+    label = "END";
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Interpreter finishes at `%s' with status %d\n",
+              label,
+              is->result);
+  for (unsigned int j = 0;
+       NULL != (cmd = &is->commands[j])->label;
+       j++)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Cleaning up cmd %s\n",
+                cmd->label);
+    cmd->cleanup (cmd->cls);
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Cleaned up cmd %s\n",
+                cmd->label);
+  }
+  if (NULL != is->task)
+  {
+    GNUNET_SCHEDULER_cancel (is->task);
+    is->task = NULL;
+  }
+  if (NULL != is->timeout_task)
+  {
+    GNUNET_SCHEDULER_cancel (is->timeout_task);
+    is->timeout_task = NULL;
+  }
+  GNUNET_free (is->commands);
+  is->rc (is->rc_cls,
+          is->result);
+  GNUNET_free (is);
 }
 
 
@@ -125,15 +212,10 @@ interpreter_next (void *cls)
 
   if (GNUNET_SYSERR == is->result)
     return; /* ignore, we already failed! */
-  if (GNUNET_TESTING_cmd_is_batch (cmd))
-  {
-    GNUNET_TESTING_cmd_batch_next (is);
-  }
-  else
-  {
-    cmd->finish_time = GNUNET_TIME_absolute_get ();
+  cmd->finish_time = GNUNET_TIME_absolute_get ();
+  if ( (! GNUNET_TESTING_cmd_is_batch_ (cmd)) ||
+       (! GNUNET_TESTING_cmd_batch_next_ (cmd->cls)) )
     is->ip++;
-  }
   if (0 == (ipc % 1000))
   {
     if (0 != ipc)
@@ -150,26 +232,24 @@ interpreter_next (void *cls)
 }
 
 
-/**
- * Current command failed, clean up and fail the test case.
- *
- * @param is interpreter of the test
- */
 void
 GNUNET_TESTING_interpreter_fail (struct GNUNET_TESTING_Interpreter *is)
 {
   struct GNUNET_TESTING_Command *cmd = &is->commands[is->ip];
 
   if (GNUNET_SYSERR == is->result)
+  {
+    GNUNET_break (0);
     return; /* ignore, we already failed! */
+  }
   if (NULL != cmd)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                 "Failed at command `%s'\n",
                 cmd->label);
-    while (GNUNET_TESTING_cmd_is_batch (cmd))
+    while (GNUNET_TESTING_cmd_is_batch_ (cmd))
     {
-      cmd = GNUNET_TESTING_cmd_batch_get_current (cmd);
+      cmd = GNUNET_TESTING_cmd_batch_get_current_ (cmd);
       GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                   "Failed in batch at command `%s'\n",
                   cmd->label);
@@ -181,29 +261,12 @@ GNUNET_TESTING_interpreter_fail (struct GNUNET_TESTING_Interpreter *is)
                 "Failed with CMD being NULL!\n");
   }
   is->result = GNUNET_SYSERR;
-  GNUNET_SCHEDULER_shutdown ();
-}
-
-
-/**
- * Create command array terminator.
- *
- * @return a end-command.
- */
-struct GNUNET_TESTING_Command
-GNUNET_TESTING_cmd_end (void)
-{
-  static struct GNUNET_TESTING_Command cmd = {
-    .label = NULL
-  };
-
-  return cmd;
+  GNUNET_assert (NULL == is->final_task);
+  is->final_task = GNUNET_SCHEDULER_add_now (&finish_test,
+                                             is);
 }
 
 
-/**
- * Obtain current label.
- */
 const char *
 GNUNET_TESTING_interpreter_get_current_label (
   struct GNUNET_TESTING_Interpreter *is)
@@ -234,12 +297,9 @@ interpreter_run (void *cls)
     GNUNET_SCHEDULER_shutdown ();
     return;
   }
-  if (NULL != cmd)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Running command `%s'\n",
-                cmd->label);
-  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Running command `%s'\n",
+              cmd->label);
   cmd->start_time
     = cmd->last_req_time
       = GNUNET_TIME_absolute_get ();
@@ -261,57 +321,6 @@ interpreter_run (void *cls)
 
 
 /**
- * Function run when the test terminates (good or bad).
- * Cleans up our state.
- *
- * @param cls the interpreter state.
- */
-static void
-do_shutdown (void *cls)
-{
-  struct GNUNET_TESTING_Interpreter *is = cls;
-  struct GNUNET_TESTING_Command *cmd;
-  const char *label;
-
-  label = is->commands[is->ip].label;
-  if (NULL == label)
-    label = "END";
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-              "Executing shutdown at `%s'\n",
-              label);
-  for (unsigned int j = 0;
-       NULL != (cmd = &is->commands[j])->label;
-       j++)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Cleaning up cmd %s\n",
-                cmd->label);
-    cmd->cleanup (cmd->cls);
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Cleaned up cmd %s\n",
-                cmd->label);
-  }
-  if (NULL != is->finish_task)
-  {
-    GNUNET_SCHEDULER_cancel (is->finish_task);
-    is->finish_task = NULL;
-  }
-  if (NULL != is->task)
-  {
-    GNUNET_SCHEDULER_cancel (is->task);
-    is->task = NULL;
-  }
-  if (NULL != is->timeout_task)
-  {
-    GNUNET_SCHEDULER_cancel (is->timeout_task);
-    is->timeout_task = NULL;
-  }
-  GNUNET_free (is->commands);
-  GNUNET_free (is);
-}
-
-
-/**
  * Function run when the test terminates (good or bad) with timeout.
  *
  * @param cls the interpreter state
@@ -323,20 +332,24 @@ do_timeout (void *cls)
 
   is->timeout_task = NULL;
   GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-              "Terminating test due to timeout\n");
-  GNUNET_SCHEDULER_shutdown ();
+              "Terminating test due to global timeout\n");
+  is->result = GNUNET_SYSERR;
+  finish_test (is);
 }
 
 
-enum GNUNET_GenericReturnValue
-GNUNET_TESTING_run (const char *cfg_filename,
-                    struct GNUNET_TESTING_Command *commands,
-                    struct GNUNET_TIME_Relative timeout)
+void
+GNUNET_TESTING_run (struct GNUNET_TESTING_Command *commands,
+                    struct GNUNET_TIME_Relative timeout,
+                    GNUNET_TESTING_ResultCallback rc,
+                    void *rc_cls)
 {
   struct GNUNET_TESTING_Interpreter *is;
   unsigned int i;
 
   is = GNUNET_new (struct GNUNET_TESTING_Interpreter);
+  is->rc = rc;
+  is->rc_cls = rc_cls;
   /* get the number of commands */
   for (i = 0; NULL != commands[i].label; i++)
     ;
@@ -349,11 +362,8 @@ GNUNET_TESTING_run (const char *cfg_filename,
     = GNUNET_SCHEDULER_add_delayed (timeout,
                                     &do_timeout,
                                     is);
-  GNUNET_SCHEDULER_add_shutdown (&do_shutdown,
-                                 is);
   is->task = GNUNET_SCHEDULER_add_now (&interpreter_run,
                                        is);
-  return GNUNET_OK;
 }
 
 
@@ -362,14 +372,46 @@ GNUNET_TESTING_run (const char *cfg_filename,
  */
 struct MainParams
 {
-  const char *cfg_filename;
+
+  /**
+   * NULL-label terminated array of commands.
+   */
   struct GNUNET_TESTING_Command *commands;
+
+  /**
+   * Global timeout for the test.
+   */ 
   struct GNUNET_TIME_Relative timeout;
+
+  /**
+   * Set to #EXIT_FAILURE on error.
+   */ 
   int rv;
 };
 
 
 /**
+ * Function called with the final result of the test.
+ *
+ * @param cls the `struct MainParams`
+ * @param rv #GNUNET_OK if the test passed
+ */
+static void
+handle_result (void *cls,
+               enum GNUNET_GenericReturnValue rv)
+{
+  struct MainParams *mp = cls;
+
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Test exits with status %d\n",
+              rv);
+  if (GNUNET_OK != rv)
+    mp->rv = EXIT_FAILURE;
+  GNUNET_SCHEDULER_shutdown ();
+}
+
+
+/**
  * Main function to run the test cases.
  *
  * @param cls a `struct MainParams *`
@@ -379,24 +421,18 @@ loop_run (void *cls)
 {
   struct MainParams *mp = cls;
 
-  if (GNUNET_OK !=
-      GNUNET_TESTING_run (mp->cfg_filename,
-                          mp->commands,
-                          mp->timeout))
-  {
-    GNUNET_break (0);
-    mp->rv = EXIT_FAILURE;
-  }
+  GNUNET_TESTING_run (mp->commands,
+                      mp->timeout,
+                      &handle_result,
+                      mp);
 }
 
 
 int
-GNUNET_TESTING_main (const char *cfg_filename,
-                     struct GNUNET_TESTING_Command *commands,
+GNUNET_TESTING_main (struct GNUNET_TESTING_Command *commands,
                      struct GNUNET_TIME_Relative timeout)
 {
   struct MainParams mp = {
-    .cfg_filename = cfg_filename,
     .commands = commands,
     .timeout = timeout,
     .rv = EXIT_SUCCESS