PQ: implementing #7014: support testing database version is current

4 files changed, 201 insertions, 66 deletions

diff --git a/src/include/gnunet_pq_lib.h b/src/include/gnunet_pq_lib.h
index fe3fabbea..6a2227581 100644
--- a/src/include/gnunet_pq_lib.h
+++ b/src/include/gnunet_pq_lib.h
@@ -470,7 +470,8 @@ GNUNET_PQ_result_spec_absolute_time_nbo (const char *name,
  * @return array entry for the result specification to use
  */
 struct GNUNET_PQ_ResultSpec
-GNUNET_PQ_result_spec_uint16 (const char *name, uint16_t *u16);
+GNUNET_PQ_result_spec_uint16 (const char *name,
+                              uint16_t *u16);
 
 
 /**
@@ -481,7 +482,8 @@ GNUNET_PQ_result_spec_uint16 (const char *name, uint16_t *u16);
  * @return array entry for the result specification to use
  */
 struct GNUNET_PQ_ResultSpec
-GNUNET_PQ_result_spec_uint32 (const char *name, uint32_t *u32);
+GNUNET_PQ_result_spec_uint32 (const char *name,
+                              uint32_t *u32);
 
 
 /**
@@ -492,7 +494,8 @@ GNUNET_PQ_result_spec_uint32 (const char *name, uint32_t *u32);
  * @return array entry for the result specification to use
  */
 struct GNUNET_PQ_ResultSpec
-GNUNET_PQ_result_spec_uint64 (const char *name, uint64_t *u64);
+GNUNET_PQ_result_spec_uint64 (const char *name,
+                              uint64_t *u64);
 
 
 /* ************************* pq.c functions ************************ */
@@ -641,11 +644,11 @@ GNUNET_PQ_eval_prepared_multi_select (struct GNUNET_PQ_Context *db,
  *         codes to `enum GNUNET_DB_QueryStatus`.
  */
 enum GNUNET_DB_QueryStatus
-GNUNET_PQ_eval_prepared_singleton_select (struct GNUNET_PQ_Context *db,
-                                          const char *statement_name,
-                                          const struct
-                                          GNUNET_PQ_QueryParam *params,
-                                          struct GNUNET_PQ_ResultSpec *rs);
+GNUNET_PQ_eval_prepared_singleton_select (
+  struct GNUNET_PQ_Context *db,
+  const char *statement_name,
+  const struct GNUNET_PQ_QueryParam *params,
+  struct GNUNET_PQ_ResultSpec *rs);
 
 
 /* ******************** pq_prepare.c functions ************** */
@@ -772,7 +775,7 @@ GNUNET_PQ_make_try_execute (const char *sql);
  * @return #GNUNET_OK on success (modulo statements where errors can be ignored)
  *         #GNUNET_SYSERR on error
  */
-int
+enum GNUNET_GenericReturnValue
 GNUNET_PQ_exec_statements (struct GNUNET_PQ_Context *db,
                            const struct GNUNET_PQ_ExecuteStatement *es);
 
@@ -781,6 +784,29 @@ GNUNET_PQ_exec_statements (struct GNUNET_PQ_Context *db,
 
 
 /**
+ * Flags to control PQ operation.
+ */
+enum GNUNET_PQ_Options
+{
+  /**
+   * Traditional default. Do nothing special.
+   */
+  GNUNET_PQ_FLAG_NONE = 0,
+
+  /**
+   * Dropping database. Do not attempt to initialize
+   * versioning schema if not present.
+   */
+  GNUNET_PQ_FLAG_DROP = 1,
+
+  /**
+   * Check database version is current.  Fail to connect if it is not.
+   */
+  GNUNET_PQ_FLAG_CHECK_CURRENT = 2
+};
+
+
+/**
  * Create a connection to the Postgres database using @a config_str for the
  * configuration.  Initialize logging via GNUnet's log routines and disable
  * Postgres's logger.  Also ensures that the statements in @a load_path and @a
@@ -810,6 +836,37 @@ GNUNET_PQ_connect (const char *config_str,
 
 
 /**
+ * Create a connection to the Postgres database using @a config_str for the
+ * configuration.  Initialize logging via GNUnet's log routines and disable
+ * Postgres's logger.  Also ensures that the statements in @a load_path and @a
+ * es are executed whenever we (re)connect to the database, and that the
+ * prepared statements in @a ps are "ready".  If statements in @es fail that
+ * were created with #GNUNET_PQ_make_execute(), then the entire operation
+ * fails.
+ *
+ * In @a load_path, a list of "$XXXX.sql" files is expected where $XXXX
+ * must be a sequence of contiguous integer values starting at 0000.
+ * These files are then loaded in sequence using "psql $config_str" before
+ * running statements from @e es.  The directory is inspected again on
+ * reconnect.
+ *
+ * @param config_str configuration to use
+ * @param load_path path to directory with SQL transactions to run, can be NULL
+ * @param es #GNUNET_PQ_PREPARED_STATEMENT_END-terminated
+ *            array of statements to execute upon EACH connection, can be NULL
+ * @param ps array of prepared statements to prepare, can be NULL
+ * @param flags connection flags
+ * @return NULL on error
+ */
+struct GNUNET_PQ_Context *
+GNUNET_PQ_connect2 (const char *config_str,
+                    const char *load_path,
+                    const struct GNUNET_PQ_ExecuteStatement *es,
+                    const struct GNUNET_PQ_PreparedStatement *ps,
+                    enum GNUNET_PQ_Options flags);
+
+
+/**
  * Connect to a postgres database using the configuration
  * option "CONFIG" in @a section.  Also ensures that the
  * statements in @a es are executed whenever we (re)connect to the
@@ -835,6 +892,33 @@ GNUNET_PQ_connect_with_cfg (const struct GNUNET_CONFIGURATION_Handle *cfg,
 
 
 /**
+ * Connect to a postgres database using the configuration
+ * option "CONFIG" in @a section.  Also ensures that the
+ * statements in @a es are executed whenever we (re)connect to the
+ * database, and that the prepared statements in @a ps are "ready".
+ *
+ * The caller does not have to ensure that @a es and @a ps remain allocated
+ * and initialized in memory until #GNUNET_PQ_disconnect() is called, as a copy will be made.
+ *
+ * @param cfg configuration
+ * @param section configuration section to use to get Postgres configuration options
+ * @param load_path_suffix suffix to append to the SQL_DIR in the configuration
+ * @param es #GNUNET_PQ_PREPARED_STATEMENT_END-terminated
+ *            array of statements to execute upon EACH connection, can be NULL
+ * @param ps array of prepared statements to prepare, can be NULL
+ * @param flags connection flags
+ * @return the postgres handle, NULL on error
+ */
+struct GNUNET_PQ_Context *
+GNUNET_PQ_connect_with_cfg2 (const struct GNUNET_CONFIGURATION_Handle *cfg,
+                             const char *section,
+                             const char *load_path_suffix,
+                             const struct GNUNET_PQ_ExecuteStatement *es,
+                             const struct GNUNET_PQ_PreparedStatement *ps,
+                             enum GNUNET_PQ_Options flags);
+
+
+/**
  * Reinitialize the database @a db if the connection is down.
  *
  * @param db database connection to reinitialize
diff --git a/src/pq/pq.h b/src/pq/pq.h
index d10931d99..354d85a9f 100644
--- a/src/pq/pq.h
+++ b/src/pq/pq.h
@@ -73,6 +73,11 @@ struct GNUNET_PQ_Context
    * File descriptor wrapper for @e event_task.
    */
   struct GNUNET_NETWORK_Handle *rfd;
+
+  /**
+   * Flags controlling the connection.
+   */
+  enum GNUNET_PQ_Options flags;
 };
 
 
diff --git a/src/pq/pq_connect.c b/src/pq/pq_connect.c
index a2dce3fb0..a63d5f14e 100644
--- a/src/pq/pq_connect.c
+++ b/src/pq/pq_connect.c
@@ -1,6 +1,6 @@
 /*
    This file is part of GNUnet
-   Copyright (C) 2017, 2019, 2020 GNUnet e.V.
+   Copyright (C) 2017, 2019, 2020, 2021 GNUnet e.V.
 
    GNUnet is free software: you can redistribute it and/or modify it
    under the terms of the GNU Affero General Public License as published
@@ -70,6 +70,21 @@ GNUNET_PQ_connect (const char *config_str,
                    const struct GNUNET_PQ_ExecuteStatement *es,
                    const struct GNUNET_PQ_PreparedStatement *ps)
 {
+  return GNUNET_PQ_connect2 (config_str,
+                             load_path,
+                             es,
+                             ps,
+                             GNUNET_PQ_FLAG_NONE);
+}
+
+
+struct GNUNET_PQ_Context *
+GNUNET_PQ_connect2 (const char *config_str,
+                    const char *load_path,
+                    const struct GNUNET_PQ_ExecuteStatement *es,
+                    const struct GNUNET_PQ_PreparedStatement *ps,
+                    enum GNUNET_PQ_Options flags)
+{
   struct GNUNET_PQ_Context *db;
   unsigned int elen = 0;
   unsigned int plen = 0;
@@ -82,6 +97,7 @@ GNUNET_PQ_connect (const char *config_str,
       plen++;
 
   db = GNUNET_new (struct GNUNET_PQ_Context);
+  db->flags = flags;
   db->config_str = GNUNET_strdup (config_str);
   if (NULL != load_path)
     db->load_path = GNUNET_strdup (load_path);
@@ -200,68 +216,72 @@ GNUNET_PQ_run_sql (struct GNUNET_PQ_Context *db,
               load_path);
   for (unsigned int i = 1; i<10000; i++)
   {
+    char patch_name[slen];
+    char buf[slen];
     enum GNUNET_DB_QueryStatus qs;
-    {
-      char buf[slen];
-
-      /* First, check patch actually exists */
-      GNUNET_snprintf (buf,
-                       sizeof (buf),
-                       "%s%04u.sql",
-                       load_path,
-                       i);
-      if (GNUNET_YES !=
-          GNUNET_DISK_file_test (buf))
-        return GNUNET_OK;   /* We are done */
-    }
+
+    /* First, check patch actually exists */
+    GNUNET_snprintf (buf,
+                     sizeof (buf),
+                     "%s%04u.sql",
+                     load_path,
+                     i);
+    if (GNUNET_YES !=
+        GNUNET_DISK_file_test (buf))
+      return GNUNET_OK;     /* We are done */
 
     /* Second, check with DB versioning schema if this patch was already applied,
        if so, skip it. */
+    GNUNET_snprintf (patch_name,
+                     sizeof (patch_name),
+                     "%s%04u",
+                     load_path_suffix,
+                     i);
     {
-      char patch_name[slen];
-
-      GNUNET_snprintf (patch_name,
-                       sizeof (patch_name),
-                       "%s%04u",
-                       load_path_suffix,
-                       i);
+      char *applied_by;
+      struct GNUNET_PQ_QueryParam params[] = {
+        GNUNET_PQ_query_param_string (patch_name),
+        GNUNET_PQ_query_param_end
+      };
+      struct GNUNET_PQ_ResultSpec rs[] = {
+        GNUNET_PQ_result_spec_string ("applied_by",
+                                      &applied_by),
+        GNUNET_PQ_result_spec_end
+      };
+
+      qs = GNUNET_PQ_eval_prepared_singleton_select (db,
+                                                     "gnunet_pq_check_patch",
+                                                     params,
+                                                     rs);
+      if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT == qs)
+      {
+        GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                    "Database version %s already applied by %s, skipping\n",
+                    patch_name,
+                    applied_by);
+        GNUNET_PQ_cleanup_result (rs);
+      }
+      if (GNUNET_DB_STATUS_HARD_ERROR == qs)
       {
-        char *applied_by;
-        struct GNUNET_PQ_QueryParam params[] = {
-          GNUNET_PQ_query_param_string (patch_name),
-          GNUNET_PQ_query_param_end
-        };
-        struct GNUNET_PQ_ResultSpec rs[] = {
-          GNUNET_PQ_result_spec_string ("applied_by",
-                                        &applied_by),
-          GNUNET_PQ_result_spec_end
-        };
-
-        qs = GNUNET_PQ_eval_prepared_singleton_select (db,
-                                                       "gnunet_pq_check_patch",
-                                                       params,
-                                                       rs);
-        if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT == qs)
-        {
-          GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                      "Database version %s already applied by %s, skipping\n",
-                      patch_name,
-                      applied_by);
-          GNUNET_PQ_cleanup_result (rs);
-        }
-        if (GNUNET_DB_STATUS_HARD_ERROR == qs)
-        {
-          GNUNET_break (0);
-          return GNUNET_SYSERR;
-        }
+        GNUNET_break (0);
+        return GNUNET_SYSERR;
       }
     }
     if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT == qs)
       continue; /* patch already applied, skip it */
 
-    /* patch not yet applied, run it! */
+    if (0 != (GNUNET_PQ_FLAG_CHECK_CURRENT & db->flags))
     {
-      int ret;
+      /* We are only checking, found unapplied patch, bad! */
+      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                  "Database outdated, patch %s missing. Aborting!\n",
+                  patch_name);
+      return GNUNET_SYSERR;
+    }
+    else
+    {
+      /* patch not yet applied, run it! */
+      enum GNUNET_GenericReturnValue ret;
 
       ret = apply_patch (db,
                          load_path,
@@ -334,9 +354,17 @@ GNUNET_PQ_reconnect (struct GNUNET_PQ_Context *db)
                      NULL);
     if (PGRES_COMMAND_OK != PQresultStatus (res))
     {
-      int ret;
+      enum GNUNET_GenericReturnValue ret;
 
       PQclear (res);
+      if (0 != (db->flags & GNUNET_PQ_FLAG_DROP))
+      {
+        GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                    "Failed to prepare statement to check patch level. Likely versioning schema does not exist yet. Not attempting drop!\n");
+        PQfinish (db->conn);
+        db->conn = NULL;
+        return;
+      }
       GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                   "Failed to prepare statement to check patch level. Likely versioning schema does not exist yet, loading patch level 0000!\n");
       ret = apply_patch (db,
@@ -424,6 +452,23 @@ GNUNET_PQ_connect_with_cfg (const struct GNUNET_CONFIGURATION_Handle *cfg,
                             const struct GNUNET_PQ_ExecuteStatement *es,
                             const struct GNUNET_PQ_PreparedStatement *ps)
 {
+  return GNUNET_PQ_connect_with_cfg2 (cfg,
+                                      section,
+                                      load_path_suffix,
+                                      es,
+                                      ps,
+                                      GNUNET_PQ_FLAG_NONE);
+}
+
+
+struct GNUNET_PQ_Context *
+GNUNET_PQ_connect_with_cfg2 (const struct GNUNET_CONFIGURATION_Handle *cfg,
+                             const char *section,
+                             const char *load_path_suffix,
+                             const struct GNUNET_PQ_ExecuteStatement *es,
+                             const struct GNUNET_PQ_PreparedStatement *ps,
+                             enum GNUNET_PQ_Options flags)
+{
   struct GNUNET_PQ_Context *db;
   char *conninfo;
   char *load_path;
@@ -447,10 +492,11 @@ GNUNET_PQ_connect_with_cfg (const struct GNUNET_CONFIGURATION_Handle *cfg,
                      "%s%s",
                      sp,
                      load_path_suffix);
-  db = GNUNET_PQ_connect (conninfo == NULL ? "" : conninfo,
-                          load_path,
-                          es,
-                          ps);
+  db = GNUNET_PQ_connect2 (conninfo == NULL ? "" : conninfo,
+                           load_path,
+                           es,
+                           ps,
+                           flags);
   GNUNET_free (load_path);
   GNUNET_free (sp);
   GNUNET_free (conninfo);
diff --git a/src/pq/pq_exec.c b/src/pq/pq_exec.c
index fd4feae53..464fff4b4 100644
--- a/src/pq/pq_exec.c
+++ b/src/pq/pq_exec.c
@@ -72,7 +72,7 @@ GNUNET_PQ_make_try_execute (const char *sql)
  * @return #GNUNET_OK on success (modulo statements where errors can be ignored)
  *         #GNUNET_SYSERR on error
  */
-int
+enum GNUNET_GenericReturnValue
 GNUNET_PQ_exec_statements (struct GNUNET_PQ_Context *db,
                            const struct GNUNET_PQ_ExecuteStatement *es)
 {