From 1aa0d5f3421d8598f12005ea1138c9eb24ddfd2c Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Thu, 23 Sep 2021 22:52:44 +0200 Subject: PQ: implementing #7014: support testing database version is current --- src/include/gnunet_pq_lib.h | 102 +++++++++++++++++++++++++--- src/pq/pq.h | 5 ++ src/pq/pq_connect.c | 158 ++++++++++++++++++++++++++++---------------- src/pq/pq_exec.c | 2 +- 4 files changed, 201 insertions(+), 66 deletions(-) (limited to 'src') diff --git a/src/include/gnunet_pq_lib.h b/src/include/gnunet_pq_lib.h index fe3fabbea..6a2227581 100644 --- a/src/include/gnunet_pq_lib.h +++ b/src/include/gnunet_pq_lib.h @@ -470,7 +470,8 @@ GNUNET_PQ_result_spec_absolute_time_nbo (const char *name, * @return array entry for the result specification to use */ struct GNUNET_PQ_ResultSpec -GNUNET_PQ_result_spec_uint16 (const char *name, uint16_t *u16); +GNUNET_PQ_result_spec_uint16 (const char *name, + uint16_t *u16); /** @@ -481,7 +482,8 @@ GNUNET_PQ_result_spec_uint16 (const char *name, uint16_t *u16); * @return array entry for the result specification to use */ struct GNUNET_PQ_ResultSpec -GNUNET_PQ_result_spec_uint32 (const char *name, uint32_t *u32); +GNUNET_PQ_result_spec_uint32 (const char *name, + uint32_t *u32); /** @@ -492,7 +494,8 @@ GNUNET_PQ_result_spec_uint32 (const char *name, uint32_t *u32); * @return array entry for the result specification to use */ struct GNUNET_PQ_ResultSpec -GNUNET_PQ_result_spec_uint64 (const char *name, uint64_t *u64); +GNUNET_PQ_result_spec_uint64 (const char *name, + uint64_t *u64); /* ************************* pq.c functions ************************ */ @@ -641,11 +644,11 @@ GNUNET_PQ_eval_prepared_multi_select (struct GNUNET_PQ_Context *db, * codes to `enum GNUNET_DB_QueryStatus`. */ enum GNUNET_DB_QueryStatus -GNUNET_PQ_eval_prepared_singleton_select (struct GNUNET_PQ_Context *db, - const char *statement_name, - const struct - GNUNET_PQ_QueryParam *params, - struct GNUNET_PQ_ResultSpec *rs); +GNUNET_PQ_eval_prepared_singleton_select ( + struct GNUNET_PQ_Context *db, + const char *statement_name, + const struct GNUNET_PQ_QueryParam *params, + struct GNUNET_PQ_ResultSpec *rs); /* ******************** pq_prepare.c functions ************** */ @@ -772,7 +775,7 @@ GNUNET_PQ_make_try_execute (const char *sql); * @return #GNUNET_OK on success (modulo statements where errors can be ignored) * #GNUNET_SYSERR on error */ -int +enum GNUNET_GenericReturnValue GNUNET_PQ_exec_statements (struct GNUNET_PQ_Context *db, const struct GNUNET_PQ_ExecuteStatement *es); @@ -780,6 +783,29 @@ GNUNET_PQ_exec_statements (struct GNUNET_PQ_Context *db, /* ******************** pq_connect.c functions ************** */ +/** + * Flags to control PQ operation. + */ +enum GNUNET_PQ_Options +{ + /** + * Traditional default. Do nothing special. + */ + GNUNET_PQ_FLAG_NONE = 0, + + /** + * Dropping database. Do not attempt to initialize + * versioning schema if not present. + */ + GNUNET_PQ_FLAG_DROP = 1, + + /** + * Check database version is current. Fail to connect if it is not. + */ + GNUNET_PQ_FLAG_CHECK_CURRENT = 2 +}; + + /** * Create a connection to the Postgres database using @a config_str for the * configuration. Initialize logging via GNUnet's log routines and disable @@ -809,6 +835,37 @@ GNUNET_PQ_connect (const char *config_str, const struct GNUNET_PQ_PreparedStatement *ps); +/** + * Create a connection to the Postgres database using @a config_str for the + * configuration. Initialize logging via GNUnet's log routines and disable + * Postgres's logger. Also ensures that the statements in @a load_path and @a + * es are executed whenever we (re)connect to the database, and that the + * prepared statements in @a ps are "ready". If statements in @es fail that + * were created with #GNUNET_PQ_make_execute(), then the entire operation + * fails. + * + * In @a load_path, a list of "$XXXX.sql" files is expected where $XXXX + * must be a sequence of contiguous integer values starting at 0000. + * These files are then loaded in sequence using "psql $config_str" before + * running statements from @e es. The directory is inspected again on + * reconnect. + * + * @param config_str configuration to use + * @param load_path path to directory with SQL transactions to run, can be NULL + * @param es #GNUNET_PQ_PREPARED_STATEMENT_END-terminated + * array of statements to execute upon EACH connection, can be NULL + * @param ps array of prepared statements to prepare, can be NULL + * @param flags connection flags + * @return NULL on error + */ +struct GNUNET_PQ_Context * +GNUNET_PQ_connect2 (const char *config_str, + const char *load_path, + const struct GNUNET_PQ_ExecuteStatement *es, + const struct GNUNET_PQ_PreparedStatement *ps, + enum GNUNET_PQ_Options flags); + + /** * Connect to a postgres database using the configuration * option "CONFIG" in @a section. Also ensures that the @@ -834,6 +891,33 @@ GNUNET_PQ_connect_with_cfg (const struct GNUNET_CONFIGURATION_Handle *cfg, const struct GNUNET_PQ_PreparedStatement *ps); +/** + * Connect to a postgres database using the configuration + * option "CONFIG" in @a section. Also ensures that the + * statements in @a es are executed whenever we (re)connect to the + * database, and that the prepared statements in @a ps are "ready". + * + * The caller does not have to ensure that @a es and @a ps remain allocated + * and initialized in memory until #GNUNET_PQ_disconnect() is called, as a copy will be made. + * + * @param cfg configuration + * @param section configuration section to use to get Postgres configuration options + * @param load_path_suffix suffix to append to the SQL_DIR in the configuration + * @param es #GNUNET_PQ_PREPARED_STATEMENT_END-terminated + * array of statements to execute upon EACH connection, can be NULL + * @param ps array of prepared statements to prepare, can be NULL + * @param flags connection flags + * @return the postgres handle, NULL on error + */ +struct GNUNET_PQ_Context * +GNUNET_PQ_connect_with_cfg2 (const struct GNUNET_CONFIGURATION_Handle *cfg, + const char *section, + const char *load_path_suffix, + const struct GNUNET_PQ_ExecuteStatement *es, + const struct GNUNET_PQ_PreparedStatement *ps, + enum GNUNET_PQ_Options flags); + + /** * Reinitialize the database @a db if the connection is down. * diff --git a/src/pq/pq.h b/src/pq/pq.h index d10931d99..354d85a9f 100644 --- a/src/pq/pq.h +++ b/src/pq/pq.h @@ -73,6 +73,11 @@ struct GNUNET_PQ_Context * File descriptor wrapper for @e event_task. */ struct GNUNET_NETWORK_Handle *rfd; + + /** + * Flags controlling the connection. + */ + enum GNUNET_PQ_Options flags; }; diff --git a/src/pq/pq_connect.c b/src/pq/pq_connect.c index a2dce3fb0..a63d5f14e 100644 --- a/src/pq/pq_connect.c +++ b/src/pq/pq_connect.c @@ -1,6 +1,6 @@ /* This file is part of GNUnet - Copyright (C) 2017, 2019, 2020 GNUnet e.V. + Copyright (C) 2017, 2019, 2020, 2021 GNUnet e.V. GNUnet is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published @@ -69,6 +69,21 @@ GNUNET_PQ_connect (const char *config_str, const char *load_path, const struct GNUNET_PQ_ExecuteStatement *es, const struct GNUNET_PQ_PreparedStatement *ps) +{ + return GNUNET_PQ_connect2 (config_str, + load_path, + es, + ps, + GNUNET_PQ_FLAG_NONE); +} + + +struct GNUNET_PQ_Context * +GNUNET_PQ_connect2 (const char *config_str, + const char *load_path, + const struct GNUNET_PQ_ExecuteStatement *es, + const struct GNUNET_PQ_PreparedStatement *ps, + enum GNUNET_PQ_Options flags) { struct GNUNET_PQ_Context *db; unsigned int elen = 0; @@ -82,6 +97,7 @@ GNUNET_PQ_connect (const char *config_str, plen++; db = GNUNET_new (struct GNUNET_PQ_Context); + db->flags = flags; db->config_str = GNUNET_strdup (config_str); if (NULL != load_path) db->load_path = GNUNET_strdup (load_path); @@ -200,68 +216,72 @@ GNUNET_PQ_run_sql (struct GNUNET_PQ_Context *db, load_path); for (unsigned int i = 1; i<10000; i++) { + char patch_name[slen]; + char buf[slen]; enum GNUNET_DB_QueryStatus qs; - { - char buf[slen]; - - /* First, check patch actually exists */ - GNUNET_snprintf (buf, - sizeof (buf), - "%s%04u.sql", - load_path, - i); - if (GNUNET_YES != - GNUNET_DISK_file_test (buf)) - return GNUNET_OK; /* We are done */ - } + + /* First, check patch actually exists */ + GNUNET_snprintf (buf, + sizeof (buf), + "%s%04u.sql", + load_path, + i); + if (GNUNET_YES != + GNUNET_DISK_file_test (buf)) + return GNUNET_OK; /* We are done */ /* Second, check with DB versioning schema if this patch was already applied, if so, skip it. */ + GNUNET_snprintf (patch_name, + sizeof (patch_name), + "%s%04u", + load_path_suffix, + i); { - char patch_name[slen]; - - GNUNET_snprintf (patch_name, - sizeof (patch_name), - "%s%04u", - load_path_suffix, - i); + char *applied_by; + struct GNUNET_PQ_QueryParam params[] = { + GNUNET_PQ_query_param_string (patch_name), + GNUNET_PQ_query_param_end + }; + struct GNUNET_PQ_ResultSpec rs[] = { + GNUNET_PQ_result_spec_string ("applied_by", + &applied_by), + GNUNET_PQ_result_spec_end + }; + + qs = GNUNET_PQ_eval_prepared_singleton_select (db, + "gnunet_pq_check_patch", + params, + rs); + if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT == qs) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Database version %s already applied by %s, skipping\n", + patch_name, + applied_by); + GNUNET_PQ_cleanup_result (rs); + } + if (GNUNET_DB_STATUS_HARD_ERROR == qs) { - char *applied_by; - struct GNUNET_PQ_QueryParam params[] = { - GNUNET_PQ_query_param_string (patch_name), - GNUNET_PQ_query_param_end - }; - struct GNUNET_PQ_ResultSpec rs[] = { - GNUNET_PQ_result_spec_string ("applied_by", - &applied_by), - GNUNET_PQ_result_spec_end - }; - - qs = GNUNET_PQ_eval_prepared_singleton_select (db, - "gnunet_pq_check_patch", - params, - rs); - if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT == qs) - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Database version %s already applied by %s, skipping\n", - patch_name, - applied_by); - GNUNET_PQ_cleanup_result (rs); - } - if (GNUNET_DB_STATUS_HARD_ERROR == qs) - { - GNUNET_break (0); - return GNUNET_SYSERR; - } + GNUNET_break (0); + return GNUNET_SYSERR; } } if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT == qs) continue; /* patch already applied, skip it */ - /* patch not yet applied, run it! */ + if (0 != (GNUNET_PQ_FLAG_CHECK_CURRENT & db->flags)) { - int ret; + /* We are only checking, found unapplied patch, bad! */ + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Database outdated, patch %s missing. Aborting!\n", + patch_name); + return GNUNET_SYSERR; + } + else + { + /* patch not yet applied, run it! */ + enum GNUNET_GenericReturnValue ret; ret = apply_patch (db, load_path, @@ -334,9 +354,17 @@ GNUNET_PQ_reconnect (struct GNUNET_PQ_Context *db) NULL); if (PGRES_COMMAND_OK != PQresultStatus (res)) { - int ret; + enum GNUNET_GenericReturnValue ret; PQclear (res); + if (0 != (db->flags & GNUNET_PQ_FLAG_DROP)) + { + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Failed to prepare statement to check patch level. Likely versioning schema does not exist yet. Not attempting drop!\n"); + PQfinish (db->conn); + db->conn = NULL; + return; + } GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Failed to prepare statement to check patch level. Likely versioning schema does not exist yet, loading patch level 0000!\n"); ret = apply_patch (db, @@ -423,6 +451,23 @@ GNUNET_PQ_connect_with_cfg (const struct GNUNET_CONFIGURATION_Handle *cfg, const char *load_path_suffix, const struct GNUNET_PQ_ExecuteStatement *es, const struct GNUNET_PQ_PreparedStatement *ps) +{ + return GNUNET_PQ_connect_with_cfg2 (cfg, + section, + load_path_suffix, + es, + ps, + GNUNET_PQ_FLAG_NONE); +} + + +struct GNUNET_PQ_Context * +GNUNET_PQ_connect_with_cfg2 (const struct GNUNET_CONFIGURATION_Handle *cfg, + const char *section, + const char *load_path_suffix, + const struct GNUNET_PQ_ExecuteStatement *es, + const struct GNUNET_PQ_PreparedStatement *ps, + enum GNUNET_PQ_Options flags) { struct GNUNET_PQ_Context *db; char *conninfo; @@ -447,10 +492,11 @@ GNUNET_PQ_connect_with_cfg (const struct GNUNET_CONFIGURATION_Handle *cfg, "%s%s", sp, load_path_suffix); - db = GNUNET_PQ_connect (conninfo == NULL ? "" : conninfo, - load_path, - es, - ps); + db = GNUNET_PQ_connect2 (conninfo == NULL ? "" : conninfo, + load_path, + es, + ps, + flags); GNUNET_free (load_path); GNUNET_free (sp); GNUNET_free (conninfo); diff --git a/src/pq/pq_exec.c b/src/pq/pq_exec.c index fd4feae53..464fff4b4 100644 --- a/src/pq/pq_exec.c +++ b/src/pq/pq_exec.c @@ -72,7 +72,7 @@ GNUNET_PQ_make_try_execute (const char *sql) * @return #GNUNET_OK on success (modulo statements where errors can be ignored) * #GNUNET_SYSERR on error */ -int +enum GNUNET_GenericReturnValue GNUNET_PQ_exec_statements (struct GNUNET_PQ_Context *db, const struct GNUNET_PQ_ExecuteStatement *es) { -- cgit v1.2.3