Merge remote-tracking branch 'origin/master' into credentials

6 files changed, 338 insertions, 182 deletions

diff --git a/src/include/Makefile.am b/src/include/Makefile.am
index b745da125..e5abec416 100644
--- a/src/include/Makefile.am
+++ b/src/include/Makefile.am
@@ -48,6 +48,7 @@ gnunetinclude_HEADERS = \
   gnunet_datacache_plugin.h \
   gnunet_datastore_service.h \
   gnunet_datastore_plugin.h \
+  gnunet_db_lib.h \
   gnunet_dht_service.h \
   gnunet_disk_lib.h \
   gnunet_dnsparser_lib.h \
@@ -90,7 +91,6 @@ gnunetinclude_HEADERS = \
   gnunet_peerstore_service.h \
   gnunet_plugin_lib.h \
   gnunet_pq_lib.h \
-  gnunet_postgres_lib.h \
   gnunet_psycstore_plugin.h \
   gnunet_psycstore_service.h \
   gnunet_psyc_service.h \
diff --git a/src/include/gnunet_db_lib.h b/src/include/gnunet_db_lib.h
new file mode 100644
index 000000000..9356f66cb
--- /dev/null
+++ b/src/include/gnunet_db_lib.h
@@ -0,0 +1,61 @@
+/*
+  This file is part of GNUnet
+  Copyright (C) 2017 GNUnet e.V.
+
+  GNUnet is free software; you can redistribute it and/or modify it under the
+  terms of the GNU General Public License as published by the Free Software
+  Foundation; either version 3, or (at your option) any later version.
+
+  GNUnet is distributed in the hope that it will be useful, but WITHOUT ANY
+  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+  A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License along with
+  GNUnet; see the file COPYING.  If not, If not, see <http://www.gnu.org/licenses/>
+*/
+/**
+ * @file include/gnunet_db_lib.h
+ * @brief shared defintions for transactional databases
+ * @author Christian Grothoff
+ */
+#ifndef GNUNET_DB_LIB_H
+#define GNUNET_DB_LIB_H
+
+
+/**
+ * Status code returned from functions running database commands.
+ * Can be combined with a function that returns the number
+ * of results, so all non-negative values indicate success.
+ */
+enum GNUNET_DB_QueryStatus
+{
+  /**
+   * A hard error occurred, retrying will not help.
+   */
+  GNUNET_DB_STATUS_HARD_ERROR = -2,
+
+  /**
+   * A soft error occurred, retrying the transaction may succeed.
+   * Includes DEADLOCKS and SERIALIZATION errors.
+   */
+  GNUNET_DB_STATUS_SOFT_ERROR = -1,
+
+  /**
+   * The transaction succeeded, but yielded zero results.
+   * May include the case where an INSERT failed with UNIQUE
+   * violation (i.e. row already exists) or where DELETE
+   * failed to remove anything (i.e. nothing matched).
+   */
+  GNUNET_DB_STATUS_SUCCESS_NO_RESULTS = 0,
+
+  /**
+   * The transaction succeeded, and yielded one result.
+   */
+  GNUNET_DB_STATUS_SUCCESS_ONE_RESULT = 1
+
+  /* Larger values may be returned for SELECT statements
+     that returned more than one result. */
+
+};
+
+#endif
diff --git a/src/include/gnunet_postgres_lib.h b/src/include/gnunet_postgres_lib.h
deleted file mode 100644
index 66fc2a5df..000000000
--- a/src/include/gnunet_postgres_lib.h
+++ /dev/null
@@ -1,178 +0,0 @@
-/*
-     This file is part of GNUnet
-     Copyright (C) 2012 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-/**
- * @author Christian Grothoff
- *
- * @file
- * Helper library to access a PostgreSQL database
- *
- * @defgroup postgres  PostgreSQL library
- * Helper library to access a PostgreSQL database.
- * @{
- */
-#ifndef GNUNET_POSTGRES_LIB_H
-#define GNUNET_POSTGRES_LIB_H
-
-#include "gnunet_util_lib.h"
-#include <libpq-fe.h>
-
-#ifdef __cplusplus
-extern "C"
-{
-#if 0                           /* keep Emacsens' auto-indent happy */
-}
-#endif
-#endif
-
-
-/**
- * Check if the result obtained from Postgres has
- * the desired status code.  If not, log an error, clear the
- * result and return #GNUNET_SYSERR.
- *
- * @param dbh database handle
- * @param ret return value from database operation to check
- * @param expected_status desired status
- * @param command description of the command that was run
- * @param args arguments given to the command
- * @param filename name of the source file where the command was run
- * @param line line number in the source file
- * @return #GNUNET_OK if the result is acceptable
- */
-int
-GNUNET_POSTGRES_check_result_ (PGconn *dbh,
-                               PGresult *ret,
-                               int expected_status,
-                               const char *command,
-                               const char *args,
-                               const char *filename,
-                               int line);
-
-
-/**
- * Check if the result obtained from Postgres has
- * the desired status code.  If not, log an error, clear the
- * result and return #GNUNET_SYSERR.
- *
- * @param dbh database handle
- * @param ret return value from database operation to check
- * @param expected_status desired status
- * @param command description of the command that was run
- * @param args arguments given to the command
- * @return #GNUNET_OK if the result is acceptable
- */
-#define GNUNET_POSTGRES_check_result(dbh,ret,expected_status,command,args) GNUNET_POSTGRES_check_result_(dbh,ret,expected_status,command,args,__FILE__,__LINE__)
-
-
-/**
- * Run simple SQL statement (without results).
- *
- * @param dbh database handle
- * @param sql statement to run
- * @param filename filename for error reporting
- * @param line code line for error reporting
- * @return #GNUNET_OK on success
- */
-int
-GNUNET_POSTGRES_exec_ (PGconn *dbh,
-                       const char *sql,
-                       const char *filename,
-                       int line);
-
-
-/**
- * Run simple SQL statement (without results).
- *
- * @param dbh database handle
- * @param sql statement to run
- * @return #GNUNET_OK on success
- */
-#define GNUNET_POSTGRES_exec(dbh,sql) GNUNET_POSTGRES_exec_(dbh,sql,__FILE__,__LINE__)
-
-
-/**
- * Prepare SQL statement.
- *
- * @param dbh database handle
- * @param name name for the prepared SQL statement
- * @param sql SQL code to prepare
- * @param nparams number of parameters in sql
- * @param filename filename for error reporting
- * @param line code line for error reporting
- * @return #GNUNET_OK on success
- */
-int
-GNUNET_POSTGRES_prepare_ (PGconn *dbh,
-                          const char *name,
-                          const char *sql,
-                          int nparams,
-                          const char *filename,
-                          int line);
-
-
-/**
- * Prepare SQL statement.
- *
- * @param dbh database handle
- * @param name name for the prepared SQL statement
- * @param sql SQL code to prepare
- * @param nparams number of parameters in sql
- * @return #GNUNET_OK on success
- */
-#define GNUNET_POSTGRES_prepare(dbh,name,sql,nparams) GNUNET_POSTGRES_prepare_(dbh,name,sql,nparams,__FILE__,__LINE__)
-
-
-/**
- * Connect to a postgres database
- *
- * @param cfg configuration
- * @param section configuration section to use to get Postgres configuration options
- * @return the postgres handle
- */
-PGconn *
-GNUNET_POSTGRES_connect (const struct GNUNET_CONFIGURATION_Handle *cfg,
-                         const char *section);
-
-
-/**
- * Delete the row identified by the given rowid (qid
- * in postgres).
- *
- * @param dbh database handle
- * @param stmt name of the prepared statement
- * @param rowid which row to delete
- * @return #GNUNET_OK on success
- */
-int
-GNUNET_POSTGRES_delete_by_rowid (PGconn *dbh,
-                                 const char *stmt,
-                                 uint32_t rowid);
-
-
-#if 0                           /* keep Emacsens' auto-indent happy */
-{
-#endif
-#ifdef __cplusplus
-}
-#endif
-
-#endif
-
-/** @} */  /* end of group */
diff --git a/src/include/gnunet_pq_lib.h b/src/include/gnunet_pq_lib.h
index 756370b74..ed295b500 100644
--- a/src/include/gnunet_pq_lib.h
+++ b/src/include/gnunet_pq_lib.h
@@ -1,6 +1,6 @@
 /*
   This file is part of GNUnet
-  Copyright (C) 2016 GNUnet e.V.
+  Copyright (C) 2016, 2017 GNUnet e.V.
 
   GNUnet is free software; you can redistribute it and/or modify it under the
   terms of the GNU General Public License as published by the Free Software
@@ -23,6 +23,9 @@
 
 #include <libpq-fe.h>
 #include "gnunet_util_lib.h"
+#include "gnunet_db_lib.h"
+
+/* ************************* pq_query_helper.c functions ************************ */
 
 
 /**
@@ -188,6 +191,9 @@ struct GNUNET_PQ_QueryParam
 GNUNET_PQ_query_param_uint64 (const uint64_t *x);
 
 
+/* ************************* pq_result_helper.c functions ************************ */
+
+
 /**
  * Extract data from a Postgres database @a result at row @a row.
  *
@@ -412,6 +418,8 @@ GNUNET_PQ_result_spec_uint64 (const char *name,
                               uint64_t *u64);
 
 
+/* ************************* pq.c functions ************************ */
+
 /**
  * Execute a prepared statement.
  *
@@ -419,6 +427,7 @@ GNUNET_PQ_result_spec_uint64 (const char *name,
  * @param name name of the prepared statement
  * @param params parameters to the statement
  * @return postgres result
+ * @deprecated (should become an internal API)
  */
 PGresult *
 GNUNET_PQ_exec_prepared (PGconn *db_conn,
@@ -435,6 +444,7 @@ GNUNET_PQ_exec_prepared (PGconn *db_conn,
  * @return
  *   #GNUNET_YES if all results could be extracted
  *   #GNUNET_SYSERR if a result was invalid (non-existing field)
+ * @deprecated (should become an internal API)
  */
 int
 GNUNET_PQ_extract_result (PGresult *result,
@@ -452,6 +462,262 @@ void
 GNUNET_PQ_cleanup_result (struct GNUNET_PQ_ResultSpec *rs);
 
 
+/* ******************** pq_eval.c functions ************** */
+
+
+/**
+ * Check the @a result's error code to see what happened.
+ * Also logs errors.
+ *
+ * @param connection connection to execute the statement in
+ * @param statement_name name of the statement that created @a result
+ * @param result result to check
+ * @return status code from the result, mapping PQ status
+ *         codes to `enum GNUNET_DB_QueryStatus`.  Never
+ *         returns positive values as this function does
+ *         not look at the result set.
+ * @deprecated (low level, let's see if we can do with just the high-level functions)
+ */
+enum GNUNET_DB_QueryStatus
+GNUNET_PQ_eval_result (PGconn *connection,
+                       const char *statement_name,
+                       PGresult *result);
+
+
+/**
+ * Execute a named prepared @a statement that is NOT a SELECT
+ * statement in @a connnection using the given @a params.  Returns the
+ * resulting session state.
+ *
+ * @param connection connection to execute the statement in
+ * @param statement_name name of the statement
+ * @param params parameters to give to the statement (#GNUNET_PQ_query_param_end-terminated)
+ * @return status code from the result, mapping PQ status
+ *         codes to `enum GNUNET_DB_QueryStatus`.   If the
+ *         statement was a DELETE or UPDATE statement, the
+ *         number of affected rows is returned; if the
+ *         statment was an INSERT statement, and no row
+ *         was added due to a UNIQUE violation, we return
+ *         zero; if INSERT was successful, we return one.
+ */
+enum GNUNET_DB_QueryStatus
+GNUNET_PQ_eval_prepared_non_select (PGconn *connection,
+                                    const char *statement_name,
+                                    const struct GNUNET_PQ_QueryParam *params);
+
+
+/**
+ * Function to be called with the results of a SELECT statement
+ * that has returned @a num_results results.
+ *
+ * @param cls closure
+ * @param result the postgres result
+ * @param num_result the number of results in @a result
+ */
+typedef void
+(*GNUNET_PQ_PostgresResultHandler)(void *cls,
+                                   PGresult *result,
+                                   unsigned int num_results);
+
+
+/**
+ * Execute a named prepared @a statement that is a SELECT statement
+ * which may return multiple results in @a connection using the given
+ * @a params.  Call @a rh with the results.  Returns the query
+ * status including the number of results given to @a rh (possibly zero).
+ * @a rh will not have been called if the return value is negative.
+ *
+ * @param connection connection to execute the statement in
+ * @param statement_name name of the statement
+ * @param params parameters to give to the statement (#GNUNET_PQ_query_param_end-terminated)
+ * @param rh function to call with the result set, NULL to ignore
+ * @param rh_cls closure to pass to @a rh
+ * @return status code from the result, mapping PQ status
+ *         codes to `enum GNUNET_DB_QueryStatus`.
+ */
+enum GNUNET_DB_QueryStatus
+GNUNET_PQ_eval_prepared_multi_select (PGconn *connection,
+                                      const char *statement_name,
+                                      const struct GNUNET_PQ_QueryParam *params,
+                                      GNUNET_PQ_PostgresResultHandler rh,
+                                      void *rh_cls);
+
+
+/**
+ * Execute a named prepared @a statement that is a SELECT statement
+ * which must return a single result in @a connection using the given
+ * @a params.  Stores the result (if any) in @a rs, which the caller
+ * must then clean up using #GNUNET_PQ_cleanup_result() if the return
+ * value was #GNUNET_DB_STATUS_SUCCESS_ONE_RESULT.  Returns the
+ * resulting session status.
+ *
+ * @param connection connection to execute the statement in
+ * @param statement_name name of the statement
+ * @param params parameters to give to the statement (#GNUNET_PQ_query_param_end-terminated)
+ * @param[in,out] rs result specification to use for storing the result of the query
+ * @return status code from the result, mapping PQ status
+ *         codes to `enum GNUNET_DB_QueryStatus`.
+ */
+enum GNUNET_DB_QueryStatus
+GNUNET_PQ_eval_prepared_singleton_select (PGconn *connection,
+                                          const char *statement_name,
+                                          const struct GNUNET_PQ_QueryParam *params,
+                                          struct GNUNET_PQ_ResultSpec *rs);
+
+
+/* ******************** pq_prepare.c functions ************** */
+
+
+/**
+ * Information needed to prepare a list of SQL statements using
+ * #GNUNET_PQ_prepare_statements().
+ */
+struct GNUNET_PQ_PreparedStatement {
+
+  /**
+   * Name of the statement.
+   */
+  const char *name;
+
+  /**
+   * Actual SQL statement.
+   */
+  const char *sql;
+
+  /**
+   * Number of arguments included in @e sql.
+   */
+  unsigned int num_arguments;
+
+};
+
+
+/**
+ * Terminator for prepared statement list.
+ */
+#define GNUNET_PQ_PREPARED_STATEMENT_END { NULL, NULL, 0 }
+
+
+/**
+ * Create a `struct GNUNET_PQ_PreparedStatement`.
+ *
+ * @param name name of the statement
+ * @param sql actual SQL statement
+ * @param num_args number of arguments in the statement
+ * @return initialized struct
+ */
+struct GNUNET_PQ_PreparedStatement
+GNUNET_PQ_make_prepare (const char *name,
+                        const char *sql,
+                        unsigned int num_args);
+
+
+/**
+ * Request creation of prepared statements @a ps from Postgres.
+ *
+ * @param connection connection to prepare the statements for
+ * @param ps #GNUNET_PQ_PREPARED_STATEMENT_END-terminated array of prepared
+ *            statements.
+ * @return #GNUNET_OK on success,
+ *         #GNUNET_SYSERR on error
+ */
+int
+GNUNET_PQ_prepare_statements (PGconn *connection,
+                              const struct GNUNET_PQ_PreparedStatement *ps);
+
+
+/* ******************** pq_exec.c functions ************** */
+
+
+/**
+ * Information needed to run a list of SQL statements using
+ * #GNUNET_PQ_exec_statements().
+ */
+struct GNUNET_PQ_ExecuteStatement {
+
+  /**
+   * Actual SQL statement.
+   */
+  const char *sql;
+
+  /**
+   * Should we ignore errors?
+   */
+  int ignore_errors;
+
+};
+
+
+/**
+ * Terminator for executable statement list.
+ */
+#define GNUNET_PQ_EXECUTE_STATEMENT_END { NULL, GNUNET_SYSERR }
+
+
+/**
+ * Create a `struct GNUNET_PQ_ExecuteStatement` where errors are fatal.
+ *
+ * @param sql actual SQL statement
+ * @return initialized struct
+ */
+struct GNUNET_PQ_ExecuteStatement
+GNUNET_PQ_make_execute (const char *sql);
+
+
+/**
+ * Create a `struct GNUNET_PQ_ExecuteStatement` where errors should
+ * be tolerated.
+ *
+ * @param sql actual SQL statement
+ * @return initialized struct
+ */
+struct GNUNET_PQ_ExecuteStatement
+GNUNET_PQ_make_try_execute (const char *sql);
+
+
+/**
+ * Request execution of an array of statements @a es from Postgres.
+ *
+ * @param connection connection to execute the statements over
+ * @param es #GNUNET_PQ_PREPARED_STATEMENT_END-terminated array of prepared
+ *            statements.
+ * @return #GNUNET_OK on success (modulo statements where errors can be ignored)
+ *         #GNUNET_SYSERR on error
+ */
+int
+GNUNET_PQ_exec_statements (PGconn *connection,
+                           const struct GNUNET_PQ_ExecuteStatement *es);
+
+
+/* ******************** pq_connect.c functions ************** */
+
+
+/**
+ * Create a connection to the Postgres database using @a config_str
+ * for the configuration.  Initialize logging via GNUnet's log
+ * routines and disable Postgres's logger.
+ *
+ * @param config_str configuration to use
+ * @return NULL on error
+ */
+PGconn *
+GNUNET_PQ_connect (const char *config_str);
+
+
+/**
+ * Connect to a postgres database using the configuration
+ * option "CONFIG" in @a section.
+ *
+ * @param cfg configuration
+ * @param section configuration section to use to get Postgres configuration options
+ * @return the postgres handle, NULL on error
+ */
+PGconn *
+GNUNET_PQ_connect_with_cfg (const struct GNUNET_CONFIGURATION_Handle *cfg,
+                            const char *section);
+
+
+
 #endif  /* GNUNET_PQ_LIB_H_ */
 
 /* end of include/gnunet_pq_lib.h */
diff --git a/src/include/gnunet_protocols.h b/src/include/gnunet_protocols.h
index 1d8049593..455a8292b 100644
--- a/src/include/gnunet_protocols.h
+++ b/src/include/gnunet_protocols.h
@@ -1814,6 +1814,13 @@ extern "C"
  */
 #define GNUNET_MESSAGE_TYPE_SET_UNION_P2P_FULL_ELEMENT 598
 
+/**
+ * Request all missing elements from the other peer,
+ * based on their sets and the elements we previously sent
+ * with #GNUNET_MESSAGE_TYPE_SET_P2P_ELEMENTS.
+ */
+#define GNUNET_MESSAGE_TYPE_SET_UNION_P2P_OVER 599
+
 
 /*******************************************************************************
  * TESTBED LOGGER message types
diff --git a/src/include/gnunet_strings_lib.h b/src/include/gnunet_strings_lib.h
index 144780c82..897f9f161 100644
--- a/src/include/gnunet_strings_lib.h
+++ b/src/include/gnunet_strings_lib.h
@@ -281,7 +281,7 @@ GNUNET_STRINGS_get_short_name (const char *filename);
 
 
 /**
- * Convert binary data to ASCII encoding using Base32Hex (RFC 4648).
+ * Convert binary data to ASCII encoding using CrockfordBase32.
  * Does not append 0-terminator, but returns a pointer to the place where
  * it should be placed, if needed.
  *
@@ -315,7 +315,7 @@ GNUNET_STRINGS_data_to_string_alloc (const void *buf,
 
 
 /**
- * Convert Base32hex encoding back to data.
+ * Convert CrockfordBase32 encoding back to data.
  * @a out_size must match exactly the size of the data before it was encoded.
  *
  * @param enc the encoding