taler-helper-auditor-wire-debit.c (56727B)
1 /* 2 This file is part of TALER 3 Copyright (C) 2017-2024 Taler Systems SA 4 5 TALER is free software; you can redistribute it and/or modify it under the 6 terms of the GNU General Public License as published by the Free Software 7 Foundation; either version 3, or (at your option) any later version. 8 9 TALER is distributed in the hope that it will be useful, but WITHOUT ANY 10 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR 11 A PARTICULAR PURPOSE. See the GNU General Public License for more details. 12 13 You should have received a copy of the GNU General Public License along with 14 TALER; see the file COPYING. If not, see <http://www.gnu.org/licenses/> 15 */ 16 /** 17 * @file auditor/taler-helper-auditor-wire-debit.c 18 * @brief audits that wire outgoing transfers match those from an exchange 19 * database. 20 * @author Christian Grothoff 21 * @author Özgür Kesim 22 * 23 * - We check that the outgoing wire transfers match those 24 * given in the 'wire_out' and 'reserve_closures' tables; 25 * any outgoing transfer MUST have a prior justification, 26 * so if one is missing we flag it (and never remove it). 27 * - We check that all wire transfers that should 28 * have been made, were actually made. If any were not made, 29 * we flag those, but may remove those flags if we later 30 * find that the wire transfers were made (wire transfers 31 * could be delayed due to AML/KYC or core-banking issues). 32 */ 33 #include "platform.h" 34 #include <gnunet/gnunet_util_lib.h> 35 #include <gnunet/gnunet_curl_lib.h> 36 #include "auditordb_lib.h" 37 /* WIRE_TRANSFER_OUT's callback typedef also lives in exchangedb_lib.h, so its 38 closure override must be established before that header is included. */ 39 struct WireAccount; 40 #define TALER_EXCHANGEDB_WIRE_TRANSFER_OUT_RESULT_CLOSURE struct WireAccount 41 #include "exchangedb_lib.h" 42 #include "taler/taler_json_lib.h" 43 #include "taler/taler_bank_service.h" 44 #include "taler/taler_signatures.h" 45 #include "report-lib.h" 46 #include "taler/taler_dbevents.h" 47 #include "auditor-database/delete_auditor_closure_lag.h" 48 #include "auditor-database/delete_wire_out_inconsistency_if_matching.h" 49 #include "auditor-database/event_listen.h" 50 #include "auditor-database/get_auditor_progress.h" 51 #include "auditor-database/get_balance.h" 52 #include "auditor-database/insert_auditor_closure_lags.h" 53 #include "auditor-database/insert_auditor_progress.h" 54 #include "auditor-database/insert_balance.h" 55 #include "auditor-database/insert_row_inconsistency.h" 56 #include "auditor-database/insert_row_minor_inconsistencies.h" 57 #include "auditor-database/insert_wire_format_inconsistency.h" 58 #include "auditor-database/insert_wire_out_inconsistency.h" 59 #include "auditor-database/preflight.h" 60 #include "auditor-database/start.h" 61 #include "exchange-database/get_profit_drain.h" 62 #include "exchange-database/preflight.h" 63 #include "exchange-database/rollback.h" 64 #include "exchange-database/iterate_reserve_closed_above_serial_id.h" 65 #include "exchange-database/iterate_wire_outs_above_serial_id_by_account.h" 66 #include "exchange-database/start_read_only.h" 67 68 69 /** 70 * Maximum number of wire transfers we process per 71 * (database) transaction. 72 */ 73 #define MAX_PER_TRANSACTION 1024 74 75 /** 76 * How much do we allow the bank and the exchange to disagree about 77 * timestamps? Should be sufficiently large to avoid bogus reports from deltas 78 * created by imperfect clock synchronization and network delay. 79 */ 80 #define TIME_TOLERANCE GNUNET_TIME_relative_multiply ( \ 81 GNUNET_TIME_UNIT_MINUTES, \ 82 15) 83 84 85 /** 86 * How long do we try to long-poll for bank wire transfers? 87 */ 88 #define MAX_LONGPOLL_DELAY GNUNET_TIME_relative_multiply ( \ 89 GNUNET_TIME_UNIT_HOURS, \ 90 1) 91 92 93 /** 94 * How long do we wait between polling for bank wire transfers at the minimum? 95 */ 96 #define MIN_LONGPOLL_DELAY GNUNET_TIME_relative_multiply ( \ 97 GNUNET_TIME_UNIT_MINUTES, \ 98 5) 99 100 101 /** 102 * Run in test mode. Exit when idle instead of 103 * going to sleep and waiting for more work. 104 */ 105 static int test_mode; 106 107 108 /** 109 * Information we keep for each supported account. 110 */ 111 struct WireAccount 112 { 113 /** 114 * Accounts are kept in a DLL. 115 */ 116 struct WireAccount *next; 117 118 /** 119 * Plugins are kept in a DLL. 120 */ 121 struct WireAccount *prev; 122 123 /** 124 * Account details. 125 */ 126 const struct TALER_EXCHANGEDB_AccountInfo *ai; 127 128 /** 129 * Active wire request for the transaction history. 130 */ 131 struct TALER_BANK_DebitHistoryHandle *dhh; 132 133 /** 134 * Task to trigger @e dhh long-polling. 135 */ 136 struct GNUNET_SCHEDULER_Task *dhh_task; 137 138 /** 139 * Time when we expect the current @e dhh long-poll 140 * to finish and we thus could begin another one. 141 */ 142 struct GNUNET_TIME_Absolute dhh_next; 143 144 /** 145 * Progress point for this account. 146 */ 147 uint64_t last_wire_out_serial_id; 148 149 /** 150 * Initial progress point for this account. 151 */ 152 uint64_t start_wire_out_serial_id; 153 154 /** 155 * Where we are in the outbound transaction history. 156 */ 157 uint64_t wire_off_out; 158 159 /** 160 * Label under which we store our pp's reserve_in_serial_id. 161 */ 162 char *label_wire_out_serial_id; 163 164 /** 165 * Label under which we store our wire_off_out. 166 */ 167 char *label_wire_off_out; 168 }; 169 170 171 /** 172 * Information we track for a reserve being closed. 173 */ 174 struct ReserveClosure 175 { 176 /** 177 * Row in the reserves_closed table for this action. 178 */ 179 uint64_t rowid; 180 181 /** 182 * When was the reserve closed? 183 */ 184 struct GNUNET_TIME_Timestamp execution_date; 185 186 /** 187 * Amount transferred (amount remaining minus fee). 188 */ 189 struct TALER_Amount amount; 190 191 /** 192 * Target account where the money was sent. 193 */ 194 struct TALER_FullPayto receiver_account; 195 196 /** 197 * Wire transfer subject used. 198 */ 199 struct TALER_WireTransferIdentifierRawP wtid; 200 }; 201 202 203 /** 204 * Map from H(wtid,receiver_account) to `struct ReserveClosure` entries. 205 */ 206 static struct GNUNET_CONTAINER_MultiHashMap *reserve_closures; 207 208 /** 209 * Return value from main(). 210 */ 211 static int global_ret; 212 213 /** 214 * State of the current database transaction with 215 * the auditor DB. 216 */ 217 static enum GNUNET_DB_QueryStatus global_qs; 218 219 /** 220 * Map with information about outgoing wire transfers. 221 * Maps hashes of the wire subjects (in binary encoding) 222 * to `struct ReserveOutInfo`s. 223 */ 224 static struct GNUNET_CONTAINER_MultiHashMap *out_map; 225 226 /** 227 * Head of list of wire accounts we still need to look at. 228 */ 229 static struct WireAccount *wa_head; 230 231 /** 232 * Tail of list of wire accounts we still need to look at. 233 */ 234 static struct WireAccount *wa_tail; 235 236 /** 237 * Last reserve_out / wire_out serial IDs seen. 238 */ 239 static TALER_ARL_DEF_PP (wire_reserve_close_id); 240 241 /** 242 * Total amount that was transferred too much from the exchange. 243 */ 244 static TALER_ARL_DEF_AB (total_bad_amount_out_plus); 245 246 /** 247 * Total amount that was transferred too little from the exchange. 248 */ 249 static TALER_ARL_DEF_AB (total_bad_amount_out_minus); 250 251 /** 252 * Total amount of reserve closures which the exchange did not transfer in time. 253 */ 254 static TALER_ARL_DEF_AB (total_closure_amount_lag); 255 256 /** 257 * Total amount affected by duplicate wire transfer 258 * subjects. 259 */ 260 static TALER_ARL_DEF_AB (wire_debit_duplicate_transfer_subject_total); 261 262 /** 263 * Total amount debited to exchange accounts. 264 */ 265 static TALER_ARL_DEF_AB (total_wire_out); 266 267 /** 268 * Total amount of profits drained. 269 */ 270 static TALER_ARL_DEF_AB (total_drained); 271 272 /** 273 * Amount of zero in our currency. 274 */ 275 static struct TALER_Amount zero; 276 277 /** 278 * Handle to the context for interacting with the bank. 279 */ 280 static struct GNUNET_CURL_Context *ctx; 281 282 /** 283 * Scheduler context for running the @e ctx. 284 */ 285 static struct GNUNET_CURL_RescheduleContext *rctx; 286 287 /** 288 * Should we run checks that only work for exchange-internal audits? 289 */ 290 static int internal_checks; 291 292 /** 293 * Should we ignore if the bank does not know our bank 294 * account? 295 */ 296 static int ignore_account_404; 297 298 /** 299 * Database event handler to wake us up again. 300 */ 301 static struct GNUNET_DB_EventHandler *eh; 302 303 /** 304 * The auditors's configuration. 305 */ 306 static const struct GNUNET_CONFIGURATION_Handle *cfg; 307 308 309 /** 310 * Entry in map with wire information we expect to obtain from the 311 * #TALER_ARL_edb later. 312 */ 313 struct WireTransferOutInfo 314 { 315 316 /** 317 * Hash of the wire transfer subject. 318 */ 319 struct GNUNET_HashCode subject_hash; 320 321 /** 322 * Expected details about the wire transfer. 323 */ 324 struct TALER_BANK_DebitDetails details; 325 326 }; 327 328 329 /** 330 * Free entry in #out_map. 331 * 332 * @param cls NULL 333 * @param key unused key 334 * @param value the `struct WireTransferOutInfo` to free 335 * @return #GNUNET_OK 336 */ 337 static enum GNUNET_GenericReturnValue 338 free_roi (void *cls, 339 const struct GNUNET_HashCode *key, 340 void *value) 341 { 342 struct WireTransferOutInfo *roi = value; 343 344 (void) cls; 345 GNUNET_assert (GNUNET_YES == 346 GNUNET_CONTAINER_multihashmap_remove (out_map, 347 key, 348 roi)); 349 GNUNET_free (roi); 350 return GNUNET_OK; 351 } 352 353 354 /** 355 * Free entry in #reserve_closures. 356 * 357 * @param cls NULL 358 * @param key unused key 359 * @param value the `struct ReserveClosure` to free 360 * @return #GNUNET_OK 361 */ 362 static enum GNUNET_GenericReturnValue 363 free_rc (void *cls, 364 const struct GNUNET_HashCode *key, 365 void *value) 366 { 367 struct ReserveClosure *rc = value; 368 369 (void) cls; 370 GNUNET_assert (GNUNET_YES == 371 GNUNET_CONTAINER_multihashmap_remove (reserve_closures, 372 key, 373 rc)); 374 GNUNET_free (rc->receiver_account.full_payto); 375 GNUNET_free (rc); 376 return GNUNET_OK; 377 } 378 379 380 /** 381 * Task run on shutdown. 382 * 383 * @param cls NULL 384 */ 385 static void 386 do_shutdown (void *cls) 387 { 388 struct WireAccount *wa; 389 390 (void) cls; 391 if (NULL != eh) 392 { 393 TALER_AUDITORDB_event_listen_cancel (eh); 394 eh = NULL; 395 } 396 TALER_ARL_done (); 397 if (NULL != reserve_closures) 398 { 399 GNUNET_CONTAINER_multihashmap_iterate (reserve_closures, 400 &free_rc, 401 NULL); 402 GNUNET_CONTAINER_multihashmap_destroy (reserve_closures); 403 reserve_closures = NULL; 404 } 405 if (NULL != out_map) 406 { 407 GNUNET_CONTAINER_multihashmap_iterate (out_map, 408 &free_roi, 409 NULL); 410 GNUNET_CONTAINER_multihashmap_destroy (out_map); 411 out_map = NULL; 412 } 413 while (NULL != (wa = wa_head)) 414 { 415 if (NULL != wa->dhh_task) 416 { 417 GNUNET_SCHEDULER_cancel (wa->dhh_task); 418 wa->dhh_task = NULL; 419 } 420 if (NULL != wa->dhh) 421 { 422 TALER_BANK_debit_history_cancel (wa->dhh); 423 wa->dhh = NULL; 424 } 425 GNUNET_CONTAINER_DLL_remove (wa_head, 426 wa_tail, 427 wa); 428 GNUNET_free (wa->label_wire_out_serial_id); 429 GNUNET_free (wa->label_wire_off_out); 430 GNUNET_free (wa); 431 } 432 if (NULL != ctx) 433 { 434 GNUNET_CURL_fini (ctx); 435 ctx = NULL; 436 } 437 if (NULL != rctx) 438 { 439 GNUNET_CURL_gnunet_rc_destroy (rctx); 440 rctx = NULL; 441 } 442 TALER_EXCHANGEDB_unload_accounts (); 443 TALER_ARL_cfg = NULL; 444 } 445 446 447 /** 448 * Detect any entries in #reserve_closures that were not yet 449 * observed on the wire transfer side and update the progress 450 * point accordingly. 451 * 452 * @param cls NULL 453 * @param key unused key 454 * @param value the `struct ReserveClosure` to free 455 * @return #GNUNET_OK 456 */ 457 static enum GNUNET_GenericReturnValue 458 check_pending_rc (void *cls, 459 const struct GNUNET_HashCode *key, 460 void *value) 461 { 462 struct ReserveClosure *rc = value; 463 464 (void) cls; 465 (void) key; 466 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 467 "Missing wire transfer for closed reserve with balance %s\n", 468 TALER_amount2s (&rc->amount)); 469 if (! TALER_amount_is_zero (&rc->amount)) 470 { 471 struct TALER_AUDITORDB_ClosureLags cl = { 472 .problem_row_id = rc->rowid, 473 .account = rc->receiver_account, 474 .amount = rc->amount, 475 .deadline = rc->execution_date.abs_time, 476 .wtid = rc->wtid 477 }; 478 enum GNUNET_DB_QueryStatus qs; 479 480 qs = TALER_AUDITORDB_insert_auditor_closure_lags ( 481 TALER_ARL_adb, 482 &cl); 483 if (qs < 0) 484 { 485 global_qs = qs; 486 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 487 return GNUNET_SYSERR; 488 } 489 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_closure_amount_lag), 490 &TALER_ARL_USE_AB (total_closure_amount_lag), 491 &rc->amount); 492 } 493 return GNUNET_OK; 494 } 495 496 497 /** 498 * Compute the key under which a reserve closure for a given 499 * @a receiver_account and @a wtid would be stored. 500 * 501 * @param receiver_account payto://-URI of the account 502 * @param wtid wire transfer identifier used 503 * @param[out] key set to the key 504 */ 505 static void 506 hash_rc (const struct TALER_FullPayto receiver_account, 507 const struct TALER_WireTransferIdentifierRawP *wtid, 508 struct GNUNET_HashCode *key) 509 { 510 struct TALER_NormalizedPayto npto 511 = TALER_payto_normalize (receiver_account); 512 size_t slen = strlen (npto.normalized_payto); 513 char buf[sizeof (struct TALER_WireTransferIdentifierRawP) + slen]; 514 515 GNUNET_memcpy (buf, 516 wtid, 517 sizeof (*wtid)); 518 GNUNET_memcpy (&buf[sizeof (*wtid)], 519 npto.normalized_payto, 520 slen); 521 GNUNET_CRYPTO_hash (buf, 522 sizeof (buf), 523 key); 524 GNUNET_free (npto.normalized_payto); 525 } 526 527 528 /** 529 * Start the database transactions and begin the audit. 530 * 531 * @return transaction status code 532 */ 533 static enum GNUNET_DB_QueryStatus 534 begin_transaction (void); 535 536 537 /** 538 * Commit the transaction, checkpointing our progress in the auditor DB. 539 * 540 * @param qs transaction status so far 541 */ 542 static void 543 commit (enum GNUNET_DB_QueryStatus qs) 544 { 545 GNUNET_CONTAINER_multihashmap_iterate (reserve_closures, 546 &check_pending_rc, 547 NULL); 548 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 549 "Transaction logic ended with status %d\n", 550 qs); 551 TALER_EXCHANGEDB_rollback (TALER_ARL_edb); 552 qs = TALER_AUDITORDB_insert_balance ( 553 TALER_ARL_adb, 554 TALER_ARL_SET_AB (total_drained), 555 TALER_ARL_SET_AB (total_wire_out), 556 TALER_ARL_SET_AB (total_bad_amount_out_plus), 557 TALER_ARL_SET_AB (total_bad_amount_out_minus), 558 TALER_ARL_SET_AB (total_closure_amount_lag), 559 TALER_ARL_SET_AB (wire_debit_duplicate_transfer_subject_total), 560 TALER_ARL_SET_AB (total_wire_out), 561 NULL); 562 if (0 > qs) 563 goto handle_db_error; 564 for (struct WireAccount *wa = wa_head; 565 NULL != wa; 566 wa = wa->next) 567 { 568 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 569 "Transaction of account %s ends at %llu/%llu\n", 570 wa->ai->section_name, 571 (unsigned long long) wa->last_wire_out_serial_id, 572 (unsigned long long) wa->wire_off_out); 573 qs = TALER_AUDITORDB_insert_auditor_progress ( 574 TALER_ARL_adb, 575 wa->label_wire_out_serial_id, 576 wa->last_wire_out_serial_id, 577 wa->label_wire_off_out, 578 wa->wire_off_out, 579 NULL); 580 if (0 > qs) 581 goto handle_db_error; 582 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 583 "Transaction ends at %s=%llu for account `%s'\n", 584 wa->label_wire_out_serial_id, 585 (unsigned long long) wa->last_wire_out_serial_id, 586 wa->ai->section_name); 587 } 588 qs = TALER_AUDITORDB_insert_auditor_progress ( 589 TALER_ARL_adb, 590 TALER_ARL_SET_PP (wire_reserve_close_id), 591 NULL); 592 if (0 > qs) 593 goto handle_db_error; 594 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 595 "Concluded audit step at %llu\n", 596 (unsigned long long) TALER_ARL_USE_PP (wire_reserve_close_id)); 597 qs = TALER_AUDITORDB_commit (TALER_ARL_adb); 598 if (0 > qs) 599 goto handle_db_error; 600 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 601 "Transaction concluded!\n"); 602 if (1 == test_mode) 603 GNUNET_SCHEDULER_shutdown (); 604 return; 605 handle_db_error: 606 TALER_AUDITORDB_rollback (TALER_ARL_adb); 607 for (unsigned int max_retries = 3; max_retries>0; max_retries--) 608 { 609 if (GNUNET_DB_STATUS_HARD_ERROR == qs) 610 break; 611 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 612 "Serialization issue, trying again\n"); 613 qs = begin_transaction (); 614 } 615 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 616 "Hard database error, terminating\n"); 617 GNUNET_SCHEDULER_shutdown (); 618 } 619 620 621 /** 622 * Check that @a want is within #TIME_TOLERANCE of @a have. 623 * Otherwise report an inconsistency in row @a rowid of @a table. 624 * 625 * @param table where is the inconsistency (if any) 626 * @param rowid what is the row 627 * @param want what is the expected time 628 * @param have what is the time we got 629 * @return true on success, false to abort 630 */ 631 static bool 632 check_time_difference (const char *table, 633 uint64_t rowid, 634 struct GNUNET_TIME_Timestamp want, 635 struct GNUNET_TIME_Timestamp have) 636 { 637 struct GNUNET_TIME_Relative delta; 638 char *details; 639 640 if (GNUNET_TIME_timestamp_cmp (have, >, want)) 641 delta = GNUNET_TIME_absolute_get_difference (want.abs_time, 642 have.abs_time); 643 else 644 delta = GNUNET_TIME_absolute_get_difference (have.abs_time, 645 want.abs_time); 646 if (GNUNET_TIME_relative_cmp (delta, 647 <=, 648 TIME_TOLERANCE)) 649 return true; 650 651 GNUNET_asprintf (&details, 652 "execution date mismatch (%s)", 653 GNUNET_TIME_relative2s (delta, 654 true)); 655 { 656 struct TALER_AUDITORDB_RowMinorInconsistencies rmi = { 657 .row_table = (char *) table, 658 .problem_row = rowid, 659 .diagnostic = details 660 }; 661 enum GNUNET_DB_QueryStatus qs; 662 663 qs = TALER_AUDITORDB_insert_row_minor_inconsistencies ( 664 TALER_ARL_adb, 665 &rmi); 666 667 if (qs < 0) 668 { 669 global_qs = qs; 670 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 671 GNUNET_free (details); 672 return false; 673 } 674 } 675 GNUNET_free (details); 676 return true; 677 } 678 679 680 /** 681 * Closure for #check_rc_matches 682 */ 683 struct CheckMatchContext 684 { 685 686 /** 687 * Reserve operation looking for a match 688 */ 689 const struct WireTransferOutInfo *roi; 690 691 /** 692 * Set to true if we found a match. 693 */ 694 bool found; 695 }; 696 697 698 /** 699 * Check if any of the reserve closures match the given wire transfer. 700 * 701 * @param[in,out] cls a `struct CheckMatchContext` 702 * @param key key of @a value in #reserve_closures 703 * @param value a `struct ReserveClosure` 704 */ 705 static enum GNUNET_GenericReturnValue 706 check_rc_matches (void *cls, 707 const struct GNUNET_HashCode *key, 708 void *value) 709 { 710 struct CheckMatchContext *cmx = cls; 711 struct ReserveClosure *rc = value; 712 713 if ((0 == GNUNET_memcmp (&cmx->roi->details.wtid, 714 &rc->wtid)) && 715 (0 == TALER_full_payto_cmp (rc->receiver_account, 716 cmx->roi->details.credit_account_uri)) && 717 (0 == TALER_amount_cmp (&rc->amount, 718 &cmx->roi->details.amount))) 719 { 720 if (! check_time_difference ("reserves_closures", 721 rc->rowid, 722 rc->execution_date, 723 cmx->roi->details.execution_date)) 724 { 725 free_rc (NULL, 726 key, 727 rc); 728 return GNUNET_SYSERR; 729 } 730 cmx->found = true; 731 free_rc (NULL, 732 key, 733 rc); 734 return GNUNET_NO; 735 } 736 return GNUNET_OK; 737 } 738 739 740 /** 741 * Maximum required length for make_missing_diag(). 742 */ 743 #define MAX_DIAG_LEN 128 744 745 /** 746 * Make diagnostic string for missing wire transfer. 747 * 748 * @param[out] diag where to write the diagnostic string 749 * @param wtid wire transfer ID to include 750 */ 751 static void 752 make_missing_diag (char diag[MAX_DIAG_LEN], 753 const struct TALER_WireTransferIdentifierRawP *wtid) 754 { 755 char *wtid_s; 756 757 wtid_s = GNUNET_STRINGS_data_to_string_alloc (wtid, 758 sizeof (*wtid)); 759 GNUNET_snprintf (diag, 760 MAX_DIAG_LEN, 761 "expected outgoing wire transfer %s missing", 762 wtid_s); 763 GNUNET_free (wtid_s); 764 } 765 766 767 /** 768 * Check if an existing report on a missing wire 769 * out operation justified the @a roi. If so, 770 * clear the existing report. 771 * 772 * @param roi reserve out operation to check 773 * @return #GNUNET_YES if @a roi was justified by a previous report, 774 * #GNUNET_NO of @a roi was not justified by a previous missing report 775 * #GNUNET_SYSERR on database trouble 776 */ 777 static enum GNUNET_GenericReturnValue 778 check_reported_inconsistency (struct WireTransferOutInfo *roi) 779 { 780 char diag[MAX_DIAG_LEN]; 781 struct TALER_AUDITORDB_WireOutInconsistency woi = { 782 .wire_out_row_id = roi->details.serial_id, 783 .destination_account = roi->details.credit_account_uri, 784 .diagnostic = diag, 785 .expected = roi->details.amount, 786 .claimed = zero, 787 }; 788 enum GNUNET_DB_QueryStatus qs; 789 790 make_missing_diag (diag, 791 &roi->details.wtid); 792 qs = TALER_AUDITORDB_delete_wire_out_inconsistency_if_matching ( 793 TALER_ARL_adb, 794 &woi); 795 if (qs < 0) 796 { 797 global_qs = qs; 798 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 799 return GNUNET_SYSERR; 800 } 801 if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) 802 { 803 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, 804 "Deletion of wire out inconsistency %llu (%s, %s, %s) failed: not reported missing!\n", 805 (unsigned long long) roi->details.serial_id, 806 roi->details.credit_account_uri.full_payto, 807 diag, 808 TALER_amount2s (&roi->details.amount)); 809 return GNUNET_NO; 810 } 811 TALER_ARL_amount_subtract (&TALER_ARL_USE_AB (total_bad_amount_out_minus), 812 &TALER_ARL_USE_AB (total_bad_amount_out_minus), 813 &roi->details.amount); 814 return GNUNET_YES; 815 } 816 817 818 /** 819 * Check if a profit drain operation justified the @a roi 820 * 821 * @param roi reserve out operation to check 822 * @return #GNUNET_YES if @a roi was justified by a profit drain, 823 * #GNUNET_NO of @a roi was not justified by a proft drain 824 * #GNUNET_SYSERR on database trouble 825 */ 826 static enum GNUNET_GenericReturnValue 827 check_profit_drain (struct WireTransferOutInfo *roi) 828 { 829 enum GNUNET_DB_QueryStatus qs; 830 uint64_t serial; 831 char *account_section; 832 struct TALER_FullPayto payto_uri; 833 struct GNUNET_TIME_Timestamp request_timestamp; 834 struct TALER_Amount amount; 835 struct TALER_MasterSignatureP master_sig; 836 837 qs = TALER_EXCHANGEDB_get_profit_drain ( 838 TALER_ARL_edb, 839 &roi->details.wtid, 840 &serial, 841 &account_section, 842 &payto_uri, 843 &request_timestamp, 844 &amount, 845 &master_sig); 846 switch (qs) 847 { 848 case GNUNET_DB_STATUS_HARD_ERROR: 849 GNUNET_break (0); 850 global_ret = EXIT_FAILURE; 851 GNUNET_SCHEDULER_shutdown (); 852 return GNUNET_SYSERR; 853 case GNUNET_DB_STATUS_SOFT_ERROR: 854 /* should fail on commit later ... */ 855 GNUNET_break (0); 856 return GNUNET_SYSERR; 857 case GNUNET_DB_STATUS_SUCCESS_NO_RESULTS: 858 /* not a profit drain */ 859 return GNUNET_NO; 860 case GNUNET_DB_STATUS_SUCCESS_ONE_RESULT: 861 break; 862 } 863 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 864 "Profit drain of %s to %s found!\n", 865 TALER_amount2s (&amount), 866 payto_uri.full_payto); 867 if (GNUNET_OK != 868 TALER_exchange_offline_profit_drain_verify ( 869 &roi->details.wtid, 870 request_timestamp, 871 &amount, 872 account_section, 873 payto_uri, 874 &TALER_ARL_master_pub, 875 &master_sig)) 876 { 877 struct TALER_AUDITORDB_RowInconsistency ri = { 878 .row_id = roi->details.serial_id, 879 .row_table = (char *) "profit_drains", 880 .diagnostic = (char *) "invalid signature" 881 }; 882 883 GNUNET_break (0); 884 qs = TALER_AUDITORDB_insert_row_inconsistency ( 885 TALER_ARL_adb, 886 &ri); 887 GNUNET_free (payto_uri.full_payto); 888 GNUNET_free (account_section); 889 if (qs < 0) 890 { 891 global_qs = qs; 892 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 893 return GNUNET_SYSERR; 894 } 895 return GNUNET_NO; 896 } 897 GNUNET_free (account_section); 898 899 { 900 if (0 != 901 TALER_full_payto_normalize_and_cmp (payto_uri, 902 roi->details.credit_account_uri)) 903 { 904 struct TALER_AUDITORDB_WireOutInconsistency woi = { 905 .wire_out_row_id = serial, 906 .destination_account = roi->details.credit_account_uri, 907 .diagnostic = (char *) "profit drain wired to invalid account", 908 .expected = roi->details.amount, 909 .claimed = zero, 910 }; 911 912 qs = TALER_AUDITORDB_insert_wire_out_inconsistency ( 913 TALER_ARL_adb, 914 &woi); 915 if (qs < 0) 916 { 917 global_qs = qs; 918 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 919 GNUNET_free (payto_uri.full_payto); 920 return GNUNET_SYSERR; 921 } 922 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_bad_amount_out_plus), 923 &TALER_ARL_USE_AB (total_bad_amount_out_plus), 924 &amount); 925 GNUNET_free (payto_uri.full_payto); 926 return GNUNET_YES; /* justified, kind-of */ 927 } 928 } 929 GNUNET_free (payto_uri.full_payto); 930 if (0 != 931 TALER_amount_cmp (&amount, 932 &roi->details.amount)) 933 { 934 struct TALER_AUDITORDB_WireOutInconsistency woi = { 935 .wire_out_row_id = roi->details.serial_id, 936 .destination_account = roi->details.credit_account_uri, 937 .diagnostic = (char *) "incorrect amount drained to correct account", 938 .expected = roi->details.amount, 939 .claimed = amount, 940 }; 941 942 qs = TALER_AUDITORDB_insert_wire_out_inconsistency ( 943 TALER_ARL_adb, 944 &woi); 945 if (qs < 0) 946 { 947 global_qs = qs; 948 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 949 return GNUNET_SYSERR; 950 } 951 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_bad_amount_out_minus), 952 &TALER_ARL_USE_AB (total_bad_amount_out_minus), 953 &roi->details.amount); 954 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_bad_amount_out_plus), 955 &TALER_ARL_USE_AB (total_bad_amount_out_plus), 956 &amount); 957 return GNUNET_YES; /* justified, kind-of */ 958 } 959 /* profit drain was correct */ 960 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_drained), 961 &TALER_ARL_USE_AB (total_drained), 962 &amount); 963 return GNUNET_YES; 964 } 965 966 967 /** 968 * Check whether the given transfer was justified because it was 969 * actually previously reported as missing. 970 * 971 * @param roi the reserve out operation to check 972 * @return #GNUNET_OK on success, #GNUNET_NO if there was no 973 * matching lag, #GNUNET_SYSERR on database trouble 974 */ 975 static enum GNUNET_GenericReturnValue 976 check_closure_lag (const struct WireTransferOutInfo *roi) 977 { 978 enum GNUNET_DB_QueryStatus qs; 979 980 qs = TALER_AUDITORDB_delete_auditor_closure_lag ( 981 TALER_ARL_adb, 982 &roi->details.amount, 983 &roi->details.wtid, 984 roi->details.credit_account_uri); 985 if (qs < 0) 986 { 987 global_qs = qs; 988 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 989 return GNUNET_SYSERR; 990 } 991 if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) 992 return GNUNET_NO; 993 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 994 "Cleared closure lag: found justification\n"); 995 TALER_ARL_amount_subtract (&TALER_ARL_USE_AB (total_closure_amount_lag), 996 &TALER_ARL_USE_AB (total_closure_amount_lag), 997 &roi->details.amount); 998 return GNUNET_YES; /* found! */ 999 } 1000 1001 1002 /** 1003 * Check whether the given transfer was justified by a reserve closure or 1004 * profit drain. If not, complain that we failed to match an entry from 1005 * #out_map. This means a wire transfer was made without proper 1006 * justification. 1007 * 1008 * @param cls a `struct WireAccount` 1009 * @param key unused key 1010 * @param value the `struct WireTransferOutInfo` to report 1011 * @return #GNUNET_OK on success 1012 */ 1013 static enum GNUNET_GenericReturnValue 1014 complain_out_not_found (void *cls, 1015 const struct GNUNET_HashCode *key, 1016 void *value) 1017 { 1018 // struct WireAccount *wa = cls; 1019 struct WireTransferOutInfo *roi = value; 1020 struct GNUNET_HashCode rkey; 1021 struct CheckMatchContext cmx = { 1022 .roi = roi, 1023 .found = false 1024 }; 1025 enum GNUNET_GenericReturnValue ret; 1026 1027 (void) cls; 1028 (void) key; 1029 hash_rc (roi->details.credit_account_uri, 1030 &roi->details.wtid, 1031 &rkey); 1032 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1033 "Checking for reserve closure %s benefiting %s\n", 1034 GNUNET_h2s (&rkey), 1035 roi->details.credit_account_uri.full_payto); 1036 GNUNET_CONTAINER_multihashmap_get_multiple (reserve_closures, 1037 &rkey, 1038 &check_rc_matches, 1039 &cmx); 1040 if (cmx.found) 1041 return GNUNET_OK; 1042 ret = check_reported_inconsistency (roi); 1043 if (GNUNET_NO != ret) 1044 return ret; 1045 ret = check_profit_drain (roi); 1046 if (GNUNET_NO != ret) 1047 return ret; 1048 ret = check_closure_lag (roi); 1049 if (GNUNET_NO != ret) 1050 return ret; 1051 1052 { 1053 struct TALER_AUDITORDB_WireOutInconsistency woi = { 1054 .destination_account = roi->details.credit_account_uri, 1055 .diagnostic = (char *) "missing justification for outgoing wire transfer", 1056 .wire_out_row_id = roi->details.serial_id, 1057 .expected = zero, 1058 .claimed = roi->details.amount 1059 }; 1060 enum GNUNET_DB_QueryStatus qs; 1061 1062 qs = TALER_AUDITORDB_insert_wire_out_inconsistency ( 1063 TALER_ARL_adb, 1064 &woi); 1065 if (qs < 0) 1066 { 1067 global_qs = qs; 1068 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1069 return GNUNET_SYSERR; 1070 } 1071 } 1072 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_bad_amount_out_plus), 1073 &TALER_ARL_USE_AB (total_bad_amount_out_plus), 1074 &roi->details.amount); 1075 return GNUNET_OK; 1076 } 1077 1078 1079 /** 1080 * Function called with details about outgoing wire transfers 1081 * as claimed by the exchange DB. 1082 * 1083 * @param wa a `struct WireAccount` 1084 * @param rowid unique serial ID in wire_out table 1085 * @param date timestamp of the transfer (roughly) 1086 * @param wtid wire transfer subject 1087 * @param payto_uri wire transfer details of the receiver 1088 * @param amount amount that was wired 1089 * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR to stop 1090 */ 1091 static enum GNUNET_GenericReturnValue 1092 wire_out_cb ( 1093 struct WireAccount *wa, 1094 uint64_t rowid, 1095 struct GNUNET_TIME_Timestamp date, 1096 const struct TALER_WireTransferIdentifierRawP *wtid, 1097 const struct TALER_FullPayto payto_uri, 1098 const struct TALER_Amount *amount) 1099 { 1100 struct GNUNET_HashCode key; 1101 struct WireTransferOutInfo *roi; 1102 1103 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1104 "Exchange wire OUT #%llu at %s of %s with WTID %s\n", 1105 (unsigned long long) rowid, 1106 GNUNET_TIME_timestamp2s (date), 1107 TALER_amount2s (amount), 1108 TALER_B2S (wtid)); 1109 wa->last_wire_out_serial_id = rowid + 1; 1110 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_wire_out), 1111 &TALER_ARL_USE_AB (total_wire_out), 1112 amount); 1113 GNUNET_CRYPTO_hash (wtid, 1114 sizeof (*wtid), 1115 &key); 1116 roi = GNUNET_CONTAINER_multihashmap_get (out_map, 1117 &key); 1118 if (NULL == roi) 1119 { 1120 /* Wire transfer was not made (yet) at all (but would have been 1121 justified), so the entire amount is missing / still to be done. This 1122 is moderately harmless, it might just be that the 1123 taler-exchange-transfer tool or bank has not yet fully caught up with 1124 the transfers it should do. 1125 May be cleared later by check_reported_inconsistency() */ 1126 char diag[MAX_DIAG_LEN]; 1127 struct TALER_AUDITORDB_WireOutInconsistency woi = { 1128 .destination_account = payto_uri, 1129 .diagnostic = diag, 1130 .wire_out_row_id = rowid, 1131 .expected = *amount, 1132 .claimed = zero, 1133 }; 1134 enum GNUNET_DB_QueryStatus qs; 1135 1136 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1137 "Wire out for row %llu still missing\n", 1138 (unsigned long long) rowid); 1139 make_missing_diag (diag, 1140 wtid); 1141 qs = TALER_AUDITORDB_insert_wire_out_inconsistency ( 1142 TALER_ARL_adb, 1143 &woi); 1144 if (qs < 0) 1145 { 1146 global_qs = qs; 1147 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1148 return GNUNET_SYSERR; 1149 } 1150 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_bad_amount_out_minus), 1151 &TALER_ARL_USE_AB (total_bad_amount_out_minus), 1152 amount); 1153 return GNUNET_OK; 1154 } 1155 1156 if (0 != TALER_full_payto_normalize_and_cmp (payto_uri, 1157 roi->details.credit_account_uri)) 1158 { 1159 /* Destination bank account is wrong in actual wire transfer, so 1160 we should count the wire transfer as entirely spurious, and 1161 additionally consider the justified wire transfer as missing. */ 1162 struct TALER_AUDITORDB_WireOutInconsistency woi = { 1163 .wire_out_row_id = rowid, 1164 .destination_account = payto_uri, 1165 .diagnostic = (char *) "receiver account mismatch", 1166 .expected = *amount, 1167 .claimed = roi->details.amount, 1168 }; 1169 enum GNUNET_DB_QueryStatus qs; 1170 1171 qs = TALER_AUDITORDB_insert_wire_out_inconsistency ( 1172 TALER_ARL_adb, 1173 &woi); 1174 if (qs < 0) 1175 { 1176 global_qs = qs; 1177 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1178 return GNUNET_SYSERR; 1179 } 1180 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_bad_amount_out_plus), 1181 &TALER_ARL_USE_AB (total_bad_amount_out_plus), 1182 &roi->details.amount); 1183 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_bad_amount_out_minus), 1184 &TALER_ARL_USE_AB (total_bad_amount_out_minus), 1185 amount); 1186 GNUNET_assert (GNUNET_OK == 1187 free_roi (NULL, 1188 &key, 1189 roi)); 1190 return GNUNET_OK; 1191 } 1192 1193 if (0 != TALER_amount_cmp (&roi->details.amount, 1194 amount)) 1195 { 1196 struct TALER_AUDITORDB_WireOutInconsistency woi = { 1197 .destination_account = payto_uri, 1198 .diagnostic = (char *) "wire amount does not match", 1199 .wire_out_row_id = rowid, 1200 .expected = *amount, 1201 .claimed = roi->details.amount, 1202 }; 1203 enum GNUNET_DB_QueryStatus qs; 1204 1205 qs = TALER_AUDITORDB_insert_wire_out_inconsistency ( 1206 TALER_ARL_adb, 1207 &woi); 1208 if (qs < 0) 1209 { 1210 global_qs = qs; 1211 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1212 return GNUNET_SYSERR; 1213 } 1214 if (0 < TALER_amount_cmp (amount, 1215 &roi->details.amount)) 1216 { 1217 /* amount > roi->details.amount: wire transfer was smaller than it should have been */ 1218 struct TALER_Amount delta; 1219 1220 TALER_ARL_amount_subtract (&delta, 1221 amount, 1222 &roi->details.amount); 1223 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_bad_amount_out_minus), 1224 &TALER_ARL_USE_AB (total_bad_amount_out_minus), 1225 &delta); 1226 } 1227 else 1228 { 1229 /* roi->details.amount < amount: wire transfer was larger than it should have been */ 1230 struct TALER_Amount delta; 1231 1232 TALER_ARL_amount_subtract (&delta, 1233 &roi->details.amount, 1234 amount); 1235 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_bad_amount_out_plus), 1236 &TALER_ARL_USE_AB (total_bad_amount_out_plus), 1237 &delta); 1238 } 1239 GNUNET_assert (GNUNET_OK == 1240 free_roi (NULL, 1241 &key, 1242 roi)); 1243 return GNUNET_OK; 1244 } 1245 1246 { 1247 enum GNUNET_GenericReturnValue ret; 1248 1249 if (! check_time_difference ("wire_out", 1250 rowid, 1251 date, 1252 roi->details.execution_date)) 1253 { 1254 /* We had a database error, fail */ 1255 ret = GNUNET_SYSERR; 1256 } 1257 else 1258 { 1259 ret = GNUNET_OK; 1260 } 1261 GNUNET_assert (GNUNET_OK == 1262 free_roi (NULL, 1263 &key, 1264 roi)); 1265 return ret; 1266 } 1267 } 1268 1269 1270 /** 1271 * Main function for processing 'debit' data. We start by going over 1272 * the DEBIT transactions this time, and then verify that all of them are 1273 * justified by reserve closures, profit drains or regular outgoing 1274 * wire transfers from aggregated deposits. 1275 * 1276 * @param[in,out] wa wire account list to process 1277 */ 1278 static void 1279 process_debits (struct WireAccount *wa); 1280 1281 1282 /** 1283 * Go over the "wire_out" table of the exchange and 1284 * verify that all wire outs are in that table. 1285 * 1286 * @param[in,out] wa wire account we are processing 1287 */ 1288 static void 1289 check_exchange_wire_out (struct WireAccount *wa) 1290 { 1291 enum GNUNET_DB_QueryStatus qs; 1292 1293 GNUNET_assert (NULL == wa->dhh); 1294 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1295 "Analyzing exchange's wire OUT table for account `%s'\n", 1296 wa->ai->section_name); 1297 qs = TALER_EXCHANGEDB_iterate_wire_outs_above_serial_id_by_account ( 1298 TALER_ARL_edb, 1299 wa->ai->section_name, 1300 wa->last_wire_out_serial_id, 1301 &wire_out_cb, 1302 wa); 1303 if (0 > qs) 1304 { 1305 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1306 global_ret = EXIT_FAILURE; 1307 GNUNET_SCHEDULER_shutdown (); 1308 return; 1309 } 1310 GNUNET_CONTAINER_multihashmap_iterate (out_map, 1311 &complain_out_not_found, 1312 wa); 1313 /* clean up */ 1314 GNUNET_CONTAINER_multihashmap_iterate (out_map, 1315 &free_roi, 1316 NULL); 1317 process_debits (wa->next); 1318 } 1319 1320 1321 /** 1322 * This function is called for all transactions that 1323 * are debited from the exchange's account (outgoing 1324 * transactions). 1325 * 1326 * @param cls `struct WireAccount` with current wire account to process 1327 * @param dhr HTTP response details 1328 */ 1329 static void 1330 history_debit_cb ( 1331 void *cls, 1332 const struct TALER_BANK_DebitHistoryResponse *dhr); 1333 1334 1335 /** 1336 * Task scheduled to begin long-polling on the 1337 * bank transfer. 1338 * 1339 * @param cls a `struct WireAccount *` 1340 */ 1341 static void 1342 dh_long_poll (void *cls) 1343 { 1344 struct WireAccount *wa = cls; 1345 1346 wa->dhh_task = NULL; 1347 wa->dhh_next 1348 = GNUNET_TIME_relative_to_absolute (MIN_LONGPOLL_DELAY); 1349 GNUNET_assert (NULL == wa->dhh); 1350 wa->dhh = TALER_BANK_debit_history ( 1351 ctx, 1352 wa->ai->auth, 1353 wa->wire_off_out, 1354 MAX_PER_TRANSACTION, 1355 MAX_LONGPOLL_DELAY, 1356 &history_debit_cb, 1357 wa); 1358 if (NULL == wa->dhh) 1359 { 1360 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 1361 "Failed to start long-polling for bank transaction history for `%s'\n", 1362 wa->ai->section_name); 1363 global_ret = EXIT_FAILURE; 1364 GNUNET_SCHEDULER_shutdown (); 1365 return; 1366 } 1367 } 1368 1369 1370 static void 1371 history_debit_cb ( 1372 void *cls, 1373 const struct TALER_BANK_DebitHistoryResponse *dhr) 1374 { 1375 struct WireAccount *wa = cls; 1376 struct WireTransferOutInfo *roi; 1377 size_t slen; 1378 1379 wa->dhh = NULL; 1380 if ( (MHD_HTTP_OK == dhr->http_status) && 1381 (0 != dhr->details.ok.details_length) ) 1382 { 1383 /* As we got results, we go again *immediately* */ 1384 wa->dhh_next = GNUNET_TIME_UNIT_ZERO_ABS; 1385 } 1386 GNUNET_assert (NULL == wa->dhh_task); 1387 wa->dhh_task 1388 = GNUNET_SCHEDULER_add_at (wa->dhh_next, 1389 &dh_long_poll, 1390 wa); 1391 switch (dhr->http_status) 1392 { 1393 case MHD_HTTP_OK: 1394 for (unsigned int i = 0; i < dhr->details.ok.details_length; i++) 1395 { 1396 const struct TALER_BANK_DebitDetails *dd 1397 = &dhr->details.ok.details[i]; 1398 1399 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1400 "Analyzing bank DEBIT #%llu at %s of %s with WTID %s\n", 1401 (unsigned long long) dd->serial_id, 1402 GNUNET_TIME_timestamp2s (dd->execution_date), 1403 TALER_amount2s (&dd->amount), 1404 TALER_B2S (&dd->wtid)); 1405 wa->wire_off_out = dd->serial_id + 1; 1406 slen = strlen (dd->credit_account_uri.full_payto) + 1; 1407 roi = GNUNET_malloc (sizeof (struct WireTransferOutInfo) 1408 + slen); 1409 GNUNET_CRYPTO_hash (&dd->wtid, 1410 sizeof (dd->wtid), 1411 &roi->subject_hash); 1412 roi->details = *dd; 1413 roi->details.credit_account_uri.full_payto 1414 = (char *) &roi[1]; 1415 GNUNET_memcpy (&roi[1], 1416 dd->credit_account_uri.full_payto, 1417 slen); 1418 if (GNUNET_OK != 1419 GNUNET_CONTAINER_multihashmap_put (out_map, 1420 &roi->subject_hash, 1421 roi, 1422 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)) 1423 { 1424 struct TALER_AUDITORDB_WireFormatInconsistency wfi = { 1425 .amount = dd->amount, 1426 .wire_offset = dd->serial_id, 1427 .diagnostic = (char *) "duplicate outgoing wire transfer subject" 1428 }; 1429 enum GNUNET_DB_QueryStatus qs; 1430 1431 qs = TALER_AUDITORDB_insert_wire_format_inconsistency ( 1432 TALER_ARL_adb, 1433 &wfi); 1434 if (qs < 0) 1435 { 1436 global_qs = qs; 1437 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1438 commit (qs); 1439 return; 1440 } 1441 TALER_ARL_amount_add (&TALER_ARL_USE_AB ( 1442 wire_debit_duplicate_transfer_subject_total), 1443 &TALER_ARL_USE_AB ( 1444 wire_debit_duplicate_transfer_subject_total), 1445 &dd->amount); 1446 } 1447 } 1448 check_exchange_wire_out (wa); 1449 return; 1450 case MHD_HTTP_NO_CONTENT: 1451 check_exchange_wire_out (wa); 1452 return; 1453 case MHD_HTTP_NOT_FOUND: 1454 if (ignore_account_404) 1455 { 1456 check_exchange_wire_out (wa); 1457 return; 1458 } 1459 break; 1460 default: 1461 break; 1462 } 1463 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 1464 "Error fetching debit history of account %s: %u/%u!\n", 1465 wa->ai->section_name, 1466 dhr->http_status, 1467 (unsigned int) dhr->ec); 1468 commit (GNUNET_DB_STATUS_HARD_ERROR); 1469 global_ret = EXIT_FAILURE; 1470 GNUNET_SCHEDULER_shutdown (); 1471 return; 1472 } 1473 1474 1475 static void 1476 process_debits (struct WireAccount *wa) 1477 { 1478 /* skip accounts where DEBIT is not enabled */ 1479 while ( (NULL != wa) && 1480 (! wa->ai->debit_enabled) ) 1481 wa = wa->next; 1482 if (NULL == wa) 1483 { 1484 /* end of iteration */ 1485 commit (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT); 1486 return; 1487 } 1488 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1489 "Checking bank DEBIT records of account `%s'\n", 1490 wa->ai->section_name); 1491 if ( (NULL == wa->dhh) && 1492 (NULL == wa->dhh_task) ) 1493 { 1494 wa->dhh = TALER_BANK_debit_history ( 1495 ctx, 1496 wa->ai->auth, 1497 wa->wire_off_out, 1498 MAX_PER_TRANSACTION, 1499 GNUNET_TIME_UNIT_ZERO, 1500 &history_debit_cb, 1501 wa); 1502 if (NULL == wa->dhh) 1503 { 1504 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 1505 "Failed to obtain bank transaction history for `%s'\n", 1506 wa->ai->section_name); 1507 commit (GNUNET_DB_STATUS_HARD_ERROR); 1508 global_ret = EXIT_FAILURE; 1509 GNUNET_SCHEDULER_shutdown (); 1510 return; 1511 } 1512 } 1513 } 1514 1515 1516 /** 1517 * Function called about reserve closing operations the aggregator triggered. 1518 * 1519 * @param cls closure; NULL 1520 * @param rowid row identifier used to uniquely identify the reserve closing operation 1521 * @param execution_date when did we execute the close operation 1522 * @param amount_with_fee how much did we debit the reserve 1523 * @param closing_fee how much did we charge for closing the reserve 1524 * @param reserve_pub public key of the reserve 1525 * @param receiver_account where did we send the funds, in payto://-format 1526 * @param wtid identifier used for the wire transfer 1527 * @param close_request_row which close request triggered the operation? 1528 * 0 if it was a timeout (not used) 1529 * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR to stop 1530 */ 1531 static enum GNUNET_GenericReturnValue 1532 reserve_closed_cb ( 1533 void *cls, 1534 uint64_t rowid, 1535 struct GNUNET_TIME_Timestamp execution_date, 1536 const struct TALER_Amount *amount_with_fee, 1537 const struct TALER_Amount *closing_fee, 1538 const struct TALER_ReservePublicKeyP *reserve_pub, 1539 const struct TALER_FullPayto receiver_account, 1540 const struct TALER_WireTransferIdentifierRawP *wtid, 1541 uint64_t close_request_row) 1542 { 1543 struct ReserveClosure *rc; 1544 struct GNUNET_HashCode key; 1545 1546 (void) cls; 1547 (void) close_request_row; 1548 GNUNET_assert (TALER_ARL_USE_PP (wire_reserve_close_id) <= rowid); 1549 TALER_ARL_USE_PP (wire_reserve_close_id) = rowid + 1; 1550 rc = GNUNET_new (struct ReserveClosure); 1551 if (TALER_ARL_SR_INVALID_NEGATIVE == 1552 TALER_ARL_amount_subtract_neg (&rc->amount, 1553 amount_with_fee, 1554 closing_fee)) 1555 { 1556 struct TALER_AUDITORDB_RowInconsistency ri = { 1557 .row_id = rowid, 1558 .row_table 1559 = (char *) "reserves_closures", 1560 .diagnostic 1561 = (char *) "closing fee above reserve balance (and closed anyway)" 1562 }; 1563 enum GNUNET_DB_QueryStatus qs; 1564 1565 qs = TALER_AUDITORDB_insert_row_inconsistency ( 1566 TALER_ARL_adb, 1567 &ri); 1568 if (qs < 0) 1569 { 1570 global_qs = qs; 1571 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1572 return GNUNET_OK; 1573 } 1574 GNUNET_free (rc); 1575 return GNUNET_OK; 1576 } 1577 rc->receiver_account.full_payto 1578 = GNUNET_strdup (receiver_account.full_payto); 1579 rc->wtid = *wtid; 1580 rc->execution_date = execution_date; 1581 rc->rowid = rowid; 1582 hash_rc (rc->receiver_account, 1583 wtid, 1584 &key); 1585 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1586 "Discovered reserve closure %llu (%s) over %s benefiting %s\n", 1587 (unsigned long long) rowid, 1588 GNUNET_h2s (&key), 1589 TALER_amount2s (amount_with_fee), 1590 receiver_account.full_payto); 1591 (void) GNUNET_CONTAINER_multihashmap_put ( 1592 reserve_closures, 1593 &key, 1594 rc, 1595 GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE); 1596 return GNUNET_OK; 1597 } 1598 1599 1600 /** 1601 * Start the database transactions and begin the audit. 1602 * 1603 * @return transaction status code 1604 */ 1605 static enum GNUNET_DB_QueryStatus 1606 begin_transaction (void) 1607 { 1608 enum GNUNET_DB_QueryStatus qs; 1609 1610 if (GNUNET_SYSERR == 1611 TALER_EXCHANGEDB_preflight (TALER_ARL_edb)) 1612 { 1613 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 1614 "Failed to initialize exchange database connection.\n"); 1615 return GNUNET_DB_STATUS_HARD_ERROR; 1616 } 1617 if (GNUNET_SYSERR == 1618 TALER_AUDITORDB_preflight (TALER_ARL_adb)) 1619 { 1620 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 1621 "Failed to initialize auditor database session.\n"); 1622 return GNUNET_DB_STATUS_HARD_ERROR; 1623 } 1624 global_qs = GNUNET_DB_STATUS_SUCCESS_NO_RESULTS; 1625 if (GNUNET_OK != 1626 TALER_AUDITORDB_start (TALER_ARL_adb, 1627 "auditor-wire-debit")) 1628 { 1629 GNUNET_break (0); 1630 return GNUNET_DB_STATUS_HARD_ERROR; 1631 } 1632 if (GNUNET_OK != 1633 TALER_EXCHANGEDB_start_read_only (TALER_ARL_edb, 1634 "wire debit auditor")) 1635 { 1636 GNUNET_break (0); 1637 return GNUNET_DB_STATUS_HARD_ERROR; 1638 } 1639 qs = TALER_AUDITORDB_get_balance ( 1640 TALER_ARL_adb, 1641 TALER_ARL_GET_AB (total_drained), 1642 TALER_ARL_GET_AB (total_wire_out), 1643 TALER_ARL_GET_AB (total_bad_amount_out_plus), 1644 TALER_ARL_GET_AB (total_bad_amount_out_minus), 1645 TALER_ARL_GET_AB (total_closure_amount_lag), 1646 TALER_ARL_GET_AB (wire_debit_duplicate_transfer_subject_total), 1647 TALER_ARL_GET_AB (total_wire_out), 1648 NULL); 1649 switch (qs) 1650 { 1651 case GNUNET_DB_STATUS_HARD_ERROR: 1652 GNUNET_break (0); 1653 return qs; 1654 case GNUNET_DB_STATUS_SOFT_ERROR: 1655 GNUNET_break (0); 1656 return qs; 1657 case GNUNET_DB_STATUS_SUCCESS_NO_RESULTS: 1658 case GNUNET_DB_STATUS_SUCCESS_ONE_RESULT: 1659 break; 1660 } 1661 for (struct WireAccount *wa = wa_head; 1662 NULL != wa; 1663 wa = wa->next) 1664 { 1665 GNUNET_asprintf (&wa->label_wire_out_serial_id, 1666 "wire-%s-%s", 1667 wa->ai->section_name, 1668 "wire_out_serial_id"); 1669 GNUNET_asprintf (&wa->label_wire_off_out, 1670 "wire-%s-%s", 1671 wa->ai->section_name, 1672 "wire_off_out"); 1673 qs = TALER_AUDITORDB_get_auditor_progress ( 1674 TALER_ARL_adb, 1675 wa->label_wire_out_serial_id, 1676 &wa->last_wire_out_serial_id, 1677 wa->label_wire_off_out, 1678 &wa->wire_off_out, 1679 NULL); 1680 if (0 > qs) 1681 { 1682 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1683 return qs; 1684 } 1685 GNUNET_assert (2 == qs); 1686 wa->start_wire_out_serial_id = wa->last_wire_out_serial_id; 1687 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1688 "Resuming account %s debit audit at %llu/%llu\n", 1689 wa->ai->section_name, 1690 (unsigned long long) wa->last_wire_out_serial_id, 1691 (unsigned long long) wa->wire_off_out); 1692 } 1693 qs = TALER_AUDITORDB_get_auditor_progress ( 1694 TALER_ARL_adb, 1695 TALER_ARL_GET_PP (wire_reserve_close_id), 1696 NULL); 1697 if (0 > qs) 1698 { 1699 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1700 return qs; 1701 } 1702 if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) 1703 { 1704 GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE, 1705 "First analysis of with wire auditor, starting audit from scratch\n"); 1706 } 1707 else 1708 { 1709 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1710 "Resuming wire debit audit at %llu\n", 1711 (unsigned long long) TALER_ARL_USE_PP (wire_reserve_close_id)); 1712 } 1713 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1714 "Iterating over reserve closures from %llu\n", 1715 (unsigned long long) TALER_ARL_USE_PP (wire_reserve_close_id)); 1716 qs = TALER_EXCHANGEDB_iterate_reserve_closed_above_serial_id ( 1717 TALER_ARL_edb, 1718 TALER_ARL_USE_PP (wire_reserve_close_id), 1719 &reserve_closed_cb, 1720 NULL); 1721 if (0 > qs) 1722 { 1723 GNUNET_break (GNUNET_DB_STATUS_HARD_ERROR == qs); 1724 return GNUNET_DB_STATUS_HARD_ERROR; 1725 } 1726 process_debits (wa_head); 1727 return GNUNET_DB_STATUS_SUCCESS_NO_RESULTS; 1728 } 1729 1730 1731 /** 1732 * Function called with information about a wire account. Adds the 1733 * account to our list for processing (if it is enabled and we can 1734 * load the plugin). 1735 * 1736 * @param cls closure, NULL 1737 * @param ai account information 1738 */ 1739 static void 1740 process_account_cb (void *cls, 1741 const struct TALER_EXCHANGEDB_AccountInfo *ai) 1742 { 1743 struct WireAccount *wa; 1744 1745 (void) cls; 1746 if ( (! ai->debit_enabled) && 1747 (! ai->credit_enabled) ) 1748 return; /* not an active exchange account */ 1749 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1750 "Found exchange account `%s'\n", 1751 ai->section_name); 1752 wa = GNUNET_new (struct WireAccount); 1753 wa->ai = ai; 1754 GNUNET_CONTAINER_DLL_insert (wa_head, 1755 wa_tail, 1756 wa); 1757 } 1758 1759 1760 /** 1761 * Function called on events received from Postgres. 1762 * 1763 * @param cls closure, NULL 1764 * @param extra additional event data provided 1765 * @param extra_size number of bytes in @a extra 1766 */ 1767 static void 1768 db_notify (void *cls, 1769 const void *extra, 1770 size_t extra_size) 1771 { 1772 (void) cls; 1773 (void) extra; 1774 (void) extra_size; 1775 1776 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1777 "Received notification to wake wire helper\n"); 1778 /* If there are accounts we are still processing, abort 1779 the HTTP requests so we can start afresh. */ 1780 for (struct WireAccount *wa = wa_head; 1781 NULL != wa; 1782 wa = wa->next) 1783 { 1784 if (NULL != wa->dhh) 1785 { 1786 TALER_BANK_debit_history_cancel (wa->dhh); 1787 wa->dhh = NULL; 1788 } 1789 check_exchange_wire_out (wa); 1790 } 1791 1792 if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS != 1793 begin_transaction ()) 1794 { 1795 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 1796 "Audit failed\n"); 1797 GNUNET_break (0); 1798 global_ret = EXIT_FAILURE; 1799 GNUNET_SCHEDULER_shutdown (); 1800 return; 1801 } 1802 } 1803 1804 1805 /** 1806 * Main function that will be run. 1807 * 1808 * @param cls closure 1809 * @param args remaining command-line arguments 1810 * @param cfgfile name of the configuration file used (for saving, can be NULL!) 1811 * @param c configuration 1812 */ 1813 static void 1814 run (void *cls, 1815 char *const *args, 1816 const char *cfgfile, 1817 const struct GNUNET_CONFIGURATION_Handle *c) 1818 { 1819 (void) cls; 1820 (void) args; 1821 (void) cfgfile; 1822 cfg = c; 1823 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 1824 "Launching wire debit auditor\n"); 1825 if (GNUNET_OK != 1826 TALER_ARL_init (c)) 1827 { 1828 global_ret = EXIT_FAILURE; 1829 return; 1830 } 1831 1832 reserve_closures 1833 = GNUNET_CONTAINER_multihashmap_create (1024, 1834 GNUNET_NO); 1835 GNUNET_assert (GNUNET_OK == 1836 TALER_amount_set_zero (TALER_ARL_currency, 1837 &zero)); 1838 GNUNET_SCHEDULER_add_shutdown (&do_shutdown, 1839 NULL); 1840 ctx = GNUNET_CURL_init (&GNUNET_CURL_gnunet_scheduler_reschedule, 1841 &rctx); 1842 rctx = GNUNET_CURL_gnunet_rc_create (ctx); 1843 if (NULL == ctx) 1844 { 1845 GNUNET_break (0); 1846 global_ret = EXIT_FAILURE; 1847 return; 1848 } 1849 reserve_closures = GNUNET_CONTAINER_multihashmap_create (1024, 1850 GNUNET_NO); 1851 out_map = GNUNET_CONTAINER_multihashmap_create (1024, 1852 true); 1853 if (GNUNET_OK != 1854 TALER_EXCHANGEDB_load_accounts (TALER_ARL_cfg, 1855 TALER_EXCHANGEDB_ALO_DEBIT 1856 | TALER_EXCHANGEDB_ALO_CREDIT 1857 | TALER_EXCHANGEDB_ALO_AUTHDATA)) 1858 { 1859 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 1860 "No bank accounts configured\n"); 1861 global_ret = EXIT_NOTCONFIGURED; 1862 GNUNET_SCHEDULER_shutdown (); 1863 return; 1864 } 1865 TALER_EXCHANGEDB_find_accounts (&process_account_cb, 1866 NULL); 1867 1868 if (0 == test_mode) 1869 { 1870 struct GNUNET_DB_EventHeaderP es = { 1871 .size = htons (sizeof (es)), 1872 .type = htons (TALER_DBEVENT_EXCHANGE_AUDITOR_WAKE_HELPER_WIRE) 1873 }; 1874 1875 eh = TALER_AUDITORDB_event_listen (TALER_ARL_adb, 1876 &es, 1877 GNUNET_TIME_UNIT_FOREVER_REL, 1878 &db_notify, 1879 NULL); 1880 GNUNET_assert (NULL != eh); 1881 } 1882 if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS != 1883 begin_transaction ()) 1884 { 1885 GNUNET_break (0); 1886 global_ret = EXIT_FAILURE; 1887 GNUNET_SCHEDULER_shutdown (); 1888 return; 1889 } 1890 } 1891 1892 1893 /** 1894 * The main function of the wire auditing tool. Checks that 1895 * the exchange's records of wire transfers match that of 1896 * the wire gateway. 1897 * 1898 * @param argc number of arguments from the command line 1899 * @param argv command line arguments 1900 * @return 0 ok, 1 on error 1901 */ 1902 int 1903 main (int argc, 1904 char *const *argv) 1905 { 1906 const struct GNUNET_GETOPT_CommandLineOption options[] = { 1907 GNUNET_GETOPT_option_flag ('i', 1908 "internal", 1909 "perform checks only applicable for exchange-internal audits", 1910 &internal_checks), 1911 GNUNET_GETOPT_option_flag ('I', 1912 "ignore-not-found", 1913 "continue, even if the bank account of the exchange was not found", 1914 &ignore_account_404), 1915 GNUNET_GETOPT_option_flag ('t', 1916 "test", 1917 "run in test mode and exit when idle", 1918 &test_mode), 1919 GNUNET_GETOPT_option_timetravel ('T', 1920 "timetravel"), 1921 GNUNET_GETOPT_OPTION_END 1922 }; 1923 enum GNUNET_GenericReturnValue ret; 1924 1925 ret = GNUNET_PROGRAM_run ( 1926 TALER_AUDITOR_project_data (), 1927 argc, 1928 argv, 1929 "taler-helper-auditor-wire-debit", 1930 gettext_noop ( 1931 "Audit exchange database for consistency with the bank's outgoing wire transfers"), 1932 options, 1933 &run, 1934 NULL); 1935 if (GNUNET_SYSERR == ret) 1936 return EXIT_INVALIDARGUMENT; 1937 if (GNUNET_NO == ret) 1938 return EXIT_SUCCESS; 1939 return global_ret; 1940 } 1941 1942 1943 /* end of taler-helper-auditor-wire-debit.c */