taler-helper-auditor-wire-debit.c (57419B)
1 /* 2 This file is part of TALER 3 Copyright (C) 2017-2024 Taler Systems SA 4 5 TALER is free software; you can redistribute it and/or modify it under the 6 terms of the GNU General Public License as published by the Free Software 7 Foundation; either version 3, or (at your option) any later version. 8 9 TALER is distributed in the hope that it will be useful, but WITHOUT ANY 10 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR 11 A PARTICULAR PURPOSE. See the GNU General Public License for more details. 12 13 You should have received a copy of the GNU General Public License along with 14 TALER; see the file COPYING. If not, see <http://www.gnu.org/licenses/> 15 */ 16 /** 17 * @file auditor/taler-helper-auditor-wire-debit.c 18 * @brief audits that wire outgoing transfers match those from an exchange 19 * database. 20 * @author Christian Grothoff 21 * @author Özgür Kesim 22 * 23 * - We check that the outgoing wire transfers match those 24 * given in the 'wire_out' and 'reserve_closures' tables; 25 * any outgoing transfer MUST have a prior justification, 26 * so if one is missing we flag it (and never remove it). 27 * - We check that all wire transfers that should 28 * have been made, were actually made. If any were not made, 29 * we flag those, but may remove those flags if we later 30 * find that the wire transfers were made (wire transfers 31 * could be delayed due to AML/KYC or core-banking issues). 32 */ 33 #include "platform.h" 34 #include <gnunet/gnunet_util_lib.h> 35 #include <gnunet/gnunet_curl_lib.h> 36 #include "auditordb_lib.h" 37 #include "exchangedb_lib.h" 38 #include "taler/taler_json_lib.h" 39 #include "taler/taler_bank_service.h" 40 #include "taler/taler_signatures.h" 41 #include "report-lib.h" 42 #include "taler/taler_dbevents.h" 43 #include "auditor-database/delete_auditor_closure_lag.h" 44 #include "auditor-database/delete_wire_out_inconsistency_if_matching.h" 45 #include "auditor-database/event_listen.h" 46 #include "auditor-database/get_auditor_progress.h" 47 #include "auditor-database/get_balance.h" 48 #include "auditor-database/insert_auditor_closure_lags.h" 49 #include "auditor-database/insert_auditor_progress.h" 50 #include "auditor-database/insert_balance.h" 51 #include "auditor-database/insert_row_inconsistency.h" 52 #include "auditor-database/insert_row_minor_inconsistencies.h" 53 #include "auditor-database/insert_wire_format_inconsistency.h" 54 #include "auditor-database/insert_wire_out_inconsistency.h" 55 #include "auditor-database/preflight.h" 56 #include "auditor-database/start.h" 57 #include "auditor-database/update_auditor_progress.h" 58 #include "auditor-database/update_balance.h" 59 #include "exchange-database/get_drain_profit.h" 60 #include "exchange-database/preflight.h" 61 #include "exchange-database/rollback.h" 62 #include "exchange-database/select_reserve_closed_above_serial_id.h" 63 #include "exchange-database/select_wire_out_above_serial_id_by_account.h" 64 #include "exchange-database/start_read_only.h" 65 66 67 /** 68 * Maximum number of wire transfers we process per 69 * (database) transaction. 70 */ 71 #define MAX_PER_TRANSACTION 1024 72 73 /** 74 * How much do we allow the bank and the exchange to disagree about 75 * timestamps? Should be sufficiently large to avoid bogus reports from deltas 76 * created by imperfect clock synchronization and network delay. 77 */ 78 #define TIME_TOLERANCE GNUNET_TIME_relative_multiply ( \ 79 GNUNET_TIME_UNIT_MINUTES, \ 80 15) 81 82 83 /** 84 * How long do we try to long-poll for bank wire transfers? 85 */ 86 #define MAX_LONGPOLL_DELAY GNUNET_TIME_relative_multiply ( \ 87 GNUNET_TIME_UNIT_HOURS, \ 88 1) 89 90 91 /** 92 * How long do we wait between polling for bank wire transfers at the minimum? 93 */ 94 #define MIN_LONGPOLL_DELAY GNUNET_TIME_relative_multiply ( \ 95 GNUNET_TIME_UNIT_MINUTES, \ 96 5) 97 98 99 /** 100 * Run in test mode. Exit when idle instead of 101 * going to sleep and waiting for more work. 102 */ 103 static int test_mode; 104 105 106 /** 107 * Information we keep for each supported account. 108 */ 109 struct WireAccount 110 { 111 /** 112 * Accounts are kept in a DLL. 113 */ 114 struct WireAccount *next; 115 116 /** 117 * Plugins are kept in a DLL. 118 */ 119 struct WireAccount *prev; 120 121 /** 122 * Account details. 123 */ 124 const struct TALER_EXCHANGEDB_AccountInfo *ai; 125 126 /** 127 * Active wire request for the transaction history. 128 */ 129 struct TALER_BANK_DebitHistoryHandle *dhh; 130 131 /** 132 * Task to trigger @e dhh long-polling. 133 */ 134 struct GNUNET_SCHEDULER_Task *dhh_task; 135 136 /** 137 * Time when we expect the current @e dhh long-poll 138 * to finish and we thus could begin another one. 139 */ 140 struct GNUNET_TIME_Absolute dhh_next; 141 142 /** 143 * Progress point for this account. 144 */ 145 uint64_t last_wire_out_serial_id; 146 147 /** 148 * Initial progress point for this account. 149 */ 150 uint64_t start_wire_out_serial_id; 151 152 /** 153 * Where we are in the outbound transaction history. 154 */ 155 uint64_t wire_off_out; 156 157 /** 158 * Label under which we store our pp's reserve_in_serial_id. 159 */ 160 char *label_wire_out_serial_id; 161 162 /** 163 * Label under which we store our wire_off_out. 164 */ 165 char *label_wire_off_out; 166 }; 167 168 169 /** 170 * Information we track for a reserve being closed. 171 */ 172 struct ReserveClosure 173 { 174 /** 175 * Row in the reserves_closed table for this action. 176 */ 177 uint64_t rowid; 178 179 /** 180 * When was the reserve closed? 181 */ 182 struct GNUNET_TIME_Timestamp execution_date; 183 184 /** 185 * Amount transferred (amount remaining minus fee). 186 */ 187 struct TALER_Amount amount; 188 189 /** 190 * Target account where the money was sent. 191 */ 192 struct TALER_FullPayto receiver_account; 193 194 /** 195 * Wire transfer subject used. 196 */ 197 struct TALER_WireTransferIdentifierRawP wtid; 198 }; 199 200 201 /** 202 * Map from H(wtid,receiver_account) to `struct ReserveClosure` entries. 203 */ 204 static struct GNUNET_CONTAINER_MultiHashMap *reserve_closures; 205 206 /** 207 * Return value from main(). 208 */ 209 static int global_ret; 210 211 /** 212 * State of the current database transaction with 213 * the auditor DB. 214 */ 215 static enum GNUNET_DB_QueryStatus global_qs; 216 217 /** 218 * Map with information about outgoing wire transfers. 219 * Maps hashes of the wire subjects (in binary encoding) 220 * to `struct ReserveOutInfo`s. 221 */ 222 static struct GNUNET_CONTAINER_MultiHashMap *out_map; 223 224 /** 225 * Head of list of wire accounts we still need to look at. 226 */ 227 static struct WireAccount *wa_head; 228 229 /** 230 * Tail of list of wire accounts we still need to look at. 231 */ 232 static struct WireAccount *wa_tail; 233 234 /** 235 * Last reserve_out / wire_out serial IDs seen. 236 */ 237 static TALER_ARL_DEF_PP (wire_reserve_close_id); 238 239 /** 240 * Total amount that was transferred too much from the exchange. 241 */ 242 static TALER_ARL_DEF_AB (total_bad_amount_out_plus); 243 244 /** 245 * Total amount that was transferred too little from the exchange. 246 */ 247 static TALER_ARL_DEF_AB (total_bad_amount_out_minus); 248 249 /** 250 * Total amount of reserve closures which the exchange did not transfer in time. 251 */ 252 static TALER_ARL_DEF_AB (total_closure_amount_lag); 253 254 /** 255 * Total amount affected by duplicate wire transfer 256 * subjects. 257 */ 258 static TALER_ARL_DEF_AB (wire_debit_duplicate_transfer_subject_total); 259 260 /** 261 * Total amount debited to exchange accounts. 262 */ 263 static TALER_ARL_DEF_AB (total_wire_out); 264 265 /** 266 * Total amount of profits drained. 267 */ 268 static TALER_ARL_DEF_AB (total_drained); 269 270 /** 271 * Amount of zero in our currency. 272 */ 273 static struct TALER_Amount zero; 274 275 /** 276 * Handle to the context for interacting with the bank. 277 */ 278 static struct GNUNET_CURL_Context *ctx; 279 280 /** 281 * Scheduler context for running the @e ctx. 282 */ 283 static struct GNUNET_CURL_RescheduleContext *rctx; 284 285 /** 286 * Should we run checks that only work for exchange-internal audits? 287 */ 288 static int internal_checks; 289 290 /** 291 * Should we ignore if the bank does not know our bank 292 * account? 293 */ 294 static int ignore_account_404; 295 296 /** 297 * Database event handler to wake us up again. 298 */ 299 static struct GNUNET_DB_EventHandler *eh; 300 301 /** 302 * The auditors's configuration. 303 */ 304 static const struct GNUNET_CONFIGURATION_Handle *cfg; 305 306 307 /** 308 * Entry in map with wire information we expect to obtain from the 309 * #TALER_ARL_edb later. 310 */ 311 struct WireTransferOutInfo 312 { 313 314 /** 315 * Hash of the wire transfer subject. 316 */ 317 struct GNUNET_HashCode subject_hash; 318 319 /** 320 * Expected details about the wire transfer. 321 */ 322 struct TALER_BANK_DebitDetails details; 323 324 }; 325 326 327 /** 328 * Free entry in #out_map. 329 * 330 * @param cls NULL 331 * @param key unused key 332 * @param value the `struct WireTransferOutInfo` to free 333 * @return #GNUNET_OK 334 */ 335 static enum GNUNET_GenericReturnValue 336 free_roi (void *cls, 337 const struct GNUNET_HashCode *key, 338 void *value) 339 { 340 struct WireTransferOutInfo *roi = value; 341 342 (void) cls; 343 GNUNET_assert (GNUNET_YES == 344 GNUNET_CONTAINER_multihashmap_remove (out_map, 345 key, 346 roi)); 347 GNUNET_free (roi); 348 return GNUNET_OK; 349 } 350 351 352 /** 353 * Free entry in #reserve_closures. 354 * 355 * @param cls NULL 356 * @param key unused key 357 * @param value the `struct ReserveClosure` to free 358 * @return #GNUNET_OK 359 */ 360 static enum GNUNET_GenericReturnValue 361 free_rc (void *cls, 362 const struct GNUNET_HashCode *key, 363 void *value) 364 { 365 struct ReserveClosure *rc = value; 366 367 (void) cls; 368 GNUNET_assert (GNUNET_YES == 369 GNUNET_CONTAINER_multihashmap_remove (reserve_closures, 370 key, 371 rc)); 372 GNUNET_free (rc->receiver_account.full_payto); 373 GNUNET_free (rc); 374 return GNUNET_OK; 375 } 376 377 378 /** 379 * Task run on shutdown. 380 * 381 * @param cls NULL 382 */ 383 static void 384 do_shutdown (void *cls) 385 { 386 struct WireAccount *wa; 387 388 (void) cls; 389 if (NULL != eh) 390 { 391 TALER_AUDITORDB_event_listen_cancel (eh); 392 eh = NULL; 393 } 394 TALER_ARL_done (); 395 if (NULL != reserve_closures) 396 { 397 GNUNET_CONTAINER_multihashmap_iterate (reserve_closures, 398 &free_rc, 399 NULL); 400 GNUNET_CONTAINER_multihashmap_destroy (reserve_closures); 401 reserve_closures = NULL; 402 } 403 if (NULL != out_map) 404 { 405 GNUNET_CONTAINER_multihashmap_iterate (out_map, 406 &free_roi, 407 NULL); 408 GNUNET_CONTAINER_multihashmap_destroy (out_map); 409 out_map = NULL; 410 } 411 while (NULL != (wa = wa_head)) 412 { 413 if (NULL != wa->dhh_task) 414 { 415 GNUNET_SCHEDULER_cancel (wa->dhh_task); 416 wa->dhh_task = NULL; 417 } 418 if (NULL != wa->dhh) 419 { 420 TALER_BANK_debit_history_cancel (wa->dhh); 421 wa->dhh = NULL; 422 } 423 GNUNET_CONTAINER_DLL_remove (wa_head, 424 wa_tail, 425 wa); 426 GNUNET_free (wa->label_wire_out_serial_id); 427 GNUNET_free (wa->label_wire_off_out); 428 GNUNET_free (wa); 429 } 430 if (NULL != ctx) 431 { 432 GNUNET_CURL_fini (ctx); 433 ctx = NULL; 434 } 435 if (NULL != rctx) 436 { 437 GNUNET_CURL_gnunet_rc_destroy (rctx); 438 rctx = NULL; 439 } 440 TALER_EXCHANGEDB_unload_accounts (); 441 TALER_ARL_cfg = NULL; 442 } 443 444 445 /** 446 * Detect any entries in #reserve_closures that were not yet 447 * observed on the wire transfer side and update the progress 448 * point accordingly. 449 * 450 * @param cls NULL 451 * @param key unused key 452 * @param value the `struct ReserveClosure` to free 453 * @return #GNUNET_OK 454 */ 455 static enum GNUNET_GenericReturnValue 456 check_pending_rc (void *cls, 457 const struct GNUNET_HashCode *key, 458 void *value) 459 { 460 struct ReserveClosure *rc = value; 461 462 (void) cls; 463 (void) key; 464 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 465 "Missing wire transfer for closed reserve with balance %s\n", 466 TALER_amount2s (&rc->amount)); 467 if (! TALER_amount_is_zero (&rc->amount)) 468 { 469 struct TALER_AUDITORDB_ClosureLags cl = { 470 .problem_row_id = rc->rowid, 471 .account = rc->receiver_account, 472 .amount = rc->amount, 473 .deadline = rc->execution_date.abs_time, 474 .wtid = rc->wtid 475 }; 476 enum GNUNET_DB_QueryStatus qs; 477 478 qs = TALER_AUDITORDB_insert_auditor_closure_lags ( 479 TALER_ARL_adb, 480 &cl); 481 if (qs < 0) 482 { 483 global_qs = qs; 484 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 485 return GNUNET_SYSERR; 486 } 487 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_closure_amount_lag), 488 &TALER_ARL_USE_AB (total_closure_amount_lag), 489 &rc->amount); 490 } 491 return GNUNET_OK; 492 } 493 494 495 /** 496 * Compute the key under which a reserve closure for a given 497 * @a receiver_account and @a wtid would be stored. 498 * 499 * @param receiver_account payto://-URI of the account 500 * @param wtid wire transfer identifier used 501 * @param[out] key set to the key 502 */ 503 static void 504 hash_rc (const struct TALER_FullPayto receiver_account, 505 const struct TALER_WireTransferIdentifierRawP *wtid, 506 struct GNUNET_HashCode *key) 507 { 508 struct TALER_NormalizedPayto npto 509 = TALER_payto_normalize (receiver_account); 510 size_t slen = strlen (npto.normalized_payto); 511 char buf[sizeof (struct TALER_WireTransferIdentifierRawP) + slen]; 512 513 GNUNET_memcpy (buf, 514 wtid, 515 sizeof (*wtid)); 516 GNUNET_memcpy (&buf[sizeof (*wtid)], 517 npto.normalized_payto, 518 slen); 519 GNUNET_CRYPTO_hash (buf, 520 sizeof (buf), 521 key); 522 GNUNET_free (npto.normalized_payto); 523 } 524 525 526 /** 527 * Start the database transactions and begin the audit. 528 * 529 * @return transaction status code 530 */ 531 static enum GNUNET_DB_QueryStatus 532 begin_transaction (void); 533 534 535 /** 536 * Commit the transaction, checkpointing our progress in the auditor DB. 537 * 538 * @param qs transaction status so far 539 */ 540 static void 541 commit (enum GNUNET_DB_QueryStatus qs) 542 { 543 GNUNET_CONTAINER_multihashmap_iterate (reserve_closures, 544 &check_pending_rc, 545 NULL); 546 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 547 "Transaction logic ended with status %d\n", 548 qs); 549 TALER_EXCHANGEDB_rollback (TALER_ARL_edb); 550 qs = TALER_AUDITORDB_update_balance ( 551 TALER_ARL_adb, 552 TALER_ARL_SET_AB (total_drained), 553 TALER_ARL_SET_AB (total_wire_out), 554 TALER_ARL_SET_AB (total_bad_amount_out_plus), 555 TALER_ARL_SET_AB (total_bad_amount_out_minus), 556 TALER_ARL_SET_AB (total_closure_amount_lag), 557 TALER_ARL_SET_AB (wire_debit_duplicate_transfer_subject_total), 558 TALER_ARL_SET_AB (total_wire_out), 559 NULL); 560 if (0 > qs) 561 goto handle_db_error; 562 qs = TALER_AUDITORDB_insert_balance ( 563 TALER_ARL_adb, 564 TALER_ARL_SET_AB (total_drained), 565 TALER_ARL_SET_AB (total_wire_out), 566 TALER_ARL_SET_AB (total_bad_amount_out_plus), 567 TALER_ARL_SET_AB (total_bad_amount_out_minus), 568 TALER_ARL_SET_AB (total_closure_amount_lag), 569 TALER_ARL_SET_AB (wire_debit_duplicate_transfer_subject_total), 570 TALER_ARL_SET_AB (total_wire_out), 571 NULL); 572 if (0 > qs) 573 goto handle_db_error; 574 for (struct WireAccount *wa = wa_head; 575 NULL != wa; 576 wa = wa->next) 577 { 578 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 579 "Transaction of account %s ends at %llu/%llu\n", 580 wa->ai->section_name, 581 (unsigned long long) wa->last_wire_out_serial_id, 582 (unsigned long long) wa->wire_off_out); 583 qs = TALER_AUDITORDB_update_auditor_progress ( 584 TALER_ARL_adb, 585 wa->label_wire_out_serial_id, 586 wa->last_wire_out_serial_id, 587 wa->label_wire_off_out, 588 wa->wire_off_out, 589 NULL); 590 if (0 > qs) 591 goto handle_db_error; 592 qs = TALER_AUDITORDB_insert_auditor_progress ( 593 TALER_ARL_adb, 594 wa->label_wire_out_serial_id, 595 wa->last_wire_out_serial_id, 596 wa->label_wire_off_out, 597 wa->wire_off_out, 598 NULL); 599 if (0 > qs) 600 goto handle_db_error; 601 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 602 "Transaction ends at %s=%llu for account `%s'\n", 603 wa->label_wire_out_serial_id, 604 (unsigned long long) wa->last_wire_out_serial_id, 605 wa->ai->section_name); 606 } 607 qs = TALER_AUDITORDB_update_auditor_progress ( 608 TALER_ARL_adb, 609 TALER_ARL_SET_PP (wire_reserve_close_id), 610 NULL); 611 if (0 > qs) 612 goto handle_db_error; 613 qs = TALER_AUDITORDB_insert_auditor_progress ( 614 TALER_ARL_adb, 615 TALER_ARL_SET_PP (wire_reserve_close_id), 616 NULL); 617 if (0 > qs) 618 goto handle_db_error; 619 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 620 "Concluded audit step at %llu\n", 621 (unsigned long long) TALER_ARL_USE_PP (wire_reserve_close_id)); 622 qs = TALER_AUDITORDB_commit (TALER_ARL_adb); 623 if (0 > qs) 624 goto handle_db_error; 625 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 626 "Transaction concluded!\n"); 627 if (1 == test_mode) 628 GNUNET_SCHEDULER_shutdown (); 629 return; 630 handle_db_error: 631 TALER_AUDITORDB_rollback (TALER_ARL_adb); 632 for (unsigned int max_retries = 3; max_retries>0; max_retries--) 633 { 634 if (GNUNET_DB_STATUS_HARD_ERROR == qs) 635 break; 636 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 637 "Serialization issue, trying again\n"); 638 qs = begin_transaction (); 639 } 640 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 641 "Hard database error, terminating\n"); 642 GNUNET_SCHEDULER_shutdown (); 643 } 644 645 646 /** 647 * Check that @a want is within #TIME_TOLERANCE of @a have. 648 * Otherwise report an inconsistency in row @a rowid of @a table. 649 * 650 * @param table where is the inconsistency (if any) 651 * @param rowid what is the row 652 * @param want what is the expected time 653 * @param have what is the time we got 654 * @return true on success, false to abort 655 */ 656 static bool 657 check_time_difference (const char *table, 658 uint64_t rowid, 659 struct GNUNET_TIME_Timestamp want, 660 struct GNUNET_TIME_Timestamp have) 661 { 662 struct GNUNET_TIME_Relative delta; 663 char *details; 664 665 if (GNUNET_TIME_timestamp_cmp (have, >, want)) 666 delta = GNUNET_TIME_absolute_get_difference (want.abs_time, 667 have.abs_time); 668 else 669 delta = GNUNET_TIME_absolute_get_difference (have.abs_time, 670 want.abs_time); 671 if (GNUNET_TIME_relative_cmp (delta, 672 <=, 673 TIME_TOLERANCE)) 674 return true; 675 676 GNUNET_asprintf (&details, 677 "execution date mismatch (%s)", 678 GNUNET_TIME_relative2s (delta, 679 true)); 680 { 681 struct TALER_AUDITORDB_RowMinorInconsistencies rmi = { 682 .row_table = (char *) table, 683 .problem_row = rowid, 684 .diagnostic = details 685 }; 686 enum GNUNET_DB_QueryStatus qs; 687 688 qs = TALER_AUDITORDB_insert_row_minor_inconsistencies ( 689 TALER_ARL_adb, 690 &rmi); 691 692 if (qs < 0) 693 { 694 global_qs = qs; 695 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 696 GNUNET_free (details); 697 return false; 698 } 699 } 700 GNUNET_free (details); 701 return true; 702 } 703 704 705 /** 706 * Closure for #check_rc_matches 707 */ 708 struct CheckMatchContext 709 { 710 711 /** 712 * Reserve operation looking for a match 713 */ 714 const struct WireTransferOutInfo *roi; 715 716 /** 717 * Set to true if we found a match. 718 */ 719 bool found; 720 }; 721 722 723 /** 724 * Check if any of the reserve closures match the given wire transfer. 725 * 726 * @param[in,out] cls a `struct CheckMatchContext` 727 * @param key key of @a value in #reserve_closures 728 * @param value a `struct ReserveClosure` 729 */ 730 static enum GNUNET_GenericReturnValue 731 check_rc_matches (void *cls, 732 const struct GNUNET_HashCode *key, 733 void *value) 734 { 735 struct CheckMatchContext *cmx = cls; 736 struct ReserveClosure *rc = value; 737 738 if ((0 == GNUNET_memcmp (&cmx->roi->details.wtid, 739 &rc->wtid)) && 740 (0 == TALER_full_payto_cmp (rc->receiver_account, 741 cmx->roi->details.credit_account_uri)) && 742 (0 == TALER_amount_cmp (&rc->amount, 743 &cmx->roi->details.amount))) 744 { 745 if (! check_time_difference ("reserves_closures", 746 rc->rowid, 747 rc->execution_date, 748 cmx->roi->details.execution_date)) 749 { 750 free_rc (NULL, 751 key, 752 rc); 753 return GNUNET_SYSERR; 754 } 755 cmx->found = true; 756 free_rc (NULL, 757 key, 758 rc); 759 return GNUNET_NO; 760 } 761 return GNUNET_OK; 762 } 763 764 765 /** 766 * Maximum required length for make_missing_diag(). 767 */ 768 #define MAX_DIAG_LEN 128 769 770 /** 771 * Make diagnostic string for missing wire transfer. 772 * 773 * @param[out] diag where to write the diagnostic string 774 * @param wtid wire transfer ID to include 775 */ 776 static void 777 make_missing_diag (char diag[MAX_DIAG_LEN], 778 const struct TALER_WireTransferIdentifierRawP *wtid) 779 { 780 char *wtid_s; 781 782 wtid_s = GNUNET_STRINGS_data_to_string_alloc (wtid, 783 sizeof (*wtid)); 784 GNUNET_snprintf (diag, 785 MAX_DIAG_LEN, 786 "expected outgoing wire transfer %s missing", 787 wtid_s); 788 GNUNET_free (wtid_s); 789 } 790 791 792 /** 793 * Check if an existing report on a missing wire 794 * out operation justified the @a roi. If so, 795 * clear the existing report. 796 * 797 * @param roi reserve out operation to check 798 * @return #GNUNET_YES if @a roi was justified by a previous report, 799 * #GNUNET_NO of @a roi was not justified by a previous missing report 800 * #GNUNET_SYSERR on database trouble 801 */ 802 static enum GNUNET_GenericReturnValue 803 check_reported_inconsistency (struct WireTransferOutInfo *roi) 804 { 805 char diag[MAX_DIAG_LEN]; 806 struct TALER_AUDITORDB_WireOutInconsistency woi = { 807 .wire_out_row_id = roi->details.serial_id, 808 .destination_account = roi->details.credit_account_uri, 809 .diagnostic = diag, 810 .expected = roi->details.amount, 811 .claimed = zero, 812 }; 813 enum GNUNET_DB_QueryStatus qs; 814 815 make_missing_diag (diag, 816 &roi->details.wtid); 817 qs = TALER_AUDITORDB_delete_wire_out_inconsistency_if_matching ( 818 TALER_ARL_adb, 819 &woi); 820 if (qs < 0) 821 { 822 global_qs = qs; 823 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 824 return GNUNET_SYSERR; 825 } 826 if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) 827 { 828 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, 829 "Deletion of wire out inconsistency %llu (%s, %s, %s) failed: not reported missing!\n", 830 (unsigned long long) roi->details.serial_id, 831 roi->details.credit_account_uri.full_payto, 832 diag, 833 TALER_amount2s (&roi->details.amount)); 834 return GNUNET_NO; 835 } 836 TALER_ARL_amount_subtract (&TALER_ARL_USE_AB (total_bad_amount_out_minus), 837 &TALER_ARL_USE_AB (total_bad_amount_out_minus), 838 &roi->details.amount); 839 return GNUNET_YES; 840 } 841 842 843 /** 844 * Check if a profit drain operation justified the @a roi 845 * 846 * @param roi reserve out operation to check 847 * @return #GNUNET_YES if @a roi was justified by a profit drain, 848 * #GNUNET_NO of @a roi was not justified by a proft drain 849 * #GNUNET_SYSERR on database trouble 850 */ 851 static enum GNUNET_GenericReturnValue 852 check_profit_drain (struct WireTransferOutInfo *roi) 853 { 854 enum GNUNET_DB_QueryStatus qs; 855 uint64_t serial; 856 char *account_section; 857 struct TALER_FullPayto payto_uri; 858 struct GNUNET_TIME_Timestamp request_timestamp; 859 struct TALER_Amount amount; 860 struct TALER_MasterSignatureP master_sig; 861 862 qs = TALER_EXCHANGEDB_get_drain_profit ( 863 TALER_ARL_edb, 864 &roi->details.wtid, 865 &serial, 866 &account_section, 867 &payto_uri, 868 &request_timestamp, 869 &amount, 870 &master_sig); 871 switch (qs) 872 { 873 case GNUNET_DB_STATUS_HARD_ERROR: 874 GNUNET_break (0); 875 global_ret = EXIT_FAILURE; 876 GNUNET_SCHEDULER_shutdown (); 877 return GNUNET_SYSERR; 878 case GNUNET_DB_STATUS_SOFT_ERROR: 879 /* should fail on commit later ... */ 880 GNUNET_break (0); 881 return GNUNET_SYSERR; 882 case GNUNET_DB_STATUS_SUCCESS_NO_RESULTS: 883 /* not a profit drain */ 884 return GNUNET_NO; 885 case GNUNET_DB_STATUS_SUCCESS_ONE_RESULT: 886 break; 887 } 888 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 889 "Profit drain of %s to %s found!\n", 890 TALER_amount2s (&amount), 891 payto_uri.full_payto); 892 if (GNUNET_OK != 893 TALER_exchange_offline_profit_drain_verify ( 894 &roi->details.wtid, 895 request_timestamp, 896 &amount, 897 account_section, 898 payto_uri, 899 &TALER_ARL_master_pub, 900 &master_sig)) 901 { 902 struct TALER_AUDITORDB_RowInconsistency ri = { 903 .row_id = roi->details.serial_id, 904 .row_table = (char *) "profit_drains", 905 .diagnostic = (char *) "invalid signature" 906 }; 907 908 GNUNET_break (0); 909 qs = TALER_AUDITORDB_insert_row_inconsistency ( 910 TALER_ARL_adb, 911 &ri); 912 GNUNET_free (payto_uri.full_payto); 913 GNUNET_free (account_section); 914 if (qs < 0) 915 { 916 global_qs = qs; 917 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 918 return GNUNET_SYSERR; 919 } 920 return GNUNET_NO; 921 } 922 GNUNET_free (account_section); 923 924 { 925 if (0 != 926 TALER_full_payto_normalize_and_cmp (payto_uri, 927 roi->details.credit_account_uri)) 928 { 929 struct TALER_AUDITORDB_WireOutInconsistency woi = { 930 .wire_out_row_id = serial, 931 .destination_account = roi->details.credit_account_uri, 932 .diagnostic = (char *) "profit drain wired to invalid account", 933 .expected = roi->details.amount, 934 .claimed = zero, 935 }; 936 937 qs = TALER_AUDITORDB_insert_wire_out_inconsistency ( 938 TALER_ARL_adb, 939 &woi); 940 if (qs < 0) 941 { 942 global_qs = qs; 943 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 944 GNUNET_free (payto_uri.full_payto); 945 return GNUNET_SYSERR; 946 } 947 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_bad_amount_out_plus), 948 &TALER_ARL_USE_AB (total_bad_amount_out_plus), 949 &amount); 950 GNUNET_free (payto_uri.full_payto); 951 return GNUNET_YES; /* justified, kind-of */ 952 } 953 } 954 GNUNET_free (payto_uri.full_payto); 955 if (0 != 956 TALER_amount_cmp (&amount, 957 &roi->details.amount)) 958 { 959 struct TALER_AUDITORDB_WireOutInconsistency woi = { 960 .wire_out_row_id = roi->details.serial_id, 961 .destination_account = roi->details.credit_account_uri, 962 .diagnostic = (char *) "incorrect amount drained to correct account", 963 .expected = roi->details.amount, 964 .claimed = amount, 965 }; 966 967 qs = TALER_AUDITORDB_insert_wire_out_inconsistency ( 968 TALER_ARL_adb, 969 &woi); 970 if (qs < 0) 971 { 972 global_qs = qs; 973 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 974 return GNUNET_SYSERR; 975 } 976 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_bad_amount_out_minus), 977 &TALER_ARL_USE_AB (total_bad_amount_out_minus), 978 &roi->details.amount); 979 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_bad_amount_out_plus), 980 &TALER_ARL_USE_AB (total_bad_amount_out_plus), 981 &amount); 982 return GNUNET_YES; /* justified, kind-of */ 983 } 984 /* profit drain was correct */ 985 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_drained), 986 &TALER_ARL_USE_AB (total_drained), 987 &amount); 988 return GNUNET_YES; 989 } 990 991 992 /** 993 * Check whether the given transfer was justified because it was 994 * actually previously reported as missing. 995 * 996 * @param roi the reserve out operation to check 997 * @return #GNUNET_OK on success, #GNUNET_NO if there was no 998 * matching lag, #GNUNET_SYSERR on database trouble 999 */ 1000 static enum GNUNET_GenericReturnValue 1001 check_closure_lag (const struct WireTransferOutInfo *roi) 1002 { 1003 enum GNUNET_DB_QueryStatus qs; 1004 1005 qs = TALER_AUDITORDB_delete_auditor_closure_lag ( 1006 TALER_ARL_adb, 1007 &roi->details.amount, 1008 &roi->details.wtid, 1009 roi->details.credit_account_uri); 1010 if (qs < 0) 1011 { 1012 global_qs = qs; 1013 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1014 return GNUNET_SYSERR; 1015 } 1016 if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) 1017 return GNUNET_NO; 1018 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1019 "Cleared closure lag: found justification\n"); 1020 TALER_ARL_amount_subtract (&TALER_ARL_USE_AB (total_closure_amount_lag), 1021 &TALER_ARL_USE_AB (total_closure_amount_lag), 1022 &roi->details.amount); 1023 return GNUNET_YES; /* found! */ 1024 } 1025 1026 1027 /** 1028 * Check whether the given transfer was justified by a reserve closure or 1029 * profit drain. If not, complain that we failed to match an entry from 1030 * #out_map. This means a wire transfer was made without proper 1031 * justification. 1032 * 1033 * @param cls a `struct WireAccount` 1034 * @param key unused key 1035 * @param value the `struct WireTransferOutInfo` to report 1036 * @return #GNUNET_OK on success 1037 */ 1038 static enum GNUNET_GenericReturnValue 1039 complain_out_not_found (void *cls, 1040 const struct GNUNET_HashCode *key, 1041 void *value) 1042 { 1043 // struct WireAccount *wa = cls; 1044 struct WireTransferOutInfo *roi = value; 1045 struct GNUNET_HashCode rkey; 1046 struct CheckMatchContext cmx = { 1047 .roi = roi, 1048 .found = false 1049 }; 1050 enum GNUNET_GenericReturnValue ret; 1051 1052 (void) cls; 1053 (void) key; 1054 hash_rc (roi->details.credit_account_uri, 1055 &roi->details.wtid, 1056 &rkey); 1057 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1058 "Checking for reserve closure %s benefiting %s\n", 1059 GNUNET_h2s (&rkey), 1060 roi->details.credit_account_uri.full_payto); 1061 GNUNET_CONTAINER_multihashmap_get_multiple (reserve_closures, 1062 &rkey, 1063 &check_rc_matches, 1064 &cmx); 1065 if (cmx.found) 1066 return GNUNET_OK; 1067 ret = check_reported_inconsistency (roi); 1068 if (GNUNET_NO != ret) 1069 return ret; 1070 ret = check_profit_drain (roi); 1071 if (GNUNET_NO != ret) 1072 return ret; 1073 ret = check_closure_lag (roi); 1074 if (GNUNET_NO != ret) 1075 return ret; 1076 1077 { 1078 struct TALER_AUDITORDB_WireOutInconsistency woi = { 1079 .destination_account = roi->details.credit_account_uri, 1080 .diagnostic = (char *) "missing justification for outgoing wire transfer", 1081 .wire_out_row_id = roi->details.serial_id, 1082 .expected = zero, 1083 .claimed = roi->details.amount 1084 }; 1085 enum GNUNET_DB_QueryStatus qs; 1086 1087 qs = TALER_AUDITORDB_insert_wire_out_inconsistency ( 1088 TALER_ARL_adb, 1089 &woi); 1090 if (qs < 0) 1091 { 1092 global_qs = qs; 1093 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1094 return GNUNET_SYSERR; 1095 } 1096 } 1097 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_bad_amount_out_plus), 1098 &TALER_ARL_USE_AB (total_bad_amount_out_plus), 1099 &roi->details.amount); 1100 return GNUNET_OK; 1101 } 1102 1103 1104 /** 1105 * Function called with details about outgoing wire transfers 1106 * as claimed by the exchange DB. 1107 * 1108 * @param cls a `struct WireAccount` 1109 * @param rowid unique serial ID in wire_out table 1110 * @param date timestamp of the transfer (roughly) 1111 * @param wtid wire transfer subject 1112 * @param payto_uri wire transfer details of the receiver 1113 * @param amount amount that was wired 1114 * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR to stop 1115 */ 1116 static enum GNUNET_GenericReturnValue 1117 wire_out_cb ( 1118 void *cls, 1119 uint64_t rowid, 1120 struct GNUNET_TIME_Timestamp date, 1121 const struct TALER_WireTransferIdentifierRawP *wtid, 1122 const struct TALER_FullPayto payto_uri, 1123 const struct TALER_Amount *amount) 1124 { 1125 struct WireAccount *wa = cls; 1126 struct GNUNET_HashCode key; 1127 struct WireTransferOutInfo *roi; 1128 1129 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1130 "Exchange wire OUT #%llu at %s of %s with WTID %s\n", 1131 (unsigned long long) rowid, 1132 GNUNET_TIME_timestamp2s (date), 1133 TALER_amount2s (amount), 1134 TALER_B2S (wtid)); 1135 wa->last_wire_out_serial_id = rowid + 1; 1136 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_wire_out), 1137 &TALER_ARL_USE_AB (total_wire_out), 1138 amount); 1139 GNUNET_CRYPTO_hash (wtid, 1140 sizeof (*wtid), 1141 &key); 1142 roi = GNUNET_CONTAINER_multihashmap_get (out_map, 1143 &key); 1144 if (NULL == roi) 1145 { 1146 /* Wire transfer was not made (yet) at all (but would have been 1147 justified), so the entire amount is missing / still to be done. This 1148 is moderately harmless, it might just be that the 1149 taler-exchange-transfer tool or bank has not yet fully caught up with 1150 the transfers it should do. 1151 May be cleared later by check_reported_inconsistency() */ 1152 char diag[MAX_DIAG_LEN]; 1153 struct TALER_AUDITORDB_WireOutInconsistency woi = { 1154 .destination_account = payto_uri, 1155 .diagnostic = diag, 1156 .wire_out_row_id = rowid, 1157 .expected = *amount, 1158 .claimed = zero, 1159 }; 1160 enum GNUNET_DB_QueryStatus qs; 1161 1162 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1163 "Wire out for row %llu still missing\n", 1164 (unsigned long long) rowid); 1165 make_missing_diag (diag, 1166 wtid); 1167 qs = TALER_AUDITORDB_insert_wire_out_inconsistency ( 1168 TALER_ARL_adb, 1169 &woi); 1170 if (qs < 0) 1171 { 1172 global_qs = qs; 1173 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1174 return GNUNET_SYSERR; 1175 } 1176 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_bad_amount_out_minus), 1177 &TALER_ARL_USE_AB (total_bad_amount_out_minus), 1178 amount); 1179 return GNUNET_OK; 1180 } 1181 1182 if (0 != TALER_full_payto_normalize_and_cmp (payto_uri, 1183 roi->details.credit_account_uri)) 1184 { 1185 /* Destination bank account is wrong in actual wire transfer, so 1186 we should count the wire transfer as entirely spurious, and 1187 additionally consider the justified wire transfer as missing. */ 1188 struct TALER_AUDITORDB_WireOutInconsistency woi = { 1189 .wire_out_row_id = rowid, 1190 .destination_account = payto_uri, 1191 .diagnostic = (char *) "receiver account mismatch", 1192 .expected = *amount, 1193 .claimed = roi->details.amount, 1194 }; 1195 enum GNUNET_DB_QueryStatus qs; 1196 1197 qs = TALER_AUDITORDB_insert_wire_out_inconsistency ( 1198 TALER_ARL_adb, 1199 &woi); 1200 if (qs < 0) 1201 { 1202 global_qs = qs; 1203 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1204 return GNUNET_SYSERR; 1205 } 1206 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_bad_amount_out_plus), 1207 &TALER_ARL_USE_AB (total_bad_amount_out_plus), 1208 &roi->details.amount); 1209 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_bad_amount_out_minus), 1210 &TALER_ARL_USE_AB (total_bad_amount_out_minus), 1211 amount); 1212 GNUNET_assert (GNUNET_OK == 1213 free_roi (NULL, 1214 &key, 1215 roi)); 1216 return GNUNET_OK; 1217 } 1218 1219 if (0 != TALER_amount_cmp (&roi->details.amount, 1220 amount)) 1221 { 1222 struct TALER_AUDITORDB_WireOutInconsistency woi = { 1223 .destination_account = payto_uri, 1224 .diagnostic = (char *) "wire amount does not match", 1225 .wire_out_row_id = rowid, 1226 .expected = *amount, 1227 .claimed = roi->details.amount, 1228 }; 1229 enum GNUNET_DB_QueryStatus qs; 1230 1231 qs = TALER_AUDITORDB_insert_wire_out_inconsistency ( 1232 TALER_ARL_adb, 1233 &woi); 1234 if (qs < 0) 1235 { 1236 global_qs = qs; 1237 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1238 return GNUNET_SYSERR; 1239 } 1240 if (0 < TALER_amount_cmp (amount, 1241 &roi->details.amount)) 1242 { 1243 /* amount > roi->details.amount: wire transfer was smaller than it should have been */ 1244 struct TALER_Amount delta; 1245 1246 TALER_ARL_amount_subtract (&delta, 1247 amount, 1248 &roi->details.amount); 1249 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_bad_amount_out_minus), 1250 &TALER_ARL_USE_AB (total_bad_amount_out_minus), 1251 &delta); 1252 } 1253 else 1254 { 1255 /* roi->details.amount < amount: wire transfer was larger than it should have been */ 1256 struct TALER_Amount delta; 1257 1258 TALER_ARL_amount_subtract (&delta, 1259 &roi->details.amount, 1260 amount); 1261 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_bad_amount_out_plus), 1262 &TALER_ARL_USE_AB (total_bad_amount_out_plus), 1263 &delta); 1264 } 1265 GNUNET_assert (GNUNET_OK == 1266 free_roi (NULL, 1267 &key, 1268 roi)); 1269 return GNUNET_OK; 1270 } 1271 1272 { 1273 enum GNUNET_GenericReturnValue ret; 1274 1275 if (! check_time_difference ("wire_out", 1276 rowid, 1277 date, 1278 roi->details.execution_date)) 1279 { 1280 /* We had a database error, fail */ 1281 ret = GNUNET_SYSERR; 1282 } 1283 else 1284 { 1285 ret = GNUNET_OK; 1286 } 1287 GNUNET_assert (GNUNET_OK == 1288 free_roi (NULL, 1289 &key, 1290 roi)); 1291 return ret; 1292 } 1293 } 1294 1295 1296 /** 1297 * Main function for processing 'debit' data. We start by going over 1298 * the DEBIT transactions this time, and then verify that all of them are 1299 * justified by reserve closures, profit drains or regular outgoing 1300 * wire transfers from aggregated deposits. 1301 * 1302 * @param[in,out] wa wire account list to process 1303 */ 1304 static void 1305 process_debits (struct WireAccount *wa); 1306 1307 1308 /** 1309 * Go over the "wire_out" table of the exchange and 1310 * verify that all wire outs are in that table. 1311 * 1312 * @param[in,out] wa wire account we are processing 1313 */ 1314 static void 1315 check_exchange_wire_out (struct WireAccount *wa) 1316 { 1317 enum GNUNET_DB_QueryStatus qs; 1318 1319 GNUNET_assert (NULL == wa->dhh); 1320 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1321 "Analyzing exchange's wire OUT table for account `%s'\n", 1322 wa->ai->section_name); 1323 qs = TALER_TALER_EXCHANGEDB_select_wire_out_above_serial_id_by_account ( 1324 TALER_ARL_edb, 1325 wa->ai->section_name, 1326 wa->last_wire_out_serial_id, 1327 &wire_out_cb, 1328 wa); 1329 if (0 > qs) 1330 { 1331 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1332 global_ret = EXIT_FAILURE; 1333 GNUNET_SCHEDULER_shutdown (); 1334 return; 1335 } 1336 GNUNET_CONTAINER_multihashmap_iterate (out_map, 1337 &complain_out_not_found, 1338 wa); 1339 /* clean up */ 1340 GNUNET_CONTAINER_multihashmap_iterate (out_map, 1341 &free_roi, 1342 NULL); 1343 process_debits (wa->next); 1344 } 1345 1346 1347 /** 1348 * This function is called for all transactions that 1349 * are debited from the exchange's account (outgoing 1350 * transactions). 1351 * 1352 * @param cls `struct WireAccount` with current wire account to process 1353 * @param dhr HTTP response details 1354 */ 1355 static void 1356 history_debit_cb ( 1357 void *cls, 1358 const struct TALER_BANK_DebitHistoryResponse *dhr); 1359 1360 1361 /** 1362 * Task scheduled to begin long-polling on the 1363 * bank transfer. 1364 * 1365 * @param cls a `struct WireAccount *` 1366 */ 1367 static void 1368 dh_long_poll (void *cls) 1369 { 1370 struct WireAccount *wa = cls; 1371 1372 wa->dhh_task = NULL; 1373 wa->dhh_next 1374 = GNUNET_TIME_relative_to_absolute (MIN_LONGPOLL_DELAY); 1375 GNUNET_assert (NULL == wa->dhh); 1376 wa->dhh = TALER_BANK_debit_history ( 1377 ctx, 1378 wa->ai->auth, 1379 wa->wire_off_out, 1380 MAX_PER_TRANSACTION, 1381 MAX_LONGPOLL_DELAY, 1382 &history_debit_cb, 1383 wa); 1384 if (NULL == wa->dhh) 1385 { 1386 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 1387 "Failed to start long-polling for bank transaction history for `%s'\n", 1388 wa->ai->section_name); 1389 global_ret = EXIT_FAILURE; 1390 GNUNET_SCHEDULER_shutdown (); 1391 return; 1392 } 1393 } 1394 1395 1396 static void 1397 history_debit_cb ( 1398 void *cls, 1399 const struct TALER_BANK_DebitHistoryResponse *dhr) 1400 { 1401 struct WireAccount *wa = cls; 1402 struct WireTransferOutInfo *roi; 1403 size_t slen; 1404 1405 wa->dhh = NULL; 1406 if ( (MHD_HTTP_OK == dhr->http_status) && 1407 (0 != dhr->details.ok.details_length) ) 1408 { 1409 /* As we got results, we go again *immediately* */ 1410 wa->dhh_next = GNUNET_TIME_UNIT_ZERO_ABS; 1411 } 1412 GNUNET_assert (NULL == wa->dhh_task); 1413 wa->dhh_task 1414 = GNUNET_SCHEDULER_add_at (wa->dhh_next, 1415 &dh_long_poll, 1416 wa); 1417 switch (dhr->http_status) 1418 { 1419 case MHD_HTTP_OK: 1420 for (unsigned int i = 0; i < dhr->details.ok.details_length; i++) 1421 { 1422 const struct TALER_BANK_DebitDetails *dd 1423 = &dhr->details.ok.details[i]; 1424 1425 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1426 "Analyzing bank DEBIT #%llu at %s of %s with WTID %s\n", 1427 (unsigned long long) dd->serial_id, 1428 GNUNET_TIME_timestamp2s (dd->execution_date), 1429 TALER_amount2s (&dd->amount), 1430 TALER_B2S (&dd->wtid)); 1431 wa->wire_off_out = dd->serial_id + 1; 1432 slen = strlen (dd->credit_account_uri.full_payto) + 1; 1433 roi = GNUNET_malloc (sizeof (struct WireTransferOutInfo) 1434 + slen); 1435 GNUNET_CRYPTO_hash (&dd->wtid, 1436 sizeof (dd->wtid), 1437 &roi->subject_hash); 1438 roi->details = *dd; 1439 roi->details.credit_account_uri.full_payto 1440 = (char *) &roi[1]; 1441 GNUNET_memcpy (&roi[1], 1442 dd->credit_account_uri.full_payto, 1443 slen); 1444 if (GNUNET_OK != 1445 GNUNET_CONTAINER_multihashmap_put (out_map, 1446 &roi->subject_hash, 1447 roi, 1448 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)) 1449 { 1450 struct TALER_AUDITORDB_WireFormatInconsistency wfi = { 1451 .amount = dd->amount, 1452 .wire_offset = dd->serial_id, 1453 .diagnostic = (char *) "duplicate outgoing wire transfer subject" 1454 }; 1455 enum GNUNET_DB_QueryStatus qs; 1456 1457 qs = TALER_AUDITORDB_insert_wire_format_inconsistency ( 1458 TALER_ARL_adb, 1459 &wfi); 1460 if (qs < 0) 1461 { 1462 global_qs = qs; 1463 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1464 commit (qs); 1465 return; 1466 } 1467 TALER_ARL_amount_add (&TALER_ARL_USE_AB ( 1468 wire_debit_duplicate_transfer_subject_total), 1469 &TALER_ARL_USE_AB ( 1470 wire_debit_duplicate_transfer_subject_total), 1471 &dd->amount); 1472 } 1473 } 1474 check_exchange_wire_out (wa); 1475 return; 1476 case MHD_HTTP_NO_CONTENT: 1477 check_exchange_wire_out (wa); 1478 return; 1479 case MHD_HTTP_NOT_FOUND: 1480 if (ignore_account_404) 1481 { 1482 check_exchange_wire_out (wa); 1483 return; 1484 } 1485 break; 1486 default: 1487 break; 1488 } 1489 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 1490 "Error fetching debit history of account %s: %u/%u!\n", 1491 wa->ai->section_name, 1492 dhr->http_status, 1493 (unsigned int) dhr->ec); 1494 commit (GNUNET_DB_STATUS_HARD_ERROR); 1495 global_ret = EXIT_FAILURE; 1496 GNUNET_SCHEDULER_shutdown (); 1497 return; 1498 } 1499 1500 1501 static void 1502 process_debits (struct WireAccount *wa) 1503 { 1504 /* skip accounts where DEBIT is not enabled */ 1505 while ( (NULL != wa) && 1506 (! wa->ai->debit_enabled) ) 1507 wa = wa->next; 1508 if (NULL == wa) 1509 { 1510 /* end of iteration */ 1511 commit (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT); 1512 return; 1513 } 1514 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1515 "Checking bank DEBIT records of account `%s'\n", 1516 wa->ai->section_name); 1517 if ( (NULL == wa->dhh) && 1518 (NULL == wa->dhh_task) ) 1519 { 1520 wa->dhh = TALER_BANK_debit_history ( 1521 ctx, 1522 wa->ai->auth, 1523 wa->wire_off_out, 1524 MAX_PER_TRANSACTION, 1525 GNUNET_TIME_UNIT_ZERO, 1526 &history_debit_cb, 1527 wa); 1528 if (NULL == wa->dhh) 1529 { 1530 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 1531 "Failed to obtain bank transaction history for `%s'\n", 1532 wa->ai->section_name); 1533 commit (GNUNET_DB_STATUS_HARD_ERROR); 1534 global_ret = EXIT_FAILURE; 1535 GNUNET_SCHEDULER_shutdown (); 1536 return; 1537 } 1538 } 1539 } 1540 1541 1542 /** 1543 * Function called about reserve closing operations the aggregator triggered. 1544 * 1545 * @param cls closure; NULL 1546 * @param rowid row identifier used to uniquely identify the reserve closing operation 1547 * @param execution_date when did we execute the close operation 1548 * @param amount_with_fee how much did we debit the reserve 1549 * @param closing_fee how much did we charge for closing the reserve 1550 * @param reserve_pub public key of the reserve 1551 * @param receiver_account where did we send the funds, in payto://-format 1552 * @param wtid identifier used for the wire transfer 1553 * @param close_request_row which close request triggered the operation? 1554 * 0 if it was a timeout (not used) 1555 * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR to stop 1556 */ 1557 static enum GNUNET_GenericReturnValue 1558 reserve_closed_cb ( 1559 void *cls, 1560 uint64_t rowid, 1561 struct GNUNET_TIME_Timestamp execution_date, 1562 const struct TALER_Amount *amount_with_fee, 1563 const struct TALER_Amount *closing_fee, 1564 const struct TALER_ReservePublicKeyP *reserve_pub, 1565 const struct TALER_FullPayto receiver_account, 1566 const struct TALER_WireTransferIdentifierRawP *wtid, 1567 uint64_t close_request_row) 1568 { 1569 struct ReserveClosure *rc; 1570 struct GNUNET_HashCode key; 1571 1572 (void) cls; 1573 (void) close_request_row; 1574 GNUNET_assert (TALER_ARL_USE_PP (wire_reserve_close_id) <= rowid); 1575 TALER_ARL_USE_PP (wire_reserve_close_id) = rowid + 1; 1576 rc = GNUNET_new (struct ReserveClosure); 1577 if (TALER_ARL_SR_INVALID_NEGATIVE == 1578 TALER_ARL_amount_subtract_neg (&rc->amount, 1579 amount_with_fee, 1580 closing_fee)) 1581 { 1582 struct TALER_AUDITORDB_RowInconsistency ri = { 1583 .row_id = rowid, 1584 .row_table 1585 = (char *) "reserves_closures", 1586 .diagnostic 1587 = (char *) "closing fee above reserve balance (and closed anyway)" 1588 }; 1589 enum GNUNET_DB_QueryStatus qs; 1590 1591 qs = TALER_AUDITORDB_insert_row_inconsistency ( 1592 TALER_ARL_adb, 1593 &ri); 1594 if (qs < 0) 1595 { 1596 global_qs = qs; 1597 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1598 return GNUNET_OK; 1599 } 1600 GNUNET_free (rc); 1601 return GNUNET_OK; 1602 } 1603 rc->receiver_account.full_payto 1604 = GNUNET_strdup (receiver_account.full_payto); 1605 rc->wtid = *wtid; 1606 rc->execution_date = execution_date; 1607 rc->rowid = rowid; 1608 hash_rc (rc->receiver_account, 1609 wtid, 1610 &key); 1611 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1612 "Discovered reserve closure %llu (%s) over %s benefiting %s\n", 1613 (unsigned long long) rowid, 1614 GNUNET_h2s (&key), 1615 TALER_amount2s (amount_with_fee), 1616 receiver_account.full_payto); 1617 (void) GNUNET_CONTAINER_multihashmap_put ( 1618 reserve_closures, 1619 &key, 1620 rc, 1621 GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE); 1622 return GNUNET_OK; 1623 } 1624 1625 1626 /** 1627 * Start the database transactions and begin the audit. 1628 * 1629 * @return transaction status code 1630 */ 1631 static enum GNUNET_DB_QueryStatus 1632 begin_transaction (void) 1633 { 1634 enum GNUNET_DB_QueryStatus qs; 1635 1636 if (GNUNET_SYSERR == 1637 TALER_EXCHANGEDB_preflight (TALER_ARL_edb)) 1638 { 1639 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 1640 "Failed to initialize exchange database connection.\n"); 1641 return GNUNET_DB_STATUS_HARD_ERROR; 1642 } 1643 if (GNUNET_SYSERR == 1644 TALER_AUDITORDB_preflight (TALER_ARL_adb)) 1645 { 1646 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 1647 "Failed to initialize auditor database session.\n"); 1648 return GNUNET_DB_STATUS_HARD_ERROR; 1649 } 1650 global_qs = GNUNET_DB_STATUS_SUCCESS_NO_RESULTS; 1651 if (GNUNET_OK != 1652 TALER_AUDITORDB_start (TALER_ARL_adb)) 1653 { 1654 GNUNET_break (0); 1655 return GNUNET_DB_STATUS_HARD_ERROR; 1656 } 1657 if (GNUNET_OK != 1658 TALER_TALER_EXCHANGEDB_start_read_only (TALER_ARL_edb, 1659 "wire debit auditor")) 1660 { 1661 GNUNET_break (0); 1662 return GNUNET_DB_STATUS_HARD_ERROR; 1663 } 1664 qs = TALER_AUDITORDB_get_balance ( 1665 TALER_ARL_adb, 1666 TALER_ARL_GET_AB (total_drained), 1667 TALER_ARL_GET_AB (total_wire_out), 1668 TALER_ARL_GET_AB (total_bad_amount_out_plus), 1669 TALER_ARL_GET_AB (total_bad_amount_out_minus), 1670 TALER_ARL_GET_AB (total_closure_amount_lag), 1671 TALER_ARL_GET_AB (wire_debit_duplicate_transfer_subject_total), 1672 TALER_ARL_GET_AB (total_wire_out), 1673 NULL); 1674 switch (qs) 1675 { 1676 case GNUNET_DB_STATUS_HARD_ERROR: 1677 GNUNET_break (0); 1678 return qs; 1679 case GNUNET_DB_STATUS_SOFT_ERROR: 1680 GNUNET_break (0); 1681 return qs; 1682 case GNUNET_DB_STATUS_SUCCESS_NO_RESULTS: 1683 case GNUNET_DB_STATUS_SUCCESS_ONE_RESULT: 1684 break; 1685 } 1686 for (struct WireAccount *wa = wa_head; 1687 NULL != wa; 1688 wa = wa->next) 1689 { 1690 GNUNET_asprintf (&wa->label_wire_out_serial_id, 1691 "wire-%s-%s", 1692 wa->ai->section_name, 1693 "wire_out_serial_id"); 1694 GNUNET_asprintf (&wa->label_wire_off_out, 1695 "wire-%s-%s", 1696 wa->ai->section_name, 1697 "wire_off_out"); 1698 qs = TALER_AUDITORDB_get_auditor_progress ( 1699 TALER_ARL_adb, 1700 wa->label_wire_out_serial_id, 1701 &wa->last_wire_out_serial_id, 1702 wa->label_wire_off_out, 1703 &wa->wire_off_out, 1704 NULL); 1705 if (0 > qs) 1706 { 1707 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1708 return qs; 1709 } 1710 GNUNET_assert (2 == qs); 1711 wa->start_wire_out_serial_id = wa->last_wire_out_serial_id; 1712 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1713 "Resuming account %s debit audit at %llu/%llu\n", 1714 wa->ai->section_name, 1715 (unsigned long long) wa->last_wire_out_serial_id, 1716 (unsigned long long) wa->wire_off_out); 1717 } 1718 qs = TALER_AUDITORDB_get_auditor_progress ( 1719 TALER_ARL_adb, 1720 TALER_ARL_GET_PP (wire_reserve_close_id), 1721 NULL); 1722 if (0 > qs) 1723 { 1724 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1725 return qs; 1726 } 1727 if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) 1728 { 1729 GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE, 1730 "First analysis of with wire auditor, starting audit from scratch\n"); 1731 } 1732 else 1733 { 1734 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1735 "Resuming wire debit audit at %llu\n", 1736 (unsigned long long) TALER_ARL_USE_PP (wire_reserve_close_id)); 1737 } 1738 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1739 "Iterating over reserve closures from %llu\n", 1740 (unsigned long long) TALER_ARL_USE_PP (wire_reserve_close_id)); 1741 qs = TALER_EXCHANGEDB_select_reserve_closed_above_serial_id ( 1742 TALER_ARL_edb, 1743 TALER_ARL_USE_PP (wire_reserve_close_id), 1744 &reserve_closed_cb, 1745 NULL); 1746 if (0 > qs) 1747 { 1748 GNUNET_break (GNUNET_DB_STATUS_HARD_ERROR == qs); 1749 return GNUNET_DB_STATUS_HARD_ERROR; 1750 } 1751 process_debits (wa_head); 1752 return GNUNET_DB_STATUS_SUCCESS_NO_RESULTS; 1753 } 1754 1755 1756 /** 1757 * Function called with information about a wire account. Adds the 1758 * account to our list for processing (if it is enabled and we can 1759 * load the plugin). 1760 * 1761 * @param cls closure, NULL 1762 * @param ai account information 1763 */ 1764 static void 1765 process_account_cb (void *cls, 1766 const struct TALER_EXCHANGEDB_AccountInfo *ai) 1767 { 1768 struct WireAccount *wa; 1769 1770 (void) cls; 1771 if ( (! ai->debit_enabled) && 1772 (! ai->credit_enabled) ) 1773 return; /* not an active exchange account */ 1774 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1775 "Found exchange account `%s'\n", 1776 ai->section_name); 1777 wa = GNUNET_new (struct WireAccount); 1778 wa->ai = ai; 1779 GNUNET_CONTAINER_DLL_insert (wa_head, 1780 wa_tail, 1781 wa); 1782 } 1783 1784 1785 /** 1786 * Function called on events received from Postgres. 1787 * 1788 * @param cls closure, NULL 1789 * @param extra additional event data provided 1790 * @param extra_size number of bytes in @a extra 1791 */ 1792 static void 1793 db_notify (void *cls, 1794 const void *extra, 1795 size_t extra_size) 1796 { 1797 (void) cls; 1798 (void) extra; 1799 (void) extra_size; 1800 1801 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1802 "Received notification to wake wire helper\n"); 1803 /* If there are accounts we are still processing, abort 1804 the HTTP requests so we can start afresh. */ 1805 for (struct WireAccount *wa = wa_head; 1806 NULL != wa; 1807 wa = wa->next) 1808 { 1809 if (NULL != wa->dhh) 1810 { 1811 TALER_BANK_debit_history_cancel (wa->dhh); 1812 wa->dhh = NULL; 1813 } 1814 check_exchange_wire_out (wa); 1815 } 1816 1817 if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS != 1818 begin_transaction ()) 1819 { 1820 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 1821 "Audit failed\n"); 1822 GNUNET_break (0); 1823 global_ret = EXIT_FAILURE; 1824 GNUNET_SCHEDULER_shutdown (); 1825 return; 1826 } 1827 } 1828 1829 1830 /** 1831 * Main function that will be run. 1832 * 1833 * @param cls closure 1834 * @param args remaining command-line arguments 1835 * @param cfgfile name of the configuration file used (for saving, can be NULL!) 1836 * @param c configuration 1837 */ 1838 static void 1839 run (void *cls, 1840 char *const *args, 1841 const char *cfgfile, 1842 const struct GNUNET_CONFIGURATION_Handle *c) 1843 { 1844 (void) cls; 1845 (void) args; 1846 (void) cfgfile; 1847 cfg = c; 1848 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 1849 "Launching wire debit auditor\n"); 1850 if (GNUNET_OK != 1851 TALER_ARL_init (c)) 1852 { 1853 global_ret = EXIT_FAILURE; 1854 return; 1855 } 1856 1857 reserve_closures 1858 = GNUNET_CONTAINER_multihashmap_create (1024, 1859 GNUNET_NO); 1860 GNUNET_assert (GNUNET_OK == 1861 TALER_amount_set_zero (TALER_ARL_currency, 1862 &zero)); 1863 GNUNET_SCHEDULER_add_shutdown (&do_shutdown, 1864 NULL); 1865 ctx = GNUNET_CURL_init (&GNUNET_CURL_gnunet_scheduler_reschedule, 1866 &rctx); 1867 rctx = GNUNET_CURL_gnunet_rc_create (ctx); 1868 if (NULL == ctx) 1869 { 1870 GNUNET_break (0); 1871 global_ret = EXIT_FAILURE; 1872 return; 1873 } 1874 reserve_closures = GNUNET_CONTAINER_multihashmap_create (1024, 1875 GNUNET_NO); 1876 out_map = GNUNET_CONTAINER_multihashmap_create (1024, 1877 true); 1878 if (GNUNET_OK != 1879 TALER_EXCHANGEDB_load_accounts (TALER_ARL_cfg, 1880 TALER_EXCHANGEDB_ALO_DEBIT 1881 | TALER_EXCHANGEDB_ALO_CREDIT 1882 | TALER_EXCHANGEDB_ALO_AUTHDATA)) 1883 { 1884 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 1885 "No bank accounts configured\n"); 1886 global_ret = EXIT_NOTCONFIGURED; 1887 GNUNET_SCHEDULER_shutdown (); 1888 return; 1889 } 1890 TALER_EXCHANGEDB_find_accounts (&process_account_cb, 1891 NULL); 1892 1893 if (0 == test_mode) 1894 { 1895 struct GNUNET_DB_EventHeaderP es = { 1896 .size = htons (sizeof (es)), 1897 .type = htons (TALER_DBEVENT_EXCHANGE_AUDITOR_WAKE_HELPER_WIRE) 1898 }; 1899 1900 eh = TALER_AUDITORDB_event_listen (TALER_ARL_adb, 1901 &es, 1902 GNUNET_TIME_UNIT_FOREVER_REL, 1903 &db_notify, 1904 NULL); 1905 GNUNET_assert (NULL != eh); 1906 } 1907 if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS != 1908 begin_transaction ()) 1909 { 1910 GNUNET_break (0); 1911 global_ret = EXIT_FAILURE; 1912 GNUNET_SCHEDULER_shutdown (); 1913 return; 1914 } 1915 } 1916 1917 1918 /** 1919 * The main function of the wire auditing tool. Checks that 1920 * the exchange's records of wire transfers match that of 1921 * the wire gateway. 1922 * 1923 * @param argc number of arguments from the command line 1924 * @param argv command line arguments 1925 * @return 0 ok, 1 on error 1926 */ 1927 int 1928 main (int argc, 1929 char *const *argv) 1930 { 1931 const struct GNUNET_GETOPT_CommandLineOption options[] = { 1932 GNUNET_GETOPT_option_flag ('i', 1933 "internal", 1934 "perform checks only applicable for exchange-internal audits", 1935 &internal_checks), 1936 GNUNET_GETOPT_option_flag ('I', 1937 "ignore-not-found", 1938 "continue, even if the bank account of the exchange was not found", 1939 &ignore_account_404), 1940 GNUNET_GETOPT_option_flag ('t', 1941 "test", 1942 "run in test mode and exit when idle", 1943 &test_mode), 1944 GNUNET_GETOPT_option_timetravel ('T', 1945 "timetravel"), 1946 GNUNET_GETOPT_OPTION_END 1947 }; 1948 enum GNUNET_GenericReturnValue ret; 1949 1950 ret = GNUNET_PROGRAM_run ( 1951 TALER_AUDITOR_project_data (), 1952 argc, 1953 argv, 1954 "taler-helper-auditor-wire-debit", 1955 gettext_noop ( 1956 "Audit exchange database for consistency with the bank's outgoing wire transfers"), 1957 options, 1958 &run, 1959 NULL); 1960 if (GNUNET_SYSERR == ret) 1961 return EXIT_INVALIDARGUMENT; 1962 if (GNUNET_NO == ret) 1963 return EXIT_SUCCESS; 1964 return global_ret; 1965 } 1966 1967 1968 /* end of taler-helper-auditor-wire-debit.c */