master.cfg (53624B)
1 # -*- python -*- 2 # ex: set syntax=python: 3 4 ## 5 # This file is part of TALER 6 # (C) 2016-2025 Taler Systems SA 7 # 8 # TALER is free software; you can redistribute it and/or 9 # modify it under the terms of the GNU Affero General Public 10 # License as published by the Free Software Foundation; either 11 # version 3, or (at your option) any later version. 12 # 13 # TALER is distributed in the hope that it will be useful, 14 # but WITHOUT ANY WARRANTY; without even the implied warranty 15 # of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 16 # See the GNU General Public License for more details. 17 # 18 # You should have received a copy of the GNU General Public 19 # License along with TALER; see the file COPYING. If not, 20 # see <http://www.gnu.org/licenses/> 21 # 22 # @author Florian Dold 23 # @author Marcello Stanisci 24 # @author ng0 25 # @author Christian Grothoff 26 # @author Devan Carpenter 27 import ast 28 import configparser 29 import glob 30 import os 31 import pathlib 32 import pwd 33 import re 34 import subprocess 35 36 from buildbot.changes.pb import PBChangeSource 37 from buildbot.steps.source.git import Git 38 from buildbot.steps.shell import ShellCommand 39 from buildbot.plugins import changes 40 from buildbot.plugins import reporters 41 from buildbot.plugins import schedulers 42 from buildbot.plugins import steps 43 from buildbot.plugins import secrets, util 44 from buildbot.process import buildstep, logobserver 45 from buildbot.process.results import SKIPPED 46 from buildbot.reporters.generators.build import BuildStatusGenerator 47 from buildbot.worker import Worker 48 from twisted.internet import defer 49 50 # This is a sample buildmaster config file. It must be 51 # installed as 'master.cfg' in your buildmaster's base 52 # directory. 53 54 # This file has the following structure: 55 # - Globals: definition of global variables we use throughout 56 # + Convenience functions: helper functions useful for many jobs 57 # - Jobs: actual job definitions 58 # - General purpose: triggers and alerts shared by various jobs 59 # + general purpose notification (alerts) 60 # + general purpose triggers (schedulers) 61 # - Actual buildbot configuration object initialization 62 63 ################################################################# 64 ######################### GLOBALS ############################### 65 ################################################################# 66 67 # The 'workers' list defines the set of recognized workers. 68 # Each element is a Worker object, specifying a unique worker 69 # name and password. The same worker name and password must 70 # be configured on the worker. 71 WORKERS = [] 72 73 # 'services' is a list of BuildbotService items like reporter 74 # targets. The status of each build will be pushed to these 75 # targets. buildbot/reporters/*.py has a variety to choose from, 76 # like IRC bots. 77 78 79 class MessageFormatterWithStdout(reporters.MessageFormatter): 80 def buildAdditionalContext(self, master, ctx): 81 stdout = [] 82 for step in ctx["build"]["steps"]: 83 for log in step["logs"]: 84 all_logs = log["content"]["content"].splitlines() 85 # Including only what the script printed on stdout. 86 for line in all_logs: 87 if re.search("^o", line): 88 stdout.append(line[1:]) 89 ctx.update(dict(stdout="\n".join(stdout))) 90 91 92 ##################################################### 93 # Commit message triggers # 94 # # 95 # This checks for triggers in the commit messages # 96 # !tarball will trigger a release build # 97 # !coverity will trigger a coverity check # 98 ##################################################### 99 def checkForTarballTrigger(change): 100 if "!tarball" in change.comments: 101 return True 102 return False 103 def checkForCoverityTrigger(change): 104 if "!coverity" in change.comments: 105 return True 106 107 SERVICES = [] 108 109 # The 'builders' list defines the Builders, which tell Buildbot 110 # how to perform a build: what steps, and which workers can execute 111 # them. Note that any particular build will only take place on 112 # one worker. 113 BUILDERS = [] 114 115 # Configures the Schedulers, which decide how to react to incoming 116 # changes. 117 SCHEDULERS = [] 118 119 # Array of builders to be scheduled every night. 120 NIGHTLY_TRIGGERS=[] 121 122 # Array of builders to be scheduled whenever any of the code Git repos change 123 CODECHANGE_TRIGGERS = [] 124 125 # Array of builders to be scheduled whenever the taler-typescript-core or 126 # taler-deployment change 127 WALLETCHANGE_TRIGGERS = [] 128 129 # Array of builder names for which build status reports should be sent 130 # via e-mail 131 EMAIL_ALERTS = [] 132 133 # Array of email address for which build status reports shoudl be sent 134 BUILDER_EMAIL_ADDRESSES = [] 135 136 ############ Convenience functions ################# 137 138 # Create a FACTORY with a taler-deployment.git checkout as the first step. 139 def create_factory_with_deployment(): 140 f = util.BuildFactory() 141 update_deployment(f) 142 return f 143 144 # Convenience function that checks out a Git repository. 145 # First argument is the URL of the Git to clone, second 146 # the desired branch. Default is 'master'. 147 def git_step(repo, target_branch="master"): 148 return Git( 149 repourl=repo, 150 mode="full", 151 method="fresh", 152 logEnviron=False, 153 alwaysUseLatest=True, 154 haltOnFailure=True, 155 branch=target_branch 156 ) 157 158 # FIXME: document this function! 159 def add_default_step(bldr, step): 160 worker_script = ".buildbot/%(prop:workername)s_" + step + ".sh" 161 default_script = ".buildbot/" + step + ".sh" 162 cmd = '[ -f %s ] && ls -1 %s || exit 0' % (worker_script,worker_script) 163 bldr.addStep(steps.SetPropertyFromCommand(command=util.Interpolate(cmd), 164 hideStepIf=True, 165 property='buildbotScript_%s' % step, 166 name="Checking for worker-specific %s script" % step)) 167 cmd = '[ -f %s ] && ls -1 %s || exit 0' % (default_script,default_script) 168 bldr.addStep(steps.SetPropertyFromCommand(doStepIf=(util.Property('buildbotScript_%s' % step) != util.Interpolate(worker_script)), 169 hideStepIf=True, 170 command=util.Interpolate(cmd), 171 property='buildbotScript_%s' % step, 172 name="Checking for %s script" % step)) 173 bldr.addStep(steps.ShellCommand(command=util.Property('buildbotScript_%s' % step), 174 haltOnFailure=True, 175 env={'PATH': "${HOME}/.local/bin:$HOME/bin:${PATH}"}, 176 doStepIf=(util.Property('buildbotScript_%s' % step) != None), 177 hideStepIf=lambda results, s: results==SKIPPED, 178 name="Executing %s step" % step)) 179 180 181 182 # Convenience function that runs 'make check' in a 183 # directory of the code inside of a netjail. 184 def jailed_check(package, srcdirs): 185 return steps.ShellSequence( 186 name="Tests of " + package, 187 description="Testing " + package, 188 descriptionDone="Pass", 189 commands=map(lambda srcdir: util.ShellArg(command=["sudo", "/usr/local/bin/netjail.sh", "/home/container-worker/taler-deployment/buildbot/with-postgres.sh", "bash", "-c", "'cd src/"+srcdir+" make install check'"]), srcdirs), 190 env={'PATH': "${HOME}/local/bin:${PATH}"}, 191 workdir="../../sources/" + package 192 ) 193 194 # Convenience function that checks out the deployment. 195 def update_deployment(factory): 196 factory.addStep(steps.ShellSequence( 197 name="update taler-deployment", 198 description="removing old deployment and fetching fresh repository", 199 descriptionDone="Deployment updated", 200 commands=[ 201 util.ShellArg(command=["rm", "-rf", "taler-deployment"]), 202 util.ShellArg(command=["git", "clone", "git://git.taler.net/taler-deployment"]), 203 ], 204 haltOnFailure=True, 205 workdir="../.." 206 )) 207 208 # Convenience function that builds and runs a container. 209 def container_add_step(HALT_ON_FAILURE, 210 WARN_ON_FAILURE, 211 CONTAINER_BUILD, 212 CONTAINER_NAME, 213 factory, 214 WORK_DIR, 215 repoName, 216 stepName, 217 CONTAINER_ARCH="amd64", 218 jobCmd="/workdir/contrib/ci/ci.sh", 219 containerFile="contrib/ci/Containerfile"): 220 print(f"HALT_ON_FAILURE: {HALT_ON_FAILURE}, WARN_ON_FAILURE: {WARN_ON_FAILURE}, CONTAINER_BUILD: {CONTAINER_BUILD}, CONTAINER_NAME: {CONTAINER_NAME}") 221 222 runCommand = ["podman", "--transient-store", "run", "--rm", 223 "--arch", CONTAINER_ARCH, 224 "--log-driver=none", 225 "--add-host", "taler.host.internal:10.0.2.2", 226 "--network", "slirp4netns:allow_host_loopback=true", 227 "--env", util.Interpolate("CI_COMMIT_REF=%(prop:got_revision:-%(src::revision:-unknown)s)s"), 228 "--env", util.Interpolate("CI_GIT_BRANCH=%(src::branch)s"), 229 "--env", util.Interpolate("CI_PROJECT_NAME=%(src::project)s"), 230 "--cap-add", "SYS_ADMIN,CAP_SYS_CHROOT", 231 "--volume", f"{WORK_DIR}:/workdir", 232 "--volume", "/home/container-worker/ephemeral_ci_artifacts:/artifacts", 233 "--volume", "/home/container-worker/persistent_ci_artifacts:/persistent_artifacts", 234 "--volume", "/home/container-worker/mounted_files/ci_container_id_ed25519:/root/.ssh/id_ed25519:ro", 235 "--volume", "/home/container-worker/mounted_files/container_known_hosts:/root/.ssh/known_hosts:ro", 236 "--workdir", "/workdir"] 237 238 # Inputs directory 239 inputs_path = f"/home/container-worker/container_inputs/{repoName}" 240 print(f"Checking that {inputs_path} exists") 241 if os.path.isdir(inputs_path): 242 print(f"Adding {inputs_path}") 243 runCommand += ["--volume", f"{inputs_path}:/inputs:ro"] 244 else: 245 print(f"Inputs directory not found at {inputs_path}") 246 247 if CONTAINER_BUILD: 248 runCommand += ["--volume", f"/run/user/{pwd.getpwnam('container-worker').pw_uid}/podman/podman.sock:/run/podman/podman.sock", 249 "--security-opt", "label=disable"] 250 251 runCommand += [CONTAINER_NAME, jobCmd] 252 253 # Halting jobs (including deployment) remain fatal even if they also 254 # request warnings. Only non-halting jobs may be advisory. 255 advisory = WARN_ON_FAILURE and not HALT_ON_FAILURE 256 runArg = util.ShellArg(command=runCommand, 257 logname='run inside container', 258 flunkOnFailure=not advisory, 259 warnOnFailure=advisory, 260 haltOnFailure=HALT_ON_FAILURE) 261 262 buildArg = util.ShellArg(command=["podman", "build", "-t", CONTAINER_NAME, 263 "--arch", CONTAINER_ARCH, 264 "-f", containerFile, "."], 265 logname='build container', haltOnFailure=True) 266 267 if not CONTAINER_BUILD: 268 return steps.ShellSequence( 269 name=stepName, 270 commands=[runArg], 271 haltOnFailure=HALT_ON_FAILURE, 272 flunkOnFailure=True, 273 warnOnWarnings=True, 274 workdir=WORK_DIR 275 ) 276 else: 277 return steps.ShellSequence( 278 name=stepName, 279 commands=[buildArg, runArg], 280 haltOnFailure=HALT_ON_FAILURE, 281 flunkOnFailure=True, 282 warnOnWarnings=True, 283 workdir=WORK_DIR 284 ) 285 286 ################################################################## 287 ######################## JOBS #################################### 288 ################################################################## 289 290 # For every job, we have (in this order!): 291 # - worker(s): hosts/users that run the job 292 # - factory: list of steps that define what to do 293 # - builder: gives the job a name and binds it to the factory and worker 294 # - (OPTIONAL) alerts: notifications to trigger when the job fails 295 # Pre-defined: EMAIL_ALERTS 296 # - scheduler: rules that define when to run the job 297 # Pre-defined: NIGHTLY_TRIGGERS, CODECHANGE_TRIGGERS, WALLETCHANGE_TRIGGERS 298 299 ################ 1: BUILDMASTER JOB ################################### 300 301 ## 302 # This worker restarts the buildmaster itself on 303 # changes to this file. 304 # Location: /home/buildbot-master 305 WORKERS.append(Worker("buildmaster-worker", "buildmaster-pass")) 306 307 BUILDMASTER_FACTORY = create_factory_with_deployment() 308 BUILDMASTER_FACTORY.addStep( 309 ShellCommand( 310 name="restart buildmaster", 311 description="trigger buildmaster restart with new configuration", 312 descriptionDone="Buildmaster updated", 313 command=["pkill", "-HUP", "-A", "-u", "buildbot-master", "-f", "/usr/bin/python3"], 314 workdir="../.." 315 ) 316 ) 317 318 BUILDERS.append(util.BuilderConfig( 319 name="buildmaster-builder", 320 workernames=["buildmaster-worker"], 321 factory=BUILDMASTER_FACTORY 322 )) 323 324 EMAIL_ALERTS.append("buildmaster-builder") 325 326 # Buildmaster is notified whenever deployment.git changes 327 SCHEDULERS.append(schedulers.SingleBranchScheduler( 328 name="buildmaster-scheduler", 329 change_filter=util.ChangeFilter( 330 branch="master", 331 project_re="(taler-deployment)" 332 ), 333 treeStableTimer=None, 334 builderNames=["buildmaster-builder"] 335 )) 336 337 ################ 2: WEBSITE JOB ################################### 338 339 ## 340 # This worker builds Websites: www and stage. 341 # 342 WORKERS.append(Worker("sites-worker", "sites-pass")) 343 344 SITES_FACTORY = create_factory_with_deployment() 345 SITES_FACTORY.addStep( 346 ShellCommand( 347 name="build Web sites", 348 description="Building all the Taler homepages", 349 descriptionDone="Sites built.", 350 command=["./build-sites.sh"], 351 workdir="../../taler-deployment/worker-sites", 352 haltOnFailure=True 353 ) 354 ) 355 356 BUILDERS.append(util.BuilderConfig( 357 name="sites-builder", workernames=["sites-worker"], factory=SITES_FACTORY 358 )) 359 360 #EMAIL_ALERTS.append("sites-builder") 361 362 363 # The web page changed if 'taler-www', 'taler-tutorials', 364 # 'taler-docs' or 'twister' changed 365 def web_page(change): 366 _change = change.asDict() 367 repo = _change.get("project") 368 if repo in ["taler-docs", "taler-tutorials", "taler-www", "twister", "taler-deployment", "buywith", "taler-ops-www", "taler-systems-www", "anastasis-www", "pepsi"]: 369 return True 370 return False 371 372 373 # Sites are re-build whenever taler-deployment, taler-www, or twister changes. 374 SCHEDULERS.append(schedulers.SingleBranchScheduler( 375 name="sites-scheduler", 376 builderNames=["sites-builder"], 377 change_filter=util.ChangeFilter( 378 branch_re="(master|stable)", 379 filter_fn=web_page 380 ), 381 treeStableTimer=None 382 )) 383 384 385 ################ 3: 'check links' JOB ################################### 386 387 ## 388 # linkchecker worker checks for dead links in the Website 389 # Location: /home/linkchecker-worker 390 WORKERS.append(Worker("linkchecker-worker", "linkchecker-pass")) 391 392 # linkchecker FACTORY 393 LINKCHECKER_FACTORY = create_factory_with_deployment() 394 LINKCHECKER_FACTORY.addStep( 395 ShellCommand( 396 name="linkchecker", 397 description="Check taler.net website for broken links && Notify", 398 descriptionDone="Results of wget in buildbot logs.", 399 command=["/home/linkchecker-worker/taler-deployment/worker-linkchecker/linkchecker.sh"], 400 workdir="/home/linkchecker-worker", 401 haltOnFailure=True, 402 timeout=7200 # 2 hours 403 ) 404 ) 405 406 # linkchecker BUILDER 407 # worker at linkchecker@taler.net 408 BUILDERS.append(util.BuilderConfig( 409 name="linkchecker-builder", 410 workernames="linkchecker-worker", 411 factory=LINKCHECKER_FACTORY 412 )) 413 414 docs_generator = BuildStatusGenerator( 415 mode=('change', 'problem', 'failing', 'exception',), 416 builders=[ 417 'linkchecker-builder', 418 ], 419 message_formatter=reporters.MessageFormatter( 420 template_type='plain', 421 want_logs_content=True 422 ) 423 ) 424 425 426 SERVICES.append(reporters.MailNotifier( 427 fromaddr="bb@taler.net", 428 generators=[docs_generator], 429 sendToInterestedUsers=False, 430 useTls=False, 431 relayhost="localhost", 432 smtpPort=25, 433 dumpMailsToLog=True, 434 extraRecipients=['linkcheck@taler.net'] 435 )) 436 437 # SERVICES.append(tipReserveEmails) 438 439 NIGHTLY_TRIGGERS.append("linkchecker-builder") 440 441 #################### 442 ## GNUnet workers ## 443 #################### 444 445 WORKERS.append(Worker("firefly-x86_64-amdepyc", "pass")) 446 WORKERS.append(Worker("mp-amd64-openbsd", "Tai7zeic")) 447 WORKERS.append(Worker("schanzen-aarch64-fedora-meson", "eWi9keet")) 448 449 450 SCHEDULERS.append(schedulers.AnyBranchScheduler( 451 name="gnunet", 452 change_filter=util.ChangeFilter(branch_re='master', 453 repository='git://git.gnunet.org/gnunet.git'), 454 treeStableTimer=None, 455 builderNames=["gnunet-debian-x86_64", 456 "gnunet-fedora-aarch64"])) 457 458 SCHEDULERS.append(schedulers.AnyBranchScheduler( 459 name="gnunet-dev", 460 change_filter=util.ChangeFilter(branch_re='dev/.+', 461 repository='git://git.gnunet.org/gnunet.git'), 462 treeStableTimer=None, 463 builderNames=["gnunet-debian-x86_64-dev", 464 "gnunet-fedora-aarch64-dev"])) 465 466 SCHEDULERS.append(schedulers.SingleBranchScheduler( 467 name="tagged_release", 468 change_filter=util.ChangeFilter(branch_re='.*v[0-9]*[.][0-9]*[.][0-9]*$', 469 repository='git://git.gnunet.org/gnunet.git'), 470 treeStableTimer=None, 471 builderNames=["gnunet_release"])) 472 473 SCHEDULERS.append(schedulers.SingleBranchScheduler( 474 name="tarball", 475 onlyImportant=True, 476 change_filter=util.ChangeFilter(branch='master', 477 repository='git://git.gnunet.org/gnunet.git'), 478 fileIsImportant=checkForTarballTrigger, 479 treeStableTimer=None, 480 builderNames=["gnunet_release"])) 481 482 483 # Build a tarball nightly 484 SCHEDULERS.append(schedulers.Nightly(name='nightly', 485 change_filter=util.ChangeFilter(branch='master', 486 repository='git://git.gnunet.org/gnunet.git'), 487 builderNames=['gnunet_release'], 488 onlyIfChanged=True, 489 hour=6, minute=0)) 490 491 SCHEDULERS.append(schedulers.SingleBranchScheduler( 492 name="gnunet_cov", 493 onlyImportant=True, 494 change_filter=util.ChangeFilter(branch='master', 495 repository='git://git.gnunet.org/gnunet.git'), 496 fileIsImportant=checkForCoverityTrigger, 497 treeStableTimer=None, 498 builderNames=["gnunet_coverity"])) 499 500 SCHEDULERS.append(schedulers.SingleBranchScheduler( 501 name="gnunet_rpm", 502 change_filter=util.ChangeFilter(branch='dev/schanzen/copr', 503 repository='git://git.gnunet.org/gnunet-rpm.git'), 504 treeStableTimer=None, 505 builderNames=["gnunet_rpm_copr"])) 506 SCHEDULERS.append(schedulers.SingleBranchScheduler( 507 name="registrar_scheduler", 508 change_filter=util.ChangeFilter(branch='master', 509 repository='git://git.gnunet.org/gnunet-gns-registrar.git'), 510 treeStableTimer=None, 511 builderNames=["gnunet-gns-registrar"])) 512 513 514 # Schedule a coverity pass monthly 515 SCHEDULERS.append(schedulers.Nightly(name='nightly_cov', 516 change_filter=util.ChangeFilter(branch='master', 517 repository='git://git.gnunet.org/gnunet.git'), 518 builderNames=['gnunet_coverity'], 519 onlyIfChanged=True, 520 dayOfMonth=0, hour=3, minute=0)) 521 522 523 524 525 # 526 # WEBSITES 527 # 528 SCHEDULERS.append(schedulers.SingleBranchScheduler( 529 name="www_master_scheduler", 530 change_filter=util.ChangeFilter(branch='master', 531 repository='git://git.gnunet.org/www.git'), 532 treeStableTimer=None, 533 builderNames=["stage.gnunet.org"])) 534 535 SCHEDULERS.append(schedulers.SingleBranchScheduler( 536 name="www_stable_scheduler", 537 change_filter=util.ChangeFilter(branch='stable', 538 repository='git://git.gnunet.org/www.git'), 539 treeStableTimer=None, 540 builderNames=["www.gnunet.org"])) 541 SCHEDULERS.append(schedulers.SingleBranchScheduler( 542 name="bib_scheduler", 543 change_filter=util.ChangeFilter(branch='master', 544 repository='git://git.gnunet.org/gnunetbib.git'), 545 treeStableTimer=None, 546 builderNames=["bib.gnunet.org"])) 547 SCHEDULERS.append(schedulers.SingleBranchScheduler( 548 name="reclaim_www_scheduler", 549 change_filter=util.ChangeFilter(branch='master', 550 repository='git://git.gnunet.org/www-reclaim.git'), 551 treeStableTimer=None, 552 builderNames=["reclaim.gnunet.org"])) 553 SCHEDULERS.append(schedulers.SingleBranchScheduler( 554 name="rest_scheduler", 555 change_filter=util.ChangeFilter(branch='master', 556 repository='git://git.gnunet.org/gnunet-rest-api.git'), 557 treeStableTimer=None, 558 builderNames=["rest.gnunet.org"])) 559 SCHEDULERS.append(schedulers.SingleBranchScheduler( 560 name="lsd_scheduler", 561 change_filter=util.ChangeFilter(branch='master', 562 repository_re='git://git.gnunet.org/lsd.*'), 563 treeStableTimer=None, 564 builderNames=["lsd.gnunet.org"])) 565 SCHEDULERS.append(schedulers.SingleBranchScheduler( 566 name="gana_scheduler", 567 change_filter=util.ChangeFilter(branch='master', 568 repository='git://git.gnunet.org/gana.git'), 569 treeStableTimer=None, 570 builderNames=["gana.gnunet.org"])) 571 SCHEDULERS.append(schedulers.SingleBranchScheduler( 572 name="doc_scheduler", 573 change_filter=util.ChangeFilter(branch='master', 574 repository='git://git.gnunet.org/gnunet-handbook.git'), 575 treeStableTimer=None, 576 builderNames=["doc.gnunet.org"])) 577 578 579 # 580 # Buildbot self-reconfigure 581 # 582 SCHEDULERS.append(schedulers.SingleBranchScheduler( 583 name="bb_reload_scheduler", 584 change_filter=util.ChangeFilter(branch='master', 585 repository='git://git.gnunet.org/buildbot-ci.git'), 586 treeStableTimer=None, 587 builderNames=["buildbot"])) 588 589 590 591 def add_default_pipeline(bldr): 592 add_default_step(bldr, "build") 593 add_default_step(bldr, "install") 594 add_default_step(bldr, "test") 595 add_default_step(bldr, "deploy") 596 597 598 factory = util.BuildFactory() 599 600 601 ########################### 602 # GNUnet # 603 ########################### 604 gnunet_make_step = steps.ShellSequence( 605 name=util.Interpolate("GNUnet build"), 606 env={'GNUNET_PREFIX': '/tmp/gnunet-buildbot/lib', 607 'PATH': ["/tmp/gnunet-buildbot/bin", "${PATH}"], 608 'TMPDIR': '/tmp/gnunet/'}, 609 haltOnFailure=True, 610 commands=[ 611 util.ShellArg(command=['./bootstrap'], logname='bootstrap', haltOnFailure=True), 612 util.ShellArg(command=['./configure', 613 "--prefix=/tmp/gnunet-buildbot", 614 "--enable-experimental", 615 "--enable-logging=verbose"], 616 logname='configure', 617 haltOnFailure=True), 618 util.ShellArg(command=['make'], 619 logname='make', 620 haltOnFailure=True), 621 util.ShellArg(command=['make', 'install'], 622 logname='make install', 623 haltOnFailure=True), 624 #util.ShellArg(command=['sudo', '/home/buildbot/bin/netjail.sh', 'bash', '-c', "'make check'"], 625 # logname='make check', 626 # warnOnFailure=True, 627 # flunkOnFailure=False), # make check has issues. 628 util.ShellArg(command=['make', 'uninstall'], 629 logname='make uninstall', 630 haltOnFailure=True)] 631 ) 632 633 gnunet_build_steps = [ 634 steps.Git(repourl='git://git.gnunet.org/gnunet.git', 635 mode='full', method='fresh'), 636 gnunet_make_step 637 ] 638 639 factory.addSteps(gnunet_build_steps) 640 641 ############################ 642 # COVERITY # 643 # Occurs: 1st day of month # 644 ############################ 645 cov_factory = util.BuildFactory() 646 cov_factory.addStep(steps.Git(repourl='git://git.gnunet.org/gnunet.git', mode='full', method='fresh')) 647 cov_factory.addStep(steps.ShellSequence( 648 name=util.Interpolate("Git rev. %(prop:got_revision)s build"), 649 env={'PATH': "${HOME}/.local/bin:${HOME}/bin:${PATH}"}, 650 commands=[ 651 util.ShellArg(command=['./bootstrap'], logname='bootstrap'), 652 util.ShellArg(command=['./configure', 653 "--prefix=/tmp/gnunet-buildbot", 654 "--enable-experimental=true"], 655 logname="configure"), 656 util.ShellArg(command=['cov-build', '--dir', 'cov-int', 'make'], logname='cov-build'), 657 ])) 658 cov_factory.addStep(steps.ShellCommand(command=['tar', 'czf', 'coverity.tar.gz', 'cov-int/'], 659 haltOnFailure=True, 660 name="Packing up")) 661 cov_factory.addStep(steps.ShellCommand(command=['curl', 662 '--form', util.Interpolate('token=%(secret:coverity_token)s'), 663 '--form', 'email=mschanzenbach@posteo.de', 664 '--form', 'version="git master"', 665 '--form', 'file=@./coverity.tar.gz', 666 '--form', 'description="Buildbot triggered build"', 667 'https://scan.coverity.com/builds?project=GNUnet%2Fgnunet'], 668 haltOnFailure=True, 669 name="Sending")) 670 671 672 ################################################# 673 # RELEASE BUILD # 674 # Occurs: # 675 # 1. Nightly # 676 # 2. Upon a pushed release tag (vX.Y.Z) # 677 # 3. With a commit message containing !tarball # 678 ################################################# 679 dist_factory = util.BuildFactory() 680 # https://github.com/buildbot/buildbot/issues/6539 change to "copy" when fixed 681 dist_factory.addStep(steps.Git(repourl='git://git.gnunet.org/gnunet.git', mode='full', method='clobber')) 682 683 dist_factory.addStep(steps.ShellSequence( 684 name=util.Interpolate("Git rev. %(prop:got_revision)s build"), 685 haltOnFailure=True, 686 commands=[ 687 util.ShellArg(command=['./bootstrap'], 688 logname='bootstrap', 689 haltOnFailure=True), 690 util.ShellArg(command=['./configure'], 691 logname='configure', 692 haltOnFailure=True), 693 util.ShellArg(command=['git', 'status'], 694 logname='status before dist', 695 haltOnFailure=True), 696 util.ShellArg(command=['make', 'dist'], 697 logname="dist", 698 haltOnFailure=True), 699 util.ShellArg(command=['make', 'doxygen'], 700 haltOnFailure=True, 701 logname="Doxygen") 702 ])) 703 704 # Get version number of tarball 705 cmdmeson = r'ls -1 build/meson-dist/gnunet-*.tar.gz | sed "s/build\/meson-dist\/gnunet-//" | sed "s/.tar.gz//" | tail -n1' 706 #cmd = 'git describe --tags | sed "s/^v//"' 707 dist_factory.addStep(steps.SetPropertyFromCommand(hideStepIf=False, 708 command=cmdmeson, 709 property='gnunet_meson_releasever', 710 name="Getting release version")) 711 dist_factory.addStep(steps.ShellCommand(command=["tar", 712 "xf", 713 util.Interpolate('build/meson-dist/gnunet-%(prop:gnunet_meson_releasever)s.tar.gz'), 714 '-C', 'build'], 715 haltOnFailure=True, 716 name="Extracting tarball")) 717 718 # Make doxygen 719 # Try to build dist package 720 dist_factory.addStep(steps.ShellSequence( 721 workdir=util.Interpolate('build/build/gnunet-%(prop:gnunet_meson_releasever)s'), 722 name=util.Interpolate("GNUnet %(prop:gnunet_meson_releasever)s tarball build"), 723 env={'GNUNET_PREFIX': '/tmp/gnunet-buildbot/lib', 724 'PATH': ["/tmp/gnunet-buildbot/bin", "${PATH}"]}, 725 commands=[ 726 util.ShellArg(command=['mkdir', '-p', '$TMPDIR'],logname='tmpdir',haltOnFailure=True), 727 util.ShellArg(command=['./configure', 728 "--prefix=/tmp/gnunet-buildbot", 729 "--enable-experimental=true", 730 "--enable-logging=verbose", 731 "--mesonbuilddir=tarball_build"], 732 logname='setup', 733 haltOnFailure=True), 734 util.ShellArg(command=['make'], 735 logname='compile', 736 haltOnFailure=True), 737 util.ShellArg(command=['make', 'install'], 738 logname='install', 739 haltOnFailure=True), 740 util.ShellArg(command=['make', 'uninstall'], 741 logname='uninstall', 742 haltOnFailure=True)] 743 )) 744 745 # Upload artifact to https://buildbot.gnunet.org/artifacts 746 dist_factory.addStep(steps.FileUpload(workersrc=util.Interpolate('build/meson-dist/gnunet-%(prop:gnunet_meson_releasever)s.tar.gz'), 747 mode=0o644, 748 masterdest=util.Interpolate("~/artifacts/gnunet-%(prop:gnunet_meson_releasever)s.tar.gz"), 749 url=util.Interpolate("https://buildbot.gnunet.org/artifacts/gnunet-%(prop:gnunet_meson_releasever)s.tar.gz"))) 750 751 752 # Update doxygen (TODO skit on nightly?) 753 dist_factory.addStep(steps.ShellSequence( 754 name=util.Interpolate("Deploy Doxygen"), 755 workdir=util.Interpolate('build/doc'), 756 commands=[ 757 util.ShellArg(command=['chmod', '-R', 'ag+rX', '../doc'], 758 logname='Permissions', 759 haltOnFailure=True), 760 util.ShellArg(command=['rsync', '-a', '--delete', 761 '../doc/doxygen', 762 'handbook@firefly.gnunet.org:~/doc_deployment/'], 763 logname='Deploy', 764 haltOnFailure=True), 765 ])) 766 767 768 769 770 ########################### 771 # Fedora COPR build # 772 ########################### 773 copr_factory = util.BuildFactory() 774 copr_factory.addStep(steps.ShellCommand(command=["curl", 775 "-H", "Content-Type: application/json", 776 "--data", "{}", 777 "-X", "POST", 778 "https://copr.fedorainfracloud.org/webhooks/custom/36992/d316c169-1482-4508-a765-cfb5c0efeb67/gnunet/"], 779 haltOnFailure=True, 780 name="Triggering Copr build")) 781 782 783 ########################### 784 # gnunet-gns-registrar # 785 ########################### 786 registrar_factory = util.BuildFactory() 787 registrar_factory.addStep(steps.Git(repourl='git://git.gnunet.org/gnunet-gns-registrar.git', mode='incremental')) 788 add_default_pipeline(registrar_factory) 789 790 ########################### 791 # stage.gnunet.org # 792 ########################### 793 www_factory = util.BuildFactory() 794 www_factory.addStep(steps.Git(repourl='git://git.gnunet.org/www.git', mode='incremental')) 795 add_default_pipeline(www_factory) 796 797 ########################### 798 # www.gnunet.org # 799 ########################### 800 www_stable_factory = util.BuildFactory() 801 www_stable_factory.addStep(steps.Git(repourl='git://git.gnunet.org/www.git', mode='incremental', branch='stable')) 802 add_default_pipeline(www_stable_factory) 803 804 ########################### 805 # bib.gnunet.org # 806 ########################### 807 bib_factory = util.BuildFactory() 808 bib_factory.addStep(steps.Git(repourl='git://git.gnunet.org/gnunetbib.git', mode='incremental')) 809 add_default_pipeline(bib_factory) 810 811 812 ########################### 813 # reclaim.gnunet.org # 814 ########################### 815 reclaim_www_stable_factory = util.BuildFactory() 816 reclaim_www_stable_factory.addStep(steps.Git(repourl='git://git.gnunet.org/www-reclaim.git', mode='incremental', branch='master')) 817 add_default_pipeline(reclaim_www_stable_factory) 818 819 820 ########################### 821 # rest.gnunet.org # 822 ########################### 823 rest_factory = util.BuildFactory() 824 rest_factory.addStep(steps.Git(repourl='git://git.gnunet.org/gnunet-rest-api.git', mode='incremental', branch='master')) 825 add_default_pipeline(rest_factory) 826 827 ########################### 828 # lsd.gnunet.org # 829 ########################### 830 lsd_factory = util.BuildFactory() 831 lsd_factory.addStep(steps.Git(repourl=util.Property('repository'), mode='full', method="clobber", branch='master')) 832 add_default_pipeline(lsd_factory) 833 834 ########################### 835 # gana.gnunet.org # 836 ########################### 837 gana_factory = util.BuildFactory() 838 gana_factory.addStep(steps.Git(repourl='git://git.gnunet.org/gana.git', mode='full', method="fresh", branch='master')) 839 add_default_pipeline(gana_factory) 840 841 ############################## 842 # doc.gnunet.org (Doc-NG) # 843 ############################## 844 doc_factory = util.BuildFactory() 845 doc_factory.addStep(steps.Git(repourl='git://git.gnunet.org/gnunet-handbook.git', alwaysUseLatest=True, mode='full', method="clobber", branch='master')) 846 add_default_pipeline(doc_factory) 847 848 849 850 ########################### 851 # Buildbot TODO delete at some point # 852 ########################### 853 bb_factory = util.BuildFactory() 854 bb_factory.addStep(steps.Git(repourl='ssh://git@git.gnunet.org/buildbot-ci.git', mode='incremental')) 855 bb_factory.addStep(steps.ShellCommand(command=["./reload_bb-master.sh"], name="Reload configuration")) 856 857 858 859 BUILDERS.append( 860 util.BuilderConfig(name="gnunet-debian-x86_64", 861 workernames=["firefly-x86_64-amdepyc"], 862 factory=factory)) 863 864 BUILDERS.append( 865 util.BuilderConfig(name="gnunet-fedora-aarch64", 866 workernames=["schanzen-aarch64-fedora-meson"], 867 factory=factory)) 868 869 BUILDERS.append( 870 util.BuilderConfig(name="gnunet-debian-x86_64-dev", 871 workernames=["firefly-x86_64-amdepyc"], 872 factory=factory)) 873 874 BUILDERS.append( 875 util.BuilderConfig(name="gnunet-fedora-aarch64-dev", 876 workernames=["schanzen-aarch64-fedora-meson"], 877 factory=factory)) 878 879 880 BUILDERS.append( 881 util.BuilderConfig(name="gnunet_release", 882 workernames=["firefly-x86_64-amdepyc"], 883 factory=dist_factory)) 884 BUILDERS.append( 885 util.BuilderConfig(name="gnunet_coverity", 886 workernames=["firefly-x86_64-amdepyc"], 887 factory=cov_factory)) 888 889 BUILDERS.append( 890 util.BuilderConfig(name="gnunet-gns-registrar", 891 workernames=["firefly-x86_64-amdepyc", "schanzen-aarch64-fedora-meson"], 892 factory=registrar_factory)) 893 BUILDERS.append( 894 util.BuilderConfig(name="stage.gnunet.org", 895 workernames=["firefly-x86_64-amdepyc"], 896 factory=www_factory)) 897 BUILDERS.append( 898 util.BuilderConfig(name="www.gnunet.org", 899 workernames=["firefly-x86_64-amdepyc"], 900 factory=www_stable_factory)) 901 BUILDERS.append( 902 util.BuilderConfig(name="bib.gnunet.org", 903 workernames=["firefly-x86_64-amdepyc"], 904 factory=bib_factory)) 905 BUILDERS.append( 906 util.BuilderConfig(name="reclaim.gnunet.org", 907 workernames=["firefly-x86_64-amdepyc"], 908 factory=reclaim_www_stable_factory)) 909 BUILDERS.append( 910 util.BuilderConfig(name="rest.gnunet.org", 911 workernames=["firefly-x86_64-amdepyc"], 912 factory=rest_factory)) 913 BUILDERS.append( 914 util.BuilderConfig(name="lsd.gnunet.org", 915 workernames=["firefly-x86_64-amdepyc"], 916 factory=lsd_factory)) 917 BUILDERS.append( 918 util.BuilderConfig(name="gana.gnunet.org", 919 workernames=["firefly-x86_64-amdepyc"], 920 factory=gana_factory)) 921 BUILDERS.append( 922 util.BuilderConfig(name="doc.gnunet.org", 923 workernames=["firefly-x86_64-amdepyc"], 924 factory=doc_factory)) 925 BUILDERS.append( 926 util.BuilderConfig(name="buildbot", 927 workernames=["firefly-x86_64-amdepyc"], 928 factory=bb_factory)) 929 BUILDERS.append( 930 util.BuilderConfig(name="gnunet_rpm_copr", 931 workernames=["firefly-x86_64-amdepyc"], 932 factory=copr_factory)) 933 934 935 bsg = reporters.BuildStatusGenerator(mode=["exception", "failing", "problem"], 936 builders=["gnunet-debian-x86_64", "gnunet-fedora-aarch64"]) 937 938 mn = reporters.MailNotifier(fromaddr="buildbot@firefly.gnunet.org", 939 sendToInterestedUsers=False, 940 extraRecipients=["gnunet-ci@gnunet.org"], 941 lookup="gnunet.org", 942 generators=[bsg]) 943 SERVICES.append(mn) 944 945 946 947 948 ############################################# 949 # 19: CONTAINER FACTORY ##################### 950 ############################################# 951 ## 952 # These factories uses the standard container worker. 953 WORKERS.append(Worker("container-worker", "container-pass")) 954 955 956 # Container Job Generator Functions 957 # Return the standard configuration for a container job. 958 def default_job_config(jobName, repoName): 959 return {jobName: {"HALT_ON_FAILURE": True, 960 "WARN_ON_FAILURE": False, 961 "CONTAINER_BUILD": True, 962 "CONTAINER_NAME": repoName, 963 "CONTAINER_ARCH": "amd64"}} 964 965 966 # Parse a job config and apply it as overrides to the standard configuration. 967 def ingest_job_config(configText, jobName, repoName): 968 configDict = default_job_config(jobName, repoName) 969 jobIni = configparser.ConfigParser() 970 jobIni.optionxform = str 971 jobIni.read_string(configText) 972 configDict[jobName].update(jobIni["build"]) 973 print(configDict) 974 return configDict 975 976 977 # Search for configs, and ingest 978 def handle_job_config(jobName, repoName, configText, configResult): 979 print(configText) 980 if configResult == util.SUCCESS: 981 print(f"Ingesting Job Config: {configText}") 982 configDict = ingest_job_config(configText, jobName, repoName) 983 print(configDict) 984 return configDict 985 else: 986 print("No job config; Using default params") 987 return default_job_config(jobName, repoName) 988 989 990 class GenerateStagesCommand(buildstep.ShellMixin, steps.BuildStep): 991 992 def __init__(self, REPO_NAME, **kwargs): 993 self.REPO_NAME = REPO_NAME 994 kwargs = self.setupShellMixin(kwargs) 995 super().__init__(**kwargs) 996 self.observer = logobserver.BufferLogObserver() 997 self.addLogObserver('stdio', self.observer) 998 999 def extract_stages(self, stdout): 1000 stages = [] 1001 for line in stdout.split('\n'): 1002 stage = str(line.strip()) 1003 if stage: 1004 stages.append(stage) 1005 return stages 1006 1007 @defer.inlineCallbacks 1008 def run(self): 1009 CONTAINER_WORKDIR = f"/home/container-worker/workspace/{self.REPO_NAME}" 1010 CI_JOBS_PATH = f"{CONTAINER_WORKDIR}/contrib/ci/jobs" 1011 # run 'ls <project_root>/contrib/ci/jobs/' to get the list of stages 1012 cmd = yield self.makeRemoteShellCommand() 1013 yield self.runCommand(cmd) 1014 jobDirs = [] 1015 1016 # if the command passes extract the list of stages 1017 result = cmd.results() 1018 if result == util.SUCCESS: 1019 jobDirs = self.extract_stages(self.observer.getStdout()) 1020 print(f"this is jobDirs list: {jobDirs}") 1021 self.configDict = {} 1022 print(f"Remote cmd stdout: {self.observer.getStdout()}") 1023 print(f"cmd.results: {cmd.results()}") 1024 for stage in jobDirs: 1025 jobDirPath = f"{CI_JOBS_PATH}/{stage}" 1026 observer = logobserver.BufferLogObserver() 1027 self.addLogObserver('stdio', observer) 1028 cmd1 = yield self.makeRemoteShellCommand( 1029 command=["cat", f"{jobDirPath}/config.ini"]) 1030 yield self.runCommand(cmd1) 1031 print(f"cmd1.results: {cmd1.results()}") 1032 print(f"Second command stdout: {observer.getStdout()}") 1033 print(f"Current stage: {stage}") 1034 print(jobDirPath) 1035 self.configDict.update( 1036 handle_job_config( 1037 stage, self.REPO_NAME, 1038 observer.getStdout(), cmd1.results())) 1039 print(self.configDict) 1040 # create a container step for each stage and 1041 # add them to the build 1042 convstr2bool = ast.literal_eval 1043 self.build.addStepsAfterCurrentStep([ 1044 container_add_step( 1045 convstr2bool( 1046 str(self.configDict[stage]["HALT_ON_FAILURE"])), 1047 convstr2bool( 1048 str(self.configDict[stage]["WARN_ON_FAILURE"])), 1049 convstr2bool( 1050 str(self.configDict[stage]["CONTAINER_BUILD"])), 1051 self.configDict[stage]["CONTAINER_NAME"], 1052 container_factory, 1053 CONTAINER_WORKDIR, 1054 self.REPO_NAME, 1055 stage, 1056 self.configDict[stage]["CONTAINER_ARCH"], 1057 f"contrib/ci/jobs/{stage}/job.sh") 1058 for stage in jobDirs 1059 ]) 1060 1061 return result 1062 1063 # List of repos to add to container factory. 1064 container_repos = ["git.gnunet.org/gnunet", 1065 "git.taler.net/anastasis", 1066 "git.taler.net/challenger", 1067 "git.taler.net/donau", 1068 "git.taler.net/exchange", 1069 "git.taler.net/libeufin", 1070 "git.taler.net/taler-rust", 1071 "git.taler.net/depolymerization", 1072 "git.taler.net/merchant", 1073 "git.taler.net/sync", 1074 "git.taler.net/taler-android", 1075 "git.taler.net/taler-mailbox", 1076 "git.taler.net/taldir", 1077 "git.taler.net/taler-typescript-core",] 1078 1079 for repo in container_repos: 1080 1081 # Factory-wide variables 1082 REPO_NAME = repo.rsplit('/', 1)[1] 1083 REPO_URL = "git://" + repo + ".git" 1084 CACHE_CHECKOUT = REPO_NAME in ( 1085 "gnunet", "exchange", "merchant", "donau", "challenger", "anastasis", 1086 "sync", "libeufin", "taler-rust", "depolymerization", "taler-mailbox", 1087 "taldir", "taler-typescript-core", 1088 ) 1089 CONTAINER_WORKDIR = f"/home/container-worker/workspace/{REPO_NAME}" 1090 CI_JOBS_PATH = f"{CONTAINER_WORKDIR}/contrib/ci/jobs" 1091 1092 # Create a factory 1093 container_factory = util.BuildFactory() 1094 container_factory.workdir = CONTAINER_WORKDIR 1095 1096 # Containers can leave files owned by root. Make them writable before cleanup. 1097 workspace_command = ( 1098 f"if test -d {CONTAINER_WORKDIR}; then " 1099 f"podman run --log-driver=none --rm --volume {CONTAINER_WORKDIR}:/workdir " 1100 "docker.io/library/debian:bookworm-slim chmod -R 777 /workdir; fi" 1101 ) 1102 if CACHE_CHECKOUT: 1103 # Replace the previous shallow clone once, then retain the full checkout. 1104 workspace_command += ( 1105 f" && if test -f {CONTAINER_WORKDIR}/.git/shallow; then " 1106 f"rm -rf {CONTAINER_WORKDIR}; fi" 1107 ) 1108 else: 1109 workspace_command += f" && rm -rf {CONTAINER_WORKDIR}" 1110 workspace_command += f" && mkdir -p {CONTAINER_WORKDIR}" 1111 1112 container_factory.addStep(ShellCommand( 1113 name="workspace", 1114 descriptionDone="Workspace directory check", 1115 command=workspace_command, 1116 haltOnFailure=True, 1117 )) 1118 1119 # "fresh" cleans build outputs but retains Git history between builds. 1120 # Debian package versions require full history and all release tags. 1121 # Other container builders use a new depth-1 clone on every build. 1122 # Will checkout value of "branch" property from job properties. 1123 # https://docs.buildbot.net/latest/manual/configuration/steps/source_git.html 1124 container_factory.addStep(Git( 1125 name="git", 1126 repourl=REPO_URL, 1127 branch=util.Interpolate('%(src::branch)s'), 1128 mode='full', 1129 method='fresh' if CACHE_CHECKOUT else 'clobber', 1130 shallow=not CACHE_CHECKOUT, 1131 tags=CACHE_CHECKOUT, 1132 submodules=True, 1133 haltOnFailure=True, 1134 )) 1135 1136 container_factory.addStep(GenerateStagesCommand( 1137 REPO_NAME, 1138 name="Generate build stages", 1139 command=f"ls {CI_JOBS_PATH}", 1140 haltOnFailure=True)) 1141 1142 BUILDERS.append(util.BuilderConfig( 1143 name=f"{REPO_NAME}-builder", 1144 workernames=["container-worker"], 1145 factory=container_factory 1146 )) 1147 1148 # Only enable this scheduler for debugging! 1149 # Will run builders with 1 minute of waiting inbetween builds 1150 # SCHEDULERS.append(schedulers.Periodic( 1151 # name=f"{REPO_NAME}-minutely", 1152 # builderNames=[f"{REPO_NAME}-builder"], 1153 # periodicBuildTimer=60 1154 # )) 1155 1156 SCHEDULERS.append(schedulers.SingleBranchScheduler( 1157 name=f"{REPO_NAME}-container-scheduler", 1158 change_filter=util.ChangeFilter( 1159 branch="master", 1160 # Anchored: ChangeFilter matches the regex against the start of 1161 # the project name only, so an unanchored "(anastasis)" would 1162 # also fire for pushes to 'anastasis-www' and 'anastasis-gtk'. 1163 project_re=f"^({REPO_NAME})$" 1164 ), 1165 treeStableTimer=30, 1166 builderNames=[f"{REPO_NAME}-builder"] 1167 )) 1168 1169 SERVICES.append(reporters.MailNotifier( 1170 fromaddr="buildbot@taler.net", 1171 # notify from pass to fail, and viceversa. 1172 generators=[BuildStatusGenerator( 1173 mode=('change','problem','failing','exception',), 1174 builders=[f"{REPO_NAME}-builder",], 1175 message_formatter=reporters.MessageFormatter( 1176 template_type='plain', 1177 want_logs_content=True, 1178 ), 1179 )], 1180 sendToInterestedUsers=False, 1181 useTls=False, 1182 relayhost="localhost", 1183 smtpPort=25, 1184 dumpMailsToLog=True, 1185 extraRecipients=[f"ci-{REPO_NAME}@taler.net"] 1186 )) 1187 1188 1189 ################ 99: debug stuff JOB ################################### 1190 1191 # This does nothing, just a starting point for a factory. 1192 DEBUG_FACTORY = util.BuildFactory() 1193 DEBUG_FACTORY.addStep( 1194 ShellCommand( 1195 name="echo debug", 1196 description="just echoing a word", 1197 descriptionDone="builder responded", 1198 command=["echo", "I'm here!"] 1199 ) 1200 ) 1201 1202 1203 ################################################################## 1204 #################### General purpose ############################# 1205 ################################################################## 1206 1207 # Compute array of the names of all of our builders 1208 BUILDER_LIST = map(lambda builder: builder.name, BUILDERS) 1209 1210 ####### GENERAL PURPOSE BUILDBOT SERVICES ####################### 1211 1212 SERVICES.append(reporters.MailNotifier( 1213 fromaddr="testbuild@taler.net", 1214 # notify from pass to fail, and viceversa. 1215 generators=[BuildStatusGenerator( 1216 mode=('change','problem','failing','exception',), 1217 builders=EMAIL_ALERTS, 1218 message_formatter=reporters.MessageFormatter( 1219 template_type='plain', 1220 want_logs_content=True, 1221 ), 1222 )], 1223 sendToInterestedUsers=False, 1224 useTls=False, 1225 relayhost="localhost", 1226 smtpPort=25, 1227 dumpMailsToLog=True, 1228 extraRecipients=BUILDER_EMAIL_ADDRESSES 1229 )) 1230 1231 1232 ############# GENERAL PURPOSE SCHEDULERS ########################## 1233 1234 # Workers that are done on wallet or deployment changes to master 1235 SCHEDULERS.append(schedulers.SingleBranchScheduler( 1236 name="taler-healthcheck-scheduler", 1237 change_filter=util.ChangeFilter( 1238 branch="master", 1239 project_re="(taler-typescript-core|taler-deployment)" 1240 ), 1241 treeStableTimer=None, 1242 builderNames=WALLETCHANGE_TRIGGERS 1243 )) 1244 1245 SCHEDULERS.append(schedulers.SingleBranchScheduler( 1246 name="all-scheduler", 1247 change_filter=util.ChangeFilter( 1248 branch_re="(master|stable)", 1249 project_re="(taler-typescript-core|exchange|" 1250 "merchant|taler-deployment|twister|sync|" 1251 "taler-merchant-demos)" 1252 ), 1253 treeStableTimer=None, 1254 builderNames=CODECHANGE_TRIGGERS 1255 )) 1256 1257 # Scheduler for all nightly builds. 1258 SCHEDULERS.append(schedulers.Nightly( 1259 name="nightly-scheduler", 1260 builderNames=list(NIGHTLY_TRIGGERS), 1261 branch="master", 1262 hour=6, 1263 minute=0 1264 )) 1265 1266 # Provide "force" button in the web UI. 1267 SCHEDULERS.append(schedulers.ForceScheduler( 1268 name="force-scheduler", 1269 buttonName="Force build", 1270 builderNames=list(BUILDER_LIST) 1271 )) 1272 1273 ######################################################### 1274 ####### Actual configuation initialization ############## 1275 ######################################################### 1276 1277 # This is the dictionary that the buildmaster pays attention to. We also use 1278 # a shorter alias to save typing. 1279 c = BuildmasterConfig = {} 1280 1281 # Secrets 1282 c['secretsProviders'] = [secrets.SecretInAFile(dirname="/home/buildbot-master/secrets")] 1283 1284 c["workers"] = WORKERS 1285 c["builders"] = BUILDERS 1286 c["schedulers"] = SCHEDULERS 1287 c["services"] = SERVICES 1288 1289 # Silence warning and allow very basic phoning home. 1290 c["buildbotNetUsageData"] = "basic" 1291 1292 c["title"] = "GNUnet Builder" 1293 c["titleURL"] = "https://buildbot.gnunet.org" 1294 1295 # the 'buildbotURL' string should point to the location where the buildbot's 1296 # internal web server is visible. 1297 c['buildbotURL'] = "https://buildbot.gnunet.org/" 1298 1299 # This specifies what database buildbot uses to store its 1300 # state. You can leave this at its default for all but the 1301 # largest installations. 1302 c["db"] = { 1303 "db_url": "sqlite:///state.sqlite", 1304 } 1305 1306 # the 'change_source' setting tells the buildmaster how it should 1307 # find out about source code changes. 1308 pbSource = changes.PBChangeSource(port="tcp:19990:interface=127.0.0.1", 1309 user="git-buildbot", 1310 passwd="Aer3eari") 1311 1312 pollGnunetSource = changes.GitPoller(repourl='git://git.gnunet.org/gnunet.git', 1313 branches=True, 1314 pollInterval=300, 1315 pollAtLaunch=True, 1316 project="gnunet") 1317 c["change_source"] = [pollGnunetSource, pbSource] 1318 1319 # 'protocols' contains information about protocols which master 1320 # will use for communicating with workers. You must define at 1321 # least 'port' option that workers could connect to your master 1322 # with this protocol. 'port' must match the value configured into 1323 # the workers (with their --master option) 1324 c["protocols"] = {"pb": {"port": "tcp:19989"}} 1325 1326 # Load admin password 1327 with open("/home/buildbot-master/secrets/admin") as fh: 1328 www_pass = fh.read().strip() 1329 1330 # minimalistic config to activate new web UI 1331 # -- formerly commented out as not packaged properly in Debian and others, see 1332 # https://bugzilla.redhat.com/show_bug.cgi?id=1557687 1333 c["www"] = { 1334 "port": "tcp:8009:interface=127.0.0.1", 1335 "default_page": 'builders', 1336 "plugins": { 1337 "waterfall_view": True, 1338 "console_view": True, 1339 "grid_view": True, 1340 }, 1341 "auth": util.UserPasswordAuth([('admin',www_pass)]), 1342 "allowed_origins": ["https://*.taler.net","https://*.gnunet.org"], 1343 "avatar_methods": [], 1344 }