master.cfg (55143B)
1 # -*- python -*- 2 # ex: set syntax=python: 3 4 ## 5 # This file is part of TALER 6 # (C) 2016-2025 Taler Systems SA 7 # 8 # TALER is free software; you can redistribute it and/or 9 # modify it under the terms of the GNU Affero General Public 10 # License as published by the Free Software Foundation; either 11 # version 3, or (at your option) any later version. 12 # 13 # TALER is distributed in the hope that it will be useful, 14 # but WITHOUT ANY WARRANTY; without even the implied warranty 15 # of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 16 # See the GNU General Public License for more details. 17 # 18 # You should have received a copy of the GNU General Public 19 # License along with TALER; see the file COPYING. If not, 20 # see <http://www.gnu.org/licenses/> 21 # 22 # @author Florian Dold 23 # @author Marcello Stanisci 24 # @author ng0 25 # @author Christian Grothoff 26 # @author Devan Carpenter 27 import ast 28 import configparser 29 import glob 30 import os 31 import pathlib 32 import pwd 33 import re 34 import subprocess 35 36 from buildbot.changes.pb import PBChangeSource 37 from buildbot.steps.source.git import Git 38 from buildbot.steps.shell import ShellCommand 39 from buildbot.plugins import changes 40 from buildbot.plugins import reporters 41 from buildbot.plugins import schedulers 42 from buildbot.plugins import steps 43 from buildbot.plugins import secrets, util 44 from buildbot.process import buildstep, logobserver 45 from buildbot.process.results import SKIPPED 46 from buildbot.reporters.generators.build import BuildStatusGenerator 47 from buildbot.worker import Worker 48 from twisted.internet import defer 49 50 # This is a sample buildmaster config file. It must be 51 # installed as 'master.cfg' in your buildmaster's base 52 # directory. 53 54 # This file has the following structure: 55 # - Globals: definition of global variables we use throughout 56 # + Convenience functions: helper functions useful for many jobs 57 # - Jobs: actual job definitions 58 # - General purpose: triggers and alerts shared by various jobs 59 # + general purpose notification (alerts) 60 # + general purpose triggers (schedulers) 61 # - Actual buildbot configuration object initialization 62 63 ################################################################# 64 ######################### GLOBALS ############################### 65 ################################################################# 66 67 # The 'workers' list defines the set of recognized workers. 68 # Each element is a Worker object, specifying a unique worker 69 # name and password. The same worker name and password must 70 # be configured on the worker. 71 WORKERS = [] 72 73 # 'services' is a list of BuildbotService items like reporter 74 # targets. The status of each build will be pushed to these 75 # targets. buildbot/reporters/*.py has a variety to choose from, 76 # like IRC bots. 77 78 79 class MessageFormatterWithStdout(reporters.MessageFormatter): 80 def buildAdditionalContext(self, master, ctx): 81 stdout = [] 82 for step in ctx["build"]["steps"]: 83 for log in step["logs"]: 84 all_logs = log["content"]["content"].splitlines() 85 # Including only what the script printed on stdout. 86 for line in all_logs: 87 if re.search("^o", line): 88 stdout.append(line[1:]) 89 ctx.update(dict(stdout="\n".join(stdout))) 90 91 92 ##################################################### 93 # Commit message triggers # 94 # # 95 # This checks for triggers in the commit messages # 96 # !tarball will trigger a release build # 97 # !coverity will trigger a coverity check # 98 ##################################################### 99 def checkForTarballTrigger(change): 100 if "!tarball" in change.comments: 101 return True 102 return False 103 def checkForCoverityTrigger(change): 104 if "!coverity" in change.comments: 105 return True 106 107 SERVICES = [] 108 109 # The 'builders' list defines the Builders, which tell Buildbot 110 # how to perform a build: what steps, and which workers can execute 111 # them. Note that any particular build will only take place on 112 # one worker. 113 BUILDERS = [] 114 115 # Configures the Schedulers, which decide how to react to incoming 116 # changes. 117 SCHEDULERS = [] 118 119 # Array of builders to be scheduled every night. 120 NIGHTLY_TRIGGERS=[] 121 122 # Array of builders to be scheduled whenever any of the code Git repos change 123 CODECHANGE_TRIGGERS = [] 124 125 # Array of builders to be scheduled whenever the taler-typescript-core or 126 # taler-deployment change 127 WALLETCHANGE_TRIGGERS = [] 128 129 # Array of builder names for which build status reports should be sent 130 # via e-mail 131 EMAIL_ALERTS = [] 132 133 # Array of email address for which build status reports shoudl be sent 134 BUILDER_EMAIL_ADDRESSES = [] 135 136 ############ Convenience functions ################# 137 138 # Create a FACTORY with a taler-deployment.git checkout as the first step. 139 def create_factory_with_deployment(): 140 f = util.BuildFactory() 141 update_deployment(f) 142 return f 143 144 # Convenience function that checks out a Git repository. 145 # First argument is the URL of the Git to clone, second 146 # the desired branch. Default is 'master'. 147 def git_step(repo, target_branch="master"): 148 return Git( 149 repourl=repo, 150 mode="full", 151 method="fresh", 152 logEnviron=False, 153 alwaysUseLatest=True, 154 haltOnFailure=True, 155 branch=target_branch 156 ) 157 158 # FIXME: document this function! 159 def add_default_step(bldr, step): 160 worker_script = ".buildbot/%(prop:workername)s_" + step + ".sh" 161 default_script = ".buildbot/" + step + ".sh" 162 cmd = '[ -f %s ] && ls -1 %s || exit 0' % (worker_script,worker_script) 163 bldr.addStep(steps.SetPropertyFromCommand(command=util.Interpolate(cmd), 164 hideStepIf=True, 165 property='buildbotScript_%s' % step, 166 name="Checking for worker-specific %s script" % step)) 167 cmd = '[ -f %s ] && ls -1 %s || exit 0' % (default_script,default_script) 168 bldr.addStep(steps.SetPropertyFromCommand(doStepIf=(util.Property('buildbotScript_%s' % step) != util.Interpolate(worker_script)), 169 hideStepIf=True, 170 command=util.Interpolate(cmd), 171 property='buildbotScript_%s' % step, 172 name="Checking for %s script" % step)) 173 bldr.addStep(steps.ShellCommand(command=util.Property('buildbotScript_%s' % step), 174 haltOnFailure=True, 175 env={'PATH': "${HOME}/.local/bin:$HOME/bin:${PATH}"}, 176 doStepIf=(util.Property('buildbotScript_%s' % step) != None), 177 hideStepIf=lambda results, s: results==SKIPPED, 178 name="Executing %s step" % step)) 179 180 181 182 # Convenience function that runs 'make check' in a 183 # directory of the code inside of a netjail. 184 def jailed_check(package, srcdirs): 185 return steps.ShellSequence( 186 name="Tests of " + package, 187 description="Testing " + package, 188 descriptionDone="Pass", 189 commands=map(lambda srcdir: util.ShellArg(command=["sudo", "/usr/local/bin/netjail.sh", "/home/container-worker/taler-deployment/buildbot/with-postgres.sh", "bash", "-c", "'cd src/"+srcdir+" make install check'"]), srcdirs), 190 env={'PATH': "${HOME}/local/bin:${PATH}"}, 191 workdir="../../sources/" + package 192 ) 193 194 # Convenience function that checks out the deployment. 195 def update_deployment(factory): 196 factory.addStep(steps.ShellSequence( 197 name="update taler-deployment", 198 description="removing old deployment and fetching fresh repository", 199 descriptionDone="Deployment updated", 200 commands=[ 201 util.ShellArg(command=["rm", "-rf", "taler-deployment"]), 202 util.ShellArg(command=["git", "clone", "git://git.taler.net/taler-deployment"]), 203 ], 204 haltOnFailure=True, 205 workdir="../.." 206 )) 207 208 # Convenience function that builds and runs a container. 209 def container_add_step(HALT_ON_FAILURE, 210 WARN_ON_FAILURE, 211 CONTAINER_BUILD, 212 CONTAINER_NAME, 213 factory, 214 WORK_DIR, 215 repoName, 216 stepName, 217 CONTAINER_ARCH="amd64", 218 jobCmd="/workdir/contrib/ci/ci.sh", 219 containerFile="contrib/ci/Containerfile"): 220 print(f"HALT_ON_FAILURE: {HALT_ON_FAILURE}, WARN_ON_FAILURE: {WARN_ON_FAILURE}, CONTAINER_BUILD: {CONTAINER_BUILD}, CONTAINER_NAME: {CONTAINER_NAME}") 221 222 # Locally built images belong to this repository, job and architecture. 223 # A copied CONTAINER_NAME must not overwrite another builder's image 224 # between its build and run commands on the shared container worker. 225 # CONTAINER_NAME selects the image only for jobs using prebuilt images. 226 imageName = (f"localhost/buildbot/{repoName}/{stepName}:{CONTAINER_ARCH}" 227 if CONTAINER_BUILD else CONTAINER_NAME) 228 229 runCommand = ["podman", "--transient-store", "run", "--rm", 230 "--arch", CONTAINER_ARCH, 231 "--log-driver=none", 232 "--add-host", "taler.host.internal:10.0.2.2", 233 "--network", "slirp4netns:allow_host_loopback=true", 234 "--env", util.Interpolate("CI_COMMIT_REF=%(prop:got_revision:-%(src::revision:-unknown)s)s"), 235 "--env", util.Interpolate("CI_GIT_BRANCH=%(src::branch)s"), 236 "--env", util.Interpolate("CI_PROJECT_NAME=%(src::project)s"), 237 "--cap-add", "SYS_ADMIN,CAP_SYS_CHROOT", 238 "--volume", f"{WORK_DIR}:/workdir", 239 "--volume", "/home/container-worker/ephemeral_ci_artifacts:/artifacts", 240 "--volume", "/home/container-worker/persistent_ci_artifacts:/persistent_artifacts", 241 "--volume", "/home/container-worker/mounted_files/ci_container_id_ed25519:/root/.ssh/id_ed25519:ro", 242 "--volume", "/home/container-worker/mounted_files/container_known_hosts:/root/.ssh/known_hosts:ro", 243 "--workdir", "/workdir"] 244 245 # Inputs directory 246 inputs_path = f"/home/container-worker/container_inputs/{repoName}" 247 print(f"Checking that {inputs_path} exists") 248 if os.path.isdir(inputs_path): 249 print(f"Adding {inputs_path}") 250 runCommand += ["--volume", f"{inputs_path}:/inputs:ro"] 251 else: 252 print(f"Inputs directory not found at {inputs_path}") 253 254 if CONTAINER_BUILD: 255 runCommand += ["--volume", f"/run/user/{pwd.getpwnam('container-worker').pw_uid}/podman/podman.sock:/run/podman/podman.sock", 256 "--security-opt", "label=disable"] 257 258 runCommand += [imageName, jobCmd] 259 260 # Halting jobs (including deployment) remain fatal even if they also 261 # request warnings. Only non-halting jobs may be advisory. 262 advisory = WARN_ON_FAILURE and not HALT_ON_FAILURE 263 runArg = util.ShellArg(command=runCommand, 264 logname='run inside container', 265 flunkOnFailure=not advisory, 266 warnOnFailure=advisory, 267 haltOnFailure=HALT_ON_FAILURE) 268 269 buildArg = util.ShellArg(command=["podman", "build", "-t", imageName, 270 "--arch", CONTAINER_ARCH, 271 "-f", containerFile, "."], 272 logname='build container', haltOnFailure=True) 273 274 if not CONTAINER_BUILD: 275 return steps.ShellSequence( 276 name=stepName, 277 commands=[runArg], 278 haltOnFailure=HALT_ON_FAILURE, 279 flunkOnFailure=True, 280 warnOnWarnings=True, 281 workdir=WORK_DIR 282 ) 283 else: 284 return steps.ShellSequence( 285 name=stepName, 286 commands=[buildArg, runArg], 287 haltOnFailure=HALT_ON_FAILURE, 288 flunkOnFailure=True, 289 warnOnWarnings=True, 290 workdir=WORK_DIR 291 ) 292 293 ################################################################## 294 ######################## JOBS #################################### 295 ################################################################## 296 297 # For every job, we have (in this order!): 298 # - worker(s): hosts/users that run the job 299 # - factory: list of steps that define what to do 300 # - builder: gives the job a name and binds it to the factory and worker 301 # - (OPTIONAL) alerts: notifications to trigger when the job fails 302 # Pre-defined: EMAIL_ALERTS 303 # - scheduler: rules that define when to run the job 304 # Pre-defined: NIGHTLY_TRIGGERS, CODECHANGE_TRIGGERS, WALLETCHANGE_TRIGGERS 305 306 ################ 1: BUILDMASTER JOB ################################### 307 308 ## 309 # This worker restarts the buildmaster itself on 310 # changes to this file. 311 # Location: /home/buildbot-master 312 WORKERS.append(Worker("buildmaster-worker", "buildmaster-pass")) 313 314 BUILDMASTER_FACTORY = create_factory_with_deployment() 315 BUILDMASTER_FACTORY.addStep( 316 ShellCommand( 317 name="restart buildmaster", 318 description="trigger buildmaster restart with new configuration", 319 descriptionDone="Buildmaster updated", 320 command=["pkill", "-HUP", "-A", "-u", "buildbot-master", "-f", "/usr/bin/python3"], 321 workdir="../.." 322 ) 323 ) 324 325 BUILDERS.append(util.BuilderConfig( 326 name="buildmaster-builder", 327 workernames=["buildmaster-worker"], 328 factory=BUILDMASTER_FACTORY 329 )) 330 331 EMAIL_ALERTS.append("buildmaster-builder") 332 333 # Buildmaster is notified whenever deployment.git changes 334 SCHEDULERS.append(schedulers.SingleBranchScheduler( 335 name="buildmaster-scheduler", 336 change_filter=util.ChangeFilter( 337 branch="master", 338 project_re="(taler-deployment)" 339 ), 340 treeStableTimer=None, 341 builderNames=["buildmaster-builder"] 342 )) 343 344 ################ 2: WEBSITE JOB ################################### 345 346 ## 347 # This worker builds Websites: www and stage. 348 # 349 WORKERS.append(Worker("sites-worker", "sites-pass")) 350 351 SITES_FACTORY = create_factory_with_deployment() 352 SITES_FACTORY.addStep( 353 ShellCommand( 354 name="build Web sites", 355 description="Building all the Taler homepages", 356 descriptionDone="Sites built.", 357 command=["./build-sites.sh"], 358 workdir="../../taler-deployment/worker-sites", 359 haltOnFailure=True 360 ) 361 ) 362 363 BUILDERS.append(util.BuilderConfig( 364 name="sites-builder", workernames=["sites-worker"], factory=SITES_FACTORY 365 )) 366 367 #EMAIL_ALERTS.append("sites-builder") 368 369 370 # The web page changed if 'taler-www', 'taler-tutorials', 371 # 'taler-docs' or 'twister' changed 372 def web_page(change): 373 _change = change.asDict() 374 repo = _change.get("project") 375 if repo in ["taler-docs", "taler-tutorials", "taler-www", "twister", "taler-deployment", "buywith", "taler-ops-www", "taler-systems-www", "anastasis-www", "pepsi"]: 376 return True 377 return False 378 379 380 # Sites are re-build whenever taler-deployment, taler-www, or twister changes. 381 SCHEDULERS.append(schedulers.SingleBranchScheduler( 382 name="sites-scheduler", 383 builderNames=["sites-builder"], 384 change_filter=util.ChangeFilter( 385 branch_re="(master|stable)", 386 filter_fn=web_page 387 ), 388 treeStableTimer=None 389 )) 390 391 392 ################ 3: 'check links' JOB ################################### 393 394 ## 395 # linkchecker worker checks for dead links in the Website 396 # Location: /home/linkchecker-worker 397 WORKERS.append(Worker("linkchecker-worker", "linkchecker-pass")) 398 399 # linkchecker FACTORY 400 LINKCHECKER_FACTORY = create_factory_with_deployment() 401 LINKCHECKER_FACTORY.addStep( 402 ShellCommand( 403 name="linkchecker", 404 description="Check taler.net website for broken links && Notify", 405 descriptionDone="Results of wget in buildbot logs.", 406 command=["/home/linkchecker-worker/taler-deployment/worker-linkchecker/linkchecker.sh"], 407 workdir="/home/linkchecker-worker", 408 haltOnFailure=True, 409 timeout=7200 # 2 hours 410 ) 411 ) 412 413 # linkchecker BUILDER 414 # worker at linkchecker@taler.net 415 BUILDERS.append(util.BuilderConfig( 416 name="linkchecker-builder", 417 workernames="linkchecker-worker", 418 factory=LINKCHECKER_FACTORY 419 )) 420 421 docs_generator = BuildStatusGenerator( 422 mode=('change', 'problem', 'failing', 'exception',), 423 builders=[ 424 'linkchecker-builder', 425 ], 426 message_formatter=reporters.MessageFormatter( 427 template_type='plain', 428 want_logs_content=True 429 ) 430 ) 431 432 433 SERVICES.append(reporters.MailNotifier( 434 fromaddr="bb@taler.net", 435 generators=[docs_generator], 436 sendToInterestedUsers=False, 437 useTls=False, 438 relayhost="localhost", 439 smtpPort=25, 440 dumpMailsToLog=True, 441 extraRecipients=['linkcheck@taler.net'] 442 )) 443 444 # SERVICES.append(tipReserveEmails) 445 446 NIGHTLY_TRIGGERS.append("linkchecker-builder") 447 448 #################### 449 ## GNUnet workers ## 450 #################### 451 452 WORKERS.append(Worker("firefly-x86_64-amdepyc", "pass")) 453 WORKERS.append(Worker("mp-amd64-openbsd", "Tai7zeic")) 454 WORKERS.append(Worker("schanzen-aarch64-fedora-meson", "eWi9keet")) 455 456 457 SCHEDULERS.append(schedulers.AnyBranchScheduler( 458 name="gnunet", 459 change_filter=util.ChangeFilter(branch_re='master', 460 repository='git://git.gnunet.org/gnunet.git'), 461 treeStableTimer=None, 462 builderNames=["gnunet-debian-x86_64", 463 "gnunet-fedora-aarch64"])) 464 465 SCHEDULERS.append(schedulers.AnyBranchScheduler( 466 name="gnunet-dev", 467 change_filter=util.ChangeFilter(branch_re='dev/.+', 468 repository='git://git.gnunet.org/gnunet.git'), 469 treeStableTimer=None, 470 builderNames=["gnunet-debian-x86_64-dev", 471 "gnunet-fedora-aarch64-dev"])) 472 473 SCHEDULERS.append(schedulers.SingleBranchScheduler( 474 name="tagged_release", 475 change_filter=util.ChangeFilter(branch_re='.*v[0-9]*[.][0-9]*[.][0-9]*$', 476 repository='git://git.gnunet.org/gnunet.git'), 477 treeStableTimer=None, 478 builderNames=["gnunet_release"])) 479 480 SCHEDULERS.append(schedulers.SingleBranchScheduler( 481 name="tarball", 482 onlyImportant=True, 483 change_filter=util.ChangeFilter(branch='master', 484 repository='git://git.gnunet.org/gnunet.git'), 485 fileIsImportant=checkForTarballTrigger, 486 treeStableTimer=None, 487 builderNames=["gnunet_release"])) 488 489 490 # Build a tarball nightly 491 SCHEDULERS.append(schedulers.Nightly(name='nightly', 492 change_filter=util.ChangeFilter(branch='master', 493 repository='git://git.gnunet.org/gnunet.git'), 494 builderNames=['gnunet_release'], 495 onlyIfChanged=True, 496 hour=6, minute=0)) 497 498 SCHEDULERS.append(schedulers.SingleBranchScheduler( 499 name="gnunet_cov", 500 onlyImportant=True, 501 change_filter=util.ChangeFilter(branch='master', 502 repository='git://git.gnunet.org/gnunet.git'), 503 fileIsImportant=checkForCoverityTrigger, 504 treeStableTimer=None, 505 builderNames=["gnunet_coverity"])) 506 507 SCHEDULERS.append(schedulers.SingleBranchScheduler( 508 name="gnunet_rpm", 509 change_filter=util.ChangeFilter(branch='dev/schanzen/copr', 510 repository='git://git.gnunet.org/gnunet-rpm.git'), 511 treeStableTimer=None, 512 builderNames=["gnunet_rpm_copr"])) 513 SCHEDULERS.append(schedulers.SingleBranchScheduler( 514 name="registrar_scheduler", 515 change_filter=util.ChangeFilter(branch='master', 516 repository='git://git.gnunet.org/gnunet-gns-registrar.git'), 517 treeStableTimer=None, 518 builderNames=["gnunet-gns-registrar"])) 519 520 521 # Schedule a coverity pass monthly 522 SCHEDULERS.append(schedulers.Nightly(name='nightly_cov', 523 change_filter=util.ChangeFilter(branch='master', 524 repository='git://git.gnunet.org/gnunet.git'), 525 builderNames=['gnunet_coverity'], 526 onlyIfChanged=True, 527 dayOfMonth=0, hour=3, minute=0)) 528 529 530 531 532 # 533 # WEBSITES 534 # 535 SCHEDULERS.append(schedulers.SingleBranchScheduler( 536 name="www_master_scheduler", 537 change_filter=util.ChangeFilter(branch='master', 538 repository='git://git.gnunet.org/www.git'), 539 treeStableTimer=None, 540 builderNames=["stage.gnunet.org"])) 541 542 SCHEDULERS.append(schedulers.SingleBranchScheduler( 543 name="www_stable_scheduler", 544 change_filter=util.ChangeFilter(branch='stable', 545 repository='git://git.gnunet.org/www.git'), 546 treeStableTimer=None, 547 builderNames=["www.gnunet.org"])) 548 SCHEDULERS.append(schedulers.SingleBranchScheduler( 549 name="bib_scheduler", 550 change_filter=util.ChangeFilter(branch='master', 551 repository='git://git.gnunet.org/gnunetbib.git'), 552 treeStableTimer=None, 553 builderNames=["bib.gnunet.org"])) 554 SCHEDULERS.append(schedulers.SingleBranchScheduler( 555 name="reclaim_www_scheduler", 556 change_filter=util.ChangeFilter(branch='master', 557 repository='git://git.gnunet.org/www-reclaim.git'), 558 treeStableTimer=None, 559 builderNames=["reclaim.gnunet.org"])) 560 SCHEDULERS.append(schedulers.SingleBranchScheduler( 561 name="rest_scheduler", 562 change_filter=util.ChangeFilter(branch='master', 563 repository='git://git.gnunet.org/gnunet-rest-api.git'), 564 treeStableTimer=None, 565 builderNames=["rest.gnunet.org"])) 566 SCHEDULERS.append(schedulers.SingleBranchScheduler( 567 name="lsd_scheduler", 568 change_filter=util.ChangeFilter(branch='master', 569 repository_re='git://git.gnunet.org/lsd.*'), 570 treeStableTimer=None, 571 builderNames=["lsd.gnunet.org"])) 572 SCHEDULERS.append(schedulers.SingleBranchScheduler( 573 name="gana_scheduler", 574 change_filter=util.ChangeFilter(branch='master', 575 repository='git://git.gnunet.org/gana.git'), 576 treeStableTimer=None, 577 builderNames=["gana.gnunet.org"])) 578 SCHEDULERS.append(schedulers.SingleBranchScheduler( 579 name="doc_scheduler", 580 change_filter=util.ChangeFilter(branch='master', 581 repository='git://git.gnunet.org/gnunet-handbook.git'), 582 treeStableTimer=None, 583 builderNames=["doc.gnunet.org"])) 584 585 586 def add_default_pipeline(bldr): 587 add_default_step(bldr, "build") 588 add_default_step(bldr, "install") 589 add_default_step(bldr, "test") 590 add_default_step(bldr, "deploy") 591 592 593 factory = util.BuildFactory() 594 595 596 ########################### 597 # GNUnet # 598 ########################### 599 gnunet_make_step = steps.ShellSequence( 600 name=util.Interpolate("GNUnet build"), 601 env={'GNUNET_PREFIX': '/tmp/gnunet-buildbot/lib', 602 'PATH': ["/tmp/gnunet-buildbot/bin", "${PATH}"], 603 'TMPDIR': '/tmp/gnunet/'}, 604 haltOnFailure=True, 605 commands=[ 606 util.ShellArg(command=['./bootstrap'], logname='bootstrap', haltOnFailure=True), 607 util.ShellArg(command=['./configure', 608 "--prefix=/tmp/gnunet-buildbot", 609 "--enable-experimental", 610 "--enable-logging=verbose"], 611 logname='configure', 612 haltOnFailure=True), 613 util.ShellArg(command=['make'], 614 logname='make', 615 haltOnFailure=True), 616 util.ShellArg(command=['make', 'install'], 617 logname='make install', 618 haltOnFailure=True), 619 #util.ShellArg(command=['sudo', '/home/buildbot/bin/netjail.sh', 'bash', '-c', "'make check'"], 620 # logname='make check', 621 # warnOnFailure=True, 622 # flunkOnFailure=False), # make check has issues. 623 util.ShellArg(command=['make', 'uninstall'], 624 logname='make uninstall', 625 haltOnFailure=True)] 626 ) 627 628 gnunet_build_steps = [ 629 steps.Git(repourl='git://git.gnunet.org/gnunet.git', 630 mode='full', method='fresh'), 631 gnunet_make_step 632 ] 633 634 factory.addSteps(gnunet_build_steps) 635 636 ############################ 637 # COVERITY # 638 # Occurs: 1st day of month # 639 ############################ 640 cov_factory = util.BuildFactory() 641 cov_factory.addStep(steps.Git(repourl='git://git.gnunet.org/gnunet.git', mode='full', method='fresh')) 642 cov_factory.addStep(steps.ShellSequence( 643 name=util.Interpolate("Git rev. %(prop:got_revision)s build"), 644 env={'PATH': "${HOME}/.local/bin:${HOME}/bin:${PATH}"}, 645 commands=[ 646 util.ShellArg(command=['./bootstrap'], logname='bootstrap'), 647 util.ShellArg(command=['./configure', 648 "--prefix=/tmp/gnunet-buildbot", 649 "--enable-experimental=true"], 650 logname="configure"), 651 util.ShellArg(command=['cov-build', '--dir', 'cov-int', 'make'], logname='cov-build'), 652 ])) 653 cov_factory.addStep(steps.ShellCommand(command=['tar', 'czf', 'coverity.tar.gz', 'cov-int/'], 654 haltOnFailure=True, 655 name="Packing up")) 656 cov_factory.addStep(steps.ShellCommand(command=['curl', 657 '--form', util.Interpolate('token=%(secret:coverity_token)s'), 658 '--form', 'email=mschanzenbach@posteo.de', 659 '--form', 'version="git master"', 660 '--form', 'file=@./coverity.tar.gz', 661 '--form', 'description="Buildbot triggered build"', 662 'https://scan.coverity.com/builds?project=GNUnet%2Fgnunet'], 663 haltOnFailure=True, 664 name="Sending")) 665 666 667 ################################################# 668 # RELEASE BUILD # 669 # Occurs: # 670 # 1. Nightly # 671 # 2. Upon a pushed release tag (vX.Y.Z) # 672 # 3. With a commit message containing !tarball # 673 ################################################# 674 dist_factory = util.BuildFactory() 675 # https://github.com/buildbot/buildbot/issues/6539 change to "copy" when fixed 676 dist_factory.addStep(steps.Git(repourl='git://git.gnunet.org/gnunet.git', mode='full', method='clobber')) 677 678 dist_factory.addStep(steps.ShellSequence( 679 name=util.Interpolate("Git rev. %(prop:got_revision)s build"), 680 haltOnFailure=True, 681 commands=[ 682 util.ShellArg(command=['./bootstrap'], 683 logname='bootstrap', 684 haltOnFailure=True), 685 util.ShellArg(command=['./configure'], 686 logname='configure', 687 haltOnFailure=True), 688 util.ShellArg(command=['git', 'status'], 689 logname='status before dist', 690 haltOnFailure=True), 691 util.ShellArg(command=['make', 'dist'], 692 logname="dist", 693 haltOnFailure=True), 694 util.ShellArg(command=['make', 'doxygen'], 695 haltOnFailure=True, 696 logname="Doxygen") 697 ])) 698 699 # Get version number of tarball 700 cmdmeson = r'ls -1 build/meson-dist/gnunet-*.tar.gz | sed "s/build\/meson-dist\/gnunet-//" | sed "s/.tar.gz//" | tail -n1' 701 #cmd = 'git describe --tags | sed "s/^v//"' 702 dist_factory.addStep(steps.SetPropertyFromCommand(hideStepIf=False, 703 command=cmdmeson, 704 property='gnunet_meson_releasever', 705 name="Getting release version")) 706 dist_factory.addStep(steps.ShellCommand(command=["tar", 707 "xf", 708 util.Interpolate('build/meson-dist/gnunet-%(prop:gnunet_meson_releasever)s.tar.gz'), 709 '-C', 'build'], 710 haltOnFailure=True, 711 name="Extracting tarball")) 712 713 # Make doxygen 714 # Try to build dist package 715 dist_factory.addStep(steps.ShellSequence( 716 workdir=util.Interpolate('build/build/gnunet-%(prop:gnunet_meson_releasever)s'), 717 name=util.Interpolate("GNUnet %(prop:gnunet_meson_releasever)s tarball build"), 718 env={'GNUNET_PREFIX': '/tmp/gnunet-buildbot/lib', 719 'PATH': ["/tmp/gnunet-buildbot/bin", "${PATH}"]}, 720 commands=[ 721 util.ShellArg(command=['mkdir', '-p', '$TMPDIR'],logname='tmpdir',haltOnFailure=True), 722 util.ShellArg(command=['./configure', 723 "--prefix=/tmp/gnunet-buildbot", 724 "--enable-experimental=true", 725 "--enable-logging=verbose", 726 "--mesonbuilddir=tarball_build"], 727 logname='setup', 728 haltOnFailure=True), 729 util.ShellArg(command=['make'], 730 logname='compile', 731 haltOnFailure=True), 732 util.ShellArg(command=['make', 'install'], 733 logname='install', 734 haltOnFailure=True), 735 util.ShellArg(command=['make', 'uninstall'], 736 logname='uninstall', 737 haltOnFailure=True)] 738 )) 739 740 # Upload artifact to https://buildbot.gnunet.org/artifacts 741 dist_factory.addStep(steps.FileUpload(workersrc=util.Interpolate('build/meson-dist/gnunet-%(prop:gnunet_meson_releasever)s.tar.gz'), 742 mode=0o644, 743 masterdest=util.Interpolate("~/artifacts/gnunet-%(prop:gnunet_meson_releasever)s.tar.gz"), 744 url=util.Interpolate("https://buildbot.gnunet.org/artifacts/gnunet-%(prop:gnunet_meson_releasever)s.tar.gz"))) 745 746 747 # Update doxygen (TODO skit on nightly?) 748 dist_factory.addStep(steps.ShellSequence( 749 name=util.Interpolate("Deploy Doxygen"), 750 workdir=util.Interpolate('build/doc'), 751 commands=[ 752 util.ShellArg(command=['chmod', '-R', 'ag+rX', '../doc'], 753 logname='Permissions', 754 haltOnFailure=True), 755 util.ShellArg(command=['rsync', '-a', '--delete', 756 '../doc/doxygen', 757 'handbook@firefly.gnunet.org:~/doc_deployment/'], 758 logname='Deploy', 759 haltOnFailure=True), 760 ])) 761 762 763 764 765 ########################### 766 # Fedora COPR build # 767 ########################### 768 copr_factory = util.BuildFactory() 769 copr_factory.addStep(steps.ShellCommand(command=["curl", 770 "-H", "Content-Type: application/json", 771 "--data", "{}", 772 "-X", "POST", 773 "https://copr.fedorainfracloud.org/webhooks/custom/36992/d316c169-1482-4508-a765-cfb5c0efeb67/gnunet/"], 774 haltOnFailure=True, 775 name="Triggering Copr build")) 776 777 778 ########################### 779 # gnunet-gns-registrar # 780 ########################### 781 registrar_factory = util.BuildFactory() 782 registrar_factory.addStep(steps.Git(repourl='git://git.gnunet.org/gnunet-gns-registrar.git', mode='incremental')) 783 add_default_pipeline(registrar_factory) 784 785 ########################### 786 # stage.gnunet.org # 787 ########################### 788 www_factory = util.BuildFactory() 789 www_factory.addStep(steps.Git(repourl='git://git.gnunet.org/www.git', mode='incremental')) 790 add_default_pipeline(www_factory) 791 792 ########################### 793 # www.gnunet.org # 794 ########################### 795 www_stable_factory = util.BuildFactory() 796 www_stable_factory.addStep(steps.Git(repourl='git://git.gnunet.org/www.git', mode='incremental', branch='stable')) 797 add_default_pipeline(www_stable_factory) 798 799 ########################### 800 # bib.gnunet.org # 801 ########################### 802 bib_factory = util.BuildFactory() 803 bib_factory.addStep(steps.Git(repourl='git://git.gnunet.org/gnunetbib.git', mode='incremental')) 804 add_default_pipeline(bib_factory) 805 806 807 ########################### 808 # reclaim.gnunet.org # 809 ########################### 810 reclaim_www_stable_factory = util.BuildFactory() 811 reclaim_www_stable_factory.addStep(steps.Git(repourl='git://git.gnunet.org/www-reclaim.git', mode='incremental', branch='master')) 812 add_default_pipeline(reclaim_www_stable_factory) 813 814 815 ########################### 816 # rest.gnunet.org # 817 ########################### 818 rest_factory = util.BuildFactory() 819 rest_factory.addStep(steps.Git(repourl='git://git.gnunet.org/gnunet-rest-api.git', mode='incremental', branch='master')) 820 add_default_pipeline(rest_factory) 821 822 ########################### 823 # lsd.gnunet.org # 824 ########################### 825 lsd_factory = util.BuildFactory() 826 lsd_factory.addStep(steps.Git(repourl=util.Property('repository'), mode='full', method="clobber", branch='master')) 827 add_default_pipeline(lsd_factory) 828 829 ########################### 830 # gana.gnunet.org # 831 ########################### 832 gana_factory = util.BuildFactory() 833 gana_factory.addStep(steps.Git(repourl='git://git.gnunet.org/gana.git', mode='full', method="fresh", branch='master')) 834 add_default_pipeline(gana_factory) 835 836 ############################## 837 # doc.gnunet.org (Doc-NG) # 838 ############################## 839 doc_factory = util.BuildFactory() 840 doc_factory.addStep(steps.Git(repourl='git://git.gnunet.org/gnunet-handbook.git', alwaysUseLatest=True, mode='full', method="clobber", branch='master')) 841 add_default_pipeline(doc_factory) 842 843 844 845 BUILDERS.append( 846 util.BuilderConfig(name="gnunet-debian-x86_64", 847 workernames=["firefly-x86_64-amdepyc"], 848 factory=factory)) 849 850 BUILDERS.append( 851 util.BuilderConfig(name="gnunet-fedora-aarch64", 852 workernames=["schanzen-aarch64-fedora-meson"], 853 factory=factory)) 854 855 BUILDERS.append( 856 util.BuilderConfig(name="gnunet-debian-x86_64-dev", 857 workernames=["firefly-x86_64-amdepyc"], 858 factory=factory)) 859 860 BUILDERS.append( 861 util.BuilderConfig(name="gnunet-fedora-aarch64-dev", 862 workernames=["schanzen-aarch64-fedora-meson"], 863 factory=factory)) 864 865 866 BUILDERS.append( 867 util.BuilderConfig(name="gnunet_release", 868 workernames=["firefly-x86_64-amdepyc"], 869 factory=dist_factory)) 870 BUILDERS.append( 871 util.BuilderConfig(name="gnunet_coverity", 872 workernames=["firefly-x86_64-amdepyc"], 873 factory=cov_factory)) 874 875 BUILDERS.append( 876 util.BuilderConfig(name="gnunet-gns-registrar", 877 workernames=["firefly-x86_64-amdepyc", "schanzen-aarch64-fedora-meson"], 878 factory=registrar_factory)) 879 BUILDERS.append( 880 util.BuilderConfig(name="stage.gnunet.org", 881 workernames=["firefly-x86_64-amdepyc"], 882 factory=www_factory)) 883 BUILDERS.append( 884 util.BuilderConfig(name="www.gnunet.org", 885 workernames=["firefly-x86_64-amdepyc"], 886 factory=www_stable_factory)) 887 BUILDERS.append( 888 util.BuilderConfig(name="bib.gnunet.org", 889 workernames=["firefly-x86_64-amdepyc"], 890 factory=bib_factory)) 891 BUILDERS.append( 892 util.BuilderConfig(name="reclaim.gnunet.org", 893 workernames=["firefly-x86_64-amdepyc"], 894 factory=reclaim_www_stable_factory)) 895 BUILDERS.append( 896 util.BuilderConfig(name="rest.gnunet.org", 897 workernames=["firefly-x86_64-amdepyc"], 898 factory=rest_factory)) 899 BUILDERS.append( 900 util.BuilderConfig(name="lsd.gnunet.org", 901 workernames=["firefly-x86_64-amdepyc"], 902 factory=lsd_factory)) 903 BUILDERS.append( 904 util.BuilderConfig(name="gana.gnunet.org", 905 workernames=["firefly-x86_64-amdepyc"], 906 factory=gana_factory)) 907 BUILDERS.append( 908 util.BuilderConfig(name="doc.gnunet.org", 909 workernames=["firefly-x86_64-amdepyc"], 910 factory=doc_factory)) 911 BUILDERS.append( 912 util.BuilderConfig(name="gnunet_rpm_copr", 913 workernames=["firefly-x86_64-amdepyc"], 914 factory=copr_factory)) 915 916 917 bsg = reporters.BuildStatusGenerator(mode=["exception", "failing", "problem"], 918 builders=["gnunet-debian-x86_64", "gnunet-fedora-aarch64"]) 919 920 mn = reporters.MailNotifier(fromaddr="buildbot@firefly.gnunet.org", 921 sendToInterestedUsers=False, 922 extraRecipients=["gnunet-ci@gnunet.org"], 923 lookup="gnunet.org", 924 generators=[bsg]) 925 SERVICES.append(mn) 926 927 928 929 930 ############################################# 931 # 19: CONTAINER FACTORY ##################### 932 ############################################# 933 ## 934 # These factories uses the standard container worker. 935 WORKERS.append(Worker("container-worker", "container-pass")) 936 937 938 # Container Job Generator Functions 939 # Return the standard configuration for a container job. 940 def default_job_config(jobName, repoName): 941 return {jobName: {"HALT_ON_FAILURE": True, 942 "WARN_ON_FAILURE": False, 943 "CONTAINER_BUILD": True, 944 "CONTAINER_NAME": repoName, 945 "CONTAINER_ARCH": "amd64"}} 946 947 948 # Parse a job config and apply it as overrides to the standard configuration. 949 def ingest_job_config(configText, jobName, repoName): 950 configDict = default_job_config(jobName, repoName) 951 jobIni = configparser.ConfigParser() 952 jobIni.optionxform = str 953 jobIni.read_string(configText) 954 configDict[jobName].update(jobIni["build"]) 955 print(configDict) 956 return configDict 957 958 959 # Search for configs, and ingest 960 def handle_job_config(jobName, repoName, configText, configResult): 961 print(configText) 962 if configResult == util.SUCCESS: 963 print(f"Ingesting Job Config: {configText}") 964 configDict = ingest_job_config(configText, jobName, repoName) 965 print(configDict) 966 return configDict 967 else: 968 print("No job config; Using default params") 969 return default_job_config(jobName, repoName) 970 971 972 class GenerateStagesCommand(buildstep.ShellMixin, steps.BuildStep): 973 974 def __init__(self, REPO_NAME, **kwargs): 975 self.REPO_NAME = REPO_NAME 976 kwargs = self.setupShellMixin(kwargs) 977 super().__init__(**kwargs) 978 self.observer = logobserver.BufferLogObserver() 979 self.addLogObserver('stdio', self.observer) 980 981 def extract_stages(self, stdout): 982 stages = [] 983 for line in stdout.split('\n'): 984 stage = str(line.strip()) 985 if stage: 986 stages.append(stage) 987 return stages 988 989 @defer.inlineCallbacks 990 def run(self): 991 CONTAINER_WORKDIR = f"/home/container-worker/workspace/{self.REPO_NAME}" 992 CI_JOBS_PATH = f"{CONTAINER_WORKDIR}/contrib/ci/jobs" 993 # run 'ls <project_root>/contrib/ci/jobs/' to get the list of stages 994 cmd = yield self.makeRemoteShellCommand() 995 yield self.runCommand(cmd) 996 jobDirs = [] 997 998 # if the command passes extract the list of stages 999 result = cmd.results() 1000 if result == util.SUCCESS: 1001 jobDirs = self.extract_stages(self.observer.getStdout()) 1002 print(f"this is jobDirs list: {jobDirs}") 1003 self.configDict = {} 1004 print(f"Remote cmd stdout: {self.observer.getStdout()}") 1005 print(f"cmd.results: {cmd.results()}") 1006 for stage in jobDirs: 1007 jobDirPath = f"{CI_JOBS_PATH}/{stage}" 1008 observer = logobserver.BufferLogObserver() 1009 self.addLogObserver('stdio', observer) 1010 cmd1 = yield self.makeRemoteShellCommand( 1011 command=["cat", f"{jobDirPath}/config.ini"]) 1012 yield self.runCommand(cmd1) 1013 print(f"cmd1.results: {cmd1.results()}") 1014 print(f"Second command stdout: {observer.getStdout()}") 1015 print(f"Current stage: {stage}") 1016 print(jobDirPath) 1017 self.configDict.update( 1018 handle_job_config( 1019 stage, self.REPO_NAME, 1020 observer.getStdout(), cmd1.results())) 1021 print(self.configDict) 1022 # create a container step for each stage and 1023 # add them to the build 1024 convstr2bool = ast.literal_eval 1025 self.build.addStepsAfterCurrentStep([ 1026 container_add_step( 1027 convstr2bool( 1028 str(self.configDict[stage]["HALT_ON_FAILURE"])), 1029 convstr2bool( 1030 str(self.configDict[stage]["WARN_ON_FAILURE"])), 1031 convstr2bool( 1032 str(self.configDict[stage]["CONTAINER_BUILD"])), 1033 self.configDict[stage]["CONTAINER_NAME"], 1034 container_factory, 1035 CONTAINER_WORKDIR, 1036 self.REPO_NAME, 1037 stage, 1038 self.configDict[stage]["CONTAINER_ARCH"], 1039 f"contrib/ci/jobs/{stage}/job.sh") 1040 for stage in jobDirs 1041 ]) 1042 1043 return result 1044 1045 # MHD2 uses a deployment-owned Debian overlay and a reviewed upstream pin. 1046 # Both architectures must pass before either is uploaded to the nightly feed. 1047 MHD2_WORKDIR = "/home/container-worker/workspace/libmicrohttpd2-ci" 1048 mhd2_factory = util.BuildFactory() 1049 mhd2_factory.addStep(Git( 1050 name="git", repourl="git://git.taler.net/taler-deployment.git", 1051 branch="master", mode="full", method="fresh", workdir=MHD2_WORKDIR, 1052 haltOnFailure=True)) 1053 for arch in ("amd64", "arm64"): 1054 mhd2_factory.addStep(container_add_step( 1055 True, False, True, "unused", mhd2_factory, MHD2_WORKDIR, 1056 "libmicrohttpd2", f"build-{arch}", arch, 1057 "buildbot/libmicrohttpd2/build.sh", 1058 "buildbot/libmicrohttpd2/Containerfile")) 1059 mhd2_factory.addStep(container_add_step( 1060 True, False, False, "localhost/buildbot/libmicrohttpd2/build-amd64:amd64", 1061 mhd2_factory, MHD2_WORKDIR, "libmicrohttpd2", "upload", "amd64", 1062 "buildbot/libmicrohttpd2/upload.sh")) 1063 BUILDERS.append(util.BuilderConfig( 1064 name="libmicrohttpd2-builder", workernames=["container-worker"], 1065 factory=mhd2_factory)) 1066 1067 1068 def mhd2_packaging_changed(change): 1069 return any(path == "buildbot/master.cfg" or path.startswith(( 1070 "buildbot/libmicrohttpd2/", 1071 "packaging/ng/buildscripts/debian-overlays/libmicrohttpd2/")) 1072 for path in change.files) 1073 1074 1075 SCHEDULERS.append(schedulers.SingleBranchScheduler( 1076 name="libmicrohttpd2-packaging-scheduler", 1077 change_filter=util.ChangeFilter(branch="master", project_re="^taler-deployment$"), 1078 fileIsImportant=mhd2_packaging_changed, treeStableTimer=30, 1079 builderNames=["libmicrohttpd2-builder"])) 1080 1081 # List of repos to add to container factory. 1082 container_repos = ["git.gnunet.org/gnunet", 1083 "git.taler.net/anastasis", 1084 "git.taler.net/challenger", 1085 "git.taler.net/donau", 1086 "git.taler.net/exchange", 1087 "git.taler.net/libeufin", 1088 "git.taler.net/taler-rust", 1089 "git.taler.net/depolymerization", 1090 "git.taler.net/merchant", 1091 "git.taler.net/sync", 1092 "git.taler.net/taler-android", 1093 "git.taler.net/taler-mailbox", 1094 "git.taler.net/taldir", 1095 "git.taler.net/taler-typescript-core",] 1096 1097 for repo in container_repos: 1098 1099 # Factory-wide variables 1100 REPO_NAME = repo.rsplit('/', 1)[1] 1101 REPO_URL = "git://" + repo + ".git" 1102 CACHE_CHECKOUT = REPO_NAME in ( 1103 "gnunet", "exchange", "merchant", "donau", "challenger", "anastasis", 1104 "sync", "libeufin", "taler-rust", "depolymerization", "taler-mailbox", 1105 "taldir", "taler-typescript-core", 1106 ) 1107 CONTAINER_WORKDIR = f"/home/container-worker/workspace/{REPO_NAME}" 1108 CI_JOBS_PATH = f"{CONTAINER_WORKDIR}/contrib/ci/jobs" 1109 1110 # Create a factory 1111 container_factory = util.BuildFactory() 1112 container_factory.workdir = CONTAINER_WORKDIR 1113 1114 # Containers can leave files owned by root. Make them writable before cleanup. 1115 workspace_command = ( 1116 f"if test -d {CONTAINER_WORKDIR}; then " 1117 f"podman run --log-driver=none --rm --volume {CONTAINER_WORKDIR}:/workdir " 1118 "docker.io/library/debian:bookworm-slim chmod -R 777 /workdir; fi" 1119 ) 1120 if CACHE_CHECKOUT: 1121 # Replace the previous shallow clone once, then retain the full checkout. 1122 workspace_command += ( 1123 f" && if test -f {CONTAINER_WORKDIR}/.git/shallow; then " 1124 f"rm -rf {CONTAINER_WORKDIR}; fi" 1125 ) 1126 else: 1127 workspace_command += f" && rm -rf {CONTAINER_WORKDIR}" 1128 workspace_command += f" && mkdir -p {CONTAINER_WORKDIR}" 1129 1130 container_factory.addStep(ShellCommand( 1131 name="workspace", 1132 descriptionDone="Workspace directory check", 1133 command=workspace_command, 1134 haltOnFailure=True, 1135 )) 1136 1137 # "fresh" cleans build outputs but retains Git history between builds. 1138 # Debian package versions require full history and all release tags. 1139 # Other container builders use a new depth-1 clone on every build. 1140 # Will checkout value of "branch" property from job properties. 1141 # https://docs.buildbot.net/latest/manual/configuration/steps/source_git.html 1142 container_factory.addStep(Git( 1143 name="git", 1144 repourl=REPO_URL, 1145 branch=util.Interpolate('%(src::branch)s'), 1146 mode='full', 1147 method='fresh' if CACHE_CHECKOUT else 'clobber', 1148 shallow=not CACHE_CHECKOUT, 1149 tags=CACHE_CHECKOUT, 1150 submodules=True, 1151 haltOnFailure=True, 1152 )) 1153 1154 container_factory.addStep(GenerateStagesCommand( 1155 REPO_NAME, 1156 name="Generate build stages", 1157 command=f"ls {CI_JOBS_PATH}", 1158 haltOnFailure=True)) 1159 1160 BUILDERS.append(util.BuilderConfig( 1161 name=f"{REPO_NAME}-builder", 1162 workernames=["container-worker"], 1163 factory=container_factory 1164 )) 1165 1166 # Only enable this scheduler for debugging! 1167 # Will run builders with 1 minute of waiting inbetween builds 1168 # SCHEDULERS.append(schedulers.Periodic( 1169 # name=f"{REPO_NAME}-minutely", 1170 # builderNames=[f"{REPO_NAME}-builder"], 1171 # periodicBuildTimer=60 1172 # )) 1173 1174 SCHEDULERS.append(schedulers.SingleBranchScheduler( 1175 name=f"{REPO_NAME}-container-scheduler", 1176 change_filter=util.ChangeFilter( 1177 branch="master", 1178 # Anchored: ChangeFilter matches the regex against the start of 1179 # the project name only, so an unanchored "(anastasis)" would 1180 # also fire for pushes to 'anastasis-www' and 'anastasis-gtk'. 1181 project_re=f"^({REPO_NAME})$" 1182 ), 1183 treeStableTimer=30, 1184 builderNames=[f"{REPO_NAME}-builder"] 1185 )) 1186 1187 SERVICES.append(reporters.MailNotifier( 1188 fromaddr="buildbot@taler.net", 1189 # notify from pass to fail, and viceversa. 1190 generators=[BuildStatusGenerator( 1191 mode=('change','problem','failing','exception',), 1192 builders=[f"{REPO_NAME}-builder",], 1193 message_formatter=reporters.MessageFormatter( 1194 template_type='plain', 1195 want_logs_content=True, 1196 ), 1197 )], 1198 sendToInterestedUsers=False, 1199 useTls=False, 1200 relayhost="localhost", 1201 smtpPort=25, 1202 dumpMailsToLog=True, 1203 extraRecipients=[f"ci-{REPO_NAME}@taler.net"] 1204 )) 1205 1206 1207 ################ 99: debug stuff JOB ################################### 1208 1209 # This does nothing, just a starting point for a factory. 1210 DEBUG_FACTORY = util.BuildFactory() 1211 DEBUG_FACTORY.addStep( 1212 ShellCommand( 1213 name="echo debug", 1214 description="just echoing a word", 1215 descriptionDone="builder responded", 1216 command=["echo", "I'm here!"] 1217 ) 1218 ) 1219 1220 1221 ################################################################## 1222 #################### General purpose ############################# 1223 ################################################################## 1224 1225 # Compute array of the names of all of our builders 1226 BUILDER_LIST = map(lambda builder: builder.name, BUILDERS) 1227 1228 ####### GENERAL PURPOSE BUILDBOT SERVICES ####################### 1229 1230 SERVICES.append(reporters.MailNotifier( 1231 fromaddr="testbuild@taler.net", 1232 # notify from pass to fail, and viceversa. 1233 generators=[BuildStatusGenerator( 1234 mode=('change','problem','failing','exception',), 1235 builders=EMAIL_ALERTS, 1236 message_formatter=reporters.MessageFormatter( 1237 template_type='plain', 1238 want_logs_content=True, 1239 ), 1240 )], 1241 sendToInterestedUsers=False, 1242 useTls=False, 1243 relayhost="localhost", 1244 smtpPort=25, 1245 dumpMailsToLog=True, 1246 extraRecipients=BUILDER_EMAIL_ADDRESSES 1247 )) 1248 1249 1250 ############# GENERAL PURPOSE SCHEDULERS ########################## 1251 1252 # Workers that are done on wallet or deployment changes to master 1253 SCHEDULERS.append(schedulers.SingleBranchScheduler( 1254 name="taler-healthcheck-scheduler", 1255 change_filter=util.ChangeFilter( 1256 branch="master", 1257 project_re="(taler-typescript-core|taler-deployment)" 1258 ), 1259 treeStableTimer=None, 1260 builderNames=WALLETCHANGE_TRIGGERS 1261 )) 1262 1263 SCHEDULERS.append(schedulers.SingleBranchScheduler( 1264 name="all-scheduler", 1265 change_filter=util.ChangeFilter( 1266 branch_re="(master|stable)", 1267 project_re="(taler-typescript-core|exchange|" 1268 "merchant|taler-deployment|twister|sync|" 1269 "taler-merchant-demos)" 1270 ), 1271 treeStableTimer=None, 1272 builderNames=CODECHANGE_TRIGGERS 1273 )) 1274 1275 # Scheduler for all nightly builds. 1276 SCHEDULERS.append(schedulers.Nightly( 1277 name="nightly-scheduler", 1278 builderNames=list(NIGHTLY_TRIGGERS), 1279 branch="master", 1280 hour=6, 1281 minute=0 1282 )) 1283 1284 # Provide "force" button in the web UI. 1285 SCHEDULERS.append(schedulers.ForceScheduler( 1286 name="force-scheduler", 1287 buttonName="Force build", 1288 builderNames=list(BUILDER_LIST) 1289 )) 1290 1291 ######################################################### 1292 ####### Actual configuation initialization ############## 1293 ######################################################### 1294 1295 # This is the dictionary that the buildmaster pays attention to. We also use 1296 # a shorter alias to save typing. 1297 c = BuildmasterConfig = {} 1298 1299 # Secrets 1300 c['secretsProviders'] = [secrets.SecretInAFile(dirname="/home/buildbot-master/secrets")] 1301 1302 c["workers"] = WORKERS 1303 c["builders"] = BUILDERS 1304 c["schedulers"] = SCHEDULERS 1305 c["services"] = SERVICES 1306 1307 # Silence warning and allow very basic phoning home. 1308 c["buildbotNetUsageData"] = "basic" 1309 1310 c["title"] = "GNUnet Builder" 1311 c["titleURL"] = "https://buildbot.gnunet.org" 1312 1313 # the 'buildbotURL' string should point to the location where the buildbot's 1314 # internal web server is visible. 1315 c['buildbotURL'] = "https://buildbot.gnunet.org/" 1316 1317 # This specifies what database buildbot uses to store its 1318 # state. You can leave this at its default for all but the 1319 # largest installations. 1320 c["db"] = { 1321 "db_url": "sqlite:///state.sqlite", 1322 } 1323 1324 # the 'change_source' setting tells the buildmaster how it should 1325 # find out about source code changes. 1326 pbSource = changes.PBChangeSource(port="tcp:19990:interface=127.0.0.1", 1327 user="git-buildbot", 1328 passwd="Aer3eari") 1329 1330 pollGnunetSource = changes.GitPoller(repourl='git://git.gnunet.org/gnunet.git', 1331 branches=True, 1332 pollInterval=300, 1333 pollAtLaunch=True, 1334 project="gnunet") 1335 c["change_source"] = [pollGnunetSource, pbSource] 1336 1337 # 'protocols' contains information about protocols which master 1338 # will use for communicating with workers. You must define at 1339 # least 'port' option that workers could connect to your master 1340 # with this protocol. 'port' must match the value configured into 1341 # the workers (with their --master option) 1342 c["protocols"] = {"pb": {"port": "tcp:19989"}} 1343 1344 # Load admin password 1345 with open("/home/buildbot-master/secrets/admin") as fh: 1346 www_pass = fh.read().strip() 1347 1348 # minimalistic config to activate new web UI 1349 # -- formerly commented out as not packaged properly in Debian and others, see 1350 # https://bugzilla.redhat.com/show_bug.cgi?id=1557687 1351 c["www"] = { 1352 "port": "tcp:8009:interface=127.0.0.1", 1353 "default_page": 'builders', 1354 "plugins": { 1355 "waterfall_view": True, 1356 "console_view": True, 1357 "grid_view": True, 1358 }, 1359 "auth": util.UserPasswordAuth([('admin',www_pass)]), 1360 "authz": util.Authz( 1361 allowRules=[ 1362 # Grants control rights (force, stop, rebuild) exclusively to the 'admins' role 1363 util.AnyControlEndpointMatcher(role="admins") 1364 ], 1365 roleMatchers=[ 1366 # Maps 'admin' user profile to the 'admins' role 1367 util.RolesFromUsername(roles=["admins"], usernames=["admin"]) 1368 ] 1369 ), 1370 "allowed_origins": ["https://*.taler.net","https://*.gnunet.org"], 1371 "avatar_methods": [], 1372 }