scripts/lttng-modules/master.groovy

   1 /**
   2  * Copyright (C) 2016-2017 - Michael Jeanson <mjeanson@efficios.com>
   3  *
   4  * This program is free software: you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation, either version 3 of the License, or
   7  * (at your option) any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16  */
  17
  18 import hudson.model.*
  19 import hudson.AbortException
  20 import hudson.console.HyperlinkNote
  21 import java.util.concurrent.CancellationException
  22 import org.eclipse.jgit.api.Git
  23 import org.eclipse.jgit.lib.Ref
  24
  25
  26 class InvalidKVersionException extends Exception {
  27   public InvalidKVersionException(String message) {
  28     super(message)
  29   }
  30 }
  31
  32 class EmptyKVersionException extends Exception {
  33   public EmptyKVersionException(String message) {
  34     super(message)
  35   }
  36 }
  37
  38 class VanillaKVersion implements Comparable<VanillaKVersion> {
  39
  40   Integer major = 0
  41   Integer majorB = 0
  42   Integer minor = 0
  43   Integer patch = 0
  44   Integer rc = Integer.MAX_VALUE
  45
  46   VanillaKVersion() {}
  47
  48   VanillaKVersion(version) {
  49     this.parse(version)
  50   }
  51
  52   static VanillaKVersion minKVersion() {
  53     return new VanillaKVersion("v0.0.0")
  54   }
  55
  56   static VanillaKVersion maxKVersion() {
  57     return new VanillaKVersion("v" + Integer.MAX_VALUE + ".0.0")
  58   }
  59
  60   static VanillaKVersion factory(version) {
  61     return new VanillaKVersion(version)
  62   }
  63
  64   def parse(version) {
  65     this.major = 0
  66     this.majorB = 0
  67     this.minor = 0
  68     this.patch = 0
  69     this.rc = Integer.MAX_VALUE
  70
  71     if (!version) {
  72       throw new EmptyKVersionException("Empty kernel version")
  73     }
  74
  75     def match = version =~ /^v(\d+)\.(\d+)(\.(\d+))?(\.(\d+))?(-rc(\d+))?$/
  76     if (!match) {
  77       throw new InvalidKVersionException("Invalid kernel version: ${version}")
  78     }
  79
  80     Integer offset = 0;
  81
  82     // Major
  83     this.major = Integer.parseInt(match.group(1))
  84     if (this.major <= 2) {
  85       offset = 2
  86       this.majorB = Integer.parseInt(match.group(2))
  87     }
  88
  89     // Minor
  90     if (match.group(2 + offset) != null) {
  91       this.minor = Integer.parseInt(match.group(2 + offset))
  92     }
  93
  94     // Patch level
  95     if (match.group(4 + offset) != null) {
  96       this.patch = Integer.parseInt(match.group(4 + offset))
  97     }
  98
  99     // RC
 100     if (match.group(8) != null) {
 101       this.rc = Integer.parseInt(match.group(8))
 102     }
 103   }
 104
 105   // Return true if this version is a release candidate
 106   Boolean isRC() {
 107     return this.rc != Integer.MAX_VALUE
 108   }
 109
 110   // Return true if both version are of the same stable branch
 111   Boolean isSameStable(VanillaKVersion o) {
 112     if (this.major != o.major) {
 113       return false
 114     }
 115     if (this.majorB != o.majorB) {
 116       return false
 117     }
 118     if (this.minor != o.minor) {
 119       return false
 120     }
 121
 122     return true
 123   }
 124
 125   @Override int compareTo(VanillaKVersion o) {
 126     if (this.major != o.major) {
 127       return Integer.compare(this.major, o.major)
 128     }
 129     if (this.majorB != o.majorB) {
 130       return Integer.compare(this.majorB, o.majorB)
 131     }
 132     if (this.minor != o.minor) {
 133       return Integer.compare(this.minor, o.minor)
 134     }
 135     if (this.patch != o.patch) {
 136       return Integer.compare(this.patch, o.patch)
 137     }
 138     if (this.rc != o.rc) {
 139       return Integer.compare(this.rc, o.rc)
 140     }
 141
 142     // Same version
 143     return 0;
 144   }
 145
 146   String toString() {
 147     String vString = "v${this.major}"
 148
 149     if (this.majorB > 0) {
 150       vString = vString.concat(".${this.majorB}")
 151     }
 152
 153     vString = vString.concat(".${this.minor}")
 154
 155     if (this.patch > 0) {
 156       vString = vString.concat(".${this.patch}")
 157     }
 158
 159     if (this.rc > 0 && this.rc < Integer.MAX_VALUE) {
 160       vString = vString.concat("-rc${this.rc}")
 161     }
 162     return vString
 163   }
 164 }
 165
 166 class UbuntuKVersion implements Comparable<UbuntuKVersion> {
 167
 168   Integer major = 0
 169   Integer minor = 0
 170   Integer patch = 0
 171   Integer umajor = 0
 172   Integer uminor = 0
 173   String suffix = ""
 174   Boolean isLTS = false
 175
 176   UbuntuKVersion() {}
 177
 178   UbuntuKVersion(version) {
 179     this.parse(version)
 180   }
 181
 182   static UbuntuKVersion minKVersion() {
 183     return new UbuntuKVersion("Ubuntu-lts-0.0.0-0.0")
 184   }
 185
 186   static UbuntuKVersion maxKVersion() {
 187     return new UbuntuKVersion("Ubuntu-" + Integer.MAX_VALUE + ".0.0-0.0")
 188   }
 189
 190   static UbuntuKVersion factory(version) {
 191     return new UbuntuKVersion(version)
 192   }
 193
 194   def parse(version) {
 195     this.major = 0
 196     this.minor = 0
 197     this.patch = 0
 198     this.umajor = 0
 199     this.uminor = 0
 200     this.suffix = "";
 201     this.isLTS = false
 202
 203     if (!version) {
 204       throw new EmptyKVersionException("Empty kernel version")
 205     }
 206
 207     //'Ubuntu-lts-4.8.0-27.29_16.04.1',
 208     //'Ubuntu-4.4.0-70.91',
 209     def match = version =~ /^Ubuntu-(lts-)??(\d+)\.(\d+)\.(\d+)-(\d+)\.(\d+)(.*)??$/
 210     if (!match) {
 211       throw new InvalidKVersionException("Invalid kernel version: ${version}")
 212     }
 213
 214     this.isLTS = match.group(1) != null
 215
 216     // Major
 217     this.major = Integer.parseInt(match.group(2))
 218
 219     // Minor
 220     this.minor = Integer.parseInt(match.group(3))
 221
 222     // Patch level
 223     this.patch = Integer.parseInt(match.group(4))
 224
 225     // Ubuntu major
 226     this.umajor = Integer.parseInt(match.group(5))
 227
 228     // Ubuntu minor
 229     this.uminor = Integer.parseInt(match.group(6))
 230
 231     if (match.group(7) != null) {
 232       this.suffix = match.group(7)
 233     }
 234   }
 235
 236   // Return true if this version is a release candidate
 237   Boolean isRC() {
 238     return false
 239   }
 240
 241   // Return true if both version are of the same stable branch
 242   Boolean isSameStable(UbuntuKVersion o) {
 243     if (this.isLTS != o.isLTS) {
 244       return false
 245     }
 246     if (this.major != o.major) {
 247       return false
 248     }
 249     if (this.minor != o.minor) {
 250       return false
 251     }
 252     if (this.patch != o.patch) {
 253       return false
 254     }
 255
 256     return true
 257   }
 258
 259   @Override int compareTo(UbuntuKVersion o) {
 260     if (this.major != o.major) {
 261       return Integer.compare(this.major, o.major)
 262     }
 263     if (this.minor != o.minor) {
 264       return Integer.compare(this.minor, o.minor)
 265     }
 266     if (this.patch != o.patch) {
 267       return Integer.compare(this.patch, o.patch)
 268     }
 269     if (this.umajor != o.umajor) {
 270       return Integer.compare(this.umajor, o.umajor)
 271     }
 272     if (this.uminor != o.uminor) {
 273       return Integer.compare(this.uminor, o.uminor)
 274     }
 275     if (this.isLTS != o.isLTS) {
 276       if (o.isLTS) {
 277         return 1
 278       } else {
 279         return -1
 280       }
 281     }
 282
 283     // Same version
 284     return 0;
 285   }
 286
 287   String toString() {
 288     String vString = "Ubuntu-"
 289
 290     if (this.isLTS) {
 291       vString = vString.concat("lts-")
 292     }
 293
 294     vString = vString.concat("${this.major}.${this.minor}.${this.patch}-${this.umajor}.${this.uminor}${this.suffix}")
 295
 296     return vString
 297   }
 298 }
 299
 300
 301 // Retrieve parameters of the current build
 302 def mversion = build.buildVariableResolver.resolve('mversion')
 303 def maxConcurrentBuild = build.buildVariableResolver.resolve('maxConcurrentBuild')
 304 def kgitrepo = build.buildVariableResolver.resolve('kgitrepo')
 305 def kverfloor_raw = build.buildVariableResolver.resolve('kverfloor')
 306 def kverceil_raw = build.buildVariableResolver.resolve('kverceil')
 307 def kverfilter = build.buildVariableResolver.resolve('kverfilter')
 308 def uversion = build.buildVariableResolver.resolve('uversion')
 309 def job = Hudson.instance.getJob(build.buildVariableResolver.resolve('kbuildjob'))
 310 def currentJobName = build.project.getFullDisplayName()
 311
 312
 313 // Get the out variable
 314 def config = new HashMap()
 315 def bindings = getBinding()
 316 config.putAll(bindings.getVariables())
 317 def out = config['out']
 318
 319
 320 // Get tags from git repository
 321 def refs = Git.lsRemoteRepository().setTags(true).setRemote(kgitrepo).call();
 322
 323 // Get kernel versions to build
 324 def kversions = []
 325 def kversionsRC = []
 326 def matchStrs = []
 327 def blacklist = []
 328 def kversionFactory = ""
 329
 330 if (uversion != null) {
 331   kversionFactory = new UbuntuKVersion()
 332   switch (uversion) {
 333     case 'xenial':
 334       matchStrs = [
 335         ~/^refs\/tags\/(Ubuntu-4\.4\.0-\d{1,3}?\.[\d]+)$/,
 336         ~/^refs\/tags\/(Ubuntu-lts-4\.8\.0-.*_16\.04\.\d+)$/,
 337         ~/^refs\/tags\/(Ubuntu-lts-4\.10\.0-.*_16\.04\.\d+)$/,
 338       ]
 339
 340       blacklist = [
 341         'Ubuntu-lts-4.10.0-7.9_16.04.1',
 342       ]
 343       break
 344
 345     case 'trusty':
 346       matchStrs = [
 347         ~/^refs\/tags\/(Ubuntu-3\.13\.0-[\d\.]+)$/,
 348         ~/^refs\/tags\/(Ubuntu-lts-.*_14\.04\.\d+)$/,
 349       ]
 350       break
 351
 352     default:
 353       println 'Unsupported Ubuntu version: ${uversion}'
 354       throw new InterruptedException()
 355       break
 356   }
 357 } else {
 358   // Vanilla
 359   kversionFactory = new VanillaKVersion()
 360   matchStrs = [
 361     ~/^refs\/tags\/(v[\d\.]+(-rc(\d+))?)$/,
 362   ]
 363 }
 364
 365 // Parse kernel versions
 366 def kverfloor = ""
 367 try {
 368     kverfloor = kversionFactory.factory(kverfloor_raw)
 369 } catch (EmptyKVersionException e) {
 370     kverfloor = kversionFactory.minKVersion()
 371 }
 372
 373 def kverceil = ""
 374 try {
 375     kverceil = kversionFactory.factory(kverceil_raw)
 376 } catch (EmptyKVersionException e) {
 377     kverceil = kversionFactory.maxKVersion()
 378 }
 379
 380 // Build a sorted list of versions to build
 381 for (ref in refs) {
 382   for (matchStr in matchStrs) {
 383     def match = ref.getName() =~ matchStr
 384     if (match && !blacklist.contains(match.group(1))) {
 385       def v = kversionFactory.factory(match.group(1))
 386
 387       if ((v >= kverfloor) && (v < kverceil)) {
 388         if (v.isRC()) {
 389           kversionsRC.add(v)
 390         } else {
 391           kversions.add(v)
 392         }
 393       }
 394     }
 395   }
 396 }
 397
 398 kversions.sort()
 399 kversionsRC.sort()
 400
 401 switch (kverfilter) {
 402   case 'stable-head':
 403     // Keep only the head of each stable branch
 404     println('Filter kernel versions to keep only the latest point release of each stable branch.')
 405
 406     for (i = 0; i < kversions.size(); i++) {
 407       def curr = kversions[i]
 408       def next = i < kversions.size() - 1 ? kversions[i + 1] : null
 409
 410       if (next != null) {
 411         if (curr.isSameStable(next)) {
 412           kversions.remove(i)
 413           i--
 414         }
 415       }
 416     }
 417     break
 418
 419   default:
 420     // No filtering of kernel versions
 421     println('No kernel versions filtering selected.')
 422     break
 423 }
 424
 425 // If the last RC version is newer than the last stable, add it to the build list
 426 if (kversionsRC.size() > 0 && kversionsRC.last() > kversions.last()) {
 427   kversions.add(kversionsRC.last())
 428 }
 429
 430 println "Building the following kernel versions:"
 431 for (k in kversions) {
 432   println k
 433 }
 434
 435 // Debug: Stop build here
 436 //throw new InterruptedException()
 437
 438 def joburl = HyperlinkNote.encodeTo('/' + job.url, job.fullDisplayName)
 439
 440 def allBuilds = []
 441 def ongoingBuild = []
 442 def failedRuns = []
 443 def isFailed = false
 444 def similarJobQueued = 0;
 445
 446 // Loop while we have kernel versions remaining or jobs running
 447 while ( kversions.size() != 0 || ongoingBuild.size() != 0 ) {
 448
 449   if(ongoingBuild.size() < maxConcurrentBuild.toInteger() && kversions.size() != 0) {
 450     def kversion = kversions.pop()
 451     def job_params = [
 452       new StringParameterValue('mversion', mversion),
 453       new StringParameterValue('ktag', kversion.toString()),
 454       new StringParameterValue('kgitrepo', kgitrepo),
 455     ]
 456
 457     // Launch the parametrized build
 458     def param_build = job.scheduleBuild2(0, new Cause.UpstreamCause(build), new ParametersAction(job_params))
 459     println "triggering ${joburl} for the ${mversion} branch on kernel ${kversion}"
 460
 461     // Add it to the ongoing build queue
 462     ongoingBuild.push(param_build)
 463
 464   } else {
 465
 466     println "Waiting... Queued: " + kversions.size() + " Running: " + ongoingBuild.size()
 467     try {
 468       Thread.sleep(10000)
 469     } catch(e) {
 470       if (e in InterruptedException) {
 471         build.setResult(hudson.model.Result.ABORTED)
 472         throw new InterruptedException()
 473       } else {
 474         throw(e)
 475       }
 476     }
 477
 478     // Abort job if a newer instance is queued
 479     similarJobQueued = Hudson.instance.queue.items.count{it.task.getFullDisplayName() == currentJobName}
 480     if ( similarJobQueued > 0 ) {
 481         build.setResult(hudson.model.Result.ABORTED)
 482         throw new InterruptedException()
 483     }
 484
 485     def i = ongoingBuild.iterator()
 486     while ( i.hasNext() ) {
 487       currentBuild = i.next()
 488       if ( currentBuild.isCancelled() || currentBuild.isDone() ) {
 489         // Remove from queue
 490         i.remove()
 491
 492         // Print results
 493         def matrixParent = currentBuild.get()
 494         allBuilds.add(matrixParent)
 495         def kernelStr = matrixParent.buildVariableResolver.resolve("ktag")
 496         println "${matrixParent.fullDisplayName} (${kernelStr}) completed with status ${matrixParent.result}"
 497
 498         // Process child runs of matrixBuild
 499         def childRuns = matrixParent.getRuns()
 500         for ( childRun in childRuns ) {
 501           println "\t${childRun.fullDisplayName} (${kernelStr}) completed with status ${childRun.result}"
 502           if (childRun.result != Result.SUCCESS) {
 503             failedRuns.add(childRun)
 504             isFailed = true
 505           }
 506         }
 507       }
 508     }
 509   }
 510 }
 511
 512 // Get log of failed runs
 513 for (failedRun in failedRuns) {
 514   println "---START---"
 515   failedRun.writeWholeLogTo(out)
 516   println "---END---"
 517 }
 518
 519 println "---Build report---"
 520 for (b in allBuilds) {
 521   def kernelStr = b.buildVariableResolver.resolve("ktag")
 522   println "${b.fullDisplayName} (${kernelStr}) completed with status ${b.result}"
 523   // Cleanup builds
 524   try {
 525     b.delete()
 526   } catch (all) {}
 527 }
 528
 529 // Mark this build failed if any child build has failed
 530 if (isFailed) {
 531   build.setResult(hudson.model.Result.FAILURE)
 532 }
 533
 534 // EOF