#!/usr/bin/env python # Copyright 2014 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """An auto-roller for GN binaries into Chromium. This script is used to update the GN binaries that a Chromium checkout uses. In order to update the binaries, one must follow four steps in order: 1. Trigger try jobs to build a new GN binary at tip-of-tree and upload the newly-built binaries into the right Google CloudStorage bucket. 2. Wait for the try jobs to complete. 3. Update the buildtools repo with the .sha1 hashes of the newly built binaries. 4. Update Chromium's DEPS file to the new version of the buildtools repo. The script has four commands that correspond to the four steps above: 'build', 'wait', 'roll_buildtools', and 'roll_deps'. The script has a fifth command, 'roll', that runs the four in order. If given no arguments, the script will run the 'roll' command. It can only be run on linux in a clean Chromium checkout; it should error out in most cases if something bad happens, but the error checking isn't yet foolproof. """ from __future__ import print_function import argparse import json import os import re import subprocess import sys import tempfile import time import urllib2 depot_tools_path = os.path.abspath(os.path.normpath(os.path.join( os.path.dirname(__file__), '..', '..', '..', 'third_party', 'depot_tools'))) if not depot_tools_path in sys.path: sys.path.insert(0, depot_tools_path) CHROMIUM_REPO = 'https://chromium.googlesource.com/chromium/src.git' COMMITISH_DIGITS = 10 UNKNOWN, PENDING, STARTED, SUCCESS = ( 'unknown', 'pending', 'started', 'success') class BuildResult(object): def __init__(self): self.masterName = '-' self.builderName = '-' self.buildNumber = '-' self.state = UNKNOWN self.sha1 = '-' self.url = '-' class GNRoller(object): def __init__(self): self.chromium_src_dir = None self.buildtools_dir = None self.old_gn_commitish = None self.new_gn_commitish = None self.old_gn_version = None self.new_gn_version = None self.reviewer = 'dpranke@chromium.org' if os.getenv('USER') == 'dpranke': self.reviewer = 'brettw@chromium.org' def Roll(self): parser = argparse.ArgumentParser() parser.usage = __doc__ parser.add_argument('command', nargs='?', default='roll', help='build|roll|roll_buildtools|roll_deps|wait' ' (%(default)s is the default)') args = parser.parse_args() command = args.command ret = self.SetUp() if not ret and command in ('roll', 'build'): ret = self.TriggerBuild() if not ret and command in ('roll', 'wait'): ret = self.WaitForBuildToFinish() if not ret and command in ('roll', 'roll_buildtools'): ret = self.RollBuildtools() if not ret and command in ('roll', 'roll_deps'): ret = self.RollDEPS() return ret def SetUp(self): if sys.platform not in ('darwin', 'linux2'): print('roll_gn is only tested and working on Linux and Mac for now.') return 1 ret, out, _ = self.Call('git config --get remote.origin.url') origin = out.strip() if ret or origin != CHROMIUM_REPO: print('Not in a Chromium repo? git config --get remote.origin.url ' 'returned %d: %s' % (ret, origin)) return 1 ret, _, _ = self.Call('git diff -q') if ret: print("Checkout is dirty, exiting") return 1 _, out, _ = self.Call('git rev-parse --show-toplevel', cwd=os.getcwd()) self.chromium_src_dir = out.strip() self.buildtools_dir = os.path.join(self.chromium_src_dir, 'buildtools') self.new_gn_commitish, self.new_gn_version = self.GetNewVersions() _, out, _ = self.Call('gn --version') self.old_gn_version = out.strip() _, out, _ = self.Call('git crrev-parse %s' % self.old_gn_version) self.old_gn_commitish = out.strip() return 0 def GetNewVersions(self): _, out, _ = self.Call('git log -1 --grep Cr-Commit-Position') commit_msg = out.splitlines() first_line = commit_msg[0] new_gn_commitish = first_line.split()[1] last_line = commit_msg[-1] new_gn_version = re.sub('.*master@{#(\d+)}', '\\1', last_line) return new_gn_commitish, new_gn_version def TriggerBuild(self): ret, _, _ = self.Call('git new-branch build_gn_%s' % self.new_gn_version) if ret: print('Failed to create a new branch for build_gn_%s' % self.new_gn_version) return 1 self.MakeDummyDepsChange() ret, out, err = self.Call('git commit -a -m "Build gn at %s"' % self.new_gn_version) if ret: print('git commit failed: %s' % out + err) return 1 print('Uploading CL to build GN at {#%s} - %s' % (self.new_gn_version, self.new_gn_commitish)) ret, out, err = self.Call('git cl upload -f') if ret: print('git-cl upload failed: %s' % out + err) return 1 print('Starting try jobs') self.Call('git-cl try -m tryserver.chromium.linux ' '-b linux_chromium_gn_upload -r %s' % self.new_gn_commitish) self.Call('git-cl try -m tryserver.chromium.mac ' '-b mac_chromium_gn_upload -r %s' % self.new_gn_commitish) self.Call('git-cl try -m tryserver.chromium.win ' '-b win8_chromium_gn_upload -r %s' % self.new_gn_commitish) return 0 def MakeDummyDepsChange(self): with open('DEPS') as fp: deps_content = fp.read() new_deps = deps_content.replace("'buildtools_revision':", "'buildtools_revision': ") with open('DEPS', 'w') as fp: fp.write(new_deps) def WaitForBuildToFinish(self): ret = self.CheckoutBuildBranch() if ret: return ret print('Checking build') results = self.CheckBuild() while (any(r.state in (PENDING, STARTED) for r in results.values())): print() print('Sleeping for 30 seconds') time.sleep(30) print('Checking build') results = self.CheckBuild() ret = 0 if all(r.state == SUCCESS for r in results.values()) else 1 if ret: print('Build failed.') else: print('Builds ready.') # Close the build CL and move off of the build branch back to whatever # we were on before. self.Call('git-cl set-close') self.MoveToLastHead() return ret def CheckoutBuildBranch(self): ret, out, err = self.Call('git checkout build_gn_%s' % self.new_gn_version) if ret: print('Failed to check out build_gn_%s' % self.new_gn_version) if out: print(out) if err: print(err, file=sys.stderr) return ret def CheckBuild(self): _, out, _ = self.Call('git-cl try-results') builders = { 'linux_chromium_gn_upload': 'linux64', 'mac_chromium_gn_upload': 'mac', 'win8_chromium_gn_upload': 'win' } results = {} for platform in ('linux64', 'mac', 'win'): results[platform] = BuildResult() state = PENDING for line in out.splitlines(): fields = line.strip().split() if fields[0] == 'Started:': state = STARTED if fields[0] == 'Successes:': state = SUCCESS elif fields[0] == 'Total': pass elif fields[0] in builders: builder = fields[0] platform = builders[builder] result = results[platform] result.masterName = ('tryserver.chromium.%s' % platform.replace('linux64', 'linux')) result.builderName = builder result.url = fields[1] if result.url.startswith('id'): result.state = PENDING else: result.state = state result.buildNumber = int(result.url[result.url.rfind('/')+1:]) for result in results.values(): if result.state == SUCCESS: url = 'https://luci-milo.appspot.com/prpc/milo.BuildInfo/Get' data = json.dumps({"buildbot": { 'masterName': result.masterName, 'builderName': result.builderName, 'buildNumber': result.buildNumber, }}) headers = { 'content-type': 'application/json', 'accept': 'application/json', } req = urllib2.Request(url, data, headers) resp = urllib2.urlopen(req) data = resp.read() resp.close() # The first line of the response is garbage; skip it. js = json.loads(data.splitlines()[1]) sha1_step_name = 'gn sha1' for step in js['step']['substep']: if step['step']['name'] == sha1_step_name: sha1 = step['step']['text'][-1] result.sha1 = sha1 for platform, r in results.items(): print(platform) print(' sha1: %s' % r.sha1) print(' state: %s' % r.state) print(' build: %s' % r.buildNumber) print(' url: %s' % r.url) print() return results def RollBuildtools(self): ret = self.CheckoutBuildBranch() if ret: return ret results = self.CheckBuild() if (len(results) < 3 or not all(r.state == SUCCESS for r in results.values()) or not all(r.sha1 != '-' for r in results.values())): print("Roll isn't done or didn't succeed, exiting:") return 1 desc = self.GetBuildtoolsDesc() self.Call('git new-branch roll_buildtools_gn_%s' % self.new_gn_version, cwd=self.buildtools_dir) for platform in results: fname = 'gn.exe.sha1' if platform == 'win' else 'gn.sha1' path = os.path.join(self.buildtools_dir, platform, fname) with open(path, 'w') as fp: fp.write('%s\n' % results[platform].sha1) desc_file = tempfile.NamedTemporaryFile(delete=False) try: desc_file.write(desc) desc_file.close() self.Call('git commit -a -F %s' % desc_file.name, cwd=self.buildtools_dir) self.Call('git-cl upload -f --send-mail', cwd=self.buildtools_dir) finally: os.remove(desc_file.name) ret, out, err = self.Call('git cl land', cwd=self.buildtools_dir) if ret: print("buildtools git cl land failed: %d" % ret) if out: print(out) if err: print(err) return ret # Fetch the revision we just committed so that RollDEPS will find it. self.Call('git fetch', cwd=self.buildtools_dir) # Reset buildtools to the new commit so that we're not still on the # merged branch. self.Call('git checkout origin/master', cwd=self.buildtools_dir) _, out, _ = self.Call('git rev-parse origin/master', cwd=self.buildtools_dir) new_buildtools_commitish = out.strip() print('Ready to roll buildtools to %s in DEPS' % new_buildtools_commitish) return 0 def RollDEPS(self): ret, _, _ = self.Call('git new-branch roll_gn_%s' % self.new_gn_version) if ret: print('Failed to create a new branch for roll_gn_%s' % self.new_gn_version) return 1 _, out, _ = self.Call('git rev-parse origin/master', cwd=self.buildtools_dir) new_buildtools_commitish = out.strip() new_deps_lines = [] old_buildtools_commitish = '' with open(os.path.join(self.chromium_src_dir, 'DEPS')) as fp: for l in fp.readlines(): m = re.match(".*'buildtools_revision':.*'(.+)',", l) if m: old_buildtools_commitish = m.group(1) new_deps_lines.append(" 'buildtools_revision': '%s',\n" % new_buildtools_commitish) else: new_deps_lines.append(l) if not old_buildtools_commitish: print('Could not update DEPS properly, exiting') return 1 with open('DEPS', 'w') as fp: fp.write(''.join(new_deps_lines)) desc = self.GetDEPSRollDesc(old_buildtools_commitish, new_buildtools_commitish) desc_file = tempfile.NamedTemporaryFile(delete=False) try: desc_file.write(desc) desc_file.close() self.Call('git commit -a -F %s' % desc_file.name) self.Call('git-cl upload -f --send-mail --use-commit-queue') finally: os.remove(desc_file.name) # Move off of the roll branch onto whatever we were on before. # Do not explicitly close the roll CL issue, however; the CQ # will close it when the roll lands, assuming it does so. self.MoveToLastHead() return 0 def MoveToLastHead(self): # When this is called, there will be a commit + a checkout as # the two most recent entries in the reflog, assuming nothing as # modified the repo while this script has been running. _, out, _ = self.Call('git reflog -2') m = re.search('moving from ([^\s]+)', out) last_head = m.group(1) self.Call('git checkout %s' % last_head) def GetBuildtoolsDesc(self): gn_changes = self.GetGNChanges() return ( 'Roll gn %s..%s (r%s:r%s)\n' '\n' '%s' '\n' 'TBR=%s\n' % ( self.old_gn_commitish[:COMMITISH_DIGITS], self.new_gn_commitish[:COMMITISH_DIGITS], self.old_gn_version, self.new_gn_version, gn_changes, self.reviewer, )) def GetDEPSRollDesc(self, old_buildtools_commitish, new_buildtools_commitish): gn_changes = self.GetGNChanges() return ( 'Roll buildtools %s..%s\n' '\n' ' In order to roll GN %s..%s (r%s:r%s) and pick up\n' ' the following changes:\n' '\n' '%s' '\n' 'TBR=%s\n' % ( old_buildtools_commitish[:COMMITISH_DIGITS], new_buildtools_commitish[:COMMITISH_DIGITS], self.old_gn_commitish[:COMMITISH_DIGITS], self.new_gn_commitish[:COMMITISH_DIGITS], self.old_gn_version, self.new_gn_version, gn_changes, self.reviewer, )) def GetGNChanges(self): _, out, _ = self.Call( "git log --pretty=' %h %s' " + "%s..%s tools/gn" % (self.old_gn_commitish, self.new_gn_commitish)) return out def Call(self, cmd, cwd=None): proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True, cwd=(cwd or self.chromium_src_dir)) out, err = proc.communicate() return proc.returncode, out or '', err or '' if __name__ == '__main__': roller = GNRoller() sys.exit(roller.Roll())