mirror of
https://github.com/klzgrad/naiveproxy.git
synced 2024-11-28 08:16:09 +03:00
462 lines
14 KiB
Python
Executable File
462 lines
14 KiB
Python
Executable File
#!/usr/bin/env python
|
|
# Copyright 2014 The Chromium Authors. All rights reserved.
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
|
|
"""An auto-roller for GN binaries into Chromium.
|
|
|
|
This script is used to update the GN binaries that a Chromium
|
|
checkout uses. In order to update the binaries, one must follow
|
|
four steps in order:
|
|
|
|
1. Trigger try jobs to build a new GN binary at tip-of-tree and upload
|
|
the newly-built binaries into the right Google CloudStorage bucket.
|
|
2. Wait for the try jobs to complete.
|
|
3. Update the buildtools repo with the .sha1 hashes of the newly built
|
|
binaries.
|
|
4. Update Chromium's DEPS file to the new version of the buildtools repo.
|
|
|
|
The script has four commands that correspond to the four steps above:
|
|
'build', 'wait', 'roll_buildtools', and 'roll_deps'.
|
|
|
|
The script has a fifth command, 'roll', that runs the four in order.
|
|
|
|
If given no arguments, the script will run the 'roll' command.
|
|
|
|
It can only be run on linux in a clean Chromium checkout; it should
|
|
error out in most cases if something bad happens, but the error checking
|
|
isn't yet foolproof.
|
|
|
|
"""
|
|
|
|
from __future__ import print_function
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import urllib2
|
|
|
|
|
|
depot_tools_path = os.path.abspath(os.path.normpath(os.path.join(
|
|
os.path.dirname(__file__), '..', '..', '..', 'third_party', 'depot_tools')))
|
|
if not depot_tools_path in sys.path:
|
|
sys.path.insert(0, depot_tools_path)
|
|
|
|
|
|
CHROMIUM_REPO = 'https://chromium.googlesource.com/chromium/src.git'
|
|
|
|
COMMITISH_DIGITS = 10
|
|
|
|
UNKNOWN, PENDING, STARTED, SUCCESS = (
|
|
'unknown', 'pending', 'started', 'success')
|
|
|
|
class BuildResult(object):
|
|
def __init__(self):
|
|
self.masterName = '-'
|
|
self.builderName = '-'
|
|
self.buildNumber = '-'
|
|
self.state = UNKNOWN
|
|
self.sha1 = '-'
|
|
self.url = '-'
|
|
|
|
|
|
class GNRoller(object):
|
|
def __init__(self):
|
|
self.chromium_src_dir = None
|
|
self.buildtools_dir = None
|
|
self.old_gn_commitish = None
|
|
self.new_gn_commitish = None
|
|
self.old_gn_version = None
|
|
self.new_gn_version = None
|
|
self.reviewer = 'dpranke@chromium.org'
|
|
if os.getenv('USER') == 'dpranke':
|
|
self.reviewer = 'brettw@chromium.org'
|
|
|
|
def Roll(self):
|
|
parser = argparse.ArgumentParser()
|
|
parser.usage = __doc__
|
|
parser.add_argument('command', nargs='?', default='roll',
|
|
help='build|roll|roll_buildtools|roll_deps|wait'
|
|
' (%(default)s is the default)')
|
|
|
|
args = parser.parse_args()
|
|
command = args.command
|
|
ret = self.SetUp()
|
|
if not ret and command in ('roll', 'build'):
|
|
ret = self.TriggerBuild()
|
|
if not ret and command in ('roll', 'wait'):
|
|
ret = self.WaitForBuildToFinish()
|
|
if not ret and command in ('roll', 'roll_buildtools'):
|
|
ret = self.RollBuildtools()
|
|
if not ret and command in ('roll', 'roll_deps'):
|
|
ret = self.RollDEPS()
|
|
|
|
return ret
|
|
|
|
def SetUp(self):
|
|
if sys.platform not in ('darwin', 'linux2'):
|
|
print('roll_gn is only tested and working on Linux and Mac for now.')
|
|
return 1
|
|
|
|
ret, out, _ = self.Call('git config --get remote.origin.url')
|
|
origin = out.strip()
|
|
if ret or origin != CHROMIUM_REPO:
|
|
print('Not in a Chromium repo? git config --get remote.origin.url '
|
|
'returned %d: %s' % (ret, origin))
|
|
return 1
|
|
|
|
ret, _, _ = self.Call('git diff -q')
|
|
if ret:
|
|
print("Checkout is dirty, exiting")
|
|
return 1
|
|
|
|
_, out, _ = self.Call('git rev-parse --show-toplevel', cwd=os.getcwd())
|
|
self.chromium_src_dir = out.strip()
|
|
self.buildtools_dir = os.path.join(self.chromium_src_dir, 'buildtools')
|
|
|
|
self.new_gn_commitish, self.new_gn_version = self.GetNewVersions()
|
|
|
|
_, out, _ = self.Call('gn --version')
|
|
self.old_gn_version = out.strip()
|
|
|
|
_, out, _ = self.Call('git crrev-parse %s' % self.old_gn_version)
|
|
self.old_gn_commitish = out.strip()
|
|
return 0
|
|
|
|
def GetNewVersions(self):
|
|
_, out, _ = self.Call('git log -1 --grep Cr-Commit-Position')
|
|
commit_msg = out.splitlines()
|
|
first_line = commit_msg[0]
|
|
new_gn_commitish = first_line.split()[1]
|
|
|
|
last_line = commit_msg[-1]
|
|
new_gn_version = re.sub('.*master@{#(\d+)}', '\\1', last_line)
|
|
|
|
return new_gn_commitish, new_gn_version
|
|
|
|
def TriggerBuild(self):
|
|
ret, _, _ = self.Call('git new-branch build_gn_%s' % self.new_gn_version)
|
|
if ret:
|
|
print('Failed to create a new branch for build_gn_%s' %
|
|
self.new_gn_version)
|
|
return 1
|
|
|
|
self.MakeDummyDepsChange()
|
|
|
|
ret, out, err = self.Call('git commit -a -m "Build gn at %s"' %
|
|
self.new_gn_version)
|
|
if ret:
|
|
print('git commit failed: %s' % out + err)
|
|
return 1
|
|
|
|
print('Uploading CL to build GN at {#%s} - %s' %
|
|
(self.new_gn_version, self.new_gn_commitish))
|
|
ret, out, err = self.Call('git cl upload -f')
|
|
if ret:
|
|
print('git-cl upload failed: %s' % out + err)
|
|
return 1
|
|
|
|
print('Starting try jobs')
|
|
self.Call('git-cl try -m tryserver.chromium.linux '
|
|
'-b linux_chromium_gn_upload -r %s' % self.new_gn_commitish)
|
|
self.Call('git-cl try -m tryserver.chromium.mac '
|
|
'-b mac_chromium_gn_upload -r %s' % self.new_gn_commitish)
|
|
self.Call('git-cl try -m tryserver.chromium.win '
|
|
'-b win_chromium_gn_upload -r %s' % self.new_gn_commitish)
|
|
|
|
return 0
|
|
|
|
def MakeDummyDepsChange(self):
|
|
with open('DEPS') as fp:
|
|
deps_content = fp.read()
|
|
new_deps = deps_content.replace("'buildtools_revision':",
|
|
"'buildtools_revision': ")
|
|
|
|
with open('DEPS', 'w') as fp:
|
|
fp.write(new_deps)
|
|
|
|
def WaitForBuildToFinish(self):
|
|
ret = self.CheckoutBuildBranch()
|
|
if ret:
|
|
return ret
|
|
|
|
print('Checking build')
|
|
results = self.CheckBuild()
|
|
while (any(r.state in (PENDING, STARTED) for r in results.values())):
|
|
print()
|
|
print('Sleeping for 30 seconds')
|
|
time.sleep(30)
|
|
print('Checking build')
|
|
results = self.CheckBuild()
|
|
|
|
ret = 0 if all(r.state == SUCCESS for r in results.values()) else 1
|
|
if ret:
|
|
print('Build failed.')
|
|
else:
|
|
print('Builds ready.')
|
|
|
|
# Close the build CL and move off of the build branch back to whatever
|
|
# we were on before.
|
|
self.Call('git-cl set-close')
|
|
self.MoveToLastHead()
|
|
|
|
return ret
|
|
|
|
def CheckoutBuildBranch(self):
|
|
ret, out, err = self.Call('git checkout build_gn_%s' % self.new_gn_version)
|
|
if ret:
|
|
print('Failed to check out build_gn_%s' % self.new_gn_version)
|
|
if out:
|
|
print(out)
|
|
if err:
|
|
print(err, file=sys.stderr)
|
|
return ret
|
|
|
|
def CheckBuild(self):
|
|
_, out, _ = self.Call('git-cl try-results')
|
|
|
|
builders = {
|
|
'linux_chromium_gn_upload': 'linux64',
|
|
'mac_chromium_gn_upload': 'mac',
|
|
'win_chromium_gn_upload': 'win'
|
|
}
|
|
|
|
results = {}
|
|
for platform in ('linux64', 'mac', 'win'):
|
|
results[platform] = BuildResult()
|
|
|
|
state = PENDING
|
|
for line in out.splitlines():
|
|
fields = line.strip().split()
|
|
if fields[0] == 'Started:':
|
|
state = STARTED
|
|
if fields[0] == 'Successes:':
|
|
state = SUCCESS
|
|
elif fields[0] == 'Total':
|
|
pass
|
|
elif fields[0] in builders:
|
|
builder = fields[0]
|
|
platform = builders[builder]
|
|
result = results[platform]
|
|
result.masterName = ('tryserver.chromium.%s' %
|
|
platform.replace('linux64', 'linux'))
|
|
result.builderName = builder
|
|
result.url = fields[1]
|
|
if result.url.startswith('id'):
|
|
result.state = PENDING
|
|
else:
|
|
result.state = state
|
|
result.buildNumber = int(result.url[result.url.rfind('/')+1:])
|
|
|
|
for result in results.values():
|
|
if result.state == SUCCESS:
|
|
url = 'https://luci-milo.appspot.com/prpc/milo.BuildInfo/Get'
|
|
data = json.dumps({"buildbot": {
|
|
'masterName': result.masterName,
|
|
'builderName': result.builderName,
|
|
'buildNumber': result.buildNumber,
|
|
}})
|
|
headers = {
|
|
'content-type': 'application/json',
|
|
'accept': 'application/json',
|
|
}
|
|
|
|
req = urllib2.Request(url, data, headers)
|
|
resp = urllib2.urlopen(req)
|
|
data = resp.read()
|
|
resp.close()
|
|
|
|
# The first line of the response is garbage; skip it.
|
|
js = json.loads(data.splitlines()[1])
|
|
|
|
sha1_step_name = 'gn sha1'
|
|
for step in js['step']['substep']:
|
|
if step['step']['name'] == sha1_step_name:
|
|
sha1 = step['step']['text'][-1]
|
|
|
|
result.sha1 = sha1
|
|
|
|
for platform, r in results.items():
|
|
print(platform)
|
|
print(' sha1: %s' % r.sha1)
|
|
print(' state: %s' % r.state)
|
|
print(' build: %s' % r.buildNumber)
|
|
print(' url: %s' % r.url)
|
|
print()
|
|
|
|
return results
|
|
|
|
def RollBuildtools(self):
|
|
ret = self.CheckoutBuildBranch()
|
|
if ret:
|
|
return ret
|
|
|
|
results = self.CheckBuild()
|
|
if (len(results) < 3 or
|
|
not all(r.state == SUCCESS for r in results.values()) or
|
|
not all(r.sha1 != '-' for r in results.values())):
|
|
print("Roll isn't done or didn't succeed, exiting:")
|
|
return 1
|
|
|
|
desc = self.GetBuildtoolsDesc()
|
|
|
|
self.Call('git new-branch roll_buildtools_gn_%s' % self.new_gn_version,
|
|
cwd=self.buildtools_dir)
|
|
|
|
for platform in results:
|
|
fname = 'gn.exe.sha1' if platform == 'win' else 'gn.sha1'
|
|
path = os.path.join(self.buildtools_dir, platform, fname)
|
|
with open(path, 'w') as fp:
|
|
fp.write('%s\n' % results[platform].sha1)
|
|
|
|
desc_file = tempfile.NamedTemporaryFile(delete=False)
|
|
try:
|
|
desc_file.write(desc)
|
|
desc_file.close()
|
|
self.Call('git commit -a -F %s' % desc_file.name,
|
|
cwd=self.buildtools_dir)
|
|
self.Call('git-cl upload -f --send-mail',
|
|
cwd=self.buildtools_dir)
|
|
finally:
|
|
os.remove(desc_file.name)
|
|
|
|
ret, out, err = self.Call('git cl land', cwd=self.buildtools_dir)
|
|
if ret:
|
|
print("buildtools git cl land failed: %d" % ret)
|
|
if out:
|
|
print(out)
|
|
if err:
|
|
print(err)
|
|
return ret
|
|
|
|
# Fetch the revision we just committed so that RollDEPS will find it.
|
|
self.Call('git fetch', cwd=self.buildtools_dir)
|
|
|
|
# Reset buildtools to the new commit so that we're not still on the
|
|
# merged branch.
|
|
self.Call('git checkout origin/master', cwd=self.buildtools_dir)
|
|
|
|
_, out, _ = self.Call('git rev-parse origin/master',
|
|
cwd=self.buildtools_dir)
|
|
new_buildtools_commitish = out.strip()
|
|
print('Ready to roll buildtools to %s in DEPS' % new_buildtools_commitish)
|
|
|
|
return 0
|
|
|
|
def RollDEPS(self):
|
|
ret, _, _ = self.Call('git new-branch roll_gn_%s' % self.new_gn_version)
|
|
if ret:
|
|
print('Failed to create a new branch for roll_gn_%s' %
|
|
self.new_gn_version)
|
|
return 1
|
|
|
|
_, out, _ = self.Call('git rev-parse origin/master',
|
|
cwd=self.buildtools_dir)
|
|
new_buildtools_commitish = out.strip()
|
|
|
|
new_deps_lines = []
|
|
old_buildtools_commitish = ''
|
|
with open(os.path.join(self.chromium_src_dir, 'DEPS')) as fp:
|
|
for l in fp.readlines():
|
|
m = re.match(".*'buildtools_revision':.*'(.+)',", l)
|
|
if m:
|
|
old_buildtools_commitish = m.group(1)
|
|
new_deps_lines.append(" 'buildtools_revision': '%s',\n" %
|
|
new_buildtools_commitish)
|
|
else:
|
|
new_deps_lines.append(l)
|
|
|
|
if not old_buildtools_commitish:
|
|
print('Could not update DEPS properly, exiting')
|
|
return 1
|
|
|
|
with open('DEPS', 'w') as fp:
|
|
fp.write(''.join(new_deps_lines))
|
|
|
|
desc = self.GetDEPSRollDesc(old_buildtools_commitish,
|
|
new_buildtools_commitish)
|
|
desc_file = tempfile.NamedTemporaryFile(delete=False)
|
|
try:
|
|
desc_file.write(desc)
|
|
desc_file.close()
|
|
self.Call('git commit -a -F %s' % desc_file.name)
|
|
self.Call('git-cl upload -f --send-mail --use-commit-queue')
|
|
finally:
|
|
os.remove(desc_file.name)
|
|
|
|
# Move off of the roll branch onto whatever we were on before.
|
|
# Do not explicitly close the roll CL issue, however; the CQ
|
|
# will close it when the roll lands, assuming it does so.
|
|
self.MoveToLastHead()
|
|
|
|
return 0
|
|
|
|
def MoveToLastHead(self):
|
|
# When this is called, there will be a commit + a checkout as
|
|
# the two most recent entries in the reflog, assuming nothing as
|
|
# modified the repo while this script has been running.
|
|
_, out, _ = self.Call('git reflog -2')
|
|
m = re.search('moving from ([^\s]+)', out)
|
|
last_head = m.group(1)
|
|
self.Call('git checkout %s' % last_head)
|
|
|
|
def GetBuildtoolsDesc(self):
|
|
gn_changes = self.GetGNChanges()
|
|
return (
|
|
'Roll gn %s..%s (r%s:r%s)\n'
|
|
'\n'
|
|
'%s'
|
|
'\n'
|
|
'TBR=%s\n' % (
|
|
self.old_gn_commitish[:COMMITISH_DIGITS],
|
|
self.new_gn_commitish[:COMMITISH_DIGITS],
|
|
self.old_gn_version,
|
|
self.new_gn_version,
|
|
gn_changes,
|
|
self.reviewer,
|
|
))
|
|
|
|
def GetDEPSRollDesc(self, old_buildtools_commitish, new_buildtools_commitish):
|
|
gn_changes = self.GetGNChanges()
|
|
|
|
return (
|
|
'Roll buildtools %s..%s\n'
|
|
'\n'
|
|
' In order to roll GN %s..%s (r%s:r%s) and pick up\n'
|
|
' the following changes:\n'
|
|
'\n'
|
|
'%s'
|
|
'\n'
|
|
'TBR=%s\n' % (
|
|
old_buildtools_commitish[:COMMITISH_DIGITS],
|
|
new_buildtools_commitish[:COMMITISH_DIGITS],
|
|
self.old_gn_commitish[:COMMITISH_DIGITS],
|
|
self.new_gn_commitish[:COMMITISH_DIGITS],
|
|
self.old_gn_version,
|
|
self.new_gn_version,
|
|
gn_changes,
|
|
self.reviewer,
|
|
))
|
|
|
|
def GetGNChanges(self):
|
|
_, out, _ = self.Call(
|
|
"git log --pretty=' %h %s' " +
|
|
"%s..%s tools/gn" % (self.old_gn_commitish, self.new_gn_commitish))
|
|
return out
|
|
|
|
def Call(self, cmd, cwd=None):
|
|
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True,
|
|
cwd=(cwd or self.chromium_src_dir))
|
|
out, err = proc.communicate()
|
|
return proc.returncode, out or '', err or ''
|
|
|
|
|
|
if __name__ == '__main__':
|
|
roller = GNRoller()
|
|
sys.exit(roller.Roll())
|