naiveproxy/tools/gn/bin/roll_gn.py

#!/usr/bin/env python
# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""An auto-roller for GN binaries into Chromium.

This script is used to update the GN binaries that a Chromium
checkout uses. In order to update the binaries, one must follow
four steps in order:

1. Trigger try jobs to build a new GN binary at tip-of-tree and upload
   the newly-built binaries into the right Google CloudStorage bucket.
2. Wait for the try jobs to complete.
3. Update the buildtools repo with the .sha1 hashes of the newly built
   binaries.
4. Update Chromium's DEPS file to the new version of the buildtools repo.

The script has four commands that correspond to the four steps above:
'build', 'wait', 'roll_buildtools', and 'roll_deps'.

The script has a fifth command, 'roll', that runs the four in order.

If given no arguments, the script will run the 'roll' command.

It can only be run on linux in a clean Chromium checkout; it should
error out in most cases if something bad happens, but the error checking
isn't yet foolproof.

"""

from __future__ import print_function

import argparse
import json
import os
import re
import subprocess
import sys
import tempfile
import time
import urllib2


depot_tools_path = os.path.abspath(os.path.normpath(os.path.join(
    os.path.dirname(__file__), '..', '..', '..', 'third_party', 'depot_tools')))
if not depot_tools_path in sys.path:
  sys.path.insert(0, depot_tools_path)


CHROMIUM_REPO = 'https://chromium.googlesource.com/chromium/src.git'

COMMITISH_DIGITS = 10

UNKNOWN, PENDING, STARTED, SUCCESS = (
  'unknown', 'pending', 'started', 'success')

class BuildResult(object):
  def __init__(self):
    self.masterName = '-'
    self.builderName = '-'
    self.buildNumber = '-'
    self.state = UNKNOWN
    self.sha1 = '-'
    self.url = '-'


class GNRoller(object):
  def __init__(self):
    self.chromium_src_dir = None
    self.buildtools_dir = None
    self.old_gn_commitish = None
    self.new_gn_commitish = None
    self.old_gn_version   = None
    self.new_gn_version = None
    self.reviewer = 'dpranke@chromium.org'
    if os.getenv('USER') == 'dpranke':
      self.reviewer = 'brettw@chromium.org'

  def Roll(self):
    parser = argparse.ArgumentParser()
    parser.usage = __doc__
    parser.add_argument('command', nargs='?', default='roll',
                        help='build|roll|roll_buildtools|roll_deps|wait'
                             ' (%(default)s is the default)')

    args = parser.parse_args()
    command = args.command
    ret = self.SetUp()
    if not ret and command in ('roll', 'build'):
      ret = self.TriggerBuild()
    if not ret and command in ('roll', 'wait'):
      ret = self.WaitForBuildToFinish()
    if not ret and command in ('roll', 'roll_buildtools'):
      ret = self.RollBuildtools()
    if not ret and command in ('roll', 'roll_deps'):
      ret = self.RollDEPS()

    return ret

  def SetUp(self):
    if sys.platform not in ('darwin', 'linux2'):
      print('roll_gn is only tested and working on Linux and Mac for now.')
      return 1

    ret, out, _ = self.Call('git config --get remote.origin.url')
    origin = out.strip()
    if ret or origin != CHROMIUM_REPO:
      print('Not in a Chromium repo? git config --get remote.origin.url '
            'returned %d: %s' % (ret, origin))
      return 1

    ret, _, _ = self.Call('git diff -q')
    if ret:
      print("Checkout is dirty, exiting")
      return 1

    _, out, _ = self.Call('git rev-parse --show-toplevel', cwd=os.getcwd())
    self.chromium_src_dir = out.strip()
    self.buildtools_dir = os.path.join(self.chromium_src_dir, 'buildtools')

    self.new_gn_commitish, self.new_gn_version = self.GetNewVersions()

    _, out, _ = self.Call('gn --version')
    self.old_gn_version = out.strip()

    _, out, _ = self.Call('git crrev-parse %s' % self.old_gn_version)
    self.old_gn_commitish = out.strip()
    return 0

  def GetNewVersions(self):
    _, out, _ = self.Call('git log -1 --grep Cr-Commit-Position')
    commit_msg = out.splitlines()
    first_line = commit_msg[0]
    new_gn_commitish = first_line.split()[1]

    last_line = commit_msg[-1]
    new_gn_version = re.sub('.*master@{#(\d+)}', '\\1', last_line)

    return new_gn_commitish, new_gn_version

  def TriggerBuild(self):
    ret, _, _ = self.Call('git new-branch build_gn_%s' % self.new_gn_version)
    if ret:
      print('Failed to create a new branch for build_gn_%s' %
            self.new_gn_version)
      return 1

    self.MakeDummyDepsChange()

    ret, out, err = self.Call('git commit -a -m "Build gn at %s"' %
                         self.new_gn_version)
    if ret:
      print('git commit failed: %s' % out + err)
      return 1

    print('Uploading CL to build GN at {#%s} - %s' %
          (self.new_gn_version, self.new_gn_commitish))
    ret, out, err = self.Call('git cl upload -f')
    if ret:
      print('git-cl upload failed: %s' % out + err)
      return 1

    print('Starting try jobs')
    self.Call('git-cl try -m tryserver.chromium.linux '
              '-b linux_chromium_gn_upload -r %s' % self.new_gn_commitish)
    self.Call('git-cl try -m tryserver.chromium.mac '
              '-b mac_chromium_gn_upload -r %s' % self.new_gn_commitish)
    self.Call('git-cl try -m tryserver.chromium.win '
              '-b win_chromium_gn_upload -r %s' % self.new_gn_commitish)

    return 0

  def MakeDummyDepsChange(self):
    with open('DEPS') as fp:
      deps_content = fp.read()
      new_deps = deps_content.replace("'buildtools_revision':",
                                      "'buildtools_revision':  ")

    with open('DEPS', 'w') as fp:
      fp.write(new_deps)

  def WaitForBuildToFinish(self):
    ret = self.CheckoutBuildBranch()
    if ret:
      return ret

    print('Checking build')
    results = self.CheckBuild()
    while (any(r.state in (PENDING, STARTED) for r in results.values())):
      print()
      print('Sleeping for 30 seconds')
      time.sleep(30)
      print('Checking build')
      results = self.CheckBuild()

    ret = 0 if all(r.state == SUCCESS for r in results.values()) else 1
    if ret:
      print('Build failed.')
    else:
      print('Builds ready.')

    # Close the build CL and move off of the build branch back to whatever
    # we were on before.
    self.Call('git-cl set-close')
    self.MoveToLastHead()

    return ret

  def CheckoutBuildBranch(self):
    ret, out, err = self.Call('git checkout build_gn_%s' % self.new_gn_version)
    if ret:
      print('Failed to check out build_gn_%s' % self.new_gn_version)
      if out:
        print(out)
      if err:
        print(err, file=sys.stderr)
    return ret

  def CheckBuild(self):
    _, out, _ = self.Call('git-cl try-results')

    builders = {
        'linux_chromium_gn_upload': 'linux64',
        'mac_chromium_gn_upload': 'mac',
        'win_chromium_gn_upload': 'win'
    }

    results = {}
    for platform in ('linux64', 'mac', 'win'):
      results[platform] = BuildResult()

    state = PENDING
    for line in out.splitlines():
      fields = line.strip().split()
      if fields[0] == 'Started:':
        state = STARTED
      if fields[0] == 'Successes:':
        state = SUCCESS
      elif fields[0] == 'Total':
        pass
      elif fields[0] in builders:
        builder = fields[0]
        platform = builders[builder]
        result = results[platform]
        result.masterName = ('tryserver.chromium.%s' %
                                platform.replace('linux64', 'linux'))
        result.builderName = builder
        result.url = fields[1]
        if result.url.startswith('id'):
          result.state = PENDING
        else:
          result.state = state
          result.buildNumber = int(result.url[result.url.rfind('/')+1:])

    for result in results.values():
      if result.state == SUCCESS:
        url = 'https://luci-milo.appspot.com/prpc/milo.BuildInfo/Get'
        data = json.dumps({"buildbot": {
            'masterName': result.masterName,
            'builderName': result.builderName,
            'buildNumber': result.buildNumber,
            }})
        headers = {
            'content-type': 'application/json',
            'accept': 'application/json',
        }

        req = urllib2.Request(url, data, headers)
        resp = urllib2.urlopen(req)
        data = resp.read()
        resp.close()

        # The first line of the response is garbage; skip it.
        js = json.loads(data.splitlines()[1])

        sha1_step_name = 'gn sha1'
        for step in js['step']['substep']:
          if step['step']['name'] == sha1_step_name:
            sha1 = step['step']['text'][-1]

        result.sha1 = sha1

    for platform, r in results.items():
      print(platform)
      print('  sha1:  %s' % r.sha1)
      print('  state: %s' % r.state)
      print('  build: %s' % r.buildNumber)
      print('  url:   %s' % r.url)
      print()

    return results

  def RollBuildtools(self):
    ret = self.CheckoutBuildBranch()
    if ret:
      return ret

    results = self.CheckBuild()
    if (len(results) < 3 or
        not all(r.state == SUCCESS for r in results.values()) or
        not all(r.sha1 != '-' for r in results.values())):
      print("Roll isn't done or didn't succeed, exiting:")
      return 1

    desc = self.GetBuildtoolsDesc()

    self.Call('git new-branch roll_buildtools_gn_%s' % self.new_gn_version,
              cwd=self.buildtools_dir)

    for platform in results:
      fname = 'gn.exe.sha1' if platform == 'win' else 'gn.sha1'
      path = os.path.join(self.buildtools_dir, platform, fname)
      with open(path, 'w') as fp:
        fp.write('%s\n' % results[platform].sha1)

    desc_file = tempfile.NamedTemporaryFile(delete=False)
    try:
      desc_file.write(desc)
      desc_file.close()
      self.Call('git commit -a -F %s' % desc_file.name,
                cwd=self.buildtools_dir)
      self.Call('git-cl upload -f --send-mail',
                cwd=self.buildtools_dir)
    finally:
      os.remove(desc_file.name)

    ret, out, err = self.Call('git cl land', cwd=self.buildtools_dir)
    if ret:
        print("buildtools git cl land failed: %d" % ret)
        if out:
            print(out)
        if err:
            print(err)
        return ret

    # Fetch the revision we just committed so that RollDEPS will find it.
    self.Call('git fetch', cwd=self.buildtools_dir)

    # Reset buildtools to the new commit so that we're not still on the
    # merged branch.
    self.Call('git checkout origin/master', cwd=self.buildtools_dir)

    _, out, _ = self.Call('git rev-parse origin/master',
                          cwd=self.buildtools_dir)
    new_buildtools_commitish = out.strip()
    print('Ready to roll buildtools to %s in DEPS' % new_buildtools_commitish)

    return 0

  def RollDEPS(self):
    ret, _, _ = self.Call('git new-branch roll_gn_%s' % self.new_gn_version)
    if ret:
      print('Failed to create a new branch for roll_gn_%s' %
            self.new_gn_version)
      return 1

    _, out, _ = self.Call('git rev-parse origin/master',
                          cwd=self.buildtools_dir)
    new_buildtools_commitish = out.strip()

    new_deps_lines = []
    old_buildtools_commitish = ''
    with open(os.path.join(self.chromium_src_dir, 'DEPS')) as fp:
      for l in fp.readlines():
        m = re.match(".*'buildtools_revision':.*'(.+)',", l)
        if m:
          old_buildtools_commitish = m.group(1)
          new_deps_lines.append("  'buildtools_revision': '%s',\n" %
                                new_buildtools_commitish)
        else:
          new_deps_lines.append(l)

    if not old_buildtools_commitish:
      print('Could not update DEPS properly, exiting')
      return 1

    with open('DEPS', 'w') as fp:
      fp.write(''.join(new_deps_lines))

    desc = self.GetDEPSRollDesc(old_buildtools_commitish,
                                new_buildtools_commitish)
    desc_file = tempfile.NamedTemporaryFile(delete=False)
    try:
      desc_file.write(desc)
      desc_file.close()
      self.Call('git commit -a -F %s' % desc_file.name)
      self.Call('git-cl upload -f --send-mail --use-commit-queue')
    finally:
      os.remove(desc_file.name)

    # Move off of the roll branch onto whatever we were on before.
    # Do not explicitly close the roll CL issue, however; the CQ
    # will close it when the roll lands, assuming it does so.
    self.MoveToLastHead()

    return 0

  def MoveToLastHead(self):
    # When this is called, there will be a commit + a checkout as
    # the two most recent entries in the reflog, assuming nothing as
    # modified the repo while this script has been running.
    _, out, _ = self.Call('git reflog -2')
    m = re.search('moving from ([^\s]+)', out)
    last_head = m.group(1)
    self.Call('git checkout %s' % last_head)

  def GetBuildtoolsDesc(self):
    gn_changes = self.GetGNChanges()
    return (
      'Roll gn %s..%s (r%s:r%s)\n'
      '\n'
      '%s'
      '\n'
      'TBR=%s\n' % (
        self.old_gn_commitish[:COMMITISH_DIGITS],
        self.new_gn_commitish[:COMMITISH_DIGITS],
        self.old_gn_version,
        self.new_gn_version,
        gn_changes,
        self.reviewer,
      ))

  def GetDEPSRollDesc(self, old_buildtools_commitish, new_buildtools_commitish):
    gn_changes = self.GetGNChanges()

    return (
      'Roll buildtools %s..%s\n'
      '\n'
      '  In order to roll GN %s..%s (r%s:r%s) and pick up\n'
      '  the following changes:\n'
      '\n'
      '%s'
      '\n'
      'TBR=%s\n' % (
        old_buildtools_commitish[:COMMITISH_DIGITS],
        new_buildtools_commitish[:COMMITISH_DIGITS],
        self.old_gn_commitish[:COMMITISH_DIGITS],
        self.new_gn_commitish[:COMMITISH_DIGITS],
        self.old_gn_version,
        self.new_gn_version,
        gn_changes,
        self.reviewer,
      ))

  def GetGNChanges(self):
    _, out, _ = self.Call(
        "git log --pretty='  %h %s' " +
        "%s..%s tools/gn" % (self.old_gn_commitish, self.new_gn_commitish))
    return out

  def Call(self, cmd, cwd=None):
    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True,
                            cwd=(cwd or self.chromium_src_dir))
    out, err = proc.communicate()
    return proc.returncode, out or '', err or ''


if __name__ == '__main__':
  roller = GNRoller()
  sys.exit(roller.Roll())