Drop in Go toolchain bootstrap scripts for the bots.
Severely trimmed version of Chrome infra's scripts.
Change-Id: I378b68be670b74fe0518de5d66e0aa8b2d709f26
Reviewed-on: https://boringssl-review.googlesource.com/3491
Reviewed-by: Adam Langley <agl@google.com>
diff --git a/util/bot/README b/util/bot/README
new file mode 100644
index 0000000..b7a4332
--- /dev/null
+++ b/util/bot/README
@@ -0,0 +1,3 @@
+This directory contains tools for setting up a hermetic toolchain on the
+continuous integration bots. It is in the repository for convenience and can be
+ignored in development.
diff --git a/util/bot/go/bootstrap.py b/util/bot/go/bootstrap.py
new file mode 100755
index 0000000..166ef3b
--- /dev/null
+++ b/util/bot/go/bootstrap.py
@@ -0,0 +1,297 @@
+#!/usr/bin/env python
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Modified from go/bootstrap.py in Chromium infrastructure's repository to patch
+# out everything but the core toolchain.
+#
+# https://chromium.googlesource.com/infra/infra/
+
+"""Prepares a local hermetic Go installation.
+
+- Downloads and unpacks the Go toolset in ../golang.
+"""
+
+import contextlib
+import logging
+import os
+import platform
+import shutil
+import stat
+import subprocess
+import sys
+import tarfile
+import tempfile
+import urllib
+import zipfile
+
+# TODO(vadimsh): Migrate to new golang.org/x/ paths once Golang moves to
+# git completely.
+
+LOGGER = logging.getLogger(__name__)
+
+
+# /path/to/util/bot
+ROOT = os.path.dirname(os.path.abspath(__file__))
+
+# Where to install Go toolset to. GOROOT would be <TOOLSET_ROOT>/go.
+TOOLSET_ROOT = os.path.join(os.path.dirname(ROOT), 'golang')
+
+# Default workspace with infra go code.
+WORKSPACE = os.path.join(ROOT, 'go')
+
+# Platform depended suffix for executable files.
+EXE_SFX = '.exe' if sys.platform == 'win32' else ''
+
+# Pinned version of Go toolset to download.
+TOOLSET_VERSION = 'go1.4'
+
+# Platform dependent portion of a download URL. See http://golang.org/dl/.
+TOOLSET_VARIANTS = {
+ ('darwin', 'x86-32'): 'darwin-386-osx10.8.tar.gz',
+ ('darwin', 'x86-64'): 'darwin-amd64-osx10.8.tar.gz',
+ ('linux2', 'x86-32'): 'linux-386.tar.gz',
+ ('linux2', 'x86-64'): 'linux-amd64.tar.gz',
+ ('win32', 'x86-32'): 'windows-386.zip',
+ ('win32', 'x86-64'): 'windows-amd64.zip',
+}
+
+# Download URL root.
+DOWNLOAD_URL_PREFIX = 'https://storage.googleapis.com/golang'
+
+
+class Failure(Exception):
+ """Bootstrap failed."""
+
+
+def get_toolset_url():
+ """URL of a platform specific Go toolset archive."""
+ # TODO(vadimsh): Support toolset for cross-compilation.
+ arch = {
+ 'amd64': 'x86-64',
+ 'x86_64': 'x86-64',
+ 'i386': 'x86-32',
+ 'x86': 'x86-32',
+ }.get(platform.machine().lower())
+ variant = TOOLSET_VARIANTS.get((sys.platform, arch))
+ if not variant:
+ # TODO(vadimsh): Compile go lang from source.
+ raise Failure('Unrecognized platform')
+ return '%s/%s.%s' % (DOWNLOAD_URL_PREFIX, TOOLSET_VERSION, variant)
+
+
+def read_file(path):
+ """Returns contents of a given file or None if not readable."""
+ assert isinstance(path, (list, tuple))
+ try:
+ with open(os.path.join(*path), 'r') as f:
+ return f.read()
+ except IOError:
+ return None
+
+
+def write_file(path, data):
+ """Writes |data| to a file."""
+ assert isinstance(path, (list, tuple))
+ with open(os.path.join(*path), 'w') as f:
+ f.write(data)
+
+
+def remove_directory(path):
+ """Recursively removes a directory."""
+ assert isinstance(path, (list, tuple))
+ p = os.path.join(*path)
+ if not os.path.exists(p):
+ return
+ LOGGER.info('Removing %s', p)
+ # Crutch to remove read-only file (.git/* in particular) on Windows.
+ def onerror(func, path, _exc_info):
+ if not os.access(path, os.W_OK):
+ os.chmod(path, stat.S_IWUSR)
+ func(path)
+ else:
+ raise
+ shutil.rmtree(p, onerror=onerror if sys.platform == 'win32' else None)
+
+
+def install_toolset(toolset_root, url):
+ """Downloads and installs Go toolset.
+
+ GOROOT would be <toolset_root>/go/.
+ """
+ if not os.path.exists(toolset_root):
+ os.makedirs(toolset_root)
+ pkg_path = os.path.join(toolset_root, url[url.rfind('/')+1:])
+
+ LOGGER.info('Downloading %s...', url)
+ download_file(url, pkg_path)
+
+ LOGGER.info('Extracting...')
+ if pkg_path.endswith('.zip'):
+ with zipfile.ZipFile(pkg_path, 'r') as f:
+ f.extractall(toolset_root)
+ elif pkg_path.endswith('.tar.gz'):
+ with tarfile.open(pkg_path, 'r:gz') as f:
+ f.extractall(toolset_root)
+ else:
+ raise Failure('Unrecognized archive format')
+
+ LOGGER.info('Validating...')
+ if not check_hello_world(toolset_root):
+ raise Failure('Something is not right, test program doesn\'t work')
+
+
+def download_file(url, path):
+ """Fetches |url| to |path|."""
+ last_progress = [0]
+ def report(a, b, c):
+ progress = int(a * b * 100.0 / c)
+ if progress != last_progress[0]:
+ print >> sys.stderr, 'Downloading... %d%%' % progress
+ last_progress[0] = progress
+ # TODO(vadimsh): Use something less crippled, something that validates SSL.
+ urllib.urlretrieve(url, path, reporthook=report)
+
+
+@contextlib.contextmanager
+def temp_dir(path):
+ """Creates a temporary directory, then deletes it."""
+ tmp = tempfile.mkdtemp(dir=path)
+ try:
+ yield tmp
+ finally:
+ remove_directory([tmp])
+
+
+def check_hello_world(toolset_root):
+ """Compiles and runs 'hello world' program to verify that toolset works."""
+ with temp_dir(toolset_root) as tmp:
+ path = os.path.join(tmp, 'hello.go')
+ write_file([path], r"""
+ package main
+ func main() { println("hello, world\n") }
+ """)
+ out = subprocess.check_output(
+ [get_go_exe(toolset_root), 'run', path],
+ env=get_go_environ(toolset_root, tmp),
+ stderr=subprocess.STDOUT)
+ if out.strip() != 'hello, world':
+ LOGGER.error('Failed to run sample program:\n%s', out)
+ return False
+ return True
+
+
+def ensure_toolset_installed(toolset_root):
+ """Installs or updates Go toolset if necessary.
+
+ Returns True if new toolset was installed.
+ """
+ installed = read_file([toolset_root, 'INSTALLED_TOOLSET'])
+ available = get_toolset_url()
+ if installed == available:
+ LOGGER.debug('Go toolset is up-to-date: %s', TOOLSET_VERSION)
+ return False
+
+ LOGGER.info('Installing Go toolset.')
+ LOGGER.info(' Old toolset is %s', installed)
+ LOGGER.info(' New toolset is %s', available)
+ remove_directory([toolset_root])
+ install_toolset(toolset_root, available)
+ LOGGER.info('Go toolset installed: %s', TOOLSET_VERSION)
+ write_file([toolset_root, 'INSTALLED_TOOLSET'], available)
+ return True
+
+
+def get_go_environ(
+ toolset_root,
+ workspace=None):
+ """Returns a copy of os.environ with added GO* environment variables.
+
+ Overrides GOROOT, GOPATH and GOBIN. Keeps everything else. Idempotent.
+
+ Args:
+ toolset_root: GOROOT would be <toolset_root>/go.
+ workspace: main workspace directory or None if compiling in GOROOT.
+ """
+ env = os.environ.copy()
+ env['GOROOT'] = os.path.join(toolset_root, 'go')
+ if workspace:
+ env['GOBIN'] = os.path.join(workspace, 'bin')
+ else:
+ env.pop('GOBIN', None)
+
+ all_go_paths = []
+ if workspace:
+ all_go_paths.append(workspace)
+ env['GOPATH'] = os.pathsep.join(all_go_paths)
+
+ # New PATH entries.
+ paths_to_add = [
+ os.path.join(env['GOROOT'], 'bin'),
+ env.get('GOBIN'),
+ ]
+
+ # Make sure not to add duplicates entries to PATH over and over again when
+ # get_go_environ is invoked multiple times.
+ path = env['PATH'].split(os.pathsep)
+ paths_to_add = [p for p in paths_to_add if p and p not in path]
+ env['PATH'] = os.pathsep.join(paths_to_add + path)
+
+ return env
+
+
+def get_go_exe(toolset_root):
+ """Returns path to go executable."""
+ return os.path.join(toolset_root, 'go', 'bin', 'go' + EXE_SFX)
+
+
+def bootstrap(logging_level):
+ """Installs all dependencies in default locations.
+
+ Supposed to be called at the beginning of some script (it modifies logger).
+
+ Args:
+ logging_level: logging level of bootstrap process.
+ """
+ logging.basicConfig()
+ LOGGER.setLevel(logging_level)
+ ensure_toolset_installed(TOOLSET_ROOT)
+
+
+def prepare_go_environ():
+ """Returns dict with environment variables to set to use Go toolset.
+
+ Installs or updates the toolset if necessary.
+ """
+ bootstrap(logging.INFO)
+ return get_go_environ(TOOLSET_ROOT, WORKSPACE)
+
+
+def find_executable(name, workspaces):
+ """Returns full path to an executable in some bin/ (in GOROOT or GOBIN)."""
+ basename = name
+ if EXE_SFX and basename.endswith(EXE_SFX):
+ basename = basename[:-len(EXE_SFX)]
+ roots = [os.path.join(TOOLSET_ROOT, 'go', 'bin')]
+ for path in workspaces:
+ roots.extend([
+ os.path.join(path, 'bin'),
+ ])
+ for root in roots:
+ full_path = os.path.join(root, basename + EXE_SFX)
+ if os.path.exists(full_path):
+ return full_path
+ return name
+
+
+def main(args):
+ if args:
+ print >> sys.stderr, sys.modules[__name__].__doc__,
+ return 2
+ bootstrap(logging.DEBUG)
+ return 0
+
+
+if __name__ == '__main__':
+ sys.exit(main(sys.argv[1:]))
diff --git a/util/bot/go/env.py b/util/bot/go/env.py
new file mode 100755
index 0000000..820968c
--- /dev/null
+++ b/util/bot/go/env.py
@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Modified from go/env.py in Chromium infrastructure's repository to patch out
+# everything but the core toolchain.
+#
+# https://chromium.googlesource.com/infra/infra/
+
+"""Can be used to point environment variable to hermetic Go toolset.
+
+Usage (on linux and mac):
+$ eval `./env.py`
+$ go version
+
+Or it can be used to wrap a command:
+
+$ ./env.py go version
+"""
+
+assert __name__ == '__main__'
+
+import imp
+import os
+import subprocess
+import sys
+
+# Do not want to mess with sys.path, load the module directly.
+bootstrap = imp.load_source(
+ 'bootstrap', os.path.join(os.path.dirname(__file__), 'bootstrap.py'))
+
+old = os.environ.copy()
+new = bootstrap.prepare_go_environ()
+
+if len(sys.argv) == 1:
+ for key, value in sorted(new.iteritems()):
+ if old.get(key) != value:
+ print 'export %s="%s"' % (key, value)
+else:
+ exe = sys.argv[1]
+ if exe == 'python':
+ exe = sys.executable
+ else:
+ # Help Windows to find the executable in new PATH, do it only when
+ # executable is referenced by name (and not by path).
+ if os.sep not in exe:
+ exe = bootstrap.find_executable(exe, [bootstrap.WORKSPACE])
+ sys.exit(subprocess.call([exe] + sys.argv[2:], env=new))
