Jenkins build script unifiying the build process for all nodes

Summary:
So far, the build process was configured for each job individually. This
was flexible in the past, when frequent changes were needed. Now, since
the CI build configurations are very stable, it is huge effort to keep
the jobs aligned during changes (e.g. Qt upgrade, build arguments etc).
This, plus the need to version control the build process has driven
the creation of this build script. The script is running on all nodes
and provides a small number of arguments to configure the build.
Python script for creating symbols is incorporated and therefore renamed.

Reviewers: #swift_pilot_client, msutcliffe

Subscribers: msutcliffe, jenkins

Differential Revision: https://dev.swift-project.org/D40

scripts/jenkins.cfg

@@ -0,0 +1,15 @@
+[General]
+qt_version: 5.9.1
+
+[Windows]
+qt_path: C:/Qt
+dump_syms: C:/bin/dump_syms.exe
+mingw_path: C:/Qt/Tools/mingw530_32/bin
+
+[Linux]
+qt_path: /opt/Qt
+dump_syms:
+
+[Darwin]
+qt_path: /Applications/Qt
+dump_syms: /usr/local/bin/dump_syms

scripts/jenkins.py

@@ -0,0 +1,387 @@
+#!/bin/env python
+
+# Copyright (C) 2017
+# swift Project Community/Contributors
+#
+# This file is part of swift project. It is subject to the license terms in the LICENSE file found in the top-level
+# directory of this distribution and at http://www.swift-project.org/license.html. No part of swift project,
+# including this file, may be copied, modified, propagated, or distributed except according to the terms
+# contained in the LICENSE file.
+
+import getopt
+import multiprocessing
+import os
+import os.path as path
+import platform
+import subprocess
+import sys
+import symbolstore
+from lib.util import get_vs_env
+
+if sys.version_info < (3, 0):
+    import ConfigParser as configparser
+else:
+    import configparser
+
+
+class Builder:
+
+    def prepare(self):
+        """
+        Prepares the build environment, e.g. setup of environmental variables.
+        Build processes will be called with the environment modified by this function.
+        """
+        print('Preparing environment ...')
+        os.environ['PATH'] += os.pathsep + self._get_qt_binary_path()
+        self._specific_prepare()
+
+    def build(self, dev_build):
+        """
+        Run the build itself. Pass dev_build=True to enable a dev build
+        """
+        print('Running build ...')
+        build_path = self._get_swift_build_path()
+        if not os.path.isdir(build_path):
+            os.makedirs(build_path)
+        os.chdir(build_path)
+        qmake_call = ['qmake']
+        if dev_build:
+            qmake_call += ['"BLACK_CONFIG+=SwiftDevBranch"']
+        qmake_call += ['-r', os.pardir]
+        subprocess.check_call(qmake_call, env=dict(os.environ))
+
+        job_arg = '-j{0}'.format(multiprocessing.cpu_count())
+        subprocess.check_call([self.make_cmd, job_arg], env=dict(os.environ))
+
+    def checks(self):
+        """
+        Runs build checks.
+        """
+        print('Installing ...')
+        build_path = self._get_swift_build_path()
+        os.chdir(build_path)
+        if self._should_run_checks():
+            subprocess.check_call([self.make_cmd, 'check'], env=dict(os.environ))
+            pass
+
+    def install(self):
+        """
+        Installs all products to the default path.
+        """
+        print('Running install()')
+        build_path = self._get_swift_build_path()
+        os.chdir(build_path)
+        if self._should_run_publish():
+            subprocess.check_call([self.make_cmd, 'publish_installer'], env=dict(os.environ))
+            pass
+        else:
+            subprocess.check_call([self.make_cmd, 'install'], env=dict(os.environ))
+            pass
+
+    def package_xswiftbus(self):
+        """
+        Packages xswiftbus as 7z compressed archive into the swift source root.
+        """
+        print('Packaging xswiftbus ...')
+        build_path = self._get_swift_build_path()
+        os.chdir(build_path)
+        archive_name = '-'.join(['xswiftbus', platform.system(), self.word_size, self.version]) + '.7z'
+        archive_path = path.abspath(path.join(os.pardir, archive_name))
+        content_path = path.abspath(path.join(os.curdir, 'dist', 'xswiftbus'))
+        subprocess.check_call(['7z', 'a', '-mx=9', archive_path, content_path], env=dict(os.environ))
+
+    def symbols(self):
+        """
+        Generates the binary symbols and archives them into a gzip archive, located in the swift source root.
+        """
+        if self._should_create_symbols():
+            build_path = self._get_swift_build_path()
+            os.chdir(build_path)
+            print('Creating symbols')
+            symbol_path = path.abspath(path.join(build_path, 'symbols'))
+            binary_path = path.abspath(path.join(build_path, 'out'))
+            symbolstore.Dumper.global_init()
+            dumper = symbolstore.get_platform_specific_dumper(dump_syms=self.dump_syms, symbol_path=symbol_path)
+            dumper.process(binary_path)
+            dumper.finish()
+            tar_filename = '-'.join(['swift', 'symbols', platform.system(), self.word_size, self.version]) + '.tar.gz'
+            tar_path = path.abspath(path.join(self._get_swift_source_path(), tar_filename))
+            dumper.pack(tar_path)
+
+    def _get_swift_source_path(self):
+        return self.__source_path
+
+    def _get_swift_build_path(self):
+        return self.__build_path
+
+    def _specific_prepare(self):
+        pass
+
+    def _get_qmake_spec(self):
+        raise NotImplementedError()
+
+    def _get_make_cmd(self):
+        raise NotImplementedError()
+
+    def _get_qt_component(self):
+        raise NotImplementedError()
+
+    def _should_run_checks(self):
+        return True
+
+    def _should_run_publish(self):
+        return True
+
+    def _should_create_symbols(self):
+        return True
+
+    def _get_qtcreator_path(self):
+        qtcreator_path = path.abspath(path.join(self.qt_path, 'Tools', 'QtCreator', 'bin'))
+        return qtcreator_path
+
+    def _get_externals_path(self):
+        qmake_spec = self._get_qmake_spec()
+        lib_path = 'lib' + self.word_size
+        return path.abspath(path.join(self._get_swift_source_path(), 'externals', qmake_spec, lib_path))
+
+    def _get_qt_binary_path(self):
+        component = self._get_qt_component()
+        if self.word_size == '64':
+            component += '_64'
+        else:
+            # special case for MSVC 32 bit component (most likely all versions). Those don't have the 32 bit suffix
+            if "msvc" not in component:
+                component += '_32'
+        qt_binary_path = path.abspath(path.join(self.qt_path, self.qt_version, '{0}'.format(component), 'bin'))
+        return qt_binary_path
+
+    def _get_config(self):
+        return self.__config
+
+    def __init__(self, config_file, word_size):
+        self.__source_path = path.abspath(path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir))
+        self.__build_path = path.abspath(path.join(self.__source_path, 'build'))
+
+        swift_project_file = path.join(self.__source_path, 'swift.pro')
+        if not os.path.isfile(swift_project_file):
+            raise RuntimeError('Cannot find swift.pro! Are we in the right directory?')
+
+        self.word_size = word_size
+
+        if not config_file:
+            config_file = path.abspath(path.join(self._get_swift_source_path(), 'scripts', 'jenkins.cfg'))
+        self.__config = configparser.SafeConfigParser()
+        self.__config.read(config_file)
+        self.qt_version = self.__config.get('General', 'qt_version')
+        self.qt_path = path.abspath(self.__config.get(platform.system(), 'qt_path'))
+        self.dump_syms = path.abspath(self.__config.get(platform.system(), 'dump_syms'))
+
+        self.make_cmd = self._get_make_cmd()
+        self.version = self.__get_swift_version()
+
+    def __get_swift_version(self):
+        version_major = '0'
+        version_minor = '0'
+        version_micro = '0'
+        version_file = path.abspath(
+            path.join(self._get_swift_source_path(), 'mkspecs', 'features', 'version')) + '.pri'
+        f = open(version_file)
+        line = f.readline()
+        while line:
+            # Remove all spaces
+            line = line.strip().replace(' ', '')
+            tokens = line.split('=')
+            if not len(tokens) == 2:
+                raise ValueError('version.pri has wrong format!')
+            if tokens[0] == 'BLACK_VER_MAJ':
+                version_major = tokens[1]
+            elif tokens[0] == 'BLACK_VER_MIN':
+                version_minor = tokens[1]
+            elif tokens[0] == 'BLACK_VER_MIC':
+                version_micro = tokens[1]
+            else:
+                pass
+            line = f.readline()
+        f.close()
+        version = '.'.join([version_major, version_minor, version_micro])
+        return version
+
+
+class MSVCBuilder(Builder):
+
+    def _specific_prepare(self):
+        os.environ['PATH'] += os.pathsep + self._get_externals_path()
+        os.environ['PATH'] += os.pathsep + 'C:/Program Files/7-Zip'
+        vs_env = get_vs_env('14.0', 'amd64')
+        os.environ.update(vs_env)
+
+    def _get_qmake_spec(self):
+        return 'win32-msvc2015'
+
+    def _get_make_cmd(self):
+        return path.abspath(path.join(self._get_qtcreator_path(), 'jom.exe'))
+
+    def _get_qt_component(self):
+        return 'msvc2015'
+
+    def __init__(self, config_file, word_size):
+        Builder.__init__(self, config_file, word_size)
+
+
+class MinGWBuilder(Builder):
+
+    def _specific_prepare(self):
+        os.environ['PATH'] += os.pathsep + self._get_externals_path()
+        gcc_path = path.abspath(self._get_config().get(platform.system(), 'mingw_path'))
+        os.environ['PATH'] += os.pathsep + gcc_path
+        os.environ['PATH'] += os.pathsep + path.abspath(path.join('c:', os.sep, 'Program Files', '7-Zip'))
+
+    def _get_qmake_spec(self):
+        return 'win32-g++'
+
+    def _get_make_cmd(self):
+        return 'mingw32-make'
+
+    def _get_qt_component(self):
+        return 'mingw53'
+
+    def _should_run_checks(self):
+        return False
+
+    def _should_run_publish(self):
+        return False
+
+    def _should_create_symbols(self):
+        return False
+
+    def __init__(self, config_file, word_size):
+        Builder.__init__(self, config_file, word_size)
+
+
+class LinuxBuilder(Builder):
+
+    def _specific_prepare(self):
+        os.environ['LD_LIBRARY_PATH'] = path.abspath(path.join(self._get_swift_build_path(), 'build', 'out', 'release', 'lib'))
+        os.environ['LD_LIBRARY_PATH'] += os.pathsep + path.abspath(path.join(self._get_swift_source_path(), 'lib'))
+        os.environ['LD_LIBRARY_PATH'] += os.pathsep + self._get_externals_path()
+        lib_path = path.abspath(path.join(self._get_swift_source_path(), 'lib'))
+        libssl = path.abspath(path.join(lib_path, 'libssl.so'))
+        libcrypto = path.abspath(path.join(lib_path, 'libcrypto.so'))
+        if not os.path.isdir(lib_path):
+            os.makedirs(lib_path)
+        if not os.path.exists(libssl):
+            os.symlink('/usr/lib/x86_64-linux-gnu/libssl.so.1.0.2', libssl)
+        if not os.path.exists(libcrypto):
+            os.symlink('/usr/lib/x86_64-linux-gnu/libcrypto.so.1.0.2', libcrypto)
+
+    def _get_qmake_spec(self):
+        return 'linux-g++'
+
+    def _get_make_cmd(self):
+        return 'make'
+
+    def _get_qt_component(self):
+        return 'gcc'
+
+    def _should_create_symbols(self):
+        return False
+
+    def __init__(self, config_file, word_size):
+        Builder.__init__(self, config_file, word_size)
+
+
+class MacOSBuilder(Builder):
+
+    def _specific_prepare(self):
+        os.environ['LD_LIBRARY_PATH'] = path.abspath(path.join(self._get_swift_build_path(), 'build', 'out', 'release', 'lib'))
+        os.environ['LD_LIBRARY_PATH'] += os.pathsep + self._get_externals_path()
+
+    def _get_qmake_spec(self):
+        return 'macx-clang'
+
+    def _get_make_cmd(self):
+        return 'make'
+
+    def _get_qt_component(self):
+        return 'clang'
+
+    def _should_create_symbols(self):
+        return True
+
+    def __init__(self, config_file, word_size):
+        Builder.__init__(self, config_file, word_size)
+
+
+def print_help():
+    supported_compilers = {'Linux': ['gcc'],
+                           'Darwin': ['clang'],
+                           'Windows': ['msvc', 'mingw']
+                           }
+    compiler_help = '|'.join(supported_compilers[platform.system()])
+    print('jenkins.py -c <config file> -w <32|64> -t <' + compiler_help + '> [-d]')
+
+
+# Entry point if called as a standalone program
+def main(argv):
+    config_file = ''
+    word_size = ''
+    tool_chain = ''
+    dev_build = False
+
+    try:
+        opts, args = getopt.getopt(argv, 'hc:w:t:d', ['config=', 'wordsize=', 'toolchain=', 'dev'])
+    except getopt.GetoptError:
+        print_help()
+        sys.exit(2)
+
+    if len(opts) < 2 or len(opts) > 4:
+        print_help()
+        sys.exit(2)
+
+    for opt, arg in opts:
+        if opt == '-h':
+            print_help()
+            sys.exit()
+        elif opt in ('-c', '--config'):
+            config_file = path.abspath(arg)
+            if not os.path.exists(config_file):
+                print('Specified config file does not exist')
+                sys.exit(2)
+        elif opt in ('-w', '--wordsize'):
+            word_size = arg
+        elif opt in ('-t', '--toolchain'):
+            tool_chain = arg
+        elif opt in ('-d', '--dev'):
+            dev_build = True
+
+    if word_size not in ['32', '64']:
+        print('Unsupported word size. Choose 32 or 64')
+        sys.exit(2)
+
+    builders = {'Linux': {
+                    'gcc': LinuxBuilder},
+                'Darwin': {
+                    'clang': MacOSBuilder},
+                'Windows': {
+                    'msvc': MSVCBuilder,
+                    'mingw': MinGWBuilder
+                }
+    }
+
+    if tool_chain not in builders[platform.system()]:
+        print('Unknown or unsupported tool chain!')
+        sys.exit(2)
+
+    builder = builders[platform.system()][tool_chain](config_file, word_size)
+
+    builder.prepare()
+    builder.build(dev_build)
+    builder.checks()
+    builder.install()
+    builder.package_xswiftbus()
+    builder.symbols()
+
+
+# run main if run directly
+if __name__ == "__main__":
+    main(sys.argv[1:])

scripts/lib/util.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+
+import itertools
+import subprocess
+import sys
+
+
+def validate_pair(ob):
+  if not (len(ob) == 2):
+    print("Unexpected result:", ob, file=sys.stderr)
+    return False
+  else:
+    return True
+
+
+def consume(iter):
+  try:
+    while True: next(iter)
+  except StopIteration:
+    pass
+
+
+def get_environment_from_batch_command(env_cmd, initial=None):
+  """
+  Take a command (either a single command or list of arguments)
+  and return the environment created after running that command.
+  Note that if the command must be a batch file or .cmd file, or the
+  changes to the environment will not be captured.
+
+  If initial is supplied, it is used as the initial environment passed
+  to the child process.
+  """
+  if not isinstance(env_cmd, (list, tuple)):
+    env_cmd = [env_cmd]
+  # Construct the command that will alter the environment.
+  env_cmd = subprocess.list2cmdline(env_cmd)
+  # Create a tag so we can tell in the output when the proc is done.
+  tag = 'END OF BATCH COMMAND'
+  # Construct a cmd.exe command to do accomplish this.
+  cmd = 'cmd.exe /s /c "{env_cmd} && echo "{tag}" && set"'.format(**vars())
+  # Launch the process.
+  proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=initial)
+  # Parse the output sent to stdout.
+  lines = proc.stdout
+  # Consume whatever output occurs until the tag is reached.
+  consume(itertools.takewhile(lambda l: tag not in l, lines))
+  # Define a way to handle each KEY=VALUE line.
+  handle_line = lambda l: l.rstrip().split('=',1)
+  # Parse key/values into pairs.
+  pairs = map(handle_line, lines)
+  # Make sure the pairs are valid.
+  valid_pairs = filter(validate_pair, pairs)
+  # Construct a dictionary of the pairs.
+  result = dict(valid_pairs)
+  # Let the process finish.
+  proc.communicate()
+  return result
+
+
+def get_vs_env(vs_version, arch):
+  """
+  Returns the env object for VS building environment.
+
+  The vs_version can be strings like "12.0" (e.g. VS2013), the arch has to
+  be one of "x86", "amd64", "arm", "x86_amd64", "x86_arm", "amd64_x86",
+  "amd64_arm", e.g. the args passed to vcvarsall.bat.
+  """
+  vsvarsall = "C:\\Program Files (x86)\\Microsoft Visual Studio {0}\\VC\\vcvarsall.bat".format(vs_version)
+  return get_environment_from_batch_command([vsvarsall, arch])

scripts/symbolstore.py

@@ -198,10 +198,11 @@ class Dumper:
         if stop_pool:
             JobPool.shutdown()
 
-    def pack(self):
-        symbol_full_path = os.path.normpath(os.path.join(self.symbol_path, ".."))
-        tar_name = os.path.join(symbol_full_path, 'symbols.tar.gz')
-        tar = tarfile.open(tar_name, "w:gz")
+    def pack(self, tar_path=None):
+        if tar_path is None:
+            symbol_full_path = os.path.normpath(os.path.join(self.symbol_path, ".."))
+            tar_path = os.path.join(symbol_full_path, 'symbols.tar.gz')
+        tar = tarfile.open(tar_path, "w:gz")
         tar.add(self.symbol_path, arcname="symbols")
 
     def process(self, *args):