diff --git a/scripts/jenkins.cfg b/scripts/jenkins.cfg new file mode 100644 index 000000000..00b52ee03 --- /dev/null +++ b/scripts/jenkins.cfg @@ -0,0 +1,15 @@ +[General] +qt_version: 5.9.1 + +[Windows] +qt_path: C:/Qt +dump_syms: C:/bin/dump_syms.exe +mingw_path: C:/Qt/Tools/mingw530_32/bin + +[Linux] +qt_path: /opt/Qt +dump_syms: + +[Darwin] +qt_path: /Applications/Qt +dump_syms: /usr/local/bin/dump_syms diff --git a/scripts/jenkins.py b/scripts/jenkins.py new file mode 100644 index 000000000..00218e638 --- /dev/null +++ b/scripts/jenkins.py @@ -0,0 +1,387 @@ +#!/bin/env python + +# Copyright (C) 2017 +# swift Project Community/Contributors +# +# This file is part of swift project. It is subject to the license terms in the LICENSE file found in the top-level +# directory of this distribution and at http://www.swift-project.org/license.html. No part of swift project, +# including this file, may be copied, modified, propagated, or distributed except according to the terms +# contained in the LICENSE file. + +import getopt +import multiprocessing +import os +import os.path as path +import platform +import subprocess +import sys +import symbolstore +from lib.util import get_vs_env + +if sys.version_info < (3, 0): + import ConfigParser as configparser +else: + import configparser + + +class Builder: + + def prepare(self): + """ + Prepares the build environment, e.g. setup of environmental variables. + Build processes will be called with the environment modified by this function. + """ + print('Preparing environment ...') + os.environ['PATH'] += os.pathsep + self._get_qt_binary_path() + self._specific_prepare() + + def build(self, dev_build): + """ + Run the build itself. Pass dev_build=True to enable a dev build + """ + print('Running build ...') + build_path = self._get_swift_build_path() + if not os.path.isdir(build_path): + os.makedirs(build_path) + os.chdir(build_path) + qmake_call = ['qmake'] + if dev_build: + qmake_call += ['"BLACK_CONFIG+=SwiftDevBranch"'] + qmake_call += ['-r', os.pardir] + subprocess.check_call(qmake_call, env=dict(os.environ)) + + job_arg = '-j{0}'.format(multiprocessing.cpu_count()) + subprocess.check_call([self.make_cmd, job_arg], env=dict(os.environ)) + + def checks(self): + """ + Runs build checks. + """ + print('Installing ...') + build_path = self._get_swift_build_path() + os.chdir(build_path) + if self._should_run_checks(): + subprocess.check_call([self.make_cmd, 'check'], env=dict(os.environ)) + pass + + def install(self): + """ + Installs all products to the default path. + """ + print('Running install()') + build_path = self._get_swift_build_path() + os.chdir(build_path) + if self._should_run_publish(): + subprocess.check_call([self.make_cmd, 'publish_installer'], env=dict(os.environ)) + pass + else: + subprocess.check_call([self.make_cmd, 'install'], env=dict(os.environ)) + pass + + def package_xswiftbus(self): + """ + Packages xswiftbus as 7z compressed archive into the swift source root. + """ + print('Packaging xswiftbus ...') + build_path = self._get_swift_build_path() + os.chdir(build_path) + archive_name = '-'.join(['xswiftbus', platform.system(), self.word_size, self.version]) + '.7z' + archive_path = path.abspath(path.join(os.pardir, archive_name)) + content_path = path.abspath(path.join(os.curdir, 'dist', 'xswiftbus')) + subprocess.check_call(['7z', 'a', '-mx=9', archive_path, content_path], env=dict(os.environ)) + + def symbols(self): + """ + Generates the binary symbols and archives them into a gzip archive, located in the swift source root. + """ + if self._should_create_symbols(): + build_path = self._get_swift_build_path() + os.chdir(build_path) + print('Creating symbols') + symbol_path = path.abspath(path.join(build_path, 'symbols')) + binary_path = path.abspath(path.join(build_path, 'out')) + symbolstore.Dumper.global_init() + dumper = symbolstore.get_platform_specific_dumper(dump_syms=self.dump_syms, symbol_path=symbol_path) + dumper.process(binary_path) + dumper.finish() + tar_filename = '-'.join(['swift', 'symbols', platform.system(), self.word_size, self.version]) + '.tar.gz' + tar_path = path.abspath(path.join(self._get_swift_source_path(), tar_filename)) + dumper.pack(tar_path) + + def _get_swift_source_path(self): + return self.__source_path + + def _get_swift_build_path(self): + return self.__build_path + + def _specific_prepare(self): + pass + + def _get_qmake_spec(self): + raise NotImplementedError() + + def _get_make_cmd(self): + raise NotImplementedError() + + def _get_qt_component(self): + raise NotImplementedError() + + def _should_run_checks(self): + return True + + def _should_run_publish(self): + return True + + def _should_create_symbols(self): + return True + + def _get_qtcreator_path(self): + qtcreator_path = path.abspath(path.join(self.qt_path, 'Tools', 'QtCreator', 'bin')) + return qtcreator_path + + def _get_externals_path(self): + qmake_spec = self._get_qmake_spec() + lib_path = 'lib' + self.word_size + return path.abspath(path.join(self._get_swift_source_path(), 'externals', qmake_spec, lib_path)) + + def _get_qt_binary_path(self): + component = self._get_qt_component() + if self.word_size == '64': + component += '_64' + else: + # special case for MSVC 32 bit component (most likely all versions). Those don't have the 32 bit suffix + if "msvc" not in component: + component += '_32' + qt_binary_path = path.abspath(path.join(self.qt_path, self.qt_version, '{0}'.format(component), 'bin')) + return qt_binary_path + + def _get_config(self): + return self.__config + + def __init__(self, config_file, word_size): + self.__source_path = path.abspath(path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir)) + self.__build_path = path.abspath(path.join(self.__source_path, 'build')) + + swift_project_file = path.join(self.__source_path, 'swift.pro') + if not os.path.isfile(swift_project_file): + raise RuntimeError('Cannot find swift.pro! Are we in the right directory?') + + self.word_size = word_size + + if not config_file: + config_file = path.abspath(path.join(self._get_swift_source_path(), 'scripts', 'jenkins.cfg')) + self.__config = configparser.SafeConfigParser() + self.__config.read(config_file) + self.qt_version = self.__config.get('General', 'qt_version') + self.qt_path = path.abspath(self.__config.get(platform.system(), 'qt_path')) + self.dump_syms = path.abspath(self.__config.get(platform.system(), 'dump_syms')) + + self.make_cmd = self._get_make_cmd() + self.version = self.__get_swift_version() + + def __get_swift_version(self): + version_major = '0' + version_minor = '0' + version_micro = '0' + version_file = path.abspath( + path.join(self._get_swift_source_path(), 'mkspecs', 'features', 'version')) + '.pri' + f = open(version_file) + line = f.readline() + while line: + # Remove all spaces + line = line.strip().replace(' ', '') + tokens = line.split('=') + if not len(tokens) == 2: + raise ValueError('version.pri has wrong format!') + if tokens[0] == 'BLACK_VER_MAJ': + version_major = tokens[1] + elif tokens[0] == 'BLACK_VER_MIN': + version_minor = tokens[1] + elif tokens[0] == 'BLACK_VER_MIC': + version_micro = tokens[1] + else: + pass + line = f.readline() + f.close() + version = '.'.join([version_major, version_minor, version_micro]) + return version + + +class MSVCBuilder(Builder): + + def _specific_prepare(self): + os.environ['PATH'] += os.pathsep + self._get_externals_path() + os.environ['PATH'] += os.pathsep + 'C:/Program Files/7-Zip' + vs_env = get_vs_env('14.0', 'amd64') + os.environ.update(vs_env) + + def _get_qmake_spec(self): + return 'win32-msvc2015' + + def _get_make_cmd(self): + return path.abspath(path.join(self._get_qtcreator_path(), 'jom.exe')) + + def _get_qt_component(self): + return 'msvc2015' + + def __init__(self, config_file, word_size): + Builder.__init__(self, config_file, word_size) + + +class MinGWBuilder(Builder): + + def _specific_prepare(self): + os.environ['PATH'] += os.pathsep + self._get_externals_path() + gcc_path = path.abspath(self._get_config().get(platform.system(), 'mingw_path')) + os.environ['PATH'] += os.pathsep + gcc_path + os.environ['PATH'] += os.pathsep + path.abspath(path.join('c:', os.sep, 'Program Files', '7-Zip')) + + def _get_qmake_spec(self): + return 'win32-g++' + + def _get_make_cmd(self): + return 'mingw32-make' + + def _get_qt_component(self): + return 'mingw53' + + def _should_run_checks(self): + return False + + def _should_run_publish(self): + return False + + def _should_create_symbols(self): + return False + + def __init__(self, config_file, word_size): + Builder.__init__(self, config_file, word_size) + + +class LinuxBuilder(Builder): + + def _specific_prepare(self): + os.environ['LD_LIBRARY_PATH'] = path.abspath(path.join(self._get_swift_build_path(), 'build', 'out', 'release', 'lib')) + os.environ['LD_LIBRARY_PATH'] += os.pathsep + path.abspath(path.join(self._get_swift_source_path(), 'lib')) + os.environ['LD_LIBRARY_PATH'] += os.pathsep + self._get_externals_path() + lib_path = path.abspath(path.join(self._get_swift_source_path(), 'lib')) + libssl = path.abspath(path.join(lib_path, 'libssl.so')) + libcrypto = path.abspath(path.join(lib_path, 'libcrypto.so')) + if not os.path.isdir(lib_path): + os.makedirs(lib_path) + if not os.path.exists(libssl): + os.symlink('/usr/lib/x86_64-linux-gnu/libssl.so.1.0.2', libssl) + if not os.path.exists(libcrypto): + os.symlink('/usr/lib/x86_64-linux-gnu/libcrypto.so.1.0.2', libcrypto) + + def _get_qmake_spec(self): + return 'linux-g++' + + def _get_make_cmd(self): + return 'make' + + def _get_qt_component(self): + return 'gcc' + + def _should_create_symbols(self): + return False + + def __init__(self, config_file, word_size): + Builder.__init__(self, config_file, word_size) + + +class MacOSBuilder(Builder): + + def _specific_prepare(self): + os.environ['LD_LIBRARY_PATH'] = path.abspath(path.join(self._get_swift_build_path(), 'build', 'out', 'release', 'lib')) + os.environ['LD_LIBRARY_PATH'] += os.pathsep + self._get_externals_path() + + def _get_qmake_spec(self): + return 'macx-clang' + + def _get_make_cmd(self): + return 'make' + + def _get_qt_component(self): + return 'clang' + + def _should_create_symbols(self): + return True + + def __init__(self, config_file, word_size): + Builder.__init__(self, config_file, word_size) + + +def print_help(): + supported_compilers = {'Linux': ['gcc'], + 'Darwin': ['clang'], + 'Windows': ['msvc', 'mingw'] + } + compiler_help = '|'.join(supported_compilers[platform.system()]) + print('jenkins.py -c -w <32|64> -t <' + compiler_help + '> [-d]') + + +# Entry point if called as a standalone program +def main(argv): + config_file = '' + word_size = '' + tool_chain = '' + dev_build = False + + try: + opts, args = getopt.getopt(argv, 'hc:w:t:d', ['config=', 'wordsize=', 'toolchain=', 'dev']) + except getopt.GetoptError: + print_help() + sys.exit(2) + + if len(opts) < 2 or len(opts) > 4: + print_help() + sys.exit(2) + + for opt, arg in opts: + if opt == '-h': + print_help() + sys.exit() + elif opt in ('-c', '--config'): + config_file = path.abspath(arg) + if not os.path.exists(config_file): + print('Specified config file does not exist') + sys.exit(2) + elif opt in ('-w', '--wordsize'): + word_size = arg + elif opt in ('-t', '--toolchain'): + tool_chain = arg + elif opt in ('-d', '--dev'): + dev_build = True + + if word_size not in ['32', '64']: + print('Unsupported word size. Choose 32 or 64') + sys.exit(2) + + builders = {'Linux': { + 'gcc': LinuxBuilder}, + 'Darwin': { + 'clang': MacOSBuilder}, + 'Windows': { + 'msvc': MSVCBuilder, + 'mingw': MinGWBuilder + } + } + + if tool_chain not in builders[platform.system()]: + print('Unknown or unsupported tool chain!') + sys.exit(2) + + builder = builders[platform.system()][tool_chain](config_file, word_size) + + builder.prepare() + builder.build(dev_build) + builder.checks() + builder.install() + builder.package_xswiftbus() + builder.symbols() + + +# run main if run directly +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/scripts/lib/__init__.py b/scripts/lib/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/lib/util.py b/scripts/lib/util.py new file mode 100644 index 000000000..8c18f1831 --- /dev/null +++ b/scripts/lib/util.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python + +from __future__ import print_function + +import itertools +import subprocess +import sys + + +def validate_pair(ob): + if not (len(ob) == 2): + print("Unexpected result:", ob, file=sys.stderr) + return False + else: + return True + + +def consume(iter): + try: + while True: next(iter) + except StopIteration: + pass + + +def get_environment_from_batch_command(env_cmd, initial=None): + """ + Take a command (either a single command or list of arguments) + and return the environment created after running that command. + Note that if the command must be a batch file or .cmd file, or the + changes to the environment will not be captured. + + If initial is supplied, it is used as the initial environment passed + to the child process. + """ + if not isinstance(env_cmd, (list, tuple)): + env_cmd = [env_cmd] + # Construct the command that will alter the environment. + env_cmd = subprocess.list2cmdline(env_cmd) + # Create a tag so we can tell in the output when the proc is done. + tag = 'END OF BATCH COMMAND' + # Construct a cmd.exe command to do accomplish this. + cmd = 'cmd.exe /s /c "{env_cmd} && echo "{tag}" && set"'.format(**vars()) + # Launch the process. + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=initial) + # Parse the output sent to stdout. + lines = proc.stdout + # Consume whatever output occurs until the tag is reached. + consume(itertools.takewhile(lambda l: tag not in l, lines)) + # Define a way to handle each KEY=VALUE line. + handle_line = lambda l: l.rstrip().split('=',1) + # Parse key/values into pairs. + pairs = map(handle_line, lines) + # Make sure the pairs are valid. + valid_pairs = filter(validate_pair, pairs) + # Construct a dictionary of the pairs. + result = dict(valid_pairs) + # Let the process finish. + proc.communicate() + return result + + +def get_vs_env(vs_version, arch): + """ + Returns the env object for VS building environment. + + The vs_version can be strings like "12.0" (e.g. VS2013), the arch has to + be one of "x86", "amd64", "arm", "x86_amd64", "x86_arm", "amd64_x86", + "amd64_arm", e.g. the args passed to vcvarsall.bat. + """ + vsvarsall = "C:\\Program Files (x86)\\Microsoft Visual Studio {0}\\VC\\vcvarsall.bat".format(vs_version) + return get_environment_from_batch_command([vsvarsall, arch]) \ No newline at end of file diff --git a/scripts/create_symbolstore.py b/scripts/symbolstore.py old mode 100755 new mode 100644 similarity index 98% rename from scripts/create_symbolstore.py rename to scripts/symbolstore.py index 166476293..592f47419 --- a/scripts/create_symbolstore.py +++ b/scripts/symbolstore.py @@ -198,10 +198,11 @@ class Dumper: if stop_pool: JobPool.shutdown() - def pack(self): - symbol_full_path = os.path.normpath(os.path.join(self.symbol_path, "..")) - tar_name = os.path.join(symbol_full_path, 'symbols.tar.gz') - tar = tarfile.open(tar_name, "w:gz") + def pack(self, tar_path=None): + if tar_path is None: + symbol_full_path = os.path.normpath(os.path.join(self.symbol_path, "..")) + tar_path = os.path.join(symbol_full_path, 'symbols.tar.gz') + tar = tarfile.open(tar_path, "w:gz") tar.add(self.symbol_path, arcname="symbols") def process(self, *args):