#!/bin/env python # Copyright (C) 2017 # swift Project Community/Contributors # # This file is part of swift project. It is subject to the license terms in the LICENSE file found in the top-level # directory of this distribution and at http://www.swift-project.org/license.html. No part of swift project, # including this file, may be copied, modified, propagated, or distributed except according to the terms # contained in the LICENSE file. import getopt import multiprocessing import os import os.path as path import platform import requests import subprocess import sys import symbolstore from lib.util import get_vs_env if sys.version_info < (3, 0): import ConfigParser as configparser else: import configparser class Builder: def prepare(self): """ Prepares the build environment, e.g. setup of environmental variables. Build processes will be called with the environment modified by this function. """ print('Preparing environment ...') os.environ['PATH'] += os.pathsep + self._get_qt_binary_path() self._specific_prepare() def build(self, jobs, dev_build): """ Run the build itself. Pass dev_build=True to enable a dev build """ print('Running build ...') build_path = self._get_swift_build_path() if not os.path.isdir(build_path): os.makedirs(build_path) os.chdir(build_path) qmake_call = ['qmake'] if dev_build: qmake_call += ['"BLACK_CONFIG+=SwiftDevBranch"'] qmake_call += ['-r', os.pardir] subprocess.check_call(qmake_call, env=dict(os.environ)) if not jobs: jobs = multiprocessing.cpu_count() jobs_arg = '-j{0}'.format(jobs) subprocess.check_call([self.make_cmd, jobs_arg], env=dict(os.environ)) def checks(self): """ Runs build checks. """ print('Installing ...') build_path = self._get_swift_build_path() os.chdir(build_path) if self._should_run_checks(): subprocess.check_call([self.make_cmd, 'check'], env=dict(os.environ)) pass def install(self): """ Installs all products to the default path. """ print('Running install()') build_path = self._get_swift_build_path() os.chdir(build_path) if self._should_run_publish(): subprocess.check_call([self.make_cmd, 'publish_installer'], env=dict(os.environ)) pass else: subprocess.check_call([self.make_cmd, 'install'], env=dict(os.environ)) pass def package_xswiftbus(self): """ Packages xswiftbus as 7z compressed archive into the swift source root. """ print('Packaging xswiftbus ...') build_path = self._get_swift_build_path() os.chdir(build_path) archive_name = '-'.join(['xswiftbus', platform.system(), self.word_size, self.version]) + '.7z' archive_path = path.abspath(path.join(os.pardir, archive_name)) content_path = path.abspath(path.join(os.curdir, 'dist', 'xswiftbus')) subprocess.check_call(['7z', 'a', '-mx=9', archive_path, content_path], env=dict(os.environ)) def symbols(self, upload_symbols): """ Generates the binary symbols and archives them into a gzip archive, located in the swift source root. """ if self._should_create_symbols(): build_path = self._get_swift_build_path() os.chdir(build_path) print('Creating symbols') symbol_path = path.abspath(path.join(build_path, 'symbols')) binary_path = path.abspath(path.join(build_path, 'out')) symbolstore.Dumper.global_init() dumper = symbolstore.get_platform_specific_dumper(dump_syms=self.dump_syms, symbol_path=symbol_path) dumper.process(binary_path) dumper.finish() tar_filename = '-'.join(['swift', 'symbols', platform.system(), self.word_size, self.version]) + '.tar.gz' tar_path = path.abspath(path.join(self._get_swift_source_path(), tar_filename)) dumper.pack(tar_path) if upload_symbols: self.__upload_symbol_files(symbol_path) def _get_swift_source_path(self): return self.__source_path def _get_swift_build_path(self): return self.__build_path def _specific_prepare(self): pass def _get_qmake_spec(self): raise NotImplementedError() def _get_make_cmd(self): raise NotImplementedError() def _get_qt_component(self): raise NotImplementedError() def _should_run_checks(self): return True def _should_run_publish(self): return True def _should_create_symbols(self): return True def _get_qtcreator_path(self): qtcreator_path = path.abspath(path.join(self.qt_path, 'Tools', 'QtCreator', 'bin')) return qtcreator_path def _get_externals_path(self): qmake_spec = self._get_qmake_spec() lib_path = 'lib' + self.word_size return path.abspath(path.join(self._get_swift_source_path(), 'externals', qmake_spec, lib_path)) def _get_qt_binary_path(self): component = self._get_qt_component() if self.word_size == '64': component += '_64' else: # special case for MSVC 32 bit component (most likely all versions). Those don't have the 32 bit suffix if "msvc" not in component: component += '_32' qt_binary_path = path.abspath(path.join(self.qt_path, self.qt_version, '{0}'.format(component), 'bin')) return qt_binary_path def _get_config(self): return self.__config def __init__(self, config_file, word_size): self.__source_path = path.abspath(path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir)) self.__build_path = path.abspath(path.join(self.__source_path, 'build')) swift_project_file = path.join(self.__source_path, 'swift.pro') if not os.path.isfile(swift_project_file): raise RuntimeError('Cannot find swift.pro! Are we in the right directory?') self.word_size = word_size if not config_file: config_file = path.abspath(path.join(self._get_swift_source_path(), 'scripts', 'jenkins.cfg')) self.__config = configparser.SafeConfigParser() self.__config.read(config_file) self.qt_version = self.__config.get('General', 'qt_version') self.qt_path = path.abspath(self.__config.get(platform.system(), 'qt_path')) self.dump_syms = path.abspath(self.__config.get(platform.system(), 'dump_syms')) self.make_cmd = self._get_make_cmd() self.version = self.__get_swift_version() def __get_swift_version(self): version_major = '0' version_minor = '0' version_micro = '0' version_file = path.abspath( path.join(self._get_swift_source_path(), 'mkspecs', 'features', 'version')) + '.pri' f = open(version_file) line = f.readline() while line: # Remove all spaces line = line.strip().replace(' ', '') tokens = line.split('=') if not len(tokens) == 2: raise ValueError('version.pri has wrong format!') if tokens[0] == 'BLACK_VER_MAJ': version_major = tokens[1] elif tokens[0] == 'BLACK_VER_MIN': version_minor = tokens[1] elif tokens[0] == 'BLACK_VER_MIC': version_micro = tokens[1] else: pass line = f.readline() f.close() version = '.'.join([version_major, version_minor, version_micro]) return version def __upload_symbol_files(self, symbol_path): print('Uploading symbols') url = 'http://crashreports.swift-project.org/symfiles' symbol_files = [os.path.join(root, name) for root, dirs, files in os.walk(symbol_path) for name in files if name.endswith('.sym')] for symbol_file in symbol_files: print ('Uploading ' + symbol_file) files = [ ('symfile', open(symbol_file, 'rb')), ] data = {'release': self.version} r = requests.post(url, files=files, data=data) r.raise_for_status() class MSVCBuilder(Builder): def _specific_prepare(self): os.environ['PATH'] += os.pathsep + self._get_externals_path() os.environ['PATH'] += os.pathsep + 'C:/Program Files/7-Zip' if self.word_size == '32': vs_env = get_vs_env('14.0', 'x86') else: vs_env = get_vs_env('14.0', 'amd64') os.environ.update(vs_env) def _get_qmake_spec(self): return 'win32-msvc2015' def _get_make_cmd(self): return path.abspath(path.join(self._get_qtcreator_path(), 'jom.exe')) def _get_qt_component(self): return 'msvc2015' def __init__(self, config_file, word_size): Builder.__init__(self, config_file, word_size) class MinGWBuilder(Builder): def _specific_prepare(self): os.environ['PATH'] += os.pathsep + self._get_externals_path() gcc_path = path.abspath(self._get_config().get(platform.system(), 'mingw_path')) os.environ['PATH'] += os.pathsep + gcc_path os.environ['PATH'] += os.pathsep + path.abspath(path.join('c:', os.sep, 'Program Files', '7-Zip')) def _get_qmake_spec(self): return 'win32-g++' def _get_make_cmd(self): return 'mingw32-make' def _get_qt_component(self): return 'mingw53' def _should_run_checks(self): return False def _should_run_publish(self): return False def _should_create_symbols(self): return False def __init__(self, config_file, word_size): Builder.__init__(self, config_file, word_size) class LinuxBuilder(Builder): def _specific_prepare(self): os.environ['LD_LIBRARY_PATH'] = path.abspath(path.join(self._get_swift_build_path(), 'build', 'out', 'release', 'lib')) os.environ['LD_LIBRARY_PATH'] += os.pathsep + path.abspath(path.join(self._get_swift_source_path(), 'lib')) os.environ['LD_LIBRARY_PATH'] += os.pathsep + self._get_externals_path() lib_path = path.abspath(path.join(self._get_swift_source_path(), 'lib')) libssl = path.abspath(path.join(lib_path, 'libssl.so')) libcrypto = path.abspath(path.join(lib_path, 'libcrypto.so')) if not os.path.isdir(lib_path): os.makedirs(lib_path) if not os.path.exists(libssl): os.symlink('/usr/lib/x86_64-linux-gnu/libssl.so.1.0.2', libssl) if not os.path.exists(libcrypto): os.symlink('/usr/lib/x86_64-linux-gnu/libcrypto.so.1.0.2', libcrypto) def _get_qmake_spec(self): return 'linux-g++' def _get_make_cmd(self): return 'make' def _get_qt_component(self): return 'gcc' def _should_create_symbols(self): return False def __init__(self, config_file, word_size): Builder.__init__(self, config_file, word_size) class MacOSBuilder(Builder): def _specific_prepare(self): os.environ['LD_LIBRARY_PATH'] = path.abspath(path.join(self._get_swift_build_path(), 'build', 'out', 'release', 'lib')) os.environ['LD_LIBRARY_PATH'] += os.pathsep + self._get_externals_path() def _get_qmake_spec(self): return 'macx-clang' def _get_make_cmd(self): return 'make' def _get_qt_component(self): return 'clang' def _should_create_symbols(self): return True def __init__(self, config_file, word_size): Builder.__init__(self, config_file, word_size) def print_help(): supported_compilers = {'Linux': ['gcc'], 'Darwin': ['clang'], 'Windows': ['msvc', 'mingw'] } compiler_help = '|'.join(supported_compilers[platform.system()]) print('jenkins.py -c -w <32|64> -t <' + compiler_help + '> [-d]') # Entry point if called as a standalone program def main(argv): config_file = '' word_size = '' tool_chain = '' dev_build = False jobs = None upload_symbols = False try: opts, args = getopt.getopt(argv, 'hc:w:t:j:du', ['config=', 'wordsize=', 'toolchain=', 'jobs=', 'dev', 'upload']) except getopt.GetoptError: print_help() sys.exit(2) if len(opts) < 2 or len(opts) > 5: print_help() sys.exit(2) for opt, arg in opts: if opt == '-h': print_help() sys.exit() elif opt in ('-c', '--config'): config_file = path.abspath(arg) if not os.path.exists(config_file): print('Specified config file does not exist') sys.exit(2) elif opt in ('-w', '--wordsize'): word_size = arg elif opt in ('-t', '--toolchain'): tool_chain = arg elif opt in ('-j', '--jobs'): jobs = arg elif opt in ('-d', '--dev'): dev_build = True elif opt in ('-u', '--upload'): upload_symbols = True if word_size not in ['32', '64']: print('Unsupported word size. Choose 32 or 64') sys.exit(2) builders = {'Linux': { 'gcc': LinuxBuilder}, 'Darwin': { 'clang': MacOSBuilder}, 'Windows': { 'msvc': MSVCBuilder, 'mingw': MinGWBuilder } } if tool_chain not in builders[platform.system()]: print('Unknown or unsupported tool chain!') sys.exit(2) builder = builders[platform.system()][tool_chain](config_file, word_size) builder.prepare() builder.build(jobs, dev_build) builder.checks() builder.install() builder.package_xswiftbus() builder.symbols(upload_symbols) # run main if run directly if __name__ == "__main__": main(sys.argv[1:])