Automatically update shared files during jenkins build

Summary:
This change adds an automatic shared files update step during the jenkins
build. This guarantees, that we always have the latest DB data packaged
into the installers.
The updated files will be written into the source directory, so install
step will automatically pick up the updated ones.

ref T158

Reviewers: #swift_pilot_client, msutcliffe

Reviewed By: #swift_pilot_client, msutcliffe

Subscribers: jenkins

Maniphest Tasks: T158

Differential Revision: https://dev.swift-project.org/D56

scripts/datastore.py

@@ -0,0 +1,272 @@
+#!/bin/env python
+
+# Copyright (C) 2017
+# swift Project Community/Contributors
+#
+# This file is part of swift project. It is subject to the license terms in the LICENSE file found in the top-level
+# directory of this distribution and at http://www.swift-project.org/license.html. No part of swift project,
+# including this file, may be copied, modified, propagated, or distributed except according to the terms
+# contained in the LICENSE file.
+
+import json
+import os
+import datetime
+import zlib
+import base64
+import io
+import ssl
+
+try:
+    from urllib.request import urlopen
+    from urllib.error import HTTPError, URLError
+except ImportError:
+    from urllib2 import urlopen, HTTPError, URLError
+
+
+class DbInfo:
+    def __init__(self):
+        self._table_name = ''
+        self._file_name = ''
+        self._last_updated = datetime.datetime(datetime.MINYEAR, 1, 1)
+
+    @property
+    def table_name(self):
+        return self._table_name
+
+    @table_name.setter
+    def table_name(self, table_name):
+        self._table_name = table_name
+
+    @property
+    def file_name(self):
+        return self._file_name
+
+    @file_name.setter
+    def file_name(self, file_name):
+        self._file_name = file_name
+
+    @property
+    def last_updated(self):
+        return self._last_updated
+
+    @last_updated.setter
+    def last_updated(self, last_updated):
+        self._last_updated = last_updated
+
+
+class DbInfosLoader:
+    def __init__(self, version):
+        self.__filename = 'dbinfo.json'
+        self.__version = version
+        self.__entities_to_file_names = {
+            'aircrafticao': 'aircrafticao.json',
+            'aircraftmodel': 'models.json',
+            'airlineicao': 'airlineicao.json',
+            'airport': 'airports.json',
+            'country': 'countries.json',
+            'distributor': 'distributors.json',
+            'livery': 'liveries.json'
+        }
+
+    def convert_json_to_db_infos(self, j):
+        db_infos = []
+        data = j['data']
+        for db in data:
+            db_info = DbInfo()
+            db_info.table_name = db['tablename']
+            if db['tablename'] in self.entities_to_file_names:
+                db_info.file_name = self.entities_to_file_names[db['tablename']]
+                last_updated = datetime.datetime.strptime(db['lastupdated'], '%Y-%m-%d %H:%M:%S')
+                db_info.last_updated = last_updated
+                db_infos.append(db_info)
+        return db_infos
+
+    @property
+    def file_name(self):
+        return self.__filename
+
+    @property
+    def version(self):
+        return self.__version
+
+    @property
+    def entities_to_file_names(self):
+        return self.__entities_to_file_names
+
+    def filename_by_table_name(self, db):
+        return self.entities_to_file_names[db]
+
+
+class DbInfosLoaderLocal(DbInfosLoader):
+    def __init__(self, version, target_path):
+        DbInfosLoader.__init__(self, version)
+        self.__db_infos = []
+        file_path = os.path.abspath(os.path.join(target_path, 'shared', 'dbdata', self.file_name))
+        if os.path.exists(file_path):
+            with io.open(file_path, 'r', encoding='utf-8') as f:
+                j = json.load(f)
+            self.__db_infos = self.convert_json_to_db_infos(j)
+
+    def last_updated_by_table_name(self, table_name):
+        for db in self.__db_infos:
+            if db.table_name == table_name:
+                return db.last_updated
+
+
+class DbInfosLoaderRemote(DbInfosLoader):
+    def __init__(self, version):
+        DbInfosLoader.__init__(self, version)
+
+    def download_from(self, host):
+        # Open the url
+        url = host + '/shared/' + self.version + '/dbdata/' + self.file_name
+        try:
+            ctx = ssl.create_default_context()
+            ctx.check_hostname = False
+            ctx.verify_mode = ssl.CERT_NONE
+            response = urlopen(url, context=ctx)
+            timestamp = datetime.datetime.strptime(response.headers['last-modified'], '%a, %d %b %Y %H:%M:%S GMT')
+            print("Getting db info " + url)
+            content = response.read().decode('utf-8')
+            j = json.loads(content)
+            db_infos = self.convert_json_to_db_infos(j)
+            return db_infos, j, timestamp
+        except HTTPError as e:
+            error = "HTTP Error: " + str(e.code) + ' ' + url
+            print(error)
+        except URLError as e:
+            error = "HTTP Error: " + e.reason + ' ' + url
+            print(error)
+
+
+def write_text_to_file(file_path, text_content, timestamp):
+    if text_content:
+        # Normalize the file endings
+        text_content = text_content.replace('\r\n', '\n')
+        with io.open(file_path, 'w', encoding='utf-8') as f:
+            f.write(text_content)
+        atime = mtime = (timestamp - datetime.datetime.utcfromtimestamp(0)).total_seconds()
+        os.utime(file_path, (atime, mtime))
+
+
+def write_json_to_file(file_path, json_content, timestamp):
+    if json_content:
+        with io.open(file_path, 'w', encoding='utf-8') as f:
+            f.write(json.dumps(json_content, ensure_ascii=False))
+        atime = mtime = (timestamp - datetime.datetime.utcfromtimestamp(0)).total_seconds()
+        os.utime(file_path, (atime, mtime))
+
+
+class BaseSync:
+    def __init__(self, host, version, target_path):
+        self.__host = host
+        self.__version = version
+        self.__target_path = target_path
+        if self.__target_path is None:
+            self.__target_path = ''
+        pass
+
+    @staticmethod
+    def __uncompress_content(compressed_content):
+        compressed_content = base64.b64decode(compressed_content)
+        content = zlib.decompress(compressed_content, 0)
+        return content
+
+    @property
+    def version(self):
+        return self.__version
+
+    @property
+    def host(self):
+        return self.__host
+
+    @property
+    def target_path(self):
+        return self.__target_path
+
+    def sync_file(self, data_type, file_name):
+        path = os.path.join(self.target_path, 'shared', data_type)
+        if not os.path.exists(path):
+            os.makedirs(path)
+
+        url = self.host + '/shared/' + self.version + '/' + data_type + '/' + file_name
+        try:
+            ctx = ssl.create_default_context()
+            ctx.check_hostname = False
+            ctx.verify_mode = ssl.CERT_NONE
+            response = urlopen(url, context=ctx)
+            timestamp = datetime.datetime.strptime(response.headers['last-modified'], '%a, %d %b %Y %H:%M:%S GMT')
+            print("Syncing file " + url)
+            file_path = os.path.join(self.target_path, 'shared', data_type, file_name)
+            content = response.read().decode('utf-8')
+            write_text_to_file(file_path, content, timestamp)
+
+        except HTTPError as e:
+            error = "HTTP Error: " + str(e.code) + ' ' + url
+            print(error)
+        except URLError as e:
+            error = "HTTP Error: " + e.reason + ' ' + url
+            print(error)
+
+
+class DbDataSync(BaseSync):
+    def __init__(self, host, version, target_path):
+        BaseSync.__init__(self, host, version, target_path)
+        self.loader_local = DbInfosLoaderLocal(version, target_path)
+        self.loader_remote = DbInfosLoaderRemote(version)
+
+    def sync(self):
+        db_infos, j, timestamp = self.loader_remote.download_from(self.host)
+        db_updated = False
+        for db_info in db_infos:
+            remote_last_updated = db_info.last_updated
+            table_name = db_info.table_name
+            file_name = self.loader_remote.filename_by_table_name(table_name)
+            local_last_updated = self.loader_local.last_updated_by_table_name(table_name)
+            if not local_last_updated or remote_last_updated > local_last_updated:
+                self.sync_file('dbdata', db_info.file_name)
+                db_updated = True
+            else:
+                print(file_name + " is up to date.")
+
+        if db_updated:
+            print('DB data successfully updated')
+
+        file_path = os.path.join(self.target_path, 'shared', 'dbdata', self.loader_local.file_name)
+        write_json_to_file(file_path, j, timestamp)
+
+
+class BootstrapSync(BaseSync):
+    def __init__(self, host, version, target_path):
+        BaseSync.__init__(self, host, version, target_path)
+
+    def sync(self):
+        self.sync_file('bootstrap', 'bootstrap.json')
+
+
+class UpdateInfoSync(BaseSync):
+    def __init__(self, host, version, target_path):
+        BaseSync.__init__(self, host, version, target_path)
+
+    def sync(self):
+        self.sync_file('updateinfo', 'distribution.json')
+
+
+def update_shared(host, version, target_path):
+    DbDataSync(host, version, target_path).sync()
+    BootstrapSync(host, version, target_path).sync()
+    UpdateInfoSync(host, version, target_path).sync()
+
+
+def main():
+    host = 'https://datastore.swift-project.org'
+    version = '0.7.0'
+    target_path = os.path.abspath(os.curdir)
+
+    DbDataSync(host, version, target_path).sync()
+    BootstrapSync(host, version, target_path).sync()
+    UpdateInfoSync(host, version, target_path).sync()
+
+
+if __name__ == '__main__':
+    main()

scripts/jenkins.cfg

@@ -1,5 +1,6 @@
 [General]
 qt_version: 5.9.1
+datastore_version: 0.7.0
 
 [Windows]
 qt_path: C:/Qt

scripts/jenkins.py

@@ -16,6 +16,7 @@ import platform
 import requests
 import subprocess
 import sys
+import datastore
 import symbolstore
 from lib.util import get_vs_env
 
@@ -36,6 +37,13 @@ class Builder:
         os.environ['PATH'] += os.pathsep + self._get_qt_binary_path()
         self._specific_prepare()
 
+        print('Updating from datastore ...')
+        host = 'https://datastore.swift-project.org'
+        datastore_version = self.__config.get('General', 'datastore_version')
+        source_path = self._get_swift_source_path()
+        shared_path = os.path.abspath(os.path.join(source_path, 'resources', 'share'))
+        datastore.update_shared(host, datastore_version, shared_path)
+
     def build(self, jobs, dev_build):
         """
         Run the build itself. Pass dev_build=True to enable a dev build
@@ -60,7 +68,7 @@ class Builder:
         """
         Runs build checks.
         """
-        print('Installing ...')
+        print('Running checks ...')
         build_path = self._get_swift_build_path()
         os.chdir(build_path)
         if self._should_run_checks():
@@ -71,7 +79,7 @@ class Builder:
         """
         Installs all products to the default path.
         """
-        print('Running install()')
+        print('Running install ...')
         build_path = self._get_swift_build_path()
         os.chdir(build_path)
         if self._should_run_publish():