mirror of
https://github.com/swift-project/pilotclient.git
synced 2026-03-22 06:45:37 +08:00
228 lines
7.2 KiB
Python
228 lines
7.2 KiB
Python
#!/bin/env python
|
|
|
|
# SPDX-FileCopyrightText: Copyright (C) 2017 swift Project Community / Contributors
|
|
# SPDX-License-Identifier: GPL-3.0-or-later OR LicenseRef-swift-pilot-client-1
|
|
|
|
import json
|
|
import os
|
|
import datetime
|
|
import zlib
|
|
import base64
|
|
import io
|
|
import requests
|
|
|
|
|
|
class DbInfo:
|
|
def __init__(self):
|
|
self._table_name = ''
|
|
self._file_name = ''
|
|
self._last_updated = datetime.datetime(datetime.MINYEAR, 1, 1)
|
|
|
|
@property
|
|
def table_name(self):
|
|
return self._table_name
|
|
|
|
@table_name.setter
|
|
def table_name(self, table_name):
|
|
self._table_name = table_name
|
|
|
|
@property
|
|
def file_name(self):
|
|
return self._file_name
|
|
|
|
@file_name.setter
|
|
def file_name(self, file_name):
|
|
self._file_name = file_name
|
|
|
|
@property
|
|
def last_updated(self):
|
|
return self._last_updated
|
|
|
|
@last_updated.setter
|
|
def last_updated(self, last_updated):
|
|
self._last_updated = last_updated
|
|
|
|
|
|
class DbInfosLoader:
|
|
def __init__(self, version):
|
|
self.__filename = 'dbinfo.json'
|
|
self.__version = version
|
|
self.__entities_to_file_names = {
|
|
'aircrafticao': 'aircrafticao.json',
|
|
'aircraftmodel': 'models.json',
|
|
'airlineicao': 'airlineicao.json',
|
|
'airport': 'airports.json',
|
|
'country': 'countries.json',
|
|
'distributor': 'distributors.json',
|
|
'livery': 'liveries.json'
|
|
}
|
|
|
|
def convert_json_to_db_infos(self, j):
|
|
db_infos = []
|
|
data = j['data']
|
|
for db in data:
|
|
db_info = DbInfo()
|
|
db_info.table_name = db['tablename']
|
|
if db['tablename'] in self.entities_to_file_names:
|
|
db_info.file_name = self.entities_to_file_names[db['tablename']]
|
|
last_updated = datetime.datetime.strptime(db['lastupdated'], '%Y-%m-%d %H:%M:%S')
|
|
db_info.last_updated = last_updated
|
|
db_infos.append(db_info)
|
|
return db_infos
|
|
|
|
@property
|
|
def file_name(self):
|
|
return self.__filename
|
|
|
|
@property
|
|
def version(self):
|
|
return self.__version
|
|
|
|
@property
|
|
def entities_to_file_names(self):
|
|
return self.__entities_to_file_names
|
|
|
|
def filename_by_table_name(self, db):
|
|
return self.entities_to_file_names[db]
|
|
|
|
|
|
class DbInfosLoaderLocal(DbInfosLoader):
|
|
def __init__(self, version, target_path):
|
|
DbInfosLoader.__init__(self, version)
|
|
self.__db_infos = []
|
|
file_path = os.path.abspath(os.path.join(target_path, 'shared', 'dbdata', self.file_name))
|
|
if os.path.exists(file_path):
|
|
with io.open(file_path, 'r', encoding='utf-8') as f:
|
|
j = json.load(f)
|
|
self.__db_infos = self.convert_json_to_db_infos(j)
|
|
|
|
def last_updated_by_table_name(self, table_name):
|
|
for db in self.__db_infos:
|
|
if db.table_name == table_name:
|
|
return db.last_updated
|
|
|
|
|
|
class DbInfosLoaderRemote(DbInfosLoader):
|
|
def __init__(self, version):
|
|
DbInfosLoader.__init__(self, version)
|
|
|
|
def download_from(self, host):
|
|
# Open the url
|
|
url = host + '/shared/' + self.version + '/dbdata/' + self.file_name
|
|
try:
|
|
response = requests.get(url)
|
|
response.raise_for_status()
|
|
timestamp = datetime.datetime.strptime(response.headers['last-modified'], '%a, %d %b %Y %H:%M:%S GMT')
|
|
print("Getting db info " + url)
|
|
j = response.json()
|
|
db_infos = self.convert_json_to_db_infos(j)
|
|
return db_infos, j, timestamp
|
|
except requests.HTTPError as e:
|
|
print(f"HTTP Error: {e}")
|
|
|
|
|
|
def write_text_to_file(file_path, text_content, timestamp):
|
|
if text_content:
|
|
# Normalize the file endings
|
|
text_content = text_content.replace('\r\n', '\n')
|
|
with io.open(file_path, 'w', encoding='utf-8') as f:
|
|
f.write(text_content)
|
|
atime = mtime = (timestamp - datetime.datetime.utcfromtimestamp(0)).total_seconds()
|
|
os.utime(file_path, (atime, mtime))
|
|
|
|
|
|
def write_json_to_file(file_path, json_content, timestamp):
|
|
if json_content:
|
|
with io.open(file_path, 'w', encoding='utf-8') as f:
|
|
f.write(json.dumps(json_content, ensure_ascii=False))
|
|
atime = mtime = (timestamp - datetime.datetime.utcfromtimestamp(0)).total_seconds()
|
|
os.utime(file_path, (atime, mtime))
|
|
|
|
|
|
class BaseSync:
|
|
def __init__(self, host, version, target_path):
|
|
self.__host = host
|
|
self.__version = version
|
|
self.__target_path = target_path
|
|
if self.__target_path is None:
|
|
self.__target_path = ''
|
|
pass
|
|
|
|
@staticmethod
|
|
def __uncompress_content(compressed_content):
|
|
compressed_content = base64.b64decode(compressed_content)
|
|
content = zlib.decompress(compressed_content, 0)
|
|
return content
|
|
|
|
@property
|
|
def version(self):
|
|
return self.__version
|
|
|
|
@property
|
|
def host(self):
|
|
return self.__host
|
|
|
|
@property
|
|
def target_path(self):
|
|
return self.__target_path
|
|
|
|
def sync_file(self, data_type, file_name):
|
|
path = os.path.join(self.target_path, 'shared', data_type)
|
|
if not os.path.exists(path):
|
|
os.makedirs(path)
|
|
|
|
url = self.host + '/shared/' + self.version + '/' + data_type + '/' + file_name
|
|
try:
|
|
response = requests.get(url)
|
|
response.raise_for_status()
|
|
timestamp = datetime.datetime.strptime(response.headers['last-modified'], '%a, %d %b %Y %H:%M:%S GMT')
|
|
print("Syncing file " + url)
|
|
file_path = os.path.join(self.target_path, 'shared', data_type, file_name)
|
|
write_text_to_file(file_path, response.text, timestamp)
|
|
|
|
except requests.HTTPError as e:
|
|
print(f"HTTP Error: {e}")
|
|
|
|
|
|
class DbDataSync(BaseSync):
|
|
def __init__(self, host, version, target_path):
|
|
BaseSync.__init__(self, host, version, target_path)
|
|
self.loader_local = DbInfosLoaderLocal(version, target_path)
|
|
self.loader_remote = DbInfosLoaderRemote(version)
|
|
|
|
def sync(self):
|
|
db_infos, j, timestamp = self.loader_remote.download_from(self.host)
|
|
db_updated = False
|
|
for db_info in db_infos:
|
|
remote_last_updated = db_info.last_updated
|
|
table_name = db_info.table_name
|
|
file_name = self.loader_remote.filename_by_table_name(table_name)
|
|
local_last_updated = self.loader_local.last_updated_by_table_name(table_name)
|
|
if not local_last_updated or remote_last_updated > local_last_updated:
|
|
self.sync_file('dbdata', db_info.file_name)
|
|
db_updated = True
|
|
else:
|
|
print(file_name + " is up to date.")
|
|
|
|
if db_updated:
|
|
print('DB data successfully updated')
|
|
|
|
file_path = os.path.join(self.target_path, 'shared', 'dbdata', self.loader_local.file_name)
|
|
write_json_to_file(file_path, j, timestamp)
|
|
|
|
|
|
def update_shared(host, version, target_path):
|
|
DbDataSync(host, version, target_path).sync()
|
|
|
|
|
|
def main():
|
|
host = 'https://datastore.swift-project.org'
|
|
version = '0.7.0'
|
|
target_path = os.path.abspath(os.curdir)
|
|
|
|
DbDataSync(host, version, target_path).sync()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|