refs #507, changed setup reader and simplied global setup and setup reader

* a single base URL (shared), derived URLs by appended path
* simplified dir structure shared with sub directories
* renamed functions
* automatically synchronize setup with DB when initialized
* trigger download info loading when setup is completed
* allow to automatically read after setup data have been synchronized
* read DB data when setup has been loaded
* allow to reload from threaded reader
* improved handling (log messages, skip reading) when data are not available
This commit is contained in:
Klaus Basan
2015-11-13 03:03:03 +01:00
committed by Mathew Sutcliffe
parent 4d4d6fcdc9
commit d131cd2d33
25 changed files with 527 additions and 267 deletions

View File

@@ -7,8 +7,8 @@ CONFIG -= qt
OTHER_FILES += data/images/flags/*.png OTHER_FILES += data/images/flags/*.png
OTHER_FILES += data/images/airlines/*.png OTHER_FILES += data/images/airlines/*.png
OTHER_FILES += swiftDB/*.* OTHER_FILES += swiftDB/*.*
OTHER_FILES += bootstrap/0.6/productive/*.* OTHER_FILES += bootstrap/productive/*.*
OTHER_FILES += bootstrap/0.6/development/*.* OTHER_FILES += bootstrap/development/*.*
OTHER_FILES += local.env.template/*.* OTHER_FILES += local.env.template/*.*
OTHER_FILES += local.env.template/bootstrap/*.* OTHER_FILES += local.env.template/bootstrap/*.*
OTHER_FILES += local.env.template/bootstrap/0.6/*.* OTHER_FILES += local.env.template/bootstrap/0.6/*.*
@@ -16,8 +16,8 @@ OTHER_FILES += local.env.template/bootstrap/0.6/*.*
COPY_FILES += $$PWD/data/images/flags/*.png COPY_FILES += $$PWD/data/images/flags/*.png
COPY_FILES += $$PWD/data/images/airlines/*.png COPY_FILES += $$PWD/data/images/airlines/*.png
COPY_FILES += $$PWD/swiftDB/*.* COPY_FILES += $$PWD/swiftDB/*.*
COPY_FILES += $$PWD/bootstrap/0.6/productive/*.* COPY_FILES += $$PWD/bootstrap/productive/*.*
COPY_FILES += $$PWD/bootstrap/0.6/development/*.* COPY_FILES += $$PWD/bootstrap/development/*.*
COPY_FILES += $$PWD/local.env.template/*.* COPY_FILES += $$PWD/local.env.template/*.*
COPY_FILES += $$PWD/local.env.template/bootstrap/*.* COPY_FILES += $$PWD/local.env.template/bootstrap/*.*
COPY_FILES += $$PWD/local.env.template/bootstrap/0.6/*.* COPY_FILES += $$PWD/local.env.template/bootstrap/0.6/*.*

View File

@@ -13,6 +13,7 @@
#include "network.h" #include "network.h"
#include "simulator.h" #include "simulator.h"
#include "context_application.h" #include "context_application.h"
#include "setupreader.h"
#include <QThread> #include <QThread>
namespace BlackCore namespace BlackCore
@@ -29,5 +30,6 @@ namespace BlackCore
BlackCore::Data::CGlobalSetup::registerMetadata(); BlackCore::Data::CGlobalSetup::registerMetadata();
BlackCore::Data::CDownload::registerMetadata(); BlackCore::Data::CDownload::registerMetadata();
BlackCore::CSetupReader::instance(); // kick off reader
} }
} // namespace } // namespace

View File

@@ -55,15 +55,14 @@ namespace BlackCore
connect(this->m_dataUpdateTimer, &QTimer::timeout, this, &CContextNetwork::requestDataUpdates); connect(this->m_dataUpdateTimer, &QTimer::timeout, this, &CContextNetwork::requestDataUpdates);
this->m_dataUpdateTimer->start(30 * 1000); this->m_dataUpdateTimer->start(30 * 1000);
// 3. data reader // 3. data reader, start reading when setup is synced with xx delay
this->m_webDataReader = new CWebDataServices(CWebReaderFlags::AllReaders, this); this->m_webDataReader = new CWebDataServices(CWebReaderFlags::AllReaders, 1000, this);
this->m_webReaderSignalConnections.append( this->m_webReaderSignalConnections.append(
this->m_webDataReader->connectDataReadSignal( this->m_webDataReader->connectDataReadSignal(
this, // the object here must be the same as in the bind this, // the object here must be the same as in the bind
std::bind(&CContextNetwork::webServiceDataRead, this, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3) std::bind(&CContextNetwork::webServiceDataRead, this, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3)
) )
); );
this->m_webDataReader->readAllInBackground(1000);
// 4. Airspace contents // 4. Airspace contents
Q_ASSERT_X(this->getRuntime()->getCContextOwnAircraft(), Q_FUNC_INFO, "this and own aircraft context must be local"); Q_ASSERT_X(this->getRuntime()->getCContextOwnAircraft(), Q_FUNC_INFO, "this and own aircraft context must be local");
@@ -570,7 +569,7 @@ namespace BlackCore
{ {
if (this->isDebugEnabled()) { CLogMessage(this, CLogCategory::contextSlot()).debug() << Q_FUNC_INFO; } if (this->isDebugEnabled()) { CLogMessage(this, CLogCategory::contextSlot()).debug() << Q_FUNC_INFO; }
Q_ASSERT_X(this->m_webDataReader, Q_FUNC_INFO, "missing reader"); Q_ASSERT_X(this->m_webDataReader, Q_FUNC_INFO, "missing reader");
this->m_webDataReader->readAtcBookingsInBackground(); this->m_webDataReader->readInBackground(BlackMisc::Network::CEntityFlags::BookingEntity);
} }
bool CContextNetwork::updateAircraftRendered(const CCallsign &callsign, bool rendered, const CIdentifier &originator) bool CContextNetwork::updateAircraftRendered(const CCallsign &callsign, bool rendered, const CIdentifier &originator)

View File

@@ -248,7 +248,6 @@ namespace BlackCore
BlackMisc::CDigestSignal m_dsAtcStationsOnlineChanged { this, &IContextNetwork::changedAtcStationsOnline, &IContextNetwork::changedAtcStationsOnlineDigest, 750, 4 }; BlackMisc::CDigestSignal m_dsAtcStationsOnlineChanged { this, &IContextNetwork::changedAtcStationsOnline, &IContextNetwork::changedAtcStationsOnlineDigest, 750, 4 };
BlackMisc::CDigestSignal m_dsAircraftsInRangeChanged { this, &IContextNetwork::changedAircraftInRange, &IContextNetwork::changedAircraftInRangeDigest, 750, 4 }; BlackMisc::CDigestSignal m_dsAircraftsInRangeChanged { this, &IContextNetwork::changedAircraftInRange, &IContextNetwork::changedAircraftInRangeDigest, 750, 4 };
//! Own aircraft from \sa CContextOwnAircraft //! Own aircraft from \sa CContextOwnAircraft
const BlackMisc::Simulation::CSimulatedAircraft ownAircraft() const; const BlackMisc::Simulation::CSimulatedAircraft ownAircraft() const;

View File

@@ -8,6 +8,7 @@
*/ */
#include "globalsetup.h" #include "globalsetup.h"
#include "blackmisc/project.h"
#include "blackmisc/math/mathutils.h" #include "blackmisc/math/mathutils.h"
#include "blackmisc/blackmiscfreefunctions.h" #include "blackmisc/blackmiscfreefunctions.h"
#include <QStringList> #include <QStringList>
@@ -22,35 +23,35 @@ namespace BlackCore
{ {
CGlobalSetup::CGlobalSetup() : CGlobalSetup::CGlobalSetup() :
ITimestampBased(0), ITimestampBased(0),
m_dbRootDirectory("http://ubuntu12/swiftdatastore/public"),
m_dbHttpPort(80), m_dbHttpPort(80),
m_dbHttpsPort(443), m_dbHttpsPort(443),
m_vatsimBookings("http://vatbook.euroutepro.com/xml2.php"), m_dbRootDirectoryUrl("http://ubuntu12/swiftdatastore/public"),
m_vatsimMetars("http://metar.vatsim.net/metar.php"), m_vatsimBookingsUrl("http://vatbook.euroutepro.com/xml2.php"),
m_vatsimDataFile(QStringList({ "http://info.vroute.net/vatsim-data.txt" })), m_vatsimMetarsUrl("http://metar.vatsim.net/metar.php"),
m_bootstrap(QStringList({ "https://vatsim-germany.org:50443/mapping/public/bootstrap", "http://ubuntu12/public/bootstrap"})), m_vatsimDataFileUrls(QStringList({ "http://info.vroute.net/vatsim-data.txt" })),
m_swiftDbDataFiles(QStringList({})), m_sharedUrls(CProject::swiftTeamDefaultServers()),
m_newsUrls(QStringList({ "http://swift-project.org/" })),
m_fsdTestServers({ CServer("swift", "swift Testserver", "vatsim-germany.org", 6809, CUser("1234567", "swift Test User", "", "123456"), true) }) m_fsdTestServers({ CServer("swift", "swift Testserver", "vatsim-germany.org", 6809, CUser("1234567", "swift Test User", "", "123456"), true) })
{ } { }
CUrl CGlobalSetup::dbIcaoReader() const CUrl CGlobalSetup::dbIcaoReaderUrl() const
{ {
return dbRootDirectory(); return dbRootDirectoryUrl();
} }
CUrl CGlobalSetup::dbModelReader() const CUrl CGlobalSetup::dbModelReaderUrl() const
{ {
return dbRootDirectory(); return dbRootDirectoryUrl();
} }
CUrl CGlobalSetup::dbHomePage() const CUrl CGlobalSetup::dbHomePageUrl() const
{ {
return dbRootDirectory().withAppendedPath("/page/index.php"); return dbRootDirectoryUrl().withAppendedPath("/page/index.php");
} }
CUrl CGlobalSetup::dbLoginService() const CUrl CGlobalSetup::dbLoginServiceUrl() const
{ {
return dbRootDirectory(). return dbRootDirectoryUrl().
withAppendedPath("/service/jsonauthenticate.php"). withAppendedPath("/service/jsonauthenticate.php").
withSwitchedScheme("https", m_dbHttpsPort); withSwitchedScheme("https", m_dbHttpsPort);
} }
@@ -68,14 +69,38 @@ namespace BlackCore
m_dbDebugFlag = debug; m_dbDebugFlag = debug;
} }
bool CGlobalSetup::hasSameType(CGlobalSetup &otherSetup) const bool CGlobalSetup::hasSameType(const CGlobalSetup &otherSetup) const
{ {
return this->isDevelopment() == otherSetup.isDevelopment(); return this->isDevelopment() == otherSetup.isDevelopment();
} }
CUrl CGlobalSetup::vatsimMetars() const CUrl CGlobalSetup::vatsimMetarsUrl() const
{ {
return this->m_vatsimMetars.withAppendedQuery("id=all"); return this->m_vatsimMetarsUrl.withAppendedQuery("id=all");
}
CUrlList CGlobalSetup::bootstrapUrls() const
{
CUrlList urls(m_sharedUrls);
return urls.appendPath(isDevelopment() ?
CGlobalSetup::versionString() + "/development/bootstrap/bootstrap.json" :
CGlobalSetup::versionString() + "/productive/bootstrap/bootstrap.json");
}
CUrlList CGlobalSetup::downloadInfoUrls() const
{
CUrlList urls(m_sharedUrls);
return urls.appendPath(isDevelopment() ?
CGlobalSetup::versionString() + "/development/download/download.json" :
CGlobalSetup::versionString() + "/productive/download/download.json");
}
CUrlList CGlobalSetup::swiftDbDataFileLocationUrls() const
{
CUrlList urls(m_sharedUrls);
return urls.appendPath(isDevelopment() ?
CGlobalSetup::versionString() + "/development/dbdata/" :
CGlobalSetup::versionString() + "/productive/dbdata/");
} }
QString CGlobalSetup::convertToQString(bool i18n) const QString CGlobalSetup::convertToQString(bool i18n) const
@@ -91,39 +116,48 @@ namespace BlackCore
s.append("For development: "); s.append("For development: ");
s.append(boolToYesNo(isDevelopment())); s.append(boolToYesNo(isDevelopment()));
s.append(separator); s.append(separator);
s.append("DB root directory: ");
s.append(dbRootDirectory().convertToQString(i18n));
s.append(separator);
s.append("ICAO DB reader: ");
s.append(dbIcaoReader().convertToQString(i18n));
s.append(separator);
s.append("Model DB reader: ");
s.append(dbModelReader().convertToQString(i18n));
s.append(separator);
s.append("DB home page: "); s.append("Download URLs: ");
s.append(dbHomePage().convertToQString(i18n)); s.append(downloadInfoUrls().toQString(i18n));
s.append(separator);
s.append("DB login service: ");
s.append(dbLoginService().convertToQString(i18n));
s.append(separator);
s.append("VATSIM bookings: ");
s.append(vatsimBookings().convertToQString(i18n));
s.append(separator);
s.append("VATSIM METARs: ");
s.append(vatsimMetars().convertToQString(i18n));
s.append(separator);
s.append("VATSIM data file: ");
s.append(vatsimDataFile().convertToQString(i18n));
s.append(separator); s.append(separator);
s.append("Bootstrap URLs: "); s.append("Bootstrap URLs: ");
s.append(bootstrapUrls().convertToQString(i18n)); s.append(bootstrapUrls().toQString(i18n));
s.append(separator);
s.append("News URLs: ");
s.append(swiftLatestNewsUrls().toQString(i18n));
s.append(separator);
s.append("DB root directory: ");
s.append(dbRootDirectoryUrl().toQString(i18n));
s.append(separator);
s.append("ICAO DB reader: ");
s.append(dbIcaoReaderUrl().toQString(i18n));
s.append(separator);
s.append("Model DB reader: ");
s.append(dbModelReaderUrl().toQString(i18n));
s.append(separator);
s.append("DB home page: ");
s.append(dbHomePageUrl().toQString(i18n));
s.append(separator);
s.append("DB login service: ");
s.append(dbLoginServiceUrl().toQString(i18n));
s.append(separator); s.append(separator);
s.append("swift DB datafile locations: "); s.append("swift DB datafile locations: ");
s.append(swiftDbDataFileLocations().convertToQString(i18n)); s.append(swiftDbDataFileLocationUrls().toQString(i18n));
s.append(separator); s.append(separator);
s.append("VATSIM bookings: ");
s.append(vatsimBookingsUrl().toQString(i18n));
s.append(separator);
s.append("VATSIM METARs: ");
s.append(vatsimMetarsUrl().toQString(i18n));
s.append(separator);
s.append("VATSIM data file: ");
s.append(vatsimDataFileUrls().toQString(i18n));
s.append(separator);
s.append("FSD test servers: "); s.append("FSD test servers: ");
s.append(fsdTestServers().convertToQString(i18n)); s.append(fsdTestServers().toQString(i18n));
return s; return s;
} }
@@ -136,23 +170,27 @@ namespace BlackCore
switch (i) switch (i)
{ {
case IndexDbRootDirectory: case IndexDbRootDirectory:
return CVariant::fromValue(this->m_dbRootDirectory); return CVariant::fromValue(this->m_dbRootDirectoryUrl);
case IndexDbHttpPort: case IndexDbHttpPort:
return CVariant::fromValue(this->m_dbHttpPort); return CVariant::fromValue(this->m_dbHttpPort);
case IndexDbHttpsPort: case IndexDbHttpsPort:
return CVariant::fromValue(this->m_dbHttpsPort); return CVariant::fromValue(this->m_dbHttpsPort);
case IndexDbLoginService: case IndexDbLoginService:
return CVariant::fromValue(this->dbLoginService()); return CVariant::fromValue(this->dbLoginServiceUrl());
case IndexVatsimData: case IndexVatsimData:
return CVariant::fromValue(this->m_vatsimDataFile); return CVariant::fromValue(this->m_vatsimDataFileUrls);
case IndexVatsimBookings: case IndexVatsimBookings:
return CVariant::fromValue(this->m_vatsimDataFile); return CVariant::fromValue(this->m_vatsimDataFileUrls);
case IndexVatsimMetars: case IndexVatsimMetars:
return CVariant::fromValue(this->m_vatsimMetars); return CVariant::fromValue(this->m_vatsimMetarsUrl);
case IndexDownload:
return CVariant::fromValue(this->downloadInfoUrls());
case IndexBootstrap: case IndexBootstrap:
return CVariant::fromValue(this->m_bootstrap); return CVariant::fromValue(this->bootstrapUrls());
case IndexSwiftDbFiles: case IndexSwiftDbFiles:
return CVariant::fromValue(this->m_swiftDbDataFiles); return CVariant::fromValue(this->swiftDbDataFileLocationUrls());
case IndexShared:
return CVariant::fromValue(this->m_sharedUrls);
default: default:
return CValueObject::propertyByIndex(index); return CValueObject::propertyByIndex(index);
} }
@@ -171,7 +209,7 @@ namespace BlackCore
switch (i) switch (i)
{ {
case IndexDbRootDirectory: case IndexDbRootDirectory:
this->m_dbRootDirectory.setPropertyByIndex(variant, index.copyFrontRemoved()); this->m_dbRootDirectoryUrl.setPropertyByIndex(variant, index.copyFrontRemoved());
break; break;
case IndexDbHttpPort: case IndexDbHttpPort:
this->m_dbHttpPort = variant.toInt(); this->m_dbHttpPort = variant.toInt();
@@ -182,19 +220,16 @@ namespace BlackCore
case IndexDbLoginService: case IndexDbLoginService:
break; break;
case IndexVatsimData: case IndexVatsimData:
this->m_vatsimDataFile = variant.value<CUrlList>(); this->m_vatsimDataFileUrls = variant.value<CUrlList>();
break; break;
case IndexVatsimBookings: case IndexVatsimBookings:
this->m_vatsimBookings.setPropertyByIndex(variant, index.copyFrontRemoved()); this->m_vatsimBookingsUrl.setPropertyByIndex(variant, index.copyFrontRemoved());
break; break;
case IndexVatsimMetars: case IndexVatsimMetars:
this->m_vatsimMetars.setPropertyByIndex(variant, index.copyFrontRemoved()); this->m_vatsimMetarsUrl.setPropertyByIndex(variant, index.copyFrontRemoved());
break; break;
case IndexBootstrap: case IndexShared:
this->m_bootstrap = variant.value<CUrlList>(); this->m_sharedUrls = variant.value<CUrlList>();
break;
case IndexSwiftDbFiles:
this->m_swiftDbDataFiles = variant.value<CUrlList>();
break; break;
default: default:
CValueObject::setPropertyByIndex(variant, index); CValueObject::setPropertyByIndex(variant, index);
@@ -204,7 +239,7 @@ namespace BlackCore
const QString &CGlobalSetup::versionString() const QString &CGlobalSetup::versionString()
{ {
static const QString v("0.6"); static const QString v("0.6.1");
return v; return v;
} }
} // ns } // ns

View File

@@ -41,7 +41,9 @@ namespace BlackCore
IndexVatsimMetars, IndexVatsimMetars,
IndexVatsimData, IndexVatsimData,
IndexSwiftDbFiles, IndexSwiftDbFiles,
IndexBootstrap IndexBootstrap,
IndexDownload,
IndexShared
}; };
//! Default constructor //! Default constructor
@@ -50,50 +52,56 @@ namespace BlackCore
//! Destructor. //! Destructor.
~CGlobalSetup() {} ~CGlobalSetup() {}
//! Root directory of DB
const BlackMisc::Network::CUrl &dbRootDirectory() const { return m_dbRootDirectory; }
//! ICAO Reader location
BlackMisc::Network::CUrl dbIcaoReader() const;
//! Model Reader protocol
BlackMisc::Network::CUrl dbModelReader() const;
//! Http port //! Http port
int dbHttpPort() const { return m_dbHttpPort; } int dbHttpPort() const { return m_dbHttpPort; }
//! Https port //! Https port
int dbHttpsPort() const { return m_dbHttpsPort; } int dbHttpsPort() const { return m_dbHttpsPort; }
//! Home page url
BlackMisc::Network::CUrl dbHomePage() const;
//! Login service
BlackMisc::Network::CUrl dbLoginService() const;
//! Debug flag //! Debug flag
bool dbDebugFlag() const; bool dbDebugFlag() const;
//! Set debug flag //! Set debug flag
void setServerDebugFlag(bool debug); void setServerDebugFlag(bool debug);
//! URL to read VATSIM bookings
const BlackMisc::Network::CUrl &vatsimBookings() const { return m_vatsimBookings; }
//! Same type? //! Same type?
bool hasSameType(CGlobalSetup &otherSetup) const; bool hasSameType(const CGlobalSetup &otherSetup) const;
//! Home page url
BlackMisc::Network::CUrl dbHomePageUrl() const;
//! Login service
BlackMisc::Network::CUrl dbLoginServiceUrl() const;
//! Root directory of DB
const BlackMisc::Network::CUrl &dbRootDirectoryUrl() const { return m_dbRootDirectoryUrl; }
//! ICAO Reader location
BlackMisc::Network::CUrl dbIcaoReaderUrl() const;
//! Model Reader protocol
BlackMisc::Network::CUrl dbModelReaderUrl() const;
//! URL to read VATSIM bookings
const BlackMisc::Network::CUrl &vatsimBookingsUrl() const { return m_vatsimBookingsUrl; }
//! VATSIM METAR URL //! VATSIM METAR URL
BlackMisc::Network::CUrl vatsimMetars() const; BlackMisc::Network::CUrl vatsimMetarsUrl() const;
//! VATSIM data file URLs //! VATSIM data file URLs
const BlackMisc::Network::CUrlList &vatsimDataFile() const { return m_vatsimDataFile; } const BlackMisc::Network::CUrlList &vatsimDataFileUrls() const { return m_vatsimDataFileUrls; }
//! Bootstrap URLs (where the data for the setup itself can be downloaded) //! Bootstrap URLs (where the data for the setup itself can be downloaded)
const BlackMisc::Network::CUrlList &bootstrapUrls() const { return m_bootstrap; } BlackMisc::Network::CUrlList bootstrapUrls() const;
//! Version files and download locations
BlackMisc::Network::CUrlList downloadInfoUrls() const;
//! Alternative locations of swift DB data files //! Alternative locations of swift DB data files
const BlackMisc::Network::CUrlList &swiftDbDataFileLocations() const { return m_swiftDbDataFiles; } BlackMisc::Network::CUrlList swiftDbDataFileLocationUrls() const;
//! Locations of swift DB news
const BlackMisc::Network::CUrlList &swiftLatestNewsUrls() const { return m_newsUrls; }
//! FSD test servers //! FSD test servers
const BlackMisc::Network::CServerList &fsdTestServers() const { return m_fsdTestServers; } const BlackMisc::Network::CServerList &fsdTestServers() const { return m_fsdTestServers; }
@@ -122,19 +130,19 @@ namespace BlackCore
private: private:
BLACK_ENABLE_TUPLE_CONVERSION(BlackCore::Data::CGlobalSetup) BLACK_ENABLE_TUPLE_CONVERSION(BlackCore::Data::CGlobalSetup)
BlackMisc::Network::CUrl m_dbRootDirectory; //!< Root directory int m_dbHttpPort = 80; //!< port
int m_dbHttpPort = 80; //!< port int m_dbHttpsPort = 443; //!< SSL port
int m_dbHttpsPort = 443; //!< SSL port bool m_development = false; //!< dev. version?
BlackMisc::Network::CUrl m_vatsimBookings; //!< ATC bookings BlackMisc::Network::CUrl m_dbRootDirectoryUrl; //!< Root directory of DB
BlackMisc::Network::CUrl m_vatsimMetars; //!< METAR data BlackMisc::Network::CUrl m_vatsimBookingsUrl; //!< ATC bookings
BlackMisc::Network::CUrlList m_vatsimDataFile; //!< Overall VATSIM data file BlackMisc::Network::CUrl m_vatsimMetarsUrl; //!< METAR data
BlackMisc::Network::CUrlList m_bootstrap; //!< where we can obtain downloads of these data BlackMisc::Network::CUrlList m_vatsimDataFileUrls; //!< Overall VATSIM data file
BlackMisc::Network::CUrlList m_swiftDbDataFiles; //!< alternative locations of the DB files, if DB is not available BlackMisc::Network::CUrlList m_sharedUrls; //!< where we can obtain shared info files such as bootstrap, ..
BlackMisc::Network::CServerList m_fsdTestServers; //!< FSD test servers BlackMisc::Network::CUrlList m_newsUrls; //!< where we can obtain latest news
bool m_development = false; //!< dev. version? BlackMisc::Network::CServerList m_fsdTestServers; //!< FSD test servers
// transient members, to be switched on/off via GUI or set from reader // transient members, to be switched on/off via GUI or set from reader
bool m_dbDebugFlag = false; //!< can trigger DEBUG on the server, so you need to know hat you are doing bool m_dbDebugFlag = false; //!< can trigger DEBUG on the server, so you need to know what you are doing
}; };
//! Trait for global setup data //! Trait for global setup data
@@ -157,14 +165,14 @@ namespace BlackCore
Q_DECLARE_METATYPE(BlackCore::Data::CGlobalSetup) Q_DECLARE_METATYPE(BlackCore::Data::CGlobalSetup)
BLACK_DECLARE_TUPLE_CONVERSION(BlackCore::Data::CGlobalSetup, ( BLACK_DECLARE_TUPLE_CONVERSION(BlackCore::Data::CGlobalSetup, (
attr(o.m_timestampMSecsSinceEpoch), attr(o.m_timestampMSecsSinceEpoch),
attr(o.m_dbRootDirectory), attr(o.m_dbRootDirectoryUrl),
attr(o.m_dbHttpPort), attr(o.m_dbHttpPort),
attr(o.m_dbHttpsPort), attr(o.m_dbHttpsPort),
attr(o.m_vatsimBookings), attr(o.m_vatsimBookingsUrl),
attr(o.m_vatsimMetars), attr(o.m_vatsimMetarsUrl),
attr(o.m_vatsimDataFile), attr(o.m_vatsimDataFileUrls),
attr(o.m_bootstrap), attr(o.m_sharedUrls),
attr(o.m_swiftDbDataFiles), attr(o.m_newsUrls),
attr(o.m_fsdTestServers), attr(o.m_fsdTestServers),
attr(o.m_development), attr(o.m_development),
attr(o.m_dbDebugFlag, flags < DisabledForJson > ()) attr(o.m_dbDebugFlag, flags < DisabledForJson > ())

View File

@@ -53,7 +53,7 @@ namespace BlackCore
if (pw.isEmpty()) { msgs.push_back(CStatusMessage(cats, CStatusMessage::SeverityError, "No password")); } if (pw.isEmpty()) { msgs.push_back(CStatusMessage(cats, CStatusMessage::SeverityError, "No password")); }
if (!msgs.isEmpty()) { return msgs; } if (!msgs.isEmpty()) { return msgs; }
CUrl url(this->m_setup.get().dbLoginService()); CUrl url(this->m_setup.get().dbLoginServiceUrl());
QString msg; QString msg;
if (!CNetworkUtils::canConnect(url, msg)) if (!CNetworkUtils::canConnect(url, msg))
{ {
@@ -84,7 +84,7 @@ namespace BlackCore
void CDatabaseAuthenticationService::logoff() void CDatabaseAuthenticationService::logoff()
{ {
CUrl url(this->m_setup.get().dbLoginService()); CUrl url(this->m_setup.get().dbLoginServiceUrl());
url.setQuery("logoff=true"); url.setQuery("logoff=true");
QNetworkRequest request(CNetworkUtils::getNetworkRequest(url)); QNetworkRequest request(CNetworkUtils::getNetworkRequest(url));
this->m_networkManager->get(request); this->m_networkManager->get(request);

View File

@@ -26,7 +26,7 @@ namespace BlackCore
void CDatabaseReader::readInBackgroundThread(CEntityFlags::Entity entities) void CDatabaseReader::readInBackgroundThread(CEntityFlags::Entity entities)
{ {
if (m_shutdown) { return; } if (isFinishedOrShutdown()) { return; }
this->m_watchdogTimer.stop(); this->m_watchdogTimer.stop();
bool s = QMetaObject::invokeMethod(this, "ps_read", Q_ARG(BlackMisc::Network::CEntityFlags::Entity, entities)); bool s = QMetaObject::invokeMethod(this, "ps_read", Q_ARG(BlackMisc::Network::CEntityFlags::Entity, entities));
Q_ASSERT_X(s, Q_FUNC_INFO, "Invoke failed"); Q_ASSERT_X(s, Q_FUNC_INFO, "Invoke failed");

View File

@@ -23,8 +23,7 @@
namespace BlackCore namespace BlackCore
{ {
//! Support for threaded based reading and parsing tasks such //! Specialized version of threaded reader for DB data
//! as data files via http, or file system and parsing (such as FSX models)
class BLACKCORE_EXPORT CDatabaseReader : public BlackMisc::CThreadedReader class BLACKCORE_EXPORT CDatabaseReader : public BlackMisc::CThreadedReader
{ {
Q_OBJECT Q_OBJECT
@@ -69,7 +68,7 @@ namespace BlackCore
protected: protected:
BlackMisc::Network::CUrl m_watchdogUrl; //!< URL for checking if alive BlackMisc::Network::CUrl m_watchdogUrl; //!< URL for checking if alive
QTimer m_watchdogTimer { this }; //!< Timer for watchdog QTimer m_watchdogTimer { this }; //!< Timer for watchdog (DB available?)
QString m_watchdogMessage; //!< Returned status message QString m_watchdogMessage; //!< Returned status message
bool m_canConnect = false; //!< Successful connection? bool m_canConnect = false; //!< Successful connection?
mutable QReadWriteLock m_watchdogLock; //!< Lock mutable QReadWriteLock m_watchdogLock; //!< Lock
@@ -85,7 +84,7 @@ namespace BlackCore
CDatabaseReader::JsonDatastoreResponse setStatusAndTransformReplyIntoDatastoreResponse(QNetworkReply *nwReply); CDatabaseReader::JsonDatastoreResponse setStatusAndTransformReplyIntoDatastoreResponse(QNetworkReply *nwReply);
private slots: private slots:
//! Watchdog //! Watchdog checking if DB is available
void ps_watchdog(); void ps_watchdog();
private: private:

View File

@@ -140,33 +140,57 @@ namespace BlackCore
if (entities.testFlag(CEntityFlags::AircraftIcaoEntity)) if (entities.testFlag(CEntityFlags::AircraftIcaoEntity))
{ {
QUrl url(getAircraftIcaoUrl()); QUrl url(getAircraftIcaoUrl());
QNetworkRequest requestAircraft(CNetworkUtils::getNetworkRequest(url)); if (!url.isEmpty())
this->m_networkManagerAircraft->get(requestAircraft); {
entitiesTriggered |= CEntityFlags::AircraftIcaoEntity; QNetworkRequest requestAircraft(CNetworkUtils::getNetworkRequest(url));
this->m_networkManagerAircraft->get(requestAircraft);
entitiesTriggered |= CEntityFlags::AircraftIcaoEntity;
}
else
{
CLogMessage(this).error("No URL for %1") << CEntityFlags::flagToString(CEntityFlags::AircraftIcaoEntity);
}
} }
if (entities.testFlag(CEntityFlags::AirlineIcaoEntity)) if (entities.testFlag(CEntityFlags::AirlineIcaoEntity))
{ {
QUrl url(getAirlineIcaoUrl()); QUrl url(getAirlineIcaoUrl());
QNetworkRequest requestAirline(CNetworkUtils::getNetworkRequest(url)); if (!url.isEmpty())
this->m_networkManagerAirlines->get(requestAirline); {
entitiesTriggered |= CEntityFlags::AirlineIcaoEntity; QNetworkRequest requestAirline(CNetworkUtils::getNetworkRequest(url));
this->m_networkManagerAirlines->get(requestAirline);
entitiesTriggered |= CEntityFlags::AirlineIcaoEntity;
}
else
{
CLogMessage(this).error("No URL for %1") << CEntityFlags::flagToString(CEntityFlags::AirlineIcaoEntity);
}
} }
if (entities.testFlag(CEntityFlags::CountryEntity)) if (entities.testFlag(CEntityFlags::CountryEntity))
{ {
QUrl url(getCountryUrl()); QUrl url(getCountryUrl());
QNetworkRequest requestCountry(CNetworkUtils::getNetworkRequest(url)); if (!url.isEmpty())
this->m_networkManagerCountries->get(requestCountry); {
entitiesTriggered |= CEntityFlags::CountryEntity; QNetworkRequest requestCountry(CNetworkUtils::getNetworkRequest(url));
this->m_networkManagerCountries->get(requestCountry);
entitiesTriggered |= CEntityFlags::CountryEntity;
}
else
{
CLogMessage(this).error("No URL for %1") << CEntityFlags::flagToString(CEntityFlags::CountryEntity);
}
} }
emit dataRead(entitiesTriggered, CEntityFlags::StartRead, 0); if (entitiesTriggered != CEntityFlags::NoEntity)
{
emit dataRead(entitiesTriggered, CEntityFlags::StartRead, 0);
}
} }
CUrl CIcaoDataReader::getBaseUrl() const CUrl CIcaoDataReader::getBaseUrl() const
{ {
CUrl baseUrl(this->m_setup.get().dbIcaoReader()); CUrl baseUrl(this->m_setup.get().dbIcaoReaderUrl());
return baseUrl; return baseUrl;
} }

View File

@@ -137,29 +137,56 @@ namespace BlackCore
CEntityFlags::Entity triggeredRead = CEntityFlags::NoEntity; CEntityFlags::Entity triggeredRead = CEntityFlags::NoEntity;
if (entity.testFlag(CEntityFlags::LiveryEntity)) if (entity.testFlag(CEntityFlags::LiveryEntity))
{ {
QNetworkRequest requestLivery(getLiveryUrl()); CUrl url(getLiveryUrl());
CNetworkUtils::ignoreSslVerification(requestLivery); if (!url.isEmpty())
this->m_networkManagerLivery->get(requestLivery); {
triggeredRead |= CEntityFlags::LiveryEntity; QNetworkRequest requestLivery(url);
CNetworkUtils::ignoreSslVerification(requestLivery);
this->m_networkManagerLivery->get(requestLivery);
triggeredRead |= CEntityFlags::LiveryEntity;
}
else
{
CLogMessage(this).error("No URL for %1") << CEntityFlags::flagToString(CEntityFlags::LiveryEntity);
}
} }
if (entity.testFlag(CEntityFlags::DistributorEntity)) if (entity.testFlag(CEntityFlags::DistributorEntity))
{ {
QNetworkRequest requestDistributor(getDistributorUrl()); CUrl url(getDistributorUrl());
CNetworkUtils::ignoreSslVerification(requestDistributor); if (!url.isEmpty())
this->m_networkManagerDistributor->get(requestDistributor); {
triggeredRead |= CEntityFlags::DistributorEntity; QNetworkRequest requestDistributor(url);
CNetworkUtils::ignoreSslVerification(requestDistributor);
this->m_networkManagerDistributor->get(requestDistributor);
triggeredRead |= CEntityFlags::DistributorEntity;
}
else
{
CLogMessage(this).error("No URL for %1") << CEntityFlags::flagToString(CEntityFlags::DistributorEntity);
}
} }
if (entity.testFlag(CEntityFlags::ModelEntity)) if (entity.testFlag(CEntityFlags::ModelEntity))
{ {
QNetworkRequest requestModel(getModelUrl()); CUrl url(getModelUrl());
CNetworkUtils::ignoreSslVerification(requestModel); if (!url.isEmpty())
this->m_networkManagerModel->get(requestModel); {
triggeredRead |= CEntityFlags::ModelEntity; QNetworkRequest requestModel(url);
CNetworkUtils::ignoreSslVerification(requestModel);
this->m_networkManagerModel->get(requestModel);
triggeredRead |= CEntityFlags::ModelEntity;
}
else
{
CLogMessage(this).error("No URL for %1") << CEntityFlags::flagToString(CEntityFlags::ModelEntity);
}
} }
emit dataRead(triggeredRead, CEntityFlags::StartRead, 0); if (triggeredRead != CEntityFlags::NoEntity)
{
emit dataRead(triggeredRead, CEntityFlags::StartRead, 0);
}
} }
void CModelDataReader::ps_parseLiveryData(QNetworkReply *nwReplyPtr) void CModelDataReader::ps_parseLiveryData(QNetworkReply *nwReplyPtr)
@@ -332,7 +359,7 @@ namespace BlackCore
CUrl CModelDataReader::getBaseUrl() const CUrl CModelDataReader::getBaseUrl() const
{ {
CUrl baseUrl(m_setup.get().dbModelReader()); CUrl baseUrl(m_setup.get().dbModelReaderUrl());
return baseUrl; return baseUrl;
} }

View File

@@ -29,63 +29,95 @@ namespace BlackCore
CSetupReader::CSetupReader(QObject *owner) : CSetupReader::CSetupReader(QObject *owner) :
CThreadedReader(owner, "CSetupReader") CThreadedReader(owner, "CSetupReader")
{ {
connect(this, &CSetupReader::setupSynchronized, this, &CSetupReader::ps_setupSyncronized);
QString localFileName; QString localFileName;
if (localFile(localFileName)) if (this->localBootstrapFile(localFileName))
{ {
// initialized by local file for testing // initialized by local file for testing
// I do not even need to start in background here // I do not even need to start in background here
CLogMessage(this).info("Using local bootstrap file: %1") << localFileName; CLogMessage(this).info("Using local bootstrap file: %1") << localFileName;
emit this->setupSynchronized(true);
} }
else else
{ {
this->m_networkManager = new QNetworkAccessManager(this); this->m_bootstrapUrls.uniqueWrite()->push_back(m_setup.get().bootstrapUrls());
this->connect(this->m_networkManager, &QNetworkAccessManager::finished, this, &CSetupReader::ps_parseSetupFile); this->m_downloadUrls.uniqueWrite()->push_back(m_setup.get().downloadInfoUrls());
this->m_networkManagerBootstrap = new QNetworkAccessManager(this);
this->connect(this->m_networkManagerBootstrap, &QNetworkAccessManager::finished, this, &CSetupReader::ps_parseSetupFile);
this->m_networkManagerDownload = new QNetworkAccessManager(this);
this->connect(this->m_networkManagerDownload, &QNetworkAccessManager::finished, this, &CSetupReader::ps_parseDownloadFile);
this->start(QThread::LowPriority); this->start(QThread::LowPriority);
} }
} }
CSetupReader &CSetupReader::instance()
{
static CSetupReader reader(QCoreApplication::instance());
return reader;
}
void CSetupReader::initialize() void CSetupReader::initialize()
{ {
// start to read by myself
CThreadedReader::initialize(); CThreadedReader::initialize();
QTimer::singleShot(500, this, &CSetupReader::ps_read); QTimer::singleShot(500, this, &CSetupReader::ps_readSetup);
} }
void CSetupReader::ps_read() void CSetupReader::ps_readSetup()
{ {
this->threadAssertCheck(); this->threadAssertCheck();
Q_ASSERT_X(this->m_networkManager, Q_FUNC_INFO, "Missing network manager"); Q_ASSERT_X(this->m_networkManagerBootstrap, Q_FUNC_INFO, "Missing network manager");
CUrlList urls(getRemainingUrls()); CUrl url(this->m_bootstrapUrls.uniqueWrite()->getNextWorkingUrl());
if (urls.isEmpty()) { return; } if (url.isEmpty())
int urlsSize = urls.size();
QUrl url(urls.getNextUrl(false));
if (urlsSize > 1)
{ {
// more than one URL one, quick check if this can be contacted CLogMessage(this).warning("Cannot read setup, failed URLs: %1") << this->m_bootstrapUrls.read()->getFailedUrls();
QString m; emit setupSynchronized(false);
if (!CNetworkUtils::canConnect(url, m)) return;
{
m_failedUrls.push_back(url);
CLogMessage(this).warning("Cannot connect to %1") << url.toString();
url = urls.getNextUrl();
}
} }
// now I have the first or second URL for reading
if (url.isEmpty()) { return; }
QNetworkRequest request(url); QNetworkRequest request(url);
CNetworkUtils::ignoreSslVerification(request); CNetworkUtils::ignoreSslVerification(request);
this->m_networkManagerBootstrap->get(request);
// request
this->m_networkManager->get(request);
} }
bool CSetupReader::localFile(QString &fileName) void CSetupReader::ps_readDownload()
{
this->threadAssertCheck();
Q_ASSERT_X(this->m_networkManagerDownload, Q_FUNC_INFO, "Missing network manager");
CUrl url(this->m_downloadUrls.uniqueWrite()->getNextWorkingUrl());
if (url.isEmpty())
{
CLogMessage(this).warning("Cannot read download (info), failed URLs: %1") << this->m_downloadUrls.read()->getFailedUrls();
return;
}
QNetworkRequest request(url);
CNetworkUtils::ignoreSslVerification(request);
this->m_networkManagerDownload->get(request);
}
void CSetupReader::ps_setupSyncronized(bool success)
{
// trigger
if (success)
{
CLogMessage(this).info("Setup synchronized, will trigger read of download information");
QTimer::singleShot(500, this, &CSetupReader::ps_readDownload);
}
else
{
CLogMessage(this).error("Setup reading failed, hence version info will not be loaded");
}
}
bool CSetupReader::localBootstrapFile(QString &fileName)
{ {
QString dir(CProject::getSwiftPrivateResourceDir()); QString dir(CProject::getSwiftPrivateResourceDir());
if (dir.isEmpty()) { return false; } if (dir.isEmpty()) { return false; }
fileName = CFileUtils::appendFilePaths(dir, "bootstrap/" + CGlobalSetup::versionString() + "/bootstrap.json"); // no version for local files, as those come withe the current code
fileName = CFileUtils::appendFilePaths(dir, "bootstrap/bootstrap.json");
QString content(CFileUtils::readFileToString(fileName)); QString content(CFileUtils::readFileToString(fileName));
if (content.isEmpty()) { return false; } if (content.isEmpty()) { return false; }
CGlobalSetup s; CGlobalSetup s;
@@ -100,53 +132,33 @@ namespace BlackCore
return CProject::useDevelopmentSetup(); return CProject::useDevelopmentSetup();
} }
CUrlList CSetupReader::getRemainingUrls() const
{
CUrlList urls(m_setup.get().bootstrapUrls().appendPath(appendPathAndFile()));
urls.removeIfIn(m_failedUrls);
return urls;
}
QString CSetupReader::appendPathAndFile()
{
return isForDevelopment() ?
CGlobalSetup::versionString() + "/development/bootstrap.json" :
CGlobalSetup::versionString() + "/productive/bootstrap.json";
}
void CSetupReader::ps_parseSetupFile(QNetworkReply *nwReplyPtr) void CSetupReader::ps_parseSetupFile(QNetworkReply *nwReplyPtr)
{ {
// wrap pointer, make sure any exit cleans up reply // wrap pointer, make sure any exit cleans up reply
// required to use delete later as object is created in a different thread // required to use delete later as object is created in a different thread
QScopedPointer<QNetworkReply, QScopedPointerDeleteLater> nwReply(nwReplyPtr); QScopedPointer<QNetworkReply, QScopedPointerDeleteLater> nwReply(nwReplyPtr);
this->threadAssertCheck();
QUrl url(nwReply->url()); QUrl url(nwReply->url());
QString urlString(url.toString()); QString urlString(url.toString());
QString replyMessage(nwReply->errorString()); QString replyMessage(nwReply->errorString());
this->threadAssertCheck();
if (this->isFinishedOrShutdown()) if (this->isFinishedOrShutdown())
{ {
CLogMessage(this).debug() << Q_FUNC_INFO; CLogMessage(this).debug() << Q_FUNC_INFO;
CLogMessage(this).info("Terminated loading bootstrap files"); CLogMessage(this).info("Terminated loading bootstrap files");
nwReply->abort(); nwReply->abort();
emit setupSynchronized(false);
return; // stop, terminate straight away, ending thread return; // stop, terminate straight away, ending thread
} }
if (nwReply->error() == QNetworkReply::NoError) if (nwReply->error() == QNetworkReply::NoError)
{ {
qint64 lastModified = -1; qint64 lastModified = this->lastModifiedMsSinceEpoch(nwReply.data());
QVariant lastModifiedQv = nwReply->header(QNetworkRequest::LastModifiedHeader);
if (lastModifiedQv.isValid() && lastModifiedQv.canConvert<QDateTime>())
{
lastModified = lastModifiedQv.value<QDateTime>().toMSecsSinceEpoch();
}
QString setupJson(nwReplyPtr->readAll()); QString setupJson(nwReplyPtr->readAll());
nwReplyPtr->close(); nwReplyPtr->close();
if (setupJson.isEmpty()) if (setupJson.isEmpty())
{ {
CLogMessage(this).info("No bootstrap setup file"); CLogMessage(this).info("No bootstrap setup file at %1") << urlString;
m_failedUrls.push_back(url);
// try next URL // try next URL
} }
else else
@@ -156,20 +168,21 @@ namespace BlackCore
loadedSetup.convertFromJson(Json::jsonObjectFromString(setupJson)); loadedSetup.convertFromJson(Json::jsonObjectFromString(setupJson));
loadedSetup.setDevelopment(isForDevelopment()); // always update, regardless what persistent setting says loadedSetup.setDevelopment(isForDevelopment()); // always update, regardless what persistent setting says
if (loadedSetup.getMSecsSinceEpoch() == 0 && lastModified > 0) { loadedSetup.setMSecsSinceEpoch(lastModified); } if (loadedSetup.getMSecsSinceEpoch() == 0 && lastModified > 0) { loadedSetup.setMSecsSinceEpoch(lastModified); }
bool sameType = loadedSetup.hasSameType(currentSetup);
qint64 currentVersionTimestamp = currentSetup.getMSecsSinceEpoch(); qint64 currentVersionTimestamp = currentSetup.getMSecsSinceEpoch();
qint64 newVersionTimestamp = loadedSetup.getMSecsSinceEpoch(); qint64 newVersionTimestamp = loadedSetup.getMSecsSinceEpoch();
bool sameVersionLoaded = sameType && (newVersionTimestamp == currentVersionTimestamp); bool sameVersionLoaded = (loadedSetup == currentSetup);
if (sameVersionLoaded) if (sameVersionLoaded)
{ {
CLogMessage(this).info("Same version loaded from %1 as already in data cache %2") << urlString << CDataCache::persistentStore(); CLogMessage(this).info("Same setup version loaded from %1 as already in data cache %2") << urlString << m_setup.getFilename();
emit setupSynchronized(true);
return; // success return; // success
} }
bool sameType = loadedSetup.hasSameType(currentSetup);
bool outdatedVersionLoaded = sameType && (newVersionTimestamp < currentVersionTimestamp); bool outdatedVersionLoaded = sameType && (newVersionTimestamp < currentVersionTimestamp);
if (outdatedVersionLoaded) if (outdatedVersionLoaded)
{ {
CLogMessage(this).info("Version loaded from %1 outdated, older than version in data cache %2") << urlString << CDataCache::persistentStore(); CLogMessage(this).info("Setup loaded from %1 outdated, older than version in data cache %2") << urlString << m_setup.getFilename();
// try next URL // try next URL
} }
else else
@@ -178,11 +191,13 @@ namespace BlackCore
if (!m.isEmpty()) if (!m.isEmpty())
{ {
CLogMessage(this).preformatted(m); CLogMessage(this).preformatted(m);
emit setupSynchronized(false);
return; // issue with cache return; // issue with cache
} }
else else
{ {
CLogMessage(this).info("Updated data cache in %1") << CDataCache::persistentStore(); CLogMessage(this).info("Setup: Updated data cache in %1") << this->m_setup.getFilename();
emit setupSynchronized(true);
return; // success return; // success
} // cache } // cache
} // outdated? } // outdated?
@@ -197,14 +212,95 @@ namespace BlackCore
} }
// try next one if any // try next one if any
const int maxTrials = 2; if (this->m_bootstrapUrls.uniqueWrite()->addFailedUrl(url))
m_failedUrls.push_back(url);
if (m_failedUrls.size() >= maxTrials) { return; }
int urlsNo = getRemainingUrls().size();
if (urlsNo >= 0)
{ {
QTimer::singleShot(500, this, &CSetupReader::ps_read); QTimer::singleShot(500, this, &CSetupReader::ps_readSetup);
}
else
{
emit setupSynchronized(false);
}
}
void CSetupReader::ps_parseDownloadFile(QNetworkReply *nwReplyPtr)
{
// wrap pointer, make sure any exit cleans up reply
// required to use delete later as object is created in a different thread
QScopedPointer<QNetworkReply, QScopedPointerDeleteLater> nwReply(nwReplyPtr);
this->threadAssertCheck();
QUrl url(nwReply->url());
QString urlString(url.toString());
QString replyMessage(nwReply->errorString());
if (this->isFinishedOrShutdown())
{
CLogMessage(this).debug() << Q_FUNC_INFO;
CLogMessage(this).info("Terminated loading download info");
nwReply->abort();
return; // stop, terminate straight away, ending thread
}
if (nwReply->error() == QNetworkReply::NoError)
{
qint64 lastModified = this->lastModifiedMsSinceEpoch(nwReply.data());
QString setupJson(nwReplyPtr->readAll());
nwReplyPtr->close();
if (setupJson.isEmpty())
{
CLogMessage(this).info("No download (info) file");
// try next URL
}
else
{
CDownload currentDownload(m_download.get()); // from cache
CDownload loadedDownload;
loadedDownload.convertFromJson(Json::jsonObjectFromString(setupJson));
loadedDownload.setDevelopment(isForDevelopment()); // always update, regardless what persistent setting says
if (loadedDownload.getMSecsSinceEpoch() == 0 && lastModified > 0) { loadedDownload.setMSecsSinceEpoch(lastModified); }
qint64 currentVersionTimestamp = currentDownload.getMSecsSinceEpoch();
qint64 newVersionTimestamp = loadedDownload.getMSecsSinceEpoch();
bool sameVersionLoaded = (loadedDownload == currentDownload);
if (sameVersionLoaded)
{
CLogMessage(this).info("Same download info loaded from %1 as already in data cache %2") << urlString << m_download.getFilename();
return; // success
}
bool sameType = loadedDownload.hasSameType(currentDownload);
bool outdatedVersionLoaded = sameType && (newVersionTimestamp < currentVersionTimestamp);
if (outdatedVersionLoaded)
{
CLogMessage(this).info("Download loaded from %1 outdated, older than version in data cache %2") << urlString << m_download.getFilename();
// try next URL
}
else
{
CStatusMessage m = m_download.set(loadedDownload);
if (!m.isEmpty())
{
CLogMessage(this).preformatted(m);
return; // issue with cache
}
else
{
CLogMessage(this).info("Download info: Updated data cache in %1") << m_download.getFilename();
return; // success
} // cache
} // outdated?
} // json empty
} // no error
else
{
// network error
CLogMessage(this).warning("Reading download info failed %1 %2") << replyMessage << urlString;
nwReply->abort();
}
// try next one if any
if (this->m_downloadUrls.uniqueWrite()->addFailedUrl(url))
{
QTimer::singleShot(500, this, &CSetupReader::ps_readSetup);
} }
} // method } // method

View File

@@ -14,7 +14,9 @@
#include "blackcore/blackcoreexport.h" #include "blackcore/blackcoreexport.h"
#include "blackmisc/threadedreader.h" #include "blackmisc/threadedreader.h"
#include "blackmisc/lockfree.h"
#include "blackcore/data/globalsetup.h" #include "blackcore/data/globalsetup.h"
#include "blackcore/data/download.h"
#include <QObject> #include <QObject>
#include <QTimer> #include <QTimer>
@@ -28,8 +30,12 @@ namespace BlackCore
Q_OBJECT Q_OBJECT
public: public:
//! Constructor //! Single instance
explicit CSetupReader(QObject *owner); static CSetupReader &instance();
signals:
//! Setup has been read
void setupSynchronized(bool success);
protected slots: protected slots:
//! \copydoc CThreadedReader::initialize //! \copydoc CThreadedReader::initialize
@@ -40,25 +46,35 @@ namespace BlackCore
//! \threadsafe //! \threadsafe
void ps_parseSetupFile(QNetworkReply *nwReply); void ps_parseSetupFile(QNetworkReply *nwReply);
//! Download has been read
//! \threadsafe
void ps_parseDownloadFile(QNetworkReply *nwReplyPtr);
//! Do reading //! Do reading
void ps_read(); void ps_readSetup();
//! Do reading
void ps_readDownload();
//! Setup has beem syncronized
void ps_setupSyncronized(bool success);
private: private:
QNetworkAccessManager *m_networkManager = nullptr; QNetworkAccessManager *m_networkManagerBootstrap = nullptr;
BlackMisc::Network::CUrlList m_failedUrls; QNetworkAccessManager *m_networkManagerDownload = nullptr;
CData<BlackCore::Data::GlobalSetup> m_setup {this}; //!< data cache BlackMisc::LockFree<BlackMisc::Network::CFailoverUrlList> m_bootstrapUrls;
BlackMisc::LockFree<BlackMisc::Network::CFailoverUrlList> m_downloadUrls;
CData<BlackCore::Data::GlobalSetup> m_setup {this}; //!< data cache setup
CData<BlackCore::Data::Download> m_download {this}; //!< data cache downloads
//! Constructor
explicit CSetupReader(QObject *owner);
//! Read by local individual file //! Read by local individual file
bool localFile(QString &fileName); bool localBootstrapFile(QString &fileName);
//! Remaining URLs failed one excluded
BlackMisc::Network::CUrlList getRemainingUrls() const;
//! Read for development environment? //! Read for development environment?
static bool isForDevelopment(); static bool isForDevelopment();
//! Bootstrap URL
static QString appendPathAndFile();
}; };
} // ns } // ns

View File

@@ -41,7 +41,7 @@ namespace BlackCore
{ {
this->threadAssertCheck(); this->threadAssertCheck();
Q_ASSERT_X(this->m_networkManager, Q_FUNC_INFO, "No network manager"); Q_ASSERT_X(this->m_networkManager, Q_FUNC_INFO, "No network manager");
QUrl url(m_setup.get().vatsimBookings()); QUrl url(m_setup.get().vatsimBookingsUrl());
if (url.isEmpty()) { return; } if (url.isEmpty()) { return; }
QNetworkRequest request(url); QNetworkRequest request(url);

View File

@@ -152,12 +152,11 @@ namespace BlackCore
void CVatsimDataFileReader::ps_read() void CVatsimDataFileReader::ps_read()
{ {
this->threadAssertCheck(); this->threadAssertCheck();
if (m_setup.get().vatsimDataFile().size() < 1) { return; }
// round robin for load balancing // round robin for load balancing
// remark: Don't use QThread to run network operations in the background // remark: Don't use QThread to run network operations in the background
// see http://qt-project.org/doc/qt-4.7/qnetworkaccessmanager.html // see http://qt-project.org/doc/qt-4.7/qnetworkaccessmanager.html
QUrl url(m_setup.get().vatsimDataFile().getNextUrl()); QUrl url(m_setup.get().vatsimDataFileUrls().getNextUrl());
if (url.isEmpty()) { return; } if (url.isEmpty()) { return; }
Q_ASSERT_X(this->m_networkManager, Q_FUNC_INFO, "Missing network manager"); Q_ASSERT_X(this->m_networkManager, Q_FUNC_INFO, "Missing network manager");
QNetworkRequest request(url); QNetworkRequest request(url);

View File

@@ -60,7 +60,7 @@ namespace BlackCore
void CVatsimMetarReader::ps_readMetars() void CVatsimMetarReader::ps_readMetars()
{ {
this->threadAssertCheck(); this->threadAssertCheck();
QUrl url(m_setup.get().vatsimMetars()); QUrl url(m_setup.get().vatsimMetarsUrl());
if (url.isEmpty()) { return; } if (url.isEmpty()) { return; }
Q_ASSERT_X(this->m_networkManager, Q_FUNC_INFO, "No network manager"); Q_ASSERT_X(this->m_networkManager, Q_FUNC_INFO, "No network manager");
QNetworkRequest request(url); QNetworkRequest request(url);

View File

@@ -16,11 +16,11 @@
#include "blackcore/vatsimdatafilereader.h" #include "blackcore/vatsimdatafilereader.h"
#include "blackcore/vatsimmetarreader.h" #include "blackcore/vatsimmetarreader.h"
#include "data/globalsetup.h" #include "data/globalsetup.h"
#include "blackmisc/network/networkutils.h"
#include "blackmisc/logmessage.h" #include "blackmisc/logmessage.h"
#include "blackmisc/fileutilities.h" #include "blackmisc/fileutilities.h"
#include "blackmisc/worker.h" #include "blackmisc/worker.h"
#include "blackmisc/json.h" #include "blackmisc/json.h"
#include "blackmisc/network/networkutils.h"
#include <QJsonObject> #include <QJsonObject>
#include <QJsonDocument> #include <QJsonDocument>
@@ -34,10 +34,12 @@ using namespace BlackMisc::Weather;
namespace BlackCore namespace BlackCore
{ {
CWebDataServices::CWebDataServices(CWebReaderFlags::WebReader readerFlags, QObject *parent) : CWebDataServices::CWebDataServices(
QObject(parent), m_readerFlags(readerFlags) CWebReaderFlags::WebReader readerFlags, int autoReadAfterSetupSynchronized, QObject *parent) :
QObject(parent), m_readerFlags(readerFlags), m_autoReadAfterSetupMs(autoReadAfterSetupSynchronized)
{ {
this->setObjectName("CWebDataReader"); this->setObjectName("CWebDataReader");
connect(&CSetupReader::instance(), &CSetupReader::setupSynchronized, this, &CWebDataServices::ps_setupRead);
this->initReaders(readerFlags); this->initReaders(readerFlags);
this->initWriters(); this->initWriters();
} }
@@ -429,7 +431,7 @@ namespace BlackCore
c = connect(this->m_icaoDataReader, &CIcaoDataReader::dataRead, this, &CWebDataServices::ps_readFromSwiftDb); c = connect(this->m_icaoDataReader, &CIcaoDataReader::dataRead, this, &CWebDataServices::ps_readFromSwiftDb);
Q_ASSERT_X(c, Q_FUNC_INFO, "ICAO reader signals"); Q_ASSERT_X(c, Q_FUNC_INFO, "ICAO reader signals");
Q_UNUSED(c); Q_UNUSED(c);
this->m_icaoDataReader->start(); this->m_icaoDataReader->start(QThread::LowPriority);
} }
// 5. Model reader // 5. Model reader
@@ -439,14 +441,14 @@ namespace BlackCore
bool c = connect(this->m_modelDataReader, &CModelDataReader::dataRead, this, &CWebDataServices::ps_readFromSwiftDb); bool c = connect(this->m_modelDataReader, &CModelDataReader::dataRead, this, &CWebDataServices::ps_readFromSwiftDb);
Q_ASSERT_X(c, Q_FUNC_INFO, "Model reader signals"); Q_ASSERT_X(c, Q_FUNC_INFO, "Model reader signals");
Q_UNUSED(c); Q_UNUSED(c);
this->m_modelDataReader->start(); this->m_modelDataReader->start(QThread::LowPriority);
} }
} }
void CWebDataServices::initWriters() void CWebDataServices::initWriters()
{ {
this->m_databaseWriter = new CDatabaseWriter( this->m_databaseWriter = new CDatabaseWriter(
m_setup.get().dbModelReader(), m_setup.get().dbModelReaderUrl(),
this); this);
} }
@@ -477,23 +479,40 @@ namespace BlackCore
} }
} }
void CWebDataServices::readAllInBackground(int delayMs) void CWebDataServices::ps_setupRead(bool success)
{
// setup has been changed
if (success)
{
if (m_autoReadAfterSetupMs >= 0)
{
CLogMessage(this).info("Setup synchronized, will trigger read of web service data");
this->readInBackground(CEntityFlags::AllEntities, m_autoReadAfterSetupMs);
}
}
else
{
CLogMessage(this).error("Failed to read setup, will not(!) triggrr web data read");
}
}
void CWebDataServices::ps_setupChanged()
{
CLogMessage(this).debug() << "Setup changed";
}
void CWebDataServices::readInBackground(CEntityFlags::Entity entities, int delayMs)
{ {
if (delayMs > 100) if (delayMs > 100)
{ {
BlackMisc::singleShot(delayMs, QThread::currentThread(), [ = ]() BlackMisc::singleShot(delayMs, QThread::currentThread(), [ = ]()
{ {
this->readAllInBackground(0); this->readInBackground(entities, 0);
}); });
} }
else else
{ {
// only readers requested will be read this->triggerRead(entities);
if (this->m_vatsimBookingReader) { this->m_vatsimBookingReader->readInBackgroundThread(); }
if (this->m_vatsimDataFileReader) { this->m_vatsimDataFileReader->readInBackgroundThread(); }
if (this->m_vatsimMetarReader) { this->m_vatsimMetarReader->readInBackgroundThread(); }
if (this->m_icaoDataReader) { this->m_icaoDataReader->readInBackgroundThread(CEntityFlags::AllIcaoAndCountries); }
if (this->m_modelDataReader) { this->m_modelDataReader->readInBackgroundThread(CEntityFlags::DistributorLiveryModel); }
} }
} }
@@ -557,10 +576,4 @@ namespace BlackCore
return s; return s;
} }
void CWebDataServices::readAtcBookingsInBackground() const
{
if (!this->m_vatsimBookingReader) { return; }
this->m_vatsimBookingReader->readInBackgroundThread();
}
} // ns } // ns

View File

@@ -23,6 +23,7 @@
#include "blackmisc/network/serverlist.h" #include "blackmisc/network/serverlist.h"
#include "blackmisc/network/voicecapabilities.h" #include "blackmisc/network/voicecapabilities.h"
#include "blackmisc/network/webdataservicesprovider.h" #include "blackmisc/network/webdataservicesprovider.h"
#include "blackmisc/network/entityflags.h"
#include "blackmisc/simulation/distributorlist.h" #include "blackmisc/simulation/distributorlist.h"
#include "blackmisc/weather/metarset.h" #include "blackmisc/weather/metarset.h"
#include "blackmisc/logcategorylist.h" #include "blackmisc/logcategorylist.h"
@@ -51,12 +52,13 @@ namespace BlackCore
public: public:
//! Constructor //! Constructor
CWebDataServices(CWebReaderFlags::WebReader readerFlags, QObject *parent = nullptr); CWebDataServices(CWebReaderFlags::WebReader readerFlags,
int autoReadAfterSetupSynchronized, QObject *parent = nullptr);
//! Shutdown //! Shutdown
void gracefulShutdown(); void gracefulShutdown();
//! Read ATC bookings //! Read ATC bookings (used to re-read)
void readAtcBookingsInBackground() const; void readAtcBookingsInBackground() const;
//! Booking reader //! Booking reader
@@ -74,7 +76,7 @@ namespace BlackCore
//! Log categories //! Log categories
static const BlackMisc::CLogCategoryList &getLogCategories(); static const BlackMisc::CLogCategoryList &getLogCategories();
// ------------------------ provider functionality ------------------------------ // ------------------------ provider functionality start ------------------------------
//! \copydoc IWebDataReaderProvider::connectDataReadSignal //! \copydoc IWebDataReaderProvider::connectDataReadSignal
//! \ingroup webdatareaderprovider //! \ingroup webdatareaderprovider
@@ -231,22 +233,22 @@ namespace BlackCore
virtual bool canConnectSwiftDb() const override; virtual bool canConnectSwiftDb() const override;
//! Save all DB data to JSON files //! Save all DB data to JSON files
//! \ingroup webdatareaderprovider
virtual bool writeDbDataToDisk(const QString &dir) const override; virtual bool writeDbDataToDisk(const QString &dir) const override;
//! Load DB data from JSON files //! Load DB data from JSON files
//! \ingroup webdatareaderprovider
virtual bool readDbDataFromDisk(const QString &dir, bool inBackground) override; virtual bool readDbDataFromDisk(const QString &dir, bool inBackground) override;
// ------------------------ provider functionality end ----------------------------
// ---------------------------------------------
// Consider to use the connect method of the provider to connect by entity
// ---------------------------------------------
public slots: public slots:
//! First read (allows to immediately read in background) //! First read (allows to immediately read in background)
void readAllInBackground(int delayMs); void readInBackground(BlackMisc::Network::CEntityFlags::Entity entities = BlackMisc::Network::CEntityFlags::AllEntities, int delayMs = 0);
signals:
//
// Consider to use the connect method of the provider to connect by entity
//
//! Model has been written
void modelWritten(const BlackMisc::Simulation::CAircraftModel &model);
private slots: private slots:
//! ATC bookings received //! ATC bookings received
@@ -261,6 +263,12 @@ namespace BlackCore
//! Read from model reader //! Read from model reader
void ps_readFromSwiftDb(BlackMisc::Network::CEntityFlags::Entity entity, BlackMisc::Network::CEntityFlags::ReadState state, int number); void ps_readFromSwiftDb(BlackMisc::Network::CEntityFlags::Entity entity, BlackMisc::Network::CEntityFlags::ReadState state, int number);
//! Setup has been read
void ps_setupRead(bool success);
//! Setup has been changed
void ps_setupChanged();
private: private:
//! Init the readers //! Init the readers
void initReaders(CWebReaderFlags::WebReader flags); void initReaders(CWebReaderFlags::WebReader flags);
@@ -269,7 +277,8 @@ namespace BlackCore
void initWriters(); void initWriters();
CWebReaderFlags::WebReader m_readerFlags = CWebReaderFlags::WebReaderFlag::None; //!< which readers are available CWebReaderFlags::WebReader m_readerFlags = CWebReaderFlags::WebReaderFlag::None; //!< which readers are available
BlackCore::CData<BlackCore::Data::GlobalSetup> m_setup {this}; //!< setup cache int m_autoReadAfterSetupMs = -1; //!< directly read all known readers after setup was syncronized
BlackCore::CData<BlackCore::Data::GlobalSetup> m_setup {this, &CWebDataServices::ps_setupChanged}; //!< setup cache
// for reading XML and VATSIM data files // for reading XML and VATSIM data files
CVatsimBookingReader *m_vatsimBookingReader = nullptr; CVatsimBookingReader *m_vatsimBookingReader = nullptr;
@@ -280,9 +289,6 @@ namespace BlackCore
// writing objects directly into DB // writing objects directly into DB
CDatabaseWriter *m_databaseWriter = nullptr; CDatabaseWriter *m_databaseWriter = nullptr;
// Setup
CSetupReader m_setupReader { this };
}; };
} // namespace } // namespace

View File

@@ -90,7 +90,7 @@ namespace BlackGui
void CDbLoginComponent::ps_setupChanged() void CDbLoginComponent::ps_setupChanged()
{ {
CUrl url(m_setup.get().dbHomePage()); CUrl url(m_setup.get().dbHomePageUrl());
ui->lbl_SwiftDB->setText("<a href=\"" + url.getFullUrl() + "\">swift DB@" + url.getHost() + "</a>"); ui->lbl_SwiftDB->setText("<a href=\"" + url.getFullUrl() + "\">swift DB@" + url.getHost() + "</a>");
ui->lbl_SwiftDB->setTextFormat(Qt::RichText); ui->lbl_SwiftDB->setTextFormat(Qt::RichText);
ui->lbl_SwiftDB->setTextInteractionFlags(Qt::TextBrowserInteraction); ui->lbl_SwiftDB->setTextInteractionFlags(Qt::TextBrowserInteraction);

View File

@@ -170,6 +170,8 @@ namespace BlackMisc
bool CProject::isNewerVersion(const QString &versionString) bool CProject::isNewerVersion(const QString &versionString)
{ {
if (versionString.isEmpty()) { return false; } if (versionString.isEmpty()) { return false; }
if (CProject::version() == versionString) { return false; }
QList<int> newer(getVersionParts(versionString)); QList<int> newer(getVersionParts(versionString));
QList<int> current(getVersionParts(version())); QList<int> current(getVersionParts(version()));
for (int i = 0; i < current.length(); i++) for (int i = 0; i < current.length(); i++)
@@ -178,7 +180,7 @@ namespace BlackMisc
if (current.at(i) > newer.at(i)) { return false; } if (current.at(i) > newer.at(i)) { return false; }
if (current.at(i) < newer.at(i)) { return true; } if (current.at(i) < newer.at(i)) { return true; }
} }
return true; return false;
} }
bool CProject::isDebugBuild() bool CProject::isDebugBuild()
@@ -295,6 +297,12 @@ namespace BlackMisc
return s; return s;
} }
const QStringList &CProject::swiftTeamDefaultServers()
{
static const QStringList s( { "https://vatsim-germany.org:50443/mapping/public/shared", "http://ubuntu12/public/bootstrap/shared"});
return s;
}
QString CProject::envVarPrivateSetupDirValue() QString CProject::envVarPrivateSetupDirValue()
{ {
return QProcessEnvironment::systemEnvironment().value(envVarPrivateSetupDir()); return QProcessEnvironment::systemEnvironment().value(envVarPrivateSetupDir());

View File

@@ -144,6 +144,9 @@ namespace BlackMisc
//! Executable name for swift data, no(!) appendix //! Executable name for swift data, no(!) appendix
static const QString &swiftDataExecutableName(); static const QString &swiftDataExecutableName();
//! swift team default servers for DB, bootstrap etc.
static const QStringList &swiftTeamDefaultServers();
private: private:
//! Constructor //! Constructor
CProject() {} CProject() {}

View File

@@ -19,6 +19,19 @@ namespace BlackMisc
m_updateTimer(new QTimer(this)) m_updateTimer(new QTimer(this))
{ } { }
qint64 CThreadedReader::lastModifiedMsSinceEpoch(QNetworkReply *nwReply) const
{
if (nwReply)
{
QVariant lastModifiedQv = nwReply->header(QNetworkRequest::LastModifiedHeader);
if (lastModifiedQv.isValid() && lastModifiedQv.canConvert<QDateTime>())
{
return lastModifiedQv.value<QDateTime>().toMSecsSinceEpoch();
}
}
return -1;
}
bool CThreadedReader::isFinishedOrShutdown() const bool CThreadedReader::isFinishedOrShutdown() const
{ {
return m_shutdown || isFinished(); return m_shutdown || isFinished();
@@ -38,14 +51,20 @@ namespace BlackMisc
void CThreadedReader::requestStop() void CThreadedReader::requestStop()
{ {
setFinished();
QMetaObject::invokeMethod(m_updateTimer, "stop"); QMetaObject::invokeMethod(m_updateTimer, "stop");
} }
void CThreadedReader::requestReload()
{
// default implementation, subclasses shall override as required
this->initialize();
}
void CThreadedReader::gracefulShutdown() void CThreadedReader::gracefulShutdown()
{ {
this->m_shutdown = true; this->m_shutdown = true;
this->requestStop(); this->requestStop();
this->quit();
} }
CThreadedReader::~CThreadedReader() CThreadedReader::~CThreadedReader()

View File

@@ -42,6 +42,10 @@ namespace BlackMisc
//! \threadsafe //! \threadsafe
void requestStop(); void requestStop();
//! Request new reading
//! \note override as required, default is to call initialize()
virtual void requestReload();
//! Destructor //! Destructor
virtual ~CThreadedReader(); virtual ~CThreadedReader();
@@ -62,13 +66,16 @@ namespace BlackMisc
void gracefulShutdown(); void gracefulShutdown();
protected: protected:
//! Constructor
CThreadedReader(QObject *owner, const QString &name);
QTimer *m_updateTimer = nullptr; //!< update timer QTimer *m_updateTimer = nullptr; //!< update timer
std::atomic<bool> m_shutdown { false }; //!< in shutdown process std::atomic<bool> m_shutdown { false }; //!< in shutdown process
mutable QReadWriteLock m_lock {QReadWriteLock::Recursive}; //!< lock which can be used from the derived classes mutable QReadWriteLock m_lock {QReadWriteLock::Recursive}; //!< lock which can be used from the derived classes
//! Constructor
CThreadedReader(QObject *owner, const QString &name);
//! When was reply last modified, -1 if N/A
qint64 lastModifiedMsSinceEpoch(QNetworkReply *nwReply) const;
//! Shutdown in progress or finished //! Shutdown in progress or finished
bool isFinishedOrShutdown() const; bool isFinishedOrShutdown() const;

View File

@@ -31,7 +31,7 @@ namespace BlackCoreTest
void CTestReaders::readIcaoData() void CTestReaders::readIcaoData()
{ {
CUrl url(m_setup.get().dbIcaoReader()); CUrl url(m_setup.get().dbIcaoReaderUrl());
if (!this->pingServer(url)) { return; } if (!this->pingServer(url)) { return; }
m_icaoReader.start(); m_icaoReader.start();
Expect e(&this->m_icaoReader); Expect e(&this->m_icaoReader);
@@ -54,7 +54,7 @@ namespace BlackCoreTest
void CTestReaders::readModelData() void CTestReaders::readModelData()
{ {
CUrl url(m_setup.get().dbModelReader()); CUrl url(m_setup.get().dbModelReaderUrl());
if (!this->pingServer(url)) { return; } if (!this->pingServer(url)) { return; }
m_modelReader.start(); m_modelReader.start();
Expect e(&this->m_modelReader); Expect e(&this->m_modelReader);