From d131cd2d33db9a09e13022c2c10c97b490a1798a Mon Sep 17 00:00:00 2001 From: Klaus Basan Date: Fri, 13 Nov 2015 03:03:03 +0100 Subject: [PATCH] refs #507, changed setup reader and simplied global setup and setup reader * a single base URL (shared), derived URLs by appended path * simplified dir structure shared with sub directories * renamed functions * automatically synchronize setup with DB when initialized * trigger download info loading when setup is completed * allow to automatically read after setup data have been synchronized * read DB data when setup has been loaded * allow to reload from threaded reader * improved handling (log messages, skip reading) when data are not available --- .../bootstrap/{0.6 => }/bootstrap.json | 0 resources/swiftresources.pro | 8 +- src/blackcore/blackcorefreefunctions.cpp | 2 + src/blackcore/context_network_impl.cpp | 7 +- src/blackcore/context_network_impl.h | 1 - src/blackcore/data/globalsetup.cpp | 155 +++++++----- src/blackcore/data/globalsetup.h | 90 +++---- src/blackcore/databaseauthentication.cpp | 4 +- src/blackcore/databasereader.cpp | 2 +- src/blackcore/databasereader.h | 7 +- src/blackcore/icaodatareader.cpp | 46 +++- src/blackcore/modeldatareader.cpp | 55 +++-- src/blackcore/setupreader.cpp | 224 +++++++++++++----- src/blackcore/setupreader.h | 42 +++- src/blackcore/vatsimbookingreader.cpp | 2 +- src/blackcore/vatsimdatafilereader.cpp | 3 +- src/blackcore/vatsimmetarreader.cpp | 2 +- src/blackcore/webdataservices.cpp | 53 +++-- src/blackcore/webdataservices.h | 38 +-- src/blackgui/components/dblogincomponent.cpp | 2 +- src/blackmisc/project.cpp | 10 +- src/blackmisc/project.h | 3 + src/blackmisc/threadedreader.cpp | 21 +- src/blackmisc/threadedreader.h | 13 +- tests/blackcore/testreaders.cpp | 4 +- 25 files changed, 527 insertions(+), 267 deletions(-) rename resources/local.env.template/bootstrap/{0.6 => }/bootstrap.json (100%) diff --git a/resources/local.env.template/bootstrap/0.6/bootstrap.json b/resources/local.env.template/bootstrap/bootstrap.json similarity index 100% rename from resources/local.env.template/bootstrap/0.6/bootstrap.json rename to resources/local.env.template/bootstrap/bootstrap.json diff --git a/resources/swiftresources.pro b/resources/swiftresources.pro index 588b855fa..1e5421279 100644 --- a/resources/swiftresources.pro +++ b/resources/swiftresources.pro @@ -7,8 +7,8 @@ CONFIG -= qt OTHER_FILES += data/images/flags/*.png OTHER_FILES += data/images/airlines/*.png OTHER_FILES += swiftDB/*.* -OTHER_FILES += bootstrap/0.6/productive/*.* -OTHER_FILES += bootstrap/0.6/development/*.* +OTHER_FILES += bootstrap/productive/*.* +OTHER_FILES += bootstrap/development/*.* OTHER_FILES += local.env.template/*.* OTHER_FILES += local.env.template/bootstrap/*.* OTHER_FILES += local.env.template/bootstrap/0.6/*.* @@ -16,8 +16,8 @@ OTHER_FILES += local.env.template/bootstrap/0.6/*.* COPY_FILES += $$PWD/data/images/flags/*.png COPY_FILES += $$PWD/data/images/airlines/*.png COPY_FILES += $$PWD/swiftDB/*.* -COPY_FILES += $$PWD/bootstrap/0.6/productive/*.* -COPY_FILES += $$PWD/bootstrap/0.6/development/*.* +COPY_FILES += $$PWD/bootstrap/productive/*.* +COPY_FILES += $$PWD/bootstrap/development/*.* COPY_FILES += $$PWD/local.env.template/*.* COPY_FILES += $$PWD/local.env.template/bootstrap/*.* COPY_FILES += $$PWD/local.env.template/bootstrap/0.6/*.* diff --git a/src/blackcore/blackcorefreefunctions.cpp b/src/blackcore/blackcorefreefunctions.cpp index e71ab70e0..7a174cdd7 100644 --- a/src/blackcore/blackcorefreefunctions.cpp +++ b/src/blackcore/blackcorefreefunctions.cpp @@ -13,6 +13,7 @@ #include "network.h" #include "simulator.h" #include "context_application.h" +#include "setupreader.h" #include namespace BlackCore @@ -29,5 +30,6 @@ namespace BlackCore BlackCore::Data::CGlobalSetup::registerMetadata(); BlackCore::Data::CDownload::registerMetadata(); + BlackCore::CSetupReader::instance(); // kick off reader } } // namespace diff --git a/src/blackcore/context_network_impl.cpp b/src/blackcore/context_network_impl.cpp index 8f1789cb8..3c3ce875b 100644 --- a/src/blackcore/context_network_impl.cpp +++ b/src/blackcore/context_network_impl.cpp @@ -55,15 +55,14 @@ namespace BlackCore connect(this->m_dataUpdateTimer, &QTimer::timeout, this, &CContextNetwork::requestDataUpdates); this->m_dataUpdateTimer->start(30 * 1000); - // 3. data reader - this->m_webDataReader = new CWebDataServices(CWebReaderFlags::AllReaders, this); + // 3. data reader, start reading when setup is synced with xx delay + this->m_webDataReader = new CWebDataServices(CWebReaderFlags::AllReaders, 1000, this); this->m_webReaderSignalConnections.append( this->m_webDataReader->connectDataReadSignal( this, // the object here must be the same as in the bind std::bind(&CContextNetwork::webServiceDataRead, this, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3) ) ); - this->m_webDataReader->readAllInBackground(1000); // 4. Airspace contents Q_ASSERT_X(this->getRuntime()->getCContextOwnAircraft(), Q_FUNC_INFO, "this and own aircraft context must be local"); @@ -570,7 +569,7 @@ namespace BlackCore { if (this->isDebugEnabled()) { CLogMessage(this, CLogCategory::contextSlot()).debug() << Q_FUNC_INFO; } Q_ASSERT_X(this->m_webDataReader, Q_FUNC_INFO, "missing reader"); - this->m_webDataReader->readAtcBookingsInBackground(); + this->m_webDataReader->readInBackground(BlackMisc::Network::CEntityFlags::BookingEntity); } bool CContextNetwork::updateAircraftRendered(const CCallsign &callsign, bool rendered, const CIdentifier &originator) diff --git a/src/blackcore/context_network_impl.h b/src/blackcore/context_network_impl.h index 3b253461e..0ec114b8a 100644 --- a/src/blackcore/context_network_impl.h +++ b/src/blackcore/context_network_impl.h @@ -248,7 +248,6 @@ namespace BlackCore BlackMisc::CDigestSignal m_dsAtcStationsOnlineChanged { this, &IContextNetwork::changedAtcStationsOnline, &IContextNetwork::changedAtcStationsOnlineDigest, 750, 4 }; BlackMisc::CDigestSignal m_dsAircraftsInRangeChanged { this, &IContextNetwork::changedAircraftInRange, &IContextNetwork::changedAircraftInRangeDigest, 750, 4 }; - //! Own aircraft from \sa CContextOwnAircraft const BlackMisc::Simulation::CSimulatedAircraft ownAircraft() const; diff --git a/src/blackcore/data/globalsetup.cpp b/src/blackcore/data/globalsetup.cpp index 0da054c2d..41880806a 100644 --- a/src/blackcore/data/globalsetup.cpp +++ b/src/blackcore/data/globalsetup.cpp @@ -8,6 +8,7 @@ */ #include "globalsetup.h" +#include "blackmisc/project.h" #include "blackmisc/math/mathutils.h" #include "blackmisc/blackmiscfreefunctions.h" #include @@ -22,35 +23,35 @@ namespace BlackCore { CGlobalSetup::CGlobalSetup() : ITimestampBased(0), - m_dbRootDirectory("http://ubuntu12/swiftdatastore/public"), m_dbHttpPort(80), m_dbHttpsPort(443), - m_vatsimBookings("http://vatbook.euroutepro.com/xml2.php"), - m_vatsimMetars("http://metar.vatsim.net/metar.php"), - m_vatsimDataFile(QStringList({ "http://info.vroute.net/vatsim-data.txt" })), - m_bootstrap(QStringList({ "https://vatsim-germany.org:50443/mapping/public/bootstrap", "http://ubuntu12/public/bootstrap"})), - m_swiftDbDataFiles(QStringList({})), + m_dbRootDirectoryUrl("http://ubuntu12/swiftdatastore/public"), + m_vatsimBookingsUrl("http://vatbook.euroutepro.com/xml2.php"), + m_vatsimMetarsUrl("http://metar.vatsim.net/metar.php"), + m_vatsimDataFileUrls(QStringList({ "http://info.vroute.net/vatsim-data.txt" })), + m_sharedUrls(CProject::swiftTeamDefaultServers()), + m_newsUrls(QStringList({ "http://swift-project.org/" })), m_fsdTestServers({ CServer("swift", "swift Testserver", "vatsim-germany.org", 6809, CUser("1234567", "swift Test User", "", "123456"), true) }) { } - CUrl CGlobalSetup::dbIcaoReader() const + CUrl CGlobalSetup::dbIcaoReaderUrl() const { - return dbRootDirectory(); + return dbRootDirectoryUrl(); } - CUrl CGlobalSetup::dbModelReader() const + CUrl CGlobalSetup::dbModelReaderUrl() const { - return dbRootDirectory(); + return dbRootDirectoryUrl(); } - CUrl CGlobalSetup::dbHomePage() const + CUrl CGlobalSetup::dbHomePageUrl() const { - return dbRootDirectory().withAppendedPath("/page/index.php"); + return dbRootDirectoryUrl().withAppendedPath("/page/index.php"); } - CUrl CGlobalSetup::dbLoginService() const + CUrl CGlobalSetup::dbLoginServiceUrl() const { - return dbRootDirectory(). + return dbRootDirectoryUrl(). withAppendedPath("/service/jsonauthenticate.php"). withSwitchedScheme("https", m_dbHttpsPort); } @@ -68,14 +69,38 @@ namespace BlackCore m_dbDebugFlag = debug; } - bool CGlobalSetup::hasSameType(CGlobalSetup &otherSetup) const + bool CGlobalSetup::hasSameType(const CGlobalSetup &otherSetup) const { return this->isDevelopment() == otherSetup.isDevelopment(); } - CUrl CGlobalSetup::vatsimMetars() const + CUrl CGlobalSetup::vatsimMetarsUrl() const { - return this->m_vatsimMetars.withAppendedQuery("id=all"); + return this->m_vatsimMetarsUrl.withAppendedQuery("id=all"); + } + + CUrlList CGlobalSetup::bootstrapUrls() const + { + CUrlList urls(m_sharedUrls); + return urls.appendPath(isDevelopment() ? + CGlobalSetup::versionString() + "/development/bootstrap/bootstrap.json" : + CGlobalSetup::versionString() + "/productive/bootstrap/bootstrap.json"); + } + + CUrlList CGlobalSetup::downloadInfoUrls() const + { + CUrlList urls(m_sharedUrls); + return urls.appendPath(isDevelopment() ? + CGlobalSetup::versionString() + "/development/download/download.json" : + CGlobalSetup::versionString() + "/productive/download/download.json"); + } + + CUrlList CGlobalSetup::swiftDbDataFileLocationUrls() const + { + CUrlList urls(m_sharedUrls); + return urls.appendPath(isDevelopment() ? + CGlobalSetup::versionString() + "/development/dbdata/" : + CGlobalSetup::versionString() + "/productive/dbdata/"); } QString CGlobalSetup::convertToQString(bool i18n) const @@ -91,39 +116,48 @@ namespace BlackCore s.append("For development: "); s.append(boolToYesNo(isDevelopment())); s.append(separator); - s.append("DB root directory: "); - s.append(dbRootDirectory().convertToQString(i18n)); - s.append(separator); - s.append("ICAO DB reader: "); - s.append(dbIcaoReader().convertToQString(i18n)); - s.append(separator); - s.append("Model DB reader: "); - s.append(dbModelReader().convertToQString(i18n)); - s.append(separator); - s.append("DB home page: "); - s.append(dbHomePage().convertToQString(i18n)); - s.append(separator); - s.append("DB login service: "); - s.append(dbLoginService().convertToQString(i18n)); - s.append(separator); - s.append("VATSIM bookings: "); - s.append(vatsimBookings().convertToQString(i18n)); - s.append(separator); - s.append("VATSIM METARs: "); - s.append(vatsimMetars().convertToQString(i18n)); - s.append(separator); - s.append("VATSIM data file: "); - s.append(vatsimDataFile().convertToQString(i18n)); + s.append("Download URLs: "); + s.append(downloadInfoUrls().toQString(i18n)); s.append(separator); s.append("Bootstrap URLs: "); - s.append(bootstrapUrls().convertToQString(i18n)); + s.append(bootstrapUrls().toQString(i18n)); + s.append(separator); + s.append("News URLs: "); + s.append(swiftLatestNewsUrls().toQString(i18n)); + s.append(separator); + + s.append("DB root directory: "); + s.append(dbRootDirectoryUrl().toQString(i18n)); + s.append(separator); + s.append("ICAO DB reader: "); + s.append(dbIcaoReaderUrl().toQString(i18n)); + s.append(separator); + s.append("Model DB reader: "); + s.append(dbModelReaderUrl().toQString(i18n)); + s.append(separator); + s.append("DB home page: "); + s.append(dbHomePageUrl().toQString(i18n)); + s.append(separator); + s.append("DB login service: "); + s.append(dbLoginServiceUrl().toQString(i18n)); s.append(separator); s.append("swift DB datafile locations: "); - s.append(swiftDbDataFileLocations().convertToQString(i18n)); + s.append(swiftDbDataFileLocationUrls().toQString(i18n)); s.append(separator); + + s.append("VATSIM bookings: "); + s.append(vatsimBookingsUrl().toQString(i18n)); + s.append(separator); + s.append("VATSIM METARs: "); + s.append(vatsimMetarsUrl().toQString(i18n)); + s.append(separator); + s.append("VATSIM data file: "); + s.append(vatsimDataFileUrls().toQString(i18n)); + s.append(separator); + s.append("FSD test servers: "); - s.append(fsdTestServers().convertToQString(i18n)); + s.append(fsdTestServers().toQString(i18n)); return s; } @@ -136,23 +170,27 @@ namespace BlackCore switch (i) { case IndexDbRootDirectory: - return CVariant::fromValue(this->m_dbRootDirectory); + return CVariant::fromValue(this->m_dbRootDirectoryUrl); case IndexDbHttpPort: return CVariant::fromValue(this->m_dbHttpPort); case IndexDbHttpsPort: return CVariant::fromValue(this->m_dbHttpsPort); case IndexDbLoginService: - return CVariant::fromValue(this->dbLoginService()); + return CVariant::fromValue(this->dbLoginServiceUrl()); case IndexVatsimData: - return CVariant::fromValue(this->m_vatsimDataFile); + return CVariant::fromValue(this->m_vatsimDataFileUrls); case IndexVatsimBookings: - return CVariant::fromValue(this->m_vatsimDataFile); + return CVariant::fromValue(this->m_vatsimDataFileUrls); case IndexVatsimMetars: - return CVariant::fromValue(this->m_vatsimMetars); + return CVariant::fromValue(this->m_vatsimMetarsUrl); + case IndexDownload: + return CVariant::fromValue(this->downloadInfoUrls()); case IndexBootstrap: - return CVariant::fromValue(this->m_bootstrap); + return CVariant::fromValue(this->bootstrapUrls()); case IndexSwiftDbFiles: - return CVariant::fromValue(this->m_swiftDbDataFiles); + return CVariant::fromValue(this->swiftDbDataFileLocationUrls()); + case IndexShared: + return CVariant::fromValue(this->m_sharedUrls); default: return CValueObject::propertyByIndex(index); } @@ -171,7 +209,7 @@ namespace BlackCore switch (i) { case IndexDbRootDirectory: - this->m_dbRootDirectory.setPropertyByIndex(variant, index.copyFrontRemoved()); + this->m_dbRootDirectoryUrl.setPropertyByIndex(variant, index.copyFrontRemoved()); break; case IndexDbHttpPort: this->m_dbHttpPort = variant.toInt(); @@ -182,19 +220,16 @@ namespace BlackCore case IndexDbLoginService: break; case IndexVatsimData: - this->m_vatsimDataFile = variant.value(); + this->m_vatsimDataFileUrls = variant.value(); break; case IndexVatsimBookings: - this->m_vatsimBookings.setPropertyByIndex(variant, index.copyFrontRemoved()); + this->m_vatsimBookingsUrl.setPropertyByIndex(variant, index.copyFrontRemoved()); break; case IndexVatsimMetars: - this->m_vatsimMetars.setPropertyByIndex(variant, index.copyFrontRemoved()); + this->m_vatsimMetarsUrl.setPropertyByIndex(variant, index.copyFrontRemoved()); break; - case IndexBootstrap: - this->m_bootstrap = variant.value(); - break; - case IndexSwiftDbFiles: - this->m_swiftDbDataFiles = variant.value(); + case IndexShared: + this->m_sharedUrls = variant.value(); break; default: CValueObject::setPropertyByIndex(variant, index); @@ -204,7 +239,7 @@ namespace BlackCore const QString &CGlobalSetup::versionString() { - static const QString v("0.6"); + static const QString v("0.6.1"); return v; } } // ns diff --git a/src/blackcore/data/globalsetup.h b/src/blackcore/data/globalsetup.h index 698c68cc2..677cae8d6 100644 --- a/src/blackcore/data/globalsetup.h +++ b/src/blackcore/data/globalsetup.h @@ -41,7 +41,9 @@ namespace BlackCore IndexVatsimMetars, IndexVatsimData, IndexSwiftDbFiles, - IndexBootstrap + IndexBootstrap, + IndexDownload, + IndexShared }; //! Default constructor @@ -50,50 +52,56 @@ namespace BlackCore //! Destructor. ~CGlobalSetup() {} - //! Root directory of DB - const BlackMisc::Network::CUrl &dbRootDirectory() const { return m_dbRootDirectory; } - - //! ICAO Reader location - BlackMisc::Network::CUrl dbIcaoReader() const; - - //! Model Reader protocol - BlackMisc::Network::CUrl dbModelReader() const; - //! Http port int dbHttpPort() const { return m_dbHttpPort; } //! Https port int dbHttpsPort() const { return m_dbHttpsPort; } - //! Home page url - BlackMisc::Network::CUrl dbHomePage() const; - - //! Login service - BlackMisc::Network::CUrl dbLoginService() const; - //! Debug flag bool dbDebugFlag() const; //! Set debug flag void setServerDebugFlag(bool debug); - //! URL to read VATSIM bookings - const BlackMisc::Network::CUrl &vatsimBookings() const { return m_vatsimBookings; } - //! Same type? - bool hasSameType(CGlobalSetup &otherSetup) const; + bool hasSameType(const CGlobalSetup &otherSetup) const; + + //! Home page url + BlackMisc::Network::CUrl dbHomePageUrl() const; + + //! Login service + BlackMisc::Network::CUrl dbLoginServiceUrl() const; + + //! Root directory of DB + const BlackMisc::Network::CUrl &dbRootDirectoryUrl() const { return m_dbRootDirectoryUrl; } + + //! ICAO Reader location + BlackMisc::Network::CUrl dbIcaoReaderUrl() const; + + //! Model Reader protocol + BlackMisc::Network::CUrl dbModelReaderUrl() const; + + //! URL to read VATSIM bookings + const BlackMisc::Network::CUrl &vatsimBookingsUrl() const { return m_vatsimBookingsUrl; } //! VATSIM METAR URL - BlackMisc::Network::CUrl vatsimMetars() const; + BlackMisc::Network::CUrl vatsimMetarsUrl() const; //! VATSIM data file URLs - const BlackMisc::Network::CUrlList &vatsimDataFile() const { return m_vatsimDataFile; } + const BlackMisc::Network::CUrlList &vatsimDataFileUrls() const { return m_vatsimDataFileUrls; } //! Bootstrap URLs (where the data for the setup itself can be downloaded) - const BlackMisc::Network::CUrlList &bootstrapUrls() const { return m_bootstrap; } + BlackMisc::Network::CUrlList bootstrapUrls() const; + + //! Version files and download locations + BlackMisc::Network::CUrlList downloadInfoUrls() const; //! Alternative locations of swift DB data files - const BlackMisc::Network::CUrlList &swiftDbDataFileLocations() const { return m_swiftDbDataFiles; } + BlackMisc::Network::CUrlList swiftDbDataFileLocationUrls() const; + + //! Locations of swift DB news + const BlackMisc::Network::CUrlList &swiftLatestNewsUrls() const { return m_newsUrls; } //! FSD test servers const BlackMisc::Network::CServerList &fsdTestServers() const { return m_fsdTestServers; } @@ -122,19 +130,19 @@ namespace BlackCore private: BLACK_ENABLE_TUPLE_CONVERSION(BlackCore::Data::CGlobalSetup) - BlackMisc::Network::CUrl m_dbRootDirectory; //!< Root directory - int m_dbHttpPort = 80; //!< port - int m_dbHttpsPort = 443; //!< SSL port - BlackMisc::Network::CUrl m_vatsimBookings; //!< ATC bookings - BlackMisc::Network::CUrl m_vatsimMetars; //!< METAR data - BlackMisc::Network::CUrlList m_vatsimDataFile; //!< Overall VATSIM data file - BlackMisc::Network::CUrlList m_bootstrap; //!< where we can obtain downloads of these data - BlackMisc::Network::CUrlList m_swiftDbDataFiles; //!< alternative locations of the DB files, if DB is not available - BlackMisc::Network::CServerList m_fsdTestServers; //!< FSD test servers - bool m_development = false; //!< dev. version? + int m_dbHttpPort = 80; //!< port + int m_dbHttpsPort = 443; //!< SSL port + bool m_development = false; //!< dev. version? + BlackMisc::Network::CUrl m_dbRootDirectoryUrl; //!< Root directory of DB + BlackMisc::Network::CUrl m_vatsimBookingsUrl; //!< ATC bookings + BlackMisc::Network::CUrl m_vatsimMetarsUrl; //!< METAR data + BlackMisc::Network::CUrlList m_vatsimDataFileUrls; //!< Overall VATSIM data file + BlackMisc::Network::CUrlList m_sharedUrls; //!< where we can obtain shared info files such as bootstrap, .. + BlackMisc::Network::CUrlList m_newsUrls; //!< where we can obtain latest news + BlackMisc::Network::CServerList m_fsdTestServers; //!< FSD test servers // transient members, to be switched on/off via GUI or set from reader - bool m_dbDebugFlag = false; //!< can trigger DEBUG on the server, so you need to know hat you are doing + bool m_dbDebugFlag = false; //!< can trigger DEBUG on the server, so you need to know what you are doing }; //! Trait for global setup data @@ -157,14 +165,14 @@ namespace BlackCore Q_DECLARE_METATYPE(BlackCore::Data::CGlobalSetup) BLACK_DECLARE_TUPLE_CONVERSION(BlackCore::Data::CGlobalSetup, ( attr(o.m_timestampMSecsSinceEpoch), - attr(o.m_dbRootDirectory), + attr(o.m_dbRootDirectoryUrl), attr(o.m_dbHttpPort), attr(o.m_dbHttpsPort), - attr(o.m_vatsimBookings), - attr(o.m_vatsimMetars), - attr(o.m_vatsimDataFile), - attr(o.m_bootstrap), - attr(o.m_swiftDbDataFiles), + attr(o.m_vatsimBookingsUrl), + attr(o.m_vatsimMetarsUrl), + attr(o.m_vatsimDataFileUrls), + attr(o.m_sharedUrls), + attr(o.m_newsUrls), attr(o.m_fsdTestServers), attr(o.m_development), attr(o.m_dbDebugFlag, flags < DisabledForJson > ()) diff --git a/src/blackcore/databaseauthentication.cpp b/src/blackcore/databaseauthentication.cpp index d727585ac..77a933ea7 100644 --- a/src/blackcore/databaseauthentication.cpp +++ b/src/blackcore/databaseauthentication.cpp @@ -53,7 +53,7 @@ namespace BlackCore if (pw.isEmpty()) { msgs.push_back(CStatusMessage(cats, CStatusMessage::SeverityError, "No password")); } if (!msgs.isEmpty()) { return msgs; } - CUrl url(this->m_setup.get().dbLoginService()); + CUrl url(this->m_setup.get().dbLoginServiceUrl()); QString msg; if (!CNetworkUtils::canConnect(url, msg)) { @@ -84,7 +84,7 @@ namespace BlackCore void CDatabaseAuthenticationService::logoff() { - CUrl url(this->m_setup.get().dbLoginService()); + CUrl url(this->m_setup.get().dbLoginServiceUrl()); url.setQuery("logoff=true"); QNetworkRequest request(CNetworkUtils::getNetworkRequest(url)); this->m_networkManager->get(request); diff --git a/src/blackcore/databasereader.cpp b/src/blackcore/databasereader.cpp index 49711cb76..12633881d 100644 --- a/src/blackcore/databasereader.cpp +++ b/src/blackcore/databasereader.cpp @@ -26,7 +26,7 @@ namespace BlackCore void CDatabaseReader::readInBackgroundThread(CEntityFlags::Entity entities) { - if (m_shutdown) { return; } + if (isFinishedOrShutdown()) { return; } this->m_watchdogTimer.stop(); bool s = QMetaObject::invokeMethod(this, "ps_read", Q_ARG(BlackMisc::Network::CEntityFlags::Entity, entities)); Q_ASSERT_X(s, Q_FUNC_INFO, "Invoke failed"); diff --git a/src/blackcore/databasereader.h b/src/blackcore/databasereader.h index 8c3c466e6..befcbf2cf 100644 --- a/src/blackcore/databasereader.h +++ b/src/blackcore/databasereader.h @@ -23,8 +23,7 @@ namespace BlackCore { - //! Support for threaded based reading and parsing tasks such - //! as data files via http, or file system and parsing (such as FSX models) + //! Specialized version of threaded reader for DB data class BLACKCORE_EXPORT CDatabaseReader : public BlackMisc::CThreadedReader { Q_OBJECT @@ -69,7 +68,7 @@ namespace BlackCore protected: BlackMisc::Network::CUrl m_watchdogUrl; //!< URL for checking if alive - QTimer m_watchdogTimer { this }; //!< Timer for watchdog + QTimer m_watchdogTimer { this }; //!< Timer for watchdog (DB available?) QString m_watchdogMessage; //!< Returned status message bool m_canConnect = false; //!< Successful connection? mutable QReadWriteLock m_watchdogLock; //!< Lock @@ -85,7 +84,7 @@ namespace BlackCore CDatabaseReader::JsonDatastoreResponse setStatusAndTransformReplyIntoDatastoreResponse(QNetworkReply *nwReply); private slots: - //! Watchdog + //! Watchdog checking if DB is available void ps_watchdog(); private: diff --git a/src/blackcore/icaodatareader.cpp b/src/blackcore/icaodatareader.cpp index 75ca98efe..8a56e09f9 100644 --- a/src/blackcore/icaodatareader.cpp +++ b/src/blackcore/icaodatareader.cpp @@ -140,33 +140,57 @@ namespace BlackCore if (entities.testFlag(CEntityFlags::AircraftIcaoEntity)) { QUrl url(getAircraftIcaoUrl()); - QNetworkRequest requestAircraft(CNetworkUtils::getNetworkRequest(url)); - this->m_networkManagerAircraft->get(requestAircraft); - entitiesTriggered |= CEntityFlags::AircraftIcaoEntity; + if (!url.isEmpty()) + { + QNetworkRequest requestAircraft(CNetworkUtils::getNetworkRequest(url)); + this->m_networkManagerAircraft->get(requestAircraft); + entitiesTriggered |= CEntityFlags::AircraftIcaoEntity; + } + else + { + CLogMessage(this).error("No URL for %1") << CEntityFlags::flagToString(CEntityFlags::AircraftIcaoEntity); + } } if (entities.testFlag(CEntityFlags::AirlineIcaoEntity)) { QUrl url(getAirlineIcaoUrl()); - QNetworkRequest requestAirline(CNetworkUtils::getNetworkRequest(url)); - this->m_networkManagerAirlines->get(requestAirline); - entitiesTriggered |= CEntityFlags::AirlineIcaoEntity; + if (!url.isEmpty()) + { + QNetworkRequest requestAirline(CNetworkUtils::getNetworkRequest(url)); + this->m_networkManagerAirlines->get(requestAirline); + entitiesTriggered |= CEntityFlags::AirlineIcaoEntity; + } + else + { + CLogMessage(this).error("No URL for %1") << CEntityFlags::flagToString(CEntityFlags::AirlineIcaoEntity); + } } if (entities.testFlag(CEntityFlags::CountryEntity)) { QUrl url(getCountryUrl()); - QNetworkRequest requestCountry(CNetworkUtils::getNetworkRequest(url)); - this->m_networkManagerCountries->get(requestCountry); - entitiesTriggered |= CEntityFlags::CountryEntity; + if (!url.isEmpty()) + { + QNetworkRequest requestCountry(CNetworkUtils::getNetworkRequest(url)); + this->m_networkManagerCountries->get(requestCountry); + entitiesTriggered |= CEntityFlags::CountryEntity; + } + else + { + CLogMessage(this).error("No URL for %1") << CEntityFlags::flagToString(CEntityFlags::CountryEntity); + } } - emit dataRead(entitiesTriggered, CEntityFlags::StartRead, 0); + if (entitiesTriggered != CEntityFlags::NoEntity) + { + emit dataRead(entitiesTriggered, CEntityFlags::StartRead, 0); + } } CUrl CIcaoDataReader::getBaseUrl() const { - CUrl baseUrl(this->m_setup.get().dbIcaoReader()); + CUrl baseUrl(this->m_setup.get().dbIcaoReaderUrl()); return baseUrl; } diff --git a/src/blackcore/modeldatareader.cpp b/src/blackcore/modeldatareader.cpp index a9e6fd32b..e6a4fa52c 100644 --- a/src/blackcore/modeldatareader.cpp +++ b/src/blackcore/modeldatareader.cpp @@ -137,29 +137,56 @@ namespace BlackCore CEntityFlags::Entity triggeredRead = CEntityFlags::NoEntity; if (entity.testFlag(CEntityFlags::LiveryEntity)) { - QNetworkRequest requestLivery(getLiveryUrl()); - CNetworkUtils::ignoreSslVerification(requestLivery); - this->m_networkManagerLivery->get(requestLivery); - triggeredRead |= CEntityFlags::LiveryEntity; + CUrl url(getLiveryUrl()); + if (!url.isEmpty()) + { + QNetworkRequest requestLivery(url); + CNetworkUtils::ignoreSslVerification(requestLivery); + this->m_networkManagerLivery->get(requestLivery); + triggeredRead |= CEntityFlags::LiveryEntity; + } + else + { + CLogMessage(this).error("No URL for %1") << CEntityFlags::flagToString(CEntityFlags::LiveryEntity); + } } if (entity.testFlag(CEntityFlags::DistributorEntity)) { - QNetworkRequest requestDistributor(getDistributorUrl()); - CNetworkUtils::ignoreSslVerification(requestDistributor); - this->m_networkManagerDistributor->get(requestDistributor); - triggeredRead |= CEntityFlags::DistributorEntity; + CUrl url(getDistributorUrl()); + if (!url.isEmpty()) + { + QNetworkRequest requestDistributor(url); + CNetworkUtils::ignoreSslVerification(requestDistributor); + this->m_networkManagerDistributor->get(requestDistributor); + triggeredRead |= CEntityFlags::DistributorEntity; + } + else + { + CLogMessage(this).error("No URL for %1") << CEntityFlags::flagToString(CEntityFlags::DistributorEntity); + } } if (entity.testFlag(CEntityFlags::ModelEntity)) { - QNetworkRequest requestModel(getModelUrl()); - CNetworkUtils::ignoreSslVerification(requestModel); - this->m_networkManagerModel->get(requestModel); - triggeredRead |= CEntityFlags::ModelEntity; + CUrl url(getModelUrl()); + if (!url.isEmpty()) + { + QNetworkRequest requestModel(url); + CNetworkUtils::ignoreSslVerification(requestModel); + this->m_networkManagerModel->get(requestModel); + triggeredRead |= CEntityFlags::ModelEntity; + } + else + { + CLogMessage(this).error("No URL for %1") << CEntityFlags::flagToString(CEntityFlags::ModelEntity); + } } - emit dataRead(triggeredRead, CEntityFlags::StartRead, 0); + if (triggeredRead != CEntityFlags::NoEntity) + { + emit dataRead(triggeredRead, CEntityFlags::StartRead, 0); + } } void CModelDataReader::ps_parseLiveryData(QNetworkReply *nwReplyPtr) @@ -332,7 +359,7 @@ namespace BlackCore CUrl CModelDataReader::getBaseUrl() const { - CUrl baseUrl(m_setup.get().dbModelReader()); + CUrl baseUrl(m_setup.get().dbModelReaderUrl()); return baseUrl; } diff --git a/src/blackcore/setupreader.cpp b/src/blackcore/setupreader.cpp index d61a4bbbe..c9fdc91fd 100644 --- a/src/blackcore/setupreader.cpp +++ b/src/blackcore/setupreader.cpp @@ -29,63 +29,95 @@ namespace BlackCore CSetupReader::CSetupReader(QObject *owner) : CThreadedReader(owner, "CSetupReader") { + connect(this, &CSetupReader::setupSynchronized, this, &CSetupReader::ps_setupSyncronized); QString localFileName; - if (localFile(localFileName)) + if (this->localBootstrapFile(localFileName)) { // initialized by local file for testing // I do not even need to start in background here CLogMessage(this).info("Using local bootstrap file: %1") << localFileName; + emit this->setupSynchronized(true); } else { - this->m_networkManager = new QNetworkAccessManager(this); - this->connect(this->m_networkManager, &QNetworkAccessManager::finished, this, &CSetupReader::ps_parseSetupFile); + this->m_bootstrapUrls.uniqueWrite()->push_back(m_setup.get().bootstrapUrls()); + this->m_downloadUrls.uniqueWrite()->push_back(m_setup.get().downloadInfoUrls()); + + this->m_networkManagerBootstrap = new QNetworkAccessManager(this); + this->connect(this->m_networkManagerBootstrap, &QNetworkAccessManager::finished, this, &CSetupReader::ps_parseSetupFile); + + this->m_networkManagerDownload = new QNetworkAccessManager(this); + this->connect(this->m_networkManagerDownload, &QNetworkAccessManager::finished, this, &CSetupReader::ps_parseDownloadFile); + this->start(QThread::LowPriority); } } + CSetupReader &CSetupReader::instance() + { + static CSetupReader reader(QCoreApplication::instance()); + return reader; + } + void CSetupReader::initialize() { + // start to read by myself CThreadedReader::initialize(); - QTimer::singleShot(500, this, &CSetupReader::ps_read); + QTimer::singleShot(500, this, &CSetupReader::ps_readSetup); } - void CSetupReader::ps_read() + void CSetupReader::ps_readSetup() { this->threadAssertCheck(); - Q_ASSERT_X(this->m_networkManager, Q_FUNC_INFO, "Missing network manager"); - CUrlList urls(getRemainingUrls()); - if (urls.isEmpty()) { return; } - int urlsSize = urls.size(); - - QUrl url(urls.getNextUrl(false)); - if (urlsSize > 1) + Q_ASSERT_X(this->m_networkManagerBootstrap, Q_FUNC_INFO, "Missing network manager"); + CUrl url(this->m_bootstrapUrls.uniqueWrite()->getNextWorkingUrl()); + if (url.isEmpty()) { - // more than one URL one, quick check if this can be contacted - QString m; - if (!CNetworkUtils::canConnect(url, m)) - { - m_failedUrls.push_back(url); - CLogMessage(this).warning("Cannot connect to %1") << url.toString(); - url = urls.getNextUrl(); - } + CLogMessage(this).warning("Cannot read setup, failed URLs: %1") << this->m_bootstrapUrls.read()->getFailedUrls(); + emit setupSynchronized(false); + return; } - - // now I have the first or second URL for reading - if (url.isEmpty()) { return; } QNetworkRequest request(url); CNetworkUtils::ignoreSslVerification(request); - - // request - this->m_networkManager->get(request); + this->m_networkManagerBootstrap->get(request); } - bool CSetupReader::localFile(QString &fileName) + void CSetupReader::ps_readDownload() + { + this->threadAssertCheck(); + Q_ASSERT_X(this->m_networkManagerDownload, Q_FUNC_INFO, "Missing network manager"); + CUrl url(this->m_downloadUrls.uniqueWrite()->getNextWorkingUrl()); + if (url.isEmpty()) + { + CLogMessage(this).warning("Cannot read download (info), failed URLs: %1") << this->m_downloadUrls.read()->getFailedUrls(); + return; + } + QNetworkRequest request(url); + CNetworkUtils::ignoreSslVerification(request); + this->m_networkManagerDownload->get(request); + } + + void CSetupReader::ps_setupSyncronized(bool success) + { + // trigger + if (success) + { + CLogMessage(this).info("Setup synchronized, will trigger read of download information"); + QTimer::singleShot(500, this, &CSetupReader::ps_readDownload); + } + else + { + CLogMessage(this).error("Setup reading failed, hence version info will not be loaded"); + } + } + + bool CSetupReader::localBootstrapFile(QString &fileName) { QString dir(CProject::getSwiftPrivateResourceDir()); if (dir.isEmpty()) { return false; } - fileName = CFileUtils::appendFilePaths(dir, "bootstrap/" + CGlobalSetup::versionString() + "/bootstrap.json"); + // no version for local files, as those come withe the current code + fileName = CFileUtils::appendFilePaths(dir, "bootstrap/bootstrap.json"); QString content(CFileUtils::readFileToString(fileName)); if (content.isEmpty()) { return false; } CGlobalSetup s; @@ -100,53 +132,33 @@ namespace BlackCore return CProject::useDevelopmentSetup(); } - CUrlList CSetupReader::getRemainingUrls() const - { - CUrlList urls(m_setup.get().bootstrapUrls().appendPath(appendPathAndFile())); - urls.removeIfIn(m_failedUrls); - return urls; - } - - QString CSetupReader::appendPathAndFile() - { - return isForDevelopment() ? - CGlobalSetup::versionString() + "/development/bootstrap.json" : - CGlobalSetup::versionString() + "/productive/bootstrap.json"; - } - void CSetupReader::ps_parseSetupFile(QNetworkReply *nwReplyPtr) { // wrap pointer, make sure any exit cleans up reply // required to use delete later as object is created in a different thread QScopedPointer nwReply(nwReplyPtr); + this->threadAssertCheck(); QUrl url(nwReply->url()); QString urlString(url.toString()); QString replyMessage(nwReply->errorString()); - this->threadAssertCheck(); if (this->isFinishedOrShutdown()) { CLogMessage(this).debug() << Q_FUNC_INFO; CLogMessage(this).info("Terminated loading bootstrap files"); nwReply->abort(); + emit setupSynchronized(false); return; // stop, terminate straight away, ending thread } if (nwReply->error() == QNetworkReply::NoError) { - qint64 lastModified = -1; - QVariant lastModifiedQv = nwReply->header(QNetworkRequest::LastModifiedHeader); - if (lastModifiedQv.isValid() && lastModifiedQv.canConvert()) - { - lastModified = lastModifiedQv.value().toMSecsSinceEpoch(); - } - + qint64 lastModified = this->lastModifiedMsSinceEpoch(nwReply.data()); QString setupJson(nwReplyPtr->readAll()); nwReplyPtr->close(); if (setupJson.isEmpty()) { - CLogMessage(this).info("No bootstrap setup file"); - m_failedUrls.push_back(url); + CLogMessage(this).info("No bootstrap setup file at %1") << urlString; // try next URL } else @@ -156,20 +168,21 @@ namespace BlackCore loadedSetup.convertFromJson(Json::jsonObjectFromString(setupJson)); loadedSetup.setDevelopment(isForDevelopment()); // always update, regardless what persistent setting says if (loadedSetup.getMSecsSinceEpoch() == 0 && lastModified > 0) { loadedSetup.setMSecsSinceEpoch(lastModified); } - bool sameType = loadedSetup.hasSameType(currentSetup); qint64 currentVersionTimestamp = currentSetup.getMSecsSinceEpoch(); qint64 newVersionTimestamp = loadedSetup.getMSecsSinceEpoch(); - bool sameVersionLoaded = sameType && (newVersionTimestamp == currentVersionTimestamp); + bool sameVersionLoaded = (loadedSetup == currentSetup); if (sameVersionLoaded) { - CLogMessage(this).info("Same version loaded from %1 as already in data cache %2") << urlString << CDataCache::persistentStore(); + CLogMessage(this).info("Same setup version loaded from %1 as already in data cache %2") << urlString << m_setup.getFilename(); + emit setupSynchronized(true); return; // success } + bool sameType = loadedSetup.hasSameType(currentSetup); bool outdatedVersionLoaded = sameType && (newVersionTimestamp < currentVersionTimestamp); if (outdatedVersionLoaded) { - CLogMessage(this).info("Version loaded from %1 outdated, older than version in data cache %2") << urlString << CDataCache::persistentStore(); + CLogMessage(this).info("Setup loaded from %1 outdated, older than version in data cache %2") << urlString << m_setup.getFilename(); // try next URL } else @@ -178,11 +191,13 @@ namespace BlackCore if (!m.isEmpty()) { CLogMessage(this).preformatted(m); + emit setupSynchronized(false); return; // issue with cache } else { - CLogMessage(this).info("Updated data cache in %1") << CDataCache::persistentStore(); + CLogMessage(this).info("Setup: Updated data cache in %1") << this->m_setup.getFilename(); + emit setupSynchronized(true); return; // success } // cache } // outdated? @@ -197,14 +212,95 @@ namespace BlackCore } // try next one if any - const int maxTrials = 2; - m_failedUrls.push_back(url); - if (m_failedUrls.size() >= maxTrials) { return; } - - int urlsNo = getRemainingUrls().size(); - if (urlsNo >= 0) + if (this->m_bootstrapUrls.uniqueWrite()->addFailedUrl(url)) { - QTimer::singleShot(500, this, &CSetupReader::ps_read); + QTimer::singleShot(500, this, &CSetupReader::ps_readSetup); + } + else + { + emit setupSynchronized(false); + } + } + + void CSetupReader::ps_parseDownloadFile(QNetworkReply *nwReplyPtr) + { + // wrap pointer, make sure any exit cleans up reply + // required to use delete later as object is created in a different thread + QScopedPointer nwReply(nwReplyPtr); + this->threadAssertCheck(); + QUrl url(nwReply->url()); + QString urlString(url.toString()); + QString replyMessage(nwReply->errorString()); + + if (this->isFinishedOrShutdown()) + { + CLogMessage(this).debug() << Q_FUNC_INFO; + CLogMessage(this).info("Terminated loading download info"); + nwReply->abort(); + return; // stop, terminate straight away, ending thread + } + + if (nwReply->error() == QNetworkReply::NoError) + { + qint64 lastModified = this->lastModifiedMsSinceEpoch(nwReply.data()); + QString setupJson(nwReplyPtr->readAll()); + nwReplyPtr->close(); + if (setupJson.isEmpty()) + { + CLogMessage(this).info("No download (info) file"); + // try next URL + } + else + { + CDownload currentDownload(m_download.get()); // from cache + CDownload loadedDownload; + loadedDownload.convertFromJson(Json::jsonObjectFromString(setupJson)); + loadedDownload.setDevelopment(isForDevelopment()); // always update, regardless what persistent setting says + if (loadedDownload.getMSecsSinceEpoch() == 0 && lastModified > 0) { loadedDownload.setMSecsSinceEpoch(lastModified); } + qint64 currentVersionTimestamp = currentDownload.getMSecsSinceEpoch(); + qint64 newVersionTimestamp = loadedDownload.getMSecsSinceEpoch(); + bool sameVersionLoaded = (loadedDownload == currentDownload); + if (sameVersionLoaded) + { + CLogMessage(this).info("Same download info loaded from %1 as already in data cache %2") << urlString << m_download.getFilename(); + return; // success + } + + bool sameType = loadedDownload.hasSameType(currentDownload); + bool outdatedVersionLoaded = sameType && (newVersionTimestamp < currentVersionTimestamp); + if (outdatedVersionLoaded) + { + CLogMessage(this).info("Download loaded from %1 outdated, older than version in data cache %2") << urlString << m_download.getFilename(); + // try next URL + } + else + { + CStatusMessage m = m_download.set(loadedDownload); + if (!m.isEmpty()) + { + CLogMessage(this).preformatted(m); + return; // issue with cache + } + else + { + CLogMessage(this).info("Download info: Updated data cache in %1") << m_download.getFilename(); + return; // success + } // cache + } // outdated? + + } // json empty + } // no error + else + { + // network error + CLogMessage(this).warning("Reading download info failed %1 %2") << replyMessage << urlString; + nwReply->abort(); + } + + // try next one if any + if (this->m_downloadUrls.uniqueWrite()->addFailedUrl(url)) + { + QTimer::singleShot(500, this, &CSetupReader::ps_readSetup); } } // method diff --git a/src/blackcore/setupreader.h b/src/blackcore/setupreader.h index dfa6e9e5f..fed5d3ee5 100644 --- a/src/blackcore/setupreader.h +++ b/src/blackcore/setupreader.h @@ -14,7 +14,9 @@ #include "blackcore/blackcoreexport.h" #include "blackmisc/threadedreader.h" +#include "blackmisc/lockfree.h" #include "blackcore/data/globalsetup.h" +#include "blackcore/data/download.h" #include #include @@ -28,8 +30,12 @@ namespace BlackCore Q_OBJECT public: - //! Constructor - explicit CSetupReader(QObject *owner); + //! Single instance + static CSetupReader &instance(); + + signals: + //! Setup has been read + void setupSynchronized(bool success); protected slots: //! \copydoc CThreadedReader::initialize @@ -40,25 +46,35 @@ namespace BlackCore //! \threadsafe void ps_parseSetupFile(QNetworkReply *nwReply); + //! Download has been read + //! \threadsafe + void ps_parseDownloadFile(QNetworkReply *nwReplyPtr); + //! Do reading - void ps_read(); + void ps_readSetup(); + + //! Do reading + void ps_readDownload(); + + //! Setup has beem syncronized + void ps_setupSyncronized(bool success); private: - QNetworkAccessManager *m_networkManager = nullptr; - BlackMisc::Network::CUrlList m_failedUrls; - CData m_setup {this}; //!< data cache + QNetworkAccessManager *m_networkManagerBootstrap = nullptr; + QNetworkAccessManager *m_networkManagerDownload = nullptr; + BlackMisc::LockFree m_bootstrapUrls; + BlackMisc::LockFree m_downloadUrls; + CData m_setup {this}; //!< data cache setup + CData m_download {this}; //!< data cache downloads + + //! Constructor + explicit CSetupReader(QObject *owner); //! Read by local individual file - bool localFile(QString &fileName); - - //! Remaining URLs failed one excluded - BlackMisc::Network::CUrlList getRemainingUrls() const; + bool localBootstrapFile(QString &fileName); //! Read for development environment? static bool isForDevelopment(); - - //! Bootstrap URL - static QString appendPathAndFile(); }; } // ns diff --git a/src/blackcore/vatsimbookingreader.cpp b/src/blackcore/vatsimbookingreader.cpp index e4e09ef78..6a2a1f2a5 100644 --- a/src/blackcore/vatsimbookingreader.cpp +++ b/src/blackcore/vatsimbookingreader.cpp @@ -41,7 +41,7 @@ namespace BlackCore { this->threadAssertCheck(); Q_ASSERT_X(this->m_networkManager, Q_FUNC_INFO, "No network manager"); - QUrl url(m_setup.get().vatsimBookings()); + QUrl url(m_setup.get().vatsimBookingsUrl()); if (url.isEmpty()) { return; } QNetworkRequest request(url); diff --git a/src/blackcore/vatsimdatafilereader.cpp b/src/blackcore/vatsimdatafilereader.cpp index e24dc7399..adf3f4a4a 100644 --- a/src/blackcore/vatsimdatafilereader.cpp +++ b/src/blackcore/vatsimdatafilereader.cpp @@ -152,12 +152,11 @@ namespace BlackCore void CVatsimDataFileReader::ps_read() { this->threadAssertCheck(); - if (m_setup.get().vatsimDataFile().size() < 1) { return; } // round robin for load balancing // remark: Don't use QThread to run network operations in the background // see http://qt-project.org/doc/qt-4.7/qnetworkaccessmanager.html - QUrl url(m_setup.get().vatsimDataFile().getNextUrl()); + QUrl url(m_setup.get().vatsimDataFileUrls().getNextUrl()); if (url.isEmpty()) { return; } Q_ASSERT_X(this->m_networkManager, Q_FUNC_INFO, "Missing network manager"); QNetworkRequest request(url); diff --git a/src/blackcore/vatsimmetarreader.cpp b/src/blackcore/vatsimmetarreader.cpp index 26ca9e66d..f798a92d9 100644 --- a/src/blackcore/vatsimmetarreader.cpp +++ b/src/blackcore/vatsimmetarreader.cpp @@ -60,7 +60,7 @@ namespace BlackCore void CVatsimMetarReader::ps_readMetars() { this->threadAssertCheck(); - QUrl url(m_setup.get().vatsimMetars()); + QUrl url(m_setup.get().vatsimMetarsUrl()); if (url.isEmpty()) { return; } Q_ASSERT_X(this->m_networkManager, Q_FUNC_INFO, "No network manager"); QNetworkRequest request(url); diff --git a/src/blackcore/webdataservices.cpp b/src/blackcore/webdataservices.cpp index ad4f42009..8eabe991a 100644 --- a/src/blackcore/webdataservices.cpp +++ b/src/blackcore/webdataservices.cpp @@ -16,11 +16,11 @@ #include "blackcore/vatsimdatafilereader.h" #include "blackcore/vatsimmetarreader.h" #include "data/globalsetup.h" +#include "blackmisc/network/networkutils.h" #include "blackmisc/logmessage.h" #include "blackmisc/fileutilities.h" #include "blackmisc/worker.h" #include "blackmisc/json.h" -#include "blackmisc/network/networkutils.h" #include #include @@ -34,10 +34,12 @@ using namespace BlackMisc::Weather; namespace BlackCore { - CWebDataServices::CWebDataServices(CWebReaderFlags::WebReader readerFlags, QObject *parent) : - QObject(parent), m_readerFlags(readerFlags) + CWebDataServices::CWebDataServices( + CWebReaderFlags::WebReader readerFlags, int autoReadAfterSetupSynchronized, QObject *parent) : + QObject(parent), m_readerFlags(readerFlags), m_autoReadAfterSetupMs(autoReadAfterSetupSynchronized) { this->setObjectName("CWebDataReader"); + connect(&CSetupReader::instance(), &CSetupReader::setupSynchronized, this, &CWebDataServices::ps_setupRead); this->initReaders(readerFlags); this->initWriters(); } @@ -429,7 +431,7 @@ namespace BlackCore c = connect(this->m_icaoDataReader, &CIcaoDataReader::dataRead, this, &CWebDataServices::ps_readFromSwiftDb); Q_ASSERT_X(c, Q_FUNC_INFO, "ICAO reader signals"); Q_UNUSED(c); - this->m_icaoDataReader->start(); + this->m_icaoDataReader->start(QThread::LowPriority); } // 5. Model reader @@ -439,14 +441,14 @@ namespace BlackCore bool c = connect(this->m_modelDataReader, &CModelDataReader::dataRead, this, &CWebDataServices::ps_readFromSwiftDb); Q_ASSERT_X(c, Q_FUNC_INFO, "Model reader signals"); Q_UNUSED(c); - this->m_modelDataReader->start(); + this->m_modelDataReader->start(QThread::LowPriority); } } void CWebDataServices::initWriters() { this->m_databaseWriter = new CDatabaseWriter( - m_setup.get().dbModelReader(), + m_setup.get().dbModelReaderUrl(), this); } @@ -477,23 +479,40 @@ namespace BlackCore } } - void CWebDataServices::readAllInBackground(int delayMs) + void CWebDataServices::ps_setupRead(bool success) + { + // setup has been changed + if (success) + { + if (m_autoReadAfterSetupMs >= 0) + { + CLogMessage(this).info("Setup synchronized, will trigger read of web service data"); + this->readInBackground(CEntityFlags::AllEntities, m_autoReadAfterSetupMs); + } + } + else + { + CLogMessage(this).error("Failed to read setup, will not(!) triggrr web data read"); + } + } + + void CWebDataServices::ps_setupChanged() + { + CLogMessage(this).debug() << "Setup changed"; + } + + void CWebDataServices::readInBackground(CEntityFlags::Entity entities, int delayMs) { if (delayMs > 100) { BlackMisc::singleShot(delayMs, QThread::currentThread(), [ = ]() { - this->readAllInBackground(0); + this->readInBackground(entities, 0); }); } else { - // only readers requested will be read - if (this->m_vatsimBookingReader) { this->m_vatsimBookingReader->readInBackgroundThread(); } - if (this->m_vatsimDataFileReader) { this->m_vatsimDataFileReader->readInBackgroundThread(); } - if (this->m_vatsimMetarReader) { this->m_vatsimMetarReader->readInBackgroundThread(); } - if (this->m_icaoDataReader) { this->m_icaoDataReader->readInBackgroundThread(CEntityFlags::AllIcaoAndCountries); } - if (this->m_modelDataReader) { this->m_modelDataReader->readInBackgroundThread(CEntityFlags::DistributorLiveryModel); } + this->triggerRead(entities); } } @@ -557,10 +576,4 @@ namespace BlackCore return s; } - void CWebDataServices::readAtcBookingsInBackground() const - { - if (!this->m_vatsimBookingReader) { return; } - this->m_vatsimBookingReader->readInBackgroundThread(); - } - } // ns diff --git a/src/blackcore/webdataservices.h b/src/blackcore/webdataservices.h index 068132459..61166e65c 100644 --- a/src/blackcore/webdataservices.h +++ b/src/blackcore/webdataservices.h @@ -23,6 +23,7 @@ #include "blackmisc/network/serverlist.h" #include "blackmisc/network/voicecapabilities.h" #include "blackmisc/network/webdataservicesprovider.h" +#include "blackmisc/network/entityflags.h" #include "blackmisc/simulation/distributorlist.h" #include "blackmisc/weather/metarset.h" #include "blackmisc/logcategorylist.h" @@ -51,12 +52,13 @@ namespace BlackCore public: //! Constructor - CWebDataServices(CWebReaderFlags::WebReader readerFlags, QObject *parent = nullptr); + CWebDataServices(CWebReaderFlags::WebReader readerFlags, + int autoReadAfterSetupSynchronized, QObject *parent = nullptr); //! Shutdown void gracefulShutdown(); - //! Read ATC bookings + //! Read ATC bookings (used to re-read) void readAtcBookingsInBackground() const; //! Booking reader @@ -74,7 +76,7 @@ namespace BlackCore //! Log categories static const BlackMisc::CLogCategoryList &getLogCategories(); - // ------------------------ provider functionality ------------------------------ + // ------------------------ provider functionality start ------------------------------ //! \copydoc IWebDataReaderProvider::connectDataReadSignal //! \ingroup webdatareaderprovider @@ -231,22 +233,22 @@ namespace BlackCore virtual bool canConnectSwiftDb() const override; //! Save all DB data to JSON files + //! \ingroup webdatareaderprovider virtual bool writeDbDataToDisk(const QString &dir) const override; //! Load DB data from JSON files + //! \ingroup webdatareaderprovider virtual bool readDbDataFromDisk(const QString &dir, bool inBackground) override; + // ------------------------ provider functionality end ---------------------------- + + // --------------------------------------------- + // Consider to use the connect method of the provider to connect by entity + // --------------------------------------------- + public slots: //! First read (allows to immediately read in background) - void readAllInBackground(int delayMs); - - signals: - // - // Consider to use the connect method of the provider to connect by entity - // - - //! Model has been written - void modelWritten(const BlackMisc::Simulation::CAircraftModel &model); + void readInBackground(BlackMisc::Network::CEntityFlags::Entity entities = BlackMisc::Network::CEntityFlags::AllEntities, int delayMs = 0); private slots: //! ATC bookings received @@ -261,6 +263,12 @@ namespace BlackCore //! Read from model reader void ps_readFromSwiftDb(BlackMisc::Network::CEntityFlags::Entity entity, BlackMisc::Network::CEntityFlags::ReadState state, int number); + //! Setup has been read + void ps_setupRead(bool success); + + //! Setup has been changed + void ps_setupChanged(); + private: //! Init the readers void initReaders(CWebReaderFlags::WebReader flags); @@ -269,7 +277,8 @@ namespace BlackCore void initWriters(); CWebReaderFlags::WebReader m_readerFlags = CWebReaderFlags::WebReaderFlag::None; //!< which readers are available - BlackCore::CData m_setup {this}; //!< setup cache + int m_autoReadAfterSetupMs = -1; //!< directly read all known readers after setup was syncronized + BlackCore::CData m_setup {this, &CWebDataServices::ps_setupChanged}; //!< setup cache // for reading XML and VATSIM data files CVatsimBookingReader *m_vatsimBookingReader = nullptr; @@ -280,9 +289,6 @@ namespace BlackCore // writing objects directly into DB CDatabaseWriter *m_databaseWriter = nullptr; - - // Setup - CSetupReader m_setupReader { this }; }; } // namespace diff --git a/src/blackgui/components/dblogincomponent.cpp b/src/blackgui/components/dblogincomponent.cpp index e526f3ace..c669e78aa 100644 --- a/src/blackgui/components/dblogincomponent.cpp +++ b/src/blackgui/components/dblogincomponent.cpp @@ -90,7 +90,7 @@ namespace BlackGui void CDbLoginComponent::ps_setupChanged() { - CUrl url(m_setup.get().dbHomePage()); + CUrl url(m_setup.get().dbHomePageUrl()); ui->lbl_SwiftDB->setText("swift DB@" + url.getHost() + ""); ui->lbl_SwiftDB->setTextFormat(Qt::RichText); ui->lbl_SwiftDB->setTextInteractionFlags(Qt::TextBrowserInteraction); diff --git a/src/blackmisc/project.cpp b/src/blackmisc/project.cpp index c3fd21502..0cb466b48 100644 --- a/src/blackmisc/project.cpp +++ b/src/blackmisc/project.cpp @@ -170,6 +170,8 @@ namespace BlackMisc bool CProject::isNewerVersion(const QString &versionString) { if (versionString.isEmpty()) { return false; } + if (CProject::version() == versionString) { return false; } + QList newer(getVersionParts(versionString)); QList current(getVersionParts(version())); for (int i = 0; i < current.length(); i++) @@ -178,7 +180,7 @@ namespace BlackMisc if (current.at(i) > newer.at(i)) { return false; } if (current.at(i) < newer.at(i)) { return true; } } - return true; + return false; } bool CProject::isDebugBuild() @@ -295,6 +297,12 @@ namespace BlackMisc return s; } + const QStringList &CProject::swiftTeamDefaultServers() + { + static const QStringList s( { "https://vatsim-germany.org:50443/mapping/public/shared", "http://ubuntu12/public/bootstrap/shared"}); + return s; + } + QString CProject::envVarPrivateSetupDirValue() { return QProcessEnvironment::systemEnvironment().value(envVarPrivateSetupDir()); diff --git a/src/blackmisc/project.h b/src/blackmisc/project.h index 3ccd5139a..af957f957 100644 --- a/src/blackmisc/project.h +++ b/src/blackmisc/project.h @@ -144,6 +144,9 @@ namespace BlackMisc //! Executable name for swift data, no(!) appendix static const QString &swiftDataExecutableName(); + //! swift team default servers for DB, bootstrap etc. + static const QStringList &swiftTeamDefaultServers(); + private: //! Constructor CProject() {} diff --git a/src/blackmisc/threadedreader.cpp b/src/blackmisc/threadedreader.cpp index 5f82d7447..e6145cea2 100644 --- a/src/blackmisc/threadedreader.cpp +++ b/src/blackmisc/threadedreader.cpp @@ -19,6 +19,19 @@ namespace BlackMisc m_updateTimer(new QTimer(this)) { } + qint64 CThreadedReader::lastModifiedMsSinceEpoch(QNetworkReply *nwReply) const + { + if (nwReply) + { + QVariant lastModifiedQv = nwReply->header(QNetworkRequest::LastModifiedHeader); + if (lastModifiedQv.isValid() && lastModifiedQv.canConvert()) + { + return lastModifiedQv.value().toMSecsSinceEpoch(); + } + } + return -1; + } + bool CThreadedReader::isFinishedOrShutdown() const { return m_shutdown || isFinished(); @@ -38,14 +51,20 @@ namespace BlackMisc void CThreadedReader::requestStop() { - setFinished(); QMetaObject::invokeMethod(m_updateTimer, "stop"); } + void CThreadedReader::requestReload() + { + // default implementation, subclasses shall override as required + this->initialize(); + } + void CThreadedReader::gracefulShutdown() { this->m_shutdown = true; this->requestStop(); + this->quit(); } CThreadedReader::~CThreadedReader() diff --git a/src/blackmisc/threadedreader.h b/src/blackmisc/threadedreader.h index 2255a0bb3..6cf4ff822 100644 --- a/src/blackmisc/threadedreader.h +++ b/src/blackmisc/threadedreader.h @@ -42,6 +42,10 @@ namespace BlackMisc //! \threadsafe void requestStop(); + //! Request new reading + //! \note override as required, default is to call initialize() + virtual void requestReload(); + //! Destructor virtual ~CThreadedReader(); @@ -62,13 +66,16 @@ namespace BlackMisc void gracefulShutdown(); protected: - //! Constructor - CThreadedReader(QObject *owner, const QString &name); - QTimer *m_updateTimer = nullptr; //!< update timer std::atomic m_shutdown { false }; //!< in shutdown process mutable QReadWriteLock m_lock {QReadWriteLock::Recursive}; //!< lock which can be used from the derived classes + //! Constructor + CThreadedReader(QObject *owner, const QString &name); + + //! When was reply last modified, -1 if N/A + qint64 lastModifiedMsSinceEpoch(QNetworkReply *nwReply) const; + //! Shutdown in progress or finished bool isFinishedOrShutdown() const; diff --git a/tests/blackcore/testreaders.cpp b/tests/blackcore/testreaders.cpp index 107e58990..388a01ae2 100644 --- a/tests/blackcore/testreaders.cpp +++ b/tests/blackcore/testreaders.cpp @@ -31,7 +31,7 @@ namespace BlackCoreTest void CTestReaders::readIcaoData() { - CUrl url(m_setup.get().dbIcaoReader()); + CUrl url(m_setup.get().dbIcaoReaderUrl()); if (!this->pingServer(url)) { return; } m_icaoReader.start(); Expect e(&this->m_icaoReader); @@ -54,7 +54,7 @@ namespace BlackCoreTest void CTestReaders::readModelData() { - CUrl url(m_setup.get().dbModelReader()); + CUrl url(m_setup.get().dbModelReaderUrl()); if (!this->pingServer(url)) { return; } m_modelReader.start(); Expect e(&this->m_modelReader);