diff --git a/src/blackcore/db/airportdatareader.cpp b/src/blackcore/db/airportdatareader.cpp index eadd66b66..d971df93d 100644 --- a/src/blackcore/db/airportdatareader.cpp +++ b/src/blackcore/db/airportdatareader.cpp @@ -16,6 +16,7 @@ using namespace BlackMisc; using namespace BlackMisc::Aviation; using namespace BlackMisc::Network; +using namespace BlackMisc::Db; namespace BlackCore { @@ -37,6 +38,45 @@ namespace BlackCore return this->getAirports().size(); } + bool CAirportDataReader::readFromJsonFilesInBackground(const QString &dir, CEntityFlags::Entity whatToRead) + { + if (dir.isEmpty() || whatToRead == CEntityFlags::NoEntity) { return false; } + QTimer::singleShot(0, this, [this, dir, whatToRead]() + { + bool s = this->readFromJsonFiles(dir, whatToRead); + Q_UNUSED(s); + }); + return true; + } + + bool CAirportDataReader::readFromJsonFiles(const QString &dir, CEntityFlags::Entity whatToRead) + { + QDir directory(dir); + if (!directory.exists()) { return false; } + BlackMisc::Network::CEntityFlags::Entity reallyRead = CEntityFlags::NoEntity; + + if (whatToRead.testFlag(CEntityFlags::AirportEntity)) + { + QString airportsJson(CFileUtils::readFileToString(CFileUtils::appendFilePaths(directory.absolutePath(), "airports.json"))); + if (!airportsJson.isEmpty()) + { + CAirportList airports; + airports.convertFromJson(airportsJson); + const int c = airports.size(); + this->m_airportCache.set(airports); + + emit dataRead(CEntityFlags::AirportEntity, CEntityFlags::ReadFinished, c); + reallyRead |= CEntityFlags::AirportEntity; + } + } + return (reallyRead & CEntityFlags::DistributorLiveryModel) == whatToRead; + } + + CEntityFlags::Entity CAirportDataReader::getSupportedEntities() const + { + return CEntityFlags::AirportEntity; + } + QDateTime CAirportDataReader::getCacheTimestamp(CEntityFlags::Entity entities) const { return entities == CEntityFlags::AirportEntity ? m_airportCache.getAvailableTimestamp() : QDateTime(); @@ -65,12 +105,17 @@ namespace BlackCore bool CAirportDataReader::hasChangedUrl(CEntityFlags::Entity entity) const { Q_UNUSED(entity); - return CDatabaseReader::isChangedUrl(this->m_readerUrlCache.get(), getBaseUrl()); + return CDatabaseReader::isChangedUrl(this->m_readerUrlCache.get(), getBaseUrl(CDbFlags::DbReading)); } - CUrl CAirportDataReader::getAirportsUrl() const + CUrl CAirportDataReader::getDbServiceBaseUrl() const { - return sApp->getGlobalSetup().getDbAirportReaderUrl().withAppendedPath("service/jsonairport.php"); + return sApp->getGlobalSetup().getDbAirportReaderUrl(); + } + + CUrl CAirportDataReader::getAirportsUrl(CDbFlags::DataRetrievalModeFlag mode) const + { + return getBaseUrl(mode).withAppendedPath(fileNameForMode(CEntityFlags::AirportEntity, mode)); } void CAirportDataReader::ps_parseAirportData(QNetworkReply *nwReply) @@ -104,18 +149,18 @@ namespace BlackCore } this->m_airportCache.set(airports, latestTimestamp); - this->updateReaderUrl(getBaseUrl()); + this->updateReaderUrl(getBaseUrl(CDbFlags::DbReading)); emit this->dataRead(CEntityFlags::AirportEntity, CEntityFlags::ReadFinished, airports.size()); } - void CAirportDataReader::ps_read(CEntityFlags::Entity entity, const QDateTime &newerThan) + void CAirportDataReader::ps_read(CEntityFlags::Entity entity, CDbFlags::DataRetrievalModeFlag mode, const QDateTime &newerThan) { this->threadAssertCheck(); if (this->isAbandoned()) { return; } if (entity.testFlag(CEntityFlags::AirportEntity)) { - CUrl url = getAirportsUrl(); + CUrl url = getAirportsUrl(mode); if (!url.isEmpty()) { if (!newerThan.isNull()) @@ -153,12 +198,5 @@ namespace BlackCore CLogMessage::preformatted(m); } } - - const CUrl &CAirportDataReader::getBaseUrl() - { - static const CUrl baseUrl(sApp->getGlobalSetup().getDbAirportReaderUrl()); - return baseUrl; - } - } // ns } // ns diff --git a/src/blackcore/db/airportdatareader.h b/src/blackcore/db/airportdatareader.h index bc57dbb9a..0fd02c485 100644 --- a/src/blackcore/db/airportdatareader.h +++ b/src/blackcore/db/airportdatareader.h @@ -32,7 +32,7 @@ namespace BlackCore public: //! Constructor - CAirportDataReader(QObject* parent, const CDatabaseReaderConfigList &config); + CAirportDataReader(QObject *parent, const CDatabaseReaderConfigList &config); //! Returns a list of all airports in the database. //! \threadsafe @@ -42,7 +42,14 @@ namespace BlackCore //! \threadsafe int getAirportsCount() const; + //! Read from static data file + bool readFromJsonFiles(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead = BlackMisc::Network::CEntityFlags::AirportEntity); + + //! Read from static data file in background + bool readFromJsonFilesInBackground(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead = BlackMisc::Network::CEntityFlags::AirportEntity); + // base class overrides + virtual BlackMisc::Network::CEntityFlags::Entity getSupportedEntities() const override; virtual QDateTime getCacheTimestamp(BlackMisc::Network::CEntityFlags::Entity entities) const override; virtual int getCacheCount(BlackMisc::Network::CEntityFlags::Entity entity) const override; virtual void synchronizeCaches(BlackMisc::Network::CEntityFlags::Entity entities) override; @@ -52,17 +59,15 @@ namespace BlackCore // base class overrides virtual void invalidateCaches(BlackMisc::Network::CEntityFlags::Entity entities) override; virtual bool hasChangedUrl(BlackMisc::Network::CEntityFlags::Entity entity) const override; - - private: - //! URL for airport list - BlackMisc::Network::CUrl getAirportsUrl() const; + virtual BlackMisc::Network::CUrl getDbServiceBaseUrl() const override; private slots: //! Parse downloaded JSON file void ps_parseAirportData(QNetworkReply *nwReply); //! Read / re-read data file - void ps_read(BlackMisc::Network::CEntityFlags::Entity entity = BlackMisc::Network::CEntityFlags::DistributorLiveryModel, const QDateTime &newerThan = QDateTime()); + void ps_read(BlackMisc::Network::CEntityFlags::Entity entity = BlackMisc::Network::CEntityFlags::DistributorLiveryModel, + BlackMisc::Db::CDbFlags::DataRetrievalModeFlag mode = BlackMisc::Db::CDbFlags::DbReading, const QDateTime &newerThan = QDateTime()); //! Airport cache changed void ps_airportCacheChanged(); @@ -79,9 +84,8 @@ namespace BlackCore //! Update reader URL void updateReaderUrl(const BlackMisc::Network::CUrl &url); - //! Base URL - //! \threadsafe - static const BlackMisc::Network::CUrl &getBaseUrl(); + //! URL for airport list + BlackMisc::Network::CUrl getAirportsUrl(BlackMisc::Db::CDbFlags::DataRetrievalModeFlag mode) const; }; } } // ns diff --git a/src/blackcore/db/databasereader.cpp b/src/blackcore/db/databasereader.cpp index 5e9576a79..8be5fae77 100644 --- a/src/blackcore/db/databasereader.cpp +++ b/src/blackcore/db/databasereader.cpp @@ -37,10 +37,12 @@ namespace BlackCore { namespace Db { + CUrl CDatabaseReader::s_workingSharedDbData; + CDatabaseReader::CDatabaseReader(QObject *owner, const CDatabaseReaderConfigList &config, const QString &name) : BlackCore::CThreadedReader(owner, name), m_config(config) { - getDbUrl(); // init the cache + CDatabaseReader::initWorkingUrls(); } void CDatabaseReader::readInBackgroundThread(CEntityFlags::Entity entities, const QDateTime &newerThan) @@ -49,9 +51,10 @@ namespace BlackCore // we accept cached cached data Q_ASSERT_X(!entities.testFlag(CEntityFlags::InfoObjectEntity), Q_FUNC_INFO, "Read info objects directly"); - const bool hasInfoObjects = this->hasInfoObjects(); // no info objects is no necessarily error, but indicates a) either data not available (DB down) or b) only caches are used + const bool hasInfoObjects = this->hasInfoObjects(); // no info objects is not necessarily error, but indicates a) either data not available (DB down) or b) only caches are used CEntityFlags::Entity allEntities = entities; CEntityFlags::Entity cachedEntities = CEntityFlags::NoEntity; + CEntityFlags::Entity dbEntities = CEntityFlags::NoEntity; CEntityFlags::Entity currentEntity = CEntityFlags::iterateDbEntities(allEntities); // CEntityFlags::InfoObjectEntity will be ignored while (currentEntity) { @@ -61,7 +64,7 @@ namespace BlackCore Q_ASSERT_X(!rm.testFlag(CDbFlags::Unspecified), Q_FUNC_INFO, "Missing retrieval mode"); if (rm.testFlag(CDbFlags::Ignore) || rm.testFlag(CDbFlags::Canceled)) { - entities &= ~currentEntity; // do not load from web database + // do not load } else if (rm.testFlag(CDbFlags::Cached)) { @@ -76,12 +79,12 @@ namespace BlackCore if (!changedUrl && cacheTimestamp >= latestEntityTimestamp && cacheTimestamp >= 0 && latestEntityTimestamp >= 0) { this->admitCaches(currentEntity); - entities &= ~currentEntity; // do not load from web database cachedEntities |= currentEntity; // read from cache CLogMessage(this).info("Using cache for %1 (%2, %3)") << currentEntityName << cacheTs.toString() << cacheTimestamp; } else { + dbEntities |= currentEntity; if (changedUrl) { CLogMessage(this).info("Data location changed, will override cache for %1") << currentEntityName; @@ -94,11 +97,9 @@ namespace BlackCore } else { - // no info objects, server down + // no info objects, server down or no info objects loaded this->admitCaches(currentEntity); - const int c = this->getCacheCount(currentEntity); - CLogMessage(this).info("No info object for %1, using cache with %2 objects") << currentEntityName << c; - entities &= ~currentEntity; // do not load from web database + CLogMessage(this).info("No info object for %1, triggered reading cache") << currentEntityName; cachedEntities |= currentEntity; // read from cache } } @@ -109,11 +110,47 @@ namespace BlackCore this->emitReadSignalPerSingleCachedEntity(cachedEntities, true); // Real read from DB - this->startReadFromDbInBackgroundThread(entities, newerThan); + this->startReadFromBackendInBackgroundThread(dbEntities, CDbFlags::DbReading, newerThan); } - void CDatabaseReader::startReadFromDbInBackgroundThread(CEntityFlags::Entity entities, const QDateTime &newerThan) + CEntityFlags::Entity CDatabaseReader::triggerLoadingDirectlyFromDb(CEntityFlags::Entity entities, const QDateTime &newerThan) { + this->startReadFromBackendInBackgroundThread(entities, CDbFlags::DbReading, newerThan); + return entities; + } + + CEntityFlags::Entity CDatabaseReader::triggerLoadingDirectlyFromSharedFiles(CEntityFlags::Entity entities, bool checkCacheTsUpfront) + { + if (entities == CEntityFlags::NoEntity) { return CEntityFlags::NoEntity; } + if (checkCacheTsUpfront) + { + CEntityFlags::Entity newerHeaderEntities = this->getEntitesWithNewerHeaderTimestamp(entities); + if (newerHeaderEntities != entities) + { + const CEntityFlags::Entity validInCacheEntities = (entities ^ newerHeaderEntities) & entities; + CLogMessage(this).info("Reduced '%1' to '%2' before triggering load of shared files (still in cache)") << CEntityFlags::flagToString(entities) << CEntityFlags::flagToString(newerHeaderEntities); + + // In case we have difference entities we treat them as they were loaded, they result from still being in the cache + // Using timer to first finish this function, then the resulting signal + if (validInCacheEntities != CEntityFlags::NoEntity) + { + QTimer::singleShot(10, this, [ = ] + { + emit this->dataRead(validInCacheEntities, CEntityFlags::ReadFinished, 0); + }); + } + if (newerHeaderEntities == CEntityFlags::NoEntity) { return CEntityFlags::NoEntity; } + entities = newerHeaderEntities; + } + } + this->startReadFromBackendInBackgroundThread(entities, CDbFlags::Shared); + return entities; + } + + void CDatabaseReader::startReadFromBackendInBackgroundThread(CEntityFlags::Entity entities, CDbFlags::DataRetrievalModeFlag mode, const QDateTime &newerThan) + { + Q_ASSERT_X(mode == CDbFlags::DbReading || mode == CDbFlags::Shared, Q_FUNC_INFO, "Wrong mode"); + // ps_read is implemented in the derived classes if (entities == CEntityFlags::NoEntity) { return; } if (!this->isNetworkAvailable()) @@ -124,6 +161,7 @@ namespace BlackCore const bool s = QMetaObject::invokeMethod(this, "ps_read", Q_ARG(BlackMisc::Network::CEntityFlags::Entity, entities), + Q_ARG(BlackMisc::Db::CDbFlags::DataRetrievalModeFlag, mode), Q_ARG(QDateTime, newerThan)); Q_ASSERT_X(s, Q_FUNC_INFO, "Invoke failed"); Q_UNUSED(s); @@ -131,6 +169,7 @@ namespace BlackCore CDatabaseReader::JsonDatastoreResponse CDatabaseReader::transformReplyIntoDatastoreResponse(QNetworkReply *nwReply) const { + Q_ASSERT_X(nwReply, Q_FUNC_INFO, "missing reply"); JsonDatastoreResponse datastoreResponse; const bool ok = this->setHeaderInfoPart(datastoreResponse, nwReply); if (ok) @@ -140,7 +179,6 @@ namespace BlackCore if (dataFileData.isEmpty()) { datastoreResponse.setMessage(CStatusMessage(this, CStatusMessage::SeverityError, "Empty response, no data")); - datastoreResponse.setUpdateTimestamp(QDateTime::currentDateTimeUtc()); } else { @@ -155,7 +193,6 @@ namespace BlackCore HeaderResponse headerResponse; const bool success = this->setHeaderInfoPart(headerResponse, nwReply); Q_UNUSED(success); - nwReply->close(); return headerResponse; } @@ -181,7 +218,7 @@ namespace BlackCore const QDateTime lastModified = nwReply->header(QNetworkRequest::LastModifiedHeader).toDateTime(); const qulonglong size = nwReply->header(QNetworkRequest::ContentLengthHeader).toULongLong(); - headerResponse.setUpdateTimestamp(lastModified); + headerResponse.setLastModifiedTimestamp(lastModified); headerResponse.setContentLengthHeader(size); if (nwReply->error() == QNetworkReply::NoError) @@ -204,7 +241,12 @@ namespace BlackCore CDatabaseReader::JsonDatastoreResponse CDatabaseReader::setStatusAndTransformReplyIntoDatastoreResponse(QNetworkReply *nwReply) { this->setReplyStatus(nwReply); - return this->transformReplyIntoDatastoreResponse(nwReply); + const CDatabaseReader::JsonDatastoreResponse dsr = this->transformReplyIntoDatastoreResponse(nwReply); + if (dsr.isSharedFile()) + { + this->receivedSharedFileHeaderNonClosing(nwReply); + } + return dsr; } CDbInfoList CDatabaseReader::infoList() const @@ -220,8 +262,27 @@ namespace BlackCore return infoList().size() > 0; } + bool CDatabaseReader::hasSharedFileHeader(const CEntityFlags::Entity entity) const + { + Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "need single entity"); + return m_sharedFileResponses.contains(entity); + } + + bool CDatabaseReader::hasSharedFileHeaders(const CEntityFlags::Entity entities) const + { + CEntityFlags::Entity myEntities = maskBySupportedEntities(entities); + CEntityFlags::Entity currentEntity = CEntityFlags::iterateDbEntities(myEntities); + while (currentEntity != CEntityFlags::NoEntity) + { + if (!hasSharedFileHeader(currentEntity)) { return false; } + currentEntity = CEntityFlags::iterateDbEntities(myEntities); + } + return true; + } + QDateTime CDatabaseReader::getLatestEntityTimestampFromInfoObjects(CEntityFlags::Entity entity) const { + Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "need single entity"); static const QDateTime e; const CDbInfoList il(infoList()); if (il.isEmpty() || entity == CEntityFlags::NoEntity) { return e; } @@ -233,31 +294,32 @@ namespace BlackCore return info.getUtcTimestamp(); } - QDateTime CDatabaseReader::getSharedFileTimestamp(CEntityFlags::Entity entity) const + QDateTime CDatabaseReader::getLatestSharedFileHeaderTimestamp(CEntityFlags::Entity entity) const { - const int key = static_cast(entity); - if (m_sharedFileResponses.contains(key)) - { - return this->m_sharedFileResponses[key].getUpdateTimestamp(); - } - else - { - return QDateTime(); - } + Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "need single entity"); + static const QDateTime e; + if (!this->hasSharedFileHeader(entity)) { return e; } + return m_sharedFileResponses[entity].getLastModifiedTimestamp(); } bool CDatabaseReader::requestHeadersOfSharedFiles(const CEntityFlags::Entity &entities) { - CEntityFlags::Entity allEntities(entities); + if (!this->isNetworkAvailable()) + { + CLogMessage(this).warning("No network, will not read shared file headers for %1") << CEntityFlags::flagToString(entities); + return false; + } + + CEntityFlags::Entity allEntities(this->maskBySupportedEntities(entities)); CEntityFlags::Entity currentEntity = CEntityFlags::iterateDbEntities(allEntities); - CUrl urlDbData = CGlobalSetup::buildDbDataDirectory(getWorkingSharedUrl()); + const CUrl urlSharedData = CGlobalSetup::buildDbDataDirectory(getWorkingDbDataFileLocationUrl()); int c = 0; while (currentEntity != CEntityFlags::NoEntity) { - const QString fileName = CDbInfo::entityToSharedFileName(currentEntity); + const QString fileName = CDbInfo::entityToSharedName(currentEntity); Q_ASSERT_X(!fileName.isEmpty(), Q_FUNC_INFO, "No file name for entity"); - CUrl url = urlDbData; + CUrl url = urlSharedData; url.appendPath(fileName); const QString entityString = CEntityFlags::flagToString(currentEntity); @@ -269,6 +331,33 @@ namespace BlackCore return c > 0; } + bool CDatabaseReader::isSharedHeaderNewerThanCacheTimestamp(CEntityFlags::Entity entity) const + { + Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "need single entity"); + const QDateTime cacheTs(this->getCacheTimestamp(entity)); + if (!cacheTs.isValid()) { return true; } // we have no cache ts + + const QDateTime hts(this->getLatestSharedFileHeaderTimestamp(entity)); + if (!hts.isValid()) { return false; } + return hts > cacheTs; + } + + CEntityFlags::Entity CDatabaseReader::getEntitesWithNewerHeaderTimestamp(CEntityFlags::Entity entities) const + { + entities = this->maskBySupportedEntities(entities); // handled by this reader + CEntityFlags::Entity currentEntity = CEntityFlags::iterateDbEntities(entities); + CEntityFlags::Entity newerEntities = CEntityFlags::NoEntity; + while (currentEntity != CEntityFlags::NoEntity) + { + if (this->isSharedHeaderNewerThanCacheTimestamp(currentEntity)) + { + newerEntities |= currentEntity; + } + currentEntity = CEntityFlags::iterateDbEntities(entities); + } + return newerEntities; + } + int CDatabaseReader::getCountFromInfoObjects(CEntityFlags::Entity entity) const { static const QDateTime e; @@ -303,14 +392,31 @@ namespace BlackCore return emitted; } + CUrl CDatabaseReader::getBaseUrl(CDbFlags::DataRetrievalModeFlag mode) const + { + Q_ASSERT_X(sApp, Q_FUNC_INFO, "Missing app object"); + switch (mode) + { + case CDbFlags::DbReading: + return getDbServiceBaseUrl().withAppendedPath("/service"); + case CDbFlags::SharedHeadersOnly: + case CDbFlags::Shared: + return CDatabaseReader::getWorkingDbDataFileLocationUrl(); + default: + qFatal("Wrong mode"); + break; + } + return CUrl(); + } + bool CDatabaseReader::isChangedUrl(const CUrl &oldUrl, const CUrl ¤tUrl) { if (oldUrl.isEmpty()) { return true; } Q_ASSERT_X(!currentUrl.isEmpty(), Q_FUNC_INFO, "No base URL"); - const QString oldS(oldUrl.getFullUrl(false)); - const QString currentS(currentUrl.getFullUrl(false)); - return oldS != currentS; + const QString old(oldUrl.getFullUrl(false)); + const QString current(currentUrl.getFullUrl(false)); + return old != current; } void CDatabaseReader::receivedSharedFileHeader(QNetworkReply *nwReplyPtr) @@ -319,13 +425,20 @@ namespace BlackCore // required to use delete later as object is created in a different thread QScopedPointer nwReply(nwReplyPtr); if (this->isAbandoned()) { return; } + this->receivedSharedFileHeaderNonClosing(nwReplyPtr); + nwReply->close(); + } - const HeaderResponse headerResponse = this->transformReplyIntoHeaderResponse(nwReply.data()); + void CDatabaseReader::receivedSharedFileHeaderNonClosing(QNetworkReply *nwReply) + { + if (this->isAbandoned()) { return; } + + const HeaderResponse headerResponse = this->transformReplyIntoHeaderResponse(nwReply); const QString fileName = nwReply->url().fileName(); const CEntityFlags::Entity entity = CEntityFlags::singleEntityByName(fileName); - const int key = static_cast(entity); - this->m_sharedFileResponses[key] = headerResponse; + this->m_sharedFileResponses[entity] = headerResponse; + CLogMessage(this).info("Received header for shared file of '%1' from '%2'") << fileName << headerResponse.getUrl().toQString(); emit this->sharedFileHeaderRead(entity, fileName, !headerResponse.hasWarningOrAboveMessage()); } @@ -348,6 +461,16 @@ namespace BlackCore return m_1stReplyReceived; } + CEntityFlags::Entity CDatabaseReader::maskBySupportedEntities(CEntityFlags::Entity entities) const + { + return entities & getSupportedEntities(); + } + + bool CDatabaseReader::supportsAnyOfEntities(CEntityFlags::Entity entities) const + { + return static_cast(maskBySupportedEntities(entities)) > 0; + } + const QString &CDatabaseReader::getStatusMessage() const { return this->m_statusMessage; @@ -370,6 +493,20 @@ namespace BlackCore } } + QString CDatabaseReader::fileNameForMode(CEntityFlags::Entity entity, CDbFlags::DataRetrievalModeFlag mode) + { + Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "needs single entity"); + switch (mode) + { + case CDbFlags::Shared: + case CDbFlags::SharedHeadersOnly: + return CDbInfo::entityToSharedName(entity); + default: + case CDbFlags::DbReading: + return CDbInfo::entityToServiceName(entity); + } + } + const CLogCategoryList &CDatabaseReader::getLogCategories() { static const BlackMisc::CLogCategoryList cats @@ -397,9 +534,9 @@ namespace BlackCore return dbUrl; } - CUrl CDatabaseReader::getWorkingSharedUrl() + CUrl CDatabaseReader::getWorkingDbDataFileLocationUrl() { - return sApp->getGlobalSetup().getSwiftSharedUrls().getRandomWorkingUrl(); + return CDatabaseReader::s_workingSharedDbData; } void CDatabaseReader::cacheHasChanged(CEntityFlags::Entity entities) @@ -413,12 +550,23 @@ namespace BlackCore return CNetworkUtils::canConnect(url); } + bool CDatabaseReader::initWorkingUrls(bool force) + { + if (!force && !CDatabaseReader::s_workingSharedDbData.isEmpty()) { return false; } + CDatabaseReader::s_workingSharedDbData = sApp->getGlobalSetup().getSwiftDbDataFileLocationUrls().getRandomWorkingUrl(); + return !CDatabaseReader::s_workingSharedDbData.isEmpty(); + } + + CUrl CDatabaseReader::getCurrentSharedDbDataUrl() + { + return CDatabaseReader::s_workingSharedDbData; + } + void CDatabaseReader::stringToDatastoreResponse(const QString &jsonContent, JsonDatastoreResponse &datastoreResponse) { if (jsonContent.isEmpty()) { datastoreResponse.setMessage(CStatusMessage(getLogCategories(), CStatusMessage::SeverityError, "Empty string, no data")); - datastoreResponse.setUpdateTimestamp(QDateTime::currentDateTimeUtc()); return; } @@ -427,14 +575,14 @@ namespace BlackCore { // directly an array, no further info datastoreResponse.setJsonArray(jsonResponse.array()); - datastoreResponse.setUpdateTimestamp(QDateTime::currentDateTimeUtc()); + datastoreResponse.setLastModifiedTimestamp(QDateTime::currentDateTimeUtc()); } else { const QJsonObject responseObject(jsonResponse.object()); datastoreResponse.setJsonArray(responseObject["data"].toArray()); const QString ts(responseObject["latest"].toString()); - datastoreResponse.setUpdateTimestamp(ts.isEmpty() ? QDateTime::currentDateTimeUtc() : CDatastoreUtility::parseTimestamp(ts)); + datastoreResponse.setLastModifiedTimestamp(ts.isEmpty() ? QDateTime::currentDateTimeUtc() : CDatastoreUtility::parseTimestamp(ts)); datastoreResponse.setRestricted(responseObject["restricted"].toBool()); } } @@ -444,5 +592,11 @@ namespace BlackCore m_jsonArray = value; m_arraySize = value.size(); } + + bool CDatabaseReader::HeaderResponse::isSharedFile() const + { + const QString fn(getUrl().getFileName()); + return CDbInfo::sharedFileNames().contains(fn, Qt::CaseInsensitive); + } } // ns } // ns diff --git a/src/blackcore/db/databasereader.h b/src/blackcore/db/databasereader.h index 75cdf7e79..cd3d95272 100644 --- a/src/blackcore/db/databasereader.h +++ b/src/blackcore/db/databasereader.h @@ -49,27 +49,27 @@ namespace BlackCore struct HeaderResponse { private: - QDateTime m_updated; //!< when was the latest update? + QDateTime m_lastModified; //!< when was the latest update? + qulonglong m_contentLengthHeader = 0; //!< content length + qint64 m_loadTimeMs = -1; //!< how long did it take to load BlackMisc::CStatusMessage m_message; //!< last error or warning BlackMisc::Network::CUrl m_url; //!< loaded url - qulonglong m_contentLengthHeader = 0; //!< content length - qint64 m_loadTimeMs = -1; //!< how long did it take to load public: //! Any timestamp? - bool hasTimestamp() const { return m_updated.isValid(); } + bool hasTimestamp() const { return m_lastModified.isValid(); } //! Is response newer? - bool isNewer(const QDateTime &ts) const { return m_updated.toMSecsSinceEpoch() > ts.toMSecsSinceEpoch(); } + bool isNewer(const QDateTime &ts) const { return m_lastModified.toMSecsSinceEpoch() > ts.toMSecsSinceEpoch(); } //! Is response newer? - bool isNewer(qint64 mSecsSinceEpoch) const { return m_updated.toMSecsSinceEpoch() > mSecsSinceEpoch; } + bool isNewer(qint64 mSecsSinceEpoch) const { return m_lastModified.toMSecsSinceEpoch() > mSecsSinceEpoch; } - //! Get the update timestamp - const QDateTime &getUpdateTimestamp() const { return m_updated; } + //! Get the "last-modified" timestamp + const QDateTime &getLastModifiedTimestamp() const { return m_lastModified; } //! Set update timestamp, default normally "last-modified" - void setUpdateTimestamp(const QDateTime &updated) { m_updated = updated; } + void setLastModifiedTimestamp(const QDateTime &updated) { m_lastModified = updated; } //! Header content length qulonglong getContentLengthHeader() const { return m_contentLengthHeader; } @@ -95,6 +95,9 @@ namespace BlackCore //! Set the loaded URL void setUrl(const BlackMisc::Network::CUrl &url) { m_url = url; } + //! Is a shared file? + bool isSharedFile() const; + //! Load time in ms (from request to response) qint64 getLoadTimeMs() const { return m_loadTimeMs; } @@ -106,9 +109,9 @@ namespace BlackCore struct JsonDatastoreResponse : public HeaderResponse { private: - QJsonArray m_jsonArray; //!< JSON array data - int m_arraySize = -1; //!< size of array, if applicable (copied to member for debugging purposes) - bool m_restricted = false; //!< restricted reponse, only changed data + QJsonArray m_jsonArray; //!< JSON array data + int m_arraySize = -1; //!< size of array, if applicable (copied to member for debugging purposes) + bool m_restricted = false; //!< restricted reponse, only changed data public: //! Any data? @@ -120,7 +123,7 @@ namespace BlackCore //! Incremental data, restricted by query? bool isRestricted() const { return m_restricted; } - //! Incremental data, restricted by query + //! Mark as restricted void setRestricted(bool restricted) { m_restricted = restricted; } //! Get the JSON array @@ -134,10 +137,16 @@ namespace BlackCore }; //! Start reading in own thread + //! \remark uses caches, info objects void readInBackgroundThread(BlackMisc::Network::CEntityFlags::Entity entities, const QDateTime &newerThan); - //! Start reading in own thread (without config/caching) - void startReadFromDbInBackgroundThread(BlackMisc::Network::CEntityFlags::Entity entities, const QDateTime &newerThan); + //! Start loading from DB in own thread + //! \remark bypass caches/config + BlackMisc::Network::CEntityFlags::Entity triggerLoadingDirectlyFromDb(BlackMisc::Network::CEntityFlags::Entity entities, const QDateTime &newerThan); + + //! Start loading from shared files in own thread + //! \remark bypass caches/config + BlackMisc::Network::CEntityFlags::Entity triggerLoadingDirectlyFromSharedFiles(BlackMisc::Network::CEntityFlags::Entity entities, bool checkCacheTsUpfront); //! Has received Ok response from server at least once? //! \threadsafe @@ -152,8 +161,17 @@ namespace BlackCore //! \threadsafe bool hasReceivedFirstReply() const; + //! Supported entities by this reader + virtual BlackMisc::Network::CEntityFlags::Entity getSupportedEntities() const = 0; + + //! Mask by supported entities + BlackMisc::Network::CEntityFlags::Entity maskBySupportedEntities(BlackMisc::Network::CEntityFlags::Entity entities) const; + + //! Is any of the given entities supported here by this reader + bool supportsAnyOfEntities(BlackMisc::Network::CEntityFlags::Entity entities) const; + //! Get cache timestamp - virtual QDateTime getCacheTimestamp(BlackMisc::Network::CEntityFlags::Entity entities) const = 0; + virtual QDateTime getCacheTimestamp(BlackMisc::Network::CEntityFlags::Entity entity) const = 0; //! Cache`s number of entities virtual int getCacheCount(BlackMisc::Network::CEntityFlags::Entity entity) const = 0; @@ -161,12 +179,24 @@ namespace BlackCore //! Info objects available? bool hasInfoObjects() const; + //! Header of shared file read (for single entity)? + bool hasSharedFileHeader(const BlackMisc::Network::CEntityFlags::Entity entity) const; + + //! Headers of shared file read (for single entity)? + bool hasSharedFileHeaders(const BlackMisc::Network::CEntityFlags::Entity entities) const; + //! Obtain latest object timestamp from info objects //! \sa BlackCore::Db::CInfoDataReader QDateTime getLatestEntityTimestampFromInfoObjects(BlackMisc::Network::CEntityFlags::Entity entity) const; - //! Timestamp of shared file for entiry - QDateTime getSharedFileTimestamp(BlackMisc::Network::CEntityFlags::Entity entity) const; + //! Header timestamp (last-modified) for shared file + QDateTime getLatestSharedFileHeaderTimestamp(BlackMisc::Network::CEntityFlags::Entity entity) const; + + //! Is the file timestamp neer than cache timestamp? + bool isSharedHeaderNewerThanCacheTimestamp(BlackMisc::Network::CEntityFlags::Entity entity) const; + + //! Those entities where the timestamp of header is newer than the cache timestamp + BlackMisc::Network::CEntityFlags::Entity getEntitesWithNewerHeaderTimestamp(BlackMisc::Network::CEntityFlags::Entity entities) const; //! Request header of shared file bool requestHeadersOfSharedFiles(const BlackMisc::Network::CEntityFlags::Entity &entities); @@ -190,24 +220,30 @@ namespace BlackCore //! swift DB server reachable? static bool canPingSwiftServer(); + //! Init the working URLs + static bool initWorkingUrls(bool force = false); + + //! Currently used URL for shared DB data + static BlackMisc::Network::CUrl getCurrentSharedDbDataUrl(); + //! Transform JSON data to response struct data - //! \private used also for samples + //! \private used also for samples, that`s why it is declared public static void stringToDatastoreResponse(const QString &jsonContent, CDatabaseReader::JsonDatastoreResponse &datastoreResponse); signals: //! Combined read signal - void dataRead(BlackMisc::Network::CEntityFlags::Entity entity, BlackMisc::Network::CEntityFlags::ReadState state, int number); + void dataRead(BlackMisc::Network::CEntityFlags::Entity entities, BlackMisc::Network::CEntityFlags::ReadState state, int number); //! Header of shared file read void sharedFileHeaderRead(BlackMisc::Network::CEntityFlags::Entity entity, const QString &fileName, bool success); protected: - CDatabaseReaderConfigList m_config; //!< DB reder configuration - QMap m_sharedFileResponses; //!< file responses of the shared files - QString m_statusMessage; //!< Returned status message from watchdog - QNetworkReply::NetworkError m_1stReplyStatus = QNetworkReply::UnknownServerError; //!< Successful connection? - bool m_1stReplyReceived = false; //!< Successful connection? Does not mean data / authorizations are correct - mutable QReadWriteLock m_statusLock; //!< Lock + CDatabaseReaderConfigList m_config; //!< DB reder configuration + QString m_statusMessage; //!< Returned status message from watchdog + bool m_1stReplyReceived = false; //!< Successful connection? Does not mean data / authorizations are correct + mutable QReadWriteLock m_statusLock; //!< Lock + QNetworkReply::NetworkError m_1stReplyStatus = QNetworkReply::UnknownServerError; //!< Successful connection? + QMap m_sharedFileResponses; //!< file responses of the shared files //! Constructor CDatabaseReader(QObject *owner, const CDatabaseReaderConfigList &config, const QString &name); @@ -225,11 +261,21 @@ namespace BlackCore //! Split into single entity and send dataRead signal BlackMisc::Network::CEntityFlags::Entity emitReadSignalPerSingleCachedEntity(BlackMisc::Network::CEntityFlags::Entity cachedEntities, bool onlyIfHasData); + //! Get the service URL, individual for each reader + virtual BlackMisc::Network::CUrl getDbServiceBaseUrl() const = 0; + + //! Base URL + BlackMisc::Network::CUrl getBaseUrl(BlackMisc::Db::CDbFlags::DataRetrievalModeFlag mode) const; + //! DB base URL static const BlackMisc::Network::CUrl &getDbUrl(); - //! Obtain a working shared URL - static BlackMisc::Network::CUrl getWorkingSharedUrl(); + //! Get the working shared URL, initialized by CDatabaseReader::initWorkingUrls + //! \remark normally constant after startup phase + static BlackMisc::Network::CUrl getWorkingDbDataFileLocationUrl(); + + //! File appendix for given mode, such as ".php" or ".json" + static QString fileNameForMode(BlackMisc::Network::CEntityFlags::Entity entity, BlackMisc::Db::CDbFlags::DataRetrievalModeFlag mode); //! \name Cache access //! @{ @@ -253,9 +299,18 @@ namespace BlackCore //! @} private: + static BlackMisc::Network::CUrl s_workingSharedDbData; //!< one chhosen URL for all DB reader objects + + //! Start reading in own thread (without config/caching) + //! \remarks can handle DB or shared file reads + void startReadFromBackendInBackgroundThread(BlackMisc::Network::CEntityFlags::Entity entities, BlackMisc::Db::CDbFlags::DataRetrievalModeFlag mode, const QDateTime &newerThan = QDateTime()); + //! Received a reply of a header for a shared file void receivedSharedFileHeader(QNetworkReply *nwReplyPtr); + //! Received a reply of a header for a shared file + void receivedSharedFileHeaderNonClosing(QNetworkReply *nwReply); + //! Check if terminated or error, otherwise split into array of objects JsonDatastoreResponse transformReplyIntoDatastoreResponse(QNetworkReply *nwReply) const; diff --git a/src/blackcore/db/icaodatareader.cpp b/src/blackcore/db/icaodatareader.cpp index 5fdebd040..6a59e3d3f 100644 --- a/src/blackcore/db/icaodatareader.cpp +++ b/src/blackcore/db/icaodatareader.cpp @@ -30,6 +30,7 @@ #include using namespace BlackMisc; +using namespace BlackMisc::Db; using namespace BlackMisc::Aviation; using namespace BlackMisc::Network; using namespace BlackCore::Data; @@ -42,7 +43,7 @@ namespace BlackCore CDatabaseReader(owner, confg, "CIcaoDataReader") { // init to avoid threading issues - getBaseUrl(); + getBaseUrl(CDbFlags::DbReading); } CAircraftIcaoCodeList CIcaoDataReader::getAircraftIcaoCodes() const @@ -122,7 +123,7 @@ namespace BlackCore return this->getCountries().size(); } - void CIcaoDataReader::ps_read(BlackMisc::Network::CEntityFlags::Entity entities, const QDateTime &newerThan) + void CIcaoDataReader::ps_read(BlackMisc::Network::CEntityFlags::Entity entities, BlackMisc::Db::CDbFlags::DataRetrievalModeFlag mode, const QDateTime &newerThan) { this->threadAssertCheck(); // runs in background thread if (this->isAbandoned()) { return; } @@ -130,7 +131,7 @@ namespace BlackCore CEntityFlags::Entity entitiesTriggered = CEntityFlags::NoEntity; if (entities.testFlag(CEntityFlags::AircraftIcaoEntity)) { - CUrl url(getAircraftIcaoUrl()); + CUrl url(getAircraftIcaoUrl(mode)); if (!url.isEmpty()) { if (!newerThan.isNull()) { url.appendQuery("newer=" + newerThan.toString(Qt::ISODate)); } @@ -145,7 +146,7 @@ namespace BlackCore if (entities.testFlag(CEntityFlags::AirlineIcaoEntity)) { - CUrl url(getAirlineIcaoUrl()); + CUrl url(getAirlineIcaoUrl(mode)); if (!url.isEmpty()) { if (!newerThan.isNull()) { url.appendQuery("newer=" + newerThan.toString(Qt::ISODate)); } @@ -160,7 +161,7 @@ namespace BlackCore if (entities.testFlag(CEntityFlags::CountryEntity)) { - CUrl url(getCountryUrl()); + CUrl url(getCountryUrl(mode)); if (!url.isEmpty()) { if (!newerThan.isNull()) { url.appendQuery("newer=" + newerThan.toString(Qt::ISODate)); } @@ -210,12 +211,6 @@ namespace BlackCore } } - const CUrl &CIcaoDataReader::getBaseUrl() - { - static const CUrl baseUrl(sApp->getGlobalSetup().getDbIcaoReaderUrl()); - return baseUrl; - } - void CIcaoDataReader::ps_parseAircraftIcaoData(QNetworkReply *nwReplyPtr) { // wrap pointer, make sure any exit cleans up reply @@ -243,7 +238,7 @@ namespace BlackCore } this->m_aircraftIcaoCache.set(codes, latestTimestamp); - this->updateReaderUrl(getBaseUrl()); + this->updateReaderUrl(getBaseUrl(CDbFlags::DbReading)); emit dataRead(CEntityFlags::AircraftIcaoEntity, CEntityFlags::ReadFinished, n); CLogMessage(this).info("Read %1 %2 from %3") << n << CEntityFlags::flagToString(CEntityFlags::AircraftIcaoEntity) << urlString; } @@ -271,7 +266,7 @@ namespace BlackCore } this->m_airlineIcaoCache.set(codes, latestTimestamp); - this->updateReaderUrl(getBaseUrl()); + this->updateReaderUrl(getBaseUrl(CDbFlags::DbReading)); emit dataRead(CEntityFlags::AirlineIcaoEntity, CEntityFlags::ReadFinished, n); CLogMessage(this).info("Read %1 %2 from %3") << n << CEntityFlags::flagToString(CEntityFlags::AirlineIcaoEntity) << urlString; } @@ -297,7 +292,7 @@ namespace BlackCore } this->m_countryCache.set(countries, latestTimestamp); - this->updateReaderUrl(getBaseUrl()); + this->updateReaderUrl(getBaseUrl(CDbFlags::DbReading)); emit dataRead(CEntityFlags::CountryEntity, CEntityFlags::ReadFinished, n); CLogMessage(this).info("Read %1 %2 from %3") << n << CEntityFlags::flagToString(CEntityFlags::CountryEntity) << urlString; } @@ -393,6 +388,11 @@ namespace BlackCore return true; } + CEntityFlags::Entity CIcaoDataReader::getSupportedEntities() const + { + return CEntityFlags::AllIcaoAndCountries; + } + void CIcaoDataReader::synchronizeCaches(CEntityFlags::Entity entities) { if (entities.testFlag(CEntityFlags::AircraftIcaoEntity)) { this->m_aircraftIcaoCache.synchronize(); } @@ -439,28 +439,27 @@ namespace BlackCore bool CIcaoDataReader::hasChangedUrl(CEntityFlags::Entity entity) const { Q_UNUSED(entity); - return CDatabaseReader::isChangedUrl(this->m_readerUrlCache.get(), getBaseUrl()); + return CDatabaseReader::isChangedUrl(this->m_readerUrlCache.get(), getBaseUrl(CDbFlags::DbReading)); } - CUrl CIcaoDataReader::getAircraftIcaoUrl(bool shared) const + CUrl CIcaoDataReader::getDbServiceBaseUrl() const { - return shared ? - getWorkingSharedUrl().withAppendedPath("jsonaircrafticao.php") : - getBaseUrl().withAppendedPath("service/jsonaircrafticao.php"); + return sApp->getGlobalSetup().getDbIcaoReaderUrl(); } - CUrl CIcaoDataReader::getAirlineIcaoUrl(bool shared) const + CUrl CIcaoDataReader::getAircraftIcaoUrl(CDbFlags::DataRetrievalModeFlag mode) const { - return shared ? - getWorkingSharedUrl().withAppendedPath("jsonairlineicao.php") : - getBaseUrl().withAppendedPath("service/jsonairlineicao.php"); + return getBaseUrl(mode).withAppendedPath(fileNameForMode(CEntityFlags::AircraftIcaoEntity, mode)); } - CUrl CIcaoDataReader::getCountryUrl(bool shared) const + CUrl CIcaoDataReader::getAirlineIcaoUrl(CDbFlags::DataRetrievalModeFlag mode) const { - return shared ? - getWorkingSharedUrl().withAppendedPath("jsoncountry.php") : - getBaseUrl().withAppendedPath("service/jsoncountry.php"); + return getBaseUrl(mode).withAppendedPath(fileNameForMode(CEntityFlags::AirlineIcaoEntity, mode)); + } + + CUrl CIcaoDataReader::getCountryUrl(CDbFlags::DataRetrievalModeFlag mode) const + { + return getBaseUrl(mode).withAppendedPath(fileNameForMode(CEntityFlags::CountryEntity, mode)); } } // ns } // ns diff --git a/src/blackcore/db/icaodatareader.h b/src/blackcore/db/icaodatareader.h index 47f184288..a8e40e8cd 100644 --- a/src/blackcore/db/icaodatareader.h +++ b/src/blackcore/db/icaodatareader.h @@ -115,6 +115,7 @@ namespace BlackCore bool writeToJsonFiles(const QString &dir) const; // cache handling for base class + virtual BlackMisc::Network::CEntityFlags::Entity getSupportedEntities() const override; virtual QDateTime getCacheTimestamp(BlackMisc::Network::CEntityFlags::Entity entity) const override; virtual int getCacheCount(BlackMisc::Network::CEntityFlags::Entity entity) const override; virtual void synchronizeCaches(BlackMisc::Network::CEntityFlags::Entity entities) override; @@ -124,6 +125,7 @@ namespace BlackCore // cache handling for base class virtual void invalidateCaches(BlackMisc::Network::CEntityFlags::Entity entities) override; virtual bool hasChangedUrl(BlackMisc::Network::CEntityFlags::Entity entity) const override; + virtual BlackMisc::Network::CUrl getDbServiceBaseUrl() const override; private slots: //! Aircraft have been read @@ -136,7 +138,8 @@ namespace BlackCore void ps_parseCountryData(QNetworkReply *nwReply); //! Read / re-read data file - void ps_read(BlackMisc::Network::CEntityFlags::Entity entities, const QDateTime &newerThan); + void ps_read(BlackMisc::Network::CEntityFlags::Entity entities, + BlackMisc::Db::CDbFlags::DataRetrievalModeFlag mode, const QDateTime &newerThan); void ps_aircraftIcaoCacheChanged(); void ps_airlineIcaoCacheChanged(); @@ -154,18 +157,14 @@ namespace BlackCore //! Update reader URL void updateReaderUrl(const BlackMisc::Network::CUrl &url); - //! Base URL - //! \threadsafe - static const BlackMisc::Network::CUrl &getBaseUrl(); + //! URL + BlackMisc::Network::CUrl getAircraftIcaoUrl(BlackMisc::Db::CDbFlags::DataRetrievalModeFlag mode) const; //! URL - BlackMisc::Network::CUrl getAircraftIcaoUrl(bool shared = false) const; + BlackMisc::Network::CUrl getAirlineIcaoUrl(BlackMisc::Db::CDbFlags::DataRetrievalModeFlag mode) const; //! URL - BlackMisc::Network::CUrl getAirlineIcaoUrl(bool shared = false) const; - - //! URL - BlackMisc::Network::CUrl getCountryUrl(bool shared = false) const; + BlackMisc::Network::CUrl getCountryUrl(BlackMisc::Db::CDbFlags::DataRetrievalModeFlag mode) const; }; } // ns } // ns diff --git a/src/blackcore/db/infodatareader.cpp b/src/blackcore/db/infodatareader.cpp index 094eacb2e..14f6f5812 100644 --- a/src/blackcore/db/infodatareader.cpp +++ b/src/blackcore/db/infodatareader.cpp @@ -28,10 +28,7 @@ namespace BlackCore { CInfoDataReader::CInfoDataReader(QObject *owner, const CDatabaseReaderConfigList &config) : CDatabaseReader(owner, config, "CInfoDataReader") - { - // init to avoid threading issues - getBaseUrl(); - } + { } CDbInfoList CInfoDataReader::getDbInfoObjects() const { @@ -89,13 +86,19 @@ namespace BlackCore return false; } - void CInfoDataReader::read(CEntityFlags::Entity entities, const QDateTime &newerThan) + CUrl CInfoDataReader::getDbServiceBaseUrl() const { - this->ps_read(entities, newerThan); + return sApp->getGlobalSetup().getDbInfoReaderUrl(); } - void CInfoDataReader::ps_read(CEntityFlags::Entity entities, const QDateTime &newerThan) + void CInfoDataReader::read(CEntityFlags::Entity entities, const QDateTime &newerThan) { + this->ps_read(entities, CDbFlags::DbReading, newerThan); + } + + void CInfoDataReader::ps_read(CEntityFlags::Entity entities, CDbFlags::DataRetrievalModeFlag mode, const QDateTime &newerThan) + { + Q_UNUSED(mode); CEntityFlags::Entity triggeredRead = CEntityFlags::NoEntity; CUrl url(getInfoObjectsUrl()); if (entities.testFlag(CEntityFlags::InfoObjectEntity)) @@ -154,15 +157,14 @@ namespace BlackCore CLogMessage(this).info("Read %1 %2 from %3") << n << CEntityFlags::flagToString(CEntityFlags::InfoObjectEntity) << urlString; } - const CUrl &CInfoDataReader::getBaseUrl() - { - static const CUrl baseUrl(sApp->getGlobalSetup().getDbInfoReaderUrl()); - return baseUrl; - } - CUrl CInfoDataReader::getInfoObjectsUrl() const { - return getBaseUrl().withAppendedPath("service/jsondbinfo.php"); + return getBaseUrl(CDbFlags::DbReading).withAppendedPath("jsondbinfo.php"); + } + + CEntityFlags::Entity CInfoDataReader::getSupportedEntities() const + { + return CEntityFlags::InfoObjectEntity; } } // namespace } // namespace diff --git a/src/blackcore/db/infodatareader.h b/src/blackcore/db/infodatareader.h index 5751aa608..250241769 100644 --- a/src/blackcore/db/infodatareader.h +++ b/src/blackcore/db/infodatareader.h @@ -49,6 +49,7 @@ namespace BlackCore BlackMisc::Network::CUrl getInfoObjectsUrl() const; // cache handling for base class + virtual BlackMisc::Network::CEntityFlags::Entity getSupportedEntities() const override; virtual QDateTime getCacheTimestamp(BlackMisc::Network::CEntityFlags::Entity entity) const override; virtual int getCacheCount(BlackMisc::Network::CEntityFlags::Entity entity) const override; virtual void synchronizeCaches(BlackMisc::Network::CEntityFlags::Entity entities) override; @@ -62,21 +63,20 @@ namespace BlackCore // cache handling for base class virtual void invalidateCaches(BlackMisc::Network::CEntityFlags::Entity entities) override; virtual bool hasChangedUrl(BlackMisc::Network::CEntityFlags::Entity entity) const override; + virtual BlackMisc::Network::CUrl getDbServiceBaseUrl() const override; private slots: //! Info objects have been read void ps_parseInfoObjectsData(QNetworkReply *nwReply); //! Read / re-read data file - void ps_read(BlackMisc::Network::CEntityFlags::Entity entities = BlackMisc::Network::CEntityFlags::InfoObjectEntity, const QDateTime &newerThan = QDateTime()); + void ps_read(BlackMisc::Network::CEntityFlags::Entity entities = BlackMisc::Network::CEntityFlags::InfoObjectEntity, + BlackMisc::Db::CDbFlags::DataRetrievalModeFlag mode = BlackMisc::Db::CDbFlags::DbReading, const QDateTime &newerThan = QDateTime()); private: BlackMisc::Db::CDbInfoList m_infoObjects; BlackMisc::Network::CUrl m_urlInfoObjects; mutable QReadWriteLock m_lockInfoObjects; - - //! Base URL - static const BlackMisc::Network::CUrl &getBaseUrl(); }; } // ns } // ns diff --git a/src/blackcore/db/modeldatareader.cpp b/src/blackcore/db/modeldatareader.cpp index 17f2c6dbc..91344cfd3 100644 --- a/src/blackcore/db/modeldatareader.cpp +++ b/src/blackcore/db/modeldatareader.cpp @@ -29,6 +29,7 @@ #include using namespace BlackMisc; +using namespace BlackMisc::Db; using namespace BlackMisc::Aviation; using namespace BlackMisc::Simulation; using namespace BlackMisc::Network; @@ -42,7 +43,7 @@ namespace BlackCore CDatabaseReader(owner, config, "CModelDataReader") { // init to avoid threading issues - getBaseUrl(); + getBaseUrl(CDbFlags::DbReading); } CLiveryList CModelDataReader::getLiveries() const @@ -146,7 +147,7 @@ namespace BlackCore getDistributorsCount() > 0; } - void CModelDataReader::ps_read(CEntityFlags::Entity entity, const QDateTime &newerThan) + void CModelDataReader::ps_read(CEntityFlags::Entity entity, CDbFlags::DataRetrievalModeFlag mode, const QDateTime &newerThan) { this->threadAssertCheck(); if (this->isAbandoned()) { return; } @@ -154,7 +155,7 @@ namespace BlackCore CEntityFlags::Entity triggeredRead = CEntityFlags::NoEntity; if (entity.testFlag(CEntityFlags::LiveryEntity)) { - CUrl url(getLiveryUrl()); + CUrl url(getLiveryUrl(mode)); if (!url.isEmpty()) { if (!newerThan.isNull()) @@ -173,7 +174,7 @@ namespace BlackCore if (entity.testFlag(CEntityFlags::DistributorEntity)) { - CUrl url(getDistributorUrl()); + CUrl url(getDistributorUrl(mode)); if (!url.isEmpty()) { if (!newerThan.isNull()) @@ -192,7 +193,7 @@ namespace BlackCore if (entity.testFlag(CEntityFlags::ModelEntity)) { - CUrl url(getModelUrl()); + CUrl url(getModelUrl(mode)); if (!url.isEmpty()) { if (!newerThan.isNull()) @@ -282,11 +283,11 @@ namespace BlackCore latestTimestamp = lastModifiedMsSinceEpoch(nwReply.data()); } this->m_liveryCache.set(liveries, latestTimestamp); - this->updateReaderUrl(getBaseUrl()); + this->updateReaderUrl(getBaseUrl(CDbFlags::DbReading)); // never emit when lock is held -> deadlock emit dataRead(CEntityFlags::LiveryEntity, res.isRestricted() ? CEntityFlags::ReadFinishedRestricted : CEntityFlags::ReadFinished, n); - CLogMessage(this).info("Read %1 %2 from %3") << n << CEntityFlags::flagToString(CEntityFlags::LiveryEntity) << urlString; + CLogMessage(this).info("Read '%1' '%2' from '%3'") << n << CEntityFlags::flagToString(CEntityFlags::ModelEntity) << urlString; } void CModelDataReader::ps_parseDistributorData(QNetworkReply *nwReplyPtr) @@ -295,7 +296,7 @@ namespace BlackCore // required to use delete later as object is created in a different thread QScopedPointer nwReply(nwReplyPtr); if (this->isAbandoned()) { return; } - QString urlString(nwReply->url().toString()); + const QString urlString(nwReply->url().toString()); CDatabaseReader::JsonDatastoreResponse res = this->setStatusAndTransformReplyIntoDatastoreResponse(nwReply.data()); if (res.hasErrorMessage()) { @@ -325,10 +326,10 @@ namespace BlackCore latestTimestamp = lastModifiedMsSinceEpoch(nwReply.data()); } this->m_distributorCache.set(distributors, latestTimestamp); - this->updateReaderUrl(getBaseUrl()); + this->updateReaderUrl(getBaseUrl(CDbFlags::DbReading)); emit dataRead(CEntityFlags::DistributorEntity, res.isRestricted() ? CEntityFlags::ReadFinishedRestricted : CEntityFlags::ReadFinished, n); - CLogMessage(this).info("Read %1 %2 from %3") << n << CEntityFlags::flagToString(CEntityFlags::DistributorEntity) << urlString; + CLogMessage(this).info("Read '%1' '%2' from '%3'") << n << CEntityFlags::flagToString(CEntityFlags::ModelEntity) << urlString; } void CModelDataReader::ps_parseModelData(QNetworkReply *nwReplyPtr) @@ -368,10 +369,10 @@ namespace BlackCore latestTimestamp = lastModifiedMsSinceEpoch(nwReply.data()); } this->m_modelCache.set(models, latestTimestamp); - this->updateReaderUrl(getBaseUrl()); + this->updateReaderUrl(getBaseUrl(CDbFlags::DbReading)); emit dataRead(CEntityFlags::ModelEntity, res.isRestricted() ? CEntityFlags::ReadFinishedRestricted : CEntityFlags::ReadFinished, n); - CLogMessage(this).info("Read %1 %2 from %3") << n << CEntityFlags::flagToString(CEntityFlags::ModelEntity) << urlString; + CLogMessage(this).info("Read '%1' '%2' from '%3'") << n << CEntityFlags::flagToString(CEntityFlags::ModelEntity) << urlString; } bool CModelDataReader::readFromJsonFiles(const QString &dir, CEntityFlags::Entity whatToRead) @@ -466,6 +467,11 @@ namespace BlackCore return true; } + CEntityFlags::Entity CModelDataReader::getSupportedEntities() const + { + return CEntityFlags::DistributorLiveryModel; + } + void CModelDataReader::synchronizeCaches(CEntityFlags::Entity entities) { if (entities.testFlag(CEntityFlags::LiveryEntity)) { this->m_liveryCache.synchronize(); } @@ -512,34 +518,27 @@ namespace BlackCore bool CModelDataReader::hasChangedUrl(CEntityFlags::Entity entity) const { Q_UNUSED(entity); - return CDatabaseReader::isChangedUrl(this->m_readerUrlCache.get(), getBaseUrl()); + return CDatabaseReader::isChangedUrl(this->m_readerUrlCache.get(), getBaseUrl(CDbFlags::DbReading)); } - const CUrl &CModelDataReader::getBaseUrl() + CUrl CModelDataReader::getDbServiceBaseUrl() const { - static const CUrl baseUrl(sApp->getGlobalSetup().getDbModelReaderUrl()); - return baseUrl; + return sApp->getGlobalSetup().getDbModelReaderUrl(); } - CUrl CModelDataReader::getLiveryUrl(bool shared) const + CUrl CModelDataReader::getLiveryUrl(CDbFlags::DataRetrievalModeFlag mode) const { - return shared ? - getBaseUrl().withAppendedPath("service/jsonlivery.php") : - getBaseUrl().withAppendedPath("service/jsonlivery.php"); + return getBaseUrl(mode).withAppendedPath(fileNameForMode(CEntityFlags::LiveryEntity, mode)); } - CUrl CModelDataReader::getDistributorUrl(bool shared) const + CUrl CModelDataReader::getDistributorUrl(CDbFlags::DataRetrievalModeFlag mode) const { - return shared ? - getBaseUrl().withAppendedPath("service/jsondistributor.php") : - getBaseUrl().withAppendedPath("service/jsondistributor.php"); + return getBaseUrl(mode).withAppendedPath(fileNameForMode(CEntityFlags::DistributorEntity, mode)); } - CUrl CModelDataReader::getModelUrl(bool shared) const + CUrl CModelDataReader::getModelUrl(CDbFlags::DataRetrievalModeFlag mode) const { - return shared ? - getBaseUrl().withAppendedPath("service/jsonaircraftmodel.php") : - getBaseUrl().withAppendedPath("service/jsonaircraftmodel.php"); + return getBaseUrl(mode).withAppendedPath(fileNameForMode(CEntityFlags::ModelEntity, mode)); } } // ns } // ns diff --git a/src/blackcore/db/modeldatareader.h b/src/blackcore/db/modeldatareader.h index b71809a17..c1ff2ed4a 100644 --- a/src/blackcore/db/modeldatareader.h +++ b/src/blackcore/db/modeldatareader.h @@ -125,6 +125,7 @@ namespace BlackCore bool writeToJsonFiles(const QString &dir) const; // cache handling for base class + virtual BlackMisc::Network::CEntityFlags::Entity getSupportedEntities() const override; virtual QDateTime getCacheTimestamp(BlackMisc::Network::CEntityFlags::Entity entity) const override; virtual int getCacheCount(BlackMisc::Network::CEntityFlags::Entity entity) const override; virtual void synchronizeCaches(BlackMisc::Network::CEntityFlags::Entity entities) override; @@ -134,7 +135,7 @@ namespace BlackCore // cache handling for base class virtual void invalidateCaches(BlackMisc::Network::CEntityFlags::Entity entities) override; virtual bool hasChangedUrl(BlackMisc::Network::CEntityFlags::Entity entity) const override; - + virtual BlackMisc::Network::CUrl getDbServiceBaseUrl() const override; private slots: //! Liveries have been read void ps_parseLiveryData(QNetworkReply *nwReply); @@ -146,7 +147,8 @@ namespace BlackCore void ps_parseModelData(QNetworkReply *nwReply); //! Read / re-read data file - void ps_read(BlackMisc::Network::CEntityFlags::Entity entity = BlackMisc::Network::CEntityFlags::DistributorLiveryModel, const QDateTime &newerThan = QDateTime()); + void ps_read(BlackMisc::Network::CEntityFlags::Entity entity = BlackMisc::Network::CEntityFlags::DistributorLiveryModel, + BlackMisc::Db::CDbFlags::DataRetrievalModeFlag mode = BlackMisc::Db::CDbFlags::DbReading, const QDateTime &newerThan = QDateTime()); void ps_liveryCacheChanged(); void ps_modelCacheChanged(); @@ -164,18 +166,14 @@ namespace BlackCore //! Update reader URL void updateReaderUrl(const BlackMisc::Network::CUrl &url); - //! Base URL - //! \threadsafe - static const BlackMisc::Network::CUrl &getBaseUrl(); - //! URL livery web service - BlackMisc::Network::CUrl getLiveryUrl(bool shared = false) const; + BlackMisc::Network::CUrl getLiveryUrl(BlackMisc::Db::CDbFlags::DataRetrievalModeFlag mode) const; //! URL distributor web service - BlackMisc::Network::CUrl getDistributorUrl(bool shared = false) const; + BlackMisc::Network::CUrl getDistributorUrl(BlackMisc::Db::CDbFlags::DataRetrievalModeFlag mode) const; //! URL model web service - BlackMisc::Network::CUrl getModelUrl(bool shared = false) const; + BlackMisc::Network::CUrl getModelUrl(BlackMisc::Db::CDbFlags::DataRetrievalModeFlag mode) const; }; } // ns } // ns