diff --git a/src/blackcore/db/airportdatareader.cpp b/src/blackcore/db/airportdatareader.cpp index 3a71184ae..e189aa287 100644 --- a/src/blackcore/db/airportdatareader.cpp +++ b/src/blackcore/db/airportdatareader.cpp @@ -12,7 +12,9 @@ #include "blackcore/application.h" #include "blackmisc/network/networkutils.h" #include "blackmisc/logmessage.h" + #include +#include using namespace BlackMisc; using namespace BlackMisc::Aviation; @@ -49,12 +51,12 @@ namespace BlackCore return this->getAirports().size(); } - bool CAirportDataReader::readFromJsonFilesInBackground(const QString &dir, CEntityFlags::Entity whatToRead) + bool CAirportDataReader::readFromJsonFilesInBackground(const QString &dir, CEntityFlags::Entity whatToRead, bool overrideNewerOnly) { if (dir.isEmpty() || whatToRead == CEntityFlags::NoEntity) { return false; } - QTimer::singleShot(0, this, [this, dir, whatToRead]() + QTimer::singleShot(0, this, [ = ]() { - const CStatusMessageList msgs = this->readFromJsonFiles(dir, whatToRead); + const CStatusMessageList msgs = this->readFromJsonFiles(dir, whatToRead, overrideNewerOnly); if (msgs.isFailure()) { CLogMessage::preformatted(msgs); @@ -63,13 +65,14 @@ namespace BlackCore return true; } - CStatusMessageList CAirportDataReader::readFromJsonFiles(const QString &dir, CEntityFlags::Entity whatToRead) + CStatusMessageList CAirportDataReader::readFromJsonFiles(const QString &dir, CEntityFlags::Entity whatToRead, bool overrideNewerOnly) { - const QString fileName = CFileUtils::appendFilePaths(dir, "airports.json"); - if (!QFile::exists(fileName)) + const QDir directory(dir); + if (!directory.exists()) { - return CStatusMessage(this).error("File '%1' does not exist") << fileName; + return CStatusMessage(this).error("Missing directory '%1'") << dir; } + whatToRead &= CEntityFlags::AirportEntity; // can handle these entities if (whatToRead == CEntityFlags::NoEntity) { @@ -78,33 +81,40 @@ namespace BlackCore int c = 0; CEntityFlags::Entity reallyRead = CEntityFlags::NoEntity; - const QJsonObject airportsJson(CDatabaseUtils::readQJsonObjectFromDatabaseFile(fileName)); - if (!airportsJson.isEmpty()) - { - try - { - CAirportList airports; - airports.convertFromJson(airportsJson); - c = airports.size(); - m_airportCache.set(airports); - emit dataRead(CEntityFlags::AirportEntity, CEntityFlags::ReadFinished, c); - reallyRead |= CEntityFlags::AirportEntity; - } - catch (const CJsonException &ex) - { - emit dataRead(CEntityFlags::AirportEntity, CEntityFlags::ReadFailed, 0); - return ex.toStatusMessage(this, QString("Reading airports from '%1'").arg(fileName)); - } - } - if ((reallyRead & CEntityFlags::AirportEntity) == CEntityFlags::AirportEntity) + CStatusMessageList msgs; + const QString fileName = CFileUtils::appendFilePaths(dir, "airports.json"); + const QFileInfo fi(fileName); + if (!fi.exists()) { - return CStatusMessage(this).info("Written %1 airports from '%2' to cache") << c << fileName; + msgs.push_back(CStatusMessage(this).warning("File '%1' does not exist") << fileName); + } + else if (!this->overrideCacheFromFile(overrideNewerOnly, fi, CEntityFlags::AirportEntity, msgs)) + { + // void } else { - return CStatusMessage(this).error("Not able to read airports from '%1'") << fileName; + const QJsonObject airportsJson(CDatabaseUtils::readQJsonObjectFromDatabaseFile(fileName)); + if (!airportsJson.isEmpty()) + { + try + { + const CAirportList airports = CAirportList::fromMultipleJsonFormats(airportsJson); + c = airports.size(); + msgs.push_back(m_airportCache.set(airports, fi.created().toUTC().toMSecsSinceEpoch())); + + emit dataRead(CEntityFlags::AirportEntity, CEntityFlags::ReadFinished, c); + reallyRead |= CEntityFlags::AirportEntity; + } + catch (const CJsonException &ex) + { + emit dataRead(CEntityFlags::AirportEntity, CEntityFlags::ReadFailed, 0); + return ex.toStatusMessage(this, QString("Reading airports from '%1'").arg(fileName)); + } + } } + return msgs; } CEntityFlags::Entity CAirportDataReader::getSupportedEntities() const @@ -166,7 +176,7 @@ namespace BlackCore CUrl CAirportDataReader::getAirportsUrl(CDbFlags::DataRetrievalModeFlag mode) const { - return getBaseUrl(mode).withAppendedPath(fileNameForMode(CEntityFlags::AirportEntity, mode)); + return this->getBaseUrl(mode).withAppendedPath(fileNameForMode(CEntityFlags::AirportEntity, mode)); } void CAirportDataReader::ps_parseAirportData(QNetworkReply *nwReplyPtr) diff --git a/src/blackcore/db/airportdatareader.h b/src/blackcore/db/airportdatareader.h index f7596edda..af4ba178b 100644 --- a/src/blackcore/db/airportdatareader.h +++ b/src/blackcore/db/airportdatareader.h @@ -49,8 +49,8 @@ namespace BlackCore int getAirportsCount() const; // data read from local data - virtual BlackMisc::CStatusMessageList readFromJsonFiles(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead) override; - virtual bool readFromJsonFilesInBackground(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead) override; + virtual BlackMisc::CStatusMessageList readFromJsonFiles(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead, bool overrideNewerOnly) override; + virtual bool readFromJsonFilesInBackground(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead, bool overrideNewerOnly) override; // base class overrides virtual BlackMisc::Network::CEntityFlags::Entity getSupportedEntities() const override; diff --git a/src/blackcore/db/databasereader.cpp b/src/blackcore/db/databasereader.cpp index f483d08ae..0fd558f37 100644 --- a/src/blackcore/db/databasereader.cpp +++ b/src/blackcore/db/databasereader.cpp @@ -15,6 +15,7 @@ #include "blackcore/application.h" #include "blackmisc/db/datastoreutility.h" #include "blackmisc/network/networkutils.h" +#include "blackmisc/network/entityflags.h" #include "blackmisc/directoryutils.h" #include "blackmisc/logcategory.h" #include "blackmisc/logcategorylist.h" @@ -23,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -568,6 +570,11 @@ namespace BlackCore return m_1stReplyReceived; } + QString CDatabaseReader::getSupportedEntitiesAsString() const + { + return CEntityFlags::flagToString(this->getSupportedEntities()); + } + CEntityFlags::Entity CDatabaseReader::maskBySupportedEntities(CEntityFlags::Entity entities) const { return entities & this->getSupportedEntities(); @@ -590,27 +597,20 @@ namespace BlackCore return m_statusMessage; } - CStatusMessageList CDatabaseReader::initFromLocalResourceFiles() + CStatusMessageList CDatabaseReader::initFromLocalResourceFiles(bool inBackground) { - static const QDateTime threshold = QDateTime::currentDateTimeUtc().addYears(-1); - const QSet eSet = CEntityFlags::asSingleEntities(this->getSupportedEntities()); - - CStatusMessageList msgs; - CEntityFlags::Entity entities = CEntityFlags::NoEntity; - for (CEntityFlags::Entity e : eSet) + const bool overrideNewerOnly = true; + if (inBackground || !CThreadUtils::isCurrentThreadObjectThread(this)) { - if (this->hasCacheTimestampNewerThan(e, threshold)) - { - entities |= e; - } - { - msgs.push_back(CStatusMessage(this).info("Will not init from local file, there are already data for '%1'") << CEntityFlags::flagToString(e)); - } + const bool s = this->readFromJsonFilesInBackground(CDirectoryUtils::staticDbFilesDirectory(), this->getSupportedEntities(), overrideNewerOnly); + return s ? + CStatusMessage(this).info("Started reading in background from '%1' of entities: '%2'") << CDirectoryUtils::staticDbFilesDirectory() << CEntityFlags::flagToString(this->getSupportedEntities()) : + CStatusMessage(this).error("Starting reading in background from '%1' of entities: '%2' failed") << CDirectoryUtils::staticDbFilesDirectory() << CEntityFlags::flagToString(this->getSupportedEntities()); + } + else + { + return this->readFromJsonFiles(CDirectoryUtils::staticDbFilesDirectory(), this->getSupportedEntities(), overrideNewerOnly); } - - const CStatusMessageList readMsgs = this->readFromJsonFiles(CDirectoryUtils::staticDbFilesDirectory(), entities); - msgs.push_back(readMsgs); - return readMsgs; } void CDatabaseReader::setReplyStatus(QNetworkReply::NetworkError status, const QString &message) @@ -631,6 +631,27 @@ namespace BlackCore } } + bool CDatabaseReader::overrideCacheFromFile(bool overrideNewerOnly, const QFileInfo &fileInfo, CEntityFlags::Entity entity, CStatusMessageList &msgs) const + { + if (!fileInfo.created().isValid()) { return false; } + if (!overrideNewerOnly) { return true; } + + const qint64 fileTs = fileInfo.created().toUTC().toMSecsSinceEpoch(); + const QDateTime cacheDateTime(this->getCacheTimestamp(entity)); + if (!cacheDateTime.isValid()) { return true; } // no cache + const qint64 cacheTs = cacheDateTime.toUTC().toMSecsSinceEpoch(); + if (fileTs > cacheTs) + { + msgs.push_back(CStatusMessage(this).info("File '%1' is newer than cache (%2)") << fileInfo.absoluteFilePath() << cacheDateTime.toUTC().toString()); + return true; + } + else + { + msgs.push_back(CStatusMessage(this).info("File '%1' is not newer than cache (%2)") << fileInfo.absoluteFilePath() << cacheDateTime.toUTC().toString()); + return true; + } + } + QString CDatabaseReader::fileNameForMode(CEntityFlags::Entity entity, CDbFlags::DataRetrievalModeFlag mode) { Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "needs single entity"); diff --git a/src/blackcore/db/databasereader.h b/src/blackcore/db/databasereader.h index 18413f252..01b5839ad 100644 --- a/src/blackcore/db/databasereader.h +++ b/src/blackcore/db/databasereader.h @@ -32,6 +32,8 @@ #include class QNetworkReply; +class QFileInfo; + namespace BlackMisc { class CLogCategoryList; } namespace BlackCore { @@ -186,6 +188,9 @@ namespace BlackCore //! Supported entities by this reader virtual BlackMisc::Network::CEntityFlags::Entity getSupportedEntities() const = 0; + //! Supported entities as string + QString getSupportedEntitiesAsString() const; + //! Mask by supported entities BlackMisc::Network::CEntityFlags::Entity maskBySupportedEntities(BlackMisc::Network::CEntityFlags::Entity entities) const; @@ -259,13 +264,13 @@ namespace BlackCore //! Init from local resource file //! \remark normally used after installation for a 1st time init - BlackMisc::CStatusMessageList initFromLocalResourceFiles(); + BlackMisc::CStatusMessageList initFromLocalResourceFiles(bool inBackground); //! Data read from local data - virtual BlackMisc::CStatusMessageList readFromJsonFiles(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead) = 0; + virtual BlackMisc::CStatusMessageList readFromJsonFiles(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead, bool overrideNewer) = 0; //! Data read from local data - virtual bool readFromJsonFilesInBackground(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead) = 0; + virtual bool readFromJsonFilesInBackground(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead, bool overrideNewer) = 0; //! Log categories static const BlackMisc::CLogCategoryList &getLogCategories(); @@ -398,6 +403,10 @@ namespace BlackCore //! Feedback about connection status //! \threadsafe void setReplyStatus(QNetworkReply *nwReply); + + //! Override cache from file + //! \threadsafe + bool overrideCacheFromFile(bool overrideNewerOnly, const QFileInfo &fileInfo, BlackMisc::Network::CEntityFlags::Entity entity, BlackMisc::CStatusMessageList &msgs) const; }; } // ns } // ns diff --git a/src/blackcore/db/databaseutils.cpp b/src/blackcore/db/databaseutils.cpp index 2fafc6ba0..a2a83c09c 100644 --- a/src/blackcore/db/databaseutils.cpp +++ b/src/blackcore/db/databaseutils.cpp @@ -14,6 +14,7 @@ #include "blackmisc/fileutils.h" using namespace BlackMisc; +using namespace BlackMisc::Json; using namespace BlackMisc::Aviation; using namespace BlackMisc::Simulation; @@ -356,8 +357,8 @@ namespace BlackCore byteData = qUncompress(ba); } while (false); - } + if (byteData.isEmpty()) { return QJsonDocument(); } return QJsonDocument::fromJson(byteData); } @@ -373,7 +374,10 @@ namespace BlackCore { const QString raw = CFileUtils::readFileToString(filename); if (raw.isEmpty()) { return QJsonObject(); } - return BlackMisc::Json::jsonObjectFromString(raw); + + // allow also compressed format + const QJsonDocument jsonDoc = CDatabaseUtils::databaseJsonToQJsonDocument(raw); + return jsonDoc.object(); } QJsonObject CDatabaseUtils::readQJsonObjectFromDatabaseFile(const QString &directory, const QString &filename) diff --git a/src/blackcore/db/icaodatareader.cpp b/src/blackcore/db/icaodatareader.cpp index fe0469153..abb087c23 100644 --- a/src/blackcore/db/icaodatareader.cpp +++ b/src/blackcore/db/icaodatareader.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -395,7 +396,7 @@ namespace BlackCore this->emitAndLogDataRead(CEntityFlags::CountryEntity, n, res); } - CStatusMessageList CIcaoDataReader::readFromJsonFiles(const QString &dir, CEntityFlags::Entity whatToRead) + CStatusMessageList CIcaoDataReader::readFromJsonFiles(const QString &dir, CEntityFlags::Entity whatToRead, bool overrideNewerOnly) { const QDir directory(dir); if (!directory.exists()) @@ -410,100 +411,126 @@ namespace BlackCore if (whatToRead.testFlag(CEntityFlags::CountryEntity)) { const QString fileName = CFileUtils::appendFilePaths(directory.absolutePath(), "countries.json"); - const QJsonObject countriesJson(CDatabaseUtils::readQJsonObjectFromDatabaseFile(fileName)); - if (countriesJson.isEmpty()) + const QFileInfo fi(fileName); + if (!fi.exists()) { - msgs.push_back(CStatusMessage(this).error("Failed to read from file/empty file '%1'") << fileName); + msgs.push_back(CStatusMessage(this).warning("File '%1' does not exist") << fileName); + } + else if (!this->overrideCacheFromFile(overrideNewerOnly, fi, CEntityFlags::CountryEntity, msgs)) + { + // void } else { - try + const QJsonObject countriesJson(CDatabaseUtils::readQJsonObjectFromDatabaseFile(fileName)); + if (countriesJson.isEmpty()) { - CCountryList countries; - countries.convertFromJson(countriesJson); - const int c = countries.size(); - m_countryCache.set(countries); - reallyRead |= CEntityFlags::CountryEntity; - emit this->dataRead(CEntityFlags::CountryEntity, CEntityFlags::ReadFinished, c); + msgs.push_back(CStatusMessage(this).error("Failed to read from file/empty file '%1'") << fileName); } - catch (const CJsonException &ex) + else { - emit this->dataRead(CEntityFlags::CountryEntity, CEntityFlags::ReadFailed, 0); - msgs.push_back(ex.toStatusMessage(this, QString("Reading countries from '%1'").arg(fileName))); + try + { + const CCountryList countries = CCountryList::fromMultipleJsonFormats(countriesJson); + const int c = countries.size(); + msgs.push_back(m_countryCache.set(countries, fi.created().toUTC().toMSecsSinceEpoch())); + reallyRead |= CEntityFlags::CountryEntity; + emit this->dataRead(CEntityFlags::CountryEntity, CEntityFlags::ReadFinished, c); + } + catch (const CJsonException &ex) + { + emit this->dataRead(CEntityFlags::CountryEntity, CEntityFlags::ReadFailed, 0); + msgs.push_back(ex.toStatusMessage(this, QString("Reading countries from '%1'").arg(fileName))); + } } } - } + } // country if (whatToRead.testFlag(CEntityFlags::AircraftIcaoEntity)) { const QString fileName = CFileUtils::appendFilePaths(directory.absolutePath(), "aircrafticao.json"); - const QJsonObject aircraftJson(CDatabaseUtils::readQJsonObjectFromDatabaseFile(fileName)); - if (aircraftJson.isEmpty()) + const QFileInfo fi(fileName); + if (!fi.exists()) { - msgs.push_back(CStatusMessage(this).error("Failed to read from file/empty file '%1'") << fileName); + msgs.push_back(CStatusMessage(this).warning("File '%1' does not exist") << fileName); + } + else if (!this->overrideCacheFromFile(overrideNewerOnly, fi, CEntityFlags::AircraftIcaoEntity, msgs)) + { + // void } else { - try + const QJsonObject aircraftJson(CDatabaseUtils::readQJsonObjectFromDatabaseFile(fileName)); + if (aircraftJson.isEmpty()) { - CAircraftIcaoCodeList aircraftIcaos; - aircraftIcaos.convertFromJson(aircraftJson); - const int c = aircraftIcaos.size(); - m_aircraftIcaoCache.set(aircraftIcaos); - reallyRead |= CEntityFlags::AircraftIcaoEntity; - emit this->dataRead(CEntityFlags::AircraftIcaoEntity, CEntityFlags::ReadFinished, c); + msgs.push_back(CStatusMessage(this).error("Failed to read from file/empty file '%1'") << fileName); } - catch (const CJsonException &ex) + else { - emit this->dataRead(CEntityFlags::AircraftIcaoEntity, CEntityFlags::ReadFailed, 0); - msgs.push_back(ex.toStatusMessage(this, QString("Reading aircraft ICAOs from '%1'").arg(fileName))); + try + { + const CAircraftIcaoCodeList aircraftIcaos = CAircraftIcaoCodeList::fromMultipleJsonFormats(aircraftJson); + const int c = aircraftIcaos.size(); + msgs.push_back(m_aircraftIcaoCache.set(aircraftIcaos, fi.created().toUTC().toMSecsSinceEpoch())); + reallyRead |= CEntityFlags::AircraftIcaoEntity; + emit this->dataRead(CEntityFlags::AircraftIcaoEntity, CEntityFlags::ReadFinished, c); + } + catch (const CJsonException &ex) + { + emit this->dataRead(CEntityFlags::AircraftIcaoEntity, CEntityFlags::ReadFailed, 0); + msgs.push_back(ex.toStatusMessage(this, QString("Reading aircraft ICAOs from '%1'").arg(fileName))); + } } } - } + } // aircraft if (whatToRead.testFlag(CEntityFlags::AirlineIcaoEntity)) { const QString fileName = CFileUtils::appendFilePaths(directory.absolutePath(), "airlineicao.json"); - const QJsonObject airlineJson(CDatabaseUtils::readQJsonObjectFromDatabaseFile(fileName)); - if (airlineJson.isEmpty()) + const QFileInfo fi(fileName); + if (!fi.exists()) { - msgs.push_back(CStatusMessage(this).error("Failed to read from file/empty file '%1'") << fileName); + msgs.push_back(CStatusMessage(this).warning("File '%1' does not exist") << fileName); + } + else if (!this->overrideCacheFromFile(overrideNewerOnly, fi, CEntityFlags::AirlineIcaoEntity, msgs)) + { + // void } else { - try + const QJsonObject airlineJson(CDatabaseUtils::readQJsonObjectFromDatabaseFile(fileName)); + if (airlineJson.isEmpty()) { - CAirlineIcaoCodeList airlineIcaos; - airlineIcaos.convertFromJson(airlineJson); - const int c = airlineIcaos.size(); - m_airlineIcaoCache.set(airlineIcaos); - reallyRead |= CEntityFlags::AirlineIcaoEntity; - emit this->dataRead(CEntityFlags::AirlineIcaoEntity, CEntityFlags::ReadFinished, c); + msgs.push_back(CStatusMessage(this).error("Failed to read from file/empty file '%1'") << fileName); } - catch (const CJsonException &ex) + else { - emit this->dataRead(CEntityFlags::AirlineIcaoEntity, CEntityFlags::ReadFailed, 0); - msgs.push_back(ex.toStatusMessage(this, QString("Reading airline ICAOs from '%1'").arg(fileName))); + try + { + const CAirlineIcaoCodeList airlineIcaos = CAirlineIcaoCodeList::fromMultipleJsonFormats(airlineJson); + const int c = airlineIcaos.size(); + msgs.push_back(m_airlineIcaoCache.set(airlineIcaos, fi.created().toUTC().toMSecsSinceEpoch())); + reallyRead |= CEntityFlags::AirlineIcaoEntity; + emit this->dataRead(CEntityFlags::AirlineIcaoEntity, CEntityFlags::ReadFinished, c); + } + catch (const CJsonException &ex) + { + emit this->dataRead(CEntityFlags::AirlineIcaoEntity, CEntityFlags::ReadFailed, 0); + msgs.push_back(ex.toStatusMessage(this, QString("Reading airline ICAOs from '%1'").arg(fileName))); + } } } - } + } // airline - if (msgs.isSuccess() && (reallyRead & CEntityFlags::DistributorLiveryModel) == whatToRead) - { - return CStatusMessage(this).info("Updated caches for '%1' from '%2'") << CEntityFlags::flagToString(reallyRead) << dir; - } - else - { - return msgs; - } + return msgs; } - bool CIcaoDataReader::readFromJsonFilesInBackground(const QString &dir, CEntityFlags::Entity whatToRead) + bool CIcaoDataReader::readFromJsonFilesInBackground(const QString &dir, CEntityFlags::Entity whatToRead, bool overrideNewerOnly) { if (dir.isEmpty() || whatToRead == CEntityFlags::NoEntity) { return false; } - QTimer::singleShot(0, this, [this, dir, whatToRead]() + QTimer::singleShot(0, this, [ = ]() { - const CStatusMessageList msgs = this->readFromJsonFiles(dir, whatToRead); + const CStatusMessageList msgs = this->readFromJsonFiles(dir, whatToRead, overrideNewerOnly); if (msgs.isFailure()) { CLogMessage::preformatted(msgs); @@ -518,22 +545,22 @@ namespace BlackCore if (!directory.exists()) { return false; } if (this->getCountriesCount() > 0) { - QString json(QJsonDocument(this->getCountries().toJson()).toJson()); - bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), "countries.json")); + const QString json(QJsonDocument(this->getCountries().toJson()).toJson()); + const bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), "countries.json")); if (!s) { return false; } } if (this->getAircraftIcaoCodesCount() > 0) { - QString json(QJsonDocument(this->getAircraftIcaoCodes().toJson()).toJson()); - bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), "aircrafticao.json")); + const QString json(QJsonDocument(this->getAircraftIcaoCodes().toJson()).toJson()); + const bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), "aircrafticao.json")); if (!s) { return false; } } if (this->getAirlineIcaoCodesCount() > 0) { - QString json(QJsonDocument(this->getAirlineIcaoCodes().toJson()).toJson()); - bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), "airlineicao.json")); + const QString json(QJsonDocument(this->getAirlineIcaoCodes().toJson()).toJson()); + const bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), "airlineicao.json")); if (!s) { return false; } } return true; diff --git a/src/blackcore/db/icaodatareader.h b/src/blackcore/db/icaodatareader.h index d70f572b1..321aa17f7 100644 --- a/src/blackcore/db/icaodatareader.h +++ b/src/blackcore/db/icaodatareader.h @@ -137,8 +137,8 @@ namespace BlackCore bool writeToJsonFiles(const QString &dir) const; // data read from local data - virtual BlackMisc::CStatusMessageList readFromJsonFiles(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead) override; - virtual bool readFromJsonFilesInBackground(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead) override; + virtual BlackMisc::CStatusMessageList readFromJsonFiles(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead, bool overrideNewerOnly) override; + virtual bool readFromJsonFilesInBackground(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead, bool overrideNewerOnly) override; // cache handling for base class virtual BlackMisc::Network::CEntityFlags::Entity getSupportedEntities() const override; diff --git a/src/blackcore/db/infodatareader.cpp b/src/blackcore/db/infodatareader.cpp index 58a854028..c3e9fdcaf 100644 --- a/src/blackcore/db/infodatareader.cpp +++ b/src/blackcore/db/infodatareader.cpp @@ -217,18 +217,21 @@ namespace BlackCore return CUrl(); } - CStatusMessageList CInfoDataReader::readFromJsonFiles(const QString &dir, CEntityFlags::Entity whatToRead) + CStatusMessageList CInfoDataReader::readFromJsonFiles(const QString &dir, CEntityFlags::Entity whatToRead, bool overrideNewer) { Q_UNUSED(dir); Q_UNUSED(whatToRead); + Q_UNUSED(overrideNewer); Q_ASSERT_X(false, Q_FUNC_INFO, "Not supported"); + return CStatusMessage(this).error("Not supported"); } - bool CInfoDataReader::readFromJsonFilesInBackground(const QString &dir, CEntityFlags::Entity whatToRead) + bool CInfoDataReader::readFromJsonFilesInBackground(const QString &dir, CEntityFlags::Entity whatToRead, bool overrideNewer) { Q_UNUSED(dir); Q_UNUSED(whatToRead); + Q_UNUSED(overrideNewer) Q_ASSERT_X(false, Q_FUNC_INFO, "Not supported"); return false; } diff --git a/src/blackcore/db/infodatareader.h b/src/blackcore/db/infodatareader.h index d2cffa30c..918bbe81d 100644 --- a/src/blackcore/db/infodatareader.h +++ b/src/blackcore/db/infodatareader.h @@ -54,8 +54,8 @@ namespace BlackCore BlackMisc::Network::CUrl getInfoObjectsUrl() const; // data read from local data - virtual BlackMisc::CStatusMessageList readFromJsonFiles(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead) override; - virtual bool readFromJsonFilesInBackground(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead) override; + virtual BlackMisc::CStatusMessageList readFromJsonFiles(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead, bool overrideNewer) override; + virtual bool readFromJsonFilesInBackground(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead, bool overrideNewer) override; // cache handling for base class: no cache handling here in that case virtual BlackMisc::Network::CEntityFlags::Entity getSupportedEntities() const override; diff --git a/src/blackcore/db/modeldatareader.cpp b/src/blackcore/db/modeldatareader.cpp index 3c49ea350..b3eed5ccf 100644 --- a/src/blackcore/db/modeldatareader.cpp +++ b/src/blackcore/db/modeldatareader.cpp @@ -18,6 +18,7 @@ #include #include +#include #include #include #include @@ -158,9 +159,9 @@ namespace BlackCore bool CModelDataReader::areAllDataRead() const { return - getLiveriesCount() > 0 && - getModelsCount() > 0 && - getDistributorsCount() > 0; + this->getLiveriesCount() > 0 && + this->getModelsCount() > 0 && + this->getDistributorsCount() > 0; } void CModelDataReader::ps_read(CEntityFlags::Entity entities, CDbFlags::DataRetrievalModeFlag mode, const QDateTime &newerThan) @@ -351,7 +352,7 @@ namespace BlackCore if (res.hasErrorMessage()) { CLogMessage::preformatted(res.lastWarningOrAbove()); - emit dataRead(CEntityFlags::ModelEntity, CEntityFlags::ReadFailed, 0); + emit this->dataRead(CEntityFlags::ModelEntity, CEntityFlags::ReadFailed, 0); return; } @@ -384,7 +385,7 @@ namespace BlackCore this->emitAndLogDataRead(CEntityFlags::ModelEntity, n, res); } - CStatusMessageList CModelDataReader::readFromJsonFiles(const QString &dir, CEntityFlags::Entity whatToRead) + CStatusMessageList CModelDataReader::readFromJsonFiles(const QString &dir, CEntityFlags::Entity whatToRead, bool overrideNewerOnly) { const QDir directory(dir); if (!directory.exists()) @@ -399,26 +400,37 @@ namespace BlackCore if (whatToRead.testFlag(CEntityFlags::LiveryEntity)) { const QString fileName = CFileUtils::appendFilePaths(directory.absolutePath(), "liveries.json"); - const QJsonObject liveriesJson(CDatabaseUtils::readQJsonObjectFromDatabaseFile(fileName)); - if (liveriesJson.isEmpty()) + const QFileInfo fi(fileName); + if (!fi.exists()) { - msgs.push_back(CStatusMessage(this).error("Failed to read from file/empty file '%1'") << fileName); + msgs.push_back(CStatusMessage(this).warning("File '%1' does not exist") << fileName); + } + else if (!this->overrideCacheFromFile(overrideNewerOnly, fi, CEntityFlags::LiveryEntity, msgs)) + { + // void } else { - try + const QJsonObject liveriesJson(CDatabaseUtils::readQJsonObjectFromDatabaseFile(fileName)); + if (liveriesJson.isEmpty()) { - CLiveryList liveries; - liveries.convertFromJson(liveriesJson); - const int c = liveries.size(); - m_liveryCache.set(liveries); - emit dataRead(CEntityFlags::LiveryEntity, CEntityFlags::ReadFinished, c); - reallyRead |= CEntityFlags::LiveryEntity; + msgs.push_back(CStatusMessage(this).error("Failed to read from file/empty file '%1'") << fileName); } - catch (const CJsonException &ex) + else { - emit dataRead(CEntityFlags::LiveryEntity, CEntityFlags::ReadFailed, 0); - msgs.push_back(ex.toStatusMessage(this, QString("Reading liveries from '%1'").arg(fileName))); + try + { + const CLiveryList liveries = CLiveryList::fromMultipleJsonFormats(liveriesJson); + const int c = liveries.size(); + msgs.push_back(m_liveryCache.set(liveries, fi.created().toUTC().toMSecsSinceEpoch())); + emit this->dataRead(CEntityFlags::LiveryEntity, CEntityFlags::ReadFinished, c); + reallyRead |= CEntityFlags::LiveryEntity; + } + catch (const CJsonException &ex) + { + emit this->dataRead(CEntityFlags::LiveryEntity, CEntityFlags::ReadFailed, 0); + msgs.push_back(ex.toStatusMessage(this, QString("Reading liveries from '%1'").arg(fileName))); + } } } } @@ -426,26 +438,37 @@ namespace BlackCore if (whatToRead.testFlag(CEntityFlags::ModelEntity)) { const QString fileName = CFileUtils::appendFilePaths(directory.absolutePath(), "models.json"); - const QJsonObject modelsJson(CDatabaseUtils::readQJsonObjectFromDatabaseFile(fileName)); - if (modelsJson.isEmpty()) + const QFileInfo fi(fileName); + if (!fi.exists()) { - msgs.push_back(CStatusMessage(this).error("Failed to read from file/empty file '%1'") << fileName); + msgs.push_back(CStatusMessage(this).warning("File '%1' does not exist") << fileName); + } + else if (!this->overrideCacheFromFile(overrideNewerOnly, fi, CEntityFlags::ModelEntity, msgs)) + { + // void } else { - try + const QJsonObject modelsJson(CDatabaseUtils::readQJsonObjectFromDatabaseFile(fileName)); + if (modelsJson.isEmpty()) { - CAircraftModelList models; - models.convertFromJson(modelsJson); - const int c = models.size(); - m_modelCache.set(models); - emit dataRead(CEntityFlags::ModelEntity, CEntityFlags::ReadFinished, c); - reallyRead |= CEntityFlags::ModelEntity; + msgs.push_back(CStatusMessage(this).error("Failed to read from file/empty file '%1'") << fileName); } - catch (const CJsonException &ex) + else { - emit dataRead(CEntityFlags::ModelEntity, CEntityFlags::ReadFailed, 0); - msgs.push_back(ex.toStatusMessage(this, QString("Reading models from '%1'").arg(fileName))); + try + { + const CAircraftModelList models = CAircraftModelList::fromMultipleJsonFormats(modelsJson); + const int c = models.size(); + msgs.push_back(m_modelCache.set(models, fi.created().toUTC().toMSecsSinceEpoch())); + emit this->dataRead(CEntityFlags::ModelEntity, CEntityFlags::ReadFinished, c); + reallyRead |= CEntityFlags::ModelEntity; + } + catch (const CJsonException &ex) + { + emit this->dataRead(CEntityFlags::ModelEntity, CEntityFlags::ReadFailed, 0); + msgs.push_back(ex.toStatusMessage(this, QString("Reading models from '%1'").arg(fileName))); + } } } } @@ -453,46 +476,50 @@ namespace BlackCore if (whatToRead.testFlag(CEntityFlags::DistributorEntity)) { const QString fileName = CFileUtils::appendFilePaths(directory.absolutePath(), "distributors.json"); - const QJsonObject distributorsJson(CDatabaseUtils::readQJsonObjectFromDatabaseFile(fileName)); - if (distributorsJson.isEmpty()) + const QFileInfo fi(fileName); + if (!fi.exists()) { - msgs.push_back(CStatusMessage(this).error("Failed to read from file/empty file '%1'") << fileName); + msgs.push_back(CStatusMessage(this).warning("File '%1' does not exist") << fileName); + } + else if (!this->overrideCacheFromFile(overrideNewerOnly, fi, CEntityFlags::DistributorEntity, msgs)) + { + // void } else { - try + const QJsonObject distributorsJson(CDatabaseUtils::readQJsonObjectFromDatabaseFile(fileName)); + if (distributorsJson.isEmpty()) { - CDistributorList distributors; - distributors.convertFromJson(distributorsJson); - const int c = distributors.size(); - m_distributorCache.set(distributors); - emit dataRead(CEntityFlags::DistributorEntity, CEntityFlags::ReadFinished, c); - reallyRead |= CEntityFlags::DistributorEntity; + msgs.push_back(CStatusMessage(this).error("Failed to read from file/empty file '%1'") << fileName); } - catch (const CJsonException &ex) + else { - emit dataRead(CEntityFlags::DistributorEntity, CEntityFlags::ReadFailed, 0); - msgs.push_back(ex.toStatusMessage(this, QString("Reading distributors from '%1'").arg(fileName))); + try + { + const CDistributorList distributors = CDistributorList::fromMultipleJsonFormats(distributorsJson); + const int c = distributors.size(); + msgs.push_back(m_distributorCache.set(distributors, fi.created().toUTC().toMSecsSinceEpoch())); + emit this->dataRead(CEntityFlags::DistributorEntity, CEntityFlags::ReadFinished, c); + reallyRead |= CEntityFlags::DistributorEntity; + } + catch (const CJsonException &ex) + { + emit this->dataRead(CEntityFlags::DistributorEntity, CEntityFlags::ReadFailed, 0); + msgs.push_back(ex.toStatusMessage(this, QString("Reading distributors from '%1'").arg(fileName))); + } } } } - if (msgs.isSuccess() && (reallyRead & CEntityFlags::DistributorLiveryModel) == whatToRead) - { - return CStatusMessage(this).info("Updated caches for '%1' from '%2'") << CEntityFlags::flagToString(reallyRead) << dir; - } - else - { - return msgs; - } + return msgs; } - bool CModelDataReader::readFromJsonFilesInBackground(const QString &dir, CEntityFlags::Entity whatToRead) + bool CModelDataReader::readFromJsonFilesInBackground(const QString &dir, CEntityFlags::Entity whatToRead, bool overrideNewerOnly) { if (dir.isEmpty() || whatToRead == CEntityFlags::NoEntity) { return false; } - QTimer::singleShot(0, this, [this, dir, whatToRead]() + QTimer::singleShot(0, this, [ = ]() { - const CStatusMessageList msgs = this->readFromJsonFiles(dir, whatToRead); + const CStatusMessageList msgs = this->readFromJsonFiles(dir, whatToRead, overrideNewerOnly); if (msgs.isFailure()) { CLogMessage::preformatted(msgs); diff --git a/src/blackcore/db/modeldatareader.h b/src/blackcore/db/modeldatareader.h index 96d51e67c..2c036d4fd 100644 --- a/src/blackcore/db/modeldatareader.h +++ b/src/blackcore/db/modeldatareader.h @@ -124,8 +124,8 @@ namespace BlackCore bool areAllDataRead() const; // data read from local data - virtual BlackMisc::CStatusMessageList readFromJsonFiles(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead) override; - virtual bool readFromJsonFilesInBackground(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead) override; + virtual BlackMisc::CStatusMessageList readFromJsonFiles(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead, bool overrideNewerOnly) override; + virtual bool readFromJsonFilesInBackground(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead, bool overrideNewerOnly) override; //! Write to JSON file bool writeToJsonFiles(const QString &dir) const; diff --git a/src/blackcore/webdataservices.cpp b/src/blackcore/webdataservices.cpp index f2238dbc6..a523b42db 100644 --- a/src/blackcore/webdataservices.cpp +++ b/src/blackcore/webdataservices.cpp @@ -1414,34 +1414,87 @@ namespace BlackCore return true; } - bool CWebDataServices::readDbDataFromDisk(const QString &dir, bool inBackground) + bool CWebDataServices::readDbDataFromDisk(const QString &dir, bool inBackground, bool overrideNewerOnly) { if (dir.isEmpty()) { return false; } const QDir directory(dir); if (!directory.exists()) { return false; } - bool s = false; + bool s1 = !m_icaoDataReader; if (m_icaoDataReader) { - if (inBackground) { return m_icaoDataReader->readFromJsonFilesInBackground(dir, m_icaoDataReader->getSupportedEntities()); } - const CStatusMessageList msgs = m_icaoDataReader->readFromJsonFiles(dir, m_icaoDataReader->getSupportedEntities()); - if (msgs.isFailure()) { CLogMessage::preformatted(msgs); } - s = msgs.isSuccess(); + // force update to background reading if reader is already in another thread + bool ib = inBackground || !CThreadUtils::isCurrentThreadObjectThread(m_icaoDataReader); + if (ib) + { + CLogMessage(this).info("Reading from disk in background: %1") << m_icaoDataReader->getSupportedEntitiesAsString(); + s1 = m_icaoDataReader->readFromJsonFilesInBackground(dir, m_icaoDataReader->getSupportedEntities(), overrideNewerOnly); + } + else + { + const CStatusMessageList msgs = m_icaoDataReader->readFromJsonFiles(dir, m_icaoDataReader->getSupportedEntities(), overrideNewerOnly); + CLogMessage::preformatted(msgs); + s1 = msgs.isSuccess(); + } } - if (s && m_modelDataReader) + + bool s2 = !m_modelDataReader; + if (m_modelDataReader) { - if (inBackground) { return m_modelDataReader->readFromJsonFilesInBackground(dir, m_modelDataReader->getSupportedEntities()); } - const CStatusMessageList msgs = m_modelDataReader->readFromJsonFiles(dir, m_modelDataReader->getSupportedEntities()); - if (msgs.isFailure()) { CLogMessage::preformatted(msgs); } - s = msgs.isSuccess(); + // force update to background reading if reader is already in another thread + bool ib = inBackground || !CThreadUtils::isCurrentThreadObjectThread(m_modelDataReader); + if (ib) + { + CLogMessage(this).info("Reading from disk in background: %1") << m_modelDataReader->getSupportedEntitiesAsString(); + s2 = m_modelDataReader->readFromJsonFilesInBackground(dir, m_modelDataReader->getSupportedEntities(), overrideNewerOnly); + } + else + { + const CStatusMessageList msgs = m_modelDataReader->readFromJsonFiles(dir, m_modelDataReader->getSupportedEntities(), overrideNewerOnly); + CLogMessage::preformatted(msgs); + s2 = msgs.isSuccess(); + } } - if (s && m_airportDataReader) + + bool s3 = !m_airportDataReader; + if (m_airportDataReader) { - if (inBackground) { return m_airportDataReader->readFromJsonFilesInBackground(dir, m_airportDataReader->getSupportedEntities()); } - const CStatusMessageList msgs = m_airportDataReader->readFromJsonFiles(dir, m_airportDataReader->getSupportedEntities()); - if (msgs.isFailure()) { CLogMessage::preformatted(msgs); } - s = msgs.isSuccess(); + // force update to background reading if reader is already in another thread + bool ib = inBackground || !CThreadUtils::isCurrentThreadObjectThread(m_airportDataReader); + if (ib) + { + CLogMessage(this).info("Reading from disk in background: %1") << m_airportDataReader->getSupportedEntitiesAsString(); + s3 = m_airportDataReader->readFromJsonFilesInBackground(dir, m_airportDataReader->getSupportedEntities(), overrideNewerOnly); + } + else + { + const CStatusMessageList msgs = m_airportDataReader->readFromJsonFiles(dir, m_airportDataReader->getSupportedEntities(), overrideNewerOnly); + CLogMessage::preformatted(msgs); + s3 = msgs.isSuccess(); + } } - return s; + + return s1 && s2 && s3; + } + + CStatusMessageList CWebDataServices::initDbCachesFromLocalResourceFiles(bool inBackground) + { + CStatusMessageList msgs; + msgs.push_back( + m_icaoDataReader ? + m_icaoDataReader->initFromLocalResourceFiles(inBackground) : + CStatusMessage(this).info("No ICAO reader") + ); + msgs.push_back( + m_modelDataReader ? + m_modelDataReader->initFromLocalResourceFiles(inBackground) : + CStatusMessage(this).info("No model reader") + ); + msgs.push_back( + m_airportDataReader ? + m_airportDataReader->initFromLocalResourceFiles(inBackground) : + CStatusMessage(this).info("No airport reader") + ); + return msgs; } } // ns diff --git a/src/blackcore/webdataservices.h b/src/blackcore/webdataservices.h index fbc7cd1ef..d5e0b2cac 100644 --- a/src/blackcore/webdataservices.h +++ b/src/blackcore/webdataservices.h @@ -445,8 +445,13 @@ namespace BlackCore //! Write data to disk (mainly for testing scenarios) bool writeDbDataToDisk(const QString &dir) const; - //! Load DB data from disk (mainly for testing scenarios) - bool readDbDataFromDisk(const QString &dir, bool inBackground); + //! Load DB data from disk (mainly for initial data load and testing scenarios) + //! \remark if the DB readers are alred in aother thread reads in background + bool readDbDataFromDisk(const QString &dir, bool inBackground, bool overrideNewerOnly); + + //! Init caches from local DB files + //! \remark the shared files coming with the installer + BlackMisc::CStatusMessageList initDbCachesFromLocalResourceFiles(bool inBackground); signals: //! Combined read signal diff --git a/src/blackgui/components/datainfoareacomponent.cpp b/src/blackgui/components/datainfoareacomponent.cpp index 30c7475cf..326bda8fd 100644 --- a/src/blackgui/components/datainfoareacomponent.cpp +++ b/src/blackgui/components/datainfoareacomponent.cpp @@ -104,11 +104,12 @@ namespace BlackGui bool CDataInfoAreaComponent::readDbDataFromResourceDir() { - const bool s = sGui && - sGui->getWebDataServices()->readDbDataFromDisk(CDirectoryUtils::staticDbFilesDirectory(), true); + if (!sGui) { return false; } + const CStatusMessageList msgs = sGui->getWebDataServices()->initDbCachesFromLocalResourceFiles(true); // info - if (s) + bool ok = false; + if (msgs.isSuccess()) { CLogMessage(this).info("Read DB data from directory: %1") << CDirectoryUtils::staticDbFilesDirectory(); ui->comp_DbAircraftIcao->showLoadIndicator(); @@ -117,12 +118,13 @@ namespace BlackGui ui->comp_DbDistributors->showLoadIndicator(); ui->comp_DbLiveries->showLoadIndicator(); ui->comp_DbModels->showLoadIndicator(); + ok = true; } else { - CLogMessage(this).error("Failed to load DB data"); + CLogMessage::preformatted(msgs); } - return s; + return ok; } QSize CDataInfoAreaComponent::getPreferredSizeWhenFloating(int areaIndex) const @@ -145,20 +147,13 @@ namespace BlackGui InfoArea area = static_cast(areaIndex); switch (area) { - case InfoAreaAircraftIcao: - return CIcons::appAircraftIcao16(); - case InfoAreaAirlineIcao: - return CIcons::appAirlineIcao16(); - case InfoAreaLiveries: - return CIcons::appLiveries16(); - case InfoAreaDistributors: - return CIcons::appDistributors16(); - case InfoAreaModels: - return CIcons::appModels16(); - case InfoAreaCountries: - return CIcons::appCountries16(); - default: - return CIcons::empty(); + case InfoAreaAircraftIcao: return CIcons::appAircraftIcao16(); + case InfoAreaAirlineIcao: return CIcons::appAirlineIcao16(); + case InfoAreaLiveries: return CIcons::appLiveries16(); + case InfoAreaDistributors: return CIcons::appDistributors16(); + case InfoAreaModels: return CIcons::appModels16(); + case InfoAreaCountries: return CIcons::appCountries16(); + default: return CIcons::empty(); } }