diff --git a/src/blackcore/db/icaodatareader.cpp b/src/blackcore/db/icaodatareader.cpp index 86a99e0df..80a3b9e89 100644 --- a/src/blackcore/db/icaodatareader.cpp +++ b/src/blackcore/db/icaodatareader.cpp @@ -141,6 +141,16 @@ namespace BlackCore return codes.smartAirlineIcaoSelector(icaoPattern, callsign); } + CAircraftCategoryList CIcaoDataReader::getAircraftCategories() const + { + return m_categoryCache.get(); + } + + int CIcaoDataReader::getAircraftCategoryCount() const + { + return this->getAircraftCategories().size(); + } + int CIcaoDataReader::getAircraftIcaoCodesCount() const { return this->getAircraftIcaoCodes().size(); @@ -165,7 +175,7 @@ namespace BlackCore { this->threadAssertCheck(); // runs in background thread if (!this->doWorkCheck()) { return; } - entities &= CEntityFlags::AllIcaoAndCountries; + entities &= CEntityFlags::AllIcaoCountriesCategory; if (!this->isInternetAccessible()) { emit this->dataRead(entities, CEntityFlags::ReadSkipped, 0); @@ -175,7 +185,7 @@ namespace BlackCore CEntityFlags::Entity entitiesTriggered = CEntityFlags::NoEntity; if (entities.testFlag(CEntityFlags::AircraftIcaoEntity)) { - CUrl url(getAircraftIcaoUrl(mode)); + CUrl url(this->getAircraftIcaoUrl(mode)); if (!url.isEmpty()) { url.appendQuery(queryLatestTimestamp(newerThan)); @@ -190,7 +200,7 @@ namespace BlackCore if (entities.testFlag(CEntityFlags::AirlineIcaoEntity)) { - CUrl url(getAirlineIcaoUrl(mode)); + CUrl url(this->getAirlineIcaoUrl(mode)); if (!url.isEmpty()) { url.appendQuery(queryLatestTimestamp(newerThan)); @@ -205,7 +215,7 @@ namespace BlackCore if (entities.testFlag(CEntityFlags::CountryEntity)) { - CUrl url(getCountryUrl(mode)); + CUrl url(this->getCountryUrl(mode)); if (!url.isEmpty()) { url.appendQuery(queryLatestTimestamp(newerThan)); @@ -218,6 +228,21 @@ namespace BlackCore } } + if (entities.testFlag(CEntityFlags::AircraftCategoryEntity)) + { + CUrl url(this->getAircraftCategoryUrl(mode)); + if (!url.isEmpty()) + { + url.appendQuery(queryLatestTimestamp(newerThan)); + this->getFromNetworkAndLog(url, { this, &CIcaoDataReader::parseAircraftCategoryData }); + entitiesTriggered |= CEntityFlags::AircraftCategoryEntity; + } + else + { + this->logNoWorkingUrl(CEntityFlags::AircraftCategoryEntity); + } + } + if (entitiesTriggered != CEntityFlags::NoEntity) { emit this->dataRead(entitiesTriggered, CEntityFlags::ReadStarted, 0); @@ -239,6 +264,11 @@ namespace BlackCore this->cacheHasChanged(CEntityFlags::CountryEntity); } + void CIcaoDataReader::aircraftCategoryCacheChanged() + { + this->cacheHasChanged(CEntityFlags::AircraftCategoryEntity); + } + void CIcaoDataReader::baseUrlCacheChanged() { // void @@ -393,7 +423,7 @@ namespace BlackCore // normally read from special DB view which already filters incomplete QTime time; time.start(); - countries = CCountryList::fromDatabaseJson(res); + countries = CCountryList::fromDatabaseJson(res); this->logParseMessage("countries", countries.size(), time.elapsed(), res); } @@ -412,6 +442,51 @@ namespace BlackCore this->emitAndLogDataRead(CEntityFlags::CountryEntity, n, res); } + void CIcaoDataReader::parseAircraftCategoryData(QNetworkReply *nwReplyPtr) + { + QScopedPointer nwReply(nwReplyPtr); + const CDatabaseReader::JsonDatastoreResponse res = this->setStatusAndTransformReplyIntoDatastoreResponse(nwReply.data()); + if (res.hasErrorMessage()) + { + CLogMessage::preformatted(res.lastWarningOrAbove()); + emit this->dataRead(CEntityFlags::AircraftCategoryEntity, CEntityFlags::ReadFailed, 0); + return; + } + + emit this->dataRead(CEntityFlags::AircraftCategoryEntity, CEntityFlags::ReadParsing, 0); + CAircraftCategoryList categories; + if (res.isRestricted()) + { + // create full list if it was just incremental + const CAircraftCategoryList incrementalCategories(CAircraftCategoryList::fromDatabaseJson(res)); + if (incrementalCategories.isEmpty()) { return; } // currently ignored + categories = this->getAircraftCategories(); + categories.replaceOrAddObjectsByKey(incrementalCategories); + } + else + { + // normally read from special DB view which already filters incomplete + QTime time; + time.start(); + categories = CAircraftCategoryList::fromDatabaseJson(res); + this->logParseMessage("categories", categories.size(), time.elapsed(), res); + } + + if (!this->doWorkCheck()) { return; } + const int n = categories.size(); + qint64 latestTimestamp = categories.latestTimestampMsecsSinceEpoch(); + if (n > 0 && latestTimestamp < 0) + { + CLogMessage(this).error(u"No timestamp in category list, setting to last modified value"); + latestTimestamp = this->lastModifiedMsSinceEpoch(nwReply.data()); + } + + m_categoryCache.set(categories, latestTimestamp); + this->updateReaderUrl(this->getBaseUrl(CDbFlags::DbReading)); + + this->emitAndLogDataRead(CEntityFlags::AircraftCategoryEntity, n, res); + } + CStatusMessageList CIcaoDataReader::readFromJsonFiles(const QString &dir, CEntityFlags::Entity whatToRead, bool overrideNewerOnly) { const QDir directory(dir); @@ -422,11 +497,11 @@ namespace BlackCore // Hint: Do not emit while locked -> deadlock CStatusMessageList msgs; - whatToRead &= CEntityFlags::AllIcaoAndCountries; + whatToRead &= CEntityFlags::AllIcaoCountriesCategory; CEntityFlags::Entity reallyRead = CEntityFlags::NoEntity; if (whatToRead.testFlag(CEntityFlags::CountryEntity)) { - const QString fileName = CFileUtils::appendFilePaths(directory.absolutePath(), "countries.json"); + const QString fileName = CFileUtils::appendFilePaths(directory.absolutePath(), CDbInfo::entityToSharedName(CEntityFlags::CountryEntity)); const QFileInfo fi(fileName); if (!fi.exists()) { @@ -464,7 +539,7 @@ namespace BlackCore if (whatToRead.testFlag(CEntityFlags::AircraftIcaoEntity)) { - const QString fileName = CFileUtils::appendFilePaths(directory.absolutePath(), "aircrafticao.json"); + const QString fileName = CFileUtils::appendFilePaths(directory.absolutePath(), CDbInfo::entityToSharedName(CEntityFlags::AircraftIcaoEntity)); const QFileInfo fi(fileName); if (!fi.exists()) { @@ -502,7 +577,7 @@ namespace BlackCore if (whatToRead.testFlag(CEntityFlags::AirlineIcaoEntity)) { - const QString fileName = CFileUtils::appendFilePaths(directory.absolutePath(), "airlineicao.json"); + const QString fileName = CFileUtils::appendFilePaths(directory.absolutePath(), CDbInfo::entityToSharedName(CEntityFlags::AirlineIcaoEntity)); const QFileInfo fi(fileName); if (!fi.exists()) { @@ -538,6 +613,44 @@ namespace BlackCore } } // airline + if (whatToRead.testFlag(CEntityFlags::AircraftCategoryEntity)) + { + const QString fileName = CFileUtils::appendFilePaths(directory.absolutePath(), CDbInfo::entityToSharedName(CEntityFlags::AircraftCategoryEntity)); + const QFileInfo fi(fileName); + if (!fi.exists()) + { + msgs.push_back(CStatusMessage(this).warning(u"File '%1' does not exist") << fileName); + } + else if (!this->overrideCacheFromFile(overrideNewerOnly, fi, CEntityFlags::AircraftCategoryEntity, msgs)) + { + // void + } + else + { + const QJsonObject aircraftCategory(CDatabaseUtils::readQJsonObjectFromDatabaseFile(fileName)); + if (aircraftCategory.isEmpty()) + { + msgs.push_back(CStatusMessage(this).error(u"Failed to read from file/empty file '%1'") << fileName); + } + else + { + try + { + const CAircraftCategoryList aircraftCategories = CAircraftCategoryList::fromMultipleJsonFormats(aircraftCategory); + const int c = aircraftCategories.size(); + msgs.push_back(m_categoryCache.set(aircraftCategories, fi.created().toUTC().toMSecsSinceEpoch())); + reallyRead |= CEntityFlags::AircraftCategoryEntity; + emit this->dataRead(CEntityFlags::AircraftCategoryEntity, CEntityFlags::ReadFinished, c); + } + catch (const CJsonException &ex) + { + emit this->dataRead(CEntityFlags::AircraftCategoryEntity, CEntityFlags::ReadFailed, 0); + msgs.push_back(ex.toStatusMessage(this, QStringLiteral("Reading categories from '%1'").arg(fileName))); + } + } + } + } // categories + return msgs; } @@ -562,50 +675,61 @@ namespace BlackCore if (this->getCountriesCount() > 0) { const QString json(QJsonDocument(this->getCountries().toJson()).toJson()); - const bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), "countries.json")); + const bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), CDbInfo::entityToSharedName(CEntityFlags::CountryEntity))); if (!s) { return false; } } if (this->getAircraftIcaoCodesCount() > 0) { const QString json(QJsonDocument(this->getAircraftIcaoCodes().toJson()).toJson()); - const bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), "aircrafticao.json")); + const bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), CDbInfo::entityToSharedName(CEntityFlags::AircraftIcaoEntity))); if (!s) { return false; } } if (this->getAirlineIcaoCodesCount() > 0) { const QString json(QJsonDocument(this->getAirlineIcaoCodes().toJson()).toJson()); - const bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), "airlineicao.json")); + const bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), CDbInfo::entityToSharedName(CEntityFlags::AirlineIcaoEntity))); if (!s) { return false; } } + + if (this->getAircraftCategoryCount() > 0) + { + const QString json(QJsonDocument(this->getAirlineIcaoCodes().toJson()).toJson()); + const bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), CDbInfo::entityToSharedName(CEntityFlags::AircraftCategoryEntity))); + if (!s) { return false; } + } + return true; } CEntityFlags::Entity CIcaoDataReader::getSupportedEntities() const { - return CEntityFlags::AllIcaoAndCountries; + return CEntityFlags::AllIcaoCountriesCategory; } void CIcaoDataReader::synchronizeCaches(CEntityFlags::Entity entities) { if (entities.testFlag(CEntityFlags::AircraftIcaoEntity)) { if (m_syncedAircraftIcaoCache) { return; } m_syncedAircraftIcaoCache = true; m_aircraftIcaoCache.synchronize(); } - if (entities.testFlag(CEntityFlags::AirlineIcaoEntity)) { if (m_syncedAirlineIcaoCache) { return; } m_syncedAirlineIcaoCache = true; m_airlineIcaoCache.synchronize(); } - if (entities.testFlag(CEntityFlags::CountryEntity)) { if (m_syncedCountryCache) { return; } m_syncedCountryCache = true; m_countryCache.synchronize(); } + if (entities.testFlag(CEntityFlags::AirlineIcaoEntity)) { if (m_syncedAirlineIcaoCache) { return; } m_syncedAirlineIcaoCache = true; m_airlineIcaoCache.synchronize(); } + if (entities.testFlag(CEntityFlags::CountryEntity)) { if (m_syncedCountryCache) { return; } m_syncedCountryCache = true; m_countryCache.synchronize(); } + if (entities.testFlag(CEntityFlags::AircraftCategoryEntity)) { if (m_syncedCategories) { return; } m_syncedCategories = true; m_categoryCache.synchronize(); } } void CIcaoDataReader::admitCaches(CEntityFlags::Entity entities) { - if (entities.testFlag(CEntityFlags::AircraftIcaoEntity)) { m_aircraftIcaoCache.admit(); } - if (entities.testFlag(CEntityFlags::AirlineIcaoEntity)) { m_airlineIcaoCache.admit(); } - if (entities.testFlag(CEntityFlags::CountryEntity)) { m_countryCache.admit(); } + if (entities.testFlag(CEntityFlags::AircraftIcaoEntity)) { m_aircraftIcaoCache.admit(); } + if (entities.testFlag(CEntityFlags::AirlineIcaoEntity)) { m_airlineIcaoCache.admit(); } + if (entities.testFlag(CEntityFlags::CountryEntity)) { m_countryCache.admit(); } + if (entities.testFlag(CEntityFlags::AircraftCategoryEntity)) { m_categoryCache.admit(); } } void CIcaoDataReader::invalidateCaches(CEntityFlags::Entity entities) { if (entities.testFlag(CEntityFlags::AircraftIcaoEntity)) { CDataCache::instance()->clearAllValues(m_aircraftIcaoCache.getKey()); } - if (entities.testFlag(CEntityFlags::AirlineIcaoEntity)) {CDataCache::instance()->clearAllValues(m_airlineIcaoCache.getKey()); } - if (entities.testFlag(CEntityFlags::CountryEntity)) {CDataCache::instance()->clearAllValues(m_countryCache.getKey()); } + if (entities.testFlag(CEntityFlags::AirlineIcaoEntity)) { CDataCache::instance()->clearAllValues(m_airlineIcaoCache.getKey()); } + if (entities.testFlag(CEntityFlags::CountryEntity)) { CDataCache::instance()->clearAllValues(m_countryCache.getKey()); } + if (entities.testFlag(CEntityFlags::AircraftCategoryEntity)) { CDataCache::instance()->clearAllValues(m_categoryCache.getKey()); } } QDateTime CIcaoDataReader::getCacheTimestamp(CEntityFlags::Entity entity) const @@ -615,6 +739,7 @@ namespace BlackCore case CEntityFlags::AircraftIcaoEntity: return m_aircraftIcaoCache.getAvailableTimestamp(); case CEntityFlags::AirlineIcaoEntity: return m_airlineIcaoCache.getAvailableTimestamp(); case CEntityFlags::CountryEntity: return m_countryCache.getAvailableTimestamp(); + case CEntityFlags::AircraftCategoryEntity: return m_categoryCache.getAvailableTimestamp(); default: return QDateTime(); } } @@ -626,6 +751,7 @@ namespace BlackCore case CEntityFlags::AircraftIcaoEntity: return m_aircraftIcaoCache.get().size(); case CEntityFlags::AirlineIcaoEntity: return m_airlineIcaoCache.get().size(); case CEntityFlags::CountryEntity: return m_countryCache.get().size(); + case CEntityFlags::AircraftCategoryEntity: return m_categoryCache.get().size(); default: return 0; } } @@ -633,9 +759,10 @@ namespace BlackCore CEntityFlags::Entity CIcaoDataReader::getEntitiesWithCacheCount() const { CEntityFlags::Entity entities = CEntityFlags::NoEntity; - if (this->getCacheCount(CEntityFlags::AircraftIcaoEntity) > 0) entities |= CEntityFlags::AircraftIcaoEntity; - if (this->getCacheCount(CEntityFlags::AirlineIcaoEntity) > 0) entities |= CEntityFlags::AirlineIcaoEntity; - if (this->getCacheCount(CEntityFlags::CountryEntity) > 0) entities |= CEntityFlags::CountryEntity; + if (this->getCacheCount(CEntityFlags::AircraftIcaoEntity) > 0) entities |= CEntityFlags::AircraftIcaoEntity; + if (this->getCacheCount(CEntityFlags::AirlineIcaoEntity) > 0) entities |= CEntityFlags::AirlineIcaoEntity; + if (this->getCacheCount(CEntityFlags::CountryEntity) > 0) entities |= CEntityFlags::CountryEntity; + if (this->getCacheCount(CEntityFlags::AircraftCategoryEntity) > 0) entities |= CEntityFlags::AircraftCategoryEntity; return entities; } @@ -643,8 +770,9 @@ namespace BlackCore { CEntityFlags::Entity entities = CEntityFlags::NoEntity; if (this->hasCacheTimestampNewerThan(CEntityFlags::AircraftIcaoEntity, threshold)) entities |= CEntityFlags::AircraftIcaoEntity; - if (this->hasCacheTimestampNewerThan(CEntityFlags::AirlineIcaoEntity, threshold)) entities |= CEntityFlags::AirlineIcaoEntity; - if (this->hasCacheTimestampNewerThan(CEntityFlags::CountryEntity, threshold)) entities |= CEntityFlags::CountryEntity; + if (this->hasCacheTimestampNewerThan(CEntityFlags::AirlineIcaoEntity, threshold)) entities |= CEntityFlags::AirlineIcaoEntity; + if (this->hasCacheTimestampNewerThan(CEntityFlags::CountryEntity, threshold)) entities |= CEntityFlags::CountryEntity; + if (this->hasCacheTimestampNewerThan(CEntityFlags::AircraftCategoryEntity, threshold)) entities |= CEntityFlags::CountryEntity; return entities; } @@ -675,5 +803,10 @@ namespace BlackCore { return this->getBaseUrl(mode).withAppendedPath(fileNameForMode(CEntityFlags::CountryEntity, mode)); } + + CUrl CIcaoDataReader::getAircraftCategoryUrl(CDbFlags::DataRetrievalModeFlag mode) const + { + return this->getBaseUrl(mode).withAppendedPath(fileNameForMode(CEntityFlags::AircraftCategoryEntity, mode)); + } } // ns } // ns diff --git a/src/blackcore/db/icaodatareader.h b/src/blackcore/db/icaodatareader.h index 53513d54d..e0f67e632 100644 --- a/src/blackcore/db/icaodatareader.h +++ b/src/blackcore/db/icaodatareader.h @@ -128,6 +128,14 @@ namespace BlackCore //! \threadsafe BlackMisc::Aviation::CAirlineIcaoCode smartAirlineIcaoSelector(const BlackMisc::Aviation::CAirlineIcaoCode &icaoPattern, const BlackMisc::Aviation::CCallsign &callsign = BlackMisc::Aviation::CCallsign()) const; + //! Get aircraft categories + //! \threadsafe + BlackMisc::Aviation::CAircraftCategoryList getAircraftCategories() const; + + //! Get aircraft category count + //! \threadsafe + int getAircraftCategoryCount() const; + //! All data read? //! \threadsafe bool areAllDataRead() const; @@ -155,12 +163,14 @@ namespace BlackCore virtual BlackMisc::Network::CUrl getDbServiceBaseUrl() const override; private: - BlackMisc::CData m_aircraftIcaoCache {this, &CIcaoDataReader::aircraftIcaoCacheChanged }; - BlackMisc::CData m_airlineIcaoCache {this, &CIcaoDataReader::airlineIcaoCacheChanged }; - BlackMisc::CData m_countryCache {this, &CIcaoDataReader::countryCacheChanged }; + BlackMisc::CData m_aircraftIcaoCache {this, &CIcaoDataReader::aircraftIcaoCacheChanged }; + BlackMisc::CData m_airlineIcaoCache {this, &CIcaoDataReader::airlineIcaoCacheChanged }; + BlackMisc::CData m_countryCache {this, &CIcaoDataReader::countryCacheChanged }; + BlackMisc::CData m_categoryCache {this, &CIcaoDataReader::aircraftCategoryCacheChanged }; std::atomic_bool m_syncedAircraftIcaoCache { false }; //!< already synchronized? std::atomic_bool m_syncedAirlineIcaoCache { false }; //!< already synchronized? std::atomic_bool m_syncedCountryCache { false }; //!< already synchronized? + std::atomic_bool m_syncedCategories { false }; //!< already synchronized? //! \copydoc CDatabaseReader::read virtual void read(BlackMisc::Network::CEntityFlags::Entity entities, @@ -178,6 +188,9 @@ namespace BlackCore //! Countries have been read void parseCountryData(QNetworkReply *nwReply); + //! Categories have been read + void parseAircraftCategoryData(QNetworkReply *nwReplyPtr); + //! Cache has changed elsewhere void aircraftIcaoCacheChanged(); @@ -187,6 +200,9 @@ namespace BlackCore //! Cache has changed elsewhere void countryCacheChanged(); + //! Cache has changed elsewhere + void aircraftCategoryCacheChanged(); + //! Cache has changed elsewhere void baseUrlCacheChanged(); @@ -201,6 +217,9 @@ namespace BlackCore //! URL BlackMisc::Network::CUrl getCountryUrl(BlackMisc::Db::CDbFlags::DataRetrievalModeFlag mode) const; + + //! URL + BlackMisc::Network::CUrl getAircraftCategoryUrl(BlackMisc::Db::CDbFlags::DataRetrievalModeFlag mode) const; }; } // ns } // ns diff --git a/src/blackmisc/network/entityflags.h b/src/blackmisc/network/entityflags.h index 3865a522d..b042dc506 100644 --- a/src/blackmisc/network/entityflags.h +++ b/src/blackmisc/network/entityflags.h @@ -50,9 +50,10 @@ namespace BlackMisc AllEntities = ((1 << 14) - 1), //!< everything AllIcaoEntities = AircraftIcaoEntity | AirlineIcaoEntity, //!< all ICAO codes AllIcaoAndCountries = AircraftIcaoEntity | AirlineIcaoEntity | CountryEntity, //!< all ICAO codes and countries + AllIcaoCountriesCategory = AllIcaoAndCountries | AircraftCategoryEntity, //!< includes category DistributorLiveryModel = DistributorEntity | LiveryEntity | ModelEntity, //!< Combined ModelMatchingEntities = AllIcaoEntities | LiveryEntity | ModelEntity, //!< all needed for model matching - AllDbEntitiesNoInfoObjects = AllIcaoAndCountries | DistributorLiveryModel | AirportEntity | AircraftCategoryEntity, //!< all DB entities, no info objects + AllDbEntitiesNoInfoObjects = AllIcaoCountriesCategory | DistributorLiveryModel | AirportEntity | AircraftCategoryEntity, //!< all DB entities, no info objects AllDbEntities = AllDbEntitiesNoInfoObjects | DbInfoObjectEntity, //!< all DB stuff AllDbEntitiesNoInfoObjectsNoAirportsAndCategories = AllIcaoAndCountries | DistributorLiveryModel //!< all DB entities, no info objects and airports };