/* Copyright (C) 2015 * swift project Community / Contributors * * This file is part of swift project. It is subject to the license terms in the LICENSE file found in the top-level * directory of this distribution and at http://www.swift-project.org/license.html. No part of swift project, * including this file, may be copied, modified, propagated, or distributed except according to the terms * contained in the LICENSE file. */ #include "blackcore/application.h" #include "blackcore/data/globalsetup.h" #include "blackcore/db/airportdatareader.h" #include "blackcore/db/infodatareader.h" #include "blackcore/db/icaodatareader.h" #include "blackcore/db/databasewriter.h" #include "blackcore/db/icaodatareader.h" #include "blackcore/db/modeldatareader.h" #include "blackcore/setupreader.h" #include "blackcore/vatsim/vatsimbookingreader.h" #include "blackcore/vatsim/vatsimdatafilereader.h" #include "blackcore/vatsim/vatsimmetarreader.h" #include "blackcore/vatsim/vatsimstatusfilereader.h" #include "blackcore/webdataservices.h" #include "blackmisc/fileutils.h" #include "blackmisc/logcategory.h" #include "blackmisc/logcategorylist.h" #include "blackmisc/logmessage.h" #include "blackmisc/restricted.h" #include "blackmisc/statusmessage.h" #include "blackmisc/worker.h" #include #include #include #include #include #include #include using namespace BlackCore; using namespace BlackCore::Db; using namespace BlackCore::Data; using namespace BlackCore::Vatsim; using namespace BlackMisc; using namespace BlackMisc::Simulation; using namespace BlackMisc::Network; using namespace BlackMisc::Aviation; using namespace BlackMisc::Weather; namespace BlackCore { CWebDataServices::CWebDataServices(CWebReaderFlags::WebReader readers, const CDatabaseReaderConfigList &dbReaderConfig, BlackMisc::Restricted, QObject *parent) : QObject(parent), m_readers(readers), m_dbReaderConfig(dbReaderConfig) { if (!sApp) { return; } // shutting down Q_ASSERT_X(QSslSocket::supportsSsl(), Q_FUNC_INFO, "Missing SSL support"); Q_ASSERT_X(sApp->isSetupAvailable(), Q_FUNC_INFO, "Setup not synchronized"); this->setObjectName("CWebDataReader"); // check if I need info objects const bool readFromSwiftDb = dbReaderConfig.possiblyReadsFromSwiftDb(); // DB read access const bool writeToSwiftDb = dbReaderConfig.possiblyWritesToSwiftDb(); // DB write access if (!readFromSwiftDb && readers.testFlag(CWebReaderFlags::InfoDataReader)) { // will remove info reader because not needed readers &= ~CWebReaderFlags::InfoDataReader; this->m_readers = readers; CLogMessage(this).info("Remove info object reader because not needed"); } // get entities to be read CEntityFlags::Entity entities = CWebReaderFlags::allEntitiesForReaders(readers); if (entities.testFlag(CEntityFlags::InfoObjectEntity)) { Q_ASSERT_X(readers.testFlag(CWebReaderFlags::InfoDataReader), Q_FUNC_INFO, "info object but no reader"); CLogMessage(this).info("Using info objects for swift DB entities"); } this->initReaders(readers, entities); // reads info objects if required if (writeToSwiftDb) { this->initWriters(); } // make sure this is called in event queue, so pending tasks cam be performed entities &= ~CEntityFlags::InfoObjectEntity; // triggered in init readers entities &= ~CEntityFlags::VatsimStatusFile; // triggered in init readers entities &= ~this->m_entitiesPeriodicallyRead; // will be triggered by timers // trigger reading // but do not start all at the same time const CEntityFlags::Entity icaoPart = entities & CEntityFlags::AllIcaoAndCountries; const CEntityFlags::Entity modelPart = entities & CEntityFlags::DistributorLiveryModel; CEntityFlags::Entity remainingEntities = entities & ~icaoPart; remainingEntities &= ~modelPart; this->readDeferredInBackground(icaoPart, 500); this->readDeferredInBackground(modelPart, 1000); this->readDeferredInBackground(remainingEntities, 1500); } CServerList CWebDataServices::getVatsimFsdServers() const { if (m_vatsimDataFileReader) { return m_vatsimDataFileReader->getFsdServers(); } return CServerList(); } CServerList CWebDataServices::getVatsimVoiceServers() const { if (m_vatsimDataFileReader) { return m_vatsimDataFileReader->getVoiceServers(); } return CServerList(); } CUrlList CWebDataServices::getVatsimMetarUrls() const { if (m_vatsimStatusReader) { return m_vatsimStatusReader->getMetarFileUrls(); } return CUrlList(); } CUrlList CWebDataServices::getVatsimDataFileUrls() const { if (m_vatsimStatusReader) { return m_vatsimStatusReader->getDataFileUrls(); } return CUrlList(); } CUserList CWebDataServices::getUsersForCallsign(const CCallsign &callsign) const { if (m_vatsimDataFileReader) { return m_vatsimDataFileReader->getUsersForCallsign(callsign); } return CUserList(); } CAtcStationList CWebDataServices::getAtcStationsForCallsign(const CCallsign &callsign) const { if (m_vatsimDataFileReader) { return m_vatsimDataFileReader->getAtcStationsForCallsign(callsign); } return CAtcStationList(); } CVoiceCapabilities CWebDataServices::getVoiceCapabilityForCallsign(const CCallsign &callsign) const { if (m_vatsimDataFileReader) { return m_vatsimDataFileReader->getVoiceCapabilityForCallsign(callsign); } return CVoiceCapabilities(); } void CWebDataServices::updateWithVatsimDataFileData(CSimulatedAircraft &aircraftToBeUdpated) const { if (m_vatsimDataFileReader) { m_vatsimDataFileReader->updateWithVatsimDataFileData(aircraftToBeUdpated); } } CStatusMessageList CWebDataServices::asyncPublishModels(const CAircraftModelList &models) const { if (m_databaseWriter) { return m_databaseWriter->asyncPublishModels(models);} return CStatusMessageList(); } void CWebDataServices::triggerReadOfInfoObjects() { initInfoObjectReaderAndTriggerRead(); } bool CWebDataServices::canConnectSwiftDb() const { if (!m_icaoDataReader && !m_modelDataReader && !m_airportDataReader && !m_infoDataReader) { return false; } // use the first one to test if (m_infoDataReader) { return m_infoDataReader->hasReceivedOkReply(); } else if (m_icaoDataReader) { return m_icaoDataReader->hasReceivedOkReply(); } else if (m_modelDataReader) { return m_modelDataReader->hasReceivedOkReply(); } else if (m_airportDataReader) { return m_airportDataReader->hasReceivedOkReply(); } return false; } void CWebDataServices::resetSignalFlags() { m_signalledEntities.clear(); m_signalledHeaders = false; } bool CWebDataServices::hasDbAircraftData() const { return (this->getModelsCount() > 0) && (this->getLiveriesCount() > 0) && (this->getDistributorsCount() > 0) && (this->getAircraftIcaoCodesCount() > 0); } void CWebDataServices::admitDbCaches(CEntityFlags::Entity entities) { // hint: all the readers use own threads if (this->m_infoDataReader) { this->m_infoDataReader->admitCaches(entities); } if (this->m_modelDataReader) { this->m_modelDataReader->admitCaches(entities); } if (this->m_icaoDataReader) { this->m_icaoDataReader->admitCaches(entities); } if (this->m_airportDataReader) { this->m_airportDataReader->admitCaches(entities); } } CEntityFlags::Entity CWebDataServices::triggerRead(CEntityFlags::Entity whatToRead, const QDateTime &newerThan) { m_initialRead = true; // read started Q_ASSERT_X(!whatToRead.testFlag(CEntityFlags::InfoObjectEntity), Q_FUNC_INFO, "Info object must be read upfront"); CEntityFlags::Entity triggeredRead = CEntityFlags::NoEntity; if (m_vatsimDataFileReader) { if (whatToRead.testFlag(CEntityFlags::VatsimDataFile)) { m_vatsimDataFileReader->readInBackgroundThread(); triggeredRead |= CEntityFlags::VatsimDataFile; } } if (m_vatsimBookingReader) { if (whatToRead.testFlag(CEntityFlags::BookingEntity)) { m_vatsimBookingReader->readInBackgroundThread(); triggeredRead |= CEntityFlags::BookingEntity; } } if (m_vatsimMetarReader) { if (whatToRead.testFlag(CEntityFlags::MetarEntity)) { m_vatsimMetarReader->readInBackgroundThread(); triggeredRead |= CEntityFlags::MetarEntity; } } if (m_airportDataReader) { if (whatToRead.testFlag(CEntityFlags::AirportEntity)) { CEntityFlags::Entity airportEntities = whatToRead & CEntityFlags::AirportEntity; m_airportDataReader->readInBackgroundThread(airportEntities, newerThan); triggeredRead |= CEntityFlags::AirportEntity; } } if (m_icaoDataReader) { if (whatToRead.testFlag(CEntityFlags::AircraftIcaoEntity) || whatToRead.testFlag(CEntityFlags::AirlineIcaoEntity) || whatToRead.testFlag(CEntityFlags::CountryEntity)) { CEntityFlags::Entity icaoEntities = whatToRead & CEntityFlags::AllIcaoAndCountries; m_icaoDataReader->readInBackgroundThread(icaoEntities, newerThan); triggeredRead |= icaoEntities; } } if (m_modelDataReader) { if (whatToRead.testFlag(CEntityFlags::LiveryEntity) || whatToRead.testFlag(CEntityFlags::DistributorEntity) || whatToRead.testFlag(CEntityFlags::ModelEntity)) { CEntityFlags::Entity modelEntities = whatToRead & CEntityFlags::DistributorLiveryModel; m_modelDataReader->readInBackgroundThread(modelEntities, newerThan); triggeredRead |= modelEntities; } } return triggeredRead; } CEntityFlags::Entity CWebDataServices::triggerLoadingDirectlyFromDb(CEntityFlags::Entity whatToRead, const QDateTime &newerThan) { CEntityFlags::Entity triggeredRead = CEntityFlags::NoEntity; if (m_infoDataReader) { this->triggerReadOfInfoObjects(); triggeredRead |= CEntityFlags::InfoObjectEntity; } if (m_icaoDataReader) { if (m_icaoDataReader->supportsAnyOfEntities(whatToRead)) { CEntityFlags::Entity icaoEntities = m_icaoDataReader->maskBySupportedEntities(whatToRead); m_icaoDataReader->triggerLoadingDirectlyFromDb(icaoEntities, newerThan); triggeredRead |= icaoEntities; } } if (m_modelDataReader) { if (m_modelDataReader->supportsAnyOfEntities(whatToRead)) { CEntityFlags::Entity modelEntities = m_modelDataReader->maskBySupportedEntities(whatToRead); m_modelDataReader->triggerLoadingDirectlyFromDb(modelEntities, newerThan); triggeredRead |= modelEntities; } } if (m_airportDataReader) { if (m_airportDataReader->supportsAnyOfEntities(whatToRead)) { CEntityFlags::Entity airportEntities = m_airportDataReader->maskBySupportedEntities(whatToRead); m_airportDataReader->triggerLoadingDirectlyFromDb(airportEntities, newerThan); triggeredRead |= airportEntities; } } return triggeredRead; } CEntityFlags::Entity CWebDataServices::triggerLoadingDirectlyFromSharedFiles(CEntityFlags::Entity whatToRead, bool checkCacheTsUpfront) { CEntityFlags::Entity triggeredRead = CEntityFlags::NoEntity; this->triggerLoadingOfSharedFilesHeaders(whatToRead); // trigger reload of headers if (m_icaoDataReader) { if (m_icaoDataReader->supportsAnyOfEntities(whatToRead)) { CEntityFlags::Entity icaoEntities = m_icaoDataReader->maskBySupportedEntities(whatToRead); triggeredRead |= m_icaoDataReader->triggerLoadingDirectlyFromSharedFiles(icaoEntities, checkCacheTsUpfront); } } if (m_modelDataReader) { if (m_modelDataReader->supportsAnyOfEntities(whatToRead)) { CEntityFlags::Entity modelEntities = m_modelDataReader->maskBySupportedEntities(whatToRead); triggeredRead |= m_modelDataReader->triggerLoadingDirectlyFromSharedFiles(modelEntities, checkCacheTsUpfront); } } if (m_airportDataReader) { if (m_airportDataReader->supportsAnyOfEntities(whatToRead)) { CEntityFlags::Entity airportEntities = m_airportDataReader->maskBySupportedEntities(whatToRead); triggeredRead |= m_airportDataReader->triggerLoadingDirectlyFromSharedFiles(airportEntities, checkCacheTsUpfront); } } return triggeredRead; } QDateTime CWebDataServices::getCacheTimestamp(CEntityFlags::Entity entity) const { Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "Need single entity"); if (CEntityFlags::anySwiftDbEntity(entity)) { const CDatabaseReader *dr = this->getDbReader(entity); if (!dr) { return QDateTime(); } return dr->getCacheTimestamp(entity); } else { // non DB entities would go here return QDateTime(); } } QDateTime CWebDataServices::getDbLatestEntityTimestamp(CEntityFlags::Entity entity) const { Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "Need single entity"); if (CEntityFlags::anySwiftDbEntity(entity)) { CInfoDataReader *ir = this->getInfoDataReader(); if (!ir) { return QDateTime(); } return ir->getLatestEntityTimestampFromInfoObjects(entity); } else { return QDateTime(); } } QDateTime CWebDataServices::getSharedFileTimestamp(CEntityFlags::Entity entity) const { const CDatabaseReader *reader = this->getDbReader(entity); if (reader) { return reader->getLatestSharedFileHeaderTimestamp(entity); } else { return QDateTime(); } } bool CWebDataServices::requestHeaderOfSharedFile(CEntityFlags::Entity entity) { Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "Need single entity"); CDatabaseReader *reader = this->getDbReader(entity); if (!reader) { return false; } return reader->requestHeadersOfSharedFiles(entity); } bool CWebDataServices::areSharedHeadersLoaded(CEntityFlags::Entity entities) const { bool hasAllHeaders = false; do { if (this->m_modelDataReader && this->m_modelDataReader->supportsAnyOfEntities(entities)) { if (!this->m_modelDataReader->hasSharedFileHeaders(entities)) { break; } } if (this->m_icaoDataReader && this->m_icaoDataReader->supportsAnyOfEntities(entities)) { if (!this->m_icaoDataReader->hasSharedFileHeaders(entities)) { break; } } if (this->m_airportDataReader && this->m_airportDataReader->supportsAnyOfEntities(entities)) { if (!this->m_airportDataReader->hasSharedFileHeaders(entities)) { break; } } hasAllHeaders = true; } while (false); return hasAllHeaders; } BlackMisc::Network::CEntityFlags::Entity CWebDataServices::getEntitiesWithNewerHeaderTimestamp(BlackMisc::Network::CEntityFlags::Entity entities) const { CEntityFlags::Entity newerEntities = CEntityFlags::NoEntity; if (this->m_airportDataReader) { newerEntities |= this->m_airportDataReader->getEntitesWithNewerHeaderTimestamp(entities); } if (this->m_icaoDataReader) { newerEntities |= this->m_icaoDataReader->getEntitesWithNewerHeaderTimestamp(entities); } if (this->m_modelDataReader) { newerEntities |= this->m_modelDataReader->getEntitesWithNewerHeaderTimestamp(entities); } return newerEntities; } int CWebDataServices::getCacheCount(CEntityFlags::Entity entity) const { Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "Need single entity"); if (CEntityFlags::anySwiftDbEntity(entity)) { CDatabaseReader *dr = this->getDbReader(entity); if (!dr) { return -1; } return dr->getCacheCount(entity); } else { // non DB/shared entities would go here return -1; } } int CWebDataServices::getDbInfoCount(CEntityFlags::Entity entity) const { Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "Need single entity"); if (CEntityFlags::anySwiftDbEntity(entity)) { CDatabaseReader *dr = this->getDbReader(entity); if (!dr) { return -1; } return dr->getCountFromInfoObjects(entity); } else { // non DB entities would go here return -1; } } CDistributorList CWebDataServices::getDistributors() const { if (m_modelDataReader) { return m_modelDataReader->getDistributors(); } return CDistributorList(); } int CWebDataServices::getDistributorsCount() const { if (m_modelDataReader) { return m_modelDataReader->getDistributorsCount(); } return 0; } CDistributor CWebDataServices::smartDistributorSelector(const CDistributor &distributor) const { if (m_modelDataReader) { return m_modelDataReader->smartDistributorSelector(distributor); } return CDistributor(); } CDistributor CWebDataServices::smartDistributorSelector(const CDistributor &distributor, const CAircraftModel &model) const { if (m_modelDataReader) { return m_modelDataReader->smartDistributorSelector(distributor, model); } return CDistributor(); } CLiveryList CWebDataServices::getLiveries() const { if (m_modelDataReader) { return m_modelDataReader->getLiveries(); } return CLiveryList(); } int CWebDataServices::getLiveriesCount() const { if (m_modelDataReader) { return m_modelDataReader->getLiveriesCount(); } return 0; } CLivery CWebDataServices::getLiveryForCombinedCode(const QString &combinedCode) const { if (m_modelDataReader) { return m_modelDataReader->getLiveryForCombinedCode(combinedCode); } return CLivery(); } CLivery CWebDataServices::getTempLiveryOrDefault() const { if (m_modelDataReader) { return m_modelDataReader->getLiveryForCombinedCode(CLivery::tempLiveryCode()); } return CLivery(); } CLivery CWebDataServices::getStdLiveryForAirlineCode(const CAirlineIcaoCode &icao) const { if (m_modelDataReader) { return m_modelDataReader->getStdLiveryForAirlineVDesignator(icao); } return CLivery(); } CLivery CWebDataServices::getLiveryForDbKey(int id) const { if (m_modelDataReader) { return m_modelDataReader->getLiveryForDbKey(id); } return CLivery(); } CLivery CWebDataServices::smartLiverySelector(const CLivery &livery) const { if (m_modelDataReader) { return m_modelDataReader->smartLiverySelector(livery); } return livery; } CAircraftModelList CWebDataServices::getModels() const { if (m_modelDataReader) { return m_modelDataReader->getModels(); } return CAircraftModelList(); } int CWebDataServices::getModelsCount() const { if (m_modelDataReader) { return m_modelDataReader->getModelsCount(); } return 0; } QSet CWebDataServices::getModelDbKeys() const { if (m_modelDataReader) { return m_modelDataReader->getModelDbKeys(); } return QSet(); } QStringList CWebDataServices::getModelStrings() const { if (m_modelDataReader) { return m_modelDataReader->getModelStringList(); } return QStringList(); } QStringList CWebDataServices::getModelCompleterStrings(bool sorted) const { if (m_modelDataReader) { return m_modelDataReader->getModels().toCompleterStrings(sorted); } return QStringList(); } CAircraftModelList CWebDataServices::getModelsForAircraftDesignatorAndLiveryCombinedCode(const QString &aircraftDesignator, const QString &combinedCode) const { if (m_modelDataReader) { return m_modelDataReader->getModelsForAircraftDesignatorAndLiveryCombinedCode(aircraftDesignator, combinedCode); } return CAircraftModelList(); } CAircraftModel CWebDataServices::getModelForModelString(const QString &modelString) const { if (m_modelDataReader) { return m_modelDataReader->getModelForModelString(modelString); } return CAircraftModel(); } CAircraftIcaoCodeList CWebDataServices::getAircraftIcaoCodes() const { if (m_icaoDataReader) { return m_icaoDataReader->getAircraftIcaoCodes(); } return CAircraftIcaoCodeList(); } int CWebDataServices::getAircraftIcaoCodesCount() const { if (m_icaoDataReader) { return m_icaoDataReader->getAircraftIcaoCodesCount(); } return 0; } CAircraftIcaoCode CWebDataServices::getAircraftIcaoCodeForDesignator(const QString &designator) const { if (m_icaoDataReader) { return m_icaoDataReader->getAircraftIcaoCodeForDesignator(designator); } return CAircraftIcaoCode(); } CAircraftIcaoCode CWebDataServices::getAircraftIcaoCodeForDbKey(int key) const { if (m_icaoDataReader) { return m_icaoDataReader->getAircraftIcaoCodeForDbKey(key); } return CAircraftIcaoCode(); } CAircraftIcaoCode CWebDataServices::smartAircraftIcaoSelector(const CAircraftIcaoCode &icao) const { if (m_icaoDataReader) { return m_icaoDataReader->smartAircraftIcaoSelector(icao); } return icao; } CAirlineIcaoCodeList CWebDataServices::getAirlineIcaoCodes() const { if (m_icaoDataReader) { return m_icaoDataReader->getAirlineIcaoCodes(); } return CAirlineIcaoCodeList(); } int CWebDataServices::getAirlineIcaoCodesCount() const { if (m_icaoDataReader) { return m_icaoDataReader->getAirlineIcaoCodesCount(); } return 0; } CAirlineIcaoCode CWebDataServices::smartAirlineIcaoSelector(const CAirlineIcaoCode &icaoPattern, const CCallsign &callsign) const { if (m_icaoDataReader) { return m_icaoDataReader->smartAirlineIcaoSelector(icaoPattern, callsign); } return CAirlineIcaoCode(); } CAirlineIcaoCode CWebDataServices::findBestMatchByCallsign(const CCallsign &callsign) const { if (callsign.isEmpty()) { return CAirlineIcaoCode(); } const CAirlineIcaoCodeList icaos(this->getAirlineIcaoCodes()); return icaos.findBestMatchByCallsign(callsign); } CAirlineIcaoCode CWebDataServices::getAirlineIcaoCodeForDbKey(int key) const { if (m_icaoDataReader) { return m_icaoDataReader->getAirlineIcaoCodeForDbKey(key); } return CAirlineIcaoCode(); } CCountryList CWebDataServices::getCountries() const { if (m_icaoDataReader) { return m_icaoDataReader->getCountries(); } return CCountryList(); } int CWebDataServices::getCountriesCount() const { if (m_icaoDataReader) { return m_icaoDataReader->getCountriesCount(); } return 0; } CCountry CWebDataServices::getCountryForName(const QString &name) const { if (m_icaoDataReader) { return m_icaoDataReader->getCountryForName(name); } return CCountry(); } CAirportList CWebDataServices::getAirports() const { if (m_airportDataReader) { return m_airportDataReader->getAirports(); } return CAirportList(); } int CWebDataServices::getAirportsCount() const { if (m_airportDataReader) { return m_airportDataReader->getAirportsCount(); } return 0; } CCountry CWebDataServices::getCountryForIsoCode(const QString &iso) const { if (m_icaoDataReader) { return m_icaoDataReader->getCountryForIsoCode(iso); } return CCountry(); } CMetarList CWebDataServices::getMetars() const { if (m_vatsimMetarReader) { return m_vatsimMetarReader->getMetars(); } return {}; } CMetar CWebDataServices::getMetarForAirport(const CAirportIcaoCode &icao) const { if (m_vatsimMetarReader) { return m_vatsimMetarReader->getMetarForAirport(icao); } return CMetar(); } int CWebDataServices::getMetarsCount() const { if (m_vatsimMetarReader) { return m_vatsimMetarReader->getMetarsCount(); } return 0; } CStatusMessageList CWebDataServices::validateForPublishing(const CAircraftModelList &models, CAircraftModelList &validModels, CAircraftModelList &invalidModels) const { CStatusMessageList msgs(models.validateForPublishing(validModels, invalidModels)); // check against existing distributors const CDistributorList distributors(this->getDistributors()); if (!distributors.isEmpty()) { // only further check the valid ones CAircraftModelList newValidModels; CStatusMessageList msgsDistributos(validModels.validateDistributors(distributors, newValidModels, invalidModels)); validModels = newValidModels; msgs.push_back(msgsDistributos); } return msgs; } CAirlineIcaoCodeList CWebDataServices::getAirlineIcaoCodesForDesignator(const QString &designator) const { if (m_icaoDataReader) { return m_icaoDataReader->getAirlineIcaoCodesForDesignator(designator); } return CAirlineIcaoCodeList(); } void CWebDataServices::gracefulShutdown() { this->disconnect(); // all signals if (this->m_vatsimMetarReader) { this->m_vatsimMetarReader->gracefulShutdown(); } if (this->m_vatsimBookingReader) { this->m_vatsimBookingReader->gracefulShutdown(); } if (this->m_vatsimDataFileReader) { this->m_vatsimDataFileReader->gracefulShutdown(); } if (this->m_vatsimStatusReader) { this->m_vatsimStatusReader->gracefulShutdown(); } if (this->m_modelDataReader) { this->m_modelDataReader->gracefulShutdown(); } if (this->m_airportDataReader) { this->m_airportDataReader->gracefulShutdown(); } if (this->m_icaoDataReader) { this->m_icaoDataReader->gracefulShutdown(); } if (this->m_infoDataReader) { this->m_infoDataReader->gracefulShutdown(); } if (this->m_databaseWriter) { this->m_databaseWriter->gracefulShutdown(); } } CUrl CWebDataServices::getDbReaderCurrentSharedDbDataUrl() const { return CDatabaseReader::getCurrentSharedDbDataUrl(); } CEntityFlags::Entity CWebDataServices::allDbEntitiesForUsedReaders() const { // obtain entities from real readers (means when reader is really used) CEntityFlags::Entity entities = CEntityFlags::NoEntity; if (this->m_icaoDataReader) { entities |= CWebReaderFlags::allEntitiesForReaders(CWebReaderFlags::IcaoDataReader); } if (this->m_modelDataReader) { entities |= CWebReaderFlags::allEntitiesForReaders(CWebReaderFlags::ModelReader); } if (this->m_airportDataReader) { entities |= CWebReaderFlags::allEntitiesForReaders(CWebReaderFlags::AirportReader); } // when we have a config, we ignore the ones not from cache or DB if (!this->m_dbReaderConfig.isEmpty()) { CEntityFlags::Entity configuredEntities = this->m_dbReaderConfig.getEntitesCachedOrReadFromDB(); entities &= configuredEntities; } entities &= CEntityFlags::AllDbEntities; // make sure to only use DB data return entities; } const CLogCategoryList &CWebDataServices::getLogCategories() { static const BlackMisc::CLogCategoryList cats { CLogCategory("swift.datareader"), CLogCategory::webservice() }; return cats; } void CWebDataServices::initReaders(CWebReaderFlags::WebReader flags, CEntityFlags::Entity entities) { // // ---- "metadata" reader, 1/2 will trigger read directly during init // CDatabaseReaderConfigList dbReaderConfig(this->m_dbReaderConfig); const bool anyDbEntities = CEntityFlags::anySwiftDbEntity(entities); const bool needsSharedHeaders = dbReaderConfig.needsSharedHeaders(entities); const bool needsInfoObjects = dbReaderConfig.possiblyReadsFromSwiftDb(); bool c = false; // for signal connect // 1a. If any DB data, read the info objects upfront if (needsInfoObjects) { const bool databaseUp = CInfoDataReader::canPingSwiftServer(); if (!databaseUp) { dbReaderConfig.markAsDbDown(); } if (anyDbEntities && flags.testFlag(CWebReaderFlags::WebReaderFlag::InfoDataReader)) { // info data reader has a special role, it will not be triggered in triggerRead() if (databaseUp) { this->initInfoObjectReaderAndTriggerRead(); } else { CLogMessage(this).warning("DB unreachable, skipping read from info data reader"); } } } // 1b. Read shared headers if needed // --> See below after readers have been connected // 2. Status file, updating the VATSIM related caches if (flags.testFlag(CWebReaderFlags::VatsimStatusReader) || flags.testFlag(CWebReaderFlags::VatsimDataReader) || flags.testFlag(CWebReaderFlags::VatsimMetarReader)) { this->m_vatsimStatusReader = new CVatsimStatusFileReader(this); this->m_vatsimStatusReader->start(QThread::LowPriority); // no timer updates from timer here QTimer::singleShot(100, this->m_vatsimStatusReader, &CVatsimStatusFileReader::readInBackgroundThread); } // ---- "normal data", triggerRead will start read, not starting directly // 3. VATSIM bookings if (flags.testFlag(CWebReaderFlags::WebReaderFlag::VatsimBookingReader)) { this->m_vatsimBookingReader = new CVatsimBookingReader(this); c = connect(this->m_vatsimBookingReader, &CVatsimBookingReader::atcBookingsRead, this, &CWebDataServices::ps_receivedBookings); Q_ASSERT_X(c, Q_FUNC_INFO, "VATSIM booking reader signals"); c = connect(this->m_vatsimBookingReader, &CVatsimBookingReader::dataRead, this, &CWebDataServices::dataRead); Q_ASSERT_X(c, Q_FUNC_INFO, "connect failed bookings"); this->m_entitiesPeriodicallyRead |= CEntityFlags::BookingEntity; this->m_vatsimBookingReader->start(QThread::LowPriority); this->m_vatsimBookingReader->startReader(); } // 4. VATSIM data file if (flags.testFlag(CWebReaderFlags::WebReaderFlag::VatsimDataReader)) { this->m_vatsimDataFileReader = new CVatsimDataFileReader(this); c = connect(this->m_vatsimDataFileReader, &CVatsimDataFileReader::dataFileRead, this, &CWebDataServices::ps_vatsimDataFileRead); Q_ASSERT_X(c, Q_FUNC_INFO, "VATSIM data reader signals"); c = connect(this->m_vatsimDataFileReader, &CVatsimDataFileReader::dataRead, this, &CWebDataServices::dataRead); Q_ASSERT_X(c, Q_FUNC_INFO, "connect failed VATSIM data file"); this->m_entitiesPeriodicallyRead |= CEntityFlags::VatsimDataFile; this->m_vatsimDataFileReader->start(QThread::LowPriority); this->m_vatsimDataFileReader->startReader(); } // 5. VATSIM METAR data if (flags.testFlag(CWebReaderFlags::WebReaderFlag::VatsimMetarReader)) { this->m_vatsimMetarReader = new CVatsimMetarReader(this); c = connect(this->m_vatsimMetarReader, &CVatsimMetarReader::metarsRead, this, &CWebDataServices::ps_receivedMetars); Q_ASSERT_X(c, Q_FUNC_INFO, "VATSIM METAR reader signals"); c = connect(this->m_vatsimMetarReader, &CVatsimMetarReader::dataRead, this, &CWebDataServices::dataRead); Q_ASSERT_X(c, Q_FUNC_INFO, "connect failed VATSIM METAR"); this->m_entitiesPeriodicallyRead |= CEntityFlags::MetarEntity; this->m_vatsimMetarReader->start(QThread::LowPriority); this->m_vatsimMetarReader->startReader(); } // 6. ICAO data reader if (flags.testFlag(CWebReaderFlags::WebReaderFlag::IcaoDataReader)) { this->m_icaoDataReader = new CIcaoDataReader(this, dbReaderConfig); c = connect(this->m_icaoDataReader, &CIcaoDataReader::dataRead, this, &CWebDataServices::ps_readFromSwiftDb); Q_ASSERT_X(c, Q_FUNC_INFO, "Cannot connect ICAO reader signals"); c = connect(this->m_icaoDataReader, &CIcaoDataReader::dataRead, this, &CWebDataServices::dataRead); Q_ASSERT_X(c, Q_FUNC_INFO, "Cannot connect ICAO reader signals"); c = connect(this->m_icaoDataReader, &CIcaoDataReader::sharedFileHeaderRead, this, &CWebDataServices::sharedFileHeaderRead); Q_ASSERT_X(c, Q_FUNC_INFO, "Cannot connect ICAO reader signals"); this->m_icaoDataReader->start(QThread::LowPriority); } // 7. Model reader if (flags.testFlag(CWebReaderFlags::WebReaderFlag::ModelReader)) { this->m_modelDataReader = new CModelDataReader(this, dbReaderConfig); c = connect(this->m_modelDataReader, &CModelDataReader::dataRead, this, &CWebDataServices::ps_readFromSwiftDb); Q_ASSERT_X(c, Q_FUNC_INFO, "Cannot connect Model reader signals"); c = connect(this->m_modelDataReader, &CModelDataReader::dataRead, this, &CWebDataServices::dataRead); Q_ASSERT_X(c, Q_FUNC_INFO, "Cannot connect Model reader signals"); c = connect(this->m_modelDataReader, &CModelDataReader::sharedFileHeaderRead, this, &CWebDataServices::sharedFileHeaderRead); Q_ASSERT_X(c, Q_FUNC_INFO, "Cannot connect Model reader signals"); this->m_modelDataReader->start(QThread::LowPriority); } // 8. Airport list reader if (flags.testFlag(CWebReaderFlags::WebReaderFlag::AirportReader)) { this->m_airportDataReader = new CAirportDataReader(this, dbReaderConfig); c = connect(this->m_airportDataReader, &CAirportDataReader::dataRead, this, &CWebDataServices::ps_readFromSwiftDb); Q_ASSERT_X(c, Q_FUNC_INFO, "Cannot connect Model reader signals"); c = connect(this->m_airportDataReader, &CAirportDataReader::dataRead, this, &CWebDataServices::dataRead); Q_ASSERT_X(c, Q_FUNC_INFO, "Cannot connect Model reader signals"); c = connect(this->m_airportDataReader, &CAirportDataReader::sharedFileHeaderRead, this, &CWebDataServices::sharedFileHeaderRead); Q_ASSERT_X(c, Q_FUNC_INFO, "Cannot connect Model reader signals"); this->m_airportDataReader->start(QThread::LowPriority); } // 1b. Read the headers if needed c = connect(this, &CWebDataServices::sharedFileHeaderRead, this, &CWebDataServices::ps_sharedFileHeaderReceived); Q_ASSERT_X(c, Q_FUNC_INFO, "Cannot connect header signal"); if (needsSharedHeaders) { QTimer::singleShot(0, [this, entities]() { this->triggerLoadingOfSharedFilesHeaders(entities); }); } Q_UNUSED(c); // signal connect flag } void CWebDataServices::initInfoObjectReaderAndTriggerRead() { if (!this->m_infoDataReader) { this->m_infoDataReader = new CInfoDataReader(this, m_dbReaderConfig); bool c = connect(this->m_infoDataReader, &CInfoDataReader::dataRead, this, &CWebDataServices::ps_readFromSwiftDb); Q_ASSERT_X(c, Q_FUNC_INFO, "Info reader connect failed"); // relay signal c = connect(this->m_infoDataReader, &CInfoDataReader::dataRead, this, &CWebDataServices::dataRead); Q_ASSERT_X(c, Q_FUNC_INFO, "Info reader connect failed"); // start in own thread this->m_infoDataReader->start(QThread::LowPriority); } // and trigger read QTimer::singleShot(0, [this]() { this->m_infoDataReader->read(CEntityFlags::InfoObjectEntity, QDateTime()); }); } CDatabaseReader *CWebDataServices::getDbReader(CEntityFlags::Entity entity) const { Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "Need single entity"); Q_ASSERT_X(CEntityFlags::anySwiftDbEntity(entity), Q_FUNC_INFO, "No swift DB entity"); const CWebReaderFlags::WebReader wr = CWebReaderFlags::entityToReader(entity); switch (wr) { case CWebReaderFlags::IcaoDataReader: return this->m_icaoDataReader; case CWebReaderFlags::ModelReader: return this->m_modelDataReader; case CWebReaderFlags::AirportReader: return this->m_airportDataReader; default: break; } return nullptr; } void CWebDataServices::initWriters() { this->m_databaseWriter = new CDatabaseWriter( sApp->getGlobalSetup().getDbRootDirectoryUrl(), this); } bool CWebDataServices::signalEntitiesRead(CEntityFlags::Entity entities) { if (m_signalledEntities.contains(entities)) { return false; } m_signalledEntities.insert(entities); return true; } void CWebDataServices::ps_receivedBookings(const CAtcStationList &stations) { CLogMessage(this).info("Read %1 ATC bookings from network") << stations.size(); } void CWebDataServices::ps_receivedMetars(const CMetarList &metars) { CLogMessage(this).info("Read %1 METARs") << metars.size(); } void CWebDataServices::ps_vatsimDataFileRead(int lines) { CLogMessage(this).info("Read VATSIM data file, %1 lines") << lines; } void CWebDataServices::ps_readFromSwiftDb(CEntityFlags::Entity entities, CEntityFlags::ReadState state, int number) { static const CLogCategoryList cats(CLogCategoryList(this).join({ CLogCategory::webservice()})); if (state == CEntityFlags::StartRead) { return; } // just started if (CEntityFlags::isWarningOrAbove(state)) { const CStatusMessage::StatusSeverity severity = CEntityFlags::flagToSeverity(state); if (severity == CStatusMessage::SeverityWarning) { CLogMessage(cats).warning("Read data %1 entries: %2 state: %3") << CEntityFlags::flagToString(entities) << number << CEntityFlags::flagToString(state); } else { CLogMessage(cats).error("Read data %1 entries: %2 state: %3") << CEntityFlags::flagToString(entities) << number << CEntityFlags::flagToString(state); } } else { CLogMessage(cats).info("Read data %1 entries: %2 state: %3") << CEntityFlags::flagToString(entities) << number << CEntityFlags::flagToString(state); } this->m_swiftDbEntitiesRead |= entities; const int allUsedEntities = static_cast(this->allDbEntitiesForUsedReaders()); if (((static_cast(this->m_swiftDbEntitiesRead)) & allUsedEntities) == allUsedEntities) { emit allSwiftDbDataRead(); } // individual signals if (state == CEntityFlags::ReadFinished || state == CEntityFlags::ReadFinishedRestricted) { if (entities.testFlag(CEntityFlags::AirportEntity) && signalEntitiesRead(CEntityFlags::AirportEntity)) { emit swiftDbAirportsRead(); } if (entities.testFlag(CEntityFlags::AirlineIcaoEntity) && signalEntitiesRead(CEntityFlags::AirlineIcaoEntity)) { emit swiftDbAirlineIcaoRead(); } if (entities.testFlag(CEntityFlags::AircraftIcaoEntity) && signalEntitiesRead(CEntityFlags::AircraftIcaoEntity)) { emit swiftDbAircraftIcaoRead(); } if (entities.testFlag(CEntityFlags::ModelEntity) && signalEntitiesRead(CEntityFlags::ModelEntity)) { emit swiftDbModelsRead(); } if (m_swiftDbEntitiesRead.testFlag(CEntityFlags::AllIcaoEntities) && signalEntitiesRead(CEntityFlags::AllIcaoEntities)) { emit swiftDbAllIcaoEntities(); } if (m_swiftDbEntitiesRead.testFlag(CEntityFlags::ModelMatchingEntities) && signalEntitiesRead(CEntityFlags::ModelMatchingEntities)) { emit swiftDbModelMatchingEntities(); } } } void CWebDataServices::ps_sharedFileHeaderReceived(CEntityFlags::Entity entity, const QString &fileName, bool success) { Q_UNUSED(entity); Q_UNUSED(fileName); Q_UNUSED(success); if (m_signalledHeaders) { return; } if (!this->areSharedHeadersLoaded()) { return; } m_signalledHeaders = true; emit this->allSwiftSharedAllHeadersReceived(); } void CWebDataServices::readDeferredInBackground(CEntityFlags::Entity entities, int delayMs) { if (entities == CEntityFlags::NoEntity) { return; } QTimer::singleShot(delayMs, [ = ]() { this->readInBackground(entities); }); } void CWebDataServices::readInBackground(CEntityFlags::Entity entities) { this->m_initialRead = true; // read started if (CEntityFlags::anySwiftDbEntity(entities)) { // with info objects wait until info objects are loaded Q_ASSERT_X(!entities.testFlag(CEntityFlags::InfoObjectEntity), Q_FUNC_INFO, "Info object must be read upfront, do not pass as entity here"); const bool waitForInfoReader = this->m_infoDataReader && !this->m_infoDataReader->areAllDataRead() && !this->m_infoDataReader->isMarkedAsFailed(); if (waitForInfoReader) { if (!this->waitForInfoObjects(entities)) { return; } // will call this function again after some time } const bool waitForSharedHeaders = m_dbReaderConfig.needsSharedHeadersLoaded(entities); if (waitForSharedHeaders) { if (!this->waitForSharedHeaders(entities)) { return; } // will call this function again after some time } } // read entities this->triggerRead(entities); } bool CWebDataServices::waitForInfoObjects(CEntityFlags::Entity entities) { const int waitForInfoObjectsMs = 1000; // ms const int maxWaitCycles = 10; // try to read if (this->m_infoObjectTrials > maxWaitCycles) { CLogMessage(this).warning("Cannot read info objects for %1 from %2") << CEntityFlags::flagToString(entities) << this->m_infoDataReader->getInfoObjectsUrl().toQString(); // continue here and read data without info objects return true; // no need wait any longer } else if (this->m_infoDataReader->hasReceivedFirstReply()) { if (this->m_infoDataReader->areAllDataRead()) { // we have all data and carry on CLogMessage(this).info("Info objects for %1 loaded (trial %2) from %3") << CEntityFlags::flagToString(entities) << this->m_infoObjectTrials << this->m_infoDataReader->getInfoObjectsUrl().toQString(); return true; // no need wait any longer } else { // we have received a response, but not all data yet if (this->m_infoDataReader->hasReceivedOkReply()) { // ok, this means we are parsing this->m_infoObjectTrials++; this->readDeferredInBackground(entities, waitForInfoObjectsMs); return false; // wait } else { // we have a response, but a failure // means server is alive, but responded with error // such an error (access, ...) normally will not go away CLogMessage(this).error("Info objects loading for %1 failed from %2, '%3'") << CEntityFlags::flagToString(entities) << this->m_infoDataReader->getInfoObjectsUrl().toQString() << this->m_infoDataReader->getStatusMessage(); this->m_infoDataReader->setMarkedAsFailed(true); return true; // no need wait any longer } } } else { // wait for 1st reply this->m_infoObjectTrials++; this->readDeferredInBackground(entities, waitForInfoObjectsMs); return false; // wait } } bool CWebDataServices::waitForSharedHeaders(CEntityFlags::Entity entities) { const int waitForHeadersMs = 1000; // ms const int maxWaitCycles = 10; // try to read if (m_sharedHeadersTrials > maxWaitCycles) { CLogMessage(this).warning("Cannot read headers for %1") << CEntityFlags::flagToString(entities); // continue here and read data without info objects return true; // no need wait any longer } else { const bool hasAllHeaders = this->areSharedHeadersLoaded(entities); m_sharedHeadersTrials++; if (hasAllHeaders) { CLogMessage(this).info("Headers loaded for %1, trail %2") << CEntityFlags::flagToString(entities) << m_sharedHeadersTrials; } else { CLogMessage(this).info("Waiting for headers of %1, trail %2") << CEntityFlags::flagToString(entities) << m_sharedHeadersTrials; this->readDeferredInBackground(entities, waitForHeadersMs); } return hasAllHeaders; } } bool CWebDataServices::writeDbDataToDisk(const QString &dir) const { if (dir.isEmpty()) { return false; } QDir directory(dir); if (!directory.exists()) { bool s = directory.mkpath(dir); if (!s) { return false; } } if (this->getModelsCount() > 0) { QString json(QJsonDocument(this->getModels().toJson()).toJson()); bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), "models.json")); if (!s) { return false; } } if (this->getLiveriesCount() > 0) { QString json(QJsonDocument(this->getLiveries().toJson()).toJson()); bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), "liveries.json")); if (!s) { return false; } } if (this->getAirportsCount() > 0) { QString json(QJsonDocument(this->getAirports().toJson()).toJson()); bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), "airports.json")); if (!s) { return false; } } return true; } bool CWebDataServices::readDbDataFromDisk(const QString &dir, bool inBackground) { if (dir.isEmpty()) { return false; } const QDir directory(dir); if (!directory.exists()) { return false; } bool s = false; if (this->m_icaoDataReader) { s = inBackground ? this->m_icaoDataReader->readFromJsonFilesInBackground(dir) : this->m_icaoDataReader->readFromJsonFiles(dir); } if (s && this->m_modelDataReader) { s = inBackground ? this->m_modelDataReader->readFromJsonFilesInBackground(dir) : this->m_modelDataReader->readFromJsonFiles(dir); } if (s && this->m_airportDataReader) { s = inBackground ? this->m_airportDataReader->readFromJsonFilesInBackground(dir) : this->m_airportDataReader->readFromJsonFiles(dir); } return s; } bool CWebDataServices::triggerLoadingOfSharedFilesHeaders(CEntityFlags::Entity requestedEntities) { CEntityFlags::Entity triggeredEntities = CEntityFlags::NoEntity; if (this->m_modelDataReader && this->m_modelDataReader->supportsAnyOfEntities(requestedEntities)) { triggeredEntities |= this->m_modelDataReader->maskBySupportedEntities(requestedEntities); this->m_modelDataReader->requestHeadersOfSharedFiles(requestedEntities); } if (this->m_icaoDataReader && this->m_icaoDataReader->supportsAnyOfEntities(requestedEntities)) { triggeredEntities |= this->m_icaoDataReader->maskBySupportedEntities(requestedEntities); this->m_icaoDataReader->requestHeadersOfSharedFiles(requestedEntities); } if (this->m_airportDataReader && this->m_airportDataReader->supportsAnyOfEntities(requestedEntities)) { triggeredEntities |= this->m_icaoDataReader->maskBySupportedEntities(requestedEntities); this->m_airportDataReader->requestHeadersOfSharedFiles(requestedEntities); } return triggeredEntities != CEntityFlags::NoEntity; } } // ns