diff --git a/src/blackcore/airspacemonitor.cpp b/src/blackcore/airspacemonitor.cpp index 27f898169..46f3180f0 100644 --- a/src/blackcore/airspacemonitor.cpp +++ b/src/blackcore/airspacemonitor.cpp @@ -93,7 +93,7 @@ namespace BlackCore // AutoConnection: this should also avoid race conditions by updating the bookings Q_ASSERT_X(sApp->getWebDataServices(), Q_FUNC_INFO, "Missing data reader"); this->connect(sApp->getWebDataServices()->getBookingReader(), &CVatsimBookingReader::atcBookingsRead, this, &CAirspaceMonitor::ps_receivedBookings); - this->connect(sApp->getWebDataServices()->getDataFileReader(), &CVatsimDataFileReader::dataFileRead, this, &CAirspaceMonitor::ps_receivedDataFile); + this->connect(sApp->getWebDataServices()->getVatsimDataFileReader(), &CVatsimDataFileReader::dataFileRead, this, &CAirspaceMonitor::ps_receivedDataFile); // Force snapshot in the main event loop this->connect(this->m_analyzer, &CAirspaceAnalyzer::airspaceAircraftSnapshot, this, &CAirspaceMonitor::airspaceAircraftSnapshot, Qt::QueuedConnection); diff --git a/src/blackcore/webdataservices.cpp b/src/blackcore/webdataservices.cpp index 84f97aac5..a73a9f496 100644 --- a/src/blackcore/webdataservices.cpp +++ b/src/blackcore/webdataservices.cpp @@ -54,12 +54,13 @@ namespace BlackCore { if (!sApp) { return; } // shutting down - Q_ASSERT_X(QSslSocket::supportsSsl(), Q_FUNC_INFO, "missing SSL support"); + Q_ASSERT_X(QSslSocket::supportsSsl(), Q_FUNC_INFO, "Missing SSL support"); Q_ASSERT_X(sApp->isSetupAvailable(), Q_FUNC_INFO, "Setup not synchronized"); this->setObjectName("CWebDataReader"); // check if I need info objects - const bool readFromSwiftDb = dbReaderConfig.possiblyReadsFromSwiftDb(); // only cached? + const bool readFromSwiftDb = dbReaderConfig.possiblyReadsFromSwiftDb(); // DB read access + const bool writeToSwiftDb = dbReaderConfig.possiblyWritesToSwiftDb(); // DB write access if (!readFromSwiftDb && readers.testFlag(CWebReaderFlags::InfoDataReader)) { // will remove info reader because not needed @@ -76,11 +77,13 @@ namespace BlackCore CLogMessage(this).info("Using info objects for swift DB entities"); } - this->initReaders(readers); // reads info object if required - this->initWriters(); + this->initReaders(readers, entities); // reads info objects if required + if (writeToSwiftDb) + { + this->initWriters(); + } // make sure this is called in event queue, so pending tasks cam be performed - // important so info objects can be read entities &= ~CEntityFlags::InfoObjectEntity; // triggered in init readers entities &= ~CEntityFlags::VatsimStatusFile; // triggered in init readers entities &= ~this->m_entitiesPeriodicallyRead; // will be triggered by timers @@ -89,12 +92,11 @@ namespace BlackCore // but do not start all at the same time const CEntityFlags::Entity icaoPart = entities & CEntityFlags::AllIcaoAndCountries; const CEntityFlags::Entity modelPart = entities & CEntityFlags::DistributorLiveryModel; - this->readDeferredInBackground(icaoPart, 1000); - this->readDeferredInBackground(modelPart, 2000); - - CEntityFlags::Entity restEntities = entities & ~icaoPart; - restEntities &= ~modelPart; - this->readDeferredInBackground(restEntities, 3000); + CEntityFlags::Entity remainingEntities = entities & ~icaoPart; + remainingEntities &= ~modelPart; + this->readDeferredInBackground(icaoPart, 500); + this->readDeferredInBackground(modelPart, 1000); + this->readDeferredInBackground(remainingEntities, 1500); } CServerList CWebDataServices::getVatsimFsdServers() const @@ -150,9 +152,14 @@ namespace BlackCore return CStatusMessageList(); } + void CWebDataServices::triggerReadOfInfoObjects() + { + initInfoObjectReaderAndTriggerRead(); + } + bool CWebDataServices::canConnectSwiftDb() const { - if (!m_icaoDataReader && !m_modelDataReader && !m_infoDataReader) { return false; } + if (!m_icaoDataReader && !m_modelDataReader && !m_airportDataReader && !m_infoDataReader) { return false; } // use the first one to test if (m_infoDataReader) @@ -167,12 +174,17 @@ namespace BlackCore { return m_modelDataReader->hasReceivedOkReply(); } + else if (m_airportDataReader) + { + return m_airportDataReader->hasReceivedOkReply(); + } return false; } void CWebDataServices::resetSignalFlags() { - m_signaledEntities.clear(); + m_signalledEntities.clear(); + m_signalledHeaders = false; } bool CWebDataServices::hasDbAircraftData() const @@ -254,45 +266,77 @@ namespace BlackCore return triggeredRead; } - CEntityFlags::Entity CWebDataServices::triggerReloadFromDb(CEntityFlags::Entity whatToRead, const QDateTime &newerThan) + CEntityFlags::Entity CWebDataServices::triggerLoadingDirectlyFromDb(CEntityFlags::Entity whatToRead, const QDateTime &newerThan) { CEntityFlags::Entity triggeredRead = CEntityFlags::NoEntity; - if (m_infoDataReader) { - // when possible update info objects - CEntityFlags::Entity infoObjectEntity = CEntityFlags::InfoObjectEntity; - m_infoDataReader->startReadFromDbInBackgroundThread(infoObjectEntity, newerThan); - triggeredRead |= infoObjectEntity; + this->triggerReadOfInfoObjects(); + triggeredRead |= CEntityFlags::InfoObjectEntity; } if (m_icaoDataReader) { - if (whatToRead.testFlag(CEntityFlags::AircraftIcaoEntity) || whatToRead.testFlag(CEntityFlags::AirlineIcaoEntity) || whatToRead.testFlag(CEntityFlags::CountryEntity)) + if (m_icaoDataReader->supportsAnyOfEntities(whatToRead)) { - CEntityFlags::Entity icaoEntities = whatToRead & CEntityFlags::AllIcaoAndCountries; - m_icaoDataReader->startReadFromDbInBackgroundThread(icaoEntities, newerThan); + CEntityFlags::Entity icaoEntities = m_icaoDataReader->maskBySupportedEntities(whatToRead); + m_icaoDataReader->triggerLoadingDirectlyFromDb(icaoEntities, newerThan); triggeredRead |= icaoEntities; } } if (m_modelDataReader) { - if (whatToRead.testFlag(CEntityFlags::LiveryEntity) || whatToRead.testFlag(CEntityFlags::DistributorEntity) || whatToRead.testFlag(CEntityFlags::ModelEntity)) + if (m_modelDataReader->supportsAnyOfEntities(whatToRead)) { - CEntityFlags::Entity modelEntities = whatToRead & CEntityFlags::DistributorLiveryModel; - m_modelDataReader->startReadFromDbInBackgroundThread(modelEntities, newerThan); + CEntityFlags::Entity modelEntities = m_modelDataReader->maskBySupportedEntities(whatToRead); + m_modelDataReader->triggerLoadingDirectlyFromDb(modelEntities, newerThan); triggeredRead |= modelEntities; } } if (m_airportDataReader) { - if (whatToRead.testFlag(CEntityFlags::AirportEntity)) + if (m_airportDataReader->supportsAnyOfEntities(whatToRead)) { - CEntityFlags::Entity airportEntity = whatToRead & CEntityFlags::AirportEntity; - m_airportDataReader->startReadFromDbInBackgroundThread(airportEntity, newerThan); - triggeredRead |= airportEntity; + CEntityFlags::Entity airportEntities = m_airportDataReader->maskBySupportedEntities(whatToRead); + m_airportDataReader->triggerLoadingDirectlyFromDb(airportEntities, newerThan); + triggeredRead |= airportEntities; + } + } + + return triggeredRead; + } + + CEntityFlags::Entity CWebDataServices::triggerLoadingDirectlyFromSharedFiles(CEntityFlags::Entity whatToRead, bool checkCacheTsUpfront) + { + CEntityFlags::Entity triggeredRead = CEntityFlags::NoEntity; + this->triggerLoadingOfSharedFilesHeaders(whatToRead); // trigger reload of headers + + if (m_icaoDataReader) + { + if (m_icaoDataReader->supportsAnyOfEntities(whatToRead)) + { + CEntityFlags::Entity icaoEntities = m_icaoDataReader->maskBySupportedEntities(whatToRead); + triggeredRead |= m_icaoDataReader->triggerLoadingDirectlyFromSharedFiles(icaoEntities, checkCacheTsUpfront); + } + } + + if (m_modelDataReader) + { + if (m_modelDataReader->supportsAnyOfEntities(whatToRead)) + { + CEntityFlags::Entity modelEntities = m_modelDataReader->maskBySupportedEntities(whatToRead); + triggeredRead |= m_modelDataReader->triggerLoadingDirectlyFromSharedFiles(modelEntities, checkCacheTsUpfront); + } + } + + if (m_airportDataReader) + { + if (m_airportDataReader->supportsAnyOfEntities(whatToRead)) + { + CEntityFlags::Entity airportEntities = m_airportDataReader->maskBySupportedEntities(whatToRead); + triggeredRead |= m_airportDataReader->triggerLoadingDirectlyFromSharedFiles(airportEntities, checkCacheTsUpfront); } } @@ -304,7 +348,7 @@ namespace BlackCore Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "Need single entity"); if (CEntityFlags::anySwiftDbEntity(entity)) { - CDatabaseReader *dr = this->getDbReader(entity); + const CDatabaseReader *dr = this->getDbReader(entity); if (!dr) { return QDateTime(); } return dr->getCacheTimestamp(entity); } @@ -335,7 +379,7 @@ namespace BlackCore const CDatabaseReader *reader = this->getDbReader(entity); if (reader) { - return reader->getSharedFileTimestamp(entity); + return reader->getLatestSharedFileHeaderTimestamp(entity); } else { @@ -351,6 +395,47 @@ namespace BlackCore return reader->requestHeadersOfSharedFiles(entity); } + bool CWebDataServices::areSharedHeadersLoaded(CEntityFlags::Entity entities) const + { + bool hasAllHeaders = false; + do + { + if (this->m_modelDataReader && this->m_modelDataReader->supportsAnyOfEntities(entities)) + { + if (!this->m_modelDataReader->hasSharedFileHeaders(entities)) { break; } + } + if (this->m_icaoDataReader && this->m_icaoDataReader->supportsAnyOfEntities(entities)) + { + if (!this->m_icaoDataReader->hasSharedFileHeaders(entities)) { break; } + } + if (this->m_airportDataReader && this->m_airportDataReader->supportsAnyOfEntities(entities)) + { + if (!this->m_airportDataReader->hasSharedFileHeaders(entities)) { break; } + } + hasAllHeaders = true; + } + while (false); + return hasAllHeaders; + } + + BlackMisc::Network::CEntityFlags::Entity CWebDataServices::getEntitiesWithNewerHeaderTimestamp(BlackMisc::Network::CEntityFlags::Entity entities) const + { + CEntityFlags::Entity newerEntities = CEntityFlags::NoEntity; + if (this->m_airportDataReader) + { + newerEntities |= this->m_airportDataReader->getEntitesWithNewerHeaderTimestamp(entities); + } + if (this->m_icaoDataReader) + { + newerEntities |= this->m_icaoDataReader->getEntitesWithNewerHeaderTimestamp(entities); + } + if (this->m_modelDataReader) + { + newerEntities |= this->m_modelDataReader->getEntitesWithNewerHeaderTimestamp(entities); + } + return newerEntities; + } + int CWebDataServices::getCacheCount(CEntityFlags::Entity entity) const { Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "Need single entity"); @@ -362,7 +447,7 @@ namespace BlackCore } else { - // non DB entities would go here + // non DB/shared entities would go here return -1; } } @@ -643,7 +728,12 @@ namespace BlackCore if (this->m_databaseWriter) { this->m_databaseWriter->gracefulShutdown(); } } - CEntityFlags::Entity CWebDataServices::allDbEntiiesForUsedReaders() const + CUrl CWebDataServices::getDbReaderCurrentSharedDbDataUrl() const + { + return CDatabaseReader::getCurrentSharedDbDataUrl(); + } + + CEntityFlags::Entity CWebDataServices::allDbEntitiesForUsedReaders() const { // obtain entities from real readers (means when reader is really used) CEntityFlags::Entity entities = CEntityFlags::NoEntity; @@ -677,41 +767,41 @@ namespace BlackCore return cats; } - void CWebDataServices::initReaders(CWebReaderFlags::WebReader flags) + void CWebDataServices::initReaders(CWebReaderFlags::WebReader flags, CEntityFlags::Entity entities) { + // // ---- "metadata" reader, 1/2 will trigger read directly during init + // - // 1. If any DB data, read the info upfront - const bool anyDbData = flags.testFlag(CWebReaderFlags::WebReaderFlag::IcaoDataReader) || flags.testFlag(CWebReaderFlags::WebReaderFlag::ModelReader); - const bool databaseUp = CInfoDataReader::canPingSwiftServer(); CDatabaseReaderConfigList dbReaderConfig(this->m_dbReaderConfig); - if (!databaseUp) { dbReaderConfig.markAsDbDown(); } - + const bool anyDbEntities = CEntityFlags::anySwiftDbEntity(entities); + const bool needsSharedHeaders = dbReaderConfig.needsSharedHeaders(entities); + const bool needsInfoObjects = dbReaderConfig.possiblyReadsFromSwiftDb(); bool c = false; // for signal connect - Q_UNUSED(c); - if (anyDbData && flags.testFlag(CWebReaderFlags::WebReaderFlag::InfoDataReader)) + + // 1a. If any DB data, read the info objects upfront + if (needsInfoObjects) { - // info data reader has a special role, it will not be triggered in triggerRead() - if (databaseUp) - { - this->m_infoDataReader = new CInfoDataReader(this, dbReaderConfig); - c = connect(this->m_infoDataReader, &CInfoDataReader::dataRead, this, &CWebDataServices::ps_readFromSwiftDb); - Q_ASSERT_X(c, Q_FUNC_INFO, "Info object connect failed"); + const bool databaseUp = CInfoDataReader::canPingSwiftServer(); + if (!databaseUp) { dbReaderConfig.markAsDbDown(); } - // relay signal - c = connect(this->m_infoDataReader, &CInfoDataReader::dataRead, this, &CWebDataServices::dataRead); - Q_ASSERT_X(c, Q_FUNC_INFO, "Info object connect failed"); - - // start reading - this->m_infoDataReader->start(QThread::LowPriority); - QTimer::singleShot(0, [this]() { this->m_infoDataReader->read(CEntityFlags::InfoObjectEntity, QDateTime()); }); - } - else + if (anyDbEntities && flags.testFlag(CWebReaderFlags::WebReaderFlag::InfoDataReader)) { - CLogMessage(this).warning("DB unrechable, skipping read from info data reader"); + // info data reader has a special role, it will not be triggered in triggerRead() + if (databaseUp) + { + this->initInfoObjectReaderAndTriggerRead(); + } + else + { + CLogMessage(this).warning("DB unreachable, skipping read from info data reader"); + } } } + // 1b. Read shared headers if needed + // --> See below after readers have been connected + // 2. Status file, updating the VATSIM related caches if (flags.testFlag(CWebReaderFlags::VatsimStatusReader) || flags.testFlag(CWebReaderFlags::VatsimDataReader) || flags.testFlag(CWebReaderFlags::VatsimMetarReader)) { @@ -749,7 +839,7 @@ namespace BlackCore this->m_vatsimDataFileReader->startReader(); } - // 5. VATSIM metar data + // 5. VATSIM METAR data if (flags.testFlag(CWebReaderFlags::WebReaderFlag::VatsimMetarReader)) { this->m_vatsimMetarReader = new CVatsimMetarReader(this); @@ -788,7 +878,7 @@ namespace BlackCore this->m_modelDataReader->start(QThread::LowPriority); } - // 6. Airport list reader + // 8. Airport list reader if (flags.testFlag(CWebReaderFlags::WebReaderFlag::AirportReader)) { this->m_airportDataReader = new CAirportDataReader(this, dbReaderConfig); @@ -801,9 +891,35 @@ namespace BlackCore this->m_airportDataReader->start(QThread::LowPriority); } - // Trigger Shared file headers loading - //! \todo refine, check if really needed to load the headers here - QTimer::singleShot(0, [this]() { this->triggerLoadingOfSharedFilesHeaders(); }); + // 1b. Read the headers if needed + c = connect(this, &CWebDataServices::sharedFileHeaderRead, this, &CWebDataServices::ps_sharedFileHeaderReceived); + Q_ASSERT_X(c, Q_FUNC_INFO, "Cannot connect header signal"); + if (needsSharedHeaders) + { + QTimer::singleShot(0, [this, entities]() { this->triggerLoadingOfSharedFilesHeaders(entities); }); + } + + Q_UNUSED(c); // signal connect flag + } + + void CWebDataServices::initInfoObjectReaderAndTriggerRead() + { + if (!this->m_infoDataReader) + { + this->m_infoDataReader = new CInfoDataReader(this, m_dbReaderConfig); + bool c = connect(this->m_infoDataReader, &CInfoDataReader::dataRead, this, &CWebDataServices::ps_readFromSwiftDb); + Q_ASSERT_X(c, Q_FUNC_INFO, "Info reader connect failed"); + + // relay signal + c = connect(this->m_infoDataReader, &CInfoDataReader::dataRead, this, &CWebDataServices::dataRead); + Q_ASSERT_X(c, Q_FUNC_INFO, "Info reader connect failed"); + + // start in own thread + this->m_infoDataReader->start(QThread::LowPriority); + } + + // and trigger read + QTimer::singleShot(0, [this]() { this->m_infoDataReader->read(CEntityFlags::InfoObjectEntity, QDateTime()); }); } CDatabaseReader *CWebDataServices::getDbReader(CEntityFlags::Entity entity) const @@ -832,8 +948,8 @@ namespace BlackCore bool CWebDataServices::signalEntitiesRead(CEntityFlags::Entity entities) { - if (m_signaledEntities.contains(entities)) { return false; } - m_signaledEntities.insert(entities); + if (m_signalledEntities.contains(entities)) { return false; } + m_signalledEntities.insert(entities); return true; } @@ -875,7 +991,7 @@ namespace BlackCore } this->m_swiftDbEntitiesRead |= entities; - const int allUsedEntities = static_cast(this->allDbEntiiesForUsedReaders()); + const int allUsedEntities = static_cast(this->allDbEntitiesForUsedReaders()); if (((static_cast(this->m_swiftDbEntitiesRead)) & allUsedEntities) == allUsedEntities) { emit allSwiftDbDataRead(); @@ -899,6 +1015,18 @@ namespace BlackCore } } + void CWebDataServices::ps_sharedFileHeaderReceived(CEntityFlags::Entity entity, const QString &fileName, bool success) + { + Q_UNUSED(entity); + Q_UNUSED(fileName); + Q_UNUSED(success); + if (m_signalledHeaders) { return; } + + if (!this->areSharedHeadersLoaded()) { return; } + m_signalledHeaders = true; + emit this->allSwiftSharedAllHeadersReceived(); + } + void CWebDataServices::readDeferredInBackground(CEntityFlags::Entity entities, int delayMs) { if (entities == CEntityFlags::NoEntity) { return; } @@ -912,63 +1040,20 @@ namespace BlackCore { this->m_initialRead = true; // read started - const int waitForInfoObjects = 1000; // ms - const int maxWaitCycles = 10; - - // with info objects wait until info objects are loaded - Q_ASSERT_X(!entities.testFlag(CEntityFlags::InfoObjectEntity), Q_FUNC_INFO, "Info object must be read upfront"); - const bool readFromInfoReader = this->m_infoDataReader && !this->m_infoDataReader->areAllDataRead() && !this->m_infoDataReader->isMarkedAsFailed(); - if (readFromInfoReader && CEntityFlags::anySwiftDbEntity(entities)) + if (CEntityFlags::anySwiftDbEntity(entities)) { - // try to read - if (this->m_infoObjectTrials > maxWaitCycles) + // with info objects wait until info objects are loaded + Q_ASSERT_X(!entities.testFlag(CEntityFlags::InfoObjectEntity), Q_FUNC_INFO, "Info object must be read upfront, do not pass as entity here"); + const bool waitForInfoReader = this->m_infoDataReader && !this->m_infoDataReader->areAllDataRead() && !this->m_infoDataReader->isMarkedAsFailed(); + if (waitForInfoReader) { - CLogMessage(this).warning("Cannot read info objects for %1 from %2") - << CEntityFlags::flagToString(entities) - << this->m_infoDataReader->getInfoObjectsUrl().toQString(); - // continue here and read data without info objects + if (!this->waitForInfoObjects(entities)) { return; } // will call this function again after some time } - else if (this->m_infoDataReader->hasReceivedFirstReply()) + + const bool waitForSharedHeaders = m_dbReaderConfig.needsSharedHeadersLoaded(entities); + if (waitForSharedHeaders) { - if (this->m_infoDataReader->areAllDataRead()) - { - // we have all data and carry on - CLogMessage(this).info("Info objects for %1 loaded (trial %2) from %3") - << CEntityFlags::flagToString(entities) - << this->m_infoObjectTrials - << this->m_infoDataReader->getInfoObjectsUrl().toQString(); - // continue here and read data - } - else - { - // we have received a response, but not all data yet - if (this->m_infoDataReader->hasReceivedOkReply()) - { - // ok, this means we are parsing - this->m_infoObjectTrials++; - this->readDeferredInBackground(entities, waitForInfoObjects); - return; - } - else - { - // we have a response, but a failure - // means server is alive, but responded with error - // such an error (access, ...) normally will not go away - CLogMessage(this).error("Info objects loading for %1 failed from %2, '%3'") - << CEntityFlags::flagToString(entities) - << this->m_infoDataReader->getInfoObjectsUrl().toQString() - << this->m_infoDataReader->getStatusMessage(); - this->m_infoDataReader->setMarkedAsFailed(true); - // continue here and read data - } - } - } - else - { - // wait for 1st reply - this->m_infoObjectTrials++; - this->readDeferredInBackground(entities, waitForInfoObjects); - return; + if (!this->waitForSharedHeaders(entities)) { return; } // will call this function again after some time } } @@ -976,6 +1061,93 @@ namespace BlackCore this->triggerRead(entities); } + bool CWebDataServices::waitForInfoObjects(CEntityFlags::Entity entities) + { + const int waitForInfoObjectsMs = 1000; // ms + const int maxWaitCycles = 10; + + // try to read + if (this->m_infoObjectTrials > maxWaitCycles) + { + CLogMessage(this).warning("Cannot read info objects for %1 from %2") + << CEntityFlags::flagToString(entities) + << this->m_infoDataReader->getInfoObjectsUrl().toQString(); + // continue here and read data without info objects + return true; // no need wait any longer + } + else if (this->m_infoDataReader->hasReceivedFirstReply()) + { + if (this->m_infoDataReader->areAllDataRead()) + { + // we have all data and carry on + CLogMessage(this).info("Info objects for %1 loaded (trial %2) from %3") + << CEntityFlags::flagToString(entities) + << this->m_infoObjectTrials + << this->m_infoDataReader->getInfoObjectsUrl().toQString(); + return true; // no need wait any longer + } + else + { + // we have received a response, but not all data yet + if (this->m_infoDataReader->hasReceivedOkReply()) + { + // ok, this means we are parsing + this->m_infoObjectTrials++; + this->readDeferredInBackground(entities, waitForInfoObjectsMs); + return false; // wait + } + else + { + // we have a response, but a failure + // means server is alive, but responded with error + // such an error (access, ...) normally will not go away + CLogMessage(this).error("Info objects loading for %1 failed from %2, '%3'") + << CEntityFlags::flagToString(entities) + << this->m_infoDataReader->getInfoObjectsUrl().toQString() + << this->m_infoDataReader->getStatusMessage(); + this->m_infoDataReader->setMarkedAsFailed(true); + return true; // no need wait any longer + } + } + } + else + { + // wait for 1st reply + this->m_infoObjectTrials++; + this->readDeferredInBackground(entities, waitForInfoObjectsMs); + return false; // wait + } + } + + bool CWebDataServices::waitForSharedHeaders(CEntityFlags::Entity entities) + { + const int waitForHeadersMs = 1000; // ms + const int maxWaitCycles = 10; + + // try to read + if (m_sharedHeadersTrials > maxWaitCycles) + { + CLogMessage(this).warning("Cannot read headers for %1") << CEntityFlags::flagToString(entities); + // continue here and read data without info objects + return true; // no need wait any longer + } + else + { + const bool hasAllHeaders = this->areSharedHeadersLoaded(entities); + m_sharedHeadersTrials++; + if (hasAllHeaders) + { + CLogMessage(this).info("Headers loaded for %1, trail %2") << CEntityFlags::flagToString(entities) << m_sharedHeadersTrials; + } + else + { + CLogMessage(this).info("Waiting for headers of %1, trail %2") << CEntityFlags::flagToString(entities) << m_sharedHeadersTrials; + this->readDeferredInBackground(entities, waitForHeadersMs); + } + return hasAllHeaders; + } + } + bool CWebDataServices::writeDbDataToDisk(const QString &dir) const { if (dir.isEmpty()) { return false; } @@ -1000,14 +1172,10 @@ namespace BlackCore if (!s) { return false; } } - if (m_icaoDataReader) + if (this->getAirportsCount() > 0) { - bool s = m_icaoDataReader->writeToJsonFiles(directory.absolutePath()); - if (!s) { return false; } - } - if (m_modelDataReader) - { - bool s = m_modelDataReader->writeToJsonFiles(directory.absolutePath()); + QString json(QJsonDocument(this->getAirports().toJson()).toJson()); + bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), "airports.json")); if (!s) { return false; } } @@ -1033,38 +1201,32 @@ namespace BlackCore this->m_modelDataReader->readFromJsonFilesInBackground(dir) : this->m_modelDataReader->readFromJsonFiles(dir); } + if (s && this->m_airportDataReader) + { + s = inBackground ? + this->m_airportDataReader->readFromJsonFilesInBackground(dir) : + this->m_airportDataReader->readFromJsonFiles(dir); + } return s; } bool CWebDataServices::triggerLoadingOfSharedFilesHeaders(CEntityFlags::Entity requestedEntities) { CEntityFlags::Entity triggeredEntities = CEntityFlags::NoEntity; - if (this->m_modelDataReader) + if (this->m_modelDataReader && this->m_modelDataReader->supportsAnyOfEntities(requestedEntities)) { - const CEntityFlags::Entity entities = requestedEntities & CWebReaderFlags::allEntitiesForReaders(CWebReaderFlags::ModelReader); - if (entities != CEntityFlags::NoEntity) - { - triggeredEntities |= entities; - this->m_modelDataReader->requestHeadersOfSharedFiles(entities); - } + triggeredEntities |= this->m_modelDataReader->maskBySupportedEntities(requestedEntities); + this->m_modelDataReader->requestHeadersOfSharedFiles(requestedEntities); } - if (this->m_icaoDataReader) + if (this->m_icaoDataReader && this->m_icaoDataReader->supportsAnyOfEntities(requestedEntities)) { - const CEntityFlags::Entity entities = requestedEntities & CWebReaderFlags::allEntitiesForReaders(CWebReaderFlags::IcaoDataReader); - if (entities != CEntityFlags::NoEntity) - { - triggeredEntities |= entities; - this->m_icaoDataReader->requestHeadersOfSharedFiles(entities); - } + triggeredEntities |= this->m_icaoDataReader->maskBySupportedEntities(requestedEntities); + this->m_icaoDataReader->requestHeadersOfSharedFiles(requestedEntities); } - if (this->m_airportDataReader) + if (this->m_airportDataReader && this->m_airportDataReader->supportsAnyOfEntities(requestedEntities)) { - const CEntityFlags::Entity entities = requestedEntities & CWebReaderFlags::allEntitiesForReaders(CWebReaderFlags::AirportReader); - if (entities != CEntityFlags::NoEntity) - { - triggeredEntities |= entities; - this->m_airportDataReader->requestHeadersOfSharedFiles(entities); - } + triggeredEntities |= this->m_icaoDataReader->maskBySupportedEntities(requestedEntities); + this->m_airportDataReader->requestHeadersOfSharedFiles(requestedEntities); } return triggeredEntities != CEntityFlags::NoEntity; } diff --git a/src/blackcore/webdataservices.h b/src/blackcore/webdataservices.h index 1e5c942f7..5cd42dac8 100644 --- a/src/blackcore/webdataservices.h +++ b/src/blackcore/webdataservices.h @@ -98,18 +98,21 @@ namespace BlackCore //! Read ATC bookings (used to re-read) void readAtcBookingsInBackground() const; + //! Data file reader + Vatsim::CVatsimDataFileReader *getVatsimDataFileReader() const { return m_vatsimDataFileReader; } + //! Booking reader Vatsim::CVatsimBookingReader *getBookingReader() const { return m_vatsimBookingReader; } - //! Data file reader - Vatsim::CVatsimDataFileReader *getDataFileReader() const { return m_vatsimDataFileReader; } - //! Metar reader Vatsim::CVatsimMetarReader *getMetarReader() const { return m_vatsimMetarReader; } //! Info data reader Db::CInfoDataReader *getInfoDataReader() const { return m_infoDataReader; } + //! Currently used URL for shared DB data + BlackMisc::Network::CUrl getDbReaderCurrentSharedDbDataUrl() const; + //! DB writer class Db::CDatabaseWriter *getDatabaseWriter() const { return m_databaseWriter; } @@ -117,7 +120,7 @@ namespace BlackCore CWebReaderFlags::WebReader getReaderFlags() const { return m_readers; } //! All DB entities for those readers used and not ignored - BlackMisc::Network::CEntityFlags::Entity allDbEntiiesForUsedReaders() const; + BlackMisc::Network::CEntityFlags::Entity allDbEntitiesForUsedReaders() const; //! FSD servers //! \threadsafe @@ -314,12 +317,18 @@ namespace BlackCore //! Publish models to database BlackMisc::CStatusMessageList asyncPublishModels(const BlackMisc::Simulation::CAircraftModelList &models) const; + //! Trigger read of info objects + void triggerReadOfInfoObjects(); + //! Trigger read of new data //! \note requires info objects loaded upfront and uses the full cache logic BlackMisc::Network::CEntityFlags::Entity triggerRead(BlackMisc::Network::CEntityFlags::Entity whatToRead, const QDateTime &newerThan = QDateTime()); //! Trigger reload from DB, only loads the DB data and bypasses the caches checks and info objects - BlackMisc::Network::CEntityFlags::Entity triggerReloadFromDb(BlackMisc::Network::CEntityFlags::Entity whatToRead, const QDateTime &newerThan = QDateTime()); + BlackMisc::Network::CEntityFlags::Entity triggerLoadingDirectlyFromDb(BlackMisc::Network::CEntityFlags::Entity whatToRead, const QDateTime &newerThan = QDateTime()); + + //! Trigger reload from shared files, only loads the data and bypasses caches + BlackMisc::Network::CEntityFlags::Entity triggerLoadingDirectlyFromSharedFiles(BlackMisc::Network::CEntityFlags::Entity whatToRead, bool checkCacheTsUpfront); //! Trigger loading of the HTTP headers for the shared files //! \note allows to obtain the timestamps @@ -339,6 +348,12 @@ namespace BlackCore //! Request (updated) HTTP header for shared file of entity bool requestHeaderOfSharedFile(BlackMisc::Network::CEntityFlags::Entity entity); + //! Are the shared headers for the given entities loaded + bool areSharedHeadersLoaded(BlackMisc::Network::CEntityFlags::Entity entities = BlackMisc::Network::CEntityFlags::AllDbEntities) const; + + //! Those entities where the timestamp of header is newer than the cache timestamp + BlackMisc::Network::CEntityFlags::Entity getEntitiesWithNewerHeaderTimestamp(BlackMisc::Network::CEntityFlags::Entity entities) const; + //! Cache count for entity //! \threadsafe int getCacheCount(BlackMisc::Network::CEntityFlags::Entity entity) const; @@ -369,8 +384,8 @@ namespace BlackCore //! Combined read signal void dataRead(BlackMisc::Network::CEntityFlags::Entity entity, BlackMisc::Network::CEntityFlags::ReadState state, int number); - //! All swift DB data have been read - void allSwiftDbDataRead(); + //! Header of shared file read + void sharedFileHeaderRead(BlackMisc::Network::CEntityFlags::Entity entity, const QString &fileName, bool success); // simplified signals // 1) simple signature @@ -378,6 +393,12 @@ namespace BlackCore //! \name Simplified read signals //! @{ + //! All swift DB data have been read + void allSwiftDbDataRead(); + + //! All headers received + void allSwiftSharedAllHeadersReceived(); + //! All models read void swiftDbModelsRead(); @@ -397,14 +418,12 @@ namespace BlackCore void swiftDbModelMatchingEntities(); //! @} - //! //! Header of shared file read - void sharedFileHeaderRead(BlackMisc::Network::CEntityFlags::Entity entity, const QString &fileName, bool success); - public slots: //! Call CWebDataServices::readInBackground by single shot void readDeferredInBackground(BlackMisc::Network::CEntityFlags::Entity entities, int delayMs); //! First read (allows to immediately read in background) + //! \remark ensures info objects (if and only if needed) are read upfront void readInBackground(BlackMisc::Network::CEntityFlags::Entity entities = BlackMisc::Network::CEntityFlags::AllEntities); private slots: @@ -420,9 +439,15 @@ namespace BlackCore //! Read finished from reader void ps_readFromSwiftDb(BlackMisc::Network::CEntityFlags::Entity entities, BlackMisc::Network::CEntityFlags::ReadState state, int number); + //! A shared file header has been received + void ps_sharedFileHeaderReceived(BlackMisc::Network::CEntityFlags::Entity entity, const QString &fileName, bool success); + private: //! Init the readers - void initReaders(CWebReaderFlags::WebReader flags); + void initReaders(CWebReaderFlags::WebReader flags, BlackMisc::Network::CEntityFlags::Entity entities); + + //! Init the info objects readers + void initInfoObjectReaderAndTriggerRead(); //! DB reader for given entity Db::CDatabaseReader *getDbReader(BlackMisc::Network::CEntityFlags::Entity entity) const; @@ -433,13 +458,21 @@ namespace BlackCore //! Remember this entity/those enties already have been signaled bool signalEntitiesRead(BlackMisc::Network::CEntityFlags::Entity entities); - CWebReaderFlags::WebReader m_readers = CWebReaderFlags::WebReaderFlag::None; //!< which readers are available - BlackCore::Db::CDatabaseReaderConfigList m_dbReaderConfig; //!< how to read DB data + //! Wait for info objects to be read + bool waitForInfoObjects(BlackMisc::Network::CEntityFlags::Entity entities); + + //! Wait for shared headers to be read + bool waitForSharedHeaders(BlackMisc::Network::CEntityFlags::Entity entities); + + CWebReaderFlags::WebReader m_readers = CWebReaderFlags::WebReaderFlag::None; //!< which readers are available + BlackCore::Db::CDatabaseReaderConfigList m_dbReaderConfig; //!< how to read DB data BlackMisc::Network::CEntityFlags::Entity m_entitiesPeriodicallyRead = BlackMisc::Network::CEntityFlags::NoEntity; //!< those entities which are permanently updated by timers - BlackMisc::Network::CEntityFlags::Entity m_swiftDbEntitiesRead = BlackMisc::Network::CEntityFlags::NoEntity; //!< entities read - bool m_initialRead = false; //!< Initial read started - int m_infoObjectTrials = 0; //!< Tried to read info objects - QSet m_signaledEntities; //!< remember signales entites + BlackMisc::Network::CEntityFlags::Entity m_swiftDbEntitiesRead = BlackMisc::Network::CEntityFlags::NoEntity; //!< entities read + bool m_initialRead = false; //!< Initial read started + bool m_signalledHeaders = false; //!< headers loading has been signalled + int m_infoObjectTrials = 0; //!< Tried to read info objects + int m_sharedHeadersTrials = 0; //!< Tried to read shared file headers + QSet m_signalledEntities; //!< remember signalled entites // for reading XML and VATSIM data files Vatsim::CVatsimStatusFileReader *m_vatsimStatusReader = nullptr; diff --git a/src/blackgui/components/datainfoareacomponent.cpp b/src/blackgui/components/datainfoareacomponent.cpp index 7dff0657a..ed1bbcd79 100644 --- a/src/blackgui/components/datainfoareacomponent.cpp +++ b/src/blackgui/components/datainfoareacomponent.cpp @@ -165,7 +165,7 @@ namespace BlackGui void CDataInfoAreaComponent::requestUpdateOfAllDbData() { - sGui->getWebDataServices()->triggerReloadFromDb(CEntityFlags::AllDbEntitiesNoInfoObjectsNoAirports, QDateTime()); + sGui->getWebDataServices()->triggerLoadingDirectlyFromDb(CEntityFlags::AllDbEntitiesNoInfoObjectsNoAirports, QDateTime()); } void CDataInfoAreaComponent::requestUpdatedData(CEntityFlags::Entity entity) diff --git a/src/blackgui/components/dbaircrafticaocomponent.cpp b/src/blackgui/components/dbaircrafticaocomponent.cpp index bbeb1a5ed..1d59dfbe6 100644 --- a/src/blackgui/components/dbaircrafticaocomponent.cpp +++ b/src/blackgui/components/dbaircrafticaocomponent.cpp @@ -70,7 +70,7 @@ namespace BlackGui void CDbAircraftIcaoComponent::ps_reload() { if (!sGui) { return; } - sGui->getWebDataServices()->triggerReloadFromDb(CEntityFlags::AircraftIcaoEntity, QDateTime()); + sGui->getWebDataServices()->triggerLoadingDirectlyFromDb(CEntityFlags::AircraftIcaoEntity, QDateTime()); } } // ns } // ns diff --git a/src/blackgui/components/dbairlineicaocomponent.cpp b/src/blackgui/components/dbairlineicaocomponent.cpp index 9f1fc3925..b021c2c50 100644 --- a/src/blackgui/components/dbairlineicaocomponent.cpp +++ b/src/blackgui/components/dbairlineicaocomponent.cpp @@ -63,7 +63,7 @@ namespace BlackGui void CDbAirlineIcaoComponent::ps_reload() { if (!sGui || !sGui->hasWebDataServices()) { return; } - sGui->getWebDataServices()->triggerReloadFromDb(CEntityFlags::AirlineIcaoEntity, QDateTime()); + sGui->getWebDataServices()->triggerLoadingDirectlyFromDb(CEntityFlags::AirlineIcaoEntity, QDateTime()); } } // ns } // ns diff --git a/src/blackgui/components/dbcountrycomponent.cpp b/src/blackgui/components/dbcountrycomponent.cpp index f4561d924..84f40702e 100644 --- a/src/blackgui/components/dbcountrycomponent.cpp +++ b/src/blackgui/components/dbcountrycomponent.cpp @@ -58,7 +58,7 @@ namespace BlackGui void CDbCountryComponent::ps_reload() { if (!sGui || !sGui->getWebDataServices()) { return; } - sApp->getWebDataServices()->triggerReloadFromDb(CEntityFlags::CountryEntity); + sApp->getWebDataServices()->triggerLoadingDirectlyFromDb(CEntityFlags::CountryEntity); } } // ns } // ns diff --git a/src/blackgui/components/dbdistributorcomponent.cpp b/src/blackgui/components/dbdistributorcomponent.cpp index 84088f104..86c988178 100644 --- a/src/blackgui/components/dbdistributorcomponent.cpp +++ b/src/blackgui/components/dbdistributorcomponent.cpp @@ -60,7 +60,7 @@ namespace BlackGui void CDbDistributorComponent::ps_reload() { if (!sGui) { return; } - sGui->getWebDataServices()->triggerReloadFromDb(CEntityFlags::DistributorEntity); + sGui->getWebDataServices()->triggerLoadingDirectlyFromDb(CEntityFlags::DistributorEntity); } } // ns } // ns diff --git a/src/blackgui/components/dbliverycomponent.cpp b/src/blackgui/components/dbliverycomponent.cpp index 6a0d09b5c..41504f38d 100644 --- a/src/blackgui/components/dbliverycomponent.cpp +++ b/src/blackgui/components/dbliverycomponent.cpp @@ -71,7 +71,7 @@ namespace BlackGui void CDbLiveryComponent::ps_reload() { if (!sGui) { return; } - sGui->getWebDataServices()->triggerReloadFromDb(CEntityFlags::LiveryEntity); + sGui->getWebDataServices()->triggerLoadingDirectlyFromDb(CEntityFlags::LiveryEntity); } } // ns } // ns diff --git a/src/blackgui/components/dbmodelcomponent.cpp b/src/blackgui/components/dbmodelcomponent.cpp index 457699b76..c5b05a334 100644 --- a/src/blackgui/components/dbmodelcomponent.cpp +++ b/src/blackgui/components/dbmodelcomponent.cpp @@ -73,7 +73,7 @@ namespace BlackGui CAircraftModel model(ui->tvp_AircraftModel->container().latestObject()); ts = model.getUtcTimestamp(); } - sGui->getWebDataServices()->triggerReloadFromDb(CEntityFlags::ModelEntity, ts); + sGui->getWebDataServices()->triggerLoadingDirectlyFromDb(CEntityFlags::ModelEntity, ts); } void CDbModelComponent::ps_modelsRead(CEntityFlags::Entity entity, CEntityFlags::ReadState readState, int count) @@ -91,7 +91,7 @@ namespace BlackGui void CDbModelComponent::ps_reload() { if (!sGui) { return; } - sGui->getWebDataServices()->triggerReloadFromDb(CEntityFlags::ModelEntity); + sGui->getWebDataServices()->triggerLoadingDirectlyFromDb(CEntityFlags::ModelEntity); } void CDbModelComponent::ps_onStyleSheetChanged()