From 0696eed04f0d5bf11704dabefd75f0a986a8b05e Mon Sep 17 00:00:00 2001 From: Klaus Basan Date: Thu, 20 Apr 2017 03:01:28 +0200 Subject: [PATCH] Ref T24, adjusted web data services * no longer using header timestamps, removed functions * renamed functions to distinguish shared/DB info data * added 2nd reader m_sharedInfoDataReader --- src/blackcore/webdataservices.cpp | 309 ++++++++++++------------------ src/blackcore/webdataservices.h | 89 +++++---- 2 files changed, 166 insertions(+), 232 deletions(-) diff --git a/src/blackcore/webdataservices.cpp b/src/blackcore/webdataservices.cpp index df32fbd93..8976dca10 100644 --- a/src/blackcore/webdataservices.cpp +++ b/src/blackcore/webdataservices.cpp @@ -72,7 +72,7 @@ namespace BlackCore // get entities to be read CEntityFlags::Entity entities = CWebReaderFlags::allEntitiesForReaders(readers); - if (entities.testFlag(CEntityFlags::InfoObjectEntity)) + if (entities.testFlag(CEntityFlags::DbInfoObjectEntity)) { Q_ASSERT_X(readers.testFlag(CWebReaderFlags::DbInfoDataReader), Q_FUNC_INFO, "info object but no reader"); CLogMessage(this).info("Using info objects for swift DB entities"); @@ -85,7 +85,7 @@ namespace BlackCore } // make sure this is called in event queue, so pending tasks cam be performed - entities &= ~CEntityFlags::InfoObjectEntity; // triggered in init readers + entities &= ~CEntityFlags::DbInfoObjectEntity; // triggered in init readers entities &= ~CEntityFlags::VatsimStatusFile; // triggered in init readers entities &= ~this->m_entitiesPeriodicallyRead; // will be triggered by timers @@ -153,11 +153,16 @@ namespace BlackCore return CStatusMessageList(); } - void CWebDataServices::triggerReadOfInfoObjects() + void CWebDataServices::triggerReadOfDbInfoObjects() { initDbInfoObjectReaderAndTriggerRead(); } + void CWebDataServices::triggerReadOfSharedInfoObjects() + { + initSharedInfoObjectReaderAndTriggerRead(); + } + bool CWebDataServices::canConnectSwiftDb() const { if (!m_icaoDataReader && !m_modelDataReader && !m_airportDataReader && !m_dbInfoDataReader) { return false; } @@ -195,8 +200,11 @@ namespace BlackCore void CWebDataServices::admitDbCaches(CEntityFlags::Entity entities) { - // hint: all the readers use own threads + // hint: those 2 are currently doing nothing, but this might change in the future if (this->m_dbInfoDataReader) { this->m_dbInfoDataReader->admitCaches(entities); } + if (this->m_sharedInfoDataReader) { this->m_sharedInfoDataReader->admitCaches(entities); } + + // hint: all the readers use own threads if (this->m_modelDataReader) { this->m_modelDataReader->admitCaches(entities); } if (this->m_icaoDataReader) { this->m_icaoDataReader->admitCaches(entities); } if (this->m_airportDataReader) { this->m_airportDataReader->admitCaches(entities); } @@ -205,7 +213,7 @@ namespace BlackCore CEntityFlags::Entity CWebDataServices::triggerRead(CEntityFlags::Entity whatToRead, const QDateTime &newerThan) { m_initialRead = true; // read started - Q_ASSERT_X(!whatToRead.testFlag(CEntityFlags::InfoObjectEntity), Q_FUNC_INFO, "Info object must be read upfront"); + Q_ASSERT_X(!whatToRead.testFlag(CEntityFlags::DbInfoObjectEntity), Q_FUNC_INFO, "Info object must be read upfront"); CEntityFlags::Entity triggeredRead = CEntityFlags::NoEntity; if (m_vatsimDataFileReader) { @@ -272,8 +280,8 @@ namespace BlackCore CEntityFlags::Entity triggeredRead = CEntityFlags::NoEntity; if (m_dbInfoDataReader) { - this->triggerReadOfInfoObjects(); - triggeredRead |= CEntityFlags::InfoObjectEntity; + this->triggerReadOfDbInfoObjects(); + triggeredRead |= CEntityFlags::DbInfoObjectEntity; } if (m_icaoDataReader) @@ -312,7 +320,7 @@ namespace BlackCore CEntityFlags::Entity CWebDataServices::triggerLoadingDirectlyFromSharedFiles(CEntityFlags::Entity whatToRead, bool checkCacheTsUpfront) { CEntityFlags::Entity triggeredRead = CEntityFlags::NoEntity; - this->triggerLoadingOfSharedFilesHeaders(whatToRead); // trigger reload of headers + this->triggerReadOfSharedInfoObjects(); // trigger reload of headers if (m_icaoDataReader) { @@ -367,7 +375,7 @@ namespace BlackCore { CInfoDataReader *ir = this->getDbInfoDataReader(); if (!ir) { return QDateTime(); } - return ir->getLatestEntityTimestampFromInfoObjects(entity); + return ir->getLatestEntityTimestampFromDbInfoObjects(entity); } else { @@ -375,12 +383,13 @@ namespace BlackCore } } - QDateTime CWebDataServices::getSharedFileTimestamp(CEntityFlags::Entity entity) const + QDateTime CWebDataServices::getSharedInfoObjectTimestamp(CEntityFlags::Entity entity) const { const CDatabaseReader *reader = this->getDbReader(entity); if (reader) { - return reader->getLatestSharedFileHeaderTimestamp(entity); + Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "need single entity"); + return reader->getLatestEntityTimestampFromSharedInfoObjects(entity); } else { @@ -388,53 +397,11 @@ namespace BlackCore } } - bool CWebDataServices::requestHeaderOfSharedFile(CEntityFlags::Entity entity) + CEntityFlags::Entity CWebDataServices::getEntitiesWithNewerSharedFile(CEntityFlags::Entity entities) const { - Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "Need single entity"); - CDatabaseReader *reader = this->getDbReader(entity); - if (!reader) { return false; } - return reader->requestHeadersOfSharedFiles(entity); - } - - bool CWebDataServices::areSharedHeadersLoaded(CEntityFlags::Entity entities) const - { - bool hasAllHeaders = false; - do - { - if (this->m_modelDataReader && this->m_modelDataReader->supportsAnyOfEntities(entities)) - { - if (!this->m_modelDataReader->hasSharedFileHeaders(entities)) { break; } - } - if (this->m_icaoDataReader && this->m_icaoDataReader->supportsAnyOfEntities(entities)) - { - if (!this->m_icaoDataReader->hasSharedFileHeaders(entities)) { break; } - } - if (this->m_airportDataReader && this->m_airportDataReader->supportsAnyOfEntities(entities)) - { - if (!this->m_airportDataReader->hasSharedFileHeaders(entities)) { break; } - } - hasAllHeaders = true; - } - while (false); - return hasAllHeaders; - } - - BlackMisc::Network::CEntityFlags::Entity CWebDataServices::getEntitiesWithNewerHeaderTimestamp(BlackMisc::Network::CEntityFlags::Entity entities) const - { - CEntityFlags::Entity newerEntities = CEntityFlags::NoEntity; - if (this->m_airportDataReader) - { - newerEntities |= this->m_airportDataReader->getEntitesWithNewerHeaderTimestamp(entities); - } - if (this->m_icaoDataReader) - { - newerEntities |= this->m_icaoDataReader->getEntitesWithNewerHeaderTimestamp(entities); - } - if (this->m_modelDataReader) - { - newerEntities |= this->m_modelDataReader->getEntitesWithNewerHeaderTimestamp(entities); - } - return newerEntities; + Q_ASSERT_X(this->m_sharedInfoDataReader, Q_FUNC_INFO, "Shared info reader was not initialized"); + if (this->m_sharedInfoDataReader->getInfoObjectCount() < 1) { return CEntityFlags::NoEntity; } + return this->m_sharedInfoDataReader->getEntitesWithNewerSharedInfoObject(entities); } int CWebDataServices::getCacheCount(CEntityFlags::Entity entity) const @@ -453,20 +420,16 @@ namespace BlackCore } } - int CWebDataServices::getDbInfoCount(CEntityFlags::Entity entity) const + int CWebDataServices::getDbInfoObjectCount(CEntityFlags::Entity entity) const { - Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "Need single entity"); - if (CEntityFlags::anySwiftDbEntity(entity)) - { - CDatabaseReader *dr = this->getDbReader(entity); - if (!dr) { return -1; } - return dr->getCountFromInfoObjects(entity); - } - else - { - // non DB entities would go here - return -1; - } + if (!this->m_dbInfoDataReader) return -1; + return this->getInfoObjectCount(entity, this->m_dbInfoDataReader); + } + + int CWebDataServices::getSharedInfoObjectCount(CEntityFlags::Entity entity) const + { + if (!this->m_sharedInfoDataReader) return -1; + return this->getInfoObjectCount(entity, this->m_sharedInfoDataReader); } CDistributorList CWebDataServices::getDistributors() const @@ -782,12 +745,12 @@ namespace BlackCore CDatabaseReaderConfigList dbReaderConfig(this->m_dbReaderConfig); const bool anyDbEntities = CEntityFlags::anySwiftDbEntity(entities); - const bool needsSharedHeaders = dbReaderConfig.needsSharedHeaders(entities); - const bool needsInfoObjects = dbReaderConfig.possiblyReadsFromSwiftDb(); + const bool needsSharedInfoObjects = dbReaderConfig.needsSharedInfoObjects(entities); + const bool needsDbInfoObjects = dbReaderConfig.possiblyReadsFromSwiftDb(); bool c = false; // for signal connect // 1a. If any DB data, read the info objects upfront - if (needsInfoObjects) + if (needsDbInfoObjects) { const bool databaseUp = CInfoDataReader::canPingSwiftServer(); if (!databaseUp) { dbReaderConfig.markAsDbDown(); } @@ -866,12 +829,10 @@ namespace BlackCore if (flags.testFlag(CWebReaderFlags::WebReaderFlag::IcaoDataReader)) { this->m_icaoDataReader = new CIcaoDataReader(this, dbReaderConfig); - c = connect(this->m_icaoDataReader, &CIcaoDataReader::dataRead, this, &CWebDataServices::ps_readFromSwiftDb); + c = connect(this->m_icaoDataReader, &CIcaoDataReader::dataRead, this, &CWebDataServices::ps_readFromSwiftReader); Q_ASSERT_X(c, Q_FUNC_INFO, "Cannot connect ICAO reader signals"); c = connect(this->m_icaoDataReader, &CIcaoDataReader::dataRead, this, &CWebDataServices::dataRead); Q_ASSERT_X(c, Q_FUNC_INFO, "Cannot connect ICAO reader signals"); - c = connect(this->m_icaoDataReader, &CIcaoDataReader::sharedFileHeaderRead, this, &CWebDataServices::sharedFileHeaderRead); - Q_ASSERT_X(c, Q_FUNC_INFO, "Cannot connect ICAO reader signals"); this->m_icaoDataReader->start(QThread::LowPriority); } @@ -879,12 +840,10 @@ namespace BlackCore if (flags.testFlag(CWebReaderFlags::WebReaderFlag::ModelReader)) { this->m_modelDataReader = new CModelDataReader(this, dbReaderConfig); - c = connect(this->m_modelDataReader, &CModelDataReader::dataRead, this, &CWebDataServices::ps_readFromSwiftDb); + c = connect(this->m_modelDataReader, &CModelDataReader::dataRead, this, &CWebDataServices::ps_readFromSwiftReader); Q_ASSERT_X(c, Q_FUNC_INFO, "Cannot connect Model reader signals"); c = connect(this->m_modelDataReader, &CModelDataReader::dataRead, this, &CWebDataServices::dataRead); Q_ASSERT_X(c, Q_FUNC_INFO, "Cannot connect Model reader signals"); - c = connect(this->m_modelDataReader, &CModelDataReader::sharedFileHeaderRead, this, &CWebDataServices::sharedFileHeaderRead); - Q_ASSERT_X(c, Q_FUNC_INFO, "Cannot connect Model reader signals"); this->m_modelDataReader->start(QThread::LowPriority); } @@ -892,23 +851,13 @@ namespace BlackCore if (flags.testFlag(CWebReaderFlags::WebReaderFlag::AirportReader)) { this->m_airportDataReader = new CAirportDataReader(this, dbReaderConfig); - c = connect(this->m_airportDataReader, &CAirportDataReader::dataRead, this, &CWebDataServices::ps_readFromSwiftDb); + c = connect(this->m_airportDataReader, &CAirportDataReader::dataRead, this, &CWebDataServices::ps_readFromSwiftReader); Q_ASSERT_X(c, Q_FUNC_INFO, "Cannot connect Model reader signals"); c = connect(this->m_airportDataReader, &CAirportDataReader::dataRead, this, &CWebDataServices::dataRead); Q_ASSERT_X(c, Q_FUNC_INFO, "Cannot connect Model reader signals"); - c = connect(this->m_airportDataReader, &CAirportDataReader::sharedFileHeaderRead, this, &CWebDataServices::sharedFileHeaderRead); - Q_ASSERT_X(c, Q_FUNC_INFO, "Cannot connect Model reader signals"); this->m_airportDataReader->start(QThread::LowPriority); } - // 1b. Read the headers if needed - c = connect(this, &CWebDataServices::sharedFileHeaderRead, this, &CWebDataServices::ps_sharedFileHeaderReceived); - Q_ASSERT_X(c, Q_FUNC_INFO, "Cannot connect header signal"); - if (needsSharedHeaders) - { - QTimer::singleShot(0, [this, entities]() { this->triggerLoadingOfSharedFilesHeaders(entities); }); - } - Q_UNUSED(c); // signal connect flag } @@ -916,8 +865,8 @@ namespace BlackCore { if (!this->m_dbInfoDataReader) { - this->m_dbInfoDataReader = new CInfoDataReader(this, m_dbReaderConfig); - bool c = connect(this->m_dbInfoDataReader, &CInfoDataReader::dataRead, this, &CWebDataServices::ps_readFromSwiftDb); + this->m_dbInfoDataReader = new CInfoDataReader(this, m_dbReaderConfig, CDbFlags::DbReading); + bool c = connect(this->m_dbInfoDataReader, &CInfoDataReader::dataRead, this, &CWebDataServices::ps_readFromSwiftReader); Q_ASSERT_X(c, Q_FUNC_INFO, "Info reader connect failed"); // relay signal @@ -933,6 +882,27 @@ namespace BlackCore QTimer::singleShot(0, [this]() { this->m_dbInfoDataReader->read(); }); } + void CWebDataServices::initSharedInfoObjectReaderAndTriggerRead() + { + if (!this->m_sharedInfoDataReader) + { + this->m_sharedInfoDataReader = new CInfoDataReader(this, m_dbReaderConfig, CDbFlags::Shared); + bool c = connect(this->m_sharedInfoDataReader, &CInfoDataReader::dataRead, this, &CWebDataServices::ps_readFromSwiftReader); + Q_ASSERT_X(c, Q_FUNC_INFO, "Info reader connect failed"); + + // relay signal + c = connect(this->m_sharedInfoDataReader, &CInfoDataReader::dataRead, this, &CWebDataServices::dataRead); + Q_ASSERT_X(c, Q_FUNC_INFO, "Info reader connect failed"); + Q_UNUSED(c); + + // start in own thread + this->m_sharedInfoDataReader->start(QThread::LowPriority); + } + + // and trigger read + QTimer::singleShot(0, [this]() { this->m_sharedInfoDataReader->read(); }); + } + CDatabaseReader *CWebDataServices::getDbReader(CEntityFlags::Entity entity) const { Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "Need single entity"); @@ -957,13 +927,29 @@ namespace BlackCore this); } - bool CWebDataServices::signalEntitiesRead(CEntityFlags::Entity entities) + bool CWebDataServices::signalEntitiesAlreadyRead(CEntityFlags::Entity entities) { if (m_signalledEntities.contains(entities)) { return false; } m_signalledEntities.insert(entities); return true; } + int CWebDataServices::getInfoObjectCount(CEntityFlags::Entity entity, CInfoDataReader *reader) const + { + Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "Need single entity"); + Q_ASSERT_X(reader, Q_FUNC_INFO, "Need reader"); + if (CEntityFlags::anySwiftDbEntity(entity)) + { + const CDbInfo info = reader->getInfoObjects().findFirstByEntityOrDefault(entity); + return info.getEntries(); + } + else + { + // non DB entities would go here + return -1; + } + } + void CWebDataServices::ps_receivedBookings(const CAtcStationList &stations) { CLogMessage(this).info("Read %1 ATC bookings from network") << stations.size(); @@ -979,7 +965,7 @@ namespace BlackCore CLogMessage(this).info("Read VATSIM data file, %1 lines") << lines; } - void CWebDataServices::ps_readFromSwiftDb(CEntityFlags::Entity entities, CEntityFlags::ReadState state, int number) + void CWebDataServices::ps_readFromSwiftReader(CEntityFlags::Entity entities, CEntityFlags::ReadState state, int number) { static const CLogCategoryList cats(CLogCategoryList(this).join({ CLogCategory::webservice()})); @@ -1011,33 +997,23 @@ namespace BlackCore // individual signals if (state == CEntityFlags::ReadFinished || state == CEntityFlags::ReadFinishedRestricted) { - if (entities.testFlag(CEntityFlags::AirportEntity) && signalEntitiesRead(CEntityFlags::AirportEntity)) { emit swiftDbAirportsRead(); } - if (entities.testFlag(CEntityFlags::AirlineIcaoEntity) && signalEntitiesRead(CEntityFlags::AirlineIcaoEntity)) { emit swiftDbAirlineIcaoRead(); } - if (entities.testFlag(CEntityFlags::AircraftIcaoEntity) && signalEntitiesRead(CEntityFlags::AircraftIcaoEntity)) { emit swiftDbAircraftIcaoRead(); } - if (entities.testFlag(CEntityFlags::ModelEntity) && signalEntitiesRead(CEntityFlags::ModelEntity)) { emit swiftDbModelsRead(); } - if (m_swiftDbEntitiesRead.testFlag(CEntityFlags::AllIcaoEntities) && signalEntitiesRead(CEntityFlags::AllIcaoEntities)) + if (entities.testFlag(CEntityFlags::AirportEntity) && signalEntitiesAlreadyRead(CEntityFlags::AirportEntity)) { emit swiftDbAirportsRead(); } + if (entities.testFlag(CEntityFlags::AirlineIcaoEntity) && signalEntitiesAlreadyRead(CEntityFlags::AirlineIcaoEntity)) { emit swiftDbAirlineIcaoRead(); } + if (entities.testFlag(CEntityFlags::AircraftIcaoEntity) && signalEntitiesAlreadyRead(CEntityFlags::AircraftIcaoEntity)) { emit swiftDbAircraftIcaoRead(); } + if (entities.testFlag(CEntityFlags::ModelEntity) && signalEntitiesAlreadyRead(CEntityFlags::ModelEntity)) { emit swiftDbModelsRead(); } + if (entities.testFlag(CEntityFlags::SharedInfoObjectEntity)) { emit sharedInfoObjectsRead(); } + + if (m_swiftDbEntitiesRead.testFlag(CEntityFlags::AllIcaoEntities) && signalEntitiesAlreadyRead(CEntityFlags::AllIcaoEntities)) { emit swiftDbAllIcaoEntities(); } - if (m_swiftDbEntitiesRead.testFlag(CEntityFlags::ModelMatchingEntities) && signalEntitiesRead(CEntityFlags::ModelMatchingEntities)) + if (m_swiftDbEntitiesRead.testFlag(CEntityFlags::ModelMatchingEntities) && signalEntitiesAlreadyRead(CEntityFlags::ModelMatchingEntities)) { emit swiftDbModelMatchingEntities(); } } } - void CWebDataServices::ps_sharedFileHeaderReceived(CEntityFlags::Entity entity, const QString &fileName, bool success) - { - Q_UNUSED(entity); - Q_UNUSED(fileName); - Q_UNUSED(success); - if (m_signalledHeaders) { return; } - - if (!this->areSharedHeadersLoaded()) { return; } - m_signalledHeaders = true; - emit this->allSwiftSharedAllHeadersReceived(); - } - void CWebDataServices::readDeferredInBackground(CEntityFlags::Entity entities, int delayMs) { if (entities == CEntityFlags::NoEntity) { return; } @@ -1050,21 +1026,20 @@ namespace BlackCore void CWebDataServices::readInBackground(CEntityFlags::Entity entities) { this->m_initialRead = true; // read started - if (CEntityFlags::anySwiftDbEntity(entities)) { // with info objects wait until info objects are loaded - Q_ASSERT_X(!entities.testFlag(CEntityFlags::InfoObjectEntity), Q_FUNC_INFO, "Info object must be read upfront, do not pass as entity here"); - const bool waitForInfoReader = this->m_dbInfoDataReader && !this->m_dbInfoDataReader->areAllDataRead() && !this->m_dbInfoDataReader->isMarkedAsFailed(); - if (waitForInfoReader) + Q_ASSERT_X(!entities.testFlag(CEntityFlags::DbInfoObjectEntity), Q_FUNC_INFO, "Info object must be read upfront, do not pass as entity here"); + const bool waitForDbInfoReader = this->m_dbInfoDataReader && !this->m_dbInfoDataReader->areAllDataRead() && !this->m_dbInfoDataReader->isMarkedAsFailed(); + if (waitForDbInfoReader) { if (!this->waitForDbInfoObjects(entities)) { return; } // will call this function again after some time } - const bool waitForSharedHeaders = m_dbReaderConfig.needsSharedHeadersLoaded(entities); - if (waitForSharedHeaders) + const bool waitForSharedInfoFile = m_dbReaderConfig.needsSharedInfoFileLoaded(entities); + if (waitForSharedInfoFile) { - if (!this->waitForSharedHeaders(entities)) { return; } // will call this function again after some time + if (!this->waitForSharedInfoObjects(entities)) { return; } // will call this function again after some time } } @@ -1074,50 +1049,54 @@ namespace BlackCore bool CWebDataServices::waitForDbInfoObjects(CEntityFlags::Entity entities) { + return this->waitForInfoObjects(entities, "DB", this->m_dbInfoDataReader, this->m_dbInfoObjectTrials); + } + + bool CWebDataServices::waitForSharedInfoObjects(CEntityFlags::Entity entities) + { + return this->waitForInfoObjects(entities, "shared", this->m_sharedInfoDataReader, this->m_sharedInfoObjectsTrials); + } + + bool CWebDataServices::waitForInfoObjects(CEntityFlags::Entity entities, const QString &info, CInfoDataReader *reader, int &trials) + { + Q_ASSERT_X(reader, Q_FUNC_INFO, "Need info data reader"); + const int waitForInfoObjectsMs = 1000; // ms const int maxWaitCycles = 10; // try to read - if (this->m_dbInfoObjectTrials > maxWaitCycles) + if (trials > maxWaitCycles) { - CLogMessage(this).warning("Cannot read info objects for '%1' from '%2'") - << CEntityFlags::flagToString(entities) - << this->m_dbInfoDataReader->getInfoObjectsUrl().toQString(); + CLogMessage(this).warning("Cannot read %1 info objects for '%2' from '%3'") << info << CEntityFlags::flagToString(entities) << reader->getInfoObjectsUrl().toQString(); + // continue here and read data without info objects - this->m_dbInfoDataReader->setMarkedAsFailed(true); + reader->setMarkedAsFailed(true); return true; // carry on, regardless of situation } - else if (this->m_dbInfoDataReader->hasReceivedFirstReply()) + else if (reader->hasReceivedFirstReply()) { - if (this->m_dbInfoDataReader->areAllDataRead()) + if (reader->areAllDataRead()) { // we have all data and carry on - CLogMessage(this).info("Info objects for '%1' loaded (trial %2) from '%3'") - << CEntityFlags::flagToString(entities) - << this->m_dbInfoObjectTrials - << this->m_dbInfoDataReader->getInfoObjectsUrl().toQString(); + CLogMessage(this).info("Info objects (%1) for '%2' loaded (trial %3) from '%4'") << info << CEntityFlags::flagToString(entities) << trials << reader->getInfoObjectsUrl().toQString(); return true; // no need to wait any longer } else { // we have received a response, but not all data yet - if (this->m_dbInfoDataReader->hasReceivedOkReply()) + if (reader->hasReceivedOkReply()) { // ok, this means we are parsing - this->m_dbInfoObjectTrials++; + trials++; this->readDeferredInBackground(entities, waitForInfoObjectsMs); return false; // wait } else { - // we have a response, but a failure - // means server is alive, but responded with error + // we have a response, but a failure, means server is alive, but responded with error // such an error (access, ...) normally will not go away - CLogMessage(this).error("Info objects loading for '%1' failed from '%2', '%3'") - << CEntityFlags::flagToString(entities) - << this->m_dbInfoDataReader->getInfoObjectsUrl().toQString() - << this->m_dbInfoDataReader->getStatusMessage(); - this->m_dbInfoDataReader->setMarkedAsFailed(true); + CLogMessage(this).error("Info objects (%1) loading for '%2' failed from '%3', '%4'") << info << CEntityFlags::flagToString(entities) << reader->getInfoObjectsUrl().toQString() << reader->getStatusMessage(); + reader->setMarkedAsFailed(true); return true; // carry on, regardless of situation } } @@ -1125,41 +1104,12 @@ namespace BlackCore else { // wait for 1st reply - this->m_dbInfoObjectTrials++; + trials++; this->readDeferredInBackground(entities, waitForInfoObjectsMs); return false; // wait } } - bool CWebDataServices::waitForSharedHeaders(CEntityFlags::Entity entities) - { - const int waitForHeadersMs = 1000; // ms - const int maxWaitCycles = 10; - - // try to read - if (m_sharedHeadersTrials > maxWaitCycles) - { - CLogMessage(this).warning("Cannot read headers for %1") << CEntityFlags::flagToString(entities); - // continue here and read data without info objects - return true; // no need wait any longer - } - else - { - const bool hasAllHeaders = this->areSharedHeadersLoaded(entities); - m_sharedHeadersTrials++; - if (hasAllHeaders) - { - CLogMessage(this).info("Headers loaded for %1, trail %2") << CEntityFlags::flagToString(entities) << m_sharedHeadersTrials; - } - else - { - CLogMessage(this).info("Waiting for headers of %1, trail %2") << CEntityFlags::flagToString(entities) << m_sharedHeadersTrials; - this->readDeferredInBackground(entities, waitForHeadersMs); - } - return hasAllHeaders; - } - } - bool CWebDataServices::writeDbDataToDisk(const QString &dir) const { if (dir.isEmpty()) { return false; } @@ -1224,25 +1174,4 @@ namespace BlackCore } return s; } - - bool CWebDataServices::triggerLoadingOfSharedFilesHeaders(CEntityFlags::Entity requestedEntities) - { - CEntityFlags::Entity triggeredEntities = CEntityFlags::NoEntity; - if (this->m_modelDataReader && this->m_modelDataReader->supportsAnyOfEntities(requestedEntities)) - { - triggeredEntities |= this->m_modelDataReader->maskBySupportedEntities(requestedEntities); - this->m_modelDataReader->requestHeadersOfSharedFiles(requestedEntities); - } - if (this->m_icaoDataReader && this->m_icaoDataReader->supportsAnyOfEntities(requestedEntities)) - { - triggeredEntities |= this->m_icaoDataReader->maskBySupportedEntities(requestedEntities); - this->m_icaoDataReader->requestHeadersOfSharedFiles(requestedEntities); - } - if (this->m_airportDataReader && this->m_airportDataReader->supportsAnyOfEntities(requestedEntities)) - { - triggeredEntities |= this->m_icaoDataReader->maskBySupportedEntities(requestedEntities); - this->m_airportDataReader->requestHeadersOfSharedFiles(requestedEntities); - } - return triggeredEntities != CEntityFlags::NoEntity; - } } // ns diff --git a/src/blackcore/webdataservices.h b/src/blackcore/webdataservices.h index e9ed44543..0d3ae5543 100644 --- a/src/blackcore/webdataservices.h +++ b/src/blackcore/webdataservices.h @@ -107,11 +107,11 @@ namespace BlackCore //! Metar reader Vatsim::CVatsimMetarReader *getMetarReader() const { return m_vatsimMetarReader; } - //! Info data reader + //! DB info data reader BlackCore::Db::CInfoDataReader *getDbInfoDataReader() const { return m_dbInfoDataReader; } - //! Currently used URL for shared DB data - BlackMisc::Network::CUrl getDbReaderCurrentSharedDbDataUrl() const; + //! Shared info data reader + BlackCore::Db::CInfoDataReader *getSharedInfoDataReader() const { return m_sharedInfoDataReader; } //! DB writer class Db::CDatabaseWriter *getDatabaseWriter() const { return m_databaseWriter; } @@ -119,6 +119,9 @@ namespace BlackCore //! Reader flags CWebReaderFlags::WebReader getReaderFlags() const { return m_readers; } + //! Currently used URL for shared DB data + BlackMisc::Network::CUrl getDbReaderCurrentSharedDbDataUrl() const; + //! All DB entities for those readers used and not ignored BlackMisc::Network::CEntityFlags::Entity allDbEntitiesForUsedReaders() const; @@ -321,8 +324,11 @@ namespace BlackCore //! Publish models to database BlackMisc::CStatusMessageList asyncPublishModels(const BlackMisc::Simulation::CAircraftModelList &models) const; - //! Trigger read of info objects - void triggerReadOfInfoObjects(); + //! Trigger read of DB info objects + void triggerReadOfDbInfoObjects(); + + //! Trigger read of shared info objects + void triggerReadOfSharedInfoObjects(); //! Trigger read of new data //! \note requires info objects loaded upfront and uses the full cache logic @@ -334,10 +340,6 @@ namespace BlackCore //! Trigger reload from shared files, only loads the data and bypasses caches BlackMisc::Network::CEntityFlags::Entity triggerLoadingDirectlyFromSharedFiles(BlackMisc::Network::CEntityFlags::Entity whatToRead, bool checkCacheTsUpfront); - //! Trigger loading of the HTTP headers for the shared files - //! \note allows to obtain the timestamps - bool triggerLoadingOfSharedFilesHeaders(BlackMisc::Network::CEntityFlags::Entity requestedEntities = BlackMisc::Network::CEntityFlags::AllDbEntities); - //! Corresponding cache timestamp if applicable //! \threadsafe QDateTime getCacheTimestamp(BlackMisc::Network::CEntityFlags::Entity entity) const; @@ -346,25 +348,25 @@ namespace BlackCore //! \threadsafe QDateTime getDbLatestEntityTimestamp(BlackMisc::Network::CEntityFlags::Entity entity) const; - //! Corresponding shared file timestamp - QDateTime getSharedFileTimestamp(BlackMisc::Network::CEntityFlags::Entity entity) const; + //! Shared info object timestamp + //! \threadsafe + QDateTime getSharedInfoObjectTimestamp(BlackMisc::Network::CEntityFlags::Entity entity) const; - //! Request (updated) HTTP header for shared file of entity - bool requestHeaderOfSharedFile(BlackMisc::Network::CEntityFlags::Entity entity); - - //! Are the shared headers for the given entities loaded - bool areSharedHeadersLoaded(BlackMisc::Network::CEntityFlags::Entity entities = BlackMisc::Network::CEntityFlags::AllDbEntities) const; - - //! Those entities where the timestamp of header is newer than the cache timestamp - BlackMisc::Network::CEntityFlags::Entity getEntitiesWithNewerHeaderTimestamp(BlackMisc::Network::CEntityFlags::Entity entities) const; + //! Entities with newer shared file (from DB Info object) + //! \threadsafe + BlackMisc::Network::CEntityFlags::Entity getEntitiesWithNewerSharedFile(BlackMisc::Network::CEntityFlags::Entity entities) const; //! Cache count for entity //! \threadsafe int getCacheCount(BlackMisc::Network::CEntityFlags::Entity entity) const; - //! Count for entity from entity objects + //! Count for entity from DB entity objects //! \threadsafe - int getDbInfoCount(BlackMisc::Network::CEntityFlags::Entity entity) const; + int getDbInfoObjectCount(BlackMisc::Network::CEntityFlags::Entity entity) const; + + //! Count for entity from shared entity objects + //! \threadsafe + int getSharedInfoObjectCount(BlackMisc::Network::CEntityFlags::Entity entity) const; //! Can connect to swift DB? bool canConnectSwiftDb() const; @@ -388,21 +390,17 @@ namespace BlackCore //! Combined read signal void dataRead(BlackMisc::Network::CEntityFlags::Entity entity, BlackMisc::Network::CEntityFlags::ReadState state, int number); - //! Header of shared file read - //! \deprecated shared file headers will be replaced by dbinfo.json - void sharedFileHeaderRead(BlackMisc::Network::CEntityFlags::Entity entity, const QString &fileName, bool success); - // simplified signals follow // 1) simple signature - // 2) fired direct after read, no need to wait for other entities + // 2) fired directly after read, no need to wait for other entities //! \name Simplified read signals //! @{ //! All swift DB data have been read void allSwiftDbDataRead(); - //! All headers received - void allSwiftSharedAllHeadersReceived(); + //! Shared info objects read + void sharedInfoObjectsRead(); //! All models read void swiftDbModelsRead(); @@ -442,10 +440,7 @@ namespace BlackCore void ps_vatsimDataFileRead(int lines); //! Read finished from reader - void ps_readFromSwiftDb(BlackMisc::Network::CEntityFlags::Entity entities, BlackMisc::Network::CEntityFlags::ReadState state, int number); - - //! A shared file header has been received - void ps_sharedFileHeaderReceived(BlackMisc::Network::CEntityFlags::Entity entity, const QString &fileName, bool success); + void ps_readFromSwiftReader(BlackMisc::Network::CEntityFlags::Entity entities, BlackMisc::Network::CEntityFlags::ReadState state, int number); private: //! Init the readers @@ -454,6 +449,9 @@ namespace BlackCore //! Init the info objects reader (DB web service) void initDbInfoObjectReaderAndTriggerRead(); + //! Init the info objects reader (shared dbinfo.json) + void initSharedInfoObjectReaderAndTriggerRead(); + //! DB reader for given entity Db::CDatabaseReader *getDbReader(BlackMisc::Network::CEntityFlags::Entity entity) const; @@ -461,23 +459,29 @@ namespace BlackCore void initWriters(); //! Remember this entity/those enties already have been signaled - bool signalEntitiesRead(BlackMisc::Network::CEntityFlags::Entity entities); + bool signalEntitiesAlreadyRead(BlackMisc::Network::CEntityFlags::Entity entities); - //! Wait for info objects to be read + //! Info object count from shared/DB info objects + int getInfoObjectCount(BlackMisc::Network::CEntityFlags::Entity entity, BlackCore::Db::CInfoDataReader *reader) const; + + //! Wait for DB info objects to be read bool waitForDbInfoObjects(BlackMisc::Network::CEntityFlags::Entity entities); - //! Wait for shared headers to be read - bool waitForSharedHeaders(BlackMisc::Network::CEntityFlags::Entity entities); + //! Wait for shared info objects to be read + bool waitForSharedInfoObjects(BlackMisc::Network::CEntityFlags::Entity entities); + + //! Wait for info objects to be read + bool waitForInfoObjects(BlackMisc::Network::CEntityFlags::Entity entities, const QString &info, BlackCore::Db::CInfoDataReader *reader, int &trials); CWebReaderFlags::WebReader m_readers = CWebReaderFlags::WebReaderFlag::None; //!< which readers are available BlackMisc::Network::CEntityFlags::Entity m_entitiesPeriodicallyRead = BlackMisc::Network::CEntityFlags::NoEntity; //!< entities permanently updated by timers BlackMisc::Network::CEntityFlags::Entity m_swiftDbEntitiesRead = BlackMisc::Network::CEntityFlags::NoEntity; //!< entities read - BlackCore::Db::CDatabaseReaderConfigList m_dbReaderConfig; //!< how to read DB data - bool m_initialRead = false; //!< Initial read started - bool m_signalledHeaders = false; //!< headers loading has been signalled - int m_dbInfoObjectTrials = 0; //!< Tried to read info objects - int m_sharedHeadersTrials = 0; //!< Tried to read shared file headers - QSet m_signalledEntities; //!< remember signalled entites + BlackCore::Db::CDatabaseReaderConfigList m_dbReaderConfig; //!< how to read DB data + bool m_initialRead = false; //!< Initial read started + bool m_signalledHeaders = false; //!< headers loading has been signalled + int m_dbInfoObjectTrials = 0; //!< Tried to read info objects + int m_sharedInfoObjectsTrials = 0; //!< Tried to read shared file headers + QSet m_signalledEntities; //!< remember signalled entites // for reading XML and VATSIM data files Vatsim::CVatsimStatusFileReader *m_vatsimStatusReader = nullptr; @@ -488,6 +492,7 @@ namespace BlackCore Db::CModelDataReader *m_modelDataReader = nullptr; Db::CAirportDataReader *m_airportDataReader = nullptr; Db::CInfoDataReader *m_dbInfoDataReader = nullptr; + Db::CInfoDataReader *m_sharedInfoDataReader = nullptr; // writing objects directly into DB Db::CDatabaseWriter *m_databaseWriter = nullptr;