refs #787, support for headers / shared files in web data services

* allow to load shared files
* utility functions for timestamps
* some functions renamed

src/blackcore/airspacemonitor.cpp

@@ -93,7 +93,7 @@ namespace BlackCore
         // AutoConnection: this should also avoid race conditions by updating the bookings
         Q_ASSERT_X(sApp->getWebDataServices(), Q_FUNC_INFO, "Missing data reader");
         this->connect(sApp->getWebDataServices()->getBookingReader(), &CVatsimBookingReader::atcBookingsRead, this, &CAirspaceMonitor::ps_receivedBookings);
-        this->connect(sApp->getWebDataServices()->getDataFileReader(), &CVatsimDataFileReader::dataFileRead, this, &CAirspaceMonitor::ps_receivedDataFile);
+        this->connect(sApp->getWebDataServices()->getVatsimDataFileReader(), &CVatsimDataFileReader::dataFileRead, this, &CAirspaceMonitor::ps_receivedDataFile);
 
         // Force snapshot in the main event loop
         this->connect(this->m_analyzer, &CAirspaceAnalyzer::airspaceAircraftSnapshot, this, &CAirspaceMonitor::airspaceAircraftSnapshot, Qt::QueuedConnection);

src/blackcore/webdataservices.cpp

@@ -54,12 +54,13 @@ namespace BlackCore
     {
         if (!sApp) { return; } // shutting down
 
-        Q_ASSERT_X(QSslSocket::supportsSsl(), Q_FUNC_INFO, "missing SSL support");
+        Q_ASSERT_X(QSslSocket::supportsSsl(), Q_FUNC_INFO, "Missing SSL support");
         Q_ASSERT_X(sApp->isSetupAvailable(), Q_FUNC_INFO, "Setup not synchronized");
         this->setObjectName("CWebDataReader");
 
         // check if I need info objects
-        const bool readFromSwiftDb = dbReaderConfig.possiblyReadsFromSwiftDb(); // only cached?
+        const bool readFromSwiftDb = dbReaderConfig.possiblyReadsFromSwiftDb(); // DB read access
+        const bool writeToSwiftDb = dbReaderConfig.possiblyWritesToSwiftDb(); // DB write access
         if (!readFromSwiftDb && readers.testFlag(CWebReaderFlags::InfoDataReader))
         {
             // will remove info reader because not needed
@@ -76,11 +77,13 @@ namespace BlackCore
             CLogMessage(this).info("Using info objects for swift DB entities");
         }
 
-        this->initReaders(readers); // reads info object if required
+        this->initReaders(readers, entities); // reads info objects if required
+        if (writeToSwiftDb)
+        {
             this->initWriters();
+        }
 
         // make sure this is called in event queue, so pending tasks cam be performed
-        // important so info objects can be read
         entities &= ~CEntityFlags::InfoObjectEntity;   // triggered in init readers
         entities &= ~CEntityFlags::VatsimStatusFile;   // triggered in init readers
         entities &= ~this->m_entitiesPeriodicallyRead; // will be triggered by timers
@@ -89,12 +92,11 @@ namespace BlackCore
         // but do not start all at the same time
         const CEntityFlags::Entity icaoPart = entities & CEntityFlags::AllIcaoAndCountries;
         const CEntityFlags::Entity modelPart = entities & CEntityFlags::DistributorLiveryModel;
-        this->readDeferredInBackground(icaoPart, 1000);
+        CEntityFlags::Entity remainingEntities = entities & ~icaoPart;
-        this->readDeferredInBackground(modelPart, 2000);
+        remainingEntities &= ~modelPart;
-
+        this->readDeferredInBackground(icaoPart, 500);
-        CEntityFlags::Entity restEntities = entities & ~icaoPart;
+        this->readDeferredInBackground(modelPart, 1000);
-        restEntities &= ~modelPart;
+        this->readDeferredInBackground(remainingEntities, 1500);
-        this->readDeferredInBackground(restEntities, 3000);
     }
 
     CServerList CWebDataServices::getVatsimFsdServers() const
@@ -150,9 +152,14 @@ namespace BlackCore
         return CStatusMessageList();
     }
 
+    void CWebDataServices::triggerReadOfInfoObjects()
+    {
+        initInfoObjectReaderAndTriggerRead();
+    }
+
     bool CWebDataServices::canConnectSwiftDb() const
     {
-        if (!m_icaoDataReader && !m_modelDataReader && !m_infoDataReader) { return false; }
+        if (!m_icaoDataReader && !m_modelDataReader && !m_airportDataReader && !m_infoDataReader) { return false; }
 
         // use the first one to test
         if (m_infoDataReader)
@@ -167,12 +174,17 @@ namespace BlackCore
         {
             return m_modelDataReader->hasReceivedOkReply();
         }
+        else if (m_airportDataReader)
+        {
+            return m_airportDataReader->hasReceivedOkReply();
+        }
         return false;
     }
 
     void CWebDataServices::resetSignalFlags()
     {
-        m_signaledEntities.clear();
+        m_signalledEntities.clear();
+        m_signalledHeaders = false;
     }
 
     bool CWebDataServices::hasDbAircraftData() const
@@ -254,45 +266,77 @@ namespace BlackCore
         return triggeredRead;
     }
 
-    CEntityFlags::Entity CWebDataServices::triggerReloadFromDb(CEntityFlags::Entity whatToRead, const QDateTime &newerThan)
+    CEntityFlags::Entity CWebDataServices::triggerLoadingDirectlyFromDb(CEntityFlags::Entity whatToRead, const QDateTime &newerThan)
     {
         CEntityFlags::Entity triggeredRead = CEntityFlags::NoEntity;
-
         if (m_infoDataReader)
         {
-            // when possible update info objects
+            this->triggerReadOfInfoObjects();
-            CEntityFlags::Entity infoObjectEntity = CEntityFlags::InfoObjectEntity;
+            triggeredRead |= CEntityFlags::InfoObjectEntity;
-            m_infoDataReader->startReadFromDbInBackgroundThread(infoObjectEntity, newerThan);
-            triggeredRead |= infoObjectEntity;
         }
 
         if (m_icaoDataReader)
         {
-            if (whatToRead.testFlag(CEntityFlags::AircraftIcaoEntity) || whatToRead.testFlag(CEntityFlags::AirlineIcaoEntity) || whatToRead.testFlag(CEntityFlags::CountryEntity))
+            if (m_icaoDataReader->supportsAnyOfEntities(whatToRead))
             {
-                CEntityFlags::Entity icaoEntities = whatToRead & CEntityFlags::AllIcaoAndCountries;
+                CEntityFlags::Entity icaoEntities = m_icaoDataReader->maskBySupportedEntities(whatToRead);
-                m_icaoDataReader->startReadFromDbInBackgroundThread(icaoEntities, newerThan);
+                m_icaoDataReader->triggerLoadingDirectlyFromDb(icaoEntities, newerThan);
                 triggeredRead |= icaoEntities;
             }
         }
 
         if (m_modelDataReader)
         {
-            if (whatToRead.testFlag(CEntityFlags::LiveryEntity) || whatToRead.testFlag(CEntityFlags::DistributorEntity) || whatToRead.testFlag(CEntityFlags::ModelEntity))
+            if (m_modelDataReader->supportsAnyOfEntities(whatToRead))
             {
-                CEntityFlags::Entity modelEntities = whatToRead & CEntityFlags::DistributorLiveryModel;
+                CEntityFlags::Entity modelEntities = m_modelDataReader->maskBySupportedEntities(whatToRead);
-                m_modelDataReader->startReadFromDbInBackgroundThread(modelEntities, newerThan);
+                m_modelDataReader->triggerLoadingDirectlyFromDb(modelEntities, newerThan);
                 triggeredRead |= modelEntities;
             }
         }
 
         if (m_airportDataReader)
         {
-            if (whatToRead.testFlag(CEntityFlags::AirportEntity))
+            if (m_airportDataReader->supportsAnyOfEntities(whatToRead))
             {
-                CEntityFlags::Entity airportEntity = whatToRead & CEntityFlags::AirportEntity;
+                CEntityFlags::Entity airportEntities = m_airportDataReader->maskBySupportedEntities(whatToRead);
-                m_airportDataReader->startReadFromDbInBackgroundThread(airportEntity, newerThan);
+                m_airportDataReader->triggerLoadingDirectlyFromDb(airportEntities, newerThan);
-                triggeredRead |= airportEntity;
+                triggeredRead |= airportEntities;
+            }
+        }
+
+        return triggeredRead;
+    }
+
+    CEntityFlags::Entity CWebDataServices::triggerLoadingDirectlyFromSharedFiles(CEntityFlags::Entity whatToRead, bool checkCacheTsUpfront)
+    {
+        CEntityFlags::Entity triggeredRead = CEntityFlags::NoEntity;
+        this->triggerLoadingOfSharedFilesHeaders(whatToRead); // trigger reload of headers
+
+        if (m_icaoDataReader)
+        {
+            if (m_icaoDataReader->supportsAnyOfEntities(whatToRead))
+            {
+                CEntityFlags::Entity icaoEntities = m_icaoDataReader->maskBySupportedEntities(whatToRead);
+                triggeredRead |= m_icaoDataReader->triggerLoadingDirectlyFromSharedFiles(icaoEntities, checkCacheTsUpfront);
+            }
+        }
+
+        if (m_modelDataReader)
+        {
+            if (m_modelDataReader->supportsAnyOfEntities(whatToRead))
+            {
+                CEntityFlags::Entity modelEntities = m_modelDataReader->maskBySupportedEntities(whatToRead);
+                triggeredRead |= m_modelDataReader->triggerLoadingDirectlyFromSharedFiles(modelEntities, checkCacheTsUpfront);
+            }
+        }
+
+        if (m_airportDataReader)
+        {
+            if (m_airportDataReader->supportsAnyOfEntities(whatToRead))
+            {
+                CEntityFlags::Entity airportEntities = m_airportDataReader->maskBySupportedEntities(whatToRead);
+                triggeredRead |= m_airportDataReader->triggerLoadingDirectlyFromSharedFiles(airportEntities, checkCacheTsUpfront);
             }
         }
 
@@ -304,7 +348,7 @@ namespace BlackCore
         Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "Need single entity");
         if (CEntityFlags::anySwiftDbEntity(entity))
         {
-            CDatabaseReader *dr = this->getDbReader(entity);
+            const CDatabaseReader *dr = this->getDbReader(entity);
             if (!dr) { return QDateTime(); }
             return dr->getCacheTimestamp(entity);
         }
@@ -335,7 +379,7 @@ namespace BlackCore
         const CDatabaseReader *reader = this->getDbReader(entity);
         if (reader)
         {
-            return reader->getSharedFileTimestamp(entity);
+            return reader->getLatestSharedFileHeaderTimestamp(entity);
         }
         else
         {
@@ -351,6 +395,47 @@ namespace BlackCore
         return reader->requestHeadersOfSharedFiles(entity);
     }
 
+    bool CWebDataServices::areSharedHeadersLoaded(CEntityFlags::Entity entities) const
+    {
+        bool hasAllHeaders = false;
+        do
+        {
+            if (this->m_modelDataReader && this->m_modelDataReader->supportsAnyOfEntities(entities))
+            {
+                if (!this->m_modelDataReader->hasSharedFileHeaders(entities)) { break; }
+            }
+            if (this->m_icaoDataReader && this->m_icaoDataReader->supportsAnyOfEntities(entities))
+            {
+                if (!this->m_icaoDataReader->hasSharedFileHeaders(entities)) { break; }
+            }
+            if (this->m_airportDataReader && this->m_airportDataReader->supportsAnyOfEntities(entities))
+            {
+                if (!this->m_airportDataReader->hasSharedFileHeaders(entities)) { break; }
+            }
+            hasAllHeaders = true;
+        }
+        while (false);
+        return hasAllHeaders;
+    }
+
+    BlackMisc::Network::CEntityFlags::Entity CWebDataServices::getEntitiesWithNewerHeaderTimestamp(BlackMisc::Network::CEntityFlags::Entity entities) const
+    {
+        CEntityFlags::Entity newerEntities = CEntityFlags::NoEntity;
+        if (this->m_airportDataReader)
+        {
+            newerEntities |= this->m_airportDataReader->getEntitesWithNewerHeaderTimestamp(entities);
+        }
+        if (this->m_icaoDataReader)
+        {
+            newerEntities |= this->m_icaoDataReader->getEntitesWithNewerHeaderTimestamp(entities);
+        }
+        if (this->m_modelDataReader)
+        {
+            newerEntities |= this->m_modelDataReader->getEntitesWithNewerHeaderTimestamp(entities);
+        }
+        return newerEntities;
+    }
+
     int CWebDataServices::getCacheCount(CEntityFlags::Entity entity) const
     {
         Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "Need single entity");
@@ -362,7 +447,7 @@ namespace BlackCore
         }
         else
         {
-            // non DB entities would go here
+            // non DB/shared entities would go here
             return -1;
         }
     }
@@ -643,7 +728,12 @@ namespace BlackCore
         if (this->m_databaseWriter)       { this->m_databaseWriter->gracefulShutdown(); }
     }
 
-    CEntityFlags::Entity CWebDataServices::allDbEntiiesForUsedReaders() const
+    CUrl CWebDataServices::getDbReaderCurrentSharedDbDataUrl() const
+    {
+        return CDatabaseReader::getCurrentSharedDbDataUrl();
+    }
+
+    CEntityFlags::Entity CWebDataServices::allDbEntitiesForUsedReaders() const
     {
         // obtain entities from real readers (means when reader is really used)
         CEntityFlags::Entity entities = CEntityFlags::NoEntity;
@@ -677,40 +767,40 @@ namespace BlackCore
         return cats;
     }
 
-    void CWebDataServices::initReaders(CWebReaderFlags::WebReader flags)
+    void CWebDataServices::initReaders(CWebReaderFlags::WebReader flags, CEntityFlags::Entity entities)
     {
+        //
         // ---- "metadata" reader, 1/2 will trigger read directly during init
+        //
 
-        // 1. If any DB data, read the info upfront
-        const bool anyDbData = flags.testFlag(CWebReaderFlags::WebReaderFlag::IcaoDataReader) || flags.testFlag(CWebReaderFlags::WebReaderFlag::ModelReader);
-        const bool databaseUp = CInfoDataReader::canPingSwiftServer();
         CDatabaseReaderConfigList dbReaderConfig(this->m_dbReaderConfig);
+        const bool anyDbEntities = CEntityFlags::anySwiftDbEntity(entities);
+        const bool needsSharedHeaders = dbReaderConfig.needsSharedHeaders(entities);
+        const bool needsInfoObjects = dbReaderConfig.possiblyReadsFromSwiftDb();
+        bool c = false; // for signal connect
+
+        // 1a. If any DB data, read the info objects upfront
+        if (needsInfoObjects)
+        {
+            const bool databaseUp = CInfoDataReader::canPingSwiftServer();
             if (!databaseUp) { dbReaderConfig.markAsDbDown(); }
 
-        bool c = false; // for signal connect
+            if (anyDbEntities && flags.testFlag(CWebReaderFlags::WebReaderFlag::InfoDataReader))
-        Q_UNUSED(c);
-        if (anyDbData && flags.testFlag(CWebReaderFlags::WebReaderFlag::InfoDataReader))
             {
                 // info data reader has a special role, it will not be triggered in triggerRead()
                 if (databaseUp)
                 {
-                this->m_infoDataReader = new CInfoDataReader(this, dbReaderConfig);
+                    this->initInfoObjectReaderAndTriggerRead();
-                c = connect(this->m_infoDataReader, &CInfoDataReader::dataRead, this, &CWebDataServices::ps_readFromSwiftDb);
-                Q_ASSERT_X(c, Q_FUNC_INFO, "Info object connect failed");
-
-                // relay signal
-                c = connect(this->m_infoDataReader, &CInfoDataReader::dataRead, this, &CWebDataServices::dataRead);
-                Q_ASSERT_X(c, Q_FUNC_INFO, "Info object connect failed");
-
-                // start reading
-                this->m_infoDataReader->start(QThread::LowPriority);
-                QTimer::singleShot(0, [this]() { this->m_infoDataReader->read(CEntityFlags::InfoObjectEntity, QDateTime()); });
                 }
                 else
                 {
-                CLogMessage(this).warning("DB unrechable, skipping read from info data reader");
+                    CLogMessage(this).warning("DB unreachable, skipping read from info data reader");
                 }
             }
+        }
+
+        // 1b. Read shared headers if needed
+        // --> See below after readers have been connected
 
         // 2. Status file, updating the VATSIM related caches
         if (flags.testFlag(CWebReaderFlags::VatsimStatusReader) || flags.testFlag(CWebReaderFlags::VatsimDataReader) || flags.testFlag(CWebReaderFlags::VatsimMetarReader))
@@ -749,7 +839,7 @@ namespace BlackCore
             this->m_vatsimDataFileReader->startReader();
         }
 
-        // 5. VATSIM metar data
+        // 5. VATSIM METAR data
         if (flags.testFlag(CWebReaderFlags::WebReaderFlag::VatsimMetarReader))
         {
             this->m_vatsimMetarReader = new CVatsimMetarReader(this);
@@ -788,7 +878,7 @@ namespace BlackCore
             this->m_modelDataReader->start(QThread::LowPriority);
         }
 
-        // 6. Airport list reader
+        // 8. Airport list reader
         if (flags.testFlag(CWebReaderFlags::WebReaderFlag::AirportReader))
         {
             this->m_airportDataReader = new CAirportDataReader(this, dbReaderConfig);
@@ -801,9 +891,35 @@ namespace BlackCore
             this->m_airportDataReader->start(QThread::LowPriority);
         }
 
-        // Trigger Shared file headers loading
+        // 1b. Read the headers if needed
-        //! \todo refine, check if really needed to load the headers here
+        c = connect(this, &CWebDataServices::sharedFileHeaderRead, this, &CWebDataServices::ps_sharedFileHeaderReceived);
-        QTimer::singleShot(0, [this]() { this->triggerLoadingOfSharedFilesHeaders(); });
+        Q_ASSERT_X(c, Q_FUNC_INFO, "Cannot connect header signal");
+        if (needsSharedHeaders)
+        {
+            QTimer::singleShot(0, [this, entities]() { this->triggerLoadingOfSharedFilesHeaders(entities); });
+        }
+
+        Q_UNUSED(c); // signal connect flag
+    }
+
+    void CWebDataServices::initInfoObjectReaderAndTriggerRead()
+    {
+        if (!this->m_infoDataReader)
+        {
+            this->m_infoDataReader = new CInfoDataReader(this, m_dbReaderConfig);
+            bool c = connect(this->m_infoDataReader, &CInfoDataReader::dataRead, this, &CWebDataServices::ps_readFromSwiftDb);
+            Q_ASSERT_X(c, Q_FUNC_INFO, "Info reader connect failed");
+
+            // relay signal
+            c = connect(this->m_infoDataReader, &CInfoDataReader::dataRead, this, &CWebDataServices::dataRead);
+            Q_ASSERT_X(c, Q_FUNC_INFO, "Info reader connect failed");
+
+            // start in own thread
+            this->m_infoDataReader->start(QThread::LowPriority);
+        }
+
+        // and trigger read
+        QTimer::singleShot(0, [this]() { this->m_infoDataReader->read(CEntityFlags::InfoObjectEntity, QDateTime()); });
     }
 
     CDatabaseReader *CWebDataServices::getDbReader(CEntityFlags::Entity entity) const
@@ -832,8 +948,8 @@ namespace BlackCore
 
     bool CWebDataServices::signalEntitiesRead(CEntityFlags::Entity entities)
     {
-        if (m_signaledEntities.contains(entities)) { return false; }
+        if (m_signalledEntities.contains(entities)) { return false; }
-        m_signaledEntities.insert(entities);
+        m_signalledEntities.insert(entities);
         return true;
     }
 
@@ -875,7 +991,7 @@ namespace BlackCore
         }
 
         this->m_swiftDbEntitiesRead |= entities;
-        const int allUsedEntities = static_cast<int>(this->allDbEntiiesForUsedReaders());
+        const int allUsedEntities = static_cast<int>(this->allDbEntitiesForUsedReaders());
         if (((static_cast<int>(this->m_swiftDbEntitiesRead)) & allUsedEntities) == allUsedEntities)
         {
             emit allSwiftDbDataRead();
@@ -899,6 +1015,18 @@ namespace BlackCore
         }
     }
 
+    void CWebDataServices::ps_sharedFileHeaderReceived(CEntityFlags::Entity entity, const QString &fileName, bool success)
+    {
+        Q_UNUSED(entity);
+        Q_UNUSED(fileName);
+        Q_UNUSED(success);
+        if (m_signalledHeaders) { return; }
+
+        if (!this->areSharedHeadersLoaded()) { return; }
+        m_signalledHeaders = true;
+        emit this->allSwiftSharedAllHeadersReceived();
+    }
+
     void CWebDataServices::readDeferredInBackground(CEntityFlags::Entity entities, int delayMs)
     {
         if (entities == CEntityFlags::NoEntity) { return; }
@@ -912,14 +1040,32 @@ namespace BlackCore
     {
         this->m_initialRead = true; // read started
 
-        const int waitForInfoObjects = 1000; // ms
+        if (CEntityFlags::anySwiftDbEntity(entities))
+        {
+            // with info objects wait until info objects are loaded
+            Q_ASSERT_X(!entities.testFlag(CEntityFlags::InfoObjectEntity), Q_FUNC_INFO, "Info object must be read upfront, do not pass as entity here");
+            const bool waitForInfoReader = this->m_infoDataReader && !this->m_infoDataReader->areAllDataRead() && !this->m_infoDataReader->isMarkedAsFailed();
+            if (waitForInfoReader)
+            {
+                if (!this->waitForInfoObjects(entities)) { return; } // will call this function again after some time
+            }
+
+            const bool waitForSharedHeaders = m_dbReaderConfig.needsSharedHeadersLoaded(entities);
+            if (waitForSharedHeaders)
+            {
+                if (!this->waitForSharedHeaders(entities)) { return; } // will call this function again after some time
+            }
+        }
+
+        // read entities
+        this->triggerRead(entities);
+    }
+
+    bool CWebDataServices::waitForInfoObjects(CEntityFlags::Entity entities)
+    {
+        const int waitForInfoObjectsMs = 1000; // ms
         const int maxWaitCycles = 10;
 
-        // with info objects wait until info objects are loaded
-        Q_ASSERT_X(!entities.testFlag(CEntityFlags::InfoObjectEntity), Q_FUNC_INFO, "Info object must be read upfront");
-        const bool readFromInfoReader = this->m_infoDataReader && !this->m_infoDataReader->areAllDataRead() && !this->m_infoDataReader->isMarkedAsFailed();
-        if (readFromInfoReader && CEntityFlags::anySwiftDbEntity(entities))
-        {
         // try to read
         if (this->m_infoObjectTrials > maxWaitCycles)
         {
@@ -927,6 +1073,7 @@ namespace BlackCore
                     << CEntityFlags::flagToString(entities)
                     << this->m_infoDataReader->getInfoObjectsUrl().toQString();
             // continue here and read data without info objects
+            return true; // no need wait any longer
         }
         else if (this->m_infoDataReader->hasReceivedFirstReply())
         {
@@ -937,7 +1084,7 @@ namespace BlackCore
                         << CEntityFlags::flagToString(entities)
                         << this->m_infoObjectTrials
                         << this->m_infoDataReader->getInfoObjectsUrl().toQString();
-                    // continue here and read data
+                return true; // no need wait any longer
             }
             else
             {
@@ -946,8 +1093,8 @@ namespace BlackCore
                 {
                     // ok, this means we are parsing
                     this->m_infoObjectTrials++;
-                        this->readDeferredInBackground(entities, waitForInfoObjects);
+                    this->readDeferredInBackground(entities, waitForInfoObjectsMs);
-                        return;
+                    return false; // wait
                 }
                 else
                 {
@@ -959,7 +1106,7 @@ namespace BlackCore
                             << this->m_infoDataReader->getInfoObjectsUrl().toQString()
                             << this->m_infoDataReader->getStatusMessage();
                     this->m_infoDataReader->setMarkedAsFailed(true);
-                        // continue here and read data
+                    return true; // no need wait any longer
                 }
             }
         }
@@ -967,13 +1114,38 @@ namespace BlackCore
         {
             // wait for 1st reply
             this->m_infoObjectTrials++;
-                this->readDeferredInBackground(entities, waitForInfoObjects);
+            this->readDeferredInBackground(entities, waitForInfoObjectsMs);
-                return;
+            return false; // wait
         }
     }
 
-        // read entities
+    bool CWebDataServices::waitForSharedHeaders(CEntityFlags::Entity entities)
-        this->triggerRead(entities);
+    {
+        const int waitForHeadersMs = 1000; // ms
+        const int maxWaitCycles = 10;
+
+        // try to read
+        if (m_sharedHeadersTrials > maxWaitCycles)
+        {
+            CLogMessage(this).warning("Cannot read headers for %1") << CEntityFlags::flagToString(entities);
+            // continue here and read data without info objects
+            return true; // no need wait any longer
+        }
+        else
+        {
+            const bool hasAllHeaders = this->areSharedHeadersLoaded(entities);
+            m_sharedHeadersTrials++;
+            if (hasAllHeaders)
+            {
+                CLogMessage(this).info("Headers loaded for %1, trail %2") << CEntityFlags::flagToString(entities) << m_sharedHeadersTrials;
+            }
+            else
+            {
+                CLogMessage(this).info("Waiting for headers of %1, trail %2") << CEntityFlags::flagToString(entities) << m_sharedHeadersTrials;
+                this->readDeferredInBackground(entities, waitForHeadersMs);
+            }
+            return hasAllHeaders;
+        }
     }
 
     bool CWebDataServices::writeDbDataToDisk(const QString &dir) const
@@ -1000,14 +1172,10 @@ namespace BlackCore
             if (!s) { return false; }
         }
 
-        if (m_icaoDataReader)
+        if (this->getAirportsCount() > 0)
         {
-            bool s = m_icaoDataReader->writeToJsonFiles(directory.absolutePath());
+            QString json(QJsonDocument(this->getAirports().toJson()).toJson());
-            if (!s) { return false; }
+            bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), "airports.json"));
-        }
-        if (m_modelDataReader)
-        {
-            bool s = m_modelDataReader->writeToJsonFiles(directory.absolutePath());
             if (!s) { return false; }
         }
 
@@ -1033,38 +1201,32 @@ namespace BlackCore
                 this->m_modelDataReader->readFromJsonFilesInBackground(dir) :
                 this->m_modelDataReader->readFromJsonFiles(dir);
         }
+        if (s && this->m_airportDataReader)
+        {
+            s = inBackground ?
+                this->m_airportDataReader->readFromJsonFilesInBackground(dir) :
+                this->m_airportDataReader->readFromJsonFiles(dir);
+        }
         return s;
     }
 
     bool CWebDataServices::triggerLoadingOfSharedFilesHeaders(CEntityFlags::Entity requestedEntities)
     {
         CEntityFlags::Entity triggeredEntities = CEntityFlags::NoEntity;
-        if (this->m_modelDataReader)
+        if (this->m_modelDataReader && this->m_modelDataReader->supportsAnyOfEntities(requestedEntities))
         {
-            const CEntityFlags::Entity entities = requestedEntities & CWebReaderFlags::allEntitiesForReaders(CWebReaderFlags::ModelReader);
+            triggeredEntities |= this->m_modelDataReader->maskBySupportedEntities(requestedEntities);
-            if (entities != CEntityFlags::NoEntity)
+            this->m_modelDataReader->requestHeadersOfSharedFiles(requestedEntities);
-            {
-                triggeredEntities |= entities;
-                this->m_modelDataReader->requestHeadersOfSharedFiles(entities);
         }
-        }
+        if (this->m_icaoDataReader && this->m_icaoDataReader->supportsAnyOfEntities(requestedEntities))
-        if (this->m_icaoDataReader)
         {
-            const CEntityFlags::Entity entities = requestedEntities & CWebReaderFlags::allEntitiesForReaders(CWebReaderFlags::IcaoDataReader);
+            triggeredEntities |= this->m_icaoDataReader->maskBySupportedEntities(requestedEntities);
-            if (entities != CEntityFlags::NoEntity)
+            this->m_icaoDataReader->requestHeadersOfSharedFiles(requestedEntities);
-            {
-                triggeredEntities |= entities;
-                this->m_icaoDataReader->requestHeadersOfSharedFiles(entities);
         }
-        }
+        if (this->m_airportDataReader && this->m_airportDataReader->supportsAnyOfEntities(requestedEntities))
-        if (this->m_airportDataReader)
         {
-            const CEntityFlags::Entity entities = requestedEntities & CWebReaderFlags::allEntitiesForReaders(CWebReaderFlags::AirportReader);
+            triggeredEntities |= this->m_icaoDataReader->maskBySupportedEntities(requestedEntities);
-            if (entities != CEntityFlags::NoEntity)
+            this->m_airportDataReader->requestHeadersOfSharedFiles(requestedEntities);
-            {
-                triggeredEntities |= entities;
-                this->m_airportDataReader->requestHeadersOfSharedFiles(entities);
-            }
         }
         return triggeredEntities != CEntityFlags::NoEntity;
     }

src/blackcore/webdataservices.h

@@ -98,18 +98,21 @@ namespace BlackCore
         //! Read ATC bookings (used to re-read)
         void readAtcBookingsInBackground() const;
 
+        //! Data file reader
+        Vatsim::CVatsimDataFileReader *getVatsimDataFileReader() const { return m_vatsimDataFileReader; }
+
         //! Booking reader
         Vatsim::CVatsimBookingReader *getBookingReader() const { return m_vatsimBookingReader; }
 
-        //! Data file reader
-        Vatsim::CVatsimDataFileReader *getDataFileReader() const { return m_vatsimDataFileReader; }
-
         //! Metar reader
         Vatsim::CVatsimMetarReader *getMetarReader() const { return m_vatsimMetarReader; }
 
         //! Info data reader
         Db::CInfoDataReader *getInfoDataReader() const { return m_infoDataReader; }
 
+        //! Currently used URL for shared DB data
+        BlackMisc::Network::CUrl getDbReaderCurrentSharedDbDataUrl() const;
+
         //! DB writer class
         Db::CDatabaseWriter *getDatabaseWriter() const { return m_databaseWriter; }
 
@@ -117,7 +120,7 @@ namespace BlackCore
         CWebReaderFlags::WebReader getReaderFlags() const { return m_readers; }
 
         //! All DB entities for those readers used and not ignored
-        BlackMisc::Network::CEntityFlags::Entity allDbEntiiesForUsedReaders() const;
+        BlackMisc::Network::CEntityFlags::Entity allDbEntitiesForUsedReaders() const;
 
         //! FSD servers
         //! \threadsafe
@@ -314,12 +317,18 @@ namespace BlackCore
         //! Publish models to database
         BlackMisc::CStatusMessageList asyncPublishModels(const BlackMisc::Simulation::CAircraftModelList &models) const;
 
+        //! Trigger read of info objects
+        void triggerReadOfInfoObjects();
+
         //! Trigger read of new data
         //! \note requires info objects loaded upfront and uses the full cache logic
         BlackMisc::Network::CEntityFlags::Entity triggerRead(BlackMisc::Network::CEntityFlags::Entity whatToRead, const QDateTime &newerThan = QDateTime());
 
         //! Trigger reload from DB, only loads the DB data and bypasses the caches checks and info objects
-        BlackMisc::Network::CEntityFlags::Entity triggerReloadFromDb(BlackMisc::Network::CEntityFlags::Entity whatToRead, const QDateTime &newerThan = QDateTime());
+        BlackMisc::Network::CEntityFlags::Entity triggerLoadingDirectlyFromDb(BlackMisc::Network::CEntityFlags::Entity whatToRead, const QDateTime &newerThan = QDateTime());
+
+        //! Trigger reload from shared files, only loads the data and bypasses caches
+        BlackMisc::Network::CEntityFlags::Entity triggerLoadingDirectlyFromSharedFiles(BlackMisc::Network::CEntityFlags::Entity whatToRead, bool checkCacheTsUpfront);
 
         //! Trigger loading of the HTTP headers for the shared files
         //! \note allows to obtain the timestamps
@@ -339,6 +348,12 @@ namespace BlackCore
         //! Request (updated) HTTP header for shared file of entity
         bool requestHeaderOfSharedFile(BlackMisc::Network::CEntityFlags::Entity entity);
 
+        //! Are the shared headers for the given entities loaded
+        bool areSharedHeadersLoaded(BlackMisc::Network::CEntityFlags::Entity entities = BlackMisc::Network::CEntityFlags::AllDbEntities) const;
+
+        //! Those entities where the timestamp of header is newer than the cache timestamp
+        BlackMisc::Network::CEntityFlags::Entity getEntitiesWithNewerHeaderTimestamp(BlackMisc::Network::CEntityFlags::Entity entities) const;
+
         //! Cache count for entity
         //! \threadsafe
         int getCacheCount(BlackMisc::Network::CEntityFlags::Entity entity) const;
@@ -369,8 +384,8 @@ namespace BlackCore
         //! Combined read signal
         void dataRead(BlackMisc::Network::CEntityFlags::Entity entity, BlackMisc::Network::CEntityFlags::ReadState state, int number);
 
-        //! All swift DB data have been read
+        //! Header of shared file read
-        void allSwiftDbDataRead();
+        void sharedFileHeaderRead(BlackMisc::Network::CEntityFlags::Entity entity, const QString &fileName, bool success);
 
         // simplified signals
         // 1) simple signature
@@ -378,6 +393,12 @@ namespace BlackCore
 
         //! \name Simplified read signals
         //! @{
+        //! All swift DB data have been read
+        void allSwiftDbDataRead();
+
+        //! All headers received
+        void allSwiftSharedAllHeadersReceived();
+
         //! All models read
         void swiftDbModelsRead();
 
@@ -397,14 +418,12 @@ namespace BlackCore
         void swiftDbModelMatchingEntities();
         //! @}
 
-        //! //! Header of shared file read
-        void sharedFileHeaderRead(BlackMisc::Network::CEntityFlags::Entity entity, const QString &fileName, bool success);
-
     public slots:
         //! Call CWebDataServices::readInBackground by single shot
         void readDeferredInBackground(BlackMisc::Network::CEntityFlags::Entity entities, int delayMs);
 
         //! First read (allows to immediately read in background)
+        //! \remark ensures info objects (if and only if needed) are read upfront
         void readInBackground(BlackMisc::Network::CEntityFlags::Entity entities = BlackMisc::Network::CEntityFlags::AllEntities);
 
     private slots:
@@ -420,9 +439,15 @@ namespace BlackCore
         //! Read finished from reader
         void ps_readFromSwiftDb(BlackMisc::Network::CEntityFlags::Entity entities, BlackMisc::Network::CEntityFlags::ReadState state, int number);
 
+        //! A shared file header has been received
+        void ps_sharedFileHeaderReceived(BlackMisc::Network::CEntityFlags::Entity entity, const QString &fileName, bool success);
+
     private:
         //! Init the readers
-        void initReaders(CWebReaderFlags::WebReader flags);
+        void initReaders(CWebReaderFlags::WebReader flags, BlackMisc::Network::CEntityFlags::Entity entities);
+
+        //! Init the info objects readers
+        void initInfoObjectReaderAndTriggerRead();
 
         //! DB reader for given entity
         Db::CDatabaseReader *getDbReader(BlackMisc::Network::CEntityFlags::Entity entity) const;
@@ -433,13 +458,21 @@ namespace BlackCore
         //! Remember this entity/those enties already have been signaled
         bool signalEntitiesRead(BlackMisc::Network::CEntityFlags::Entity entities);
 
+        //! Wait for info objects to be read
+        bool waitForInfoObjects(BlackMisc::Network::CEntityFlags::Entity entities);
+
+        //! Wait for shared headers to be read
+        bool waitForSharedHeaders(BlackMisc::Network::CEntityFlags::Entity entities);
+
         CWebReaderFlags::WebReader               m_readers = CWebReaderFlags::WebReaderFlag::None; //!< which readers are available
         BlackCore::Db::CDatabaseReaderConfigList m_dbReaderConfig;                                 //!< how to read DB data
         BlackMisc::Network::CEntityFlags::Entity m_entitiesPeriodicallyRead = BlackMisc::Network::CEntityFlags::NoEntity; //!< those entities which are permanently updated by timers
         BlackMisc::Network::CEntityFlags::Entity m_swiftDbEntitiesRead      = BlackMisc::Network::CEntityFlags::NoEntity; //!< entities read
         bool                                     m_initialRead = false;      //!< Initial read started
+        bool                                     m_signalledHeaders = false; //!< headers loading has been signalled
         int                                      m_infoObjectTrials = 0;     //!< Tried to read info objects
-        QSet<BlackMisc::Network::CEntityFlags::Entity> m_signaledEntities; //!< remember signales entites
+        int                                      m_sharedHeadersTrials = 0;  //!< Tried to read shared file headers
+        QSet<BlackMisc::Network::CEntityFlags::Entity> m_signalledEntities;  //!< remember signalled entites
 
         // for reading XML and VATSIM data files
         Vatsim::CVatsimStatusFileReader *m_vatsimStatusReader   = nullptr;

src/blackgui/components/datainfoareacomponent.cpp

@@ -165,7 +165,7 @@ namespace BlackGui
 
         void CDataInfoAreaComponent::requestUpdateOfAllDbData()
         {
-            sGui->getWebDataServices()->triggerReloadFromDb(CEntityFlags::AllDbEntitiesNoInfoObjectsNoAirports, QDateTime());
+            sGui->getWebDataServices()->triggerLoadingDirectlyFromDb(CEntityFlags::AllDbEntitiesNoInfoObjectsNoAirports, QDateTime());
         }
 
         void CDataInfoAreaComponent::requestUpdatedData(CEntityFlags::Entity entity)

src/blackgui/components/dbaircrafticaocomponent.cpp

@@ -70,7 +70,7 @@ namespace BlackGui
         void CDbAircraftIcaoComponent::ps_reload()
         {
             if (!sGui) { return; }
-            sGui->getWebDataServices()->triggerReloadFromDb(CEntityFlags::AircraftIcaoEntity, QDateTime());
+            sGui->getWebDataServices()->triggerLoadingDirectlyFromDb(CEntityFlags::AircraftIcaoEntity, QDateTime());
         }
     } // ns
 } // ns

src/blackgui/components/dbairlineicaocomponent.cpp

@@ -63,7 +63,7 @@ namespace BlackGui
         void CDbAirlineIcaoComponent::ps_reload()
         {
             if (!sGui || !sGui->hasWebDataServices()) { return; }
-            sGui->getWebDataServices()->triggerReloadFromDb(CEntityFlags::AirlineIcaoEntity, QDateTime());
+            sGui->getWebDataServices()->triggerLoadingDirectlyFromDb(CEntityFlags::AirlineIcaoEntity, QDateTime());
         }
     } // ns
 } // ns

src/blackgui/components/dbcountrycomponent.cpp

@@ -58,7 +58,7 @@ namespace BlackGui
         void CDbCountryComponent::ps_reload()
         {
             if (!sGui || !sGui->getWebDataServices()) { return; }
-            sApp->getWebDataServices()->triggerReloadFromDb(CEntityFlags::CountryEntity);
+            sApp->getWebDataServices()->triggerLoadingDirectlyFromDb(CEntityFlags::CountryEntity);
         }
     } // ns
 } // ns

src/blackgui/components/dbdistributorcomponent.cpp

@@ -60,7 +60,7 @@ namespace BlackGui
         void CDbDistributorComponent::ps_reload()
         {
             if (!sGui) { return; }
-            sGui->getWebDataServices()->triggerReloadFromDb(CEntityFlags::DistributorEntity);
+            sGui->getWebDataServices()->triggerLoadingDirectlyFromDb(CEntityFlags::DistributorEntity);
         }
     } // ns
 } // ns

src/blackgui/components/dbliverycomponent.cpp

@@ -71,7 +71,7 @@ namespace BlackGui
         void CDbLiveryComponent::ps_reload()
         {
             if (!sGui) { return; }
-            sGui->getWebDataServices()->triggerReloadFromDb(CEntityFlags::LiveryEntity);
+            sGui->getWebDataServices()->triggerLoadingDirectlyFromDb(CEntityFlags::LiveryEntity);
         }
     } // ns
 } // ns

src/blackgui/components/dbmodelcomponent.cpp

@@ -73,7 +73,7 @@ namespace BlackGui
                 CAircraftModel model(ui->tvp_AircraftModel->container().latestObject());
                 ts = model.getUtcTimestamp();
             }
-            sGui->getWebDataServices()->triggerReloadFromDb(CEntityFlags::ModelEntity, ts);
+            sGui->getWebDataServices()->triggerLoadingDirectlyFromDb(CEntityFlags::ModelEntity, ts);
         }
 
         void CDbModelComponent::ps_modelsRead(CEntityFlags::Entity entity, CEntityFlags::ReadState readState, int count)
@@ -91,7 +91,7 @@ namespace BlackGui
         void CDbModelComponent::ps_reload()
         {
             if (!sGui) { return; }
-            sGui->getWebDataServices()->triggerReloadFromDb(CEntityFlags::ModelEntity);
+            sGui->getWebDataServices()->triggerLoadingDirectlyFromDb(CEntityFlags::ModelEntity);
         }
 
         void CDbModelComponent::ps_onStyleSheetChanged()