refs #748, further airport from DB follow up fixes

* Ignore mode, not all applications need to load everything (mapping tool no airports)
* removed signal in airport reader which is already in base class
* added "virtual" keyword
* fixed another syncronize typo
* added allDbEntiiesUsed() to also reflect the config (ignore flag)
* ensure allSwiftDbData are signaled correctly even with ignore flag

src/blackcore/db/airportdatareader.cpp

@@ -1,4 +1,4 @@
-/* Copyright (C) 2015
+/* Copyright (C) 2016
  * swift project Community / Contributors
  *
  * This file is part of swift project. It is subject to the license terms in the LICENSE file found in the top-level

src/blackcore/db/airportdatareader.h

@@ -1,4 +1,4 @@
-/* Copyright (C) 2015
+/* Copyright (C) 2016
  * swift project Community / Contributors
  *
  * This file is part of swift project. It is subject to the license terms in the LICENSE file found in the top-level
@@ -30,10 +30,6 @@ namespace BlackCore
         {
             Q_OBJECT
 
-        signals:
-            //! Emitted when data is parsed
-            void dataRead(BlackMisc::Network::CEntityFlags::Entity entity, BlackMisc::Network::CEntityFlags::ReadState state, int number);
-
         public:
             //! Constructor
             CAirportDataReader(QObject* parent, const CDatabaseReaderConfigList &config);
@@ -42,21 +38,15 @@ namespace BlackCore
             //! \threadsafe
             BlackMisc::Aviation::CAirportList getAirports() const;
 
-            //! \copydoc BlackCore::Db::CDatabaseReader::getCacheTimestamp()
+            // base class overrides
-            QDateTime getCacheTimestamp(BlackMisc::Network::CEntityFlags::Entity entities) const override;
+            virtual QDateTime getCacheTimestamp(BlackMisc::Network::CEntityFlags::Entity entities) const override;
-
+            virtual int getCacheCount(BlackMisc::Network::CEntityFlags::Entity entity) const override;
-            //! \copydoc BlackCore::Db::CDatabaseReader::getCacheCount()
-            int getCacheCount(BlackMisc::Network::CEntityFlags::Entity entity) const override;
 
         protected:
-            //! \copydoc BlackCore::Db::CDatabaseReader::synchronizeCaches()
+            // base class overrides
-            void synchronizeCaches(BlackMisc::Network::CEntityFlags::Entity entities) override;
+            virtual void synchronizeCaches(BlackMisc::Network::CEntityFlags::Entity entities) override;
-
+            virtual void invalidateCaches(BlackMisc::Network::CEntityFlags::Entity entities) override;
-            //! \copydoc BlackCore::Db::CDatabaseReader::invalidateCaches()
+            virtual bool hasChangedUrl(BlackMisc::Network::CEntityFlags::Entity entity) const override;
-            void invalidateCaches(BlackMisc::Network::CEntityFlags::Entity entities) override;
-
-            //! \copydoc BlackCore::Db::CDatabaseReader::hasChangedUrl()
-            bool hasChangedUrl(BlackMisc::Network::CEntityFlags::Entity entity) const override;
 
         private:
             //! URL for airport list
@@ -80,7 +70,6 @@ namespace BlackCore
 
             //! Reader URL (we read from where?) used to detect changes of location
             BlackMisc::CData<BlackCore::Data::TDbModelReaderBaseUrl> m_readerUrlCache {this, &CAirportDataReader::ps_baseUrlCacheChanged };
-
         };
     }
 } // ns

src/blackcore/db/databasereader.cpp

@@ -48,7 +48,7 @@ namespace BlackCore
 
             // we accept cached cached data
             Q_ASSERT_X(!entities.testFlag(CEntityFlags::InfoObjectEntity), Q_FUNC_INFO, "Read info objects directly");
-            const bool hasInfoObjects = this->hasInfoObjects(); // no info objects is no necessarily error, but indicates a) either data not available or b) only caches is used
+            const bool hasInfoObjects = this->hasInfoObjects(); // no info objects is no necessarily error, but indicates a) either data not available (DB down) or b) only caches are used
             CEntityFlags::Entity allEntities    = entities;
             CEntityFlags::Entity cachedEntities = CEntityFlags::NoEntity;
             CEntityFlags::Entity currentEntity  = CEntityFlags::iterateDbEntities(allEntities); // CEntityFlags::InfoObjectEntity will be ignored
@@ -56,7 +56,13 @@ namespace BlackCore
             {
                 const CDatabaseReaderConfig config(this->getConfigForEntity(currentEntity));
                 const QString currentEntityName = CEntityFlags::flagToString(currentEntity);
-                if (config.getRetrievalMode().testFlag(CDbFlags::Cached))
+                const CDbFlags::DataRetrievalMode rm = config.getRetrievalMode();
+                Q_ASSERT_X(!rm.testFlag(CDbFlags::Unspecified), Q_FUNC_INFO, "Missing retrieval mode");
+                if (rm.testFlag(CDbFlags::Ignore) || rm.testFlag(CDbFlags::Canceled))
+                {
+                    entities &= ~currentEntity; // do not load from web database
+                }
+                else if (rm.testFlag(CDbFlags::Cached))
                 {
                     if (hasInfoObjects)
                     {
@@ -69,7 +75,7 @@ namespace BlackCore
                         if (!changedUrl && cacheTimestamp >= latestEntityTimestamp && cacheTimestamp >= 0 && latestEntityTimestamp >= 0)
                         {
                             this->synchronizeCaches(currentEntity);
-                            entities &= ~currentEntity; // do not load from web
+                            entities &= ~currentEntity; // do not load from web database
                             cachedEntities |= currentEntity; // read from cache
                             CLogMessage(this).info("Using cache for %1 (%2, %3)") << currentEntityName << cacheTs.toString() << cacheTimestamp;
                         }
@@ -91,7 +97,7 @@ namespace BlackCore
                         this->synchronizeCaches(currentEntity);
                         const int c = this->getCacheCount(currentEntity);
                         CLogMessage(this).info("No info object for %1, using cache with %2 objects") << currentEntityName << c;
-                        entities &= ~currentEntity; // do not load from web
+                        entities &= ~currentEntity; // do not load from web database
                         cachedEntities |= currentEntity; // read from cache
                     }
                 }
@@ -207,6 +213,7 @@ namespace BlackCore
 
         void CDatabaseReader::emitReadSignalPerSingleCachedEntity(CEntityFlags::Entity cachedEntities)
         {
+            if (cachedEntities == CEntityFlags::NoEntity) { return; }
             CEntityFlags::Entity cachedEntitiesToEmit = cachedEntities;
             CEntityFlags::Entity currentCachedEntity  = CEntityFlags::iterateDbEntities(cachedEntitiesToEmit);
             while (currentCachedEntity)

src/blackcore/db/databasereaderconfig.cpp

@@ -66,6 +66,12 @@ namespace BlackCore
             return (this->getRetrievalMode().testFlag(CDbFlags::DbDirect) || this->getRetrievalMode().testFlag(CDbFlags::Shared));
         }
 
+        bool CDatabaseReaderConfig::possiblyReadsFromCache() const
+        {
+            if (!this->isValid()) { return false; }
+            return (this->getRetrievalMode().testFlag(CDbFlags::Cached) || this->getRetrievalMode().testFlag(CDbFlags::CacheThenDb) || this->getRetrievalMode().testFlag(CDbFlags::CacheThenShared));
+        }
+
         bool CDatabaseReaderConfig::isValid() const
         {
             return this->m_entities != BlackMisc::Network::CEntityFlags::NoEntity;
@@ -118,6 +124,19 @@ namespace BlackCore
             return false;
         }
 
+        CEntityFlags::Entity CDatabaseReaderConfigList::getEntitesCachedOrReadFromDB() const
+        {
+            CEntityFlags::Entity entities = CEntityFlags::NoEntity;
+            for (const CDatabaseReaderConfig &config : *this)
+            {
+                if (config.possiblyReadsFromSwiftDb() || config.possiblyReadsFromCache())
+                {
+                    entities |= config.getEntities();
+                }
+            }
+            return entities;
+        }
+
         CDatabaseReaderConfigList CDatabaseReaderConfigList::forMappingTool()
         {
             const CTime timeout(5.0, CTimeUnit::min());
@@ -125,6 +144,7 @@ namespace BlackCore
             CDatabaseReaderConfigList l;
             l.push_back(CDatabaseReaderConfig(CEntityFlags::AircraftIcaoEntity, retrievalFlags, timeout));
             l.push_back(CDatabaseReaderConfig(CEntityFlags::AirlineIcaoEntity, retrievalFlags, timeout));
+            l.push_back(CDatabaseReaderConfig(CEntityFlags::AirportEntity, CDbFlags::Ignore, timeout)); // not needed in mapping tool
             l.push_back(CDatabaseReaderConfig(CEntityFlags::DistributorEntity, retrievalFlags, timeout));
             l.push_back(CDatabaseReaderConfig(CEntityFlags::ModelEntity, retrievalFlags, timeout));
             l.push_back(CDatabaseReaderConfig(CEntityFlags::LiveryEntity, retrievalFlags, timeout));
@@ -139,6 +159,7 @@ namespace BlackCore
             CDatabaseReaderConfigList l;
             l.push_back(CDatabaseReaderConfig(CEntityFlags::AircraftIcaoEntity, retrievalFlags, timeout));
             l.push_back(CDatabaseReaderConfig(CEntityFlags::AirlineIcaoEntity, retrievalFlags, timeout));
+            l.push_back(CDatabaseReaderConfig(CEntityFlags::AirportEntity, retrievalFlags, timeout));
             l.push_back(CDatabaseReaderConfig(CEntityFlags::DistributorEntity, retrievalFlags, timeout));
             l.push_back(CDatabaseReaderConfig(CEntityFlags::ModelEntity, retrievalFlags, timeout));
             l.push_back(CDatabaseReaderConfig(CEntityFlags::LiveryEntity, retrievalFlags, timeout));
@@ -158,6 +179,7 @@ namespace BlackCore
             CDatabaseReaderConfigList l;
             l.push_back(CDatabaseReaderConfig(CEntityFlags::AircraftIcaoEntity, retrievalFlags, timeout));
             l.push_back(CDatabaseReaderConfig(CEntityFlags::AirlineIcaoEntity, retrievalFlags, timeout));
+            l.push_back(CDatabaseReaderConfig(CEntityFlags::AirportEntity, retrievalFlags, timeout));
             l.push_back(CDatabaseReaderConfig(CEntityFlags::DistributorEntity, retrievalFlags, timeout));
             l.push_back(CDatabaseReaderConfig(CEntityFlags::ModelEntity, retrievalFlags, timeout));
             l.push_back(CDatabaseReaderConfig(CEntityFlags::LiveryEntity, retrievalFlags, timeout));

src/blackcore/db/databasereaderconfig.h

@@ -32,7 +32,7 @@ namespace BlackCore
 {
     namespace Db
     {
-        //! Details how to read
+        //! Details how to read a certain entity
         class BLACKCORE_EXPORT CDatabaseReaderConfig : public BlackMisc::CValueObject<CDatabaseReaderConfig>
         {
         public:
@@ -62,6 +62,9 @@ namespace BlackCore
             //! Will read from swift DB
             bool possiblyReadsFromSwiftDb() const;
 
+            //! Will read from cache
+            bool possiblyReadsFromCache() const;
+
             //! Fully initialized
             bool isValid() const;
 
@@ -103,6 +106,9 @@ namespace BlackCore
             //! Will read from swift DB
             bool possiblyReadsFromSwiftDb() const;
 
+            //! Entities which will use cache or DB, so no canceled or ignored ones
+            BlackMisc::Network::CEntityFlags::Entity getEntitesCachedOrReadFromDB() const;
+
             //! Init for mapping tool
             static CDatabaseReaderConfigList forMappingTool();
 

src/blackcore/webdataservices.cpp

@@ -170,7 +170,7 @@ namespace BlackCore
         return false;
     }
 
-    void CWebDataServices::syncronizeDbCaches(CEntityFlags::Entity entities)
+    void CWebDataServices::synchronizeDbCaches(CEntityFlags::Entity entities)
     {
         if (this->m_modelDataReader) { this->m_modelDataReader->synchronizeCaches(entities); }
         if (this->m_icaoDataReader) { this->m_icaoDataReader->synchronizeCaches(entities); }
@@ -577,6 +577,34 @@ namespace BlackCore
         if (this->m_databaseWriter)       { this->m_databaseWriter->gracefulShutdown(); }
     }
 
+    CEntityFlags::Entity CWebDataServices::allDbEntiiesUsed() const
+    {
+        // obtain entities from real readers (means when reader is really used)
+        CEntityFlags::Entity entities = CEntityFlags::NoEntity;
+        if (this->m_icaoDataReader)
+        {
+            entities |=  CWebReaderFlags::allEntitiesForReaders(CWebReaderFlags::IcaoDataReader);
+        }
+        if (this->m_modelDataReader)
+        {
+            entities |=  CWebReaderFlags::allEntitiesForReaders(CWebReaderFlags::ModelReader);
+        }
+        if (this->m_airportDataReader)
+        {
+            entities |=  CWebReaderFlags::allEntitiesForReaders(CWebReaderFlags::AirportReader);
+        }
+
+        // when we have a config, we ignore the ones not from cache or DB
+        if (!this->m_dbReaderConfig.isEmpty())
+        {
+            CEntityFlags::Entity configuredEntities = this->m_dbReaderConfig.getEntitesCachedOrReadFromDB();
+            entities &= configuredEntities;
+        }
+
+        entities &= CEntityFlags::AllDbEntities; // make sure to only use DB data
+        return entities;
+    }
+
     const CLogCategoryList &CWebDataServices::getLogCategories()
     {
         static const BlackMisc::CLogCategoryList cats { CLogCategory("swift.datareader"), CLogCategory::webservice() };
@@ -760,7 +788,8 @@ namespace BlackCore
         }
 
         this->m_swiftDbEntitiesRead |= entity;
-        if (((static_cast<int>(this->m_swiftDbEntitiesRead)) & static_cast<int>(CEntityFlags::AllDbEntities)) > 0)
+        const int allUsedEntities = static_cast<int>(this->allDbEntiiesUsed());
+        if (((static_cast<int>(this->m_swiftDbEntitiesRead)) & allUsedEntities) == allUsedEntities)
         {
             emit allSwiftDbDataRead();
         }
@@ -782,7 +811,7 @@ namespace BlackCore
 
     void CWebDataServices::readInBackground(CEntityFlags::Entity entities)
     {
-        m_initialRead = true; // read started
+        this->m_initialRead = true; // read started
 
         const int waitForInfoObjects = 1000; // ms
         const int maxWaitCycles = 10;

src/blackcore/webdataservices.h

@@ -116,6 +116,9 @@ namespace BlackCore
         //! Reader flags
         CWebReaderFlags::WebReader getReaderFlags() const { return m_readers; }
 
+        //! All DB entities for those used and not ignored
+        BlackMisc::Network::CEntityFlags::Entity allDbEntiiesUsed() const;
+
         //! FSD servers
         //! \threadsafe
         BlackMisc::Network::CServerList getVatsimFsdServers() const;
@@ -328,8 +331,8 @@ namespace BlackCore
         //! Can connect to swift DB?
         bool canConnectSwiftDb() const;
 
-        //! Syncronize all DB caches
+        //! Synchronize all DB caches
-        void syncronizeDbCaches(BlackMisc::Network::CEntityFlags::Entity entities);
+        void synchronizeDbCaches(BlackMisc::Network::CEntityFlags::Entity entities);
 
         //! Write data to disk (mainly for testing scenarios)
         bool writeDbDataToDisk(const QString &dir) const;

src/blackgui/components/dbloadoverviewcomponent.cpp

@@ -156,7 +156,7 @@ namespace BlackGui
 
         void CDbLoadOverviewComponent::synchronizeCaches()
         {
-            sGui->getWebDataServices()->syncronizeDbCaches(CEntityFlags::AllDbEntities);
+            sGui->getWebDataServices()->synchronizeDbCaches(CEntityFlags::AllDbEntities);
         }
 
         void CDbLoadOverviewComponent::ps_reloadPressed()

src/blackmisc/db/dbflags.cpp

@@ -27,6 +27,8 @@ namespace BlackMisc
             switch (flag)
             {
             case Unspecified: return "Unspecified";
+            case Ignore: return "Ignore";
+            case Canceled: return "Canceled";
             case DbDirect: return "Direct DB access";
             case Shared: return "Shared data";
             case Cached: return "Cached data";
@@ -40,7 +42,8 @@ namespace BlackMisc
         {
             QStringList list;
             if (mode.testFlag(Unspecified)) list << "Unspecified";
-            if (mode.testFlag(Canceled)) list << "Unspecified";
+            if (mode.testFlag(Canceled)) list << "Canceled";
+            if (mode.testFlag(Ignore)) list << "Ignore";
 
             if (mode.testFlag(DbDirect)) list << "Direct DB access";
             if (mode.testFlag(Shared)) list << "Shared data";

src/blackmisc/db/dbflags.h

@@ -35,7 +35,8 @@ namespace BlackMisc
                 DbDirect               = 1 << 0,              //!< directly from DB
                 Shared                 = 1 << 1,              //!< shared directory
                 Cached                 = 1 << 2,              //!< from cache
-                Canceled               = 1 << 3,              //!< cancel DB reading
+                Canceled               = 1 << 3,              //!< canceled DB reading
+                Ignore                 = 1 << 4,              //!< ignore this entity
                 CacheThenDb            = DbDirect | Cached,   //!< Cache when possible, otherwise DB
                 CacheThenShared        = Shared | Cached      //!< Cache when possible, otherwise shared
 

src/blackmisc/network/entityflags.cpp

@@ -24,6 +24,7 @@ namespace BlackMisc
             {
             case AircraftIcaoEntity: return "Aircraft ICAO";
             case AirlineIcaoEntity: return "Airline ICAO";
+            case AirportEntity: return "Airport";
             case AllEntities: return "All";
             case AllIcaoAndCountries: return "All ICAO + country";
             case AllIcaoEntities: return "All ICAO";
@@ -36,7 +37,6 @@ namespace BlackMisc
             case NoEntity: return "no data";
             case VatsimDataFile: return "VATSIM data file";
             case VatsimStatusFile: return "VATSIM status file";
-            case AirportEntity: return "Airport";
             default:
                 BLACK_VERIFY_X(false, Q_FUNC_INFO, "wrong flags");
                 return "wrong flags";
@@ -48,6 +48,7 @@ namespace BlackMisc
             QStringList list;
             if (flag.testFlag(AircraftIcaoEntity)) list << "Aircraft ICAO";
             if (flag.testFlag(AirlineIcaoEntity)) list << "Airline ICAO";
+            if (flag.testFlag(AirportEntity)) list << "Airport";
             if (flag.testFlag(BookingEntity)) list << "VATSIM bookings";
             if (flag.testFlag(CountryEntity)) list << "Country";
             if (flag.testFlag(DistributorEntity)) list << "Distributor";
@@ -57,7 +58,6 @@ namespace BlackMisc
             if (flag.testFlag(NoEntity)) list << "no data";
             if (flag.testFlag(VatsimDataFile)) list << "VATSIM data file";
             if (flag.testFlag(VatsimStatusFile)) list << "VATSIM status file";
-            if (flag.testFlag(AirportEntity)) list << "Airport";
             return list.join(',');
         }
 
@@ -112,6 +112,7 @@ namespace BlackMisc
             if (entities == NoEntity || entities == InfoObjectEntity) { return NoEntity; }
             if (entities.testFlag(AircraftIcaoEntity)) { entities &= ~AircraftIcaoEntity; return AircraftIcaoEntity; }
             if (entities.testFlag(AirlineIcaoEntity))  { entities &= ~AirlineIcaoEntity; return AirlineIcaoEntity; }
+            if (entities.testFlag(AirportEntity))      { entities &= ~AirportEntity; return AirportEntity; }
             if (entities.testFlag(LiveryEntity))       { entities &= ~LiveryEntity; return LiveryEntity; }
             if (entities.testFlag(CountryEntity))      { entities &= ~CountryEntity; return CountryEntity; }
             if (entities.testFlag(ModelEntity))        { entities &= ~ModelEntity; return ModelEntity; }