refs #649, added caches to model data reader

* trait
* const fix distributor list

src/blackcore/data/dbcaches.cpp

@@ -49,11 +49,22 @@ namespace BlackCore
             return ll;
         }
 
+        const CDistributorList &DbDistributorCache::defaultValue()
+        {
+            static const CDistributorList dl;
+            return dl;
+        }
+
         const BlackMisc::Network::CUrl &DbIcaoReaderBaseUrl::defaultValue()
         {
             static const CUrl url;
             return url;
         }
 
+        const CUrl &DbModelReaderBaseUrl::defaultValue()
+        {
+            static const CUrl url;
+            return url;
+        }
     } // ns
 } // ns

src/blackcore/data/dbcaches.h

@@ -14,6 +14,7 @@
 
 #include "blackmisc/datacache.h"
 #include "blackmisc/simulation/aircraftmodellist.h"
+#include "blackmisc/simulation/distributorlist.h"
 #include "blackmisc/aviation/airlineicaocodelist.h"
 #include "blackmisc/aviation/aircrafticaocodelist.h"
 #include "blackmisc/aviation/liverylist.h"
@@ -40,6 +41,32 @@ namespace BlackCore
             static const char *key() { return "dbmodelcache"; }
         };
 
+        //! Trait for DB distributor cache
+        struct DbDistributorCache : public BlackMisc::CDataTrait<BlackMisc::Simulation::CDistributorList>
+        {
+            //! Default value
+            static const BlackMisc::Simulation::CDistributorList &defaultValue();
+
+            //! Defer loading (no currently small)
+            static constexpr bool isDeferred() { return false; }
+
+            //! Key in data cache
+            static const char *key() { return "dbdistributorcache"; }
+        };
+
+        //! Trait for DB liveries
+        struct DbLiveryCache : public BlackMisc::CDataTrait<BlackMisc::Aviation::CLiveryList>
+        {
+            //! Default value
+            static const BlackMisc::Aviation::CLiveryList &defaultValue();
+
+            //! Defer loading
+            static constexpr bool isDeferred() { return true; }
+
+            //! Key in data cache
+            static const char *key() { return "dbliverycache"; }
+        };
+
         //! Trait for DB airline ICAO codes
         struct DbAirlineIcaoCache : public BlackMisc::CDataTrait<BlackMisc::Aviation::CAirlineIcaoCodeList>
         {
@@ -80,19 +107,6 @@ namespace BlackCore
             static const char *key() { return "dbcountrycache"; }
         };
 
-        //! Trait for DB liveries
-        struct DbLiveryCache : public BlackMisc::CDataTrait<BlackMisc::Aviation::CLiveryList>
-        {
-            //! Default value
-            static const BlackMisc::Aviation::CLiveryList &defaultValue();
-
-            //! Defer loading
-            static constexpr bool isDeferred() { return true; }
-
-            //! Key in data cache
-            static const char *key() { return "dbliverycache"; }
-        };
-
         //! Trait for ICAO reader base URL
         struct DbIcaoReaderBaseUrl : public BlackMisc::CDataTrait<BlackMisc::Network::CUrl>
         {
@@ -105,6 +119,19 @@ namespace BlackCore
             //! Key in data cache
             static const char *key() { return "dbicaoreaderurl"; }
         };
+
+        //! Trait for ICAO reader base URL
+        struct DbModelReaderBaseUrl : public BlackMisc::CDataTrait<BlackMisc::Network::CUrl>
+        {
+            //! Default value
+            static const BlackMisc::Network::CUrl &defaultValue();
+
+            //! First load is synchronous
+            static constexpr bool isPinned() { return true; }
+
+            //! Key in data cache
+            static const char *key() { return "dbmodelreaderurl"; }
+        };
     } // ns
 } // ns
 

src/blackcore/db/databasereader.h

@@ -152,6 +152,8 @@ namespace BlackCore
             //! Split into single entity and send dataRead signal
             void emitReadSignalPerSingleCachedEntity(BlackMisc::Network::CEntityFlags::Entity cachedEntities);
 
+            // ---------------- cache access ------------------
+
             //! Syncronize caches for given entities
             virtual void syncronizeCaches(BlackMisc::Network::CEntityFlags::Entity entities) = 0;
 

src/blackcore/db/icaodatareader.h

@@ -146,7 +146,9 @@ namespace BlackCore
             BlackMisc::CData<BlackCore::Data::DbAircraftIcaoCache> m_aircraftIcaoCache {this, &CIcaoDataReader::ps_aircraftIcaoCacheChanged };
             BlackMisc::CData<BlackCore::Data::DbAirlineIcaoCache>  m_airlineIcaoCache  {this, &CIcaoDataReader::ps_airlineIcaoCacheChanged };
             BlackMisc::CData<BlackCore::Data::DbCountryCache>      m_countryCache      {this, &CIcaoDataReader::ps_countryCacheChanged };
-            BlackMisc::CData<BlackCore::Data::DbIcaoReaderBaseUrl> m_readerUrlCache    {this, &CIcaoDataReader::ps_baseUrlCacheChanged };
+
+            //! Reader URL (we read from where?) used to detect changes of location
+            BlackMisc::CData<BlackCore::Data::DbIcaoReaderBaseUrl> m_readerUrlCache {this, &CIcaoDataReader::ps_baseUrlCacheChanged };
 
             //! Update reader URL
             void updateReaderUrl(const BlackMisc::Network::CUrl &url);

src/blackcore/db/modeldatareader.cpp

@@ -47,97 +47,89 @@ namespace BlackCore
 
         CLiveryList CModelDataReader::getLiveries() const
         {
-            QReadLocker l(&m_lockLivery);
+            return this->m_liveryCache.getCopy();
-            return m_liveries;
         }
 
         CLivery CModelDataReader::getLiveryForCombinedCode(const QString &combinedCode) const
         {
             if (!CLivery::isValidCombinedCode(combinedCode)) { return CLivery(); }
-            CLiveryList liveries(getLiveries());
+            const CLiveryList liveries(getLiveries());
             return liveries.findByCombinedCode(combinedCode);
         }
 
         CLivery CModelDataReader::getStdLiveryForAirlineCode(const CAirlineIcaoCode &icao) const
         {
             if (!icao.hasValidDesignator()) { return CLivery(); }
-            CLiveryList liveries(getLiveries());
+            const CLiveryList liveries(getLiveries());
             return liveries.findStdLiveryByAirlineIcaoDesignator(icao);
         }
 
         CLivery CModelDataReader::getLiveryForDbKey(int id) const
         {
             if (id < 0) { return CLivery(); }
-            CLiveryList liveries(getLiveries());
+            const CLiveryList liveries(getLiveries());
             return liveries.findByKey(id);
         }
 
         CLivery CModelDataReader::smartLiverySelector(const CLivery &liveryPattern) const
         {
-            CLiveryList liveries(getLiveries()); // thread safe copy
+            const CLiveryList liveries(getLiveries()); // thread safe copy
             return liveries.smartLiverySelector(liveryPattern);
         }
 
         CDistributorList CModelDataReader::getDistributors() const
         {
-            QReadLocker l(&m_lockDistributor);
+            return m_distributorCache.getCopy();
-            return m_distributors;
         }
 
         CAircraftModelList CModelDataReader::getModels() const
         {
-            QReadLocker l(&m_lockModels);
+            return m_modelCache.getCopy();
-            return m_models;
         }
 
         CAircraftModel CModelDataReader::getModelForModelString(const QString &modelString) const
         {
             if (modelString.isEmpty()) { return CAircraftModel(); }
-            CAircraftModelList models(getModels());
+            const CAircraftModelList models(getModels());
             return models.findFirstByModelStringOrDefault(modelString);
         }
 
         CAircraftModelList CModelDataReader::getModelsForAircraftDesignatorAndLiveryCombinedCode(const QString &aircraftDesignator, const QString &combinedCode)
         {
             if (aircraftDesignator.isEmpty()) { return CAircraftModelList(); }
-            CAircraftModelList models(getModels());
+            const CAircraftModelList models(getModels());
             return models.findByAircraftDesignatorAndLiveryCombinedCode(aircraftDesignator, combinedCode);
         }
 
         int CModelDataReader::getLiveriesCount() const
         {
-            QReadLocker l(&m_lockLivery);
+            return this->getLiveries().size();
-            return m_liveries.size();
         }
 
         int CModelDataReader::getDistributorsCount() const
         {
-            QReadLocker l(&m_lockDistributor);
+            return this->getDistributors().size();
-            return m_distributors.size();
         }
 
         CDistributor CModelDataReader::smartDistributorSelector(const CDistributor &distributorPattern) const
         {
-            CDistributorList distributors(getDistributors()); // thread safe copy
+            const CDistributorList distributors(getDistributors()); // thread safe copy
             return distributors.smartDistributorSelector(distributorPattern);
         }
 
         int CModelDataReader::getModelsCount() const
         {
-            QReadLocker l(&m_lockModels);
+            return this->getModels().size();
-            return m_models.size();
         }
 
         QList<int> CModelDataReader::getModelDbKeys() const
         {
-            QReadLocker l(&m_lockModels);
+            return this->getModels().toDbKeyList();
-            return m_models.toDbKeyList();
         }
 
         QStringList CModelDataReader::getModelStrings() const
         {
-            QReadLocker l(&m_lockModels);
+            return this->getModels().getModelStrings(false);
-            return m_models.getModelStrings(false);
         }
 
         bool CModelDataReader::areAllDataRead() const
@@ -217,6 +209,37 @@ namespace BlackCore
             }
         }
 
+        void CModelDataReader::ps_liveryCacheChanged()
+        {
+            // void
+        }
+
+        void CModelDataReader::ps_modelCacheChanged()
+        {
+            // void
+        }
+
+        void CModelDataReader::ps_distributorCacheChanged()
+        {
+            // void
+        }
+
+        void CModelDataReader::ps_baseUrlCacheChanged()
+        {
+            // void
+        }
+
+        void CModelDataReader::updateReaderUrl(const CUrl &url)
+        {
+            const CUrl current = this->m_readerUrlCache.getCopy();
+            if (current == url) { return; }
+            const CStatusMessage m = this->m_readerUrlCache.set(url);
+            if (m.isFailure())
+            {
+                CLogMessage::preformatted(m);
+            }
+        }
+
         void CModelDataReader::ps_parseLiveryData(QNetworkReply *nwReplyPtr)
         {
             // wrap pointer, make sure any exit cleans up reply
@@ -245,16 +268,18 @@ namespace BlackCore
                 liveries  = CLiveryList::fromDatabaseJson(res);
             }
 
-            // this part needs to be synchronized
             int n = liveries.size();
+            qint64 latestTimestamp = liveries.latestTimestampMsecsSinceEpoch();
+            if (n > 0 && latestTimestamp < 0)
             {
-                QWriteLocker wl(&this->m_lockLivery);
+                CLogMessage(this).error("No timestamp in livery list, setting to last modified value");
-                this->m_liveries = liveries;
+                latestTimestamp = lastModifiedMsSinceEpoch(nwReply.data());
             }
+            this->m_liveryCache.set(liveries, latestTimestamp);
+            this->updateReaderUrl(this->getBaseUrl());
 
             // never emit when lock is held -> deadlock
-            emit dataRead(CEntityFlags::LiveryEntity,
+            emit dataRead(CEntityFlags::LiveryEntity, res.isRestricted() ? CEntityFlags::ReadFinishedRestricted : CEntityFlags::ReadFinished, n);
-                          res.isRestricted() ? CEntityFlags::ReadFinishedRestricted : CEntityFlags::ReadFinished, n);
             CLogMessage(this).info("Read %1 %2 from %3") << n << CEntityFlags::flagToString(CEntityFlags::LiveryEntity) << urlString;
         }
 
@@ -286,14 +311,17 @@ namespace BlackCore
                 distributors = CDistributorList::fromDatabaseJson(res);
             }
 
-            // this part needs to be synchronized
             int n = distributors.size();
+            qint64 latestTimestamp = distributors.latestTimestampMsecsSinceEpoch();
+            if (n > 0 && latestTimestamp < 0)
             {
-                QWriteLocker wl(&this->m_lockDistributor);
+                CLogMessage(this).error("No timestamp in distributor list, setting to last modified value");
-                this->m_distributors = distributors;
+                latestTimestamp = lastModifiedMsSinceEpoch(nwReply.data());
             }
-            emit dataRead(CEntityFlags::DistributorEntity,
+            this->m_distributorCache.set(distributors, latestTimestamp);
-                          res.isRestricted() ? CEntityFlags::ReadFinishedRestricted : CEntityFlags::ReadFinished, n);
+            this->updateReaderUrl(this->getBaseUrl());
+
+            emit dataRead(CEntityFlags::DistributorEntity, res.isRestricted() ? CEntityFlags::ReadFinishedRestricted : CEntityFlags::ReadFinished, n);
             CLogMessage(this).info("Read %1 %2 from %3") << n << CEntityFlags::flagToString(CEntityFlags::DistributorEntity) << urlString;
         }
 
@@ -327,13 +355,16 @@ namespace BlackCore
 
             // syncronized update
             int n = models.size();
+            qint64 latestTimestamp = models.latestTimestampMsecsSinceEpoch();
+            if (n > 0 && latestTimestamp < 0)
             {
-                QWriteLocker wl(&this->m_lockModels);
+                CLogMessage(this).error("No timestamp in model list, setting to last modified value");
-                this->m_models = models;
+                latestTimestamp = lastModifiedMsSinceEpoch(nwReply.data());
             }
+            this->m_modelCache.set(models, latestTimestamp);
+            this->updateReaderUrl(this->getBaseUrl());
 
-            emit dataRead(CEntityFlags::ModelEntity,
+            emit dataRead(CEntityFlags::ModelEntity, res.isRestricted() ? CEntityFlags::ReadFinishedRestricted : CEntityFlags::ReadFinished, n);
-                          res.isRestricted() ? CEntityFlags::ReadFinishedRestricted : CEntityFlags::ReadFinished, n);
             CLogMessage(this).info("Read %1 %2 from %3") << n << CEntityFlags::flagToString(CEntityFlags::ModelEntity) << urlString;
         }
 
@@ -351,11 +382,8 @@ namespace BlackCore
                     CLiveryList liveries;
                     liveries.convertFromJson(liveriesJson);
                     int c = liveries.size();
-                    {
+                    this->m_liveryCache.set(liveries);
-                        QWriteLocker l(&m_lockLivery);
+
-                        m_liveries = liveries;
-                    }
-                    // never emit when lcok is held -> deadlock
                     emit dataRead(CEntityFlags::LiveryEntity, CEntityFlags::ReadFinished, c);
                     reallyRead |= CEntityFlags::LiveryEntity;
                 }
@@ -369,10 +397,8 @@ namespace BlackCore
                     CAircraftModelList models;
                     models.convertFromJson(Json::jsonObjectFromString(modelsJson));
                     int c = models.size();
-                    {
+                    this->m_modelCache.set(models);
-                        QWriteLocker l(&m_lockModels);
+
-                        m_models = models;
-                    }
                     emit dataRead(CEntityFlags::ModelEntity, CEntityFlags::ReadFinished, c);
                     reallyRead |= CEntityFlags::ModelEntity;
                 }
@@ -386,12 +412,10 @@ namespace BlackCore
                     CDistributorList distributors;
                     distributors.convertFromJson(Json::jsonObjectFromString(distributorsJson));
                     int c = distributors.size();
-                    {
+                    this->m_distributorCache.set(distributors);
-                        QWriteLocker l(&m_lockDistributor);
+
-                        m_distributors = distributors;
-                    }
-                    reallyRead |= CEntityFlags::DistributorEntity;
                     emit dataRead(CEntityFlags::DistributorEntity, CEntityFlags::ReadFinished, c);
+                    reallyRead |= CEntityFlags::DistributorEntity;
                 }
             }
 
@@ -438,30 +462,44 @@ namespace BlackCore
 
         void CModelDataReader::syncronizeCaches(CEntityFlags::Entity entities)
         {
-            Q_UNUSED(entities);
+            if (entities.testFlag(CEntityFlags::LiveryEntity)) { this->m_liveryCache.synchronize(); }
+            if (entities.testFlag(CEntityFlags::ModelEntity))  { this->m_modelCache.synchronize(); }
+            if (entities.testFlag(CEntityFlags::DistributorEntity)) { this->m_distributorCache.synchronize(); }
         }
 
         void CModelDataReader::invalidateCaches(CEntityFlags::Entity entities)
         {
-            Q_UNUSED(entities);
+            if (entities.testFlag(CEntityFlags::LiveryEntity)) { CDataCache::instance()->clearAllValues(this->m_liveryCache.getKey()); }
+            if (entities.testFlag(CEntityFlags::ModelEntity))  { CDataCache::instance()->clearAllValues(this->m_modelCache.getKey()); }
+            if (entities.testFlag(CEntityFlags::DistributorEntity)) { CDataCache::instance()->clearAllValues(this->m_distributorCache.getKey()); }
         }
 
         QDateTime CModelDataReader::getCacheTimestamp(CEntityFlags::Entity entity) const
         {
-            Q_UNUSED(entity);
+            switch (entity)
-            return QDateTime();
+            {
+            case CEntityFlags::LiveryEntity:      return this->m_liveryCache.getAvailableTimestamp();
+            case CEntityFlags::ModelEntity:       return this->m_modelCache.getAvailableTimestamp();
+            case CEntityFlags::DistributorEntity: return this->m_distributorCache.getAvailableTimestamp();
+            default: return QDateTime();
+            }
         }
 
         int CModelDataReader::getCacheCount(CEntityFlags::Entity entity) const
         {
-            Q_UNUSED(entity);
+            switch (entity)
-            return 0;
+            {
+            case CEntityFlags::LiveryEntity:      return this->m_liveryCache.getCopy().size();
+            case CEntityFlags::ModelEntity:       return this->m_modelCache.getCopy().size();
+            case CEntityFlags::DistributorEntity: return this->m_distributorCache.getCopy().size();
+            default: return 0;
+            }
         }
 
         bool CModelDataReader::hasChangedUrl(CEntityFlags::Entity entity) const
         {
             Q_UNUSED(entity);
-            return CDatabaseReader::isChangedUrl(CUrl(), this->getBaseUrl());
+            return CDatabaseReader::isChangedUrl(this->m_readerUrlCache.getCopy(), this->getBaseUrl());
         }
 
         const CUrl &CModelDataReader::getBaseUrl()

src/blackcore/db/modeldatareader.h

@@ -13,6 +13,7 @@
 #define BLACKCORE_MODELDATAREADER_H
 
 #include "blackcore/blackcoreexport.h"
+#include "blackcore/data/dbcaches.h"
 #include "blackcore/db/databasereader.h"
 #include "blackmisc/aviation/airlineicaocode.h"
 #include "blackmisc/aviation/livery.h"
@@ -142,17 +143,21 @@ namespace BlackCore
             //! Read / re-read data file
             void ps_read(BlackMisc::Network::CEntityFlags::Entity entity = BlackMisc::Network::CEntityFlags::DistributorLiveryModel, const QDateTime &newerThan = QDateTime());
 
-        private:
+            void ps_liveryCacheChanged();
-            BlackMisc::Aviation::CLiveryList          m_liveries;
+            void ps_modelCacheChanged();
-            BlackMisc::Simulation::CDistributorList   m_distributors;
+            void ps_distributorCacheChanged();
-            BlackMisc::Simulation::CAircraftModelList m_models;
+            void ps_baseUrlCacheChanged();
-            BlackMisc::Network::CUrl m_urlLiveries;
-            BlackMisc::Network::CUrl m_urlDistributors;
-            BlackMisc::Network::CUrl m_urlModels;
 
-            mutable QReadWriteLock m_lockDistributor;
+        private:
-            mutable QReadWriteLock m_lockLivery;
+            BlackMisc::CData<BlackCore::Data::DbLiveryCache>      m_liveryCache {this, &CModelDataReader::ps_liveryCacheChanged };
-            mutable QReadWriteLock m_lockModels;
+            BlackMisc::CData<BlackCore::Data::DbModelCache>       m_modelCache  {this, &CModelDataReader::ps_modelCacheChanged };
+            BlackMisc::CData<BlackCore::Data::DbDistributorCache> m_distributorCache {this, &CModelDataReader::ps_distributorCacheChanged };
+
+            //! Reader URL (we read from where?) used to detect changes of location
+            BlackMisc::CData<BlackCore::Data::DbModelReaderBaseUrl> m_readerUrlCache {this, &CModelDataReader::ps_baseUrlCacheChanged };
+
+            //! Update reader URL
+            void updateReaderUrl(const BlackMisc::Network::CUrl &url);
 
             //! Base URL
             //! \threadsafe

src/blackmisc/simulation/distributorlist.cpp

@@ -32,7 +32,7 @@ namespace BlackMisc
             return CDistributor();
         }
 
-        CDistributor CDistributorList::smartDistributorSelector(const CDistributor &distributorPattern)
+        CDistributor CDistributorList::smartDistributorSelector(const CDistributor &distributorPattern) const
         {
             if (distributorPattern.hasValidDbKey())
             {

src/blackmisc/simulation/distributorlist.h

@@ -49,7 +49,7 @@ namespace BlackMisc
             CDistributor findByKeyOrAlias(const QString &keyOrAlias) const;
 
             //! Best match by given pattern
-            CDistributor smartDistributorSelector(const CDistributor &distributorPattern);
+            CDistributor smartDistributorSelector(const CDistributor &distributorPattern) const;
 
             //! At least is matching key or alias
             bool matchesAnyKeyOrAlias(const QString &keyOrAlias) const;