Ref T237, init/read from JSON files

* multi format compatible (automatically detect format)
* flag if cache values shall be overridden
* init from resource files (those are the files coming with the installer)
* automatically read in background if reader is (already) in its own thread (otherwise cache.set() ASSERT)

src/blackcore/db/airportdatareader.cpp

@@ -12,7 +12,9 @@
 #include "blackcore/application.h"
 #include "blackmisc/network/networkutils.h"
 #include "blackmisc/logmessage.h"
+
 #include <QNetworkReply>
+#include <QFileInfo>
 
 using namespace BlackMisc;
 using namespace BlackMisc::Aviation;
@@ -49,12 +51,12 @@ namespace BlackCore
             return this->getAirports().size();
         }
 
-        bool CAirportDataReader::readFromJsonFilesInBackground(const QString &dir, CEntityFlags::Entity whatToRead)
+        bool CAirportDataReader::readFromJsonFilesInBackground(const QString &dir, CEntityFlags::Entity whatToRead, bool overrideNewerOnly)
         {
             if (dir.isEmpty() || whatToRead == CEntityFlags::NoEntity) { return false; }
-            QTimer::singleShot(0, this, [this, dir, whatToRead]()
+            QTimer::singleShot(0, this, [ = ]()
             {
-                const CStatusMessageList msgs = this->readFromJsonFiles(dir, whatToRead);
+                const CStatusMessageList msgs = this->readFromJsonFiles(dir, whatToRead, overrideNewerOnly);
                 if (msgs.isFailure())
                 {
                     CLogMessage::preformatted(msgs);
@@ -63,13 +65,14 @@ namespace BlackCore
             return true;
         }
 
-        CStatusMessageList CAirportDataReader::readFromJsonFiles(const QString &dir, CEntityFlags::Entity whatToRead)
+        CStatusMessageList CAirportDataReader::readFromJsonFiles(const QString &dir, CEntityFlags::Entity whatToRead, bool overrideNewerOnly)
         {
-            const QString fileName = CFileUtils::appendFilePaths(dir, "airports.json");
-            if (!QFile::exists(fileName))
+            const QDir directory(dir);
+            if (!directory.exists())
             {
-                return CStatusMessage(this).error("File '%1' does not exist") << fileName;
+                return CStatusMessage(this).error("Missing directory '%1'") << dir;
             }
+
             whatToRead &= CEntityFlags::AirportEntity; // can handle these entities
             if (whatToRead == CEntityFlags::NoEntity)
             {
@@ -78,33 +81,40 @@ namespace BlackCore
 
             int c = 0;
             CEntityFlags::Entity reallyRead = CEntityFlags::NoEntity;
-            const QJsonObject airportsJson(CDatabaseUtils::readQJsonObjectFromDatabaseFile(fileName));
-            if (!airportsJson.isEmpty())
-            {
-                try
-                {
-                    CAirportList airports;
-                    airports.convertFromJson(airportsJson);
-                    c = airports.size();
-                    m_airportCache.set(airports);
 
-                    emit dataRead(CEntityFlags::AirportEntity, CEntityFlags::ReadFinished, c);
-                    reallyRead |= CEntityFlags::AirportEntity;
-                }
-                catch (const CJsonException &ex)
-                {
-                    emit dataRead(CEntityFlags::AirportEntity, CEntityFlags::ReadFailed, 0);
-                    return ex.toStatusMessage(this, QString("Reading airports from '%1'").arg(fileName));
-                }
-            }
-            if ((reallyRead & CEntityFlags::AirportEntity) == CEntityFlags::AirportEntity)
+            CStatusMessageList msgs;
+            const QString fileName = CFileUtils::appendFilePaths(dir, "airports.json");
+            const QFileInfo fi(fileName);
+            if (!fi.exists())
             {
-                return CStatusMessage(this).info("Written %1 airports from '%2' to cache") << c << fileName;
+                msgs.push_back(CStatusMessage(this).warning("File '%1' does not exist") << fileName);
+            }
+            else if (!this->overrideCacheFromFile(overrideNewerOnly, fi, CEntityFlags::AirportEntity, msgs))
+            {
+                // void
             }
             else
             {
-                return CStatusMessage(this).error("Not able to read airports from '%1'") << fileName;
+                const QJsonObject airportsJson(CDatabaseUtils::readQJsonObjectFromDatabaseFile(fileName));
+                if (!airportsJson.isEmpty())
+                {
+                    try
+                    {
+                        const CAirportList airports = CAirportList::fromMultipleJsonFormats(airportsJson);
+                        c = airports.size();
+                        msgs.push_back(m_airportCache.set(airports, fi.created().toUTC().toMSecsSinceEpoch()));
+
+                        emit dataRead(CEntityFlags::AirportEntity, CEntityFlags::ReadFinished, c);
+                        reallyRead |= CEntityFlags::AirportEntity;
+                    }
+                    catch (const CJsonException &ex)
+                    {
+                        emit dataRead(CEntityFlags::AirportEntity, CEntityFlags::ReadFailed, 0);
+                        return ex.toStatusMessage(this, QString("Reading airports from '%1'").arg(fileName));
+                    }
+                }
             }
+            return msgs;
         }
 
         CEntityFlags::Entity CAirportDataReader::getSupportedEntities() const
@@ -166,7 +176,7 @@ namespace BlackCore
 
         CUrl CAirportDataReader::getAirportsUrl(CDbFlags::DataRetrievalModeFlag mode) const
         {
-            return getBaseUrl(mode).withAppendedPath(fileNameForMode(CEntityFlags::AirportEntity, mode));
+            return this->getBaseUrl(mode).withAppendedPath(fileNameForMode(CEntityFlags::AirportEntity, mode));
         }
 
         void CAirportDataReader::ps_parseAirportData(QNetworkReply *nwReplyPtr)

src/blackcore/db/airportdatareader.h

@@ -49,8 +49,8 @@ namespace BlackCore
             int getAirportsCount() const;
 
             // data read from local data
-            virtual BlackMisc::CStatusMessageList readFromJsonFiles(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead) override;
-            virtual bool readFromJsonFilesInBackground(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead) override;
+            virtual BlackMisc::CStatusMessageList readFromJsonFiles(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead, bool overrideNewerOnly) override;
+            virtual bool readFromJsonFilesInBackground(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead, bool overrideNewerOnly) override;
 
             // base class overrides
             virtual BlackMisc::Network::CEntityFlags::Entity getSupportedEntities() const override;

src/blackcore/db/databasereader.cpp

@@ -15,6 +15,7 @@
 #include "blackcore/application.h"
 #include "blackmisc/db/datastoreutility.h"
 #include "blackmisc/network/networkutils.h"
+#include "blackmisc/network/entityflags.h"
 #include "blackmisc/directoryutils.h"
 #include "blackmisc/logcategory.h"
 #include "blackmisc/logcategorylist.h"
@@ -23,6 +24,7 @@
 #include <QByteArray>
 #include <QJsonDocument>
 #include <QJsonObject>
+#include <QFileInfo>
 #include <QJsonValueRef>
 #include <QMetaObject>
 #include <QNetworkReply>
@@ -568,6 +570,11 @@ namespace BlackCore
             return m_1stReplyReceived;
         }
 
+        QString CDatabaseReader::getSupportedEntitiesAsString() const
+        {
+            return CEntityFlags::flagToString(this->getSupportedEntities());
+        }
+
         CEntityFlags::Entity CDatabaseReader::maskBySupportedEntities(CEntityFlags::Entity entities) const
         {
             return entities & this->getSupportedEntities();
@@ -590,27 +597,20 @@ namespace BlackCore
             return m_statusMessage;
         }
 
-        CStatusMessageList CDatabaseReader::initFromLocalResourceFiles()
+        CStatusMessageList CDatabaseReader::initFromLocalResourceFiles(bool inBackground)
         {
-            static const QDateTime threshold = QDateTime::currentDateTimeUtc().addYears(-1);
-            const QSet<CEntityFlags::Entity> eSet = CEntityFlags::asSingleEntities(this->getSupportedEntities());
-
-            CStatusMessageList msgs;
-            CEntityFlags::Entity entities = CEntityFlags::NoEntity;
-            for (CEntityFlags::Entity e : eSet)
+            const bool overrideNewerOnly = true;
+            if (inBackground || !CThreadUtils::isCurrentThreadObjectThread(this))
             {
-                if (this->hasCacheTimestampNewerThan(e, threshold))
-                {
-                    entities |= e;
-                }
-                {
-                    msgs.push_back(CStatusMessage(this).info("Will not init from local file, there are already data for '%1'") << CEntityFlags::flagToString(e));
-                }
+                const bool s = this->readFromJsonFilesInBackground(CDirectoryUtils::staticDbFilesDirectory(), this->getSupportedEntities(), overrideNewerOnly);
+                return s ?
+                       CStatusMessage(this).info("Started reading in background from '%1' of entities: '%2'") << CDirectoryUtils::staticDbFilesDirectory() << CEntityFlags::flagToString(this->getSupportedEntities()) :
+                       CStatusMessage(this).error("Starting reading in background from '%1' of entities: '%2' failed") << CDirectoryUtils::staticDbFilesDirectory() << CEntityFlags::flagToString(this->getSupportedEntities());
+            }
+            else
+            {
+                return this->readFromJsonFiles(CDirectoryUtils::staticDbFilesDirectory(), this->getSupportedEntities(), overrideNewerOnly);
             }
-
-            const CStatusMessageList readMsgs = this->readFromJsonFiles(CDirectoryUtils::staticDbFilesDirectory(), entities);
-            msgs.push_back(readMsgs);
-            return readMsgs;
         }
 
         void CDatabaseReader::setReplyStatus(QNetworkReply::NetworkError status, const QString &message)
@@ -631,6 +631,27 @@ namespace BlackCore
             }
         }
 
+        bool CDatabaseReader::overrideCacheFromFile(bool overrideNewerOnly, const QFileInfo &fileInfo, CEntityFlags::Entity entity, CStatusMessageList &msgs) const
+        {
+            if (!fileInfo.created().isValid()) { return false; }
+            if (!overrideNewerOnly) { return true; }
+
+            const qint64 fileTs = fileInfo.created().toUTC().toMSecsSinceEpoch();
+            const QDateTime cacheDateTime(this->getCacheTimestamp(entity));
+            if (!cacheDateTime.isValid()) { return true; } // no cache
+            const qint64 cacheTs = cacheDateTime.toUTC().toMSecsSinceEpoch();
+            if (fileTs > cacheTs)
+            {
+                msgs.push_back(CStatusMessage(this).info("File '%1' is newer than cache (%2)") << fileInfo.absoluteFilePath() << cacheDateTime.toUTC().toString());
+                return true;
+            }
+            else
+            {
+                msgs.push_back(CStatusMessage(this).info("File '%1' is not newer than cache (%2)") << fileInfo.absoluteFilePath() << cacheDateTime.toUTC().toString());
+                return true;
+            }
+        }
+
         QString CDatabaseReader::fileNameForMode(CEntityFlags::Entity entity, CDbFlags::DataRetrievalModeFlag mode)
         {
             Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "needs single entity");

src/blackcore/db/databasereader.h

@@ -32,6 +32,8 @@
 #include <QNetworkReply>
 
 class QNetworkReply;
+class QFileInfo;
+
 namespace BlackMisc { class CLogCategoryList; }
 namespace BlackCore
 {
@@ -186,6 +188,9 @@ namespace BlackCore
             //! Supported entities by this reader
             virtual BlackMisc::Network::CEntityFlags::Entity getSupportedEntities() const = 0;
 
+            //! Supported entities as string
+            QString getSupportedEntitiesAsString() const;
+
             //! Mask by supported entities
             BlackMisc::Network::CEntityFlags::Entity maskBySupportedEntities(BlackMisc::Network::CEntityFlags::Entity entities) const;
 
@@ -259,13 +264,13 @@ namespace BlackCore
 
             //! Init from local resource file
             //! \remark normally used after installation for a 1st time init
-            BlackMisc::CStatusMessageList initFromLocalResourceFiles();
+            BlackMisc::CStatusMessageList initFromLocalResourceFiles(bool inBackground);
 
             //! Data read from local data
-            virtual BlackMisc::CStatusMessageList readFromJsonFiles(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead) = 0;
+            virtual BlackMisc::CStatusMessageList readFromJsonFiles(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead, bool overrideNewer) = 0;
 
             //! Data read from local data
-            virtual bool readFromJsonFilesInBackground(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead) = 0;
+            virtual bool readFromJsonFilesInBackground(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead, bool overrideNewer) = 0;
 
             //! Log categories
             static const BlackMisc::CLogCategoryList &getLogCategories();
@@ -398,6 +403,10 @@ namespace BlackCore
             //! Feedback about connection status
             //! \threadsafe
             void setReplyStatus(QNetworkReply *nwReply);
+
+            //! Override cache from file
+            //! \threadsafe
+            bool overrideCacheFromFile(bool overrideNewerOnly, const QFileInfo &fileInfo, BlackMisc::Network::CEntityFlags::Entity entity, BlackMisc::CStatusMessageList &msgs) const;
         };
     } // ns
 } // ns

src/blackcore/db/databaseutils.cpp

@@ -14,6 +14,7 @@
 #include "blackmisc/fileutils.h"
 
 using namespace BlackMisc;
+using namespace BlackMisc::Json;
 using namespace BlackMisc::Aviation;
 using namespace BlackMisc::Simulation;
 
@@ -356,8 +357,8 @@ namespace BlackCore
                     byteData = qUncompress(ba);
                 }
                 while (false);
-
             }
+
             if (byteData.isEmpty()) { return QJsonDocument(); }
             return QJsonDocument::fromJson(byteData);
         }
@@ -373,7 +374,10 @@ namespace BlackCore
         {
             const QString raw = CFileUtils::readFileToString(filename);
             if (raw.isEmpty()) { return QJsonObject(); }
-            return BlackMisc::Json::jsonObjectFromString(raw);
+
+            // allow also compressed format
+            const QJsonDocument jsonDoc = CDatabaseUtils::databaseJsonToQJsonDocument(raw);
+            return jsonDoc.object();
         }
 
         QJsonObject CDatabaseUtils::readQJsonObjectFromDatabaseFile(const QString &directory, const QString &filename)

src/blackcore/db/icaodatareader.cpp

@@ -19,6 +19,7 @@
 #include <QDateTime>
 #include <QDir>
 #include <QFlags>
+#include <QFileInfo>
 #include <QJsonDocument>
 #include <QNetworkReply>
 #include <QReadLocker>
@@ -395,7 +396,7 @@ namespace BlackCore
             this->emitAndLogDataRead(CEntityFlags::CountryEntity, n, res);
         }
 
-        CStatusMessageList CIcaoDataReader::readFromJsonFiles(const QString &dir, CEntityFlags::Entity whatToRead)
+        CStatusMessageList CIcaoDataReader::readFromJsonFiles(const QString &dir, CEntityFlags::Entity whatToRead, bool overrideNewerOnly)
         {
             const QDir directory(dir);
             if (!directory.exists())
@@ -410,100 +411,126 @@ namespace BlackCore
             if (whatToRead.testFlag(CEntityFlags::CountryEntity))
             {
                 const QString fileName = CFileUtils::appendFilePaths(directory.absolutePath(), "countries.json");
-                const QJsonObject countriesJson(CDatabaseUtils::readQJsonObjectFromDatabaseFile(fileName));
-                if (countriesJson.isEmpty())
+                const QFileInfo fi(fileName);
+                if (!fi.exists())
                 {
-                    msgs.push_back(CStatusMessage(this).error("Failed to read from file/empty file '%1'") << fileName);
+                    msgs.push_back(CStatusMessage(this).warning("File '%1' does not exist") << fileName);
+                }
+                else if (!this->overrideCacheFromFile(overrideNewerOnly, fi, CEntityFlags::CountryEntity, msgs))
+                {
+                    // void
                 }
                 else
                 {
-                    try
+                    const QJsonObject countriesJson(CDatabaseUtils::readQJsonObjectFromDatabaseFile(fileName));
+                    if (countriesJson.isEmpty())
                     {
-                        CCountryList countries;
-                        countries.convertFromJson(countriesJson);
-                        const int c = countries.size();
-                        m_countryCache.set(countries);
-                        reallyRead |= CEntityFlags::CountryEntity;
-                        emit this->dataRead(CEntityFlags::CountryEntity, CEntityFlags::ReadFinished, c);
+                        msgs.push_back(CStatusMessage(this).error("Failed to read from file/empty file '%1'") << fileName);
                     }
-                    catch (const CJsonException &ex)
+                    else
                     {
-                        emit this->dataRead(CEntityFlags::CountryEntity, CEntityFlags::ReadFailed, 0);
-                        msgs.push_back(ex.toStatusMessage(this, QString("Reading countries from '%1'").arg(fileName)));
+                        try
+                        {
+                            const CCountryList countries = CCountryList::fromMultipleJsonFormats(countriesJson);
+                            const int c = countries.size();
+                            msgs.push_back(m_countryCache.set(countries, fi.created().toUTC().toMSecsSinceEpoch()));
+                            reallyRead |= CEntityFlags::CountryEntity;
+                            emit this->dataRead(CEntityFlags::CountryEntity, CEntityFlags::ReadFinished, c);
+                        }
+                        catch (const CJsonException &ex)
+                        {
+                            emit this->dataRead(CEntityFlags::CountryEntity, CEntityFlags::ReadFailed, 0);
+                            msgs.push_back(ex.toStatusMessage(this, QString("Reading countries from '%1'").arg(fileName)));
+                        }
                     }
                 }
-            }
+            } // country
 
             if (whatToRead.testFlag(CEntityFlags::AircraftIcaoEntity))
             {
                 const QString fileName = CFileUtils::appendFilePaths(directory.absolutePath(), "aircrafticao.json");
-                const QJsonObject aircraftJson(CDatabaseUtils::readQJsonObjectFromDatabaseFile(fileName));
-                if (aircraftJson.isEmpty())
+                const QFileInfo fi(fileName);
+                if (!fi.exists())
                 {
-                    msgs.push_back(CStatusMessage(this).error("Failed to read from file/empty file '%1'") << fileName);
+                    msgs.push_back(CStatusMessage(this).warning("File '%1' does not exist") << fileName);
+                }
+                else if (!this->overrideCacheFromFile(overrideNewerOnly, fi, CEntityFlags::AircraftIcaoEntity, msgs))
+                {
+                    // void
                 }
                 else
                 {
-                    try
+                    const QJsonObject aircraftJson(CDatabaseUtils::readQJsonObjectFromDatabaseFile(fileName));
+                    if (aircraftJson.isEmpty())
                     {
-                        CAircraftIcaoCodeList aircraftIcaos;
-                        aircraftIcaos.convertFromJson(aircraftJson);
-                        const int c = aircraftIcaos.size();
-                        m_aircraftIcaoCache.set(aircraftIcaos);
-                        reallyRead |= CEntityFlags::AircraftIcaoEntity;
-                        emit this->dataRead(CEntityFlags::AircraftIcaoEntity, CEntityFlags::ReadFinished, c);
+                        msgs.push_back(CStatusMessage(this).error("Failed to read from file/empty file '%1'") << fileName);
                     }
-                    catch (const CJsonException &ex)
+                    else
                     {
-                        emit this->dataRead(CEntityFlags::AircraftIcaoEntity, CEntityFlags::ReadFailed, 0);
-                        msgs.push_back(ex.toStatusMessage(this, QString("Reading aircraft ICAOs from '%1'").arg(fileName)));
+                        try
+                        {
+                            const CAircraftIcaoCodeList aircraftIcaos = CAircraftIcaoCodeList::fromMultipleJsonFormats(aircraftJson);
+                            const int c = aircraftIcaos.size();
+                            msgs.push_back(m_aircraftIcaoCache.set(aircraftIcaos, fi.created().toUTC().toMSecsSinceEpoch()));
+                            reallyRead |= CEntityFlags::AircraftIcaoEntity;
+                            emit this->dataRead(CEntityFlags::AircraftIcaoEntity, CEntityFlags::ReadFinished, c);
+                        }
+                        catch (const CJsonException &ex)
+                        {
+                            emit this->dataRead(CEntityFlags::AircraftIcaoEntity, CEntityFlags::ReadFailed, 0);
+                            msgs.push_back(ex.toStatusMessage(this, QString("Reading aircraft ICAOs from '%1'").arg(fileName)));
+                        }
                     }
                 }
-            }
+            } // aircraft
 
             if (whatToRead.testFlag(CEntityFlags::AirlineIcaoEntity))
             {
                 const QString fileName = CFileUtils::appendFilePaths(directory.absolutePath(), "airlineicao.json");
-                const QJsonObject airlineJson(CDatabaseUtils::readQJsonObjectFromDatabaseFile(fileName));
-                if (airlineJson.isEmpty())
+                const QFileInfo fi(fileName);
+                if (!fi.exists())
                 {
-                    msgs.push_back(CStatusMessage(this).error("Failed to read from file/empty file '%1'") << fileName);
+                    msgs.push_back(CStatusMessage(this).warning("File '%1' does not exist") << fileName);
+                }
+                else if (!this->overrideCacheFromFile(overrideNewerOnly, fi, CEntityFlags::AirlineIcaoEntity, msgs))
+                {
+                    // void
                 }
                 else
                 {
-                    try
+                    const QJsonObject airlineJson(CDatabaseUtils::readQJsonObjectFromDatabaseFile(fileName));
+                    if (airlineJson.isEmpty())
                     {
-                        CAirlineIcaoCodeList airlineIcaos;
-                        airlineIcaos.convertFromJson(airlineJson);
-                        const int c = airlineIcaos.size();
-                        m_airlineIcaoCache.set(airlineIcaos);
-                        reallyRead |= CEntityFlags::AirlineIcaoEntity;
-                        emit this->dataRead(CEntityFlags::AirlineIcaoEntity, CEntityFlags::ReadFinished, c);
+                        msgs.push_back(CStatusMessage(this).error("Failed to read from file/empty file '%1'") << fileName);
                     }
-                    catch (const CJsonException &ex)
+                    else
                     {
-                        emit this->dataRead(CEntityFlags::AirlineIcaoEntity, CEntityFlags::ReadFailed, 0);
-                        msgs.push_back(ex.toStatusMessage(this, QString("Reading airline ICAOs from '%1'").arg(fileName)));
+                        try
+                        {
+                            const CAirlineIcaoCodeList airlineIcaos = CAirlineIcaoCodeList::fromMultipleJsonFormats(airlineJson);
+                            const int c = airlineIcaos.size();
+                            msgs.push_back(m_airlineIcaoCache.set(airlineIcaos, fi.created().toUTC().toMSecsSinceEpoch()));
+                            reallyRead |= CEntityFlags::AirlineIcaoEntity;
+                            emit this->dataRead(CEntityFlags::AirlineIcaoEntity, CEntityFlags::ReadFinished, c);
+                        }
+                        catch (const CJsonException &ex)
+                        {
+                            emit this->dataRead(CEntityFlags::AirlineIcaoEntity, CEntityFlags::ReadFailed, 0);
+                            msgs.push_back(ex.toStatusMessage(this, QString("Reading airline ICAOs from '%1'").arg(fileName)));
+                        }
                     }
                 }
-            }
+            } // airline
 
-            if (msgs.isSuccess() && (reallyRead & CEntityFlags::DistributorLiveryModel) == whatToRead)
-            {
-                return CStatusMessage(this).info("Updated caches for '%1' from '%2'") << CEntityFlags::flagToString(reallyRead) << dir;
-            }
-            else
-            {
-                return msgs;
-            }
+            return msgs;
         }
 
-        bool CIcaoDataReader::readFromJsonFilesInBackground(const QString &dir, CEntityFlags::Entity whatToRead)
+        bool CIcaoDataReader::readFromJsonFilesInBackground(const QString &dir, CEntityFlags::Entity whatToRead, bool overrideNewerOnly)
         {
             if (dir.isEmpty() || whatToRead == CEntityFlags::NoEntity) { return false; }
-            QTimer::singleShot(0, this, [this, dir, whatToRead]()
+            QTimer::singleShot(0, this, [ = ]()
             {
-                const CStatusMessageList msgs = this->readFromJsonFiles(dir, whatToRead);
+                const CStatusMessageList msgs = this->readFromJsonFiles(dir, whatToRead, overrideNewerOnly);
                 if (msgs.isFailure())
                 {
                     CLogMessage::preformatted(msgs);
@@ -518,22 +545,22 @@ namespace BlackCore
             if (!directory.exists()) { return false; }
             if (this->getCountriesCount() > 0)
             {
-                QString json(QJsonDocument(this->getCountries().toJson()).toJson());
-                bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), "countries.json"));
+                const QString json(QJsonDocument(this->getCountries().toJson()).toJson());
+                const bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), "countries.json"));
                 if (!s) { return false; }
             }
 
             if (this->getAircraftIcaoCodesCount() > 0)
             {
-                QString json(QJsonDocument(this->getAircraftIcaoCodes().toJson()).toJson());
-                bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), "aircrafticao.json"));
+                const QString json(QJsonDocument(this->getAircraftIcaoCodes().toJson()).toJson());
+                const bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), "aircrafticao.json"));
                 if (!s) { return false; }
             }
 
             if (this->getAirlineIcaoCodesCount() > 0)
             {
-                QString json(QJsonDocument(this->getAirlineIcaoCodes().toJson()).toJson());
-                bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), "airlineicao.json"));
+                const QString json(QJsonDocument(this->getAirlineIcaoCodes().toJson()).toJson());
+                const bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), "airlineicao.json"));
                 if (!s) { return false; }
             }
             return true;

src/blackcore/db/icaodatareader.h

@@ -137,8 +137,8 @@ namespace BlackCore
             bool writeToJsonFiles(const QString &dir) const;
 
             // data read from local data
-            virtual BlackMisc::CStatusMessageList readFromJsonFiles(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead) override;
-            virtual bool readFromJsonFilesInBackground(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead) override;
+            virtual BlackMisc::CStatusMessageList readFromJsonFiles(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead, bool overrideNewerOnly) override;
+            virtual bool readFromJsonFilesInBackground(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead, bool overrideNewerOnly) override;
 
             // cache handling for base class
             virtual BlackMisc::Network::CEntityFlags::Entity getSupportedEntities() const override;

src/blackcore/db/infodatareader.cpp

@@ -217,18 +217,21 @@ namespace BlackCore
             return CUrl();
         }
 
-        CStatusMessageList CInfoDataReader::readFromJsonFiles(const QString &dir, CEntityFlags::Entity whatToRead)
+        CStatusMessageList CInfoDataReader::readFromJsonFiles(const QString &dir, CEntityFlags::Entity whatToRead, bool overrideNewer)
         {
             Q_UNUSED(dir);
             Q_UNUSED(whatToRead);
+            Q_UNUSED(overrideNewer);
             Q_ASSERT_X(false, Q_FUNC_INFO, "Not supported");
+
             return CStatusMessage(this).error("Not supported");
         }
 
-        bool CInfoDataReader::readFromJsonFilesInBackground(const QString &dir, CEntityFlags::Entity whatToRead)
+        bool CInfoDataReader::readFromJsonFilesInBackground(const QString &dir, CEntityFlags::Entity whatToRead, bool overrideNewer)
         {
             Q_UNUSED(dir);
             Q_UNUSED(whatToRead);
+            Q_UNUSED(overrideNewer)
             Q_ASSERT_X(false, Q_FUNC_INFO, "Not supported");
             return false;
         }

src/blackcore/db/infodatareader.h

@@ -54,8 +54,8 @@ namespace BlackCore
             BlackMisc::Network::CUrl getInfoObjectsUrl() const;
 
             // data read from local data
-            virtual BlackMisc::CStatusMessageList readFromJsonFiles(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead) override;
-            virtual bool readFromJsonFilesInBackground(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead) override;
+            virtual BlackMisc::CStatusMessageList readFromJsonFiles(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead, bool overrideNewer) override;
+            virtual bool readFromJsonFilesInBackground(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead, bool overrideNewer) override;
 
             // cache handling for base class: no cache handling here in that case
             virtual BlackMisc::Network::CEntityFlags::Entity getSupportedEntities() const override;

src/blackcore/db/modeldatareader.cpp

@@ -18,6 +18,7 @@
 
 #include <QDir>
 #include <QFlags>
+#include <QFileInfo>
 #include <QJsonDocument>
 #include <QNetworkReply>
 #include <QReadLocker>
@@ -158,9 +159,9 @@ namespace BlackCore
         bool CModelDataReader::areAllDataRead() const
         {
             return
-                getLiveriesCount() > 0 &&
-                getModelsCount() > 0 &&
-                getDistributorsCount() > 0;
+                this->getLiveriesCount() > 0 &&
+                this->getModelsCount() > 0 &&
+                this->getDistributorsCount() > 0;
         }
 
         void CModelDataReader::ps_read(CEntityFlags::Entity entities, CDbFlags::DataRetrievalModeFlag mode, const QDateTime &newerThan)
@@ -351,7 +352,7 @@ namespace BlackCore
             if (res.hasErrorMessage())
             {
                 CLogMessage::preformatted(res.lastWarningOrAbove());
-                emit dataRead(CEntityFlags::ModelEntity, CEntityFlags::ReadFailed, 0);
+                emit this->dataRead(CEntityFlags::ModelEntity, CEntityFlags::ReadFailed, 0);
                 return;
             }
 
@@ -384,7 +385,7 @@ namespace BlackCore
             this->emitAndLogDataRead(CEntityFlags::ModelEntity, n, res);
         }
 
-        CStatusMessageList CModelDataReader::readFromJsonFiles(const QString &dir, CEntityFlags::Entity whatToRead)
+        CStatusMessageList CModelDataReader::readFromJsonFiles(const QString &dir, CEntityFlags::Entity whatToRead, bool overrideNewerOnly)
         {
             const QDir directory(dir);
             if (!directory.exists())
@@ -399,26 +400,37 @@ namespace BlackCore
             if (whatToRead.testFlag(CEntityFlags::LiveryEntity))
             {
                 const QString fileName = CFileUtils::appendFilePaths(directory.absolutePath(), "liveries.json");
-                const QJsonObject liveriesJson(CDatabaseUtils::readQJsonObjectFromDatabaseFile(fileName));
-                if (liveriesJson.isEmpty())
+                const QFileInfo fi(fileName);
+                if (!fi.exists())
                 {
-                    msgs.push_back(CStatusMessage(this).error("Failed to read from file/empty file '%1'") << fileName);
+                    msgs.push_back(CStatusMessage(this).warning("File '%1' does not exist") << fileName);
+                }
+                else if (!this->overrideCacheFromFile(overrideNewerOnly, fi, CEntityFlags::LiveryEntity, msgs))
+                {
+                    // void
                 }
                 else
                 {
-                    try
+                    const QJsonObject liveriesJson(CDatabaseUtils::readQJsonObjectFromDatabaseFile(fileName));
+                    if (liveriesJson.isEmpty())
                     {
-                        CLiveryList liveries;
-                        liveries.convertFromJson(liveriesJson);
-                        const int c = liveries.size();
-                        m_liveryCache.set(liveries);
-                        emit dataRead(CEntityFlags::LiveryEntity, CEntityFlags::ReadFinished, c);
-                        reallyRead |= CEntityFlags::LiveryEntity;
+                        msgs.push_back(CStatusMessage(this).error("Failed to read from file/empty file '%1'") << fileName);
                     }
-                    catch (const CJsonException &ex)
+                    else
                     {
-                        emit dataRead(CEntityFlags::LiveryEntity, CEntityFlags::ReadFailed, 0);
-                        msgs.push_back(ex.toStatusMessage(this, QString("Reading liveries from '%1'").arg(fileName)));
+                        try
+                        {
+                            const CLiveryList liveries = CLiveryList::fromMultipleJsonFormats(liveriesJson);
+                            const int c = liveries.size();
+                            msgs.push_back(m_liveryCache.set(liveries, fi.created().toUTC().toMSecsSinceEpoch()));
+                            emit this->dataRead(CEntityFlags::LiveryEntity, CEntityFlags::ReadFinished, c);
+                            reallyRead |= CEntityFlags::LiveryEntity;
+                        }
+                        catch (const CJsonException &ex)
+                        {
+                            emit this->dataRead(CEntityFlags::LiveryEntity, CEntityFlags::ReadFailed, 0);
+                            msgs.push_back(ex.toStatusMessage(this, QString("Reading liveries from '%1'").arg(fileName)));
+                        }
                     }
                 }
             }
@@ -426,26 +438,37 @@ namespace BlackCore
             if (whatToRead.testFlag(CEntityFlags::ModelEntity))
             {
                 const QString fileName = CFileUtils::appendFilePaths(directory.absolutePath(), "models.json");
-                const QJsonObject modelsJson(CDatabaseUtils::readQJsonObjectFromDatabaseFile(fileName));
-                if (modelsJson.isEmpty())
+                const QFileInfo fi(fileName);
+                if (!fi.exists())
                 {
-                    msgs.push_back(CStatusMessage(this).error("Failed to read from file/empty file '%1'") << fileName);
+                    msgs.push_back(CStatusMessage(this).warning("File '%1' does not exist") << fileName);
+                }
+                else if (!this->overrideCacheFromFile(overrideNewerOnly, fi, CEntityFlags::ModelEntity, msgs))
+                {
+                    // void
                 }
                 else
                 {
-                    try
+                    const QJsonObject modelsJson(CDatabaseUtils::readQJsonObjectFromDatabaseFile(fileName));
+                    if (modelsJson.isEmpty())
                     {
-                        CAircraftModelList models;
-                        models.convertFromJson(modelsJson);
-                        const int c = models.size();
-                        m_modelCache.set(models);
-                        emit dataRead(CEntityFlags::ModelEntity, CEntityFlags::ReadFinished, c);
-                        reallyRead |= CEntityFlags::ModelEntity;
+                        msgs.push_back(CStatusMessage(this).error("Failed to read from file/empty file '%1'") << fileName);
                     }
-                    catch (const CJsonException &ex)
+                    else
                     {
-                        emit dataRead(CEntityFlags::ModelEntity, CEntityFlags::ReadFailed, 0);
-                        msgs.push_back(ex.toStatusMessage(this, QString("Reading models from '%1'").arg(fileName)));
+                        try
+                        {
+                            const CAircraftModelList models = CAircraftModelList::fromMultipleJsonFormats(modelsJson);
+                            const int c = models.size();
+                            msgs.push_back(m_modelCache.set(models, fi.created().toUTC().toMSecsSinceEpoch()));
+                            emit this->dataRead(CEntityFlags::ModelEntity, CEntityFlags::ReadFinished, c);
+                            reallyRead |= CEntityFlags::ModelEntity;
+                        }
+                        catch (const CJsonException &ex)
+                        {
+                            emit this->dataRead(CEntityFlags::ModelEntity, CEntityFlags::ReadFailed, 0);
+                            msgs.push_back(ex.toStatusMessage(this, QString("Reading models from '%1'").arg(fileName)));
+                        }
                     }
                 }
             }
@@ -453,46 +476,50 @@ namespace BlackCore
             if (whatToRead.testFlag(CEntityFlags::DistributorEntity))
             {
                 const QString fileName = CFileUtils::appendFilePaths(directory.absolutePath(), "distributors.json");
-                const QJsonObject distributorsJson(CDatabaseUtils::readQJsonObjectFromDatabaseFile(fileName));
-                if (distributorsJson.isEmpty())
+                const QFileInfo fi(fileName);
+                if (!fi.exists())
                 {
-                    msgs.push_back(CStatusMessage(this).error("Failed to read from file/empty file '%1'") << fileName);
+                    msgs.push_back(CStatusMessage(this).warning("File '%1' does not exist") << fileName);
+                }
+                else if (!this->overrideCacheFromFile(overrideNewerOnly, fi, CEntityFlags::DistributorEntity, msgs))
+                {
+                    // void
                 }
                 else
                 {
-                    try
+                    const QJsonObject distributorsJson(CDatabaseUtils::readQJsonObjectFromDatabaseFile(fileName));
+                    if (distributorsJson.isEmpty())
                     {
-                        CDistributorList distributors;
-                        distributors.convertFromJson(distributorsJson);
-                        const int c = distributors.size();
-                        m_distributorCache.set(distributors);
-                        emit dataRead(CEntityFlags::DistributorEntity, CEntityFlags::ReadFinished, c);
-                        reallyRead |= CEntityFlags::DistributorEntity;
+                        msgs.push_back(CStatusMessage(this).error("Failed to read from file/empty file '%1'") << fileName);
                     }
-                    catch (const CJsonException &ex)
+                    else
                     {
-                        emit dataRead(CEntityFlags::DistributorEntity, CEntityFlags::ReadFailed, 0);
-                        msgs.push_back(ex.toStatusMessage(this, QString("Reading distributors from '%1'").arg(fileName)));
+                        try
+                        {
+                            const CDistributorList distributors = CDistributorList::fromMultipleJsonFormats(distributorsJson);
+                            const int c = distributors.size();
+                            msgs.push_back(m_distributorCache.set(distributors, fi.created().toUTC().toMSecsSinceEpoch()));
+                            emit this->dataRead(CEntityFlags::DistributorEntity, CEntityFlags::ReadFinished, c);
+                            reallyRead |= CEntityFlags::DistributorEntity;
+                        }
+                        catch (const CJsonException &ex)
+                        {
+                            emit this->dataRead(CEntityFlags::DistributorEntity, CEntityFlags::ReadFailed, 0);
+                            msgs.push_back(ex.toStatusMessage(this, QString("Reading distributors from '%1'").arg(fileName)));
+                        }
                     }
                 }
             }
 
-            if (msgs.isSuccess() && (reallyRead & CEntityFlags::DistributorLiveryModel) == whatToRead)
-            {
-                return CStatusMessage(this).info("Updated caches for '%1' from '%2'") << CEntityFlags::flagToString(reallyRead) << dir;
-            }
-            else
-            {
-                return msgs;
-            }
+            return msgs;
         }
 
-        bool CModelDataReader::readFromJsonFilesInBackground(const QString &dir, CEntityFlags::Entity whatToRead)
+        bool CModelDataReader::readFromJsonFilesInBackground(const QString &dir, CEntityFlags::Entity whatToRead, bool overrideNewerOnly)
         {
             if (dir.isEmpty() || whatToRead == CEntityFlags::NoEntity) { return false; }
-            QTimer::singleShot(0, this, [this, dir, whatToRead]()
+            QTimer::singleShot(0, this, [ = ]()
             {
-                const CStatusMessageList msgs = this->readFromJsonFiles(dir, whatToRead);
+                const CStatusMessageList msgs = this->readFromJsonFiles(dir, whatToRead, overrideNewerOnly);
                 if (msgs.isFailure())
                 {
                     CLogMessage::preformatted(msgs);

src/blackcore/db/modeldatareader.h

@@ -124,8 +124,8 @@ namespace BlackCore
             bool areAllDataRead() const;
 
             // data read from local data
-            virtual BlackMisc::CStatusMessageList readFromJsonFiles(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead) override;
-            virtual bool readFromJsonFilesInBackground(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead) override;
+            virtual BlackMisc::CStatusMessageList readFromJsonFiles(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead, bool overrideNewerOnly) override;
+            virtual bool readFromJsonFilesInBackground(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead, bool overrideNewerOnly) override;
 
             //! Write to JSON file
             bool writeToJsonFiles(const QString &dir) const;