refs #475, allow to load DB data from disk.

* Will allow to load data if DB is not accessible / available
* Also allows to save, so a special set can be saved for testing/forensic.
* fixed status bar, as received data no longer means DB is alive

src/blackcore/icaodatareader.cpp

@@ -10,9 +10,12 @@
 #include "blackmisc/sequence.h"
 #include "blackmisc/logmessage.h"
 #include "blackmisc/networkutils.h"
+#include "blackmisc/fileutilities.h"
+#include "blackmisc/json.h"
 #include "icaodatareader.h"
-
 #include <QRegularExpression>
+#include <QJsonDocument>
+#include <QJsonObject>
 
 using namespace BlackMisc;
 using namespace BlackMisc::Aviation;
@@ -206,11 +209,10 @@ namespace BlackCore
         CAircraftIcaoCodeList codes = CAircraftIcaoCodeList::fromDatabaseJson(array);
 
         // this part needs to be synchronized
-        int n;
+        int n = codes.size();
         {
             QWriteLocker wl(&this->m_lockAircraft);
             this->m_aircraftIcaos = codes;
-            n = codes.size();
         }
         emit dataRead(CEntityFlags::AircraftIcaoEntity, CEntityFlags::ReadFinished, n);
     }
@@ -227,11 +229,10 @@ namespace BlackCore
         CAirlineIcaoCodeList codes = CAirlineIcaoCodeList::fromDatabaseJson(array);
 
         // this part needs to be synchronized
-        int n;
+        int n = codes.size();
         {
             QWriteLocker wl(&this->m_lockAirline);
             this->m_airlineIcaos = codes;
-            n = codes.size();
         }
         emit dataRead(CEntityFlags::AirlineIcaoEntity, CEntityFlags::ReadFinished, n);
     }
@@ -248,11 +249,10 @@ namespace BlackCore
         CCountryList countries = CCountryList::fromDatabaseJson(array);
 
         // this part needs to be synchronized
-        int n;
+        int n = m_countries.size();
         {
             QWriteLocker wl(&this->m_lockCountry);
             this->m_countries = countries;
-            n = m_countries.size();
         }
         emit dataRead(CEntityFlags::CountryEntity, CEntityFlags::ReadFinished, n);
     }
@@ -266,6 +266,103 @@ namespace BlackCore
         return cm;
     }
 
+    bool CIcaoDataReader::readFromJsonFiles(const QString &dir, CEntityFlags::Entity whatToRead)
+    {
+        QDir directory(dir);
+        if (!directory.exists()) { return false; }
+
+        CEntityFlags::Entity reallyRead = CEntityFlags::NoEntity;
+        if (whatToRead.testFlag(CEntityFlags::CountryEntity))
+        {
+            QString countriesJson(CFileUtils::readFileToString(CFileUtils::appendFilePaths(directory.absolutePath(), "countries.json")));
+            if (!countriesJson.isEmpty())
+            {
+                CCountryList countries;
+                countries.convertFromJson(Json::jsonObjectFromString(countriesJson));
+                int c = countries.size();
+                {
+                    QWriteLocker l(&m_lockCountry);
+                    m_countries = countries;
+                }
+                // Do not emit while locked -> deadlock
+                reallyRead |= CEntityFlags::CountryEntity;
+                emit dataRead(CEntityFlags::CountryEntity, CEntityFlags::ReadFinished, c);
+            }
+        }
+
+        if (whatToRead.testFlag(CEntityFlags::AircraftIcaoEntity))
+        {
+            QString aircraftJson(CFileUtils::readFileToString(CFileUtils::appendFilePaths(directory.absolutePath(), "aircrafticao.json")));
+            if (!aircraftJson.isEmpty())
+            {
+                CAircraftIcaoCodeList aircraftIcaos;
+                aircraftIcaos.convertFromJson(Json::jsonObjectFromString(aircraftJson));
+                int c = aircraftIcaos.size();
+                {
+                    QWriteLocker l(&m_lockAircraft);
+                    m_aircraftIcaos = aircraftIcaos;
+                }
+                reallyRead |= CEntityFlags::AircraftIcaoEntity;
+                emit dataRead(CEntityFlags::AircraftIcaoEntity, CEntityFlags::ReadFinished, c);
+            }
+        }
+
+        if (whatToRead.testFlag(CEntityFlags::AirlineIcaoEntity))
+        {
+            QString airlineJson(CFileUtils::readFileToString(CFileUtils::appendFilePaths(directory.absolutePath(), "airlineicao.json")));
+            if (!airlineJson.isEmpty())
+            {
+                CAirlineIcaoCodeList airlineIcaos;
+                airlineIcaos.convertFromJson(Json::jsonObjectFromString(airlineJson));
+                int c = airlineIcaos.size();
+                {
+                    QWriteLocker l(&m_lockAirline);
+                    m_airlineIcaos = airlineIcaos;
+                }
+                reallyRead |= CEntityFlags::AirlineIcaoEntity;
+                emit dataRead(CEntityFlags::AirlineIcaoEntity, CEntityFlags::ReadFinished, c);
+            }
+        }
+        return (whatToRead & CEntityFlags::AllIcaoAndCountries) == reallyRead;
+    }
+
+    CWorker *CIcaoDataReader::readFromJsonFilesInBackground(const QString &dir, CEntityFlags::Entity whatToRead)
+    {
+        CWorker *worker = BlackMisc::CWorker::fromTask(this, "CIcaoDataReader::readFromJsonFilesInBackground", [this, dir, whatToRead]()
+        {
+            bool s = this->readFromJsonFiles(dir, whatToRead);
+            Q_UNUSED(s);
+        });
+        return worker;
+    }
+
+    bool CIcaoDataReader::writeToJsonFiles(const QString &dir) const
+    {
+        QDir directory(dir);
+        if (!directory.exists()) { return false; }
+        if (this->getCountriesCount() > 0)
+        {
+            QString json(QJsonDocument(this->getCountries().toJson()).toJson());
+            bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), "countries.json"));
+            if (!s) { return false; }
+        }
+
+        if (this->getAircraftIcaoCodesCount() > 0)
+        {
+            QString json(QJsonDocument(this->getAircraftIcaoCodes().toJson()).toJson());
+            bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), "aircrafticao.json"));
+            if (!s) { return false; }
+        }
+
+        if (this->getAirlineIcaoCodesCount() > 0)
+        {
+            QString json(QJsonDocument(this->getAirlineIcaoCodes().toJson()).toJson());
+            bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), "airlineicao.json"));
+            if (!s) { return false; }
+        }
+        return true;
+    }
+
     QString CIcaoDataReader::getAircraftIcaoUrl(const QString &protocol, const QString &server, const QString &baseUrl)
     {
         return CNetworkUtils::buildUrl(protocol, server, baseUrl, "service/jsonaircrafticao.php");

src/blackcore/icaodatareader.h

@@ -101,6 +101,15 @@ namespace BlackCore
         //! \copydoc CDatabaseReader::canConnect()
         using CDatabaseReader::canConnect;
 
+        //! Read from static DB data file
+        bool readFromJsonFiles(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead = BlackMisc::Network::CEntityFlags::AllIcaoAndCountries);
+
+        //! Read from static DB data file
+        BlackMisc::CWorker *readFromJsonFilesInBackground(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead = BlackMisc::Network::CEntityFlags::AllIcaoAndCountries);
+
+        //! Write to static DB data file
+        bool writeToJsonFiles(const QString &dir) const;
+
     signals:
         //! Combined read signal
         void dataRead(BlackMisc::Network::CEntityFlags::Entity entity, BlackMisc::Network::CEntityFlags::ReadState state, int number);

src/blackcore/modeldatareader.cpp

@@ -10,9 +10,12 @@
 #include "blackmisc/sequence.h"
 #include "blackmisc/logmessage.h"
 #include "blackmisc/networkutils.h"
+#include "blackmisc/fileutilities.h"
 #include "modeldatareader.h"
 
 #include <QRegularExpression>
+#include <QJsonDocument>
+#include <QJsonObject>
 
 using namespace BlackMisc;
 using namespace BlackMisc::Aviation;
@@ -210,12 +213,12 @@ namespace BlackCore
         CLiveryList liveries = CLiveryList::fromDatabaseJson(array);
 
         // this part needs to be synchronized
-        int n = 0;
+        int n = liveries.size();
         {
             QWriteLocker wl(&this->m_lockLivery);
             this->m_liveries = liveries;
-            n = liveries.size();
         }
+        // never emit when lcok is held -> deadlock
         emit dataRead(CEntityFlags::LiveryEntity, CEntityFlags::ReadFinished, n);
     }
 
@@ -231,11 +234,10 @@ namespace BlackCore
         CDistributorList distributors = CDistributorList::fromDatabaseJson(array);
 
         // this part needs to be synchronized
-        int n = 0;
+        int n = distributors.size();
         {
             QWriteLocker wl(&this->m_lockDistributor);
             this->m_distributors = distributors;
-            n = distributors.size();
         }
         emit dataRead(CEntityFlags::DistributorEntity, CEntityFlags::ReadFinished, n);
     }
@@ -252,15 +254,112 @@ namespace BlackCore
         CAircraftModelList models = CAircraftModelList::fromDatabaseJson(array);
 
         // this part needs to be synchronized
-        int n = 0;
+        int n = models.size();
         {
             QWriteLocker wl(&this->m_lockModels);
             this->m_models = models;
-            n = models.size();
         }
         emit dataRead(CEntityFlags::ModelEntity, CEntityFlags::ReadFinished, n);
     }
 
+    bool CModelDataReader::readFromJsonFiles(const QString &dir, CEntityFlags::Entity whatToRead)
+    {
+        QDir directory(dir);
+        if (!directory.exists()) { return false; }
+        BlackMisc::Network::CEntityFlags::Entity reallyRead = CEntityFlags::NoEntity;
+
+        if (whatToRead.testFlag(CEntityFlags::LiveryEntity))
+        {
+            QString liveriesJson(CFileUtils::readFileToString(CFileUtils::appendFilePaths(directory.absolutePath(), "liveries.json")));
+            if (!liveriesJson.isEmpty())
+            {
+                CLiveryList liveries;
+                liveries.convertFromJson(Json::jsonObjectFromString(liveriesJson));
+                int c = liveries.size();
+                {
+                    QWriteLocker l(&m_lockLivery);
+                    m_liveries = liveries;
+                }
+                // never emit when lcok is held -> deadlock
+                emit dataRead(CEntityFlags::LiveryEntity, CEntityFlags::ReadFinished, c);
+                reallyRead |= CEntityFlags::LiveryEntity;
+            }
+        }
+
+        if (whatToRead.testFlag(CEntityFlags::ModelEntity))
+        {
+            QString modelsJson(CFileUtils::readFileToString(CFileUtils::appendFilePaths(directory.absolutePath(), "models.json")));
+            if (!modelsJson.isEmpty())
+            {
+                CAircraftModelList models;
+                models.convertFromJson(Json::jsonObjectFromString(modelsJson));
+                int c = models.size();
+                {
+                    QWriteLocker l(&m_lockModels);
+                    m_models = models;
+                }
+                emit dataRead(CEntityFlags::ModelEntity, CEntityFlags::ReadFinished, c);
+                reallyRead |= CEntityFlags::ModelEntity;
+            }
+        }
+
+        if (whatToRead.testFlag(CEntityFlags::DistributorEntity))
+        {
+            QString distributorsJson(CFileUtils::readFileToString(CFileUtils::appendFilePaths(directory.absolutePath(), "distributors.json")));
+            if (!distributorsJson.isEmpty())
+            {
+                CDistributorList distributors;
+                distributors.convertFromJson(Json::jsonObjectFromString(distributorsJson));
+                int c = distributors.size();
+                {
+                    QWriteLocker l(&m_lockDistributor);
+                    m_distributors = distributors;
+                }
+                reallyRead |= CEntityFlags::DistributorEntity;
+                emit dataRead(CEntityFlags::DistributorEntity, CEntityFlags::ReadFinished, c);
+            }
+        }
+
+        return (reallyRead & CEntityFlags::DistributorLiveryModel) == whatToRead;
+    }
+
+    CWorker *CModelDataReader::readFromJsonFilesInBackground(const QString &dir, CEntityFlags::Entity whatToRead)
+    {
+        CWorker *worker = BlackMisc::CWorker::fromTask(this, "CModelDataReader::readFromJsonFilesInBackground", [this, dir, whatToRead]()
+        {
+            bool s = this->readFromJsonFiles(dir, whatToRead);
+            Q_UNUSED(s);
+        });
+        return worker;
+    }
+
+    bool CModelDataReader::writeToJsonFiles(const QString &dir) const
+    {
+        QDir directory(dir);
+        if (!directory.exists()) { return false; }
+        if (this->getLiveriesCount() > 0)
+        {
+            QString json(QJsonDocument(this->getLiveries().toJson()).toJson());
+            bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), "liveries.json"));
+            if (!s) { return false; }
+        }
+
+        if (this->getModelsCount() > 0)
+        {
+            QString json(QJsonDocument(this->getModels().toJson()).toJson());
+            bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), "models.json"));
+            if (!s) { return false; }
+        }
+
+        if (this->getDistributorsCount() > 0)
+        {
+            QString json(QJsonDocument(this->getDistributors().toJson()).toJson());
+            bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), "distributors.json"));
+            if (!s) { return false; }
+        }
+        return true;
+    }
+
     QString CModelDataReader::getLiveryUrl(const QString &protocol, const QString &server, const QString &baseUrl)
     {
         return CNetworkUtils::buildUrl(protocol, server, baseUrl, "service/jsonlivery.php");

src/blackcore/modeldatareader.h

@@ -93,6 +93,15 @@ namespace BlackCore
         //! \copydoc CDatabaseReader::canConnect()
         using CDatabaseReader::canConnect;
 
+        //! Write to JSON file
+        bool readFromJsonFiles(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead = BlackMisc::Network::CEntityFlags::DistributorLiveryModel);
+
+        //! Read from static DB data file
+        BlackMisc::CWorker *readFromJsonFilesInBackground(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead = BlackMisc::Network::CEntityFlags::DistributorLiveryModel);
+
+        //! Write to JSON file
+        bool writeToJsonFiles(const QString &dir) const;
+
     signals:
         //! Combined read signal
         void dataRead(BlackMisc::Network::CEntityFlags::Entity entity, BlackMisc::Network::CEntityFlags::ReadState state, int number);

src/blackcore/webdataservices.cpp

@@ -16,7 +16,11 @@
 #include "blackcore/vatsimmetarreader.h"
 #include "settings/global_reader_settings.h"
 #include "blackmisc/logmessage.h"
+#include "blackmisc/fileutilities.h"
 #include "blackmisc/worker.h"
+#include "blackmisc/json.h"
+#include <QJsonObject>
+#include <QJsonDocument>
 
 using namespace BlackCore;
 using namespace BlackCore::Settings;
@@ -474,6 +478,80 @@ namespace BlackCore
         }
     }
 
+    bool CWebDataServices::writeDbDataToDisk(const QString &dir) const
+    {
+        if (dir.isEmpty()) { return false; }
+        QDir directory(dir);
+        if (!directory.exists())
+        {
+            bool s = directory.mkpath(dir);
+            if (!s) { return false; }
+        }
+
+        if (this->getModelsCount() > 0)
+        {
+            QString json(QJsonDocument(this->getModels().toJson()).toJson());
+            bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), "models.json"));
+            if (!s) { return false; }
+        }
+
+        if (this->getLiveriesCount() > 0)
+        {
+            QString json(QJsonDocument(this->getLiveries().toJson()).toJson());
+            bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), "liveries.json"));
+            if (!s) { return false; }
+        }
+
+        if (m_icaoDataReader)
+        {
+            bool s = m_icaoDataReader->writeToJsonFiles(directory.absolutePath());
+            if (!s) { return false; }
+        }
+        if (m_modelDataReader)
+        {
+            bool s = m_modelDataReader->writeToJsonFiles(directory.absolutePath());
+            if (!s) { return false; }
+        }
+
+        return true;
+    }
+
+    bool CWebDataServices::readDbDataFromDisk(const QString &dir, bool inBackground)
+    {
+        if (dir.isEmpty()) { return false; }
+        QDir directory(dir);
+        if (!directory.exists()) { return false; }
+
+        if (this->m_icaoDataReader)
+        {
+            bool s = false;
+            if (inBackground)
+            {
+                s = (this->m_icaoDataReader->readFromJsonFilesInBackground(dir) != nullptr);
+            }
+            else
+            {
+                s = this->m_icaoDataReader->readFromJsonFiles(dir);
+            }
+            if (!s) { return false; }
+        }
+        if (this->m_modelDataReader)
+        {
+            bool s = false;
+            if (inBackground)
+            {
+                s = (this->m_modelDataReader->readFromJsonFilesInBackground(dir) != nullptr);
+            }
+            else
+            {
+                s = this->m_modelDataReader->readFromJsonFiles(dir);
+            }
+            if (!s) { return false; }
+        }
+
+        return true;
+    }
+
     void CWebDataServices::readAtcBookingsInBackground() const
     {
         if (!this->m_vatsimBookingReader) { return; }

src/blackcore/webdataservices.h

@@ -25,6 +25,7 @@
 #include "blackmisc/weather/metarset.h"
 #include "blackmisc/logcategorylist.h"
 #include "blackmisc/countrylist.h"
+#include "blackmisc/project.h"
 #include <QObject>
 
 namespace BlackCore
@@ -227,6 +228,12 @@ namespace BlackCore
         //! \ingroup webdatareaderprovider
         virtual bool canConnectSwiftDb() const override;
 
+        //! Save all DB data to JSON files
+        virtual bool writeDbDataToDisk(const QString &dir) const override;
+
+        //! Load DB data from JSON files
+        virtual bool readDbDataFromDisk(const QString &dir, bool inBackground) override;
+
     public slots:
         //! First read (allows to immediately read in background)
         void readAllInBackground(int delayMs);