refs #649, refs #656, move swift DB classes to own namespace/directory

src/blackcore/db/databaseauthentication.cpp

@@ -0,0 +1,154 @@
+/* Copyright (C) 2015
+ * swift project Community / Contributors
+ *
+ * This file is part of swift project. It is subject to the license terms in the LICENSE file found in the top-level
+ * directory of this distribution and at http://www.swift-project.org/license.html. No part of swift project,
+ * including this file, may be copied, modified, propagated, or distributed except according to the terms
+ * contained in the LICENSE file.
+ */
+
+#include "blackcore/application.h"
+#include "blackcore/data/globalsetup.h"
+#include "blackcore/db/databaseauthentication.h"
+#include "blackmisc/json.h"
+#include "blackmisc/logcategory.h"
+#include "blackmisc/logcategorylist.h"
+#include "blackmisc/logmessage.h"
+#include "blackmisc/network/authenticateduser.h"
+#include "blackmisc/network/networkutils.h"
+#include "blackmisc/network/rolelist.h"
+#include "blackmisc/network/url.h"
+#include "blackmisc/statusmessage.h"
+
+#include <QByteArray>
+#include <QJsonObject>
+#include <QNetworkReply>
+#include <QNetworkRequest>
+#include <QScopedPointer>
+#include <QScopedPointerDeleteLater>
+#include <QString>
+#include <QUrl>
+#include <QUrlQuery>
+
+using namespace BlackMisc;
+using namespace BlackMisc::Network;
+
+namespace BlackCore
+{
+    namespace Db
+    {
+        CDatabaseAuthenticationService::CDatabaseAuthenticationService(QObject *parent) :
+            QObject(parent)
+        {
+            // void
+        }
+
+        void CDatabaseAuthenticationService::gracefulShutdown()
+        {
+            if (this->m_shutdown) { return; }
+            this->m_shutdown = true;
+            this->logoff();
+        }
+
+        CStatusMessageList CDatabaseAuthenticationService::login(const QString &username, const QString &password)
+        {
+            CStatusMessageList msgs;
+            static const CLogCategoryList cats(CLogCategoryList(this).join({ CLogCategory::validation()}));
+
+            if (this->m_shutdown) { msgs.push_back(CStatusMessage(cats, CStatusMessage::SeverityError, "Shutdown in progress")); return msgs; }
+
+            QString un(username.trimmed());
+            QString pw(password.trimmed());
+            if (un.isEmpty()) { msgs.push_back(CStatusMessage(cats, CStatusMessage::SeverityError, "No user name/id")); }
+            if (pw.isEmpty()) { msgs.push_back(CStatusMessage(cats, CStatusMessage::SeverityError, "No password")); }
+            if (!msgs.isEmpty()) { return msgs; }
+
+            const CUrl url(sApp->getGlobalSetup().getDbLoginServiceUrl());
+            QString msg;
+            if (!CNetworkUtils::canConnect(url, msg))
+            {
+                msgs.push_back(CStatusMessage(cats, CStatusMessage::SeverityError, msg));
+                return msgs;
+            }
+
+            QUrlQuery params;
+            params.addQueryItem("username", un);
+            params.addQueryItem("password", pw);
+            if (sApp->getGlobalSetup().dbDebugFlag()) { CNetworkUtils::addDebugFlag(params); }
+
+            QString query = params.toString();
+            const QNetworkRequest request(CNetworkUtils::getNetworkRequest(url, CNetworkUtils::PostUrlEncoded));
+            sApp->postToNetwork(request, query.toUtf8(), { this, &CDatabaseAuthenticationService::ps_parseServerResponse});
+            QString rm("Sent request to authentication server %1");
+            msgs.push_back(CStatusMessage(cats, CStatusMessage::SeverityInfo, rm.arg(url.toQString())));
+            return msgs;
+        }
+
+        void CDatabaseAuthenticationService::logoff()
+        {
+            CUrl url(sApp->getGlobalSetup().getDbLoginServiceUrl());
+            url.setQuery("logoff=true");
+            QNetworkRequest request(CNetworkUtils::getNetworkRequest(url));
+            sApp->getFromNetwork(request, { this, &CDatabaseAuthenticationService::ps_parseServerResponse });
+            this->m_swiftDbUser.set(CAuthenticatedUser());
+        }
+
+        void CDatabaseAuthenticationService::ps_parseServerResponse(QNetworkReply *nwReplyPtr)
+        {
+            // always cleanup reply
+            QScopedPointer<QNetworkReply, QScopedPointerDeleteLater> nwReply(nwReplyPtr);
+
+            if (this->m_shutdown) { return; }
+            QString urlString(nwReply->url().toString());
+            if (urlString.toLower().contains("logoff"))
+            {
+                sApp->deleteAllCookies();
+                emit logoffFinished();
+                return;
+            }
+
+            if (nwReply->error() == QNetworkReply::NoError)
+            {
+                QString json(nwReply->readAll());
+                if (json.isEmpty())
+                {
+                    CLogMessage(this).error("Authentication failed, no response from %1") << urlString;
+                    return;
+                }
+                QJsonObject jsonObj(Json::jsonObjectFromString(json));
+                CAuthenticatedUser user(CAuthenticatedUser::fromDatabaseJson(jsonObj));
+
+                CStatusMessageList msgs;
+                static const CLogCategoryList cats(CLogCategoryList(this).join({ CLogCategory::validation()}));
+
+                if (!user.isAuthenticated() || !user.isValid())
+                {
+                    msgs.push_back(CStatusMessage(cats, CStatusMessage::SeverityError, "Cannot login, user or password wrong"));
+                }
+                else
+                {
+                    if (!user.isEnabled())
+                    {
+                        msgs.push_back(CStatusMessage(cats, CStatusMessage::SeverityError, "User is disabled"));
+                    }
+                    if (user.getRoles().isEmpty())
+                    {
+                        msgs.push_back(CStatusMessage(cats, CStatusMessage::SeverityError, "User has no roles"));
+                    }
+                }
+                this->m_swiftDbUser.set(user);
+                emit userAuthenticationFinished(user, msgs);
+            }
+            else
+            {
+                CLogMessage(this).error("Authentication failed, %1") << nwReply->errorString();
+                return;
+            }
+        }
+
+        void CDatabaseAuthenticationService::ps_userChanged()
+        {
+            // code goes here
+        }
+    } // ns
+} // ns

src/blackcore/db/databaseauthentication.h

@@ -0,0 +1,70 @@
+/* Copyright (C) 2015
+ * swift project Community / Contributors
+ *
+ * This file is part of swift project. It is subject to the license terms in the LICENSE file found in the top-level
+ * directory of this distribution and at http://www.swift-project.org/license.html. No part of swift project,
+ * including this file, may be copied, modified, propagated, or distributed except according to the terms
+ * contained in the LICENSE file.
+ */
+
+//! \file
+
+#ifndef BLACKCORE_DATABASE_CDATABASEUATHENTICATIONSERVICE_H
+#define BLACKCORE_DATABASE_CDATABASEUATHENTICATIONSERVICE_H
+
+#include "blackcore/blackcoreexport.h"
+#include "blackcore/data/authenticateduser.h"
+#include "blackmisc/datacache.h"
+#include "blackmisc/statusmessagelist.h"
+
+#include <QObject>
+#include <QString>
+
+class QNetworkReply;
+namespace BlackMisc { namespace Network { class CAuthenticatedUser; } }
+
+namespace BlackCore
+{
+    namespace Db
+    {
+        //! Databse user used with swift DB. Features role and cookie handling.
+        class BLACKCORE_EXPORT  CDatabaseAuthenticationService: public QObject
+        {
+            Q_OBJECT
+
+        public:
+            //! Constructor
+            CDatabaseAuthenticationService(QObject *parent = nullptr);
+
+            //! Shutdown
+            void gracefulShutdown();
+
+        public slots:
+            //! Try to login to authentication web service
+            BlackMisc::CStatusMessageList login(const QString &id, const QString &password);
+
+            //! Logoff
+            void logoff();
+
+        signals:
+            //! User authenticated
+            void userAuthenticationFinished(const BlackMisc::Network::CAuthenticatedUser &user, const BlackMisc::CStatusMessageList &loginStatus);
+
+            //! Logoff completed
+            void logoffFinished();
+
+        private slots:
+            //! Parse login answer
+            void ps_parseServerResponse(QNetworkReply *nwReplyPtr);
+
+            //! User object changed
+            void ps_userChanged();
+
+        private:
+            BlackMisc::CData<BlackCore::Data::AuthenticatedDbUser> m_swiftDbUser {this, &CDatabaseAuthenticationService::ps_userChanged};
+            bool m_shutdown = false;
+        };
+    } // ns
+} // ns
+
+#endif // guard

src/blackcore/db/databasewriter.cpp

@@ -0,0 +1,143 @@
+/* Copyright (C) 2015
+ * swift project Community / Contributors
+ *
+ * This file is part of swift project. It is subject to the license terms in the LICENSE file found in the top-level
+ * directory of this distribution and at http://www.swift-project.org/license.html. No part of swift project,
+ * including this file, may be copied, modified, propagated, or distributed except according to the terms
+ * contained in the LICENSE file.
+ */
+
+#include "blackcore/application.h"
+#include "blackcore/data/globalsetup.h"
+#include "blackcore/db/databasewriter.h"
+#include "blackmisc/db/datastoreutility.h"
+#include "blackmisc/logcategory.h"
+#include "blackmisc/logcategorylist.h"
+#include "blackmisc/network/networkutils.h"
+#include "blackmisc/statusmessage.h"
+
+#include <QHttpMultiPart>
+#include <QNetworkReply>
+#include <QNetworkRequest>
+#include <QScopedPointer>
+#include <QScopedPointerDeleteLater>
+#include <QString>
+#include <QUrl>
+#include <QtGlobal>
+
+using namespace BlackMisc;
+using namespace BlackMisc::Network;
+using namespace BlackMisc::Simulation;
+
+namespace BlackCore
+{
+    namespace Db
+    {
+        CDatabaseWriter::CDatabaseWriter(const Network::CUrl &baseUrl, QObject *parent) :
+            QObject(parent),
+            m_modelPublishUrl(getModelPublishUrl(baseUrl))
+        {
+            // void
+        }
+
+        CStatusMessageList CDatabaseWriter::asyncPublishModels(const CAircraftModelList &models)
+        {
+            CStatusMessageList msgs;
+            if (m_shutdown)
+            {
+                msgs.push_back(CStatusMessage(CStatusMessage::SeverityWarning, "Database writer shuts down"));
+                return msgs;
+            }
+
+            if (m_pendingReply)
+            {
+                msgs.push_back(CStatusMessage(CStatusMessage::SeverityWarning, "Another write operation in progress"));
+                return msgs;
+            }
+
+            QUrl url(m_modelPublishUrl.toQUrl());
+            QNetworkRequest request(url);
+            CNetworkUtils::ignoreSslVerification(request);
+            QHttpMultiPart *multiPart = new QHttpMultiPart(QHttpMultiPart::FormDataType, this);
+            multiPart->append(CNetworkUtils::getJsonTextMultipart(models.toDatabaseJson()));
+            if (sApp->getGlobalSetup().dbDebugFlag())
+            {
+                multiPart->append(CNetworkUtils::getMultipartWithDebugFlag());
+            }
+
+            m_pendingReply = sApp->postToNetwork(request, multiPart, { this, &CDatabaseWriter::ps_postResponse});
+            multiPart->setParent(m_pendingReply);
+            return msgs;
+        }
+
+        void CDatabaseWriter::gracefulShutdown()
+        {
+            m_shutdown = true;
+            if (m_pendingReply)
+            {
+                m_pendingReply->abort();
+                m_pendingReply = nullptr;
+            }
+        }
+
+        void CDatabaseWriter::ps_postResponse(QNetworkReply *nwReplyPtr)
+        {
+            static const CLogCategoryList cats(CLogCategoryList(this).join({ CLogCategory::swiftDbWebservice()}));
+            QScopedPointer<QNetworkReply, QScopedPointerDeleteLater> nwReply(nwReplyPtr);
+            m_pendingReply = nullptr;
+            QUrl url(nwReply->url());
+            QString urlString(url.toString());
+
+            if (m_shutdown)
+            {
+                nwReply->abort();
+                return;
+            }
+
+            if (nwReply->error() == QNetworkReply::NoError)
+            {
+                const QString dataFileData(nwReply->readAll().trimmed());
+                nwReply->close(); // close asap
+                if (dataFileData.isEmpty())
+                {
+                    const CStatusMessageList msgs({CStatusMessage(cats, CStatusMessage::SeverityError, "No response data from " + urlString)});
+                    emit published(CAircraftModelList(), CAircraftModelList(), msgs);
+                    return;
+                }
+
+                CAircraftModelList modelsPublished;
+                CAircraftModelList modelsSkipped;
+                CStatusMessageList msgs;
+                bool success = CDatastoreUtility::parseSwiftPublishResponse(dataFileData, modelsPublished, modelsSkipped, msgs);
+                emit published(modelsPublished, modelsSkipped, msgs);
+                Q_UNUSED(success);
+            }
+            else
+            {
+                QString error = nwReply->errorString();
+                nwReply->close(); // close asap
+                const CStatusMessageList msgs( {CStatusMessage(cats, CStatusMessage::SeverityError, "HTTP error: " + error)});
+                emit published(CAircraftModelList(), CAircraftModelList(), msgs);
+            }
+        }
+
+        CUrl CDatabaseWriter::getModelPublishUrl(const Network::CUrl &baseUrl)
+        {
+            return baseUrl.withAppendedPath("service/publishmodels.php");
+        }
+
+        QList<QByteArray> CDatabaseWriter::splitData(const QByteArray &data, int size)
+        {
+            if (data.size() <= size) { return QList<QByteArray>({data}); }
+            int pos = 0, arrsize = data.size();
+            QList<QByteArray> arrays;
+            while (pos < arrsize)
+            {
+                QByteArray arr = data.mid(pos, size);
+                arrays << arr;
+                pos += arr.size();
+            }
+            return arrays;
+        }
+    } // ns
+} // ns

src/blackcore/db/databasewriter.h

@@ -0,0 +1,67 @@
+/* Copyright (C) 2015
+ * swift project Community / Contributors
+ *
+ * This file is part of swift project. It is subject to the license terms in the LICENSE file found in the top-level
+ * directory of this distribution and at http://www.swift-project.org/license.html. No part of swift project,
+ * including this file, may be copied, modified, propagated, or distributed except according to the terms
+ * contained in the LICENSE file.
+ */
+
+//! \file
+
+#ifndef BLACKCORE_DATABASE_WRITER_H
+#define BLACKCORE_DATABASE_WRITER_H
+
+#include "blackcore/blackcoreexport.h"
+#include "blackmisc/network/url.h"
+#include "blackmisc/simulation/aircraftmodellist.h"
+#include "blackmisc/statusmessagelist.h"
+
+#include <QByteArray>
+#include <QList>
+#include <QObject>
+
+class QNetworkReply;
+
+namespace BlackCore
+{
+    namespace Db
+    {
+        //! Write to the swift DB
+        class BLACKCORE_EXPORT CDatabaseWriter : public QObject
+        {
+            Q_OBJECT
+
+        public:
+            //! Constructor
+            CDatabaseWriter(const BlackMisc::Network::CUrl &baseUrl, QObject *parent);
+
+            //! Write model to DB
+            BlackMisc::CStatusMessageList asyncPublishModels(const BlackMisc::Simulation::CAircraftModelList &models);
+
+            //! Shutdown
+            void gracefulShutdown();
+
+        signals:
+            //! Published models, the response to \sa asyncPublishModels
+            void published(const BlackMisc::Simulation::CAircraftModelList &modelsPublished, const BlackMisc::Simulation::CAircraftModelList &modelsSkipped, const BlackMisc::CStatusMessageList &messages);
+
+        private slots:
+            //! Post response
+            void ps_postResponse(QNetworkReply *nwReplyPtr);
+
+        private:
+            BlackMisc::Network::CUrl                       m_modelPublishUrl;
+            QNetworkReply                                 *m_pendingReply = nullptr;
+            bool                                           m_shutdown = false;
+
+            //! URL model web service
+            static BlackMisc::Network::CUrl getModelPublishUrl(const BlackMisc::Network::CUrl &baseUrl);
+
+            //! Split data array
+            static QList<QByteArray> splitData(const QByteArray &data, int size);
+        };
+    } // ns
+} // ns
+
+#endif // guard

src/blackcore/db/modeldatareader.cpp

@@ -0,0 +1,473 @@
+/* Copyright (C) 2015
+ * swift project Community / Contributors
+ *
+ * This file is part of swift project. It is subject to the license terms in the LICENSE file found in the top-level
+ * directory of this distribution and at http://www.swift-project.org/license.html. No part of swift project,
+ * including this file, may be copied, modified, propagated, or distributed except according to the terms
+ * contained in the LICENSE file.
+ */
+
+#include "blackcore/application.h"
+#include "blackcore/data/globalsetup.h"
+#include "blackcore/db/modeldatareader.h"
+#include "blackmisc/fileutils.h"
+#include "blackmisc/json.h"
+#include "blackmisc/logmessage.h"
+#include "blackmisc/statusmessage.h"
+
+#include <QDir>
+#include <QFlags>
+#include <QJsonDocument>
+#include <QNetworkReply>
+#include <QReadLocker>
+#include <QScopedPointer>
+#include <QScopedPointerDeleteLater>
+#include <QTimer>
+#include <QUrl>
+#include <QWriteLocker>
+#include <Qt>
+#include <QtGlobal>
+
+using namespace BlackMisc;
+using namespace BlackMisc::Aviation;
+using namespace BlackMisc::Simulation;
+using namespace BlackMisc::Network;
+using namespace BlackCore::Data;
+
+namespace BlackCore
+{
+    namespace Db
+    {
+        CModelDataReader::CModelDataReader(QObject *owner, const CDatabaseReaderConfigList &config) :
+            CDatabaseReader(owner, config, "CModelDataReader")
+        {
+            // void
+        }
+
+        CLiveryList CModelDataReader::getLiveries() const
+        {
+            QReadLocker l(&m_lockLivery);
+            return m_liveries;
+        }
+
+        CLivery CModelDataReader::getLiveryForCombinedCode(const QString &combinedCode) const
+        {
+            if (!CLivery::isValidCombinedCode(combinedCode)) { return CLivery(); }
+            CLiveryList liveries(getLiveries());
+            return liveries.findByCombinedCode(combinedCode);
+        }
+
+        CLivery CModelDataReader::getStdLiveryForAirlineCode(const CAirlineIcaoCode &icao) const
+        {
+            if (!icao.hasValidDesignator()) { return CLivery(); }
+            CLiveryList liveries(getLiveries());
+            return liveries.findStdLiveryByAirlineIcaoDesignator(icao);
+        }
+
+        CLivery CModelDataReader::getLiveryForDbKey(int id) const
+        {
+            if (id < 0) { return CLivery(); }
+            CLiveryList liveries(getLiveries());
+            return liveries.findByKey(id);
+        }
+
+        CLivery CModelDataReader::smartLiverySelector(const CLivery &liveryPattern) const
+        {
+            CLiveryList liveries(getLiveries()); // thread safe copy
+            return liveries.smartLiverySelector(liveryPattern);
+        }
+
+        CDistributorList CModelDataReader::getDistributors() const
+        {
+            QReadLocker l(&m_lockDistributor);
+            return m_distributors;
+        }
+
+        CAircraftModelList CModelDataReader::getModels() const
+        {
+            QReadLocker l(&m_lockModels);
+            return m_models;
+        }
+
+        CAircraftModel CModelDataReader::getModelForModelString(const QString &modelString) const
+        {
+            if (modelString.isEmpty()) { return CAircraftModel(); }
+            CAircraftModelList models(getModels());
+            return models.findFirstByModelStringOrDefault(modelString);
+        }
+
+        CAircraftModelList CModelDataReader::getModelsForAircraftDesignatorAndLiveryCombinedCode(const QString &aircraftDesignator, const QString &combinedCode)
+        {
+            if (aircraftDesignator.isEmpty()) { return CAircraftModelList(); }
+            CAircraftModelList models(getModels());
+            return models.findByAircraftDesignatorAndLiveryCombinedCode(aircraftDesignator, combinedCode);
+        }
+
+        int CModelDataReader::getLiveriesCount() const
+        {
+            QReadLocker l(&m_lockLivery);
+            return m_liveries.size();
+        }
+
+        int CModelDataReader::getDistributorsCount() const
+        {
+            QReadLocker l(&m_lockDistributor);
+            return m_distributors.size();
+        }
+
+        CDistributor CModelDataReader::smartDistributorSelector(const CDistributor &distributorPattern) const
+        {
+            CDistributorList distributors(getDistributors()); // thread safe copy
+            return distributors.smartDistributorSelector(distributorPattern);
+        }
+
+        int CModelDataReader::getModelsCount() const
+        {
+            QReadLocker l(&m_lockModels);
+            return m_models.size();
+        }
+
+        QList<int> CModelDataReader::getModelDbKeys() const
+        {
+            QReadLocker l(&m_lockModels);
+            return m_models.toDbKeyList();
+        }
+
+        QStringList CModelDataReader::getModelStrings() const
+        {
+            QReadLocker l(&m_lockModels);
+            return m_models.getModelStrings(false);
+        }
+
+        bool CModelDataReader::areAllDataRead() const
+        {
+            return
+                getLiveriesCount() > 0 &&
+                getModelsCount() > 0 &&
+                getDistributorsCount() > 0;
+        }
+
+        void CModelDataReader::ps_read(CEntityFlags::Entity entity, const QDateTime &newerThan)
+        {
+            this->threadAssertCheck();
+
+            CEntityFlags::Entity triggeredRead = CEntityFlags::NoEntity;
+            if (entity.testFlag(CEntityFlags::LiveryEntity))
+            {
+                CUrl url(getLiveryUrl());
+                if (!url.isEmpty())
+                {
+                    if (!newerThan.isNull())
+                    {
+                        const QString tss(newerThan.toString(Qt::ISODate));
+                        url.appendQuery(QString(parameterLatestTimestamp() + "=" + tss));
+                    }
+                    sApp->getFromNetwork(url, { this, &CModelDataReader::ps_parseLiveryData});
+                    triggeredRead |= CEntityFlags::LiveryEntity;
+                }
+                else
+                {
+                    CLogMessage(this).error("No URL for %1") << CEntityFlags::flagToString(CEntityFlags::LiveryEntity);
+                }
+            }
+
+            if (entity.testFlag(CEntityFlags::DistributorEntity))
+            {
+                CUrl url(getDistributorUrl());
+                if (!url.isEmpty())
+                {
+                    if (!newerThan.isNull())
+                    {
+                        const QString tss(newerThan.toString(Qt::ISODate));
+                        url.appendQuery(QString(parameterLatestTimestamp() + "=" + tss));
+                    }
+                    sApp->getFromNetwork(url, { this, &CModelDataReader::ps_parseDistributorData});
+                    triggeredRead |= CEntityFlags::DistributorEntity;
+                }
+                else
+                {
+                    CLogMessage(this).error("No URL for %1") << CEntityFlags::flagToString(CEntityFlags::DistributorEntity);
+                }
+            }
+
+            if (entity.testFlag(CEntityFlags::ModelEntity))
+            {
+                CUrl url(getModelUrl());
+                if (!url.isEmpty())
+                {
+                    if (!newerThan.isNull())
+                    {
+                        const QString tss(newerThan.toString(Qt::ISODate));
+                        url.appendQuery(QString(parameterLatestTimestamp() + "=" + tss));
+                    }
+                    sApp->getFromNetwork(url, { this, &CModelDataReader::ps_parseModelData});
+                    triggeredRead |= CEntityFlags::ModelEntity;
+                }
+                else
+                {
+                    CLogMessage(this).error("No URL for %1") << CEntityFlags::flagToString(CEntityFlags::ModelEntity);
+                }
+            }
+
+            if (triggeredRead != CEntityFlags::NoEntity)
+            {
+                emit dataRead(triggeredRead, CEntityFlags::StartRead, 0);
+            }
+        }
+
+        void CModelDataReader::ps_parseLiveryData(QNetworkReply *nwReplyPtr)
+        {
+            // wrap pointer, make sure any exit cleans up reply
+            // required to use delete later as object is created in a different thread
+            QScopedPointer<QNetworkReply, QScopedPointerDeleteLater> nwReply(nwReplyPtr);
+            QString urlString(nwReply->url().toString());
+            CDatabaseReader::JsonDatastoreResponse res = this->setStatusAndTransformReplyIntoDatastoreResponse(nwReply.data());
+            if (res.hasErrorMessage())
+            {
+                CLogMessage::preformatted(res.lastWarningOrAbove());
+                emit dataRead(CEntityFlags::LiveryEntity, CEntityFlags::ReadFailed, 0);
+                return;
+            }
+
+            // get all or incremental set of distributor
+            CLiveryList liveries;
+            if (res.isRestricted())
+            {
+                // create full list if it was just incremental
+                liveries = this->getLiveries();
+                liveries.replaceOrAddObjectsByKey(CLiveryList::fromDatabaseJson(res));
+            }
+            else
+            {
+                liveries  = CLiveryList::fromDatabaseJson(res);
+            }
+
+            // this part needs to be synchronized
+            int n = liveries.size();
+            {
+                QWriteLocker wl(&this->m_lockLivery);
+                this->m_liveries = liveries;
+            }
+
+            // never emit when lock is held -> deadlock
+            emit dataRead(CEntityFlags::LiveryEntity,
+                          res.isRestricted() ? CEntityFlags::ReadFinishedRestricted : CEntityFlags::ReadFinished, n);
+            CLogMessage(this).info("Read %1 %2 from %3") << n << CEntityFlags::flagToString(CEntityFlags::LiveryEntity) << urlString;
+        }
+
+        void CModelDataReader::ps_parseDistributorData(QNetworkReply *nwReplyPtr)
+        {
+            QScopedPointer<QNetworkReply, QScopedPointerDeleteLater> nwReply(nwReplyPtr);
+            QString urlString(nwReply->url().toString());
+            CDatabaseReader::JsonDatastoreResponse res = this->setStatusAndTransformReplyIntoDatastoreResponse(nwReply.data());
+            if (res.hasErrorMessage())
+            {
+                CLogMessage::preformatted(res.lastWarningOrAbove());
+                emit dataRead(CEntityFlags::DistributorEntity, CEntityFlags::ReadFailed, 0);
+                return;
+            }
+
+            // get all or incremental set of distributor
+            CDistributorList distributors;
+            if (res.isRestricted())
+            {
+                // create full list if it was just incremental
+                distributors = this->getDistributors();
+                distributors.replaceOrAddObjectsByKey(CDistributorList::fromDatabaseJson(res));
+            }
+            else
+            {
+                distributors = CDistributorList::fromDatabaseJson(res);
+            }
+
+            // this part needs to be synchronized
+            int n = distributors.size();
+            {
+                QWriteLocker wl(&this->m_lockDistributor);
+                this->m_distributors = distributors;
+            }
+            emit dataRead(CEntityFlags::DistributorEntity,
+                          res.isRestricted() ? CEntityFlags::ReadFinishedRestricted : CEntityFlags::ReadFinished, n);
+            CLogMessage(this).info("Read %1 %2 from %3") << n << CEntityFlags::flagToString(CEntityFlags::DistributorEntity) << urlString;
+        }
+
+        void CModelDataReader::ps_parseModelData(QNetworkReply *nwReplyPtr)
+        {
+            QScopedPointer<QNetworkReply, QScopedPointerDeleteLater> nwReply(nwReplyPtr);
+            QString urlString(nwReply->url().toString());
+            CDatabaseReader::JsonDatastoreResponse res = this->setStatusAndTransformReplyIntoDatastoreResponse(nwReply.data());
+            if (res.hasErrorMessage())
+            {
+                CLogMessage::preformatted(res.lastWarningOrAbove());
+                emit dataRead(CEntityFlags::ModelEntity, CEntityFlags::ReadFailed, 0);
+                return;
+            }
+
+            // get all or incremental set of models
+            CAircraftModelList models;
+            if (res.isRestricted())
+            {
+                // create full list if it was just incremental
+                models = this->getModels();
+                models.replaceOrAddObjectsByKey(CAircraftModelList::fromDatabaseJson(res));
+            }
+            else
+            {
+                models = CAircraftModelList::fromDatabaseJson(res);
+            }
+
+            // syncronized update
+            int n = models.size();
+            {
+                QWriteLocker wl(&this->m_lockModels);
+                this->m_models = models;
+            }
+
+            emit dataRead(CEntityFlags::ModelEntity,
+                          res.isRestricted() ? CEntityFlags::ReadFinishedRestricted : CEntityFlags::ReadFinished, n);
+            CLogMessage(this).info("Read %1 %2 from %3") << n << CEntityFlags::flagToString(CEntityFlags::ModelEntity) << urlString;
+        }
+
+        bool CModelDataReader::readFromJsonFiles(const QString &dir, CEntityFlags::Entity whatToRead)
+        {
+            QDir directory(dir);
+            if (!directory.exists()) { return false; }
+            BlackMisc::Network::CEntityFlags::Entity reallyRead = CEntityFlags::NoEntity;
+
+            if (whatToRead.testFlag(CEntityFlags::LiveryEntity))
+            {
+                QString liveriesJson(CFileUtils::readFileToString(CFileUtils::appendFilePaths(directory.absolutePath(), "liveries.json")));
+                if (!liveriesJson.isEmpty())
+                {
+                    CLiveryList liveries;
+                    liveries.convertFromJson(liveriesJson);
+                    int c = liveries.size();
+                    {
+                        QWriteLocker l(&m_lockLivery);
+                        m_liveries = liveries;
+                    }
+                    // never emit when lcok is held -> deadlock
+                    emit dataRead(CEntityFlags::LiveryEntity, CEntityFlags::ReadFinished, c);
+                    reallyRead |= CEntityFlags::LiveryEntity;
+                }
+            }
+
+            if (whatToRead.testFlag(CEntityFlags::ModelEntity))
+            {
+                QString modelsJson(CFileUtils::readFileToString(CFileUtils::appendFilePaths(directory.absolutePath(), "models.json")));
+                if (!modelsJson.isEmpty())
+                {
+                    CAircraftModelList models;
+                    models.convertFromJson(Json::jsonObjectFromString(modelsJson));
+                    int c = models.size();
+                    {
+                        QWriteLocker l(&m_lockModels);
+                        m_models = models;
+                    }
+                    emit dataRead(CEntityFlags::ModelEntity, CEntityFlags::ReadFinished, c);
+                    reallyRead |= CEntityFlags::ModelEntity;
+                }
+            }
+
+            if (whatToRead.testFlag(CEntityFlags::DistributorEntity))
+            {
+                QString distributorsJson(CFileUtils::readFileToString(CFileUtils::appendFilePaths(directory.absolutePath(), "distributors.json")));
+                if (!distributorsJson.isEmpty())
+                {
+                    CDistributorList distributors;
+                    distributors.convertFromJson(Json::jsonObjectFromString(distributorsJson));
+                    int c = distributors.size();
+                    {
+                        QWriteLocker l(&m_lockDistributor);
+                        m_distributors = distributors;
+                    }
+                    reallyRead |= CEntityFlags::DistributorEntity;
+                    emit dataRead(CEntityFlags::DistributorEntity, CEntityFlags::ReadFinished, c);
+                }
+            }
+
+            return (reallyRead & CEntityFlags::DistributorLiveryModel) == whatToRead;
+        }
+
+        bool CModelDataReader::readFromJsonFilesInBackground(const QString &dir, CEntityFlags::Entity whatToRead)
+        {
+            if (dir.isEmpty() || whatToRead == CEntityFlags::NoEntity) { return false; }
+            QTimer::singleShot(0, this, [this, dir, whatToRead]()
+            {
+                bool s = this->readFromJsonFiles(dir, whatToRead);
+                Q_UNUSED(s);
+            });
+            return true;
+        }
+
+        bool CModelDataReader::writeToJsonFiles(const QString &dir) const
+        {
+            QDir directory(dir);
+            if (!directory.exists()) { return false; }
+            if (this->getLiveriesCount() > 0)
+            {
+                QString json(QJsonDocument(this->getLiveries().toJson()).toJson());
+                bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), "liveries.json"));
+                if (!s) { return false; }
+            }
+
+            if (this->getModelsCount() > 0)
+            {
+                QString json(QJsonDocument(this->getModels().toJson()).toJson());
+                bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), "models.json"));
+                if (!s) { return false; }
+            }
+
+            if (this->getDistributorsCount() > 0)
+            {
+                QString json(QJsonDocument(this->getDistributors().toJson()).toJson());
+                bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), "distributors.json"));
+                if (!s) { return false; }
+            }
+            return true;
+        }
+
+        void CModelDataReader::syncronizeCaches(CEntityFlags::Entity entities)
+        {
+            Q_UNUSED(entities);
+        }
+
+        void CModelDataReader::invalidateCaches(CEntityFlags::Entity entities)
+        {
+            Q_UNUSED(entities);
+        }
+
+        QDateTime CModelDataReader::getCacheTimestamp(CEntityFlags::Entity entity)
+        {
+            Q_UNUSED(entity);
+            return QDateTime();
+        }
+
+        CUrl CModelDataReader::getBaseUrl() const
+        {
+            const CUrl baseUrl(sApp->getGlobalSetup().getDbModelReaderUrl());
+            return baseUrl;
+        }
+
+        CUrl CModelDataReader::getLiveryUrl(bool shared) const
+        {
+            return shared ?
+                   getBaseUrl().withAppendedPath("service/jsonlivery.php") :
+                   getBaseUrl().withAppendedPath("service/jsonlivery.php");
+        }
+
+        CUrl CModelDataReader::getDistributorUrl(bool shared) const
+        {
+            return shared ?
+                   getBaseUrl().withAppendedPath("service/jsondistributor.php") :
+                   getBaseUrl().withAppendedPath("service/jsondistributor.php");
+        }
+
+        CUrl CModelDataReader::getModelUrl(bool shared) const
+        {
+            return shared ?
+                   getBaseUrl().withAppendedPath("service/jsonaircraftmodel.php") :
+                   getBaseUrl().withAppendedPath("service/jsonaircraftmodel.php");
+        }
+    } // ns
+} // ns

src/blackcore/db/modeldatareader.h

@@ -0,0 +1,174 @@
+/* Copyright (C) 2015
+ * swift project Community / Contributors
+ *
+ * This file is part of swift project. It is subject to the license terms in the LICENSE file found in the top-level
+ * directory of this distribution and at http://www.swift-project.org/license.html. No part of swift project,
+ * including this file, may be copied, modified, propagated, or distributed except according to the terms
+ * contained in the LICENSE file.
+ */
+
+//! \file
+
+#ifndef BLACKCORE_MODELDATAREADER_H
+#define BLACKCORE_MODELDATAREADER_H
+
+#include "blackcore/blackcoreexport.h"
+#include "blackcore/db/databasereader.h"
+#include "blackmisc/aviation/airlineicaocode.h"
+#include "blackmisc/aviation/livery.h"
+#include "blackmisc/aviation/liverylist.h"
+#include "blackmisc/network/entityflags.h"
+#include "blackmisc/network/url.h"
+#include "blackmisc/simulation/aircraftmodel.h"
+#include "blackmisc/simulation/aircraftmodellist.h"
+#include "blackmisc/simulation/distributor.h"
+#include "blackmisc/simulation/distributorlist.h"
+
+#include <QDateTime>
+#include <QList>
+#include <QObject>
+#include <QReadWriteLock>
+#include <QString>
+#include <QStringList>
+
+class QNetworkReply;
+
+namespace BlackCore
+{
+    namespace Db
+    {
+        //! Read model related data from Database
+        class BLACKCORE_EXPORT CModelDataReader : public CDatabaseReader
+        {
+            Q_OBJECT
+
+        public:
+            //! Constructor
+            explicit CModelDataReader(QObject *owner, const CDatabaseReaderConfigList &config);
+
+            //! Get aircraft liveries
+            //! \threadsafe
+            BlackMisc::Aviation::CLiveryList getLiveries() const;
+
+            //! Get aircraft livery for code
+            //! \threadsafe
+            BlackMisc::Aviation::CLivery getLiveryForCombinedCode(const QString &combinedCode) const;
+
+            //! Get aircraft livery for ICAO code
+            //! \threadsafe
+            BlackMisc::Aviation::CLivery getStdLiveryForAirlineCode(const BlackMisc::Aviation::CAirlineIcaoCode &icao) const;
+
+            //! Get aircraft livery for id
+            //! \threadsafe
+            BlackMisc::Aviation::CLivery getLiveryForDbKey(int id) const;
+
+            //! Best match specified by livery
+            //! \threadsafe
+            BlackMisc::Aviation::CLivery smartLiverySelector(const BlackMisc::Aviation::CLivery &livery) const;
+
+            //! Get distributors (of models)
+            //! \threadsafe
+            BlackMisc::Simulation::CDistributorList getDistributors() const;
+
+            //! Get models
+            //! \threadsafe
+            BlackMisc::Simulation::CAircraftModelList getModels() const;
+
+            //! Get model for string
+            //! \threadsafe
+            BlackMisc::Simulation::CAircraftModel getModelForModelString(const QString &modelString) const;
+
+            //! Get model for designator/combined code
+            //! \threadsafe
+            BlackMisc::Simulation::CAircraftModelList getModelsForAircraftDesignatorAndLiveryCombinedCode(const QString &aircraftDesignator, const QString &combinedCode);
+
+            //! Get aircraft liveries count
+            //! \threadsafe
+            int getLiveriesCount() const;
+
+            //! Get model distributors count
+            //! \threadsafe
+            int getDistributorsCount() const;
+
+            //! Best match specified by distributor
+            //! \threadsafe
+            BlackMisc::Simulation::CDistributor smartDistributorSelector(const BlackMisc::Simulation::CDistributor &distributorPattern) const;
+
+            //! Get models count
+            //! \threadsafe
+            int getModelsCount() const;
+
+            //! Get model keys
+            //! \threadsafe
+            QList<int> getModelDbKeys() const;
+
+            //! Get model keys
+            //! \threadsafe
+            QStringList getModelStrings() const;
+
+            //! All data read?
+            //! \threadsafe
+            bool areAllDataRead() const;
+
+            //! Read to JSON file
+            bool readFromJsonFiles(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead = BlackMisc::Network::CEntityFlags::DistributorLiveryModel);
+
+            //! Read from static DB data file
+            bool readFromJsonFilesInBackground(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead = BlackMisc::Network::CEntityFlags::DistributorLiveryModel);
+
+            //! Write to JSON file
+            bool writeToJsonFiles(const QString &dir) const;
+
+        signals:
+            //! Combined read signal
+            void dataRead(BlackMisc::Network::CEntityFlags::Entity entity, BlackMisc::Network::CEntityFlags::ReadState state, int number);
+
+        protected:
+            //! \name cache handling for base class
+            //! @{
+            virtual void syncronizeCaches(BlackMisc::Network::CEntityFlags::Entity entities) override;
+            virtual void invalidateCaches(BlackMisc::Network::CEntityFlags::Entity entities) override;
+            virtual QDateTime getCacheTimestamp(BlackMisc::Network::CEntityFlags::Entity entity) override;
+            //! @}
+
+        private slots:
+            //! Liveries have been read
+            void ps_parseLiveryData(QNetworkReply *nwReply);
+
+            //! Distributors have been read
+            void ps_parseDistributorData(QNetworkReply *nwReply);
+
+            //! Models have been read
+            void ps_parseModelData(QNetworkReply *nwReply);
+
+            //! Read / re-read data file
+            void ps_read(BlackMisc::Network::CEntityFlags::Entity entity = BlackMisc::Network::CEntityFlags::DistributorLiveryModel, const QDateTime &newerThan = QDateTime());
+
+        private:
+            BlackMisc::Aviation::CLiveryList          m_liveries;
+            BlackMisc::Simulation::CDistributorList   m_distributors;
+            BlackMisc::Simulation::CAircraftModelList m_models;
+            BlackMisc::Network::CUrl m_urlLiveries;
+            BlackMisc::Network::CUrl m_urlDistributors;
+            BlackMisc::Network::CUrl m_urlModels;
+
+            mutable QReadWriteLock m_lockDistributor;
+            mutable QReadWriteLock m_lockLivery;
+            mutable QReadWriteLock m_lockModels;
+
+            //! Base URL
+            BlackMisc::Network::CUrl getBaseUrl() const;
+
+            //! URL livery web service
+            BlackMisc::Network::CUrl getLiveryUrl(bool shared = false) const;
+
+            //! URL distributor web service
+            BlackMisc::Network::CUrl getDistributorUrl(bool shared = false) const;
+
+            //! URL model web service
+            BlackMisc::Network::CUrl getModelUrl(bool shared = false) const;
+        };
+    } // ns
+} // ns
+
+#endif // guard