From 954ddfb2e7f9ce227b4e6fccd955ab110ad78c6c Mon Sep 17 00:00:00 2001 From: Klaus Basan Date: Tue, 24 May 2016 00:36:06 +0200 Subject: [PATCH] refs #649, refs #656, using caches in ICAO data reader * access to info data reader (get DB metadata) * adjusted database base class to support caches, info objects * moved classes to subdir --- src/blackcore/databasereader.cpp | 186 ------------ src/blackcore/databasereader.h | 139 --------- src/blackcore/db/databasereader.cpp | 250 ++++++++++++++++ src/blackcore/db/databasereader.h | 172 +++++++++++ src/blackcore/db/icaodatareader.cpp | 432 ++++++++++++++++++++++++++++ src/blackcore/db/icaodatareader.h | 171 +++++++++++ src/blackcore/icaodatareader.cpp | 376 ------------------------ src/blackcore/icaodatareader.h | 153 ---------- src/blackcore/webdataservices.cpp | 187 +++++++----- src/blackcore/webdataservices.h | 43 ++- 10 files changed, 1171 insertions(+), 938 deletions(-) delete mode 100644 src/blackcore/databasereader.cpp delete mode 100644 src/blackcore/databasereader.h create mode 100644 src/blackcore/db/databasereader.cpp create mode 100644 src/blackcore/db/databasereader.h create mode 100644 src/blackcore/db/icaodatareader.cpp create mode 100644 src/blackcore/db/icaodatareader.h delete mode 100644 src/blackcore/icaodatareader.cpp delete mode 100644 src/blackcore/icaodatareader.h diff --git a/src/blackcore/databasereader.cpp b/src/blackcore/databasereader.cpp deleted file mode 100644 index 3ed1de83c..000000000 --- a/src/blackcore/databasereader.cpp +++ /dev/null @@ -1,186 +0,0 @@ -/* Copyright (C) 2015 - * swift project Community / Contributors - * - * This file is part of swift project. It is subject to the license terms in the LICENSE file found in the top-level - * directory of this distribution and at http://www.swift-project.org/license.html. No part of swift project, - * including this file, may be copied, modified, propagated, or distributed except according to the terms - * contained in the LICENSE file. - */ - -#include "blackcore/databasereader.h" -#include "blackmisc/datastoreutility.h" -#include "blackmisc/logcategory.h" -#include "blackmisc/logcategorylist.h" -#include "blackmisc/network/networkutils.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace BlackMisc; -using namespace BlackMisc::Network; - -namespace BlackCore -{ - CDatabaseReader::CDatabaseReader(QObject *owner, const QString &name) : - BlackMisc::CThreadedReader(owner, name) - { - connect(&m_watchdogTimer, &QTimer::timeout, this, &CDatabaseReader::ps_watchdog); - } - - const CLogCategoryList &CDatabaseReader::getLogCategories() - { - static const BlackMisc::CLogCategoryList cats { BlackMisc::CLogCategory::swiftDbWebservice(), BlackMisc::CLogCategory::mapping() }; - return cats; - } - - void CDatabaseReader::readInBackgroundThread(CEntityFlags::Entity entities, const QDateTime &newerThan) - { - if (isAbandoned()) { return; } - this->m_watchdogTimer.stop(); - // ps_read is implemented in the derived classes - bool s = QMetaObject::invokeMethod(this, "ps_read", - Q_ARG(BlackMisc::Network::CEntityFlags::Entity, entities), - Q_ARG(QDateTime, newerThan)); - Q_ASSERT_X(s, Q_FUNC_INFO, "Invoke failed"); - Q_UNUSED(s); - } - - CDatabaseReader::JsonDatastoreResponse CDatabaseReader::transformReplyIntoDatastoreResponse(QNetworkReply *nwReply) const - { - this->threadAssertCheck(); - static const CLogCategoryList cats(CLogCategoryList(this).join({ CLogCategory::webservice()})); - - JsonDatastoreResponse datastoreResponse; - if (this->isAbandoned()) - { - nwReply->abort(); - datastoreResponse.setMessage(CStatusMessage(cats, CStatusMessage::SeverityError, "Terminated data parsing process")); - return datastoreResponse; // stop, terminate straight away, ending thread - } - - if (nwReply->error() == QNetworkReply::NoError) - { - const QString dataFileData = nwReply->readAll().trimmed(); - nwReply->close(); // close asap - if (dataFileData.isEmpty()) - { - datastoreResponse.setMessage(CStatusMessage(cats, CStatusMessage::SeverityError, "Empty response, no data")); - datastoreResponse.m_updated = QDateTime::currentDateTimeUtc(); - return datastoreResponse; - } - - QJsonDocument jsonResponse = QJsonDocument::fromJson(dataFileData.toUtf8()); - if (jsonResponse.isArray()) - { - // directly an array, no further info - datastoreResponse.m_jsonArray = jsonResponse.array(); - datastoreResponse.m_updated = QDateTime::currentDateTimeUtc(); - } - else - { - QJsonObject responseObject(jsonResponse.object()); - datastoreResponse.m_jsonArray = responseObject["data"].toArray(); - QString ts(responseObject["latest"].toString()); - datastoreResponse.m_updated = ts.isEmpty() ? QDateTime::currentDateTimeUtc() : CDatastoreUtility::parseTimestamp(ts); - datastoreResponse.m_restricted = responseObject["restricted"].toBool(); - } - return datastoreResponse; - } - - // no valid response - QString error(nwReply->errorString()); - QString url(nwReply->url().toString()); - nwReply->abort(); - datastoreResponse.setMessage(CStatusMessage(cats, CStatusMessage::SeverityError, - QString("Reading data failed: " + error + " " + url))); - return datastoreResponse; - } - - CDatabaseReader::JsonDatastoreResponse CDatabaseReader::setStatusAndTransformReplyIntoDatastoreResponse(QNetworkReply *nwReply) - { - setConnectionStatus(nwReply); - return transformReplyIntoDatastoreResponse(nwReply); - } - - void CDatabaseReader::setWatchdogUrl(const CUrl &url) - { - bool start; - { - QWriteLocker wl(&this->m_watchdogLock); - m_watchdogUrl = url; - start = url.isEmpty(); - } - if (start) - { - m_watchdogTimer.start(30 * 1000); - } - else - { - m_watchdogTimer.stop(); - } - } - - bool CDatabaseReader::canConnect() const - { - QReadLocker rl(&this->m_watchdogLock); - return m_canConnect; - } - - bool CDatabaseReader::canConnect(QString &message) const - { - QReadLocker rl(&this->m_watchdogLock); - message = m_watchdogMessage; - return m_canConnect; - } - - void CDatabaseReader::setConnectionStatus(bool ok, const QString &message) - { - { - QWriteLocker wl(&this->m_watchdogLock); - this->m_watchdogMessage = message; - this->m_canConnect = ok; - if (this->m_watchdogUrl.isEmpty()) { return; } - } - this->m_updateTimer->start(); // restart - } - - void CDatabaseReader::setConnectionStatus(QNetworkReply *nwReply) - { - Q_ASSERT_X(nwReply, Q_FUNC_INFO, "Missing network reply"); - if (nwReply->isFinished()) - { - if (nwReply->error() == QNetworkReply::NoError) - { - setConnectionStatus(true); - } - else - { - setConnectionStatus(false, nwReply->errorString()); - } - } - } - - void CDatabaseReader::ps_watchdog() - { - CUrl url; - { - QReadLocker rl(&this->m_watchdogLock); - url = this->m_watchdogUrl; - } - if (url.isEmpty()) - { - this->m_watchdogTimer.stop(); - return; - } - QString m; - bool ok = CNetworkUtils::canConnect(url, m); - this->setConnectionStatus(ok, m); - } -} // namespace diff --git a/src/blackcore/databasereader.h b/src/blackcore/databasereader.h deleted file mode 100644 index 44dd07409..000000000 --- a/src/blackcore/databasereader.h +++ /dev/null @@ -1,139 +0,0 @@ -/* Copyright (C) 2015 - * swift project Community / Contributors - * - * This file is part of swift project. It is subject to the license terms in the LICENSE file found in the top-level - * directory of this distribution and at http://www.swift-project.org/license.html. No part of swift project, - * including this file, may be copied, modified, propagated, or distributed except according to the terms - * contained in the LICENSE file. - */ - -//! \file - -#ifndef BLACKCORE_DATABASE_READER_H -#define BLACKCORE_DATABASE_READER_H - -#include "blackcore/blackcoreexport.h" -#include "blackmisc/network/entityflags.h" -#include "blackmisc/network/url.h" -#include "blackmisc/statusmessage.h" -#include "blackmisc/threadedreader.h" - -#include -#include -#include -#include -#include -#include -#include - -class QNetworkReply; - -namespace BlackMisc { class CLogCategoryList; } - -namespace BlackCore -{ - //! Specialized version of threaded reader for DB data - class BLACKCORE_EXPORT CDatabaseReader : public BlackMisc::CThreadedReader - { - Q_OBJECT - - public: - //! Response from our database - struct JsonDatastoreResponse - { - QJsonArray m_jsonArray; //!< JSON array data - QDateTime m_updated; //!< when was the latest updated? - bool m_restricted = false; //!< restricted reponse, only data changed - BlackMisc::CStatusMessage m_message; //!< last error or warning - - //! Any data? - bool isEmpty() const { return m_jsonArray.isEmpty(); } - - //! Number of elements - int size() const { return m_jsonArray.size(); } - - //! Any timestamp? - bool hasTimestamp() const { return m_updated.isValid(); } - - //! Is response newer? - bool isNewer(const QDateTime &ts) const { return m_updated.toMSecsSinceEpoch() > ts.toMSecsSinceEpoch(); } - - //! Is response newer? - bool isNewer(qint64 mSecsSinceEpoch) const { return m_updated.toMSecsSinceEpoch() > mSecsSinceEpoch; } - - //! Incremental data - bool isRestricted() const { return m_restricted; } - - //! Error message? - bool hasErrorMessage() const { return m_message.getSeverity() == BlackMisc::CStatusMessage::SeverityError; } - - //! Warning or error message? - bool hasWarningOrAboveMessage() const { return m_message.isWarningOrAbove(); } - - //! Last error or warning - const BlackMisc::CStatusMessage &lastWarningOrAbove() const { return m_message; } - - //! Set the error/warning message - void setMessage(const BlackMisc::CStatusMessage &lastErrorOrWarning) { m_message = lastErrorOrWarning; } - - //! Get the JSON array - QJsonArray getJsonArray() const { return m_jsonArray; } - - //! Set the JSON array - void setJsonArray(const QJsonArray &value) { m_jsonArray = value; } - - //! Implicit conversion - operator QJsonArray() const { return m_jsonArray; } - }; - - //! Log categories - static const BlackMisc::CLogCategoryList &getLogCategories(); - - //! Start reading in own thread - void readInBackgroundThread(BlackMisc::Network::CEntityFlags::Entity entities, const QDateTime &newerThan); - - //! Can connect to DB - //! \threadsafe - bool canConnect() const; - - //! Can connect to server? - //! \return message why connect failed - //! \threadsafe - bool canConnect(QString &message) const; - - protected: - BlackMisc::Network::CUrl m_watchdogUrl; //!< URL for checking if alive - QTimer m_watchdogTimer { this }; //!< Timer for watchdog (DB available?) - QString m_watchdogMessage; //!< Returned status message from watchdog - bool m_canConnect = false; //!< Successful connection? - mutable QReadWriteLock m_watchdogLock; //!< Lock - - //! Constructor - CDatabaseReader(QObject *owner, const QString &name); - - //! Watchdog URL, empty means no checking - //! \threadsafe - void setWatchdogUrl(const BlackMisc::Network::CUrl &url); - - //! Check if terminated or error, otherwise split into array of objects - CDatabaseReader::JsonDatastoreResponse setStatusAndTransformReplyIntoDatastoreResponse(QNetworkReply *nwReply); - - private slots: - //! Watchdog checking if DB is available - void ps_watchdog(); - - private: - //! Check if terminated or error, otherwise split into array of objects - JsonDatastoreResponse transformReplyIntoDatastoreResponse(QNetworkReply *nwReply) const; - - //! Feedback about connection status - //! \threadsafe - void setConnectionStatus(bool ok, const QString &message = ""); - - //! Feedback about connection status - //! \threadsafe - void setConnectionStatus(QNetworkReply *nwReply); - }; -} // namespace - -#endif // guard diff --git a/src/blackcore/db/databasereader.cpp b/src/blackcore/db/databasereader.cpp new file mode 100644 index 000000000..765be344e --- /dev/null +++ b/src/blackcore/db/databasereader.cpp @@ -0,0 +1,250 @@ +/* Copyright (C) 2015 + * swift project Community / Contributors + * + * This file is part of swift project. It is subject to the license terms in the LICENSE file found in the top-level + * directory of this distribution and at http://www.swift-project.org/license.html. No part of swift project, + * including this file, may be copied, modified, propagated, or distributed except according to the terms + * contained in the LICENSE file. + */ + +#include "blackcore/db/databasereader.h" +#include "blackcore/db/infodatareader.h" +#include "blackcore/webdataservices.h" +#include "blackcore/application.h" +#include "blackmisc/db/datastoreutility.h" +#include "blackmisc/network/networkutils.h" +#include "blackmisc/logcategory.h" +#include "blackmisc/logcategorylist.h" +#include "blackmisc/logmessage.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace BlackMisc; +using namespace BlackMisc::Db; +using namespace BlackMisc::Network; +using namespace BlackCore; + +namespace BlackCore +{ + namespace Db + { + CDatabaseReader::CDatabaseReader(QObject *owner, const CDatabaseReaderConfigList &config, const QString &name) : + BlackMisc::CThreadedReader(owner, name), m_config(config) + { + this->m_sharedUrl = sApp->getGlobalSetup().getSwiftSharedUrls().getRandomWorkingUrl(); + } + + void CDatabaseReader::readInBackgroundThread(CEntityFlags::Entity entities, const QDateTime &newerThan) + { + if (isAbandoned()) { return; } + + // we accept cached cached data + Q_ASSERT_X(!entities.testFlag(CEntityFlags::InfoObjectEntity), Q_FUNC_INFO, "Read info objects directly"); + CEntityFlags::Entity allEntities = entities; + CEntityFlags::Entity currentEntity = CEntityFlags::iterateDbEntities(allEntities); // CEntityFlags::InfoObjectEntity will be ignored + const bool hasInfoObjects = this->hasInfoObjects(); + while (currentEntity) + { + const CDatabaseReaderConfig config(this->getConfigForEntity(currentEntity)); + if (config.getRetrievalMode().testFlag(CDbFlags::Cached)) + { + if (hasInfoObjects) + { + const QDateTime cacheTs(this->getCacheTimestamp(currentEntity)); + const QDateTime latestEntityTs(this->getLatestEntityTimestamp(currentEntity)); + const qint64 cacheTimestamp = cacheTs.isValid() ? cacheTs.toMSecsSinceEpoch() : -1; + const qint64 latestEntityTimestamp = latestEntityTs.isValid() ? latestEntityTs.toMSecsSinceEpoch() : -1; + Q_ASSERT_X(latestEntityTimestamp > 0, Q_FUNC_INFO, "Missing timestamp"); + if (cacheTimestamp >= latestEntityTimestamp && cacheTimestamp > 0) + { + this->syncronizeCaches(currentEntity); + entities &= ~currentEntity; // do not load from web + CLogMessage(this).info("Using cache for %1 (%2, %3)") + << CEntityFlags::flagToString(currentEntity) + << cacheTs.toString() << cacheTimestamp; + } + else + { + CLogMessage(this).info("Cache for %1 outdated, latest entity (%2, %3)") + << CEntityFlags::flagToString(currentEntity) + << latestEntityTs.toString() << latestEntityTimestamp; + } + } + else + { + this->syncronizeCaches(currentEntity); + CLogMessage(this).info("No info object for %1, using cache") << CEntityFlags::flagToString(currentEntity); + } + } + currentEntity = CEntityFlags::iterateDbEntities(allEntities); + } + + // ps_read is implemented in the derived classes + if (entities == CEntityFlags::NoEntity) { return; } + const bool s = QMetaObject::invokeMethod(this, "ps_read", + Q_ARG(BlackMisc::Network::CEntityFlags::Entity, entities), + Q_ARG(QDateTime, newerThan)); + Q_ASSERT_X(s, Q_FUNC_INFO, "Invoke failed"); + Q_UNUSED(s); + } + + CDatabaseReader::JsonDatastoreResponse CDatabaseReader::transformReplyIntoDatastoreResponse(QNetworkReply *nwReply) const + { + this->threadAssertCheck(); + static const CLogCategoryList cats(CLogCategoryList(this).join({ CLogCategory::webservice()})); + + JsonDatastoreResponse datastoreResponse; + if (this->isAbandoned()) + { + nwReply->abort(); + datastoreResponse.setMessage(CStatusMessage(cats, CStatusMessage::SeverityError, "Terminated data parsing process")); + return datastoreResponse; // stop, terminate straight away, ending thread + } + + if (nwReply->error() == QNetworkReply::NoError) + { + const QString dataFileData = nwReply->readAll().trimmed(); + nwReply->close(); // close asap + if (dataFileData.isEmpty()) + { + datastoreResponse.setMessage(CStatusMessage(cats, CStatusMessage::SeverityError, "Empty response, no data")); + datastoreResponse.m_updated = QDateTime::currentDateTimeUtc(); + return datastoreResponse; + } + + QJsonDocument jsonResponse = QJsonDocument::fromJson(dataFileData.toUtf8()); + if (jsonResponse.isArray()) + { + // directly an array, no further info + datastoreResponse.m_jsonArray = jsonResponse.array(); + datastoreResponse.m_updated = QDateTime::currentDateTimeUtc(); + } + else + { + QJsonObject responseObject(jsonResponse.object()); + datastoreResponse.m_jsonArray = responseObject["data"].toArray(); + QString ts(responseObject["latest"].toString()); + datastoreResponse.m_updated = ts.isEmpty() ? QDateTime::currentDateTimeUtc() : CDatastoreUtility::parseTimestamp(ts); + datastoreResponse.m_restricted = responseObject["restricted"].toBool(); + } + return datastoreResponse; + } + + // no valid response + QString error(nwReply->errorString()); + QString url(nwReply->url().toString()); + nwReply->abort(); + datastoreResponse.setMessage(CStatusMessage(cats, CStatusMessage::SeverityError, + QString("Reading data failed: " + error + " " + url))); + return datastoreResponse; + } + + CDatabaseReader::JsonDatastoreResponse CDatabaseReader::setStatusAndTransformReplyIntoDatastoreResponse(QNetworkReply *nwReply) + { + this->setConnectionStatus(nwReply); + return this->transformReplyIntoDatastoreResponse(nwReply); + } + + CDbInfoList CDatabaseReader::infoList() const + { + static const CDbInfoList e; + if (!sApp->hasWebDataServices()) { return e; } + if (!sApp->getWebDataServices()->getInfoDataReader()) { return e; } + return sApp->getWebDataServices()->getInfoDataReader()->getDbInfoObjects(); + } + + bool CDatabaseReader::hasInfoObjects() const + { + return infoList().size() > 0; + } + + QDateTime CDatabaseReader::getLatestEntityTimestamp(CEntityFlags::Entity entity) const + { + static const QDateTime e; + const CDbInfoList il(infoList()); + if (il.isEmpty() || entity == CEntityFlags::NoEntity) { return e; } + CDbInfo info = il.findFirstByEntityOrDefault(entity); + if (!info.isValid()) { return e; } + return info.getUtcTimestamp(); + } + + CDatabaseReaderConfig CDatabaseReader::getConfigForEntity(CEntityFlags::Entity entity) const + { + return this->m_config.findFirstOrDefaultForEntity(entity); + } + + bool CDatabaseReader::canConnect() const + { + QReadLocker rl(&this->m_statusLock); + return m_canConnect; + } + + bool CDatabaseReader::canConnect(QString &message) const + { + QReadLocker rl(&this->m_statusLock); + message = m_statusMessage; + return m_canConnect; + } + + CUrl CDatabaseReader::getWorkingSharedUrl() const + { + return this->m_sharedUrl; + } + + const QString &CDatabaseReader::getStatusMessage() const + { + return this->m_statusMessage; + } + + void CDatabaseReader::setConnectionStatus(bool ok, const QString &message) + { + { + QWriteLocker wl(&this->m_statusLock); + this->m_statusMessage = message; + this->m_canConnect = ok; + } + } + + void CDatabaseReader::setConnectionStatus(QNetworkReply *nwReply) + { + Q_ASSERT_X(nwReply, Q_FUNC_INFO, "Missing network reply"); + if (nwReply->isFinished()) + { + if (nwReply->error() == QNetworkReply::NoError) + { + setConnectionStatus(true); + } + else + { + setConnectionStatus(false, nwReply->errorString()); + } + } + } + + const CLogCategoryList &CDatabaseReader::getLogCategories() + { + static const BlackMisc::CLogCategoryList cats { BlackMisc::CLogCategory::swiftDbWebservice(), BlackMisc::CLogCategory::mapping() }; + return cats; + } + + const QString &CDatabaseReader::parameterLatestTimestamp() + { + static const QString p("latestTimestamp"); + return p; + } + + const QString &CDatabaseReader::parameterLatestId() + { + static const QString p("latestId"); + return p; + } + } // ns +} // ns diff --git a/src/blackcore/db/databasereader.h b/src/blackcore/db/databasereader.h new file mode 100644 index 000000000..d89593d7a --- /dev/null +++ b/src/blackcore/db/databasereader.h @@ -0,0 +1,172 @@ +/* Copyright (C) 2015 + * swift project Community / Contributors + * + * This file is part of swift project. It is subject to the license terms in the LICENSE file found in the top-level + * directory of this distribution and at http://www.swift-project.org/license.html. No part of swift project, + * including this file, may be copied, modified, propagated, or distributed except according to the terms + * contained in the LICENSE file. + */ + +//! \file + +#ifndef BLACKCORE_DB_DATABASEREADER_H +#define BLACKCORE_DB_DATABASEREADER_H + +#include "blackcore/blackcoreexport.h" +#include "blackcore/db/databasereaderconfig.h" +#include "blackmisc/db/dbinfolist.h" +#include "blackmisc/pq/time.h" +#include "blackmisc/network/url.h" +#include "blackmisc/statusmessage.h" +#include "blackmisc/threadedreader.h" +#include "blackmisc/sequence.h" +#include "blackmisc/valueobject.h" + +#include +#include +#include +#include +#include +#include +#include + +class QNetworkReply; +namespace BlackMisc { class CLogCategoryList; } + +namespace BlackCore +{ + namespace Db + { + //! Specialized version of threaded reader for DB data + class BLACKCORE_EXPORT CDatabaseReader : public BlackMisc::CThreadedReader + { + Q_OBJECT + + public: + //! Response from our database + struct JsonDatastoreResponse + { + QJsonArray m_jsonArray; //!< JSON array data + QDateTime m_updated; //!< when was the latest updated? + bool m_restricted = false; //!< restricted reponse, only data changed + BlackMisc::CStatusMessage m_message; //!< last error or warning + + //! Any data? + bool isEmpty() const { return m_jsonArray.isEmpty(); } + + //! Number of elements + int size() const { return m_jsonArray.size(); } + + //! Any timestamp? + bool hasTimestamp() const { return m_updated.isValid(); } + + //! Is response newer? + bool isNewer(const QDateTime &ts) const { return m_updated.toMSecsSinceEpoch() > ts.toMSecsSinceEpoch(); } + + //! Is response newer? + bool isNewer(qint64 mSecsSinceEpoch) const { return m_updated.toMSecsSinceEpoch() > mSecsSinceEpoch; } + + //! Incremental data + bool isRestricted() const { return m_restricted; } + + //! Error message? + bool hasErrorMessage() const { return m_message.getSeverity() == BlackMisc::CStatusMessage::SeverityError; } + + //! Warning or error message? + bool hasWarningOrAboveMessage() const { return m_message.isWarningOrAbove(); } + + //! Last error or warning + const BlackMisc::CStatusMessage &lastWarningOrAbove() const { return m_message; } + + //! Set the error/warning message + void setMessage(const BlackMisc::CStatusMessage &lastErrorOrWarning) { m_message = lastErrorOrWarning; } + + //! Get the JSON array + QJsonArray getJsonArray() const { return m_jsonArray; } + + //! Set the JSON array + void setJsonArray(const QJsonArray &value) { m_jsonArray = value; } + + //! Implicit conversion + operator QJsonArray() const { return m_jsonArray; } + }; + + //! Start reading in own thread + void readInBackgroundThread(BlackMisc::Network::CEntityFlags::Entity entities, const QDateTime &newerThan); + + //! Can connect to DB + //! \threadsafe + bool canConnect() const; + + //! Can connect to server? + //! \return message why connect failed + //! \threadsafe + bool canConnect(QString &message) const; + + //! Obtain a working shared URL + BlackMisc::Network::CUrl getWorkingSharedUrl() const; + + //! Status message (error message) + const QString &getStatusMessage() const; + + //! Log categories + static const BlackMisc::CLogCategoryList &getLogCategories(); + + //! Name of latest timestamp + static const QString ¶meterLatestTimestamp(); + + //! Name of parameter for latest id + static const QString ¶meterLatestId(); + + protected: + CDatabaseReaderConfigList m_config; //!< DB reder configuration + BlackMisc::Network::CUrl m_sharedUrl; //!< URL for checking if alive + QString m_statusMessage; //!< Returned status message from watchdog + bool m_canConnect = false; //!< Successful connection? + mutable QReadWriteLock m_statusLock; //!< Lock + + //! Constructor + CDatabaseReader(QObject *owner, const CDatabaseReaderConfigList &config, const QString &name); + + //! Check if terminated or error, otherwise split into array of objects + CDatabaseReader::JsonDatastoreResponse setStatusAndTransformReplyIntoDatastoreResponse(QNetworkReply *nwReply); + + //! Info list (latest data timestamp) + //! \sa BlackCore::Db::CInfoDataReader + BlackMisc::Db::CDbInfoList infoList() const; + + //! Info objects available? + bool hasInfoObjects() const; + + //! Obtain latest object timestamp + //! \sa BlackCore::Db::CInfoDataReader + QDateTime getLatestEntityTimestamp(BlackMisc::Network::CEntityFlags::Entity entity) const; + + //! Config for given entity + CDatabaseReaderConfig getConfigForEntity(BlackMisc::Network::CEntityFlags::Entity entity) const; + + //! Syncronize caches for given entities + virtual void syncronizeCaches(BlackMisc::Network::CEntityFlags::Entity entities) = 0; + + //! Cache`s timestamp for given entity + virtual QDateTime getCacheTimestamp(BlackMisc::Network::CEntityFlags::Entity entities) = 0; + + //! Invalidate the caches for given entities + virtual void invalidateCaches(BlackMisc::Network::CEntityFlags::Entity entities) = 0; + + private: + //! Check if terminated or error, otherwise split into array of objects + JsonDatastoreResponse transformReplyIntoDatastoreResponse(QNetworkReply *nwReply) const; + + //! Feedback about connection status + //! \threadsafe + void setConnectionStatus(bool ok, const QString &message = ""); + + //! Feedback about connection status + //! \threadsafe + void setConnectionStatus(QNetworkReply *nwReply); + }; + } // ns +} // ns + +#endif // guard diff --git a/src/blackcore/db/icaodatareader.cpp b/src/blackcore/db/icaodatareader.cpp new file mode 100644 index 000000000..d7c6b1947 --- /dev/null +++ b/src/blackcore/db/icaodatareader.cpp @@ -0,0 +1,432 @@ +/* Copyright (C) 2015 + * swift project Community / Contributors + * + * This file is part of swift project. It is subject to the license terms in the LICENSE file found in the top-level + * directory of this distribution and at http://www.swift-project.org/license.html. No part of swift project, + * including this file, may be copied, modified, propagated, or distributed except according to the terms + * contained in the LICENSE file. + */ + +#include "blackcore/application.h" +#include "blackcore/data/globalsetup.h" +#include "blackcore/db/icaodatareader.h" +#include "blackmisc/fileutils.h" +#include "blackmisc/json.h" +#include "blackmisc/logmessage.h" +#include "blackmisc/statusmessage.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace BlackMisc; +using namespace BlackMisc::Aviation; +using namespace BlackMisc::Network; +using namespace BlackCore::Data; + +namespace BlackCore +{ + namespace Db + { + CIcaoDataReader::CIcaoDataReader(QObject *owner, const CDatabaseReaderConfigList &confg) : + CDatabaseReader(owner, confg, "CIcaoDataReader") + { + // void + } + + CAircraftIcaoCodeList CIcaoDataReader::getAircraftIcaoCodes() const + { + return m_aircraftIcaoCache.getCopy(); + } + + CAircraftIcaoCode CIcaoDataReader::getAircraftIcaoCodeForDesignator(const QString &designator) const + { + return getAircraftIcaoCodes().findFirstByDesignatorAndRank(designator); + } + + CAircraftIcaoCode CIcaoDataReader::getAircraftIcaoCodeForDbKey(int key) const + { + return getAircraftIcaoCodes().findByKey(key); + } + + CAirlineIcaoCodeList CIcaoDataReader::getAirlineIcaoCodes() const + { + return m_airlineIcaoCache.getCopy(); + } + + CAircraftIcaoCode CIcaoDataReader::smartAircraftIcaoSelector(const CAircraftIcaoCode &icaoPattern) const + { + CAircraftIcaoCodeList codes(getAircraftIcaoCodes()); // thread safe copy + return codes.smartAircraftIcaoSelector(icaoPattern); // sorted by rank + } + + CCountryList CIcaoDataReader::getCountries() const + { + return m_countryCache.getCopy(); + } + + CCountry CIcaoDataReader::getCountryForIsoCode(const QString &isoCode) const + { + return this->getCountries().findByIsoCode(isoCode); + } + + CCountry CIcaoDataReader::getCountryForName(const QString &name) const + { + return this->getCountries().findBestMatchByCountryName(name); + } + + CAirlineIcaoCodeList CIcaoDataReader::getAirlineIcaoCodesForDesignator(const QString &designator) const + { + return this->getAirlineIcaoCodes().findByVDesignator(designator); + } + + CAirlineIcaoCode CIcaoDataReader::getAirlineIcaoCodeForDbKey(int key) const + { + return this->getAirlineIcaoCodes().findByKey(key); + } + + CAirlineIcaoCode CIcaoDataReader::smartAirlineIcaoSelector(const CAirlineIcaoCode &icaoPattern) const + { + CAirlineIcaoCodeList codes(this->getAirlineIcaoCodes()); // thread safe copy + return codes.smartAirlineIcaoSelector(icaoPattern); + } + + int CIcaoDataReader::getAircraftIcaoCodesCount() const + { + return this->getAircraftIcaoCodes().size(); + } + + int CIcaoDataReader::getAirlineIcaoCodesCount() const + { + return this->getAirlineIcaoCodes().size(); + } + + bool CIcaoDataReader::areAllDataRead() const + { + return getCountriesCount() > 0 && getAirlineIcaoCodesCount() > 0 && getAircraftIcaoCodesCount() > 0; + } + + int CIcaoDataReader::getCountriesCount() const + { + return this->getCountries().size(); + } + + void CIcaoDataReader::ps_read(BlackMisc::Network::CEntityFlags::Entity entities, const QDateTime &newerThan) + { + this->threadAssertCheck(); // runs in background thread + + CEntityFlags::Entity entitiesTriggered = CEntityFlags::NoEntity; + if (entities.testFlag(CEntityFlags::AircraftIcaoEntity)) + { + CUrl url(getAircraftIcaoUrl()); + if (!url.isEmpty()) + { + if (!newerThan.isNull()) { url.appendQuery("newer=" + newerThan.toString(Qt::ISODate)); } + sApp->getFromNetwork(url, { this, &CIcaoDataReader::ps_parseAircraftIcaoData }); + entitiesTriggered |= CEntityFlags::AircraftIcaoEntity; + } + else + { + CLogMessage(this).error("No URL for %1") << CEntityFlags::flagToString(CEntityFlags::AircraftIcaoEntity); + } + } + + if (entities.testFlag(CEntityFlags::AirlineIcaoEntity)) + { + CUrl url(getAirlineIcaoUrl()); + if (!url.isEmpty()) + { + if (!newerThan.isNull()) { url.appendQuery("newer=" + newerThan.toString(Qt::ISODate)); } + sApp->getFromNetwork(url, { this, &CIcaoDataReader::ps_parseAirlineIcaoData }); + entitiesTriggered |= CEntityFlags::AirlineIcaoEntity; + } + else + { + CLogMessage(this).error("No URL for %1") << CEntityFlags::flagToString(CEntityFlags::AirlineIcaoEntity); + } + } + + if (entities.testFlag(CEntityFlags::CountryEntity)) + { + CUrl url(getCountryUrl()); + if (!url.isEmpty()) + { + if (!newerThan.isNull()) { url.appendQuery("newer=" + newerThan.toString(Qt::ISODate)); } + sApp->getFromNetwork(url, { this, &CIcaoDataReader::ps_parseCountryData }); + entitiesTriggered |= CEntityFlags::CountryEntity; + } + else + { + CLogMessage(this).error("No URL for %1") << CEntityFlags::flagToString(CEntityFlags::CountryEntity); + } + } + + if (entitiesTriggered != CEntityFlags::NoEntity) + { + emit dataRead(entitiesTriggered, CEntityFlags::StartRead, 0); + } + } + + void CIcaoDataReader::ps_aircraftIcaoCacheChanged() + { + // void + } + + void CIcaoDataReader::ps_airlineIcaoCacheChanged() + { + // void + } + + void CIcaoDataReader::ps_countryCacheChanged() + { + // void + } + + void CIcaoDataReader::ps_baseUrlCacheChanged() + { + // void + } + + void CIcaoDataReader::updateReaderUrl(const CUrl &url) + { + const CUrl current = this->m_readerUrlCache.getCopy(); + if (current == url) { return; } + const CStatusMessage m = this->m_readerUrlCache.set(url); + if (m.isFailure()) + { + CLogMessage::preformatted(m); + } + } + + CUrl CIcaoDataReader::getBaseUrl() const + { + const CUrl baseUrl(sApp->getGlobalSetup().getDbIcaoReaderUrl()); + return baseUrl; + } + + void CIcaoDataReader::ps_parseAircraftIcaoData(QNetworkReply *nwReplyPtr) + { + // wrap pointer, make sure any exit cleans up reply + // required to use delete later as object is created in a different thread + QScopedPointer nwReply(nwReplyPtr); + QString urlString(nwReply->url().toString()); + CDatabaseReader::JsonDatastoreResponse res = this->setStatusAndTransformReplyIntoDatastoreResponse(nwReply.data()); + if (res.hasErrorMessage()) + { + CLogMessage::preformatted(res.lastWarningOrAbove()); + emit dataRead(CEntityFlags::AircraftIcaoEntity, CEntityFlags::ReadFailed, 0); + return; + } + const CAircraftIcaoCodeList codes = CAircraftIcaoCodeList::fromDatabaseJson(res); + const int n = codes.size(); + qint64 latestTimestamp = codes.latestTimestampMsecsSinceEpoch(); + if (n > 0 && latestTimestamp < 0) + { + CLogMessage(this).error("No timestamp in aircraft ICAO list, setting to last modified value"); + latestTimestamp = lastModifiedMsSinceEpoch(nwReply.data()); + } + + this->m_aircraftIcaoCache.set(codes, latestTimestamp); + this->updateReaderUrl(this->getBaseUrl()); + emit dataRead(CEntityFlags::AircraftIcaoEntity, CEntityFlags::ReadFinished, n); + CLogMessage(this).info("Read %1 %2 from %3") << n << CEntityFlags::flagToString(CEntityFlags::AircraftIcaoEntity) << urlString; + } + + void CIcaoDataReader::ps_parseAirlineIcaoData(QNetworkReply *nwReplyPtr) + { + QScopedPointer nwReply(nwReplyPtr); + QString urlString(nwReply->url().toString()); + CDatabaseReader::JsonDatastoreResponse res = this->setStatusAndTransformReplyIntoDatastoreResponse(nwReply.data()); + if (res.hasErrorMessage()) + { + CLogMessage::preformatted(res.lastWarningOrAbove()); + emit dataRead(CEntityFlags::AirlineIcaoEntity, CEntityFlags::ReadFailed, 0); + return; + } + const CAirlineIcaoCodeList codes = CAirlineIcaoCodeList::fromDatabaseJson(res); + const int n = codes.size(); + qint64 latestTimestamp = codes.latestTimestampMsecsSinceEpoch(); + if (n > 0 && latestTimestamp < 0) + { + CLogMessage(this).error("No timestamp in airline ICAO list, setting to last modified value"); + latestTimestamp = lastModifiedMsSinceEpoch(nwReply.data()); + } + + this->m_airlineIcaoCache.set(codes, latestTimestamp); + emit dataRead(CEntityFlags::AirlineIcaoEntity, CEntityFlags::ReadFinished, n); + CLogMessage(this).info("Read %1 %2 from %3") << n << CEntityFlags::flagToString(CEntityFlags::AirlineIcaoEntity) << urlString; + } + + void CIcaoDataReader::ps_parseCountryData(QNetworkReply *nwReplyPtr) + { + QScopedPointer nwReply(nwReplyPtr); + QString urlString(nwReply->url().toString()); + CDatabaseReader::JsonDatastoreResponse res = this->setStatusAndTransformReplyIntoDatastoreResponse(nwReply.data()); + if (res.hasErrorMessage()) + { + CLogMessage::preformatted(res.lastWarningOrAbove()); + emit dataRead(CEntityFlags::CountryEntity, CEntityFlags::ReadFailed, 0); + return; + } + const CCountryList countries = CCountryList::fromDatabaseJson(res); + const int n = countries.size(); + qint64 latestTimestamp = countries.latestTimestampMsecsSinceEpoch(); + if (n > 0 && latestTimestamp < 0) + { + CLogMessage(this).error("No timestamp in country list, setting to last modified value"); + latestTimestamp = lastModifiedMsSinceEpoch(nwReply.data()); + } + + this->m_countryCache.set(countries, latestTimestamp); + emit dataRead(CEntityFlags::CountryEntity, CEntityFlags::ReadFinished, n); + CLogMessage(this).info("Read %1 %2 from %3") << n << CEntityFlags::flagToString(CEntityFlags::CountryEntity) << urlString; + } + + bool CIcaoDataReader::readFromJsonFiles(const QString &dir, CEntityFlags::Entity whatToRead) + { + QDir directory(dir); + if (!directory.exists()) { return false; } + + CEntityFlags::Entity reallyRead = CEntityFlags::NoEntity; + if (whatToRead.testFlag(CEntityFlags::CountryEntity)) + { + QString countriesJson(CFileUtils::readFileToString(CFileUtils::appendFilePaths(directory.absolutePath(), "countries.json"))); + if (!countriesJson.isEmpty()) + { + CCountryList countries; + countries.convertFromJson(Json::jsonObjectFromString(countriesJson)); + const int c = countries.size(); + this->m_countryCache.set(countries); + + // Do not emit while locked -> deadlock + reallyRead |= CEntityFlags::CountryEntity; + emit dataRead(CEntityFlags::CountryEntity, CEntityFlags::ReadFinished, c); + } + } + + if (whatToRead.testFlag(CEntityFlags::AircraftIcaoEntity)) + { + QString aircraftJson(CFileUtils::readFileToString(CFileUtils::appendFilePaths(directory.absolutePath(), "aircrafticao.json"))); + if (!aircraftJson.isEmpty()) + { + CAircraftIcaoCodeList aircraftIcaos; + aircraftIcaos.convertFromJson(Json::jsonObjectFromString(aircraftJson)); + const int c = aircraftIcaos.size(); + this->m_aircraftIcaoCache.set(aircraftIcaos); + + reallyRead |= CEntityFlags::AircraftIcaoEntity; + emit dataRead(CEntityFlags::AircraftIcaoEntity, CEntityFlags::ReadFinished, c); + } + } + + if (whatToRead.testFlag(CEntityFlags::AirlineIcaoEntity)) + { + QString airlineJson(CFileUtils::readFileToString(CFileUtils::appendFilePaths(directory.absolutePath(), "airlineicao.json"))); + if (!airlineJson.isEmpty()) + { + CAirlineIcaoCodeList airlineIcaos; + airlineIcaos.convertFromJson(Json::jsonObjectFromString(airlineJson)); + const int c = airlineIcaos.size(); + this->m_airlineIcaoCache.set(airlineIcaos); + reallyRead |= CEntityFlags::AirlineIcaoEntity; + emit dataRead(CEntityFlags::AirlineIcaoEntity, CEntityFlags::ReadFinished, c); + } + } + return (whatToRead & CEntityFlags::AllIcaoAndCountries) == reallyRead; + } + + bool CIcaoDataReader::readFromJsonFilesInBackground(const QString &dir, CEntityFlags::Entity whatToRead) + { + if (dir.isEmpty() || whatToRead == CEntityFlags::NoEntity) { return false; } + QTimer::singleShot(0, this, [this, dir, whatToRead]() + { + bool s = this->readFromJsonFiles(dir, whatToRead); + Q_UNUSED(s); + }); + return true; + } + + bool CIcaoDataReader::writeToJsonFiles(const QString &dir) const + { + QDir directory(dir); + if (!directory.exists()) { return false; } + if (this->getCountriesCount() > 0) + { + QString json(QJsonDocument(this->getCountries().toJson()).toJson()); + bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), "countries.json")); + if (!s) { return false; } + } + + if (this->getAircraftIcaoCodesCount() > 0) + { + QString json(QJsonDocument(this->getAircraftIcaoCodes().toJson()).toJson()); + bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), "aircrafticao.json")); + if (!s) { return false; } + } + + if (this->getAirlineIcaoCodesCount() > 0) + { + QString json(QJsonDocument(this->getAirlineIcaoCodes().toJson()).toJson()); + bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), "airlineicao.json")); + if (!s) { return false; } + } + return true; + } + + void CIcaoDataReader::syncronizeCaches(CEntityFlags::Entity entities) + { + if (entities.testFlag(CEntityFlags::AircraftIcaoEntity)) { this->m_aircraftIcaoCache.synchronize(); } + if (entities.testFlag(CEntityFlags::AirlineIcaoEntity)) { this->m_airlineIcaoCache.synchronize(); } + if (entities.testFlag(CEntityFlags::CountryEntity)) { this->m_countryCache.synchronize(); } + } + + void CIcaoDataReader::invalidateCaches(CEntityFlags::Entity entities) + { + if (entities.testFlag(CEntityFlags::AircraftIcaoEntity)) { CDataCache::instance()->clearAllValues(this->m_aircraftIcaoCache.getKey()); } + if (entities.testFlag(CEntityFlags::AirlineIcaoEntity)) {CDataCache::instance()->clearAllValues(this->m_airlineIcaoCache.getKey()); } + if (entities.testFlag(CEntityFlags::CountryEntity)) {CDataCache::instance()->clearAllValues(this->m_countryCache.getKey()); } + } + + QDateTime CIcaoDataReader::getCacheTimestamp(CEntityFlags::Entity entity) + { + switch (entity) + { + case CEntityFlags::AircraftIcaoEntity: return this->m_aircraftIcaoCache.getTimestamp(); + case CEntityFlags::AirlineIcaoEntity: return this->m_airlineIcaoCache.getTimestamp(); + case CEntityFlags::CountryEntity: return this->m_countryCache.getTimestamp(); + default: return QDateTime(); + } + } + + CUrl CIcaoDataReader::getAircraftIcaoUrl(bool shared) const + { + return shared ? + getWorkingSharedUrl().withAppendedPath("jsonaircrafticao.php") : + getBaseUrl().withAppendedPath("service/jsonaircrafticao.php"); + } + + CUrl CIcaoDataReader::getAirlineIcaoUrl(bool shared) const + { + return shared ? + getWorkingSharedUrl().withAppendedPath("jsonairlineicao.php") : + getBaseUrl().withAppendedPath("service/jsonairlineicao.php"); + } + + CUrl CIcaoDataReader::getCountryUrl(bool shared) const + { + return shared ? + getWorkingSharedUrl().withAppendedPath("jsoncountry.php") : + getBaseUrl().withAppendedPath("service/jsoncountry.php"); + } + } // ns +} // ns diff --git a/src/blackcore/db/icaodatareader.h b/src/blackcore/db/icaodatareader.h new file mode 100644 index 000000000..ff7ffdd8b --- /dev/null +++ b/src/blackcore/db/icaodatareader.h @@ -0,0 +1,171 @@ +/* Copyright (C) 2015 + * swift project Community / Contributors + * + * This file is part of swift project. It is subject to the license terms in the LICENSE file found in the top-level + * directory of this distribution and at http://www.swift-project.org/license.html. No part of swift project, + * including this file, may be copied, modified, propagated, or distributed except according to the terms + * contained in the LICENSE file. + */ + +//! \file + +#ifndef BLACKCORE_ICAODATAREADER_H +#define BLACKCORE_ICAODATAREADER_H + +#include "blackcore/blackcoreexport.h" +#include "blackcore/db/databasereader.h" +#include "blackcore/data/dbcaches.h" +#include "blackmisc/aviation/aircrafticaocode.h" +#include "blackmisc/aviation/aircrafticaocodelist.h" +#include "blackmisc/aviation/airlineicaocode.h" +#include "blackmisc/aviation/airlineicaocodelist.h" +#include "blackmisc/network/entityflags.h" +#include "blackmisc/network/url.h" +#include "blackmisc/country.h" +#include "blackmisc/countrylist.h" +#include "blackmisc/datacache.h" + +#include +#include +#include + +class QDateTime; +class QNetworkReply; + +namespace BlackCore +{ + namespace Db + { + //! Read ICAO data from Database + class BLACKCORE_EXPORT CIcaoDataReader : public CDatabaseReader + { + Q_OBJECT + + public: + //! Constructor + explicit CIcaoDataReader(QObject *owner, const CDatabaseReaderConfigList &confg); + + //! Get aircraft ICAO information + //! \threadsafe + BlackMisc::Aviation::CAircraftIcaoCodeList getAircraftIcaoCodes() const; + + //! Get aircraft ICAO information for designator + //! \threadsafe + BlackMisc::Aviation::CAircraftIcaoCode getAircraftIcaoCodeForDesignator(const QString &designator) const; + + //! Get aircraft ICAO information for key + //! \threadsafe + BlackMisc::Aviation::CAircraftIcaoCode getAircraftIcaoCodeForDbKey(int key) const; + + //! Get airline ICAO information + //! \threadsafe + BlackMisc::Aviation::CAirlineIcaoCodeList getAirlineIcaoCodes() const; + + //! Get best match for incomplete aircraft ICAO code + //! \threadsafe + BlackMisc::Aviation::CAircraftIcaoCode smartAircraftIcaoSelector(const BlackMisc::Aviation::CAircraftIcaoCode &icaoPattern) const; + + //! Get countries + //! \threadsafe + BlackMisc::CCountryList getCountries() const; + + //! Get countries count + //! \threadsafe + int getCountriesCount() const; + + //! Get country for ISO code + //! \threadsafe + BlackMisc::CCountry getCountryForIsoCode(const QString &isoCode) const; + + //! Get country for ISO name + //! \threadsafe + BlackMisc::CCountry getCountryForName(const QString &name) const; + + //! Get airline ICAO information for designator + //! \threadsafe + BlackMisc::Aviation::CAirlineIcaoCodeList getAirlineIcaoCodesForDesignator(const QString &designator) const; + + //! Get airline ICAO information for key + //! \threadsafe + BlackMisc::Aviation::CAirlineIcaoCode getAirlineIcaoCodeForDbKey(int key) const; + + //! Get best match for incomplete airline ICAO code + //! \threadsafe + BlackMisc::Aviation::CAirlineIcaoCode smartAirlineIcaoSelector(const BlackMisc::Aviation::CAirlineIcaoCode &icaoPattern) const; + + //! Get aircraft ICAO information count + //! \threadsafe + int getAircraftIcaoCodesCount() const; + + //! Get airline ICAO information count + //! \threadsafe + int getAirlineIcaoCodesCount() const; + + //! All data read? + //! \threadsafe + bool areAllDataRead() const; + + //! Read from static DB data file + bool readFromJsonFiles(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead = BlackMisc::Network::CEntityFlags::AllIcaoAndCountries); + + //! Read from static DB data file + bool readFromJsonFilesInBackground(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead = BlackMisc::Network::CEntityFlags::AllIcaoAndCountries); + + //! Write to static DB data file + bool writeToJsonFiles(const QString &dir) const; + + signals: + //! Combined read signal + void dataRead(BlackMisc::Network::CEntityFlags::Entity entity, BlackMisc::Network::CEntityFlags::ReadState state, int number); + + protected: + //! \name cache handling for base class + //! @{ + virtual void syncronizeCaches(BlackMisc::Network::CEntityFlags::Entity entities) override; + virtual void invalidateCaches(BlackMisc::Network::CEntityFlags::Entity entities) override; + virtual QDateTime getCacheTimestamp(BlackMisc::Network::CEntityFlags::Entity entity) override; + //! @} + + private slots: + //! Aircraft have been read + void ps_parseAircraftIcaoData(QNetworkReply *nwReply); + + //! Airlines have been read + void ps_parseAirlineIcaoData(QNetworkReply *nwReply); + + //! Airlines have been read + void ps_parseCountryData(QNetworkReply *nwReply); + + //! Read / re-read data file + void ps_read(BlackMisc::Network::CEntityFlags::Entity entities, const QDateTime &newerThan); + + void ps_aircraftIcaoCacheChanged(); + void ps_airlineIcaoCacheChanged(); + void ps_countryCacheChanged(); + void ps_baseUrlCacheChanged(); + + private: + BlackMisc::CData m_aircraftIcaoCache {this, &CIcaoDataReader::ps_aircraftIcaoCacheChanged }; + BlackMisc::CData m_airlineIcaoCache {this, &CIcaoDataReader::ps_airlineIcaoCacheChanged }; + BlackMisc::CData m_countryCache {this, &CIcaoDataReader::ps_countryCacheChanged }; + BlackMisc::CData m_readerUrlCache {this, &CIcaoDataReader::ps_baseUrlCacheChanged }; + + //! Update reader URL + void updateReaderUrl(const BlackMisc::Network::CUrl &url); + + //! Base URL + BlackMisc::Network::CUrl getBaseUrl() const; + + //! URL + BlackMisc::Network::CUrl getAircraftIcaoUrl(bool shared = false) const; + + //! URL + BlackMisc::Network::CUrl getAirlineIcaoUrl(bool shared = false) const; + + //! URL + BlackMisc::Network::CUrl getCountryUrl(bool shared = false) const; + }; + } // ns +} // ns + +#endif // guard diff --git a/src/blackcore/icaodatareader.cpp b/src/blackcore/icaodatareader.cpp deleted file mode 100644 index e91fd568b..000000000 --- a/src/blackcore/icaodatareader.cpp +++ /dev/null @@ -1,376 +0,0 @@ -/* Copyright (C) 2015 - * swift project Community / Contributors - * - * This file is part of swift project. It is subject to the license terms in the LICENSE file found in the top-level - * directory of this distribution and at http://www.swift-project.org/license.html. No part of swift project, - * including this file, may be copied, modified, propagated, or distributed except according to the terms - * contained in the LICENSE file. - */ - -#include "blackcore/application.h" -#include "blackcore/data/globalsetup.h" -#include "blackcore/icaodatareader.h" -#include "blackmisc/fileutils.h" -#include "blackmisc/json.h" -#include "blackmisc/logmessage.h" -#include "blackmisc/statusmessage.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace BlackMisc; -using namespace BlackMisc::Aviation; -using namespace BlackMisc::Network; -using namespace BlackCore::Data; - -namespace BlackCore -{ - CIcaoDataReader::CIcaoDataReader(QObject *owner) : - CDatabaseReader(owner, "CIcaoDataReader") - { - // void - } - - CAircraftIcaoCodeList CIcaoDataReader::getAircraftIcaoCodes() const - { - QReadLocker l(&m_lockAircraft); - return m_aircraftIcaos; - } - - CAircraftIcaoCode CIcaoDataReader::getAircraftIcaoCodeForDesignator(const QString &designator) const - { - return getAircraftIcaoCodes().findFirstByDesignatorAndRank(designator); - } - - CAircraftIcaoCode CIcaoDataReader::getAircraftIcaoCodeForDbKey(int key) const - { - return getAircraftIcaoCodes().findByKey(key); - } - - CAirlineIcaoCodeList CIcaoDataReader::getAirlineIcaoCodes() const - { - QReadLocker l(&m_lockAirline); - return m_airlineIcaos; - } - - CAircraftIcaoCode CIcaoDataReader::smartAircraftIcaoSelector(const CAircraftIcaoCode &icaoPattern) const - { - CAircraftIcaoCodeList codes(getAircraftIcaoCodes()); // thread safe copy - return codes.smartAircraftIcaoSelector(icaoPattern); // sorted by rank - } - - CCountryList CIcaoDataReader::getCountries() const - { - QReadLocker l(&m_lockCountry); - return m_countries; - } - - CCountry CIcaoDataReader::getCountryForIsoCode(const QString &isoCode) const - { - QReadLocker l(&m_lockCountry); - return m_countries.findByIsoCode(isoCode); - } - - CCountry CIcaoDataReader::getCountryForName(const QString &name) const - { - QReadLocker l(&m_lockCountry); - return m_countries.findBestMatchByCountryName(name); - } - - CAirlineIcaoCodeList CIcaoDataReader::getAirlineIcaoCodesForDesignator(const QString &designator) const - { - return getAirlineIcaoCodes().findByVDesignator(designator); - } - - CAirlineIcaoCode CIcaoDataReader::getAirlineIcaoCodeForDbKey(int key) const - { - return getAirlineIcaoCodes().findByKey(key); - } - - CAirlineIcaoCode CIcaoDataReader::smartAirlineIcaoSelector(const CAirlineIcaoCode &icaoPattern) const - { - CAirlineIcaoCodeList codes(this->getAirlineIcaoCodes()); // thread safe copy - return codes.smartAirlineIcaoSelector(icaoPattern); - } - - int CIcaoDataReader::getAircraftIcaoCodesCount() const - { - QReadLocker l(&m_lockAircraft); - return m_aircraftIcaos.size(); - } - - int CIcaoDataReader::getAirlineIcaoCodesCount() const - { - QReadLocker l(&m_lockAirline); - return m_airlineIcaos.size(); - } - - bool CIcaoDataReader::areAllDataRead() const - { - return getCountriesCount() > 0 && getAirlineIcaoCodesCount() > 0 && getAircraftIcaoCodesCount() > 0; - } - - int CIcaoDataReader::getCountriesCount() const - { - QReadLocker l(&m_lockCountry); - return m_countries.size(); - } - - void CIcaoDataReader::ps_read(BlackMisc::Network::CEntityFlags::Entity entities, const QDateTime &newerThan) - { - this->threadAssertCheck(); // runs in background thread - - CEntityFlags::Entity entitiesTriggered = CEntityFlags::NoEntity; - if (entities.testFlag(CEntityFlags::AircraftIcaoEntity)) - { - CUrl url(getAircraftIcaoUrl()); - if (!url.isEmpty()) - { - if (!newerThan.isNull()) { url.appendQuery("newer=" + newerThan.toString(Qt::ISODate)); } - sApp->getFromNetwork(url, { this, &CIcaoDataReader::ps_parseAircraftIcaoData }); - entitiesTriggered |= CEntityFlags::AircraftIcaoEntity; - } - else - { - CLogMessage(this).error("No URL for %1") << CEntityFlags::flagToString(CEntityFlags::AircraftIcaoEntity); - } - } - - if (entities.testFlag(CEntityFlags::AirlineIcaoEntity)) - { - CUrl url(getAirlineIcaoUrl()); - if (!url.isEmpty()) - { - if (!newerThan.isNull()) { url.appendQuery("newer=" + newerThan.toString(Qt::ISODate)); } - sApp->getFromNetwork(url, { this, &CIcaoDataReader::ps_parseAirlineIcaoData }); - entitiesTriggered |= CEntityFlags::AirlineIcaoEntity; - } - else - { - CLogMessage(this).error("No URL for %1") << CEntityFlags::flagToString(CEntityFlags::AirlineIcaoEntity); - } - } - - if (entities.testFlag(CEntityFlags::CountryEntity)) - { - CUrl url(getCountryUrl()); - if (!url.isEmpty()) - { - if (!newerThan.isNull()) { url.appendQuery("newer=" + newerThan.toString(Qt::ISODate)); } - sApp->getFromNetwork(url, { this, &CIcaoDataReader::ps_parseCountryData }); - entitiesTriggered |= CEntityFlags::CountryEntity; - } - else - { - CLogMessage(this).error("No URL for %1") << CEntityFlags::flagToString(CEntityFlags::CountryEntity); - } - } - - if (entitiesTriggered != CEntityFlags::NoEntity) - { - emit dataRead(entitiesTriggered, CEntityFlags::StartRead, 0); - } - } - - CUrl CIcaoDataReader::getBaseUrl() const - { - const CUrl baseUrl(sApp->getGlobalSetup().getDbIcaoReaderUrl()); - return baseUrl; - } - - void CIcaoDataReader::ps_parseAircraftIcaoData(QNetworkReply *nwReplyPtr) - { - // wrap pointer, make sure any exit cleans up reply - // required to use delete later as object is created in a different thread - QScopedPointer nwReply(nwReplyPtr); - QString urlString(nwReply->url().toString()); - CDatabaseReader::JsonDatastoreResponse res = this->setStatusAndTransformReplyIntoDatastoreResponse(nwReply.data()); - if (res.hasErrorMessage()) - { - CLogMessage::preformatted(res.lastWarningOrAbove()); - emit dataRead(CEntityFlags::AircraftIcaoEntity, CEntityFlags::ReadFailed, 0); - return; - } - CAircraftIcaoCodeList codes = CAircraftIcaoCodeList::fromDatabaseJson(res); - - // this part needs to be synchronized - int n = codes.size(); - { - QWriteLocker wl(&this->m_lockAircraft); - this->m_aircraftIcaos = codes; - } - emit dataRead(CEntityFlags::AircraftIcaoEntity, CEntityFlags::ReadFinished, n); - CLogMessage(this).info("Read %1 %2 from %3") << n << CEntityFlags::flagToString(CEntityFlags::AircraftIcaoEntity) << urlString; - } - - void CIcaoDataReader::ps_parseAirlineIcaoData(QNetworkReply *nwReplyPtr) - { - QScopedPointer nwReply(nwReplyPtr); - QString urlString(nwReply->url().toString()); - CDatabaseReader::JsonDatastoreResponse res = this->setStatusAndTransformReplyIntoDatastoreResponse(nwReply.data()); - if (res.hasErrorMessage()) - { - CLogMessage::preformatted(res.lastWarningOrAbove()); - emit dataRead(CEntityFlags::AirlineIcaoEntity, CEntityFlags::ReadFailed, 0); - return; - } - CAirlineIcaoCodeList codes = CAirlineIcaoCodeList::fromDatabaseJson(res); - - // this part needs to be synchronized - int n = codes.size(); - { - QWriteLocker wl(&this->m_lockAirline); - this->m_airlineIcaos = codes; - } - emit dataRead(CEntityFlags::AirlineIcaoEntity, CEntityFlags::ReadFinished, n); - CLogMessage(this).info("Read %1 %2 from %3") << n << CEntityFlags::flagToString(CEntityFlags::AirlineIcaoEntity) << urlString; - } - - void CIcaoDataReader::ps_parseCountryData(QNetworkReply *nwReplyPtr) - { - QScopedPointer nwReply(nwReplyPtr); - QString urlString(nwReply->url().toString()); - CDatabaseReader::JsonDatastoreResponse res = this->setStatusAndTransformReplyIntoDatastoreResponse(nwReply.data()); - if (res.hasErrorMessage()) - { - CLogMessage::preformatted(res.lastWarningOrAbove()); - emit dataRead(CEntityFlags::CountryEntity, CEntityFlags::ReadFailed, 0); - return; - } - CCountryList countries = CCountryList::fromDatabaseJson(res); - - // this part needs to be synchronized - int n = m_countries.size(); - { - QWriteLocker wl(&this->m_lockCountry); - this->m_countries = countries; - } - emit dataRead(CEntityFlags::CountryEntity, CEntityFlags::ReadFinished, n); - CLogMessage(this).info("Read %1 %2 from %3") << n << CEntityFlags::flagToString(CEntityFlags::CountryEntity) << urlString; - } - - bool CIcaoDataReader::readFromJsonFiles(const QString &dir, CEntityFlags::Entity whatToRead) - { - QDir directory(dir); - if (!directory.exists()) { return false; } - - CEntityFlags::Entity reallyRead = CEntityFlags::NoEntity; - if (whatToRead.testFlag(CEntityFlags::CountryEntity)) - { - QString countriesJson(CFileUtils::readFileToString(CFileUtils::appendFilePaths(directory.absolutePath(), "countries.json"))); - if (!countriesJson.isEmpty()) - { - CCountryList countries; - countries.convertFromJson(Json::jsonObjectFromString(countriesJson)); - int c = countries.size(); - { - QWriteLocker l(&m_lockCountry); - m_countries = countries; - } - // Do not emit while locked -> deadlock - reallyRead |= CEntityFlags::CountryEntity; - emit dataRead(CEntityFlags::CountryEntity, CEntityFlags::ReadFinished, c); - } - } - - if (whatToRead.testFlag(CEntityFlags::AircraftIcaoEntity)) - { - QString aircraftJson(CFileUtils::readFileToString(CFileUtils::appendFilePaths(directory.absolutePath(), "aircrafticao.json"))); - if (!aircraftJson.isEmpty()) - { - CAircraftIcaoCodeList aircraftIcaos; - aircraftIcaos.convertFromJson(Json::jsonObjectFromString(aircraftJson)); - int c = aircraftIcaos.size(); - { - QWriteLocker l(&m_lockAircraft); - m_aircraftIcaos = aircraftIcaos; - } - reallyRead |= CEntityFlags::AircraftIcaoEntity; - emit dataRead(CEntityFlags::AircraftIcaoEntity, CEntityFlags::ReadFinished, c); - } - } - - if (whatToRead.testFlag(CEntityFlags::AirlineIcaoEntity)) - { - QString airlineJson(CFileUtils::readFileToString(CFileUtils::appendFilePaths(directory.absolutePath(), "airlineicao.json"))); - if (!airlineJson.isEmpty()) - { - CAirlineIcaoCodeList airlineIcaos; - airlineIcaos.convertFromJson(Json::jsonObjectFromString(airlineJson)); - int c = airlineIcaos.size(); - { - QWriteLocker l(&m_lockAirline); - m_airlineIcaos = airlineIcaos; - } - reallyRead |= CEntityFlags::AirlineIcaoEntity; - emit dataRead(CEntityFlags::AirlineIcaoEntity, CEntityFlags::ReadFinished, c); - } - } - return (whatToRead & CEntityFlags::AllIcaoAndCountries) == reallyRead; - } - - bool CIcaoDataReader::readFromJsonFilesInBackground(const QString &dir, CEntityFlags::Entity whatToRead) - { - if (dir.isEmpty() || whatToRead == CEntityFlags::NoEntity) { return false; } - QTimer::singleShot(0, this, [this, dir, whatToRead]() - { - bool s = this->readFromJsonFiles(dir, whatToRead); - Q_UNUSED(s); - }); - return true; - } - - bool CIcaoDataReader::writeToJsonFiles(const QString &dir) const - { - QDir directory(dir); - if (!directory.exists()) { return false; } - if (this->getCountriesCount() > 0) - { - QString json(QJsonDocument(this->getCountries().toJson()).toJson()); - bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), "countries.json")); - if (!s) { return false; } - } - - if (this->getAircraftIcaoCodesCount() > 0) - { - QString json(QJsonDocument(this->getAircraftIcaoCodes().toJson()).toJson()); - bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), "aircrafticao.json")); - if (!s) { return false; } - } - - if (this->getAirlineIcaoCodesCount() > 0) - { - QString json(QJsonDocument(this->getAirlineIcaoCodes().toJson()).toJson()); - bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), "airlineicao.json")); - if (!s) { return false; } - } - return true; - } - - CUrl CIcaoDataReader::getAircraftIcaoUrl() const - { - return getBaseUrl().withAppendedPath("service/jsonaircrafticao.php"); - } - - CUrl CIcaoDataReader::getAirlineIcaoUrl() const - { - return getBaseUrl().withAppendedPath("service/jsonairlineicao.php"); - } - - CUrl CIcaoDataReader::getCountryUrl() const - { - return getBaseUrl().withAppendedPath("service/jsoncountry.php"); - } - -} // namespace diff --git a/src/blackcore/icaodatareader.h b/src/blackcore/icaodatareader.h deleted file mode 100644 index a5d92ff9d..000000000 --- a/src/blackcore/icaodatareader.h +++ /dev/null @@ -1,153 +0,0 @@ -/* Copyright (C) 2015 - * swift project Community / Contributors - * - * This file is part of swift project. It is subject to the license terms in the LICENSE file found in the top-level - * directory of this distribution and at http://www.swift-project.org/license.html. No part of swift project, - * including this file, may be copied, modified, propagated, or distributed except according to the terms - * contained in the LICENSE file. - */ - -//! \file - -#ifndef BLACKCORE_ICAODATAREADER_H -#define BLACKCORE_ICAODATAREADER_H - -#include "blackcore/blackcoreexport.h" -#include "blackcore/databasereader.h" -#include "blackmisc/aviation/aircrafticaocode.h" -#include "blackmisc/aviation/aircrafticaocodelist.h" -#include "blackmisc/aviation/airlineicaocode.h" -#include "blackmisc/aviation/airlineicaocodelist.h" -#include "blackmisc/country.h" -#include "blackmisc/countrylist.h" -#include "blackmisc/network/entityflags.h" -#include "blackmisc/network/url.h" - -#include -#include -#include - -class QDateTime; -class QNetworkReply; - -namespace BlackCore -{ - //! Read ICAO data from Database - class BLACKCORE_EXPORT CIcaoDataReader : public CDatabaseReader - { - Q_OBJECT - - public: - //! Constructor - explicit CIcaoDataReader(QObject *owner); - - //! Get aircraft ICAO information - //! \threadsafe - BlackMisc::Aviation::CAircraftIcaoCodeList getAircraftIcaoCodes() const; - - //! Get aircraft ICAO information for designator - //! \threadsafe - BlackMisc::Aviation::CAircraftIcaoCode getAircraftIcaoCodeForDesignator(const QString &designator) const; - - //! Get aircraft ICAO information for key - //! \threadsafe - BlackMisc::Aviation::CAircraftIcaoCode getAircraftIcaoCodeForDbKey(int key) const; - - //! Get airline ICAO information - //! \threadsafe - BlackMisc::Aviation::CAirlineIcaoCodeList getAirlineIcaoCodes() const; - - //! Get best match for incomplete aircraft ICAO code - //! \threadsafe - BlackMisc::Aviation::CAircraftIcaoCode smartAircraftIcaoSelector(const BlackMisc::Aviation::CAircraftIcaoCode &icaoPattern) const; - - //! Get countries - //! \threadsafe - BlackMisc::CCountryList getCountries() const; - - //! Get countries count - //! \threadsafe - int getCountriesCount() const; - - //! Get country for ISO code - //! \threadsafe - BlackMisc::CCountry getCountryForIsoCode(const QString &isoCode) const; - - //! Get country for ISO name - //! \threadsafe - BlackMisc::CCountry getCountryForName(const QString &name) const; - - //! Get airline ICAO information for designator - //! \threadsafe - BlackMisc::Aviation::CAirlineIcaoCodeList getAirlineIcaoCodesForDesignator(const QString &designator) const; - - //! Get airline ICAO information for key - //! \threadsafe - BlackMisc::Aviation::CAirlineIcaoCode getAirlineIcaoCodeForDbKey(int key) const; - - //! Get best match for incomplete airline ICAO code - //! \threadsafe - BlackMisc::Aviation::CAirlineIcaoCode smartAirlineIcaoSelector(const BlackMisc::Aviation::CAirlineIcaoCode &icaoPattern) const; - - //! Get aircraft ICAO information count - //! \threadsafe - int getAircraftIcaoCodesCount() const; - - //! Get airline ICAO information count - //! \threadsafe - int getAirlineIcaoCodesCount() const; - - //! All data read? - //! \threadsafe - bool areAllDataRead() const; - - //! Read from static DB data file - bool readFromJsonFiles(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead = BlackMisc::Network::CEntityFlags::AllIcaoAndCountries); - - //! Read from static DB data file - bool readFromJsonFilesInBackground(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead = BlackMisc::Network::CEntityFlags::AllIcaoAndCountries); - - //! Write to static DB data file - bool writeToJsonFiles(const QString &dir) const; - - signals: - //! Combined read signal - void dataRead(BlackMisc::Network::CEntityFlags::Entity entity, BlackMisc::Network::CEntityFlags::ReadState state, int number); - - private slots: - //! Aircraft have been read - void ps_parseAircraftIcaoData(QNetworkReply *nwReply); - - //! Airlines have been read - void ps_parseAirlineIcaoData(QNetworkReply *nwReply); - - //! Airlines have been read - void ps_parseCountryData(QNetworkReply *nwReply); - - //! Read / re-read data file - void ps_read(BlackMisc::Network::CEntityFlags::Entity entities, const QDateTime &newerThan); - - private: - BlackMisc::Aviation::CAircraftIcaoCodeList m_aircraftIcaos; - BlackMisc::Aviation::CAirlineIcaoCodeList m_airlineIcaos; - BlackMisc::CCountryList m_countries; - - mutable QReadWriteLock m_lockAirline; - mutable QReadWriteLock m_lockAircraft; - mutable QReadWriteLock m_lockCountry; - - //! Base URL - BlackMisc::Network::CUrl getBaseUrl() const; - - //! URL - BlackMisc::Network::CUrl getAircraftIcaoUrl() const; - - //! URL - BlackMisc::Network::CUrl getAirlineIcaoUrl() const; - - //! URL - BlackMisc::Network::CUrl getCountryUrl() const; - }; -} - -#endif // guard diff --git a/src/blackcore/webdataservices.cpp b/src/blackcore/webdataservices.cpp index 621a8f170..9f15a6b55 100644 --- a/src/blackcore/webdataservices.cpp +++ b/src/blackcore/webdataservices.cpp @@ -9,9 +9,12 @@ #include "blackcore/application.h" #include "blackcore/data/globalsetup.h" -#include "blackcore/databasewriter.h" -#include "blackcore/icaodatareader.h" -#include "blackcore/modeldatareader.h" +#include "blackcore/db/infodatareader.h" +#include "blackcore/db/icaodatareader.h" +#include "blackcore/db/databasewriter.h" +#include "blackcore/db/icaodatareader.h" +#include "blackcore/db/modeldatareader.h" +#include "blackcore/setupreader.h" #include "blackcore/vatsimbookingreader.h" #include "blackcore/vatsimdatafilereader.h" #include "blackcore/vatsimmetarreader.h" @@ -34,7 +37,9 @@ #include using namespace BlackCore; +using namespace BlackCore::Db; using namespace BlackCore::Data; +using namespace BlackCore::Db; using namespace BlackMisc; using namespace BlackMisc::Simulation; using namespace BlackMisc::Network; @@ -43,8 +48,8 @@ using namespace BlackMisc::Weather; namespace BlackCore { - CWebDataServices::CWebDataServices(CWebReaderFlags::WebReader readerFlags, CWebReaderFlags::DbReaderHint hint, BlackMisc::Restricted, QObject *parent) : - QObject(parent), m_readerFlags(readerFlags), m_dbHint(hint) + CWebDataServices::CWebDataServices(CWebReaderFlags::WebReader readerFlags, const CDatabaseReaderConfigList &dbReaderConfig, BlackMisc::Restricted, QObject *parent) : + QObject(parent), m_readerFlags(readerFlags), m_dbReaderConfig(dbReaderConfig) { if (!sApp) { return; } // shutting down @@ -54,44 +59,14 @@ namespace BlackCore this->initReaders(readerFlags); this->initWriters(); - bool c = false; - Q_UNUSED(c); - if (m_readerFlags.testFlag(CWebReaderFlags::WebReaderFlag::IcaoDataReader)) - { - Q_ASSERT_X(this->m_icaoDataReader, Q_FUNC_INFO, "Missing reader ICAO"); - c = connect(this->m_icaoDataReader, &CIcaoDataReader::dataRead, this, &CWebDataServices::dataRead); - Q_ASSERT_X(c, Q_FUNC_INFO, "connect failed ICAO"); - } + const bool withInfoData = m_readerFlags.testFlag(CWebReaderFlags::WebReaderFlag::InfoDataReader); + CEntityFlags::Entity entities = CEntityFlags::AllEntities; + entities ^= CEntityFlags::InfoObjectEntity; // 2 liner because of gcc error: invalid conversion from 'int' to 'BlackMisc::Network::CEntityFlags::EntityFlag' + if (withInfoData) { CLogMessage(this).info("Using info objects for swift DB objects"); } - if (m_readerFlags.testFlag(CWebReaderFlags::WebReaderFlag::ModelReader)) - { - Q_ASSERT_X(this->m_modelDataReader, Q_FUNC_INFO, "Missing reader models"); - c = connect(this->m_modelDataReader, &CModelDataReader::dataRead, this, &CWebDataServices::dataRead); - Q_ASSERT_X(c, Q_FUNC_INFO, "connect failed models"); - } - - if (m_readerFlags.testFlag(CWebReaderFlags::WebReaderFlag::VatsimBookingReader)) - { - Q_ASSERT_X(this->m_vatsimBookingReader, Q_FUNC_INFO, "Missing reader bookings"); - c = connect(this->m_vatsimBookingReader, &CVatsimBookingReader::dataRead, this, &CWebDataServices::dataRead); - Q_ASSERT_X(c, Q_FUNC_INFO, "connect failed bookings"); - } - - if (m_readerFlags.testFlag(CWebReaderFlags::WebReaderFlag::VatsimDataReader)) - { - Q_ASSERT_X(this->m_vatsimDataFileReader, Q_FUNC_INFO, "Missing reader data file"); - c = connect(this->m_vatsimDataFileReader, &CVatsimDataFileReader::dataRead, this, &CWebDataServices::dataRead); - Q_ASSERT_X(c, Q_FUNC_INFO, "connect failed VATSIM data file"); - } - - if (m_readerFlags.testFlag(CWebReaderFlags::WebReaderFlag::VatsimMetarReader)) - { - Q_ASSERT_X(this->m_vatsimMetarReader, Q_FUNC_INFO, "Missing reader metars"); - c = connect(this->m_vatsimMetarReader, &CVatsimMetarReader::dataRead, this, &CWebDataServices::dataRead); - Q_ASSERT_X(c, Q_FUNC_INFO, "connect failed VATSIM METAR"); - } - - this->readInBackground(CEntityFlags::AllEntities, 500); + // make sure this is called in event queue, so pending tasks cam be performed + // important so info objects can be read + this->singleShotReadInBackground(entities, 1000); } CServerList CWebDataServices::getVatsimFsdServers() const @@ -149,10 +124,14 @@ namespace BlackCore bool CWebDataServices::canConnectSwiftDb() const { - if (!m_icaoDataReader && !m_modelDataReader) { return false; } + if (!m_icaoDataReader && !m_modelDataReader && !m_infoDataReader) { return false; } // use the first one to test - if (m_icaoDataReader) + if (m_infoDataReader) + { + return m_infoDataReader->canConnect(); + } + else if (m_icaoDataReader) { return m_icaoDataReader->canConnect(); } @@ -165,7 +144,8 @@ namespace BlackCore CEntityFlags::Entity CWebDataServices::triggerRead(CEntityFlags::Entity whatToRead, const QDateTime &newerThan) { - m_initialRead = true; + m_initialRead = true; // read started + Q_ASSERT_X(!whatToRead.testFlag(CEntityFlags::InfoObjectEntity), Q_FUNC_INFO, "Info object must be read upfront"); CEntityFlags::Entity triggeredRead = CEntityFlags::NoEntity; if (m_vatsimDataFileReader) { @@ -417,6 +397,7 @@ namespace BlackCore if (this->m_vatsimMetarReader) { this->m_vatsimMetarReader->gracefulShutdown(); } if (this->m_modelDataReader) { this->m_modelDataReader->gracefulShutdown(); } if (this->m_icaoDataReader) { this->m_icaoDataReader->gracefulShutdown(); } + if (this->m_infoDataReader) { this->m_infoDataReader->gracefulShutdown(); } if (this->m_databaseWriter) { this->m_databaseWriter->gracefulShutdown(); } } @@ -428,7 +409,29 @@ namespace BlackCore void CWebDataServices::initReaders(CWebReaderFlags::WebReader flags) { - // 1. Status file, updating the cache + // ---- "metadata" reader, 1/2 will trigger read directly during init + + // 1. If any DB data, read the info upfront + const bool anyDbData = flags.testFlag(CWebReaderFlags::WebReaderFlag::IcaoDataReader) || flags.testFlag(CWebReaderFlags::WebReaderFlag::ModelReader); + const CDatabaseReaderConfigList dbReaderConfig(this->m_dbReaderConfig); + bool c = false; // signal connect + Q_UNUSED(c); + + if (anyDbData && flags.testFlag(CWebReaderFlags::WebReaderFlag::InfoDataReader)) + { + this->m_infoDataReader = new CInfoDataReader(this, dbReaderConfig); + c = connect(this->m_infoDataReader, &CInfoDataReader::dataRead, this, &CWebDataServices::ps_readFromSwiftDb); + Q_ASSERT_X(c, Q_FUNC_INFO, "ICAO info object signals"); + c = connect(this->m_infoDataReader, &CInfoDataReader::dataRead, this, &CWebDataServices::dataRead); + Q_ASSERT_X(c, Q_FUNC_INFO, "connect failed info data"); + + // info data reader has a special role, it will not be triggered in triggerRead() + this->m_infoDataReader->start(QThread::LowPriority); + // directly call read + QTimer::singleShot(0, [this]() { this->m_infoDataReader->read(CEntityFlags::InfoObjectEntity, QDateTime()); }); + } + + // 2. Status file, updating the VATSIM related caches if (flags.testFlag(CWebReaderFlags::WebReaderFlag::VatsimDataReader) || flags.testFlag(CWebReaderFlags::WebReaderFlag::VatsimMetarReader)) { this->m_vatsimStatusReader = new CVatsimStatusFileReader(this); @@ -437,57 +440,63 @@ namespace BlackCore QTimer::singleShot(100, this->m_vatsimStatusReader, &CVatsimStatusFileReader::readInBackgroundThread); } - // 2. VATSIM bookings + // ---- "normal data", triggerRead will start read, not starting directly + + // 3. VATSIM bookings if (flags.testFlag(CWebReaderFlags::WebReaderFlag::VatsimBookingReader)) { this->m_vatsimBookingReader = new CVatsimBookingReader(this); - bool c = connect(this->m_vatsimBookingReader, &CVatsimBookingReader::atcBookingsRead, this, &CWebDataServices::ps_receivedBookings); + c = connect(this->m_vatsimBookingReader, &CVatsimBookingReader::atcBookingsRead, this, &CWebDataServices::ps_receivedBookings); Q_ASSERT_X(c, Q_FUNC_INFO, "VATSIM booking reader signals"); - Q_UNUSED(c); + c = connect(this->m_vatsimBookingReader, &CVatsimBookingReader::dataRead, this, &CWebDataServices::dataRead); + Q_ASSERT_X(c, Q_FUNC_INFO, "connect failed bookings"); this->m_vatsimBookingReader->start(QThread::LowPriority); this->m_vatsimBookingReader->setInterval(3 * 60 * 1000); } - // 3. VATSIM data file + // 4. VATSIM data file if (flags.testFlag(CWebReaderFlags::WebReaderFlag::VatsimDataReader)) { this->m_vatsimDataFileReader = new CVatsimDataFileReader(this); - bool c = connect(this->m_vatsimDataFileReader, &CVatsimDataFileReader::dataFileRead, this, &CWebDataServices::ps_dataFileRead); + c = connect(this->m_vatsimDataFileReader, &CVatsimDataFileReader::dataFileRead, this, &CWebDataServices::ps_dataFileRead); Q_ASSERT_X(c, Q_FUNC_INFO, "VATSIM data reader signals"); - Q_UNUSED(c); + c = connect(this->m_vatsimDataFileReader, &CVatsimDataFileReader::dataRead, this, &CWebDataServices::dataRead); + Q_ASSERT_X(c, Q_FUNC_INFO, "connect failed VATSIM data file"); this->m_vatsimDataFileReader->start(QThread::LowPriority); this->m_vatsimDataFileReader->setInterval(90 * 1000); } - // 4. VATSIM metar data + // 5. VATSIM metar data if (flags.testFlag(CWebReaderFlags::WebReaderFlag::VatsimMetarReader)) { this->m_vatsimMetarReader = new CVatsimMetarReader(this); - bool c = connect(this->m_vatsimMetarReader, &CVatsimMetarReader::metarsRead, this, &CWebDataServices::ps_receivedMetars); + c = connect(this->m_vatsimMetarReader, &CVatsimMetarReader::metarsRead, this, &CWebDataServices::ps_receivedMetars); Q_ASSERT_X(c, Q_FUNC_INFO, "VATSIM METAR reader signals"); - Q_UNUSED(c); + c = connect(this->m_vatsimMetarReader, &CVatsimMetarReader::dataRead, this, &CWebDataServices::dataRead); + Q_ASSERT_X(c, Q_FUNC_INFO, "connect failed VATSIM METAR"); this->m_vatsimMetarReader->start(QThread::LowPriority); this->m_vatsimMetarReader->setInterval(90 * 1000); } - // 5. ICAO data reader + // 6. ICAO data reader if (flags.testFlag(CWebReaderFlags::WebReaderFlag::IcaoDataReader)) { - bool c; - this->m_icaoDataReader = new CIcaoDataReader(this); + this->m_icaoDataReader = new CIcaoDataReader(this, dbReaderConfig); c = connect(this->m_icaoDataReader, &CIcaoDataReader::dataRead, this, &CWebDataServices::ps_readFromSwiftDb); Q_ASSERT_X(c, Q_FUNC_INFO, "ICAO reader signals"); - Q_UNUSED(c); + c = connect(this->m_icaoDataReader, &CIcaoDataReader::dataRead, this, &CWebDataServices::dataRead); + Q_ASSERT_X(c, Q_FUNC_INFO, "Cannot connect ICAO reader signals"); this->m_icaoDataReader->start(QThread::LowPriority); } - // 6. Model reader + // 7. Model reader if (flags.testFlag(CWebReaderFlags::WebReaderFlag::ModelReader)) { - this->m_modelDataReader = new CModelDataReader(this); - bool c = connect(this->m_modelDataReader, &CModelDataReader::dataRead, this, &CWebDataServices::ps_readFromSwiftDb); + this->m_modelDataReader = new CModelDataReader(this, dbReaderConfig); + c = connect(this->m_modelDataReader, &CModelDataReader::dataRead, this, &CWebDataServices::ps_readFromSwiftDb); Q_ASSERT_X(c, Q_FUNC_INFO, "Model reader signals"); - Q_UNUSED(c); + c = connect(this->m_modelDataReader, &CModelDataReader::dataRead, this, &CWebDataServices::dataRead); + Q_ASSERT_X(c, Q_FUNC_INFO, "connect failed models"); this->m_modelDataReader->start(QThread::LowPriority); } } @@ -540,15 +549,56 @@ namespace BlackCore // void } + void CWebDataServices::singleShotReadInBackground(CEntityFlags::Entity entities, int delayMs) + { + QTimer::singleShot(delayMs, [ = ]() + { + this->readInBackground(entities, delayMs); + }); + } + void CWebDataServices::readInBackground(CEntityFlags::Entity entities, int delayMs) { - m_initialRead = true; + m_initialRead = true; // read started + + const int waitForInfoObjects = 1000; // ms + const int maxWaitCycles = 6; + + // with info objects wait until info objects are loaded + Q_ASSERT_X(!entities.testFlag(CEntityFlags::InfoObjectEntity), Q_FUNC_INFO, "Info object must be read upfront"); + if (this->m_infoDataReader && CEntityFlags::anySwiftDbEntity(entities)) + { + // try to read + if (this->m_infoObjectTrials > maxWaitCycles) + { + CLogMessage(this).error("Cannot read info objects from %1") << this->m_infoDataReader->getInfoObjectsUrl().toQString(); + } + else if (this->m_infoDataReader->canConnect()) + { + // read, but no idea if succesful/failure + if (this->m_infoDataReader->getDbInfoObjectCount() > 0) + { + CLogMessage(this).info("Info objects loaded from %1") << this->m_infoDataReader->getInfoObjectsUrl().toQString(); + } + else + { + CLogMessage(this).error("Info objects loading failed from %1, '%2'") + << this->m_infoDataReader->getInfoObjectsUrl().toQString() + << this->m_infoDataReader->getStatusMessage(); + } + } + else + { + // postpone by some time + this->m_infoObjectTrials++; + this->singleShotReadInBackground(entities, waitForInfoObjects); + return; + } + } + if (delayMs > 100) { - BlackMisc::singleShot(delayMs, QThread::currentThread(), [ = ]() - { - this->readInBackground(entities, 0); - }); + this->singleShotReadInBackground(entities, 0); } else { @@ -615,5 +665,4 @@ namespace BlackCore } return s; } - } // ns diff --git a/src/blackcore/webdataservices.h b/src/blackcore/webdataservices.h index 2652be576..8336e6624 100644 --- a/src/blackcore/webdataservices.h +++ b/src/blackcore/webdataservices.h @@ -14,6 +14,7 @@ #include "blackcore/blackcoreexport.h" #include "blackcore/webreaderflags.h" +#include "blackcore/db/databasereader.h" #include "blackmisc/aviation/aircrafticaocode.h" #include "blackmisc/aviation/aircrafticaocodelist.h" #include "blackmisc/aviation/airlineicaocode.h" @@ -48,6 +49,8 @@ namespace BlackMisc { class CLogCategoryList; + template class Restricted; + namespace Aviation { class CCallsign; } namespace Simulation { class CSimulatedAircraft; } } @@ -55,14 +58,19 @@ namespace BlackMisc namespace BlackCore { class CApplication; - class CDatabaseWriter; - class CIcaoDataReader; - class CModelDataReader; class CVatsimBookingReader; class CVatsimDataFileReader; class CVatsimMetarReader; class CVatsimStatusFileReader; + namespace Db + { + class CDatabaseWriter; + class CIcaoDataReader; + class CModelDataReader; + class CInfoDataReader; + } + /*! * Encapsulates reading data from web sources */ @@ -76,7 +84,7 @@ namespace BlackCore static const BlackMisc::CLogCategoryList &getLogCategories(); //! Constructor, only allowed from BlackCore::CApplication - CWebDataServices(CWebReaderFlags::WebReader readerFlags, CWebReaderFlags:: DbReaderHint hint, BlackMisc::Restricted, QObject *parent = nullptr); + CWebDataServices(CWebReaderFlags::WebReader readerFlags, const BlackCore::Db::CDatabaseReaderConfigList &dbReaderConfig, BlackMisc::Restricted, QObject *parent = nullptr); //! Shutdown void gracefulShutdown(); @@ -93,15 +101,15 @@ namespace BlackCore //! Metar reader CVatsimMetarReader *getMetarReader() const { return m_vatsimMetarReader; } + //! Info data reader + Db::CInfoDataReader *getInfoDataReader() const { return m_infoDataReader; } + //! DB writer class - CDatabaseWriter *getDatabaseWriter() const { return m_databaseWriter; } + Db::CDatabaseWriter *getDatabaseWriter() const { return m_databaseWriter; } //! Reader flags CWebReaderFlags::WebReader getReaderFlags() const { return m_readerFlags; } - //! Reader flags - CWebReaderFlags::DbReaderHint getDbHint() const { return m_dbHint; } - //! FSD servers //! \threadsafe BlackMisc::Network::CServerList getVatsimFsdServers() const; @@ -295,7 +303,7 @@ namespace BlackCore //! Data file has been read void ps_dataFileRead(int lines); - //! Read from model reader + //! Read finished from reader void ps_readFromSwiftDb(BlackMisc::Network::CEntityFlags::Entity entity, BlackMisc::Network::CEntityFlags::ReadState state, int number); //! Setup changed @@ -308,20 +316,25 @@ namespace BlackCore //! Init the writers void initWriters(); - CWebReaderFlags::WebReader m_readerFlags = CWebReaderFlags::WebReaderFlag::None; //!< which readers are available - CWebReaderFlags::DbReaderHint m_dbHint = CWebReaderFlags::NoHint; //!< how to read DB data - bool m_initialRead = false; //!< Initial read conducted + //! Call CWebDataServices::readInBackground by single shot + void singleShotReadInBackground(BlackMisc::Network::CEntityFlags::Entity entities, int delayMs); + + CWebReaderFlags::WebReader m_readerFlags = CWebReaderFlags::WebReaderFlag::None; //!< which readers are available + BlackCore::Db::CDatabaseReaderConfigList m_dbReaderConfig; //!< how to read DB data + bool m_initialRead = false; //!< Initial read started + int m_infoObjectTrials = 0; //!< Tried to read info objects // for reading XML and VATSIM data files CVatsimStatusFileReader *m_vatsimStatusReader = nullptr; CVatsimBookingReader *m_vatsimBookingReader = nullptr; CVatsimDataFileReader *m_vatsimDataFileReader = nullptr; CVatsimMetarReader *m_vatsimMetarReader = nullptr; - CIcaoDataReader *m_icaoDataReader = nullptr; - CModelDataReader *m_modelDataReader = nullptr; + Db::CIcaoDataReader *m_icaoDataReader = nullptr; + Db::CModelDataReader *m_modelDataReader = nullptr; + Db::CInfoDataReader *m_infoDataReader = nullptr; // writing objects directly into DB - CDatabaseWriter *m_databaseWriter = nullptr; + Db::CDatabaseWriter *m_databaseWriter = nullptr; }; } // namespace