flightsimulatorplatfrom/pilotclient
1
0
Fork 0
mirror of https://github.com/swift-project/pilotclient.git synced 2026-04-14 16:55:36 +08:00
Code Issues Packages Projects Releases Wiki Activity

refs #649, refs #656, using caches in ICAO data reader

Browse Source 
* access to info data reader (get DB metadata)
* adjusted database base class to support caches, info objects
* moved classes to subdir
This commit is contained in:
Klaus Basan
2016-05-24 00:36:06 +02:00
parent 6b7e05077c
commit 954ddfb2e7
10 changed files with 1171 additions and 938 deletions
Download Patch File Download Diff File Expand all files Collapse all files

186
src/blackcore/databasereader.cpp
View File

@@ -1,186 +0,0 @@
/* Copyright (C) 2015
* swift project Community / Contributors
*
* This file is part of swift project. It is subject to the license terms in the LICENSE file found in the top-level
* directory of this distribution and at http://www.swift-project.org/license.html. No part of swift project,
* including this file, may be copied, modified, propagated, or distributed except according to the terms
* contained in the LICENSE file.
*/
#include "blackcore/databasereader.h"
#include "blackmisc/datastoreutility.h"
#include "blackmisc/logcategory.h"
#include "blackmisc/logcategorylist.h"
#include "blackmisc/network/networkutils.h"
#include <QByteArray>
#include <QJsonDocument>
#include <QJsonObject>
#include <QJsonValueRef>
#include <QMetaObject>
#include <QNetworkReply>
#include <QReadLocker>
#include <QUrl>
#include <QWriteLocker>
using namespace BlackMisc;
using namespace BlackMisc::Network;
namespace BlackCore
{
CDatabaseReader::CDatabaseReader(QObject *owner, const QString &name) :
BlackMisc::CThreadedReader(owner, name)
{
connect(&m_watchdogTimer, &QTimer::timeout, this, &CDatabaseReader::ps_watchdog);
}
const CLogCategoryList &CDatabaseReader::getLogCategories()
{
static const BlackMisc::CLogCategoryList cats { BlackMisc::CLogCategory::swiftDbWebservice(), BlackMisc::CLogCategory::mapping() };
return cats;
}
void CDatabaseReader::readInBackgroundThread(CEntityFlags::Entity entities, const QDateTime &newerThan)
{
if (isAbandoned()) { return; }
this->m_watchdogTimer.stop();
// ps_read is implemented in the derived classes
bool s = QMetaObject::invokeMethod(this, "ps_read",
Q_ARG(BlackMisc::Network::CEntityFlags::Entity, entities),
Q_ARG(QDateTime, newerThan));
Q_ASSERT_X(s, Q_FUNC_INFO, "Invoke failed");
Q_UNUSED(s);
}
CDatabaseReader::JsonDatastoreResponse CDatabaseReader::transformReplyIntoDatastoreResponse(QNetworkReply *nwReply) const
{
this->threadAssertCheck();
static const CLogCategoryList cats(CLogCategoryList(this).join({ CLogCategory::webservice()}));
JsonDatastoreResponse datastoreResponse;
if (this->isAbandoned())
{
nwReply->abort();
datastoreResponse.setMessage(CStatusMessage(cats, CStatusMessage::SeverityError, "Terminated data parsing process"));
return datastoreResponse; // stop, terminate straight away, ending thread
}
if (nwReply->error() == QNetworkReply::NoError)
{
const QString dataFileData = nwReply->readAll().trimmed();
nwReply->close(); // close asap
if (dataFileData.isEmpty())
{
datastoreResponse.setMessage(CStatusMessage(cats, CStatusMessage::SeverityError, "Empty response, no data"));
datastoreResponse.m_updated = QDateTime::currentDateTimeUtc();
return datastoreResponse;
}
QJsonDocument jsonResponse = QJsonDocument::fromJson(dataFileData.toUtf8());
if (jsonResponse.isArray())
{
// directly an array, no further info
datastoreResponse.m_jsonArray = jsonResponse.array();
datastoreResponse.m_updated = QDateTime::currentDateTimeUtc();
}
else
{
QJsonObject responseObject(jsonResponse.object());
datastoreResponse.m_jsonArray = responseObject["data"].toArray();
QString ts(responseObject["latest"].toString());
datastoreResponse.m_updated = ts.isEmpty() ? QDateTime::currentDateTimeUtc() : CDatastoreUtility::parseTimestamp(ts);
datastoreResponse.m_restricted = responseObject["restricted"].toBool();
}
return datastoreResponse;
}
// no valid response
QString error(nwReply->errorString());
QString url(nwReply->url().toString());
nwReply->abort();
datastoreResponse.setMessage(CStatusMessage(cats, CStatusMessage::SeverityError,
QString("Reading data failed: " + error + " " + url)));
return datastoreResponse;
}
CDatabaseReader::JsonDatastoreResponse CDatabaseReader::setStatusAndTransformReplyIntoDatastoreResponse(QNetworkReply *nwReply)
{
setConnectionStatus(nwReply);
return transformReplyIntoDatastoreResponse(nwReply);
}
void CDatabaseReader::setWatchdogUrl(const CUrl &url)
{
bool start;
{
QWriteLocker wl(&this->m_watchdogLock);
m_watchdogUrl = url;
start = url.isEmpty();
}
if (start)
{
m_watchdogTimer.start(30 * 1000);
}
else
{
m_watchdogTimer.stop();
}
}
bool CDatabaseReader::canConnect() const
{
QReadLocker rl(&this->m_watchdogLock);
return m_canConnect;
}
bool CDatabaseReader::canConnect(QString &message) const
{
QReadLocker rl(&this->m_watchdogLock);
message = m_watchdogMessage;
return m_canConnect;
}
void CDatabaseReader::setConnectionStatus(bool ok, const QString &message)
{
{
QWriteLocker wl(&this->m_watchdogLock);
this->m_watchdogMessage = message;
this->m_canConnect = ok;
if (this->m_watchdogUrl.isEmpty()) { return; }
}
this->m_updateTimer->start(); // restart
}
void CDatabaseReader::setConnectionStatus(QNetworkReply *nwReply)
{
Q_ASSERT_X(nwReply, Q_FUNC_INFO, "Missing network reply");
if (nwReply->isFinished())
{
if (nwReply->error() == QNetworkReply::NoError)
{
setConnectionStatus(true);
}
else
{
setConnectionStatus(false, nwReply->errorString());
}
}
}
void CDatabaseReader::ps_watchdog()
{
CUrl url;
{
QReadLocker rl(&this->m_watchdogLock);
url = this->m_watchdogUrl;
}
if (url.isEmpty())
{
this->m_watchdogTimer.stop();
return;
}
QString m;
bool ok = CNetworkUtils::canConnect(url, m);
this->setConnectionStatus(ok, m);
}
} // namespace

139
src/blackcore/databasereader.h
View File

@@ -1,139 +0,0 @@
/* Copyright (C) 2015
* swift project Community / Contributors
*
* This file is part of swift project. It is subject to the license terms in the LICENSE file found in the top-level
* directory of this distribution and at http://www.swift-project.org/license.html. No part of swift project,
* including this file, may be copied, modified, propagated, or distributed except according to the terms
* contained in the LICENSE file.
*/
//! \file
#ifndef BLACKCORE_DATABASE_READER_H
#define BLACKCORE_DATABASE_READER_H
#include "blackcore/blackcoreexport.h"
#include "blackmisc/network/entityflags.h"
#include "blackmisc/network/url.h"
#include "blackmisc/statusmessage.h"
#include "blackmisc/threadedreader.h"
#include <QDateTime>
#include <QJsonArray>
#include <QObject>
#include <QReadWriteLock>
#include <QString>
#include <QTimer>
#include <QtGlobal>
class QNetworkReply;
namespace BlackMisc { class CLogCategoryList; }
namespace BlackCore
{
//! Specialized version of threaded reader for DB data
class BLACKCORE_EXPORT CDatabaseReader : public BlackMisc::CThreadedReader
{
Q_OBJECT
public:
//! Response from our database
struct JsonDatastoreResponse
{
QJsonArray m_jsonArray; //!< JSON array data
QDateTime m_updated; //!< when was the latest updated?
bool m_restricted = false; //!< restricted reponse, only data changed
BlackMisc::CStatusMessage m_message; //!< last error or warning
//! Any data?
bool isEmpty() const { return m_jsonArray.isEmpty(); }
//! Number of elements
int size() const { return m_jsonArray.size(); }
//! Any timestamp?
bool hasTimestamp() const { return m_updated.isValid(); }
//! Is response newer?
bool isNewer(const QDateTime &ts) const { return m_updated.toMSecsSinceEpoch() > ts.toMSecsSinceEpoch(); }
//! Is response newer?
bool isNewer(qint64 mSecsSinceEpoch) const { return m_updated.toMSecsSinceEpoch() > mSecsSinceEpoch; }
//! Incremental data
bool isRestricted() const { return m_restricted; }
//! Error message?
bool hasErrorMessage() const { return m_message.getSeverity() == BlackMisc::CStatusMessage::SeverityError; }
//! Warning or error message?
bool hasWarningOrAboveMessage() const { return m_message.isWarningOrAbove(); }
//! Last error or warning
const BlackMisc::CStatusMessage &lastWarningOrAbove() const { return m_message; }
//! Set the error/warning message
void setMessage(const BlackMisc::CStatusMessage &lastErrorOrWarning) { m_message = lastErrorOrWarning; }
//! Get the JSON array
QJsonArray getJsonArray() const { return m_jsonArray; }
//! Set the JSON array
void setJsonArray(const QJsonArray &value) { m_jsonArray = value; }
//! Implicit conversion
operator QJsonArray() const { return m_jsonArray; }
};
//! Log categories
static const BlackMisc::CLogCategoryList &getLogCategories();
//! Start reading in own thread
void readInBackgroundThread(BlackMisc::Network::CEntityFlags::Entity entities, const QDateTime &newerThan);
//! Can connect to DB
//! \threadsafe
bool canConnect() const;
//! Can connect to server?
//! \return message why connect failed
//! \threadsafe
bool canConnect(QString &message) const;
protected:
BlackMisc::Network::CUrl m_watchdogUrl; //!< URL for checking if alive
QTimer m_watchdogTimer { this }; //!< Timer for watchdog (DB available?)
QString m_watchdogMessage; //!< Returned status message from watchdog
bool m_canConnect = false; //!< Successful connection?
mutable QReadWriteLock m_watchdogLock; //!< Lock
//! Constructor
CDatabaseReader(QObject *owner, const QString &name);
//! Watchdog URL, empty means no checking
//! \threadsafe
void setWatchdogUrl(const BlackMisc::Network::CUrl &url);
//! Check if terminated or error, otherwise split into array of objects
CDatabaseReader::JsonDatastoreResponse setStatusAndTransformReplyIntoDatastoreResponse(QNetworkReply *nwReply);
private slots:
//! Watchdog checking if DB is available
void ps_watchdog();
private:
//! Check if terminated or error, otherwise split into array of objects
JsonDatastoreResponse transformReplyIntoDatastoreResponse(QNetworkReply *nwReply) const;
//! Feedback about connection status
//! \threadsafe
void setConnectionStatus(bool ok, const QString &message = "");
//! Feedback about connection status
//! \threadsafe
void setConnectionStatus(QNetworkReply *nwReply);
};
} // namespace
#endif // guard

250
src/blackcore/db/databasereader.cpp Normal file
View File

@@ -0,0 +1,250 @@
/* Copyright (C) 2015
* swift project Community / Contributors
*
* This file is part of swift project. It is subject to the license terms in the LICENSE file found in the top-level
* directory of this distribution and at http://www.swift-project.org/license.html. No part of swift project,
* including this file, may be copied, modified, propagated, or distributed except according to the terms
* contained in the LICENSE file.
*/
#include "blackcore/db/databasereader.h"
#include "blackcore/db/infodatareader.h"
#include "blackcore/webdataservices.h"
#include "blackcore/application.h"
#include "blackmisc/db/datastoreutility.h"
#include "blackmisc/network/networkutils.h"
#include "blackmisc/logcategory.h"
#include "blackmisc/logcategorylist.h"
#include "blackmisc/logmessage.h"
#include <QByteArray>
#include <QJsonDocument>
#include <QJsonObject>
#include <QJsonValueRef>
#include <QMetaObject>
#include <QNetworkReply>
#include <QReadLocker>
#include <QUrl>
#include <QWriteLocker>
using namespace BlackMisc;
using namespace BlackMisc::Db;
using namespace BlackMisc::Network;
using namespace BlackCore;
namespace BlackCore
{
namespace Db
{
CDatabaseReader::CDatabaseReader(QObject *owner, const CDatabaseReaderConfigList &config, const QString &name) :
BlackMisc::CThreadedReader(owner, name), m_config(config)
{
this->m_sharedUrl = sApp->getGlobalSetup().getSwiftSharedUrls().getRandomWorkingUrl();
}
void CDatabaseReader::readInBackgroundThread(CEntityFlags::Entity entities, const QDateTime &newerThan)
{
if (isAbandoned()) { return; }
// we accept cached cached data
Q_ASSERT_X(!entities.testFlag(CEntityFlags::InfoObjectEntity), Q_FUNC_INFO, "Read info objects directly");
CEntityFlags::Entity allEntities = entities;
CEntityFlags::Entity currentEntity = CEntityFlags::iterateDbEntities(allEntities); // CEntityFlags::InfoObjectEntity will be ignored
const bool hasInfoObjects = this->hasInfoObjects();
while (currentEntity)
{
const CDatabaseReaderConfig config(this->getConfigForEntity(currentEntity));
if (config.getRetrievalMode().testFlag(CDbFlags::Cached))
{
if (hasInfoObjects)
{
const QDateTime cacheTs(this->getCacheTimestamp(currentEntity));
const QDateTime latestEntityTs(this->getLatestEntityTimestamp(currentEntity));
const qint64 cacheTimestamp = cacheTs.isValid() ? cacheTs.toMSecsSinceEpoch() : -1;
const qint64 latestEntityTimestamp = latestEntityTs.isValid() ? latestEntityTs.toMSecsSinceEpoch() : -1;
Q_ASSERT_X(latestEntityTimestamp > 0, Q_FUNC_INFO, "Missing timestamp");
if (cacheTimestamp >= latestEntityTimestamp && cacheTimestamp > 0)
{
this->syncronizeCaches(currentEntity);
entities &= ~currentEntity; // do not load from web
CLogMessage(this).info("Using cache for %1 (%2, %3)")
<< CEntityFlags::flagToString(currentEntity)
<< cacheTs.toString() << cacheTimestamp;
}
else
{
CLogMessage(this).info("Cache for %1 outdated, latest entity (%2, %3)")
<< CEntityFlags::flagToString(currentEntity)
<< latestEntityTs.toString() << latestEntityTimestamp;
}
}
else
{
this->syncronizeCaches(currentEntity);
CLogMessage(this).info("No info object for %1, using cache") << CEntityFlags::flagToString(currentEntity);
}
}
currentEntity = CEntityFlags::iterateDbEntities(allEntities);
}
// ps_read is implemented in the derived classes
if (entities == CEntityFlags::NoEntity) { return; }
const bool s = QMetaObject::invokeMethod(this, "ps_read",
Q_ARG(BlackMisc::Network::CEntityFlags::Entity, entities),
Q_ARG(QDateTime, newerThan));
Q_ASSERT_X(s, Q_FUNC_INFO, "Invoke failed");
Q_UNUSED(s);
}
CDatabaseReader::JsonDatastoreResponse CDatabaseReader::transformReplyIntoDatastoreResponse(QNetworkReply *nwReply) const
{
this->threadAssertCheck();
static const CLogCategoryList cats(CLogCategoryList(this).join({ CLogCategory::webservice()}));
JsonDatastoreResponse datastoreResponse;
if (this->isAbandoned())
{
nwReply->abort();
datastoreResponse.setMessage(CStatusMessage(cats, CStatusMessage::SeverityError, "Terminated data parsing process"));
return datastoreResponse; // stop, terminate straight away, ending thread
}
if (nwReply->error() == QNetworkReply::NoError)
{
const QString dataFileData = nwReply->readAll().trimmed();
nwReply->close(); // close asap
if (dataFileData.isEmpty())
{
datastoreResponse.setMessage(CStatusMessage(cats, CStatusMessage::SeverityError, "Empty response, no data"));
datastoreResponse.m_updated = QDateTime::currentDateTimeUtc();
return datastoreResponse;
}
QJsonDocument jsonResponse = QJsonDocument::fromJson(dataFileData.toUtf8());
if (jsonResponse.isArray())
{
// directly an array, no further info
datastoreResponse.m_jsonArray = jsonResponse.array();
datastoreResponse.m_updated = QDateTime::currentDateTimeUtc();
}
else
{
QJsonObject responseObject(jsonResponse.object());
datastoreResponse.m_jsonArray = responseObject["data"].toArray();
QString ts(responseObject["latest"].toString());
datastoreResponse.m_updated = ts.isEmpty() ? QDateTime::currentDateTimeUtc() : CDatastoreUtility::parseTimestamp(ts);
datastoreResponse.m_restricted = responseObject["restricted"].toBool();
}
return datastoreResponse;
}
// no valid response
QString error(nwReply->errorString());
QString url(nwReply->url().toString());
nwReply->abort();
datastoreResponse.setMessage(CStatusMessage(cats, CStatusMessage::SeverityError,
QString("Reading data failed: " + error + " " + url)));
return datastoreResponse;
}
CDatabaseReader::JsonDatastoreResponse CDatabaseReader::setStatusAndTransformReplyIntoDatastoreResponse(QNetworkReply *nwReply)
{
this->setConnectionStatus(nwReply);
return this->transformReplyIntoDatastoreResponse(nwReply);
}
CDbInfoList CDatabaseReader::infoList() const
{
static const CDbInfoList e;
if (!sApp->hasWebDataServices()) { return e; }
if (!sApp->getWebDataServices()->getInfoDataReader()) { return e; }
return sApp->getWebDataServices()->getInfoDataReader()->getDbInfoObjects();
}
bool CDatabaseReader::hasInfoObjects() const
{
return infoList().size() > 0;
}
QDateTime CDatabaseReader::getLatestEntityTimestamp(CEntityFlags::Entity entity) const
{
static const QDateTime e;
const CDbInfoList il(infoList());
if (il.isEmpty() || entity == CEntityFlags::NoEntity) { return e; }
CDbInfo info = il.findFirstByEntityOrDefault(entity);
if (!info.isValid()) { return e; }
return info.getUtcTimestamp();
}
CDatabaseReaderConfig CDatabaseReader::getConfigForEntity(CEntityFlags::Entity entity) const
{
return this->m_config.findFirstOrDefaultForEntity(entity);
}
bool CDatabaseReader::canConnect() const
{
QReadLocker rl(&this->m_statusLock);
return m_canConnect;
}
bool CDatabaseReader::canConnect(QString &message) const
{
QReadLocker rl(&this->m_statusLock);
message = m_statusMessage;
return m_canConnect;
}
CUrl CDatabaseReader::getWorkingSharedUrl() const
{
return this->m_sharedUrl;
}
const QString &CDatabaseReader::getStatusMessage() const
{
return this->m_statusMessage;
}
void CDatabaseReader::setConnectionStatus(bool ok, const QString &message)
{
{
QWriteLocker wl(&this->m_statusLock);
this->m_statusMessage = message;
this->m_canConnect = ok;
}
}
void CDatabaseReader::setConnectionStatus(QNetworkReply *nwReply)
{
Q_ASSERT_X(nwReply, Q_FUNC_INFO, "Missing network reply");
if (nwReply->isFinished())
{
if (nwReply->error() == QNetworkReply::NoError)
{
setConnectionStatus(true);
}
else
{
setConnectionStatus(false, nwReply->errorString());
}
}
}
const CLogCategoryList &CDatabaseReader::getLogCategories()
{
static const BlackMisc::CLogCategoryList cats { BlackMisc::CLogCategory::swiftDbWebservice(), BlackMisc::CLogCategory::mapping() };
return cats;
}
const QString &CDatabaseReader::parameterLatestTimestamp()
{
static const QString p("latestTimestamp");
return p;
}
const QString &CDatabaseReader::parameterLatestId()
{
static const QString p("latestId");
return p;
}
} // ns
} // ns

172
src/blackcore/db/databasereader.h Normal file
View File

@@ -0,0 +1,172 @@
/* Copyright (C) 2015
* swift project Community / Contributors
*
* This file is part of swift project. It is subject to the license terms in the LICENSE file found in the top-level
* directory of this distribution and at http://www.swift-project.org/license.html. No part of swift project,
* including this file, may be copied, modified, propagated, or distributed except according to the terms
* contained in the LICENSE file.
*/
//! \file
#ifndef BLACKCORE_DB_DATABASEREADER_H
#define BLACKCORE_DB_DATABASEREADER_H
#include "blackcore/blackcoreexport.h"
#include "blackcore/db/databasereaderconfig.h"
#include "blackmisc/db/dbinfolist.h"
#include "blackmisc/pq/time.h"
#include "blackmisc/network/url.h"
#include "blackmisc/statusmessage.h"
#include "blackmisc/threadedreader.h"
#include "blackmisc/sequence.h"
#include "blackmisc/valueobject.h"
#include <QDateTime>
#include <QJsonArray>
#include <QObject>
#include <QReadWriteLock>
#include <QString>
#include <QTimer>
#include <QtGlobal>
class QNetworkReply;
namespace BlackMisc { class CLogCategoryList; }
namespace BlackCore
{
namespace Db
{
//! Specialized version of threaded reader for DB data
class BLACKCORE_EXPORT CDatabaseReader : public BlackMisc::CThreadedReader
{
Q_OBJECT
public:
//! Response from our database
struct JsonDatastoreResponse
{
QJsonArray m_jsonArray; //!< JSON array data
QDateTime m_updated; //!< when was the latest updated?
bool m_restricted = false; //!< restricted reponse, only data changed
BlackMisc::CStatusMessage m_message; //!< last error or warning
//! Any data?
bool isEmpty() const { return m_jsonArray.isEmpty(); }
//! Number of elements
int size() const { return m_jsonArray.size(); }
//! Any timestamp?
bool hasTimestamp() const { return m_updated.isValid(); }
//! Is response newer?
bool isNewer(const QDateTime &ts) const { return m_updated.toMSecsSinceEpoch() > ts.toMSecsSinceEpoch(); }
//! Is response newer?
bool isNewer(qint64 mSecsSinceEpoch) const { return m_updated.toMSecsSinceEpoch() > mSecsSinceEpoch; }
//! Incremental data
bool isRestricted() const { return m_restricted; }
//! Error message?
bool hasErrorMessage() const { return m_message.getSeverity() == BlackMisc::CStatusMessage::SeverityError; }
//! Warning or error message?
bool hasWarningOrAboveMessage() const { return m_message.isWarningOrAbove(); }
//! Last error or warning
const BlackMisc::CStatusMessage &lastWarningOrAbove() const { return m_message; }
//! Set the error/warning message
void setMessage(const BlackMisc::CStatusMessage &lastErrorOrWarning) { m_message = lastErrorOrWarning; }
//! Get the JSON array
QJsonArray getJsonArray() const { return m_jsonArray; }
//! Set the JSON array
void setJsonArray(const QJsonArray &value) { m_jsonArray = value; }
//! Implicit conversion
operator QJsonArray() const { return m_jsonArray; }
};
//! Start reading in own thread
void readInBackgroundThread(BlackMisc::Network::CEntityFlags::Entity entities, const QDateTime &newerThan);
//! Can connect to DB
//! \threadsafe
bool canConnect() const;
//! Can connect to server?
//! \return message why connect failed
//! \threadsafe
bool canConnect(QString &message) const;
//! Obtain a working shared URL
BlackMisc::Network::CUrl getWorkingSharedUrl() const;
//! Status message (error message)
const QString &getStatusMessage() const;
//! Log categories
static const BlackMisc::CLogCategoryList &getLogCategories();
//! Name of latest timestamp
static const QString &parameterLatestTimestamp();
//! Name of parameter for latest id
static const QString &parameterLatestId();
protected:
CDatabaseReaderConfigList m_config; //!< DB reder configuration
BlackMisc::Network::CUrl m_sharedUrl; //!< URL for checking if alive
QString m_statusMessage; //!< Returned status message from watchdog
bool m_canConnect = false; //!< Successful connection?
mutable QReadWriteLock m_statusLock; //!< Lock
//! Constructor
CDatabaseReader(QObject *owner, const CDatabaseReaderConfigList &config, const QString &name);
//! Check if terminated or error, otherwise split into array of objects
CDatabaseReader::JsonDatastoreResponse setStatusAndTransformReplyIntoDatastoreResponse(QNetworkReply *nwReply);
//! Info list (latest data timestamp)
//! \sa BlackCore::Db::CInfoDataReader
BlackMisc::Db::CDbInfoList infoList() const;
//! Info objects available?
bool hasInfoObjects() const;
//! Obtain latest object timestamp
//! \sa BlackCore::Db::CInfoDataReader
QDateTime getLatestEntityTimestamp(BlackMisc::Network::CEntityFlags::Entity entity) const;
//! Config for given entity
CDatabaseReaderConfig getConfigForEntity(BlackMisc::Network::CEntityFlags::Entity entity) const;
//! Syncronize caches for given entities
virtual void syncronizeCaches(BlackMisc::Network::CEntityFlags::Entity entities) = 0;
//! Cache`s timestamp for given entity
virtual QDateTime getCacheTimestamp(BlackMisc::Network::CEntityFlags::Entity entities) = 0;
//! Invalidate the caches for given entities
virtual void invalidateCaches(BlackMisc::Network::CEntityFlags::Entity entities) = 0;
private:
//! Check if terminated or error, otherwise split into array of objects
JsonDatastoreResponse transformReplyIntoDatastoreResponse(QNetworkReply *nwReply) const;
//! Feedback about connection status
//! \threadsafe
void setConnectionStatus(bool ok, const QString &message = "");
//! Feedback about connection status
//! \threadsafe
void setConnectionStatus(QNetworkReply *nwReply);
};
} // ns
} // ns
#endif // guard

432
src/blackcore/db/icaodatareader.cpp Normal file
View File

@@ -0,0 +1,432 @@
/* Copyright (C) 2015
* swift project Community / Contributors
*
* This file is part of swift project. It is subject to the license terms in the LICENSE file found in the top-level
* directory of this distribution and at http://www.swift-project.org/license.html. No part of swift project,
* including this file, may be copied, modified, propagated, or distributed except according to the terms
* contained in the LICENSE file.
*/
#include "blackcore/application.h"
#include "blackcore/data/globalsetup.h"
#include "blackcore/db/icaodatareader.h"
#include "blackmisc/fileutils.h"
#include "blackmisc/json.h"
#include "blackmisc/logmessage.h"
#include "blackmisc/statusmessage.h"
#include <QDateTime>
#include <QDir>
#include <QFlags>
#include <QJsonDocument>
#include <QNetworkReply>
#include <QReadLocker>
#include <QScopedPointer>
#include <QScopedPointerDeleteLater>
#include <QTimer>
#include <QUrl>
#include <QWriteLocker>
#include <Qt>
#include <QtGlobal>
using namespace BlackMisc;
using namespace BlackMisc::Aviation;
using namespace BlackMisc::Network;
using namespace BlackCore::Data;
namespace BlackCore
{
namespace Db
{
CIcaoDataReader::CIcaoDataReader(QObject *owner, const CDatabaseReaderConfigList &confg) :
CDatabaseReader(owner, confg, "CIcaoDataReader")
{
// void
}
CAircraftIcaoCodeList CIcaoDataReader::getAircraftIcaoCodes() const
{
return m_aircraftIcaoCache.getCopy();
}
CAircraftIcaoCode CIcaoDataReader::getAircraftIcaoCodeForDesignator(const QString &designator) const
{
return getAircraftIcaoCodes().findFirstByDesignatorAndRank(designator);
}
CAircraftIcaoCode CIcaoDataReader::getAircraftIcaoCodeForDbKey(int key) const
{
return getAircraftIcaoCodes().findByKey(key);
}
CAirlineIcaoCodeList CIcaoDataReader::getAirlineIcaoCodes() const
{
return m_airlineIcaoCache.getCopy();
}
CAircraftIcaoCode CIcaoDataReader::smartAircraftIcaoSelector(const CAircraftIcaoCode &icaoPattern) const
{
CAircraftIcaoCodeList codes(getAircraftIcaoCodes()); // thread safe copy
return codes.smartAircraftIcaoSelector(icaoPattern); // sorted by rank
}
CCountryList CIcaoDataReader::getCountries() const
{
return m_countryCache.getCopy();
}
CCountry CIcaoDataReader::getCountryForIsoCode(const QString &isoCode) const
{
return this->getCountries().findByIsoCode(isoCode);
}
CCountry CIcaoDataReader::getCountryForName(const QString &name) const
{
return this->getCountries().findBestMatchByCountryName(name);
}
CAirlineIcaoCodeList CIcaoDataReader::getAirlineIcaoCodesForDesignator(const QString &designator) const
{
return this->getAirlineIcaoCodes().findByVDesignator(designator);
}
CAirlineIcaoCode CIcaoDataReader::getAirlineIcaoCodeForDbKey(int key) const
{
return this->getAirlineIcaoCodes().findByKey(key);
}
CAirlineIcaoCode CIcaoDataReader::smartAirlineIcaoSelector(const CAirlineIcaoCode &icaoPattern) const
{
CAirlineIcaoCodeList codes(this->getAirlineIcaoCodes()); // thread safe copy
return codes.smartAirlineIcaoSelector(icaoPattern);
}
int CIcaoDataReader::getAircraftIcaoCodesCount() const
{
return this->getAircraftIcaoCodes().size();
}
int CIcaoDataReader::getAirlineIcaoCodesCount() const
{
return this->getAirlineIcaoCodes().size();
}
bool CIcaoDataReader::areAllDataRead() const
{
return getCountriesCount() > 0 && getAirlineIcaoCodesCount() > 0 && getAircraftIcaoCodesCount() > 0;
}
int CIcaoDataReader::getCountriesCount() const
{
return this->getCountries().size();
}
void CIcaoDataReader::ps_read(BlackMisc::Network::CEntityFlags::Entity entities, const QDateTime &newerThan)
{
this->threadAssertCheck(); // runs in background thread
CEntityFlags::Entity entitiesTriggered = CEntityFlags::NoEntity;
if (entities.testFlag(CEntityFlags::AircraftIcaoEntity))
{
CUrl url(getAircraftIcaoUrl());
if (!url.isEmpty())
{
if (!newerThan.isNull()) { url.appendQuery("newer=" + newerThan.toString(Qt::ISODate)); }
sApp->getFromNetwork(url, { this, &CIcaoDataReader::ps_parseAircraftIcaoData });
entitiesTriggered |= CEntityFlags::AircraftIcaoEntity;
}
else
{
CLogMessage(this).error("No URL for %1") << CEntityFlags::flagToString(CEntityFlags::AircraftIcaoEntity);
}
}
if (entities.testFlag(CEntityFlags::AirlineIcaoEntity))
{
CUrl url(getAirlineIcaoUrl());
if (!url.isEmpty())
{
if (!newerThan.isNull()) { url.appendQuery("newer=" + newerThan.toString(Qt::ISODate)); }
sApp->getFromNetwork(url, { this, &CIcaoDataReader::ps_parseAirlineIcaoData });
entitiesTriggered |= CEntityFlags::AirlineIcaoEntity;
}
else
{
CLogMessage(this).error("No URL for %1") << CEntityFlags::flagToString(CEntityFlags::AirlineIcaoEntity);
}
}
if (entities.testFlag(CEntityFlags::CountryEntity))
{
CUrl url(getCountryUrl());
if (!url.isEmpty())
{
if (!newerThan.isNull()) { url.appendQuery("newer=" + newerThan.toString(Qt::ISODate)); }
sApp->getFromNetwork(url, { this, &CIcaoDataReader::ps_parseCountryData });
entitiesTriggered |= CEntityFlags::CountryEntity;
}
else
{
CLogMessage(this).error("No URL for %1") << CEntityFlags::flagToString(CEntityFlags::CountryEntity);
}
}
if (entitiesTriggered != CEntityFlags::NoEntity)
{
emit dataRead(entitiesTriggered, CEntityFlags::StartRead, 0);
}
}
void CIcaoDataReader::ps_aircraftIcaoCacheChanged()
{
// void
}
void CIcaoDataReader::ps_airlineIcaoCacheChanged()
{
// void
}
void CIcaoDataReader::ps_countryCacheChanged()
{
// void
}
void CIcaoDataReader::ps_baseUrlCacheChanged()
{
// void
}
void CIcaoDataReader::updateReaderUrl(const CUrl &url)
{
const CUrl current = this->m_readerUrlCache.getCopy();
if (current == url) { return; }
const CStatusMessage m = this->m_readerUrlCache.set(url);
if (m.isFailure())
{
CLogMessage::preformatted(m);
}
}
CUrl CIcaoDataReader::getBaseUrl() const
{
const CUrl baseUrl(sApp->getGlobalSetup().getDbIcaoReaderUrl());
return baseUrl;
}
void CIcaoDataReader::ps_parseAircraftIcaoData(QNetworkReply *nwReplyPtr)
{
// wrap pointer, make sure any exit cleans up reply
// required to use delete later as object is created in a different thread
QScopedPointer<QNetworkReply, QScopedPointerDeleteLater> nwReply(nwReplyPtr);
QString urlString(nwReply->url().toString());
CDatabaseReader::JsonDatastoreResponse res = this->setStatusAndTransformReplyIntoDatastoreResponse(nwReply.data());
if (res.hasErrorMessage())
{
CLogMessage::preformatted(res.lastWarningOrAbove());
emit dataRead(CEntityFlags::AircraftIcaoEntity, CEntityFlags::ReadFailed, 0);
return;
}
const CAircraftIcaoCodeList codes = CAircraftIcaoCodeList::fromDatabaseJson(res);
const int n = codes.size();
qint64 latestTimestamp = codes.latestTimestampMsecsSinceEpoch();
if (n > 0 && latestTimestamp < 0)
{
CLogMessage(this).error("No timestamp in aircraft ICAO list, setting to last modified value");
latestTimestamp = lastModifiedMsSinceEpoch(nwReply.data());
}
this->m_aircraftIcaoCache.set(codes, latestTimestamp);
this->updateReaderUrl(this->getBaseUrl());
emit dataRead(CEntityFlags::AircraftIcaoEntity, CEntityFlags::ReadFinished, n);
CLogMessage(this).info("Read %1 %2 from %3") << n << CEntityFlags::flagToString(CEntityFlags::AircraftIcaoEntity) << urlString;
}
void CIcaoDataReader::ps_parseAirlineIcaoData(QNetworkReply *nwReplyPtr)
{
QScopedPointer<QNetworkReply, QScopedPointerDeleteLater> nwReply(nwReplyPtr);
QString urlString(nwReply->url().toString());
CDatabaseReader::JsonDatastoreResponse res = this->setStatusAndTransformReplyIntoDatastoreResponse(nwReply.data());
if (res.hasErrorMessage())
{
CLogMessage::preformatted(res.lastWarningOrAbove());
emit dataRead(CEntityFlags::AirlineIcaoEntity, CEntityFlags::ReadFailed, 0);
return;
}
const CAirlineIcaoCodeList codes = CAirlineIcaoCodeList::fromDatabaseJson(res);
const int n = codes.size();
qint64 latestTimestamp = codes.latestTimestampMsecsSinceEpoch();
if (n > 0 && latestTimestamp < 0)
{
CLogMessage(this).error("No timestamp in airline ICAO list, setting to last modified value");
latestTimestamp = lastModifiedMsSinceEpoch(nwReply.data());
}
this->m_airlineIcaoCache.set(codes, latestTimestamp);
emit dataRead(CEntityFlags::AirlineIcaoEntity, CEntityFlags::ReadFinished, n);
CLogMessage(this).info("Read %1 %2 from %3") << n << CEntityFlags::flagToString(CEntityFlags::AirlineIcaoEntity) << urlString;
}
void CIcaoDataReader::ps_parseCountryData(QNetworkReply *nwReplyPtr)
{
QScopedPointer<QNetworkReply, QScopedPointerDeleteLater> nwReply(nwReplyPtr);
QString urlString(nwReply->url().toString());
CDatabaseReader::JsonDatastoreResponse res = this->setStatusAndTransformReplyIntoDatastoreResponse(nwReply.data());
if (res.hasErrorMessage())
{
CLogMessage::preformatted(res.lastWarningOrAbove());
emit dataRead(CEntityFlags::CountryEntity, CEntityFlags::ReadFailed, 0);
return;
}
const CCountryList countries = CCountryList::fromDatabaseJson(res);
const int n = countries.size();
qint64 latestTimestamp = countries.latestTimestampMsecsSinceEpoch();
if (n > 0 && latestTimestamp < 0)
{
CLogMessage(this).error("No timestamp in country list, setting to last modified value");
latestTimestamp = lastModifiedMsSinceEpoch(nwReply.data());
}
this->m_countryCache.set(countries, latestTimestamp);
emit dataRead(CEntityFlags::CountryEntity, CEntityFlags::ReadFinished, n);
CLogMessage(this).info("Read %1 %2 from %3") << n << CEntityFlags::flagToString(CEntityFlags::CountryEntity) << urlString;
}
bool CIcaoDataReader::readFromJsonFiles(const QString &dir, CEntityFlags::Entity whatToRead)
{
QDir directory(dir);
if (!directory.exists()) { return false; }
CEntityFlags::Entity reallyRead = CEntityFlags::NoEntity;
if (whatToRead.testFlag(CEntityFlags::CountryEntity))
{
QString countriesJson(CFileUtils::readFileToString(CFileUtils::appendFilePaths(directory.absolutePath(), "countries.json")));
if (!countriesJson.isEmpty())
{
CCountryList countries;
countries.convertFromJson(Json::jsonObjectFromString(countriesJson));
const int c = countries.size();
this->m_countryCache.set(countries);
// Do not emit while locked -> deadlock
reallyRead |= CEntityFlags::CountryEntity;
emit dataRead(CEntityFlags::CountryEntity, CEntityFlags::ReadFinished, c);
}
}
if (whatToRead.testFlag(CEntityFlags::AircraftIcaoEntity))
{
QString aircraftJson(CFileUtils::readFileToString(CFileUtils::appendFilePaths(directory.absolutePath(), "aircrafticao.json")));
if (!aircraftJson.isEmpty())
{
CAircraftIcaoCodeList aircraftIcaos;
aircraftIcaos.convertFromJson(Json::jsonObjectFromString(aircraftJson));
const int c = aircraftIcaos.size();
this->m_aircraftIcaoCache.set(aircraftIcaos);
reallyRead |= CEntityFlags::AircraftIcaoEntity;
emit dataRead(CEntityFlags::AircraftIcaoEntity, CEntityFlags::ReadFinished, c);
}
}
if (whatToRead.testFlag(CEntityFlags::AirlineIcaoEntity))
{
QString airlineJson(CFileUtils::readFileToString(CFileUtils::appendFilePaths(directory.absolutePath(), "airlineicao.json")));
if (!airlineJson.isEmpty())
{
CAirlineIcaoCodeList airlineIcaos;
airlineIcaos.convertFromJson(Json::jsonObjectFromString(airlineJson));
const int c = airlineIcaos.size();
this->m_airlineIcaoCache.set(airlineIcaos);
reallyRead |= CEntityFlags::AirlineIcaoEntity;
emit dataRead(CEntityFlags::AirlineIcaoEntity, CEntityFlags::ReadFinished, c);
}
}
return (whatToRead & CEntityFlags::AllIcaoAndCountries) == reallyRead;
}
bool CIcaoDataReader::readFromJsonFilesInBackground(const QString &dir, CEntityFlags::Entity whatToRead)
{
if (dir.isEmpty() || whatToRead == CEntityFlags::NoEntity) { return false; }
QTimer::singleShot(0, this, [this, dir, whatToRead]()
{
bool s = this->readFromJsonFiles(dir, whatToRead);
Q_UNUSED(s);
});
return true;
}
bool CIcaoDataReader::writeToJsonFiles(const QString &dir) const
{
QDir directory(dir);
if (!directory.exists()) { return false; }
if (this->getCountriesCount() > 0)
{
QString json(QJsonDocument(this->getCountries().toJson()).toJson());
bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), "countries.json"));
if (!s) { return false; }
}
if (this->getAircraftIcaoCodesCount() > 0)
{
QString json(QJsonDocument(this->getAircraftIcaoCodes().toJson()).toJson());
bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), "aircrafticao.json"));
if (!s) { return false; }
}
if (this->getAirlineIcaoCodesCount() > 0)
{
QString json(QJsonDocument(this->getAirlineIcaoCodes().toJson()).toJson());
bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), "airlineicao.json"));
if (!s) { return false; }
}
return true;
}
void CIcaoDataReader::syncronizeCaches(CEntityFlags::Entity entities)
{
if (entities.testFlag(CEntityFlags::AircraftIcaoEntity)) { this->m_aircraftIcaoCache.synchronize(); }
if (entities.testFlag(CEntityFlags::AirlineIcaoEntity)) { this->m_airlineIcaoCache.synchronize(); }
if (entities.testFlag(CEntityFlags::CountryEntity)) { this->m_countryCache.synchronize(); }
}
void CIcaoDataReader::invalidateCaches(CEntityFlags::Entity entities)
{
if (entities.testFlag(CEntityFlags::AircraftIcaoEntity)) { CDataCache::instance()->clearAllValues(this->m_aircraftIcaoCache.getKey()); }
if (entities.testFlag(CEntityFlags::AirlineIcaoEntity)) {CDataCache::instance()->clearAllValues(this->m_airlineIcaoCache.getKey()); }
if (entities.testFlag(CEntityFlags::CountryEntity)) {CDataCache::instance()->clearAllValues(this->m_countryCache.getKey()); }
}
QDateTime CIcaoDataReader::getCacheTimestamp(CEntityFlags::Entity entity)
{
switch (entity)
{
case CEntityFlags::AircraftIcaoEntity: return this->m_aircraftIcaoCache.getTimestamp();
case CEntityFlags::AirlineIcaoEntity: return this->m_airlineIcaoCache.getTimestamp();
case CEntityFlags::CountryEntity: return this->m_countryCache.getTimestamp();
default: return QDateTime();
}
}
CUrl CIcaoDataReader::getAircraftIcaoUrl(bool shared) const
{
return shared ?
getWorkingSharedUrl().withAppendedPath("jsonaircrafticao.php") :
getBaseUrl().withAppendedPath("service/jsonaircrafticao.php");
}
CUrl CIcaoDataReader::getAirlineIcaoUrl(bool shared) const
{
return shared ?
getWorkingSharedUrl().withAppendedPath("jsonairlineicao.php") :
getBaseUrl().withAppendedPath("service/jsonairlineicao.php");
}
CUrl CIcaoDataReader::getCountryUrl(bool shared) const
{
return shared ?
getWorkingSharedUrl().withAppendedPath("jsoncountry.php") :
getBaseUrl().withAppendedPath("service/jsoncountry.php");
}
} // ns
} // ns

171
src/blackcore/db/icaodatareader.h Normal file
View File

@@ -0,0 +1,171 @@
/* Copyright (C) 2015
* swift project Community / Contributors
*
* This file is part of swift project. It is subject to the license terms in the LICENSE file found in the top-level
* directory of this distribution and at http://www.swift-project.org/license.html. No part of swift project,
* including this file, may be copied, modified, propagated, or distributed except according to the terms
* contained in the LICENSE file.
*/
//! \file
#ifndef BLACKCORE_ICAODATAREADER_H
#define BLACKCORE_ICAODATAREADER_H
#include "blackcore/blackcoreexport.h"
#include "blackcore/db/databasereader.h"
#include "blackcore/data/dbcaches.h"
#include "blackmisc/aviation/aircrafticaocode.h"
#include "blackmisc/aviation/aircrafticaocodelist.h"
#include "blackmisc/aviation/airlineicaocode.h"
#include "blackmisc/aviation/airlineicaocodelist.h"
#include "blackmisc/network/entityflags.h"
#include "blackmisc/network/url.h"
#include "blackmisc/country.h"
#include "blackmisc/countrylist.h"
#include "blackmisc/datacache.h"
#include <QObject>
#include <QReadWriteLock>
#include <QString>
class QDateTime;
class QNetworkReply;
namespace BlackCore
{
namespace Db
{
//! Read ICAO data from Database
class BLACKCORE_EXPORT CIcaoDataReader : public CDatabaseReader
{
Q_OBJECT
public:
//! Constructor
explicit CIcaoDataReader(QObject *owner, const CDatabaseReaderConfigList &confg);
//! Get aircraft ICAO information
//! \threadsafe
BlackMisc::Aviation::CAircraftIcaoCodeList getAircraftIcaoCodes() const;
//! Get aircraft ICAO information for designator
//! \threadsafe
BlackMisc::Aviation::CAircraftIcaoCode getAircraftIcaoCodeForDesignator(const QString &designator) const;
//! Get aircraft ICAO information for key
//! \threadsafe
BlackMisc::Aviation::CAircraftIcaoCode getAircraftIcaoCodeForDbKey(int key) const;
//! Get airline ICAO information
//! \threadsafe
BlackMisc::Aviation::CAirlineIcaoCodeList getAirlineIcaoCodes() const;
//! Get best match for incomplete aircraft ICAO code
//! \threadsafe
BlackMisc::Aviation::CAircraftIcaoCode smartAircraftIcaoSelector(const BlackMisc::Aviation::CAircraftIcaoCode &icaoPattern) const;
//! Get countries
//! \threadsafe
BlackMisc::CCountryList getCountries() const;
//! Get countries count
//! \threadsafe
int getCountriesCount() const;
//! Get country for ISO code
//! \threadsafe
BlackMisc::CCountry getCountryForIsoCode(const QString &isoCode) const;
//! Get country for ISO name
//! \threadsafe
BlackMisc::CCountry getCountryForName(const QString &name) const;
//! Get airline ICAO information for designator
//! \threadsafe
BlackMisc::Aviation::CAirlineIcaoCodeList getAirlineIcaoCodesForDesignator(const QString &designator) const;
//! Get airline ICAO information for key
//! \threadsafe
BlackMisc::Aviation::CAirlineIcaoCode getAirlineIcaoCodeForDbKey(int key) const;
//! Get best match for incomplete airline ICAO code
//! \threadsafe
BlackMisc::Aviation::CAirlineIcaoCode smartAirlineIcaoSelector(const BlackMisc::Aviation::CAirlineIcaoCode &icaoPattern) const;
//! Get aircraft ICAO information count
//! \threadsafe
int getAircraftIcaoCodesCount() const;
//! Get airline ICAO information count
//! \threadsafe
int getAirlineIcaoCodesCount() const;
//! All data read?
//! \threadsafe
bool areAllDataRead() const;
//! Read from static DB data file
bool readFromJsonFiles(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead = BlackMisc::Network::CEntityFlags::AllIcaoAndCountries);
//! Read from static DB data file
bool readFromJsonFilesInBackground(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead = BlackMisc::Network::CEntityFlags::AllIcaoAndCountries);
//! Write to static DB data file
bool writeToJsonFiles(const QString &dir) const;
signals:
//! Combined read signal
void dataRead(BlackMisc::Network::CEntityFlags::Entity entity, BlackMisc::Network::CEntityFlags::ReadState state, int number);
protected:
//! \name cache handling for base class
//! @{
virtual void syncronizeCaches(BlackMisc::Network::CEntityFlags::Entity entities) override;
virtual void invalidateCaches(BlackMisc::Network::CEntityFlags::Entity entities) override;
virtual QDateTime getCacheTimestamp(BlackMisc::Network::CEntityFlags::Entity entity) override;
//! @}
private slots:
//! Aircraft have been read
void ps_parseAircraftIcaoData(QNetworkReply *nwReply);
//! Airlines have been read
void ps_parseAirlineIcaoData(QNetworkReply *nwReply);
//! Airlines have been read
void ps_parseCountryData(QNetworkReply *nwReply);
//! Read / re-read data file
void ps_read(BlackMisc::Network::CEntityFlags::Entity entities, const QDateTime &newerThan);
void ps_aircraftIcaoCacheChanged();
void ps_airlineIcaoCacheChanged();
void ps_countryCacheChanged();
void ps_baseUrlCacheChanged();
private:
BlackMisc::CData<BlackCore::Data::DbAircraftIcaoCache> m_aircraftIcaoCache {this, &CIcaoDataReader::ps_aircraftIcaoCacheChanged };
BlackMisc::CData<BlackCore::Data::DbAirlineIcaoCache> m_airlineIcaoCache {this, &CIcaoDataReader::ps_airlineIcaoCacheChanged };
BlackMisc::CData<BlackCore::Data::DbCountryCache> m_countryCache {this, &CIcaoDataReader::ps_countryCacheChanged };
BlackMisc::CData<BlackCore::Data::DbIcaoReaderBaseUrl> m_readerUrlCache {this, &CIcaoDataReader::ps_baseUrlCacheChanged };
//! Update reader URL
void updateReaderUrl(const BlackMisc::Network::CUrl &url);
//! Base URL
BlackMisc::Network::CUrl getBaseUrl() const;
//! URL
BlackMisc::Network::CUrl getAircraftIcaoUrl(bool shared = false) const;
//! URL
BlackMisc::Network::CUrl getAirlineIcaoUrl(bool shared = false) const;
//! URL
BlackMisc::Network::CUrl getCountryUrl(bool shared = false) const;
};
} // ns
} // ns
#endif // guard

376
src/blackcore/icaodatareader.cpp
View File

@@ -1,376 +0,0 @@
/* Copyright (C) 2015
* swift project Community / Contributors
*
* This file is part of swift project. It is subject to the license terms in the LICENSE file found in the top-level
* directory of this distribution and at http://www.swift-project.org/license.html. No part of swift project,
* including this file, may be copied, modified, propagated, or distributed except according to the terms
* contained in the LICENSE file.
*/
#include "blackcore/application.h"
#include "blackcore/data/globalsetup.h"
#include "blackcore/icaodatareader.h"
#include "blackmisc/fileutils.h"
#include "blackmisc/json.h"
#include "blackmisc/logmessage.h"
#include "blackmisc/statusmessage.h"
#include <QDateTime>
#include <QDir>
#include <QFlags>
#include <QJsonDocument>
#include <QNetworkReply>
#include <QReadLocker>
#include <QScopedPointer>
#include <QScopedPointerDeleteLater>
#include <QTimer>
#include <QUrl>
#include <QWriteLocker>
#include <Qt>
#include <QtGlobal>
using namespace BlackMisc;
using namespace BlackMisc::Aviation;
using namespace BlackMisc::Network;
using namespace BlackCore::Data;
namespace BlackCore
{
CIcaoDataReader::CIcaoDataReader(QObject *owner) :
CDatabaseReader(owner, "CIcaoDataReader")
{
// void
}
CAircraftIcaoCodeList CIcaoDataReader::getAircraftIcaoCodes() const
{
QReadLocker l(&m_lockAircraft);
return m_aircraftIcaos;
}
CAircraftIcaoCode CIcaoDataReader::getAircraftIcaoCodeForDesignator(const QString &designator) const
{
return getAircraftIcaoCodes().findFirstByDesignatorAndRank(designator);
}
CAircraftIcaoCode CIcaoDataReader::getAircraftIcaoCodeForDbKey(int key) const
{
return getAircraftIcaoCodes().findByKey(key);
}
CAirlineIcaoCodeList CIcaoDataReader::getAirlineIcaoCodes() const
{
QReadLocker l(&m_lockAirline);
return m_airlineIcaos;
}
CAircraftIcaoCode CIcaoDataReader::smartAircraftIcaoSelector(const CAircraftIcaoCode &icaoPattern) const
{
CAircraftIcaoCodeList codes(getAircraftIcaoCodes()); // thread safe copy
return codes.smartAircraftIcaoSelector(icaoPattern); // sorted by rank
}
CCountryList CIcaoDataReader::getCountries() const
{
QReadLocker l(&m_lockCountry);
return m_countries;
}
CCountry CIcaoDataReader::getCountryForIsoCode(const QString &isoCode) const
{
QReadLocker l(&m_lockCountry);
return m_countries.findByIsoCode(isoCode);
}
CCountry CIcaoDataReader::getCountryForName(const QString &name) const
{
QReadLocker l(&m_lockCountry);
return m_countries.findBestMatchByCountryName(name);
}
CAirlineIcaoCodeList CIcaoDataReader::getAirlineIcaoCodesForDesignator(const QString &designator) const
{
return getAirlineIcaoCodes().findByVDesignator(designator);
}
CAirlineIcaoCode CIcaoDataReader::getAirlineIcaoCodeForDbKey(int key) const
{
return getAirlineIcaoCodes().findByKey(key);
}
CAirlineIcaoCode CIcaoDataReader::smartAirlineIcaoSelector(const CAirlineIcaoCode &icaoPattern) const
{
CAirlineIcaoCodeList codes(this->getAirlineIcaoCodes()); // thread safe copy
return codes.smartAirlineIcaoSelector(icaoPattern);
}
int CIcaoDataReader::getAircraftIcaoCodesCount() const
{
QReadLocker l(&m_lockAircraft);
return m_aircraftIcaos.size();
}
int CIcaoDataReader::getAirlineIcaoCodesCount() const
{
QReadLocker l(&m_lockAirline);
return m_airlineIcaos.size();
}
bool CIcaoDataReader::areAllDataRead() const
{
return getCountriesCount() > 0 && getAirlineIcaoCodesCount() > 0 && getAircraftIcaoCodesCount() > 0;
}
int CIcaoDataReader::getCountriesCount() const
{
QReadLocker l(&m_lockCountry);
return m_countries.size();
}
void CIcaoDataReader::ps_read(BlackMisc::Network::CEntityFlags::Entity entities, const QDateTime &newerThan)
{
this->threadAssertCheck(); // runs in background thread
CEntityFlags::Entity entitiesTriggered = CEntityFlags::NoEntity;
if (entities.testFlag(CEntityFlags::AircraftIcaoEntity))
{
CUrl url(getAircraftIcaoUrl());
if (!url.isEmpty())
{
if (!newerThan.isNull()) { url.appendQuery("newer=" + newerThan.toString(Qt::ISODate)); }
sApp->getFromNetwork(url, { this, &CIcaoDataReader::ps_parseAircraftIcaoData });
entitiesTriggered |= CEntityFlags::AircraftIcaoEntity;
}
else
{
CLogMessage(this).error("No URL for %1") << CEntityFlags::flagToString(CEntityFlags::AircraftIcaoEntity);
}
}
if (entities.testFlag(CEntityFlags::AirlineIcaoEntity))
{
CUrl url(getAirlineIcaoUrl());
if (!url.isEmpty())
{
if (!newerThan.isNull()) { url.appendQuery("newer=" + newerThan.toString(Qt::ISODate)); }
sApp->getFromNetwork(url, { this, &CIcaoDataReader::ps_parseAirlineIcaoData });
entitiesTriggered |= CEntityFlags::AirlineIcaoEntity;
}
else
{
CLogMessage(this).error("No URL for %1") << CEntityFlags::flagToString(CEntityFlags::AirlineIcaoEntity);
}
}
if (entities.testFlag(CEntityFlags::CountryEntity))
{
CUrl url(getCountryUrl());
if (!url.isEmpty())
{
if (!newerThan.isNull()) { url.appendQuery("newer=" + newerThan.toString(Qt::ISODate)); }
sApp->getFromNetwork(url, { this, &CIcaoDataReader::ps_parseCountryData });
entitiesTriggered |= CEntityFlags::CountryEntity;
}
else
{
CLogMessage(this).error("No URL for %1") << CEntityFlags::flagToString(CEntityFlags::CountryEntity);
}
}
if (entitiesTriggered != CEntityFlags::NoEntity)
{
emit dataRead(entitiesTriggered, CEntityFlags::StartRead, 0);
}
}
CUrl CIcaoDataReader::getBaseUrl() const
{
const CUrl baseUrl(sApp->getGlobalSetup().getDbIcaoReaderUrl());
return baseUrl;
}
void CIcaoDataReader::ps_parseAircraftIcaoData(QNetworkReply *nwReplyPtr)
{
// wrap pointer, make sure any exit cleans up reply
// required to use delete later as object is created in a different thread
QScopedPointer<QNetworkReply, QScopedPointerDeleteLater> nwReply(nwReplyPtr);
QString urlString(nwReply->url().toString());
CDatabaseReader::JsonDatastoreResponse res = this->setStatusAndTransformReplyIntoDatastoreResponse(nwReply.data());
if (res.hasErrorMessage())
{
CLogMessage::preformatted(res.lastWarningOrAbove());
emit dataRead(CEntityFlags::AircraftIcaoEntity, CEntityFlags::ReadFailed, 0);
return;
}
CAircraftIcaoCodeList codes = CAircraftIcaoCodeList::fromDatabaseJson(res);
// this part needs to be synchronized
int n = codes.size();
{
QWriteLocker wl(&this->m_lockAircraft);
this->m_aircraftIcaos = codes;
}
emit dataRead(CEntityFlags::AircraftIcaoEntity, CEntityFlags::ReadFinished, n);
CLogMessage(this).info("Read %1 %2 from %3") << n << CEntityFlags::flagToString(CEntityFlags::AircraftIcaoEntity) << urlString;
}
void CIcaoDataReader::ps_parseAirlineIcaoData(QNetworkReply *nwReplyPtr)
{
QScopedPointer<QNetworkReply, QScopedPointerDeleteLater> nwReply(nwReplyPtr);
QString urlString(nwReply->url().toString());
CDatabaseReader::JsonDatastoreResponse res = this->setStatusAndTransformReplyIntoDatastoreResponse(nwReply.data());
if (res.hasErrorMessage())
{
CLogMessage::preformatted(res.lastWarningOrAbove());
emit dataRead(CEntityFlags::AirlineIcaoEntity, CEntityFlags::ReadFailed, 0);
return;
}
CAirlineIcaoCodeList codes = CAirlineIcaoCodeList::fromDatabaseJson(res);
// this part needs to be synchronized
int n = codes.size();
{
QWriteLocker wl(&this->m_lockAirline);
this->m_airlineIcaos = codes;
}
emit dataRead(CEntityFlags::AirlineIcaoEntity, CEntityFlags::ReadFinished, n);
CLogMessage(this).info("Read %1 %2 from %3") << n << CEntityFlags::flagToString(CEntityFlags::AirlineIcaoEntity) << urlString;
}
void CIcaoDataReader::ps_parseCountryData(QNetworkReply *nwReplyPtr)
{
QScopedPointer<QNetworkReply, QScopedPointerDeleteLater> nwReply(nwReplyPtr);
QString urlString(nwReply->url().toString());
CDatabaseReader::JsonDatastoreResponse res = this->setStatusAndTransformReplyIntoDatastoreResponse(nwReply.data());
if (res.hasErrorMessage())
{
CLogMessage::preformatted(res.lastWarningOrAbove());
emit dataRead(CEntityFlags::CountryEntity, CEntityFlags::ReadFailed, 0);
return;
}
CCountryList countries = CCountryList::fromDatabaseJson(res);
// this part needs to be synchronized
int n = m_countries.size();
{
QWriteLocker wl(&this->m_lockCountry);
this->m_countries = countries;
}
emit dataRead(CEntityFlags::CountryEntity, CEntityFlags::ReadFinished, n);
CLogMessage(this).info("Read %1 %2 from %3") << n << CEntityFlags::flagToString(CEntityFlags::CountryEntity) << urlString;
}
bool CIcaoDataReader::readFromJsonFiles(const QString &dir, CEntityFlags::Entity whatToRead)
{
QDir directory(dir);
if (!directory.exists()) { return false; }
CEntityFlags::Entity reallyRead = CEntityFlags::NoEntity;
if (whatToRead.testFlag(CEntityFlags::CountryEntity))
{
QString countriesJson(CFileUtils::readFileToString(CFileUtils::appendFilePaths(directory.absolutePath(), "countries.json")));
if (!countriesJson.isEmpty())
{
CCountryList countries;
countries.convertFromJson(Json::jsonObjectFromString(countriesJson));
int c = countries.size();
{
QWriteLocker l(&m_lockCountry);
m_countries = countries;
}
// Do not emit while locked -> deadlock
reallyRead |= CEntityFlags::CountryEntity;
emit dataRead(CEntityFlags::CountryEntity, CEntityFlags::ReadFinished, c);
}
}
if (whatToRead.testFlag(CEntityFlags::AircraftIcaoEntity))
{
QString aircraftJson(CFileUtils::readFileToString(CFileUtils::appendFilePaths(directory.absolutePath(), "aircrafticao.json")));
if (!aircraftJson.isEmpty())
{
CAircraftIcaoCodeList aircraftIcaos;
aircraftIcaos.convertFromJson(Json::jsonObjectFromString(aircraftJson));
int c = aircraftIcaos.size();
{
QWriteLocker l(&m_lockAircraft);
m_aircraftIcaos = aircraftIcaos;
}
reallyRead |= CEntityFlags::AircraftIcaoEntity;
emit dataRead(CEntityFlags::AircraftIcaoEntity, CEntityFlags::ReadFinished, c);
}
}
if (whatToRead.testFlag(CEntityFlags::AirlineIcaoEntity))
{
QString airlineJson(CFileUtils::readFileToString(CFileUtils::appendFilePaths(directory.absolutePath(), "airlineicao.json")));
if (!airlineJson.isEmpty())
{
CAirlineIcaoCodeList airlineIcaos;
airlineIcaos.convertFromJson(Json::jsonObjectFromString(airlineJson));
int c = airlineIcaos.size();
{
QWriteLocker l(&m_lockAirline);
m_airlineIcaos = airlineIcaos;
}
reallyRead |= CEntityFlags::AirlineIcaoEntity;
emit dataRead(CEntityFlags::AirlineIcaoEntity, CEntityFlags::ReadFinished, c);
}
}
return (whatToRead & CEntityFlags::AllIcaoAndCountries) == reallyRead;
}
bool CIcaoDataReader::readFromJsonFilesInBackground(const QString &dir, CEntityFlags::Entity whatToRead)
{
if (dir.isEmpty() || whatToRead == CEntityFlags::NoEntity) { return false; }
QTimer::singleShot(0, this, [this, dir, whatToRead]()
{
bool s = this->readFromJsonFiles(dir, whatToRead);
Q_UNUSED(s);
});
return true;
}
bool CIcaoDataReader::writeToJsonFiles(const QString &dir) const
{
QDir directory(dir);
if (!directory.exists()) { return false; }
if (this->getCountriesCount() > 0)
{
QString json(QJsonDocument(this->getCountries().toJson()).toJson());
bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), "countries.json"));
if (!s) { return false; }
}
if (this->getAircraftIcaoCodesCount() > 0)
{
QString json(QJsonDocument(this->getAircraftIcaoCodes().toJson()).toJson());
bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), "aircrafticao.json"));
if (!s) { return false; }
}
if (this->getAirlineIcaoCodesCount() > 0)
{
QString json(QJsonDocument(this->getAirlineIcaoCodes().toJson()).toJson());
bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), "airlineicao.json"));
if (!s) { return false; }
}
return true;
}
CUrl CIcaoDataReader::getAircraftIcaoUrl() const
{
return getBaseUrl().withAppendedPath("service/jsonaircrafticao.php");
}
CUrl CIcaoDataReader::getAirlineIcaoUrl() const
{
return getBaseUrl().withAppendedPath("service/jsonairlineicao.php");
}
CUrl CIcaoDataReader::getCountryUrl() const
{
return getBaseUrl().withAppendedPath("service/jsoncountry.php");
}
} // namespace

153
src/blackcore/icaodatareader.h
View File

@@ -1,153 +0,0 @@
/* Copyright (C) 2015
* swift project Community / Contributors
*
* This file is part of swift project. It is subject to the license terms in the LICENSE file found in the top-level
* directory of this distribution and at http://www.swift-project.org/license.html. No part of swift project,
* including this file, may be copied, modified, propagated, or distributed except according to the terms
* contained in the LICENSE file.
*/
//! \file
#ifndef BLACKCORE_ICAODATAREADER_H
#define BLACKCORE_ICAODATAREADER_H
#include "blackcore/blackcoreexport.h"
#include "blackcore/databasereader.h"
#include "blackmisc/aviation/aircrafticaocode.h"
#include "blackmisc/aviation/aircrafticaocodelist.h"
#include "blackmisc/aviation/airlineicaocode.h"
#include "blackmisc/aviation/airlineicaocodelist.h"
#include "blackmisc/country.h"
#include "blackmisc/countrylist.h"
#include "blackmisc/network/entityflags.h"
#include "blackmisc/network/url.h"
#include <QObject>
#include <QReadWriteLock>
#include <QString>
class QDateTime;
class QNetworkReply;
namespace BlackCore
{
//! Read ICAO data from Database
class BLACKCORE_EXPORT CIcaoDataReader : public CDatabaseReader
{
Q_OBJECT
public:
//! Constructor
explicit CIcaoDataReader(QObject *owner);
//! Get aircraft ICAO information
//! \threadsafe
BlackMisc::Aviation::CAircraftIcaoCodeList getAircraftIcaoCodes() const;
//! Get aircraft ICAO information for designator
//! \threadsafe
BlackMisc::Aviation::CAircraftIcaoCode getAircraftIcaoCodeForDesignator(const QString &designator) const;
//! Get aircraft ICAO information for key
//! \threadsafe
BlackMisc::Aviation::CAircraftIcaoCode getAircraftIcaoCodeForDbKey(int key) const;
//! Get airline ICAO information
//! \threadsafe
BlackMisc::Aviation::CAirlineIcaoCodeList getAirlineIcaoCodes() const;
//! Get best match for incomplete aircraft ICAO code
//! \threadsafe
BlackMisc::Aviation::CAircraftIcaoCode smartAircraftIcaoSelector(const BlackMisc::Aviation::CAircraftIcaoCode &icaoPattern) const;
//! Get countries
//! \threadsafe
BlackMisc::CCountryList getCountries() const;
//! Get countries count
//! \threadsafe
int getCountriesCount() const;
//! Get country for ISO code
//! \threadsafe
BlackMisc::CCountry getCountryForIsoCode(const QString &isoCode) const;
//! Get country for ISO name
//! \threadsafe
BlackMisc::CCountry getCountryForName(const QString &name) const;
//! Get airline ICAO information for designator
//! \threadsafe
BlackMisc::Aviation::CAirlineIcaoCodeList getAirlineIcaoCodesForDesignator(const QString &designator) const;
//! Get airline ICAO information for key
//! \threadsafe
BlackMisc::Aviation::CAirlineIcaoCode getAirlineIcaoCodeForDbKey(int key) const;
//! Get best match for incomplete airline ICAO code
//! \threadsafe
BlackMisc::Aviation::CAirlineIcaoCode smartAirlineIcaoSelector(const BlackMisc::Aviation::CAirlineIcaoCode &icaoPattern) const;
//! Get aircraft ICAO information count
//! \threadsafe
int getAircraftIcaoCodesCount() const;
//! Get airline ICAO information count
//! \threadsafe
int getAirlineIcaoCodesCount() const;
//! All data read?
//! \threadsafe
bool areAllDataRead() const;
//! Read from static DB data file
bool readFromJsonFiles(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead = BlackMisc::Network::CEntityFlags::AllIcaoAndCountries);
//! Read from static DB data file
bool readFromJsonFilesInBackground(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead = BlackMisc::Network::CEntityFlags::AllIcaoAndCountries);
//! Write to static DB data file
bool writeToJsonFiles(const QString &dir) const;
signals:
//! Combined read signal
void dataRead(BlackMisc::Network::CEntityFlags::Entity entity, BlackMisc::Network::CEntityFlags::ReadState state, int number);
private slots:
//! Aircraft have been read
void ps_parseAircraftIcaoData(QNetworkReply *nwReply);
//! Airlines have been read
void ps_parseAirlineIcaoData(QNetworkReply *nwReply);
//! Airlines have been read
void ps_parseCountryData(QNetworkReply *nwReply);
//! Read / re-read data file
void ps_read(BlackMisc::Network::CEntityFlags::Entity entities, const QDateTime &newerThan);
private:
BlackMisc::Aviation::CAircraftIcaoCodeList m_aircraftIcaos;
BlackMisc::Aviation::CAirlineIcaoCodeList m_airlineIcaos;
BlackMisc::CCountryList m_countries;
mutable QReadWriteLock m_lockAirline;
mutable QReadWriteLock m_lockAircraft;
mutable QReadWriteLock m_lockCountry;
//! Base URL
BlackMisc::Network::CUrl getBaseUrl() const;
//! URL
BlackMisc::Network::CUrl getAircraftIcaoUrl() const;
//! URL
BlackMisc::Network::CUrl getAirlineIcaoUrl() const;
//! URL
BlackMisc::Network::CUrl getCountryUrl() const;
};
}
#endif // guard

187
src/blackcore/webdataservices.cpp
View File

@@ -9,9 +9,12 @@
#include "blackcore/application.h"
#include "blackcore/data/globalsetup.h"
#include "blackcore/databasewriter.h"
#include "blackcore/icaodatareader.h"
#include "blackcore/modeldatareader.h"
#include "blackcore/db/infodatareader.h"
#include "blackcore/db/icaodatareader.h"
#include "blackcore/db/databasewriter.h"
#include "blackcore/db/icaodatareader.h"
#include "blackcore/db/modeldatareader.h"
#include "blackcore/setupreader.h"
#include "blackcore/vatsimbookingreader.h"
#include "blackcore/vatsimdatafilereader.h"
#include "blackcore/vatsimmetarreader.h"
@@ -34,7 +37,9 @@
#include <QtGlobal>
using namespace BlackCore;
using namespace BlackCore::Db;
using namespace BlackCore::Data;
using namespace BlackCore::Db;
using namespace BlackMisc;
using namespace BlackMisc::Simulation;
using namespace BlackMisc::Network;
@@ -43,8 +48,8 @@ using namespace BlackMisc::Weather;
namespace BlackCore
{
CWebDataServices::CWebDataServices(CWebReaderFlags::WebReader readerFlags, CWebReaderFlags::DbReaderHint hint, BlackMisc::Restricted<CApplication>, QObject *parent) :
QObject(parent), m_readerFlags(readerFlags), m_dbHint(hint)
CWebDataServices::CWebDataServices(CWebReaderFlags::WebReader readerFlags, const CDatabaseReaderConfigList &dbReaderConfig, BlackMisc::Restricted<CApplication>, QObject *parent) :
QObject(parent), m_readerFlags(readerFlags), m_dbReaderConfig(dbReaderConfig)
{
if (!sApp) { return; } // shutting down
@@ -54,44 +59,14 @@ namespace BlackCore
this->initReaders(readerFlags);
this->initWriters();
bool c = false;
Q_UNUSED(c);
if (m_readerFlags.testFlag(CWebReaderFlags::WebReaderFlag::IcaoDataReader))
{
Q_ASSERT_X(this->m_icaoDataReader, Q_FUNC_INFO, "Missing reader ICAO");
c = connect(this->m_icaoDataReader, &CIcaoDataReader::dataRead, this, &CWebDataServices::dataRead);
Q_ASSERT_X(c, Q_FUNC_INFO, "connect failed ICAO");
}
const bool withInfoData = m_readerFlags.testFlag(CWebReaderFlags::WebReaderFlag::InfoDataReader);
CEntityFlags::Entity entities = CEntityFlags::AllEntities;
entities ^= CEntityFlags::InfoObjectEntity; // 2 liner because of gcc error: invalid conversion from 'int' to 'BlackMisc::Network::CEntityFlags::EntityFlag'
if (withInfoData) { CLogMessage(this).info("Using info objects for swift DB objects"); }
if (m_readerFlags.testFlag(CWebReaderFlags::WebReaderFlag::ModelReader))
{
Q_ASSERT_X(this->m_modelDataReader, Q_FUNC_INFO, "Missing reader models");
c = connect(this->m_modelDataReader, &CModelDataReader::dataRead, this, &CWebDataServices::dataRead);
Q_ASSERT_X(c, Q_FUNC_INFO, "connect failed models");
}
if (m_readerFlags.testFlag(CWebReaderFlags::WebReaderFlag::VatsimBookingReader))
{
Q_ASSERT_X(this->m_vatsimBookingReader, Q_FUNC_INFO, "Missing reader bookings");
c = connect(this->m_vatsimBookingReader, &CVatsimBookingReader::dataRead, this, &CWebDataServices::dataRead);
Q_ASSERT_X(c, Q_FUNC_INFO, "connect failed bookings");
}
if (m_readerFlags.testFlag(CWebReaderFlags::WebReaderFlag::VatsimDataReader))
{
Q_ASSERT_X(this->m_vatsimDataFileReader, Q_FUNC_INFO, "Missing reader data file");
c = connect(this->m_vatsimDataFileReader, &CVatsimDataFileReader::dataRead, this, &CWebDataServices::dataRead);
Q_ASSERT_X(c, Q_FUNC_INFO, "connect failed VATSIM data file");
}
if (m_readerFlags.testFlag(CWebReaderFlags::WebReaderFlag::VatsimMetarReader))
{
Q_ASSERT_X(this->m_vatsimMetarReader, Q_FUNC_INFO, "Missing reader metars");
c = connect(this->m_vatsimMetarReader, &CVatsimMetarReader::dataRead, this, &CWebDataServices::dataRead);
Q_ASSERT_X(c, Q_FUNC_INFO, "connect failed VATSIM METAR");
}
this->readInBackground(CEntityFlags::AllEntities, 500);
// make sure this is called in event queue, so pending tasks cam be performed
// important so info objects can be read
this->singleShotReadInBackground(entities, 1000);
}
CServerList CWebDataServices::getVatsimFsdServers() const
@@ -149,10 +124,14 @@ namespace BlackCore
bool CWebDataServices::canConnectSwiftDb() const
{
if (!m_icaoDataReader && !m_modelDataReader) { return false; }
if (!m_icaoDataReader && !m_modelDataReader && !m_infoDataReader) { return false; }
// use the first one to test
if (m_icaoDataReader)
if (m_infoDataReader)
{
return m_infoDataReader->canConnect();
}
else if (m_icaoDataReader)
{
return m_icaoDataReader->canConnect();
}
@@ -165,7 +144,8 @@ namespace BlackCore
CEntityFlags::Entity CWebDataServices::triggerRead(CEntityFlags::Entity whatToRead, const QDateTime &newerThan)
{
m_initialRead = true;
m_initialRead = true; // read started
Q_ASSERT_X(!whatToRead.testFlag(CEntityFlags::InfoObjectEntity), Q_FUNC_INFO, "Info object must be read upfront");
CEntityFlags::Entity triggeredRead = CEntityFlags::NoEntity;
if (m_vatsimDataFileReader)
{
@@ -417,6 +397,7 @@ namespace BlackCore
if (this->m_vatsimMetarReader) { this->m_vatsimMetarReader->gracefulShutdown(); }
if (this->m_modelDataReader) { this->m_modelDataReader->gracefulShutdown(); }
if (this->m_icaoDataReader) { this->m_icaoDataReader->gracefulShutdown(); }
if (this->m_infoDataReader) { this->m_infoDataReader->gracefulShutdown(); }
if (this->m_databaseWriter) { this->m_databaseWriter->gracefulShutdown(); }
}
@@ -428,7 +409,29 @@ namespace BlackCore
void CWebDataServices::initReaders(CWebReaderFlags::WebReader flags)
{
// 1. Status file, updating the cache
// ---- "metadata" reader, 1/2 will trigger read directly during init
// 1. If any DB data, read the info upfront
const bool anyDbData = flags.testFlag(CWebReaderFlags::WebReaderFlag::IcaoDataReader) || flags.testFlag(CWebReaderFlags::WebReaderFlag::ModelReader);
const CDatabaseReaderConfigList dbReaderConfig(this->m_dbReaderConfig);
bool c = false; // signal connect
Q_UNUSED(c);
if (anyDbData && flags.testFlag(CWebReaderFlags::WebReaderFlag::InfoDataReader))
{
this->m_infoDataReader = new CInfoDataReader(this, dbReaderConfig);
c = connect(this->m_infoDataReader, &CInfoDataReader::dataRead, this, &CWebDataServices::ps_readFromSwiftDb);
Q_ASSERT_X(c, Q_FUNC_INFO, "ICAO info object signals");
c = connect(this->m_infoDataReader, &CInfoDataReader::dataRead, this, &CWebDataServices::dataRead);
Q_ASSERT_X(c, Q_FUNC_INFO, "connect failed info data");
// info data reader has a special role, it will not be triggered in triggerRead()
this->m_infoDataReader->start(QThread::LowPriority);
// directly call read
QTimer::singleShot(0, [this]() { this->m_infoDataReader->read(CEntityFlags::InfoObjectEntity, QDateTime()); });
}
// 2. Status file, updating the VATSIM related caches
if (flags.testFlag(CWebReaderFlags::WebReaderFlag::VatsimDataReader) || flags.testFlag(CWebReaderFlags::WebReaderFlag::VatsimMetarReader))
{
this->m_vatsimStatusReader = new CVatsimStatusFileReader(this);
@@ -437,57 +440,63 @@ namespace BlackCore
QTimer::singleShot(100, this->m_vatsimStatusReader, &CVatsimStatusFileReader::readInBackgroundThread);
}
// 2. VATSIM bookings
// ---- "normal data", triggerRead will start read, not starting directly
// 3. VATSIM bookings
if (flags.testFlag(CWebReaderFlags::WebReaderFlag::VatsimBookingReader))
{
this->m_vatsimBookingReader = new CVatsimBookingReader(this);
bool c = connect(this->m_vatsimBookingReader, &CVatsimBookingReader::atcBookingsRead, this, &CWebDataServices::ps_receivedBookings);
c = connect(this->m_vatsimBookingReader, &CVatsimBookingReader::atcBookingsRead, this, &CWebDataServices::ps_receivedBookings);
Q_ASSERT_X(c, Q_FUNC_INFO, "VATSIM booking reader signals");
Q_UNUSED(c);
c = connect(this->m_vatsimBookingReader, &CVatsimBookingReader::dataRead, this, &CWebDataServices::dataRead);
Q_ASSERT_X(c, Q_FUNC_INFO, "connect failed bookings");
this->m_vatsimBookingReader->start(QThread::LowPriority);
this->m_vatsimBookingReader->setInterval(3 * 60 * 1000);
}
// 3. VATSIM data file
// 4. VATSIM data file
if (flags.testFlag(CWebReaderFlags::WebReaderFlag::VatsimDataReader))
{
this->m_vatsimDataFileReader = new CVatsimDataFileReader(this);
bool c = connect(this->m_vatsimDataFileReader, &CVatsimDataFileReader::dataFileRead, this, &CWebDataServices::ps_dataFileRead);
c = connect(this->m_vatsimDataFileReader, &CVatsimDataFileReader::dataFileRead, this, &CWebDataServices::ps_dataFileRead);
Q_ASSERT_X(c, Q_FUNC_INFO, "VATSIM data reader signals");
Q_UNUSED(c);
c = connect(this->m_vatsimDataFileReader, &CVatsimDataFileReader::dataRead, this, &CWebDataServices::dataRead);
Q_ASSERT_X(c, Q_FUNC_INFO, "connect failed VATSIM data file");
this->m_vatsimDataFileReader->start(QThread::LowPriority);
this->m_vatsimDataFileReader->setInterval(90 * 1000);
}
// 4. VATSIM metar data
// 5. VATSIM metar data
if (flags.testFlag(CWebReaderFlags::WebReaderFlag::VatsimMetarReader))
{
this->m_vatsimMetarReader = new CVatsimMetarReader(this);
bool c = connect(this->m_vatsimMetarReader, &CVatsimMetarReader::metarsRead, this, &CWebDataServices::ps_receivedMetars);
c = connect(this->m_vatsimMetarReader, &CVatsimMetarReader::metarsRead, this, &CWebDataServices::ps_receivedMetars);
Q_ASSERT_X(c, Q_FUNC_INFO, "VATSIM METAR reader signals");
Q_UNUSED(c);
c = connect(this->m_vatsimMetarReader, &CVatsimMetarReader::dataRead, this, &CWebDataServices::dataRead);
Q_ASSERT_X(c, Q_FUNC_INFO, "connect failed VATSIM METAR");
this->m_vatsimMetarReader->start(QThread::LowPriority);
this->m_vatsimMetarReader->setInterval(90 * 1000);
}
// 5. ICAO data reader
// 6. ICAO data reader
if (flags.testFlag(CWebReaderFlags::WebReaderFlag::IcaoDataReader))
{
bool c;
this->m_icaoDataReader = new CIcaoDataReader(this);
this->m_icaoDataReader = new CIcaoDataReader(this, dbReaderConfig);
c = connect(this->m_icaoDataReader, &CIcaoDataReader::dataRead, this, &CWebDataServices::ps_readFromSwiftDb);
Q_ASSERT_X(c, Q_FUNC_INFO, "ICAO reader signals");
Q_UNUSED(c);
c = connect(this->m_icaoDataReader, &CIcaoDataReader::dataRead, this, &CWebDataServices::dataRead);
Q_ASSERT_X(c, Q_FUNC_INFO, "Cannot connect ICAO reader signals");
this->m_icaoDataReader->start(QThread::LowPriority);
}
// 6. Model reader
// 7. Model reader
if (flags.testFlag(CWebReaderFlags::WebReaderFlag::ModelReader))
{
this->m_modelDataReader = new CModelDataReader(this);
bool c = connect(this->m_modelDataReader, &CModelDataReader::dataRead, this, &CWebDataServices::ps_readFromSwiftDb);
this->m_modelDataReader = new CModelDataReader(this, dbReaderConfig);
c = connect(this->m_modelDataReader, &CModelDataReader::dataRead, this, &CWebDataServices::ps_readFromSwiftDb);
Q_ASSERT_X(c, Q_FUNC_INFO, "Model reader signals");
Q_UNUSED(c);
c = connect(this->m_modelDataReader, &CModelDataReader::dataRead, this, &CWebDataServices::dataRead);
Q_ASSERT_X(c, Q_FUNC_INFO, "connect failed models");
this->m_modelDataReader->start(QThread::LowPriority);
}
}
@@ -540,15 +549,56 @@ namespace BlackCore
// void
}
void CWebDataServices::singleShotReadInBackground(CEntityFlags::Entity entities, int delayMs)
{
QTimer::singleShot(delayMs, [ = ]()
{
this->readInBackground(entities, delayMs);
});
}
void CWebDataServices::readInBackground(CEntityFlags::Entity entities, int delayMs)
{
m_initialRead = true;
m_initialRead = true; // read started
const int waitForInfoObjects = 1000; // ms
const int maxWaitCycles = 6;
// with info objects wait until info objects are loaded
Q_ASSERT_X(!entities.testFlag(CEntityFlags::InfoObjectEntity), Q_FUNC_INFO, "Info object must be read upfront");
if (this->m_infoDataReader && CEntityFlags::anySwiftDbEntity(entities))
{
// try to read
if (this->m_infoObjectTrials > maxWaitCycles)
{
CLogMessage(this).error("Cannot read info objects from %1") << this->m_infoDataReader->getInfoObjectsUrl().toQString();
}
else if (this->m_infoDataReader->canConnect())
{
// read, but no idea if succesful/failure
if (this->m_infoDataReader->getDbInfoObjectCount() > 0)
{
CLogMessage(this).info("Info objects loaded from %1") << this->m_infoDataReader->getInfoObjectsUrl().toQString();
}
else
{
CLogMessage(this).error("Info objects loading failed from %1, '%2'")
<< this->m_infoDataReader->getInfoObjectsUrl().toQString()
<< this->m_infoDataReader->getStatusMessage();
}
}
else
{
// postpone by some time
this->m_infoObjectTrials++;
this->singleShotReadInBackground(entities, waitForInfoObjects);
return;
}
}
if (delayMs > 100)
{
BlackMisc::singleShot(delayMs, QThread::currentThread(), [ = ]()
{
this->readInBackground(entities, 0);
});
this->singleShotReadInBackground(entities, 0);
}
else
{
@@ -615,5 +665,4 @@ namespace BlackCore
}
return s;
}
} // ns

43
src/blackcore/webdataservices.h
View File

@@ -14,6 +14,7 @@
#include "blackcore/blackcoreexport.h"
#include "blackcore/webreaderflags.h"
#include "blackcore/db/databasereader.h"
#include "blackmisc/aviation/aircrafticaocode.h"
#include "blackmisc/aviation/aircrafticaocodelist.h"
#include "blackmisc/aviation/airlineicaocode.h"
@@ -48,6 +49,8 @@
namespace BlackMisc
{
class CLogCategoryList;
template <typename T> class Restricted;
namespace Aviation { class CCallsign; }
namespace Simulation { class CSimulatedAircraft; }
}
@@ -55,14 +58,19 @@ namespace BlackMisc
namespace BlackCore
{
class CApplication;
class CDatabaseWriter;
class CIcaoDataReader;
class CModelDataReader;
class CVatsimBookingReader;
class CVatsimDataFileReader;
class CVatsimMetarReader;
class CVatsimStatusFileReader;
namespace Db
{
class CDatabaseWriter;
class CIcaoDataReader;
class CModelDataReader;
class CInfoDataReader;
}
/*!
* Encapsulates reading data from web sources
*/
@@ -76,7 +84,7 @@ namespace BlackCore
static const BlackMisc::CLogCategoryList &getLogCategories();
//! Constructor, only allowed from BlackCore::CApplication
CWebDataServices(CWebReaderFlags::WebReader readerFlags, CWebReaderFlags:: DbReaderHint hint, BlackMisc::Restricted<CApplication>, QObject *parent = nullptr);
CWebDataServices(CWebReaderFlags::WebReader readerFlags, const BlackCore::Db::CDatabaseReaderConfigList &dbReaderConfig, BlackMisc::Restricted<CApplication>, QObject *parent = nullptr);
//! Shutdown
void gracefulShutdown();
@@ -93,15 +101,15 @@ namespace BlackCore
//! Metar reader
CVatsimMetarReader *getMetarReader() const { return m_vatsimMetarReader; }
//! Info data reader
Db::CInfoDataReader *getInfoDataReader() const { return m_infoDataReader; }
//! DB writer class
CDatabaseWriter *getDatabaseWriter() const { return m_databaseWriter; }
Db::CDatabaseWriter *getDatabaseWriter() const { return m_databaseWriter; }
//! Reader flags
CWebReaderFlags::WebReader getReaderFlags() const { return m_readerFlags; }
//! Reader flags
CWebReaderFlags::DbReaderHint getDbHint() const { return m_dbHint; }
//! FSD servers
//! \threadsafe
BlackMisc::Network::CServerList getVatsimFsdServers() const;
@@ -295,7 +303,7 @@ namespace BlackCore
//! Data file has been read
void ps_dataFileRead(int lines);
//! Read from model reader
//! Read finished from reader
void ps_readFromSwiftDb(BlackMisc::Network::CEntityFlags::Entity entity, BlackMisc::Network::CEntityFlags::ReadState state, int number);
//! Setup changed
@@ -308,20 +316,25 @@ namespace BlackCore
//! Init the writers
void initWriters();
CWebReaderFlags::WebReader m_readerFlags = CWebReaderFlags::WebReaderFlag::None; //!< which readers are available
CWebReaderFlags::DbReaderHint m_dbHint = CWebReaderFlags::NoHint; //!< how to read DB data
bool m_initialRead = false; //!< Initial read conducted
//! Call CWebDataServices::readInBackground by single shot
void singleShotReadInBackground(BlackMisc::Network::CEntityFlags::Entity entities, int delayMs);
CWebReaderFlags::WebReader m_readerFlags = CWebReaderFlags::WebReaderFlag::None; //!< which readers are available
BlackCore::Db::CDatabaseReaderConfigList m_dbReaderConfig; //!< how to read DB data
bool m_initialRead = false; //!< Initial read started
int m_infoObjectTrials = 0; //!< Tried to read info objects
// for reading XML and VATSIM data files
CVatsimStatusFileReader *m_vatsimStatusReader = nullptr;
CVatsimBookingReader *m_vatsimBookingReader = nullptr;
CVatsimDataFileReader *m_vatsimDataFileReader = nullptr;
CVatsimMetarReader *m_vatsimMetarReader = nullptr;
CIcaoDataReader *m_icaoDataReader = nullptr;
CModelDataReader *m_modelDataReader = nullptr;
Db::CIcaoDataReader *m_icaoDataReader = nullptr;
Db::CModelDataReader *m_modelDataReader = nullptr;
Db::CInfoDataReader *m_infoDataReader = nullptr;
// writing objects directly into DB
CDatabaseWriter *m_databaseWriter = nullptr;
Db::CDatabaseWriter *m_databaseWriter = nullptr;
};
} // namespace