refs #649, refs #656, using caches in ICAO data reader

* access to info data reader (get DB metadata)
* adjusted database base class to support caches, info objects
* moved classes to subdir
This commit is contained in:
Klaus Basan
2016-05-24 00:36:06 +02:00
parent 6b7e05077c
commit 954ddfb2e7
10 changed files with 1171 additions and 938 deletions

View File

@@ -0,0 +1,250 @@
/* Copyright (C) 2015
* swift project Community / Contributors
*
* This file is part of swift project. It is subject to the license terms in the LICENSE file found in the top-level
* directory of this distribution and at http://www.swift-project.org/license.html. No part of swift project,
* including this file, may be copied, modified, propagated, or distributed except according to the terms
* contained in the LICENSE file.
*/
#include "blackcore/db/databasereader.h"
#include "blackcore/db/infodatareader.h"
#include "blackcore/webdataservices.h"
#include "blackcore/application.h"
#include "blackmisc/db/datastoreutility.h"
#include "blackmisc/network/networkutils.h"
#include "blackmisc/logcategory.h"
#include "blackmisc/logcategorylist.h"
#include "blackmisc/logmessage.h"
#include <QByteArray>
#include <QJsonDocument>
#include <QJsonObject>
#include <QJsonValueRef>
#include <QMetaObject>
#include <QNetworkReply>
#include <QReadLocker>
#include <QUrl>
#include <QWriteLocker>
using namespace BlackMisc;
using namespace BlackMisc::Db;
using namespace BlackMisc::Network;
using namespace BlackCore;
namespace BlackCore
{
namespace Db
{
CDatabaseReader::CDatabaseReader(QObject *owner, const CDatabaseReaderConfigList &config, const QString &name) :
BlackMisc::CThreadedReader(owner, name), m_config(config)
{
this->m_sharedUrl = sApp->getGlobalSetup().getSwiftSharedUrls().getRandomWorkingUrl();
}
void CDatabaseReader::readInBackgroundThread(CEntityFlags::Entity entities, const QDateTime &newerThan)
{
if (isAbandoned()) { return; }
// we accept cached cached data
Q_ASSERT_X(!entities.testFlag(CEntityFlags::InfoObjectEntity), Q_FUNC_INFO, "Read info objects directly");
CEntityFlags::Entity allEntities = entities;
CEntityFlags::Entity currentEntity = CEntityFlags::iterateDbEntities(allEntities); // CEntityFlags::InfoObjectEntity will be ignored
const bool hasInfoObjects = this->hasInfoObjects();
while (currentEntity)
{
const CDatabaseReaderConfig config(this->getConfigForEntity(currentEntity));
if (config.getRetrievalMode().testFlag(CDbFlags::Cached))
{
if (hasInfoObjects)
{
const QDateTime cacheTs(this->getCacheTimestamp(currentEntity));
const QDateTime latestEntityTs(this->getLatestEntityTimestamp(currentEntity));
const qint64 cacheTimestamp = cacheTs.isValid() ? cacheTs.toMSecsSinceEpoch() : -1;
const qint64 latestEntityTimestamp = latestEntityTs.isValid() ? latestEntityTs.toMSecsSinceEpoch() : -1;
Q_ASSERT_X(latestEntityTimestamp > 0, Q_FUNC_INFO, "Missing timestamp");
if (cacheTimestamp >= latestEntityTimestamp && cacheTimestamp > 0)
{
this->syncronizeCaches(currentEntity);
entities &= ~currentEntity; // do not load from web
CLogMessage(this).info("Using cache for %1 (%2, %3)")
<< CEntityFlags::flagToString(currentEntity)
<< cacheTs.toString() << cacheTimestamp;
}
else
{
CLogMessage(this).info("Cache for %1 outdated, latest entity (%2, %3)")
<< CEntityFlags::flagToString(currentEntity)
<< latestEntityTs.toString() << latestEntityTimestamp;
}
}
else
{
this->syncronizeCaches(currentEntity);
CLogMessage(this).info("No info object for %1, using cache") << CEntityFlags::flagToString(currentEntity);
}
}
currentEntity = CEntityFlags::iterateDbEntities(allEntities);
}
// ps_read is implemented in the derived classes
if (entities == CEntityFlags::NoEntity) { return; }
const bool s = QMetaObject::invokeMethod(this, "ps_read",
Q_ARG(BlackMisc::Network::CEntityFlags::Entity, entities),
Q_ARG(QDateTime, newerThan));
Q_ASSERT_X(s, Q_FUNC_INFO, "Invoke failed");
Q_UNUSED(s);
}
CDatabaseReader::JsonDatastoreResponse CDatabaseReader::transformReplyIntoDatastoreResponse(QNetworkReply *nwReply) const
{
this->threadAssertCheck();
static const CLogCategoryList cats(CLogCategoryList(this).join({ CLogCategory::webservice()}));
JsonDatastoreResponse datastoreResponse;
if (this->isAbandoned())
{
nwReply->abort();
datastoreResponse.setMessage(CStatusMessage(cats, CStatusMessage::SeverityError, "Terminated data parsing process"));
return datastoreResponse; // stop, terminate straight away, ending thread
}
if (nwReply->error() == QNetworkReply::NoError)
{
const QString dataFileData = nwReply->readAll().trimmed();
nwReply->close(); // close asap
if (dataFileData.isEmpty())
{
datastoreResponse.setMessage(CStatusMessage(cats, CStatusMessage::SeverityError, "Empty response, no data"));
datastoreResponse.m_updated = QDateTime::currentDateTimeUtc();
return datastoreResponse;
}
QJsonDocument jsonResponse = QJsonDocument::fromJson(dataFileData.toUtf8());
if (jsonResponse.isArray())
{
// directly an array, no further info
datastoreResponse.m_jsonArray = jsonResponse.array();
datastoreResponse.m_updated = QDateTime::currentDateTimeUtc();
}
else
{
QJsonObject responseObject(jsonResponse.object());
datastoreResponse.m_jsonArray = responseObject["data"].toArray();
QString ts(responseObject["latest"].toString());
datastoreResponse.m_updated = ts.isEmpty() ? QDateTime::currentDateTimeUtc() : CDatastoreUtility::parseTimestamp(ts);
datastoreResponse.m_restricted = responseObject["restricted"].toBool();
}
return datastoreResponse;
}
// no valid response
QString error(nwReply->errorString());
QString url(nwReply->url().toString());
nwReply->abort();
datastoreResponse.setMessage(CStatusMessage(cats, CStatusMessage::SeverityError,
QString("Reading data failed: " + error + " " + url)));
return datastoreResponse;
}
CDatabaseReader::JsonDatastoreResponse CDatabaseReader::setStatusAndTransformReplyIntoDatastoreResponse(QNetworkReply *nwReply)
{
this->setConnectionStatus(nwReply);
return this->transformReplyIntoDatastoreResponse(nwReply);
}
CDbInfoList CDatabaseReader::infoList() const
{
static const CDbInfoList e;
if (!sApp->hasWebDataServices()) { return e; }
if (!sApp->getWebDataServices()->getInfoDataReader()) { return e; }
return sApp->getWebDataServices()->getInfoDataReader()->getDbInfoObjects();
}
bool CDatabaseReader::hasInfoObjects() const
{
return infoList().size() > 0;
}
QDateTime CDatabaseReader::getLatestEntityTimestamp(CEntityFlags::Entity entity) const
{
static const QDateTime e;
const CDbInfoList il(infoList());
if (il.isEmpty() || entity == CEntityFlags::NoEntity) { return e; }
CDbInfo info = il.findFirstByEntityOrDefault(entity);
if (!info.isValid()) { return e; }
return info.getUtcTimestamp();
}
CDatabaseReaderConfig CDatabaseReader::getConfigForEntity(CEntityFlags::Entity entity) const
{
return this->m_config.findFirstOrDefaultForEntity(entity);
}
bool CDatabaseReader::canConnect() const
{
QReadLocker rl(&this->m_statusLock);
return m_canConnect;
}
bool CDatabaseReader::canConnect(QString &message) const
{
QReadLocker rl(&this->m_statusLock);
message = m_statusMessage;
return m_canConnect;
}
CUrl CDatabaseReader::getWorkingSharedUrl() const
{
return this->m_sharedUrl;
}
const QString &CDatabaseReader::getStatusMessage() const
{
return this->m_statusMessage;
}
void CDatabaseReader::setConnectionStatus(bool ok, const QString &message)
{
{
QWriteLocker wl(&this->m_statusLock);
this->m_statusMessage = message;
this->m_canConnect = ok;
}
}
void CDatabaseReader::setConnectionStatus(QNetworkReply *nwReply)
{
Q_ASSERT_X(nwReply, Q_FUNC_INFO, "Missing network reply");
if (nwReply->isFinished())
{
if (nwReply->error() == QNetworkReply::NoError)
{
setConnectionStatus(true);
}
else
{
setConnectionStatus(false, nwReply->errorString());
}
}
}
const CLogCategoryList &CDatabaseReader::getLogCategories()
{
static const BlackMisc::CLogCategoryList cats { BlackMisc::CLogCategory::swiftDbWebservice(), BlackMisc::CLogCategory::mapping() };
return cats;
}
const QString &CDatabaseReader::parameterLatestTimestamp()
{
static const QString p("latestTimestamp");
return p;
}
const QString &CDatabaseReader::parameterLatestId()
{
static const QString p("latestId");
return p;
}
} // ns
} // ns

View File

@@ -0,0 +1,172 @@
/* Copyright (C) 2015
* swift project Community / Contributors
*
* This file is part of swift project. It is subject to the license terms in the LICENSE file found in the top-level
* directory of this distribution and at http://www.swift-project.org/license.html. No part of swift project,
* including this file, may be copied, modified, propagated, or distributed except according to the terms
* contained in the LICENSE file.
*/
//! \file
#ifndef BLACKCORE_DB_DATABASEREADER_H
#define BLACKCORE_DB_DATABASEREADER_H
#include "blackcore/blackcoreexport.h"
#include "blackcore/db/databasereaderconfig.h"
#include "blackmisc/db/dbinfolist.h"
#include "blackmisc/pq/time.h"
#include "blackmisc/network/url.h"
#include "blackmisc/statusmessage.h"
#include "blackmisc/threadedreader.h"
#include "blackmisc/sequence.h"
#include "blackmisc/valueobject.h"
#include <QDateTime>
#include <QJsonArray>
#include <QObject>
#include <QReadWriteLock>
#include <QString>
#include <QTimer>
#include <QtGlobal>
class QNetworkReply;
namespace BlackMisc { class CLogCategoryList; }
namespace BlackCore
{
namespace Db
{
//! Specialized version of threaded reader for DB data
class BLACKCORE_EXPORT CDatabaseReader : public BlackMisc::CThreadedReader
{
Q_OBJECT
public:
//! Response from our database
struct JsonDatastoreResponse
{
QJsonArray m_jsonArray; //!< JSON array data
QDateTime m_updated; //!< when was the latest updated?
bool m_restricted = false; //!< restricted reponse, only data changed
BlackMisc::CStatusMessage m_message; //!< last error or warning
//! Any data?
bool isEmpty() const { return m_jsonArray.isEmpty(); }
//! Number of elements
int size() const { return m_jsonArray.size(); }
//! Any timestamp?
bool hasTimestamp() const { return m_updated.isValid(); }
//! Is response newer?
bool isNewer(const QDateTime &ts) const { return m_updated.toMSecsSinceEpoch() > ts.toMSecsSinceEpoch(); }
//! Is response newer?
bool isNewer(qint64 mSecsSinceEpoch) const { return m_updated.toMSecsSinceEpoch() > mSecsSinceEpoch; }
//! Incremental data
bool isRestricted() const { return m_restricted; }
//! Error message?
bool hasErrorMessage() const { return m_message.getSeverity() == BlackMisc::CStatusMessage::SeverityError; }
//! Warning or error message?
bool hasWarningOrAboveMessage() const { return m_message.isWarningOrAbove(); }
//! Last error or warning
const BlackMisc::CStatusMessage &lastWarningOrAbove() const { return m_message; }
//! Set the error/warning message
void setMessage(const BlackMisc::CStatusMessage &lastErrorOrWarning) { m_message = lastErrorOrWarning; }
//! Get the JSON array
QJsonArray getJsonArray() const { return m_jsonArray; }
//! Set the JSON array
void setJsonArray(const QJsonArray &value) { m_jsonArray = value; }
//! Implicit conversion
operator QJsonArray() const { return m_jsonArray; }
};
//! Start reading in own thread
void readInBackgroundThread(BlackMisc::Network::CEntityFlags::Entity entities, const QDateTime &newerThan);
//! Can connect to DB
//! \threadsafe
bool canConnect() const;
//! Can connect to server?
//! \return message why connect failed
//! \threadsafe
bool canConnect(QString &message) const;
//! Obtain a working shared URL
BlackMisc::Network::CUrl getWorkingSharedUrl() const;
//! Status message (error message)
const QString &getStatusMessage() const;
//! Log categories
static const BlackMisc::CLogCategoryList &getLogCategories();
//! Name of latest timestamp
static const QString &parameterLatestTimestamp();
//! Name of parameter for latest id
static const QString &parameterLatestId();
protected:
CDatabaseReaderConfigList m_config; //!< DB reder configuration
BlackMisc::Network::CUrl m_sharedUrl; //!< URL for checking if alive
QString m_statusMessage; //!< Returned status message from watchdog
bool m_canConnect = false; //!< Successful connection?
mutable QReadWriteLock m_statusLock; //!< Lock
//! Constructor
CDatabaseReader(QObject *owner, const CDatabaseReaderConfigList &config, const QString &name);
//! Check if terminated or error, otherwise split into array of objects
CDatabaseReader::JsonDatastoreResponse setStatusAndTransformReplyIntoDatastoreResponse(QNetworkReply *nwReply);
//! Info list (latest data timestamp)
//! \sa BlackCore::Db::CInfoDataReader
BlackMisc::Db::CDbInfoList infoList() const;
//! Info objects available?
bool hasInfoObjects() const;
//! Obtain latest object timestamp
//! \sa BlackCore::Db::CInfoDataReader
QDateTime getLatestEntityTimestamp(BlackMisc::Network::CEntityFlags::Entity entity) const;
//! Config for given entity
CDatabaseReaderConfig getConfigForEntity(BlackMisc::Network::CEntityFlags::Entity entity) const;
//! Syncronize caches for given entities
virtual void syncronizeCaches(BlackMisc::Network::CEntityFlags::Entity entities) = 0;
//! Cache`s timestamp for given entity
virtual QDateTime getCacheTimestamp(BlackMisc::Network::CEntityFlags::Entity entities) = 0;
//! Invalidate the caches for given entities
virtual void invalidateCaches(BlackMisc::Network::CEntityFlags::Entity entities) = 0;
private:
//! Check if terminated or error, otherwise split into array of objects
JsonDatastoreResponse transformReplyIntoDatastoreResponse(QNetworkReply *nwReply) const;
//! Feedback about connection status
//! \threadsafe
void setConnectionStatus(bool ok, const QString &message = "");
//! Feedback about connection status
//! \threadsafe
void setConnectionStatus(QNetworkReply *nwReply);
};
} // ns
} // ns
#endif // guard

View File

@@ -0,0 +1,432 @@
/* Copyright (C) 2015
* swift project Community / Contributors
*
* This file is part of swift project. It is subject to the license terms in the LICENSE file found in the top-level
* directory of this distribution and at http://www.swift-project.org/license.html. No part of swift project,
* including this file, may be copied, modified, propagated, or distributed except according to the terms
* contained in the LICENSE file.
*/
#include "blackcore/application.h"
#include "blackcore/data/globalsetup.h"
#include "blackcore/db/icaodatareader.h"
#include "blackmisc/fileutils.h"
#include "blackmisc/json.h"
#include "blackmisc/logmessage.h"
#include "blackmisc/statusmessage.h"
#include <QDateTime>
#include <QDir>
#include <QFlags>
#include <QJsonDocument>
#include <QNetworkReply>
#include <QReadLocker>
#include <QScopedPointer>
#include <QScopedPointerDeleteLater>
#include <QTimer>
#include <QUrl>
#include <QWriteLocker>
#include <Qt>
#include <QtGlobal>
using namespace BlackMisc;
using namespace BlackMisc::Aviation;
using namespace BlackMisc::Network;
using namespace BlackCore::Data;
namespace BlackCore
{
namespace Db
{
CIcaoDataReader::CIcaoDataReader(QObject *owner, const CDatabaseReaderConfigList &confg) :
CDatabaseReader(owner, confg, "CIcaoDataReader")
{
// void
}
CAircraftIcaoCodeList CIcaoDataReader::getAircraftIcaoCodes() const
{
return m_aircraftIcaoCache.getCopy();
}
CAircraftIcaoCode CIcaoDataReader::getAircraftIcaoCodeForDesignator(const QString &designator) const
{
return getAircraftIcaoCodes().findFirstByDesignatorAndRank(designator);
}
CAircraftIcaoCode CIcaoDataReader::getAircraftIcaoCodeForDbKey(int key) const
{
return getAircraftIcaoCodes().findByKey(key);
}
CAirlineIcaoCodeList CIcaoDataReader::getAirlineIcaoCodes() const
{
return m_airlineIcaoCache.getCopy();
}
CAircraftIcaoCode CIcaoDataReader::smartAircraftIcaoSelector(const CAircraftIcaoCode &icaoPattern) const
{
CAircraftIcaoCodeList codes(getAircraftIcaoCodes()); // thread safe copy
return codes.smartAircraftIcaoSelector(icaoPattern); // sorted by rank
}
CCountryList CIcaoDataReader::getCountries() const
{
return m_countryCache.getCopy();
}
CCountry CIcaoDataReader::getCountryForIsoCode(const QString &isoCode) const
{
return this->getCountries().findByIsoCode(isoCode);
}
CCountry CIcaoDataReader::getCountryForName(const QString &name) const
{
return this->getCountries().findBestMatchByCountryName(name);
}
CAirlineIcaoCodeList CIcaoDataReader::getAirlineIcaoCodesForDesignator(const QString &designator) const
{
return this->getAirlineIcaoCodes().findByVDesignator(designator);
}
CAirlineIcaoCode CIcaoDataReader::getAirlineIcaoCodeForDbKey(int key) const
{
return this->getAirlineIcaoCodes().findByKey(key);
}
CAirlineIcaoCode CIcaoDataReader::smartAirlineIcaoSelector(const CAirlineIcaoCode &icaoPattern) const
{
CAirlineIcaoCodeList codes(this->getAirlineIcaoCodes()); // thread safe copy
return codes.smartAirlineIcaoSelector(icaoPattern);
}
int CIcaoDataReader::getAircraftIcaoCodesCount() const
{
return this->getAircraftIcaoCodes().size();
}
int CIcaoDataReader::getAirlineIcaoCodesCount() const
{
return this->getAirlineIcaoCodes().size();
}
bool CIcaoDataReader::areAllDataRead() const
{
return getCountriesCount() > 0 && getAirlineIcaoCodesCount() > 0 && getAircraftIcaoCodesCount() > 0;
}
int CIcaoDataReader::getCountriesCount() const
{
return this->getCountries().size();
}
void CIcaoDataReader::ps_read(BlackMisc::Network::CEntityFlags::Entity entities, const QDateTime &newerThan)
{
this->threadAssertCheck(); // runs in background thread
CEntityFlags::Entity entitiesTriggered = CEntityFlags::NoEntity;
if (entities.testFlag(CEntityFlags::AircraftIcaoEntity))
{
CUrl url(getAircraftIcaoUrl());
if (!url.isEmpty())
{
if (!newerThan.isNull()) { url.appendQuery("newer=" + newerThan.toString(Qt::ISODate)); }
sApp->getFromNetwork(url, { this, &CIcaoDataReader::ps_parseAircraftIcaoData });
entitiesTriggered |= CEntityFlags::AircraftIcaoEntity;
}
else
{
CLogMessage(this).error("No URL for %1") << CEntityFlags::flagToString(CEntityFlags::AircraftIcaoEntity);
}
}
if (entities.testFlag(CEntityFlags::AirlineIcaoEntity))
{
CUrl url(getAirlineIcaoUrl());
if (!url.isEmpty())
{
if (!newerThan.isNull()) { url.appendQuery("newer=" + newerThan.toString(Qt::ISODate)); }
sApp->getFromNetwork(url, { this, &CIcaoDataReader::ps_parseAirlineIcaoData });
entitiesTriggered |= CEntityFlags::AirlineIcaoEntity;
}
else
{
CLogMessage(this).error("No URL for %1") << CEntityFlags::flagToString(CEntityFlags::AirlineIcaoEntity);
}
}
if (entities.testFlag(CEntityFlags::CountryEntity))
{
CUrl url(getCountryUrl());
if (!url.isEmpty())
{
if (!newerThan.isNull()) { url.appendQuery("newer=" + newerThan.toString(Qt::ISODate)); }
sApp->getFromNetwork(url, { this, &CIcaoDataReader::ps_parseCountryData });
entitiesTriggered |= CEntityFlags::CountryEntity;
}
else
{
CLogMessage(this).error("No URL for %1") << CEntityFlags::flagToString(CEntityFlags::CountryEntity);
}
}
if (entitiesTriggered != CEntityFlags::NoEntity)
{
emit dataRead(entitiesTriggered, CEntityFlags::StartRead, 0);
}
}
void CIcaoDataReader::ps_aircraftIcaoCacheChanged()
{
// void
}
void CIcaoDataReader::ps_airlineIcaoCacheChanged()
{
// void
}
void CIcaoDataReader::ps_countryCacheChanged()
{
// void
}
void CIcaoDataReader::ps_baseUrlCacheChanged()
{
// void
}
void CIcaoDataReader::updateReaderUrl(const CUrl &url)
{
const CUrl current = this->m_readerUrlCache.getCopy();
if (current == url) { return; }
const CStatusMessage m = this->m_readerUrlCache.set(url);
if (m.isFailure())
{
CLogMessage::preformatted(m);
}
}
CUrl CIcaoDataReader::getBaseUrl() const
{
const CUrl baseUrl(sApp->getGlobalSetup().getDbIcaoReaderUrl());
return baseUrl;
}
void CIcaoDataReader::ps_parseAircraftIcaoData(QNetworkReply *nwReplyPtr)
{
// wrap pointer, make sure any exit cleans up reply
// required to use delete later as object is created in a different thread
QScopedPointer<QNetworkReply, QScopedPointerDeleteLater> nwReply(nwReplyPtr);
QString urlString(nwReply->url().toString());
CDatabaseReader::JsonDatastoreResponse res = this->setStatusAndTransformReplyIntoDatastoreResponse(nwReply.data());
if (res.hasErrorMessage())
{
CLogMessage::preformatted(res.lastWarningOrAbove());
emit dataRead(CEntityFlags::AircraftIcaoEntity, CEntityFlags::ReadFailed, 0);
return;
}
const CAircraftIcaoCodeList codes = CAircraftIcaoCodeList::fromDatabaseJson(res);
const int n = codes.size();
qint64 latestTimestamp = codes.latestTimestampMsecsSinceEpoch();
if (n > 0 && latestTimestamp < 0)
{
CLogMessage(this).error("No timestamp in aircraft ICAO list, setting to last modified value");
latestTimestamp = lastModifiedMsSinceEpoch(nwReply.data());
}
this->m_aircraftIcaoCache.set(codes, latestTimestamp);
this->updateReaderUrl(this->getBaseUrl());
emit dataRead(CEntityFlags::AircraftIcaoEntity, CEntityFlags::ReadFinished, n);
CLogMessage(this).info("Read %1 %2 from %3") << n << CEntityFlags::flagToString(CEntityFlags::AircraftIcaoEntity) << urlString;
}
void CIcaoDataReader::ps_parseAirlineIcaoData(QNetworkReply *nwReplyPtr)
{
QScopedPointer<QNetworkReply, QScopedPointerDeleteLater> nwReply(nwReplyPtr);
QString urlString(nwReply->url().toString());
CDatabaseReader::JsonDatastoreResponse res = this->setStatusAndTransformReplyIntoDatastoreResponse(nwReply.data());
if (res.hasErrorMessage())
{
CLogMessage::preformatted(res.lastWarningOrAbove());
emit dataRead(CEntityFlags::AirlineIcaoEntity, CEntityFlags::ReadFailed, 0);
return;
}
const CAirlineIcaoCodeList codes = CAirlineIcaoCodeList::fromDatabaseJson(res);
const int n = codes.size();
qint64 latestTimestamp = codes.latestTimestampMsecsSinceEpoch();
if (n > 0 && latestTimestamp < 0)
{
CLogMessage(this).error("No timestamp in airline ICAO list, setting to last modified value");
latestTimestamp = lastModifiedMsSinceEpoch(nwReply.data());
}
this->m_airlineIcaoCache.set(codes, latestTimestamp);
emit dataRead(CEntityFlags::AirlineIcaoEntity, CEntityFlags::ReadFinished, n);
CLogMessage(this).info("Read %1 %2 from %3") << n << CEntityFlags::flagToString(CEntityFlags::AirlineIcaoEntity) << urlString;
}
void CIcaoDataReader::ps_parseCountryData(QNetworkReply *nwReplyPtr)
{
QScopedPointer<QNetworkReply, QScopedPointerDeleteLater> nwReply(nwReplyPtr);
QString urlString(nwReply->url().toString());
CDatabaseReader::JsonDatastoreResponse res = this->setStatusAndTransformReplyIntoDatastoreResponse(nwReply.data());
if (res.hasErrorMessage())
{
CLogMessage::preformatted(res.lastWarningOrAbove());
emit dataRead(CEntityFlags::CountryEntity, CEntityFlags::ReadFailed, 0);
return;
}
const CCountryList countries = CCountryList::fromDatabaseJson(res);
const int n = countries.size();
qint64 latestTimestamp = countries.latestTimestampMsecsSinceEpoch();
if (n > 0 && latestTimestamp < 0)
{
CLogMessage(this).error("No timestamp in country list, setting to last modified value");
latestTimestamp = lastModifiedMsSinceEpoch(nwReply.data());
}
this->m_countryCache.set(countries, latestTimestamp);
emit dataRead(CEntityFlags::CountryEntity, CEntityFlags::ReadFinished, n);
CLogMessage(this).info("Read %1 %2 from %3") << n << CEntityFlags::flagToString(CEntityFlags::CountryEntity) << urlString;
}
bool CIcaoDataReader::readFromJsonFiles(const QString &dir, CEntityFlags::Entity whatToRead)
{
QDir directory(dir);
if (!directory.exists()) { return false; }
CEntityFlags::Entity reallyRead = CEntityFlags::NoEntity;
if (whatToRead.testFlag(CEntityFlags::CountryEntity))
{
QString countriesJson(CFileUtils::readFileToString(CFileUtils::appendFilePaths(directory.absolutePath(), "countries.json")));
if (!countriesJson.isEmpty())
{
CCountryList countries;
countries.convertFromJson(Json::jsonObjectFromString(countriesJson));
const int c = countries.size();
this->m_countryCache.set(countries);
// Do not emit while locked -> deadlock
reallyRead |= CEntityFlags::CountryEntity;
emit dataRead(CEntityFlags::CountryEntity, CEntityFlags::ReadFinished, c);
}
}
if (whatToRead.testFlag(CEntityFlags::AircraftIcaoEntity))
{
QString aircraftJson(CFileUtils::readFileToString(CFileUtils::appendFilePaths(directory.absolutePath(), "aircrafticao.json")));
if (!aircraftJson.isEmpty())
{
CAircraftIcaoCodeList aircraftIcaos;
aircraftIcaos.convertFromJson(Json::jsonObjectFromString(aircraftJson));
const int c = aircraftIcaos.size();
this->m_aircraftIcaoCache.set(aircraftIcaos);
reallyRead |= CEntityFlags::AircraftIcaoEntity;
emit dataRead(CEntityFlags::AircraftIcaoEntity, CEntityFlags::ReadFinished, c);
}
}
if (whatToRead.testFlag(CEntityFlags::AirlineIcaoEntity))
{
QString airlineJson(CFileUtils::readFileToString(CFileUtils::appendFilePaths(directory.absolutePath(), "airlineicao.json")));
if (!airlineJson.isEmpty())
{
CAirlineIcaoCodeList airlineIcaos;
airlineIcaos.convertFromJson(Json::jsonObjectFromString(airlineJson));
const int c = airlineIcaos.size();
this->m_airlineIcaoCache.set(airlineIcaos);
reallyRead |= CEntityFlags::AirlineIcaoEntity;
emit dataRead(CEntityFlags::AirlineIcaoEntity, CEntityFlags::ReadFinished, c);
}
}
return (whatToRead & CEntityFlags::AllIcaoAndCountries) == reallyRead;
}
bool CIcaoDataReader::readFromJsonFilesInBackground(const QString &dir, CEntityFlags::Entity whatToRead)
{
if (dir.isEmpty() || whatToRead == CEntityFlags::NoEntity) { return false; }
QTimer::singleShot(0, this, [this, dir, whatToRead]()
{
bool s = this->readFromJsonFiles(dir, whatToRead);
Q_UNUSED(s);
});
return true;
}
bool CIcaoDataReader::writeToJsonFiles(const QString &dir) const
{
QDir directory(dir);
if (!directory.exists()) { return false; }
if (this->getCountriesCount() > 0)
{
QString json(QJsonDocument(this->getCountries().toJson()).toJson());
bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), "countries.json"));
if (!s) { return false; }
}
if (this->getAircraftIcaoCodesCount() > 0)
{
QString json(QJsonDocument(this->getAircraftIcaoCodes().toJson()).toJson());
bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), "aircrafticao.json"));
if (!s) { return false; }
}
if (this->getAirlineIcaoCodesCount() > 0)
{
QString json(QJsonDocument(this->getAirlineIcaoCodes().toJson()).toJson());
bool s = CFileUtils::writeStringToFileInBackground(json, CFileUtils::appendFilePaths(directory.absolutePath(), "airlineicao.json"));
if (!s) { return false; }
}
return true;
}
void CIcaoDataReader::syncronizeCaches(CEntityFlags::Entity entities)
{
if (entities.testFlag(CEntityFlags::AircraftIcaoEntity)) { this->m_aircraftIcaoCache.synchronize(); }
if (entities.testFlag(CEntityFlags::AirlineIcaoEntity)) { this->m_airlineIcaoCache.synchronize(); }
if (entities.testFlag(CEntityFlags::CountryEntity)) { this->m_countryCache.synchronize(); }
}
void CIcaoDataReader::invalidateCaches(CEntityFlags::Entity entities)
{
if (entities.testFlag(CEntityFlags::AircraftIcaoEntity)) { CDataCache::instance()->clearAllValues(this->m_aircraftIcaoCache.getKey()); }
if (entities.testFlag(CEntityFlags::AirlineIcaoEntity)) {CDataCache::instance()->clearAllValues(this->m_airlineIcaoCache.getKey()); }
if (entities.testFlag(CEntityFlags::CountryEntity)) {CDataCache::instance()->clearAllValues(this->m_countryCache.getKey()); }
}
QDateTime CIcaoDataReader::getCacheTimestamp(CEntityFlags::Entity entity)
{
switch (entity)
{
case CEntityFlags::AircraftIcaoEntity: return this->m_aircraftIcaoCache.getTimestamp();
case CEntityFlags::AirlineIcaoEntity: return this->m_airlineIcaoCache.getTimestamp();
case CEntityFlags::CountryEntity: return this->m_countryCache.getTimestamp();
default: return QDateTime();
}
}
CUrl CIcaoDataReader::getAircraftIcaoUrl(bool shared) const
{
return shared ?
getWorkingSharedUrl().withAppendedPath("jsonaircrafticao.php") :
getBaseUrl().withAppendedPath("service/jsonaircrafticao.php");
}
CUrl CIcaoDataReader::getAirlineIcaoUrl(bool shared) const
{
return shared ?
getWorkingSharedUrl().withAppendedPath("jsonairlineicao.php") :
getBaseUrl().withAppendedPath("service/jsonairlineicao.php");
}
CUrl CIcaoDataReader::getCountryUrl(bool shared) const
{
return shared ?
getWorkingSharedUrl().withAppendedPath("jsoncountry.php") :
getBaseUrl().withAppendedPath("service/jsoncountry.php");
}
} // ns
} // ns

View File

@@ -0,0 +1,171 @@
/* Copyright (C) 2015
* swift project Community / Contributors
*
* This file is part of swift project. It is subject to the license terms in the LICENSE file found in the top-level
* directory of this distribution and at http://www.swift-project.org/license.html. No part of swift project,
* including this file, may be copied, modified, propagated, or distributed except according to the terms
* contained in the LICENSE file.
*/
//! \file
#ifndef BLACKCORE_ICAODATAREADER_H
#define BLACKCORE_ICAODATAREADER_H
#include "blackcore/blackcoreexport.h"
#include "blackcore/db/databasereader.h"
#include "blackcore/data/dbcaches.h"
#include "blackmisc/aviation/aircrafticaocode.h"
#include "blackmisc/aviation/aircrafticaocodelist.h"
#include "blackmisc/aviation/airlineicaocode.h"
#include "blackmisc/aviation/airlineicaocodelist.h"
#include "blackmisc/network/entityflags.h"
#include "blackmisc/network/url.h"
#include "blackmisc/country.h"
#include "blackmisc/countrylist.h"
#include "blackmisc/datacache.h"
#include <QObject>
#include <QReadWriteLock>
#include <QString>
class QDateTime;
class QNetworkReply;
namespace BlackCore
{
namespace Db
{
//! Read ICAO data from Database
class BLACKCORE_EXPORT CIcaoDataReader : public CDatabaseReader
{
Q_OBJECT
public:
//! Constructor
explicit CIcaoDataReader(QObject *owner, const CDatabaseReaderConfigList &confg);
//! Get aircraft ICAO information
//! \threadsafe
BlackMisc::Aviation::CAircraftIcaoCodeList getAircraftIcaoCodes() const;
//! Get aircraft ICAO information for designator
//! \threadsafe
BlackMisc::Aviation::CAircraftIcaoCode getAircraftIcaoCodeForDesignator(const QString &designator) const;
//! Get aircraft ICAO information for key
//! \threadsafe
BlackMisc::Aviation::CAircraftIcaoCode getAircraftIcaoCodeForDbKey(int key) const;
//! Get airline ICAO information
//! \threadsafe
BlackMisc::Aviation::CAirlineIcaoCodeList getAirlineIcaoCodes() const;
//! Get best match for incomplete aircraft ICAO code
//! \threadsafe
BlackMisc::Aviation::CAircraftIcaoCode smartAircraftIcaoSelector(const BlackMisc::Aviation::CAircraftIcaoCode &icaoPattern) const;
//! Get countries
//! \threadsafe
BlackMisc::CCountryList getCountries() const;
//! Get countries count
//! \threadsafe
int getCountriesCount() const;
//! Get country for ISO code
//! \threadsafe
BlackMisc::CCountry getCountryForIsoCode(const QString &isoCode) const;
//! Get country for ISO name
//! \threadsafe
BlackMisc::CCountry getCountryForName(const QString &name) const;
//! Get airline ICAO information for designator
//! \threadsafe
BlackMisc::Aviation::CAirlineIcaoCodeList getAirlineIcaoCodesForDesignator(const QString &designator) const;
//! Get airline ICAO information for key
//! \threadsafe
BlackMisc::Aviation::CAirlineIcaoCode getAirlineIcaoCodeForDbKey(int key) const;
//! Get best match for incomplete airline ICAO code
//! \threadsafe
BlackMisc::Aviation::CAirlineIcaoCode smartAirlineIcaoSelector(const BlackMisc::Aviation::CAirlineIcaoCode &icaoPattern) const;
//! Get aircraft ICAO information count
//! \threadsafe
int getAircraftIcaoCodesCount() const;
//! Get airline ICAO information count
//! \threadsafe
int getAirlineIcaoCodesCount() const;
//! All data read?
//! \threadsafe
bool areAllDataRead() const;
//! Read from static DB data file
bool readFromJsonFiles(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead = BlackMisc::Network::CEntityFlags::AllIcaoAndCountries);
//! Read from static DB data file
bool readFromJsonFilesInBackground(const QString &dir, BlackMisc::Network::CEntityFlags::Entity whatToRead = BlackMisc::Network::CEntityFlags::AllIcaoAndCountries);
//! Write to static DB data file
bool writeToJsonFiles(const QString &dir) const;
signals:
//! Combined read signal
void dataRead(BlackMisc::Network::CEntityFlags::Entity entity, BlackMisc::Network::CEntityFlags::ReadState state, int number);
protected:
//! \name cache handling for base class
//! @{
virtual void syncronizeCaches(BlackMisc::Network::CEntityFlags::Entity entities) override;
virtual void invalidateCaches(BlackMisc::Network::CEntityFlags::Entity entities) override;
virtual QDateTime getCacheTimestamp(BlackMisc::Network::CEntityFlags::Entity entity) override;
//! @}
private slots:
//! Aircraft have been read
void ps_parseAircraftIcaoData(QNetworkReply *nwReply);
//! Airlines have been read
void ps_parseAirlineIcaoData(QNetworkReply *nwReply);
//! Airlines have been read
void ps_parseCountryData(QNetworkReply *nwReply);
//! Read / re-read data file
void ps_read(BlackMisc::Network::CEntityFlags::Entity entities, const QDateTime &newerThan);
void ps_aircraftIcaoCacheChanged();
void ps_airlineIcaoCacheChanged();
void ps_countryCacheChanged();
void ps_baseUrlCacheChanged();
private:
BlackMisc::CData<BlackCore::Data::DbAircraftIcaoCache> m_aircraftIcaoCache {this, &CIcaoDataReader::ps_aircraftIcaoCacheChanged };
BlackMisc::CData<BlackCore::Data::DbAirlineIcaoCache> m_airlineIcaoCache {this, &CIcaoDataReader::ps_airlineIcaoCacheChanged };
BlackMisc::CData<BlackCore::Data::DbCountryCache> m_countryCache {this, &CIcaoDataReader::ps_countryCacheChanged };
BlackMisc::CData<BlackCore::Data::DbIcaoReaderBaseUrl> m_readerUrlCache {this, &CIcaoDataReader::ps_baseUrlCacheChanged };
//! Update reader URL
void updateReaderUrl(const BlackMisc::Network::CUrl &url);
//! Base URL
BlackMisc::Network::CUrl getBaseUrl() const;
//! URL
BlackMisc::Network::CUrl getAircraftIcaoUrl(bool shared = false) const;
//! URL
BlackMisc::Network::CUrl getAirlineIcaoUrl(bool shared = false) const;
//! URL
BlackMisc::Network::CUrl getCountryUrl(bool shared = false) const;
};
} // ns
} // ns
#endif // guard