flightsimulatorplatfrom/pilotclient
1
0
Fork 0
mirror of https://github.com/swift-project/pilotclient.git synced 2026-03-23 07:15:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

Ref T308, loading JSON from DB optimizations

Browse Source 
- info about time consumed for parsing (so we can benchmark)
- using the "optimized" YYYYmmdd parsing
This commit is contained in:
Klaus Basan
2018-08-21 15:55:14 +02:00
parent 183280fa75
commit 12f6e006ab
7 changed files with 76 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/blackcore/db/airportdatareader.cpp
View File

@@ -209,7 +209,10 @@ namespace BlackCore
}
else
{
QTime time;
time.start();
airports = CAirportList::fromDatabaseJson(res, &inconsistent);
this->logParseMessage("airports", airports.size(), time.elapsed(), res);
}
if (!inconsistent.isEmpty())

17
src/blackcore/db/databasereader.cpp
View File

@@ -25,6 +25,7 @@
#include <QByteArray>
#include <QJsonDocument>
#include <QJsonObject>
#include <QPointer>
#include <QFileInfo>
#include <QJsonValueRef>
#include <QMetaObject>
@@ -196,8 +197,10 @@ namespace BlackCore
// Using timer to first finish this function, then the resulting signal
if (validInCacheEntities != CEntityFlags::NoEntity)
{
QPointer<CDatabaseReader> myself(this);
QTimer::singleShot(0, this, [ = ]
{
if (!myself) { return; }
emit this->dataRead(validInCacheEntities, CEntityFlags::ReadFinished, 0);
});
}
@@ -234,6 +237,7 @@ namespace BlackCore
{
const QString dataFileData = nwReply->readAll();
nwReply->close(); // close asap
datastoreResponse.setStringSize(dataFileData.size());
if (dataFileData.isEmpty())
{
datastoreResponse.setMessage(CStatusMessage(this, CStatusMessage::SeverityError, "Empty response, no data"));
@@ -662,6 +666,13 @@ namespace BlackCore
}
}
void CDatabaseReader::logParseMessage(const QString &entity, int size, int msElapsed, const CDatabaseReader::JsonDatastoreResponse &response) const
{
CLogMessage(this).info("Parsed %1 %2 in %3ms, thread '%4' | '%5'")
<< size << entity << msElapsed
<< QThread::currentThread()->objectName() << response.toQString();
}
QString CDatabaseReader::fileNameForMode(CEntityFlags::Entity entity, CDbFlags::DataRetrievalModeFlag mode)
{
Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "needs single entity");
@@ -786,6 +797,12 @@ namespace BlackCore
m_arraySize = value.size();
}
QString CDatabaseReader::JsonDatastoreResponse::toQString() const
{
static const QString s("DB: %1 | restricted: %2 | array: %3 | string size: %4 | content: %5");
return s.arg(boolToYesNo(this->isLoadedFromDb()), boolToYesNo(this->isRestricted())).arg(this->getArraySize()).arg(m_stringSize).arg(this->getContentLengthHeader());
}
bool CDatabaseReader::HeaderResponse::isSharedFile() const
{
const QString fn(getUrl().getFileName());

20
src/blackcore/db/databasereader.h
View File

@@ -132,30 +132,37 @@ namespace BlackCore
private:
QJsonArray m_jsonArray; //!< JSON array data
int m_arraySize = -1; //!< size of array, if applicable (copied to member for debugging purposes)
int m_stringSize = 0; //!< string size of JSON data
bool m_restricted = false; //!< restricted reponse, only changed data
public:
//! Any data?
bool isEmpty() const { return m_jsonArray.isEmpty(); }
//! Number of elements
int size() const { return m_jsonArray.size(); }
//! Is loaded from database
bool isLoadedFromDb() const;
//! Incremental data, restricted by query?
bool isRestricted() const { return m_restricted; }
//! Is loaded from database
bool isLoadedFromDb() const;
//! Mark as restricted
void setRestricted(bool restricted) { m_restricted = restricted; }
//! Get the JSON array
QJsonArray getJsonArray() const { return m_jsonArray; }
//! Number of elements
int getArraySize() const { return m_jsonArray.size(); }
//! Set the JSON array
void setJsonArray(const QJsonArray &value);
//! Set string size
void setStringSize(int size) { m_stringSize = size; }
//! String info
QString toQString() const;
//! Implicit conversion
operator QJsonArray() const { return m_jsonArray; }
};
@@ -411,6 +418,9 @@ namespace BlackCore
//! Override cache from file
//! \threadsafe
bool overrideCacheFromFile(bool overrideNewerOnly, const QFileInfo &fileInfo, BlackMisc::Network::CEntityFlags::Entity entity, BlackMisc::CStatusMessageList &msgs) const;
//! Parsing info message
void logParseMessage(const QString &entity, int size, int msElapsed, const JsonDatastoreResponse &response) const;
};
} // ns
} // ns

15
src/blackcore/db/icaodatareader.cpp
View File

@@ -281,8 +281,11 @@ namespace BlackCore
}
else
{
// normally read from special view which already filters incomplete
// normally read from special DB view which already filters incomplete
QTime time;
time.start();
codes = CAircraftIcaoCodeList::fromDatabaseJson(res, true, &inconsistent);
this->logParseMessage("aircraft ICAO", codes.size(), time.elapsed(), res);
}
if (!inconsistent.isEmpty())
@@ -332,8 +335,11 @@ namespace BlackCore
}
else
{
// normally read from special view which already filters incomplete
// normally read from special DB view which already filters incomplete
QTime time;
time.start();
codes = CAirlineIcaoCodeList::fromDatabaseJson(res, true, &inconsistent);
this->logParseMessage("airline ICAO", codes.size(), time.elapsed(), res);
}
if (!inconsistent.isEmpty())
@@ -380,8 +386,11 @@ namespace BlackCore
}
else
{
// normally read from special view which already filters incomplete
// normally read from special DB view which already filters incomplete
QTime time;
time.start();
countries = CCountryList::fromDatabaseJson(res);
this->logParseMessage("countries", countries.size(), time.elapsed(), res);
}
if (!this->doWorkCheck()) { return; }

9
src/blackcore/db/modeldatareader.cpp
View File

@@ -299,7 +299,10 @@ namespace BlackCore
}
else
{
QTime time;
time.start();
liveries = CLiveryList::fromDatabaseJson(res);
this->logParseMessage("liveries", liveries.size(), time.elapsed(), res);
}
if (!this->doWorkCheck()) { return; }
@@ -343,7 +346,10 @@ namespace BlackCore
}
else
{
QTime time;
time.start();
distributors = CDistributorList::fromDatabaseJson(res);
this->logParseMessage("distributors", distributors.size(), time.elapsed(), res);
}
if (!this->doWorkCheck()) { return; }
@@ -391,8 +397,7 @@ namespace BlackCore
QTime time;
time.start();
models = CAircraftModelList::fromDatabaseJson(res);
const int elapsed = time.elapsed();
CLogMessage(this).info("Parsed %1 models in %2 ms") << models.size() << elapsed;
this->logParseMessage("models", models.size(), time.elapsed(), res);
}
// synchronized update

42
src/blackmisc/aviation/aircrafticaocode.cpp
View File

@@ -738,10 +738,10 @@ namespace BlackMisc
}
// turn E to P engine
if (combinedCode.endsWith("E")) { return QStringList({ combinedCode.leftRef(2) + "P"}); }
if (combinedCode.endsWith("E")) { return QStringList({ combinedCode.leftRef(2) % QStringLiteral("P")}); }
// turn T to H plane (tilt wing to helicopter
if (combinedCode.startsWith("T")) { return QStringList({ "H" + combinedCode.rightRef(2)}); }
if (combinedCode.startsWith("T")) { return QStringList({ QStringLiteral("H") % combinedCode.rightRef(2)}); }
// based on engine count
QStringList codes;
@@ -766,18 +766,18 @@ namespace BlackMisc
return CAircraftIcaoCode();
}
const QString designator(json.value(prefix + "designator").toString());
const QString iata(json.value(prefix + "iata").toString());
const QString family(json.value(prefix + "family").toString());
const QString manufacturer(json.value(prefix + "manufacturer").toString());
const QString model(json.value(prefix + "model").toString());
const QString modelIata(json.value(prefix + "modeliata").toString());
const QString modelSwift(json.value(prefix + "modelswift").toString());
const QString type(json.value(prefix + "type").toString());
const QString engine(json.value(prefix + "engine").toString());
const int engineCount(json.value(prefix + "enginecount").toInt(-1));
const QString designator(json.value(prefix % QStringLiteral("designator")).toString());
const QString iata(json.value(prefix % QStringLiteral("iata")).toString());
const QString family(json.value(prefix % QStringLiteral("family")).toString());
const QString manufacturer(json.value(prefix % QStringLiteral("manufacturer")).toString());
const QString model(json.value(prefix % QStringLiteral("model")).toString());
const QString modelIata(json.value(prefix % QStringLiteral("modeliata")).toString());
const QString modelSwift(json.value(prefix % QStringLiteral("modelswift")).toString());
const QString type(json.value(prefix % QStringLiteral("type")).toString());
const QString engine(json.value(prefix % QStringLiteral("engine")).toString());
const int engineCount(json.value(prefix % QStringLiteral("enginecount")).toInt(-1));
const QString combined(createdCombinedString(type, engineCount, engine));
QString wtc(json.value(prefix + "wtc").toString());
QString wtc(json.value(prefix % QStringLiteral("wtc")).toString());
if (wtc.length() > 1 && wtc.contains("/"))
{
// "L/M" -> "M"
@@ -785,10 +785,10 @@ namespace BlackMisc
}
Q_ASSERT_X(wtc.length() < 2, Q_FUNC_INFO, "WTC too long");
const bool real = CDatastoreUtility::dbBoolStringToBool(json.value(prefix + "realworld").toString());
const bool legacy = CDatastoreUtility::dbBoolStringToBool(json.value(prefix + "legacy").toString());
const bool military = CDatastoreUtility::dbBoolStringToBool(json.value(prefix + "military").toString());
const int rank(json.value(prefix + "rank").toInt(10));
const bool real = CDatastoreUtility::dbBoolStringToBool(json.value(prefix % QStringLiteral("realworld")).toString());
const bool legacy = CDatastoreUtility::dbBoolStringToBool(json.value(prefix % QStringLiteral("legacy")).toString());
const bool military = CDatastoreUtility::dbBoolStringToBool(json.value(prefix % QStringLiteral("military")).toString());
const int rank(json.value(prefix % QStringLiteral("rank")).toInt(10));
CAircraftIcaoCode code(
designator, iata, family, combined, manufacturer,
@@ -802,11 +802,9 @@ namespace BlackMisc
QString CAircraftIcaoCode::createdCombinedString(const QString &type, const QString &engineCount, const QString &engine)
{
Q_ASSERT_X(engineCount.length() < 2, Q_FUNC_INFO, "Wrong engine count");
QString c(type.isEmpty() ? "-" : type.trimmed().left(1).toUpper());
c += engineCount.isEmpty() ? "-" : engineCount.trimmed();
c += engine.isEmpty() ? "-" : engine.trimmed().left(1).toUpper();
Q_ASSERT_X(c.length() == 3, Q_FUNC_INFO, "Wrong combined length");
return c;
return (type.isEmpty() ? QStringLiteral("-") : type.trimmed().left(1).toUpper()) %
(engineCount.isEmpty() ? QStringLiteral("-") : engineCount.trimmed()) %
(engine.isEmpty() ? QStringLiteral("-") : engine.trimmed().left(1).toUpper());
}
QString CAircraftIcaoCode::createdCombinedString(const QString &type, int engineCount, const QString &engine)

13
src/blackmisc/db/datastoreutility.cpp
View File

@@ -63,17 +63,8 @@ namespace BlackMisc
QDateTime CDatastoreUtility::parseTimestamp(const QString &timestamp)
{
if (!timestamp.isEmpty())
{
const QString ts(timestamp.trimmed().remove(' ').remove('-').remove(':')); // normalize
QDateTime dt = QDateTime::fromString(ts, "yyyyMMddHHmmss");
dt.setTimeZone(QTimeZone::utc());
return dt;
}
else
{
return QDateTime();
}
if (timestamp.isEmpty()) { return QDateTime(); }
return parseDateTimeStringOptimized(removeDateTimeSeparators(timestamp));
}
bool CDatastoreUtility::parseSwiftPublishResponse(const QString &jsonResponse, CAircraftModelList &publishedModels, CAircraftModelList &skippedModels, CStatusMessageList &messages, bool &directWrite)