From 12f6e006abc3dcbfeb16915a45ee8597ec792081 Mon Sep 17 00:00:00 2001 From: Klaus Basan Date: Tue, 21 Aug 2018 15:55:14 +0200 Subject: [PATCH] Ref T308, loading JSON from DB optimizations - info about time consumed for parsing (so we can benchmark) - using the "optimized" YYYYmmdd parsing --- src/blackcore/db/airportdatareader.cpp | 3 ++ src/blackcore/db/databasereader.cpp | 17 +++++++++ src/blackcore/db/databasereader.h | 20 +++++++--- src/blackcore/db/icaodatareader.cpp | 15 ++++++-- src/blackcore/db/modeldatareader.cpp | 9 ++++- src/blackmisc/aviation/aircrafticaocode.cpp | 42 ++++++++++----------- src/blackmisc/db/datastoreutility.cpp | 13 +------ 7 files changed, 76 insertions(+), 43 deletions(-) diff --git a/src/blackcore/db/airportdatareader.cpp b/src/blackcore/db/airportdatareader.cpp index 3c094ef4a..d4d13cb07 100644 --- a/src/blackcore/db/airportdatareader.cpp +++ b/src/blackcore/db/airportdatareader.cpp @@ -209,7 +209,10 @@ namespace BlackCore } else { + QTime time; + time.start(); airports = CAirportList::fromDatabaseJson(res, &inconsistent); + this->logParseMessage("airports", airports.size(), time.elapsed(), res); } if (!inconsistent.isEmpty()) diff --git a/src/blackcore/db/databasereader.cpp b/src/blackcore/db/databasereader.cpp index 1c25b1882..6cfedc114 100644 --- a/src/blackcore/db/databasereader.cpp +++ b/src/blackcore/db/databasereader.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -196,8 +197,10 @@ namespace BlackCore // Using timer to first finish this function, then the resulting signal if (validInCacheEntities != CEntityFlags::NoEntity) { + QPointer myself(this); QTimer::singleShot(0, this, [ = ] { + if (!myself) { return; } emit this->dataRead(validInCacheEntities, CEntityFlags::ReadFinished, 0); }); } @@ -234,6 +237,7 @@ namespace BlackCore { const QString dataFileData = nwReply->readAll(); nwReply->close(); // close asap + datastoreResponse.setStringSize(dataFileData.size()); if (dataFileData.isEmpty()) { datastoreResponse.setMessage(CStatusMessage(this, CStatusMessage::SeverityError, "Empty response, no data")); @@ -662,6 +666,13 @@ namespace BlackCore } } + void CDatabaseReader::logParseMessage(const QString &entity, int size, int msElapsed, const CDatabaseReader::JsonDatastoreResponse &response) const + { + CLogMessage(this).info("Parsed %1 %2 in %3ms, thread '%4' | '%5'") + << size << entity << msElapsed + << QThread::currentThread()->objectName() << response.toQString(); + } + QString CDatabaseReader::fileNameForMode(CEntityFlags::Entity entity, CDbFlags::DataRetrievalModeFlag mode) { Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "needs single entity"); @@ -786,6 +797,12 @@ namespace BlackCore m_arraySize = value.size(); } + QString CDatabaseReader::JsonDatastoreResponse::toQString() const + { + static const QString s("DB: %1 | restricted: %2 | array: %3 | string size: %4 | content: %5"); + return s.arg(boolToYesNo(this->isLoadedFromDb()), boolToYesNo(this->isRestricted())).arg(this->getArraySize()).arg(m_stringSize).arg(this->getContentLengthHeader()); + } + bool CDatabaseReader::HeaderResponse::isSharedFile() const { const QString fn(getUrl().getFileName()); diff --git a/src/blackcore/db/databasereader.h b/src/blackcore/db/databasereader.h index f90a124bb..e9ff667e0 100644 --- a/src/blackcore/db/databasereader.h +++ b/src/blackcore/db/databasereader.h @@ -132,30 +132,37 @@ namespace BlackCore private: QJsonArray m_jsonArray; //!< JSON array data int m_arraySize = -1; //!< size of array, if applicable (copied to member for debugging purposes) + int m_stringSize = 0; //!< string size of JSON data bool m_restricted = false; //!< restricted reponse, only changed data public: //! Any data? bool isEmpty() const { return m_jsonArray.isEmpty(); } - //! Number of elements - int size() const { return m_jsonArray.size(); } + //! Is loaded from database + bool isLoadedFromDb() const; //! Incremental data, restricted by query? bool isRestricted() const { return m_restricted; } - //! Is loaded from database - bool isLoadedFromDb() const; - //! Mark as restricted void setRestricted(bool restricted) { m_restricted = restricted; } //! Get the JSON array QJsonArray getJsonArray() const { return m_jsonArray; } + //! Number of elements + int getArraySize() const { return m_jsonArray.size(); } + //! Set the JSON array void setJsonArray(const QJsonArray &value); + //! Set string size + void setStringSize(int size) { m_stringSize = size; } + + //! String info + QString toQString() const; + //! Implicit conversion operator QJsonArray() const { return m_jsonArray; } }; @@ -411,6 +418,9 @@ namespace BlackCore //! Override cache from file //! \threadsafe bool overrideCacheFromFile(bool overrideNewerOnly, const QFileInfo &fileInfo, BlackMisc::Network::CEntityFlags::Entity entity, BlackMisc::CStatusMessageList &msgs) const; + + //! Parsing info message + void logParseMessage(const QString &entity, int size, int msElapsed, const JsonDatastoreResponse &response) const; }; } // ns } // ns diff --git a/src/blackcore/db/icaodatareader.cpp b/src/blackcore/db/icaodatareader.cpp index 5bb244942..6ce284904 100644 --- a/src/blackcore/db/icaodatareader.cpp +++ b/src/blackcore/db/icaodatareader.cpp @@ -281,8 +281,11 @@ namespace BlackCore } else { - // normally read from special view which already filters incomplete + // normally read from special DB view which already filters incomplete + QTime time; + time.start(); codes = CAircraftIcaoCodeList::fromDatabaseJson(res, true, &inconsistent); + this->logParseMessage("aircraft ICAO", codes.size(), time.elapsed(), res); } if (!inconsistent.isEmpty()) @@ -332,8 +335,11 @@ namespace BlackCore } else { - // normally read from special view which already filters incomplete + // normally read from special DB view which already filters incomplete + QTime time; + time.start(); codes = CAirlineIcaoCodeList::fromDatabaseJson(res, true, &inconsistent); + this->logParseMessage("airline ICAO", codes.size(), time.elapsed(), res); } if (!inconsistent.isEmpty()) @@ -380,8 +386,11 @@ namespace BlackCore } else { - // normally read from special view which already filters incomplete + // normally read from special DB view which already filters incomplete + QTime time; + time.start(); countries = CCountryList::fromDatabaseJson(res); + this->logParseMessage("countries", countries.size(), time.elapsed(), res); } if (!this->doWorkCheck()) { return; } diff --git a/src/blackcore/db/modeldatareader.cpp b/src/blackcore/db/modeldatareader.cpp index b73dd2fae..8c4cd1f10 100644 --- a/src/blackcore/db/modeldatareader.cpp +++ b/src/blackcore/db/modeldatareader.cpp @@ -299,7 +299,10 @@ namespace BlackCore } else { + QTime time; + time.start(); liveries = CLiveryList::fromDatabaseJson(res); + this->logParseMessage("liveries", liveries.size(), time.elapsed(), res); } if (!this->doWorkCheck()) { return; } @@ -343,7 +346,10 @@ namespace BlackCore } else { + QTime time; + time.start(); distributors = CDistributorList::fromDatabaseJson(res); + this->logParseMessage("distributors", distributors.size(), time.elapsed(), res); } if (!this->doWorkCheck()) { return; } @@ -391,8 +397,7 @@ namespace BlackCore QTime time; time.start(); models = CAircraftModelList::fromDatabaseJson(res); - const int elapsed = time.elapsed(); - CLogMessage(this).info("Parsed %1 models in %2 ms") << models.size() << elapsed; + this->logParseMessage("models", models.size(), time.elapsed(), res); } // synchronized update diff --git a/src/blackmisc/aviation/aircrafticaocode.cpp b/src/blackmisc/aviation/aircrafticaocode.cpp index be0825a2c..bbbb1aba1 100644 --- a/src/blackmisc/aviation/aircrafticaocode.cpp +++ b/src/blackmisc/aviation/aircrafticaocode.cpp @@ -738,10 +738,10 @@ namespace BlackMisc } // turn E to P engine - if (combinedCode.endsWith("E")) { return QStringList({ combinedCode.leftRef(2) + "P"}); } + if (combinedCode.endsWith("E")) { return QStringList({ combinedCode.leftRef(2) % QStringLiteral("P")}); } // turn T to H plane (tilt wing to helicopter - if (combinedCode.startsWith("T")) { return QStringList({ "H" + combinedCode.rightRef(2)}); } + if (combinedCode.startsWith("T")) { return QStringList({ QStringLiteral("H") % combinedCode.rightRef(2)}); } // based on engine count QStringList codes; @@ -766,18 +766,18 @@ namespace BlackMisc return CAircraftIcaoCode(); } - const QString designator(json.value(prefix + "designator").toString()); - const QString iata(json.value(prefix + "iata").toString()); - const QString family(json.value(prefix + "family").toString()); - const QString manufacturer(json.value(prefix + "manufacturer").toString()); - const QString model(json.value(prefix + "model").toString()); - const QString modelIata(json.value(prefix + "modeliata").toString()); - const QString modelSwift(json.value(prefix + "modelswift").toString()); - const QString type(json.value(prefix + "type").toString()); - const QString engine(json.value(prefix + "engine").toString()); - const int engineCount(json.value(prefix + "enginecount").toInt(-1)); + const QString designator(json.value(prefix % QStringLiteral("designator")).toString()); + const QString iata(json.value(prefix % QStringLiteral("iata")).toString()); + const QString family(json.value(prefix % QStringLiteral("family")).toString()); + const QString manufacturer(json.value(prefix % QStringLiteral("manufacturer")).toString()); + const QString model(json.value(prefix % QStringLiteral("model")).toString()); + const QString modelIata(json.value(prefix % QStringLiteral("modeliata")).toString()); + const QString modelSwift(json.value(prefix % QStringLiteral("modelswift")).toString()); + const QString type(json.value(prefix % QStringLiteral("type")).toString()); + const QString engine(json.value(prefix % QStringLiteral("engine")).toString()); + const int engineCount(json.value(prefix % QStringLiteral("enginecount")).toInt(-1)); const QString combined(createdCombinedString(type, engineCount, engine)); - QString wtc(json.value(prefix + "wtc").toString()); + QString wtc(json.value(prefix % QStringLiteral("wtc")).toString()); if (wtc.length() > 1 && wtc.contains("/")) { // "L/M" -> "M" @@ -785,10 +785,10 @@ namespace BlackMisc } Q_ASSERT_X(wtc.length() < 2, Q_FUNC_INFO, "WTC too long"); - const bool real = CDatastoreUtility::dbBoolStringToBool(json.value(prefix + "realworld").toString()); - const bool legacy = CDatastoreUtility::dbBoolStringToBool(json.value(prefix + "legacy").toString()); - const bool military = CDatastoreUtility::dbBoolStringToBool(json.value(prefix + "military").toString()); - const int rank(json.value(prefix + "rank").toInt(10)); + const bool real = CDatastoreUtility::dbBoolStringToBool(json.value(prefix % QStringLiteral("realworld")).toString()); + const bool legacy = CDatastoreUtility::dbBoolStringToBool(json.value(prefix % QStringLiteral("legacy")).toString()); + const bool military = CDatastoreUtility::dbBoolStringToBool(json.value(prefix % QStringLiteral("military")).toString()); + const int rank(json.value(prefix % QStringLiteral("rank")).toInt(10)); CAircraftIcaoCode code( designator, iata, family, combined, manufacturer, @@ -802,11 +802,9 @@ namespace BlackMisc QString CAircraftIcaoCode::createdCombinedString(const QString &type, const QString &engineCount, const QString &engine) { Q_ASSERT_X(engineCount.length() < 2, Q_FUNC_INFO, "Wrong engine count"); - QString c(type.isEmpty() ? "-" : type.trimmed().left(1).toUpper()); - c += engineCount.isEmpty() ? "-" : engineCount.trimmed(); - c += engine.isEmpty() ? "-" : engine.trimmed().left(1).toUpper(); - Q_ASSERT_X(c.length() == 3, Q_FUNC_INFO, "Wrong combined length"); - return c; + return (type.isEmpty() ? QStringLiteral("-") : type.trimmed().left(1).toUpper()) % + (engineCount.isEmpty() ? QStringLiteral("-") : engineCount.trimmed()) % + (engine.isEmpty() ? QStringLiteral("-") : engine.trimmed().left(1).toUpper()); } QString CAircraftIcaoCode::createdCombinedString(const QString &type, int engineCount, const QString &engine) diff --git a/src/blackmisc/db/datastoreutility.cpp b/src/blackmisc/db/datastoreutility.cpp index b65460df7..63b8716f0 100644 --- a/src/blackmisc/db/datastoreutility.cpp +++ b/src/blackmisc/db/datastoreutility.cpp @@ -63,17 +63,8 @@ namespace BlackMisc QDateTime CDatastoreUtility::parseTimestamp(const QString ×tamp) { - if (!timestamp.isEmpty()) - { - const QString ts(timestamp.trimmed().remove(' ').remove('-').remove(':')); // normalize - QDateTime dt = QDateTime::fromString(ts, "yyyyMMddHHmmss"); - dt.setTimeZone(QTimeZone::utc()); - return dt; - } - else - { - return QDateTime(); - } + if (timestamp.isEmpty()) { return QDateTime(); } + return parseDateTimeStringOptimized(removeDateTimeSeparators(timestamp)); } bool CDatastoreUtility::parseSwiftPublishResponse(const QString &jsonResponse, CAircraftModelList &publishedModels, CAircraftModelList &skippedModels, CStatusMessageList &messages, bool &directWrite)