Ref T308, loading JSON from DB optimizations

- info about time consumed for parsing (so we can benchmark)
- using the "optimized" YYYYmmdd parsing

src/blackcore/db/airportdatareader.cpp

@@ -209,7 +209,10 @@ namespace BlackCore
             }
             else
             {
+                QTime time;
+                time.start();
                 airports = CAirportList::fromDatabaseJson(res, &inconsistent);
+                this->logParseMessage("airports", airports.size(), time.elapsed(), res);
             }
 
             if (!inconsistent.isEmpty())

src/blackcore/db/databasereader.cpp

@@ -25,6 +25,7 @@
 #include <QByteArray>
 #include <QJsonDocument>
 #include <QJsonObject>
+#include <QPointer>
 #include <QFileInfo>
 #include <QJsonValueRef>
 #include <QMetaObject>
@@ -196,8 +197,10 @@ namespace BlackCore
                     // Using timer to first finish this function, then the resulting signal
                     if (validInCacheEntities != CEntityFlags::NoEntity)
                     {
+                        QPointer<CDatabaseReader> myself(this);
                         QTimer::singleShot(0, this, [ = ]
                         {
+                            if (!myself) { return; }
                             emit this->dataRead(validInCacheEntities, CEntityFlags::ReadFinished, 0);
                         });
                     }
@@ -234,6 +237,7 @@ namespace BlackCore
             {
                 const QString dataFileData = nwReply->readAll();
                 nwReply->close(); // close asap
+                datastoreResponse.setStringSize(dataFileData.size());
                 if (dataFileData.isEmpty())
                 {
                     datastoreResponse.setMessage(CStatusMessage(this, CStatusMessage::SeverityError, "Empty response, no data"));
@@ -662,6 +666,13 @@ namespace BlackCore
             }
         }
 
+        void CDatabaseReader::logParseMessage(const QString &entity, int size, int msElapsed, const CDatabaseReader::JsonDatastoreResponse &response) const
+        {
+            CLogMessage(this).info("Parsed %1 %2 in %3ms, thread '%4' | '%5'")
+                    << size << entity << msElapsed
+                    << QThread::currentThread()->objectName() << response.toQString();
+        }
+
         QString CDatabaseReader::fileNameForMode(CEntityFlags::Entity entity, CDbFlags::DataRetrievalModeFlag mode)
         {
             Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "needs single entity");
@@ -786,6 +797,12 @@ namespace BlackCore
             m_arraySize = value.size();
         }
 
+        QString CDatabaseReader::JsonDatastoreResponse::toQString() const
+        {
+            static const QString s("DB: %1 | restricted: %2 | array: %3 | string size: %4 | content: %5");
+            return s.arg(boolToYesNo(this->isLoadedFromDb()), boolToYesNo(this->isRestricted())).arg(this->getArraySize()).arg(m_stringSize).arg(this->getContentLengthHeader());
+        }
+
         bool CDatabaseReader::HeaderResponse::isSharedFile() const
         {
             const QString fn(getUrl().getFileName());

src/blackcore/db/databasereader.h

@@ -132,30 +132,37 @@ namespace BlackCore
             private:
                 QJsonArray m_jsonArray;          //!< JSON array data
                 int        m_arraySize  = -1;    //!< size of array, if applicable (copied to member for debugging purposes)
+                int        m_stringSize =  0;    //!< string size of JSON data
                 bool       m_restricted = false; //!< restricted reponse, only changed data
 
             public:
                 //! Any data?
                 bool isEmpty() const { return m_jsonArray.isEmpty(); }
 
-                //! Number of elements
+                //! Is loaded from database
-                int size() const { return m_jsonArray.size(); }
+                bool isLoadedFromDb() const;
 
                 //! Incremental data, restricted by query?
                 bool isRestricted() const { return m_restricted; }
 
-                //! Is loaded from database
-                bool isLoadedFromDb() const;
-
                 //! Mark as restricted
                 void setRestricted(bool restricted) { m_restricted = restricted; }
 
                 //! Get the JSON array
                 QJsonArray getJsonArray() const { return m_jsonArray; }
 
+                //! Number of elements
+                int getArraySize() const { return m_jsonArray.size(); }
+
                 //! Set the JSON array
                 void setJsonArray(const QJsonArray &value);
 
+                //! Set string size
+                void setStringSize(int size) { m_stringSize = size; }
+
+                //! String info
+                QString toQString() const;
+
                 //! Implicit conversion
                 operator QJsonArray() const { return m_jsonArray; }
             };
@@ -411,6 +418,9 @@ namespace BlackCore
             //! Override cache from file
             //! \threadsafe
             bool overrideCacheFromFile(bool overrideNewerOnly, const QFileInfo &fileInfo, BlackMisc::Network::CEntityFlags::Entity entity, BlackMisc::CStatusMessageList &msgs) const;
+
+            //! Parsing info message
+            void logParseMessage(const QString &entity, int size, int msElapsed, const JsonDatastoreResponse &response) const;
         };
     } // ns
 } // ns

src/blackcore/db/icaodatareader.cpp

@@ -281,8 +281,11 @@ namespace BlackCore
             }
             else
             {
-                // normally read from special view which already filters incomplete
+                // normally read from special DB view which already filters incomplete
+                QTime time;
+                time.start();
                 codes  = CAircraftIcaoCodeList::fromDatabaseJson(res, true, &inconsistent);
+                this->logParseMessage("aircraft ICAO", codes.size(), time.elapsed(), res);
             }
 
             if (!inconsistent.isEmpty())
@@ -332,8 +335,11 @@ namespace BlackCore
             }
             else
             {
-                // normally read from special view which already filters incomplete
+                // normally read from special DB view which already filters incomplete
+                QTime time;
+                time.start();
                 codes = CAirlineIcaoCodeList::fromDatabaseJson(res, true, &inconsistent);
+                this->logParseMessage("airline ICAO", codes.size(), time.elapsed(), res);
             }
 
             if (!inconsistent.isEmpty())
@@ -380,8 +386,11 @@ namespace BlackCore
             }
             else
             {
-                // normally read from special view which already filters incomplete
+                // normally read from special DB view which already filters incomplete
+                QTime time;
+                time.start();
                 countries  = CCountryList::fromDatabaseJson(res);
+                this->logParseMessage("countries", countries.size(), time.elapsed(), res);
             }
 
             if (!this->doWorkCheck()) { return; }

src/blackcore/db/modeldatareader.cpp

@@ -299,7 +299,10 @@ namespace BlackCore
             }
             else
             {
+                QTime time;
+                time.start();
                 liveries  = CLiveryList::fromDatabaseJson(res);
+                this->logParseMessage("liveries", liveries.size(), time.elapsed(), res);
             }
 
             if (!this->doWorkCheck()) { return; }
@@ -343,7 +346,10 @@ namespace BlackCore
             }
             else
             {
+                QTime time;
+                time.start();
                 distributors = CDistributorList::fromDatabaseJson(res);
+                this->logParseMessage("distributors", distributors.size(), time.elapsed(), res);
             }
 
             if (!this->doWorkCheck()) { return; }
@@ -391,8 +397,7 @@ namespace BlackCore
                 QTime time;
                 time.start();
                 models = CAircraftModelList::fromDatabaseJson(res);
-                const int elapsed = time.elapsed();
+                this->logParseMessage("models", models.size(), time.elapsed(), res);
-                CLogMessage(this).info("Parsed %1 models in %2 ms") << models.size() << elapsed;
             }
 
             // synchronized update

src/blackmisc/aviation/aircrafticaocode.cpp

@@ -738,10 +738,10 @@ namespace BlackMisc
             }
 
             // turn E to P engine
-            if (combinedCode.endsWith("E")) { return QStringList({ combinedCode.leftRef(2) + "P"}); }
+            if (combinedCode.endsWith("E")) { return QStringList({ combinedCode.leftRef(2) % QStringLiteral("P")}); }
 
             // turn T to H plane (tilt wing to helicopter
-            if (combinedCode.startsWith("T")) { return QStringList({ "H" + combinedCode.rightRef(2)}); }
+            if (combinedCode.startsWith("T")) { return QStringList({ QStringLiteral("H") % combinedCode.rightRef(2)}); }
 
             // based on engine count
             QStringList codes;
@@ -766,18 +766,18 @@ namespace BlackMisc
                 return CAircraftIcaoCode();
             }
 
-            const QString designator(json.value(prefix + "designator").toString());
+            const QString designator(json.value(prefix % QStringLiteral("designator")).toString());
-            const QString iata(json.value(prefix + "iata").toString());
+            const QString iata(json.value(prefix % QStringLiteral("iata")).toString());
-            const QString family(json.value(prefix + "family").toString());
+            const QString family(json.value(prefix % QStringLiteral("family")).toString());
-            const QString manufacturer(json.value(prefix + "manufacturer").toString());
+            const QString manufacturer(json.value(prefix % QStringLiteral("manufacturer")).toString());
-            const QString model(json.value(prefix + "model").toString());
+            const QString model(json.value(prefix % QStringLiteral("model")).toString());
-            const QString modelIata(json.value(prefix + "modeliata").toString());
+            const QString modelIata(json.value(prefix % QStringLiteral("modeliata")).toString());
-            const QString modelSwift(json.value(prefix + "modelswift").toString());
+            const QString modelSwift(json.value(prefix % QStringLiteral("modelswift")).toString());
-            const QString type(json.value(prefix + "type").toString());
+            const QString type(json.value(prefix % QStringLiteral("type")).toString());
-            const QString engine(json.value(prefix + "engine").toString());
+            const QString engine(json.value(prefix % QStringLiteral("engine")).toString());
-            const int engineCount(json.value(prefix + "enginecount").toInt(-1));
+            const int engineCount(json.value(prefix % QStringLiteral("enginecount")).toInt(-1));
             const QString combined(createdCombinedString(type, engineCount, engine));
-            QString wtc(json.value(prefix + "wtc").toString());
+            QString wtc(json.value(prefix % QStringLiteral("wtc")).toString());
             if (wtc.length() > 1 && wtc.contains("/"))
             {
                 // "L/M" -> "M"
@@ -785,10 +785,10 @@ namespace BlackMisc
             }
             Q_ASSERT_X(wtc.length() < 2, Q_FUNC_INFO, "WTC too long");
 
-            const bool real = CDatastoreUtility::dbBoolStringToBool(json.value(prefix + "realworld").toString());
+            const bool real = CDatastoreUtility::dbBoolStringToBool(json.value(prefix % QStringLiteral("realworld")).toString());
-            const bool legacy = CDatastoreUtility::dbBoolStringToBool(json.value(prefix + "legacy").toString());
+            const bool legacy = CDatastoreUtility::dbBoolStringToBool(json.value(prefix % QStringLiteral("legacy")).toString());
-            const bool military = CDatastoreUtility::dbBoolStringToBool(json.value(prefix + "military").toString());
+            const bool military = CDatastoreUtility::dbBoolStringToBool(json.value(prefix % QStringLiteral("military")).toString());
-            const int rank(json.value(prefix + "rank").toInt(10));
+            const int rank(json.value(prefix % QStringLiteral("rank")).toInt(10));
 
             CAircraftIcaoCode code(
                 designator, iata, family, combined, manufacturer,
@@ -802,11 +802,9 @@ namespace BlackMisc
         QString CAircraftIcaoCode::createdCombinedString(const QString &type, const QString &engineCount, const QString &engine)
         {
             Q_ASSERT_X(engineCount.length() < 2, Q_FUNC_INFO, "Wrong engine count");
-            QString c(type.isEmpty() ? "-" : type.trimmed().left(1).toUpper());
+            return (type.isEmpty() ? QStringLiteral("-") : type.trimmed().left(1).toUpper()) %
-            c += engineCount.isEmpty() ? "-" : engineCount.trimmed();
+                   (engineCount.isEmpty() ? QStringLiteral("-") : engineCount.trimmed()) %
-            c += engine.isEmpty() ? "-" : engine.trimmed().left(1).toUpper();
+                   (engine.isEmpty() ? QStringLiteral("-") : engine.trimmed().left(1).toUpper());
-            Q_ASSERT_X(c.length() == 3, Q_FUNC_INFO, "Wrong combined length");
-            return c;
         }
 
         QString CAircraftIcaoCode::createdCombinedString(const QString &type, int engineCount, const QString &engine)

src/blackmisc/db/datastoreutility.cpp

@@ -63,17 +63,8 @@ namespace BlackMisc
 
         QDateTime CDatastoreUtility::parseTimestamp(const QString &timestamp)
         {
-            if (!timestamp.isEmpty())
+            if (timestamp.isEmpty()) { return QDateTime(); }
-            {
+            return parseDateTimeStringOptimized(removeDateTimeSeparators(timestamp));
-                const QString ts(timestamp.trimmed().remove(' ').remove('-').remove(':')); // normalize
-                QDateTime dt = QDateTime::fromString(ts, "yyyyMMddHHmmss");
-                dt.setTimeZone(QTimeZone::utc());
-                return dt;
-            }
-            else
-            {
-                return QDateTime();
-            }
         }
 
         bool CDatastoreUtility::parseSwiftPublishResponse(const QString &jsonResponse, CAircraftModelList &publishedModels,  CAircraftModelList &skippedModels, CStatusMessageList &messages, bool &directWrite)