Ref T308, loading JSON from DB optimizations

- info about time consumed for parsing (so we can benchmark)
- using the "optimized" YYYYmmdd parsing

src/blackcore/db/airportdatareader.cpp

@@ -209,7 +209,10 @@ namespace BlackCore
             }
             else
             {
+                QTime time;
+                time.start();
                 airports = CAirportList::fromDatabaseJson(res, &inconsistent);
+                this->logParseMessage("airports", airports.size(), time.elapsed(), res);
             }
 
             if (!inconsistent.isEmpty())

src/blackcore/db/databasereader.cpp

@@ -25,6 +25,7 @@
 #include <QByteArray>
 #include <QJsonDocument>
 #include <QJsonObject>
+#include <QPointer>
 #include <QFileInfo>
 #include <QJsonValueRef>
 #include <QMetaObject>
@@ -196,8 +197,10 @@ namespace BlackCore
                     // Using timer to first finish this function, then the resulting signal
                     if (validInCacheEntities != CEntityFlags::NoEntity)
                     {
+                        QPointer<CDatabaseReader> myself(this);
                         QTimer::singleShot(0, this, [ = ]
                         {
+                            if (!myself) { return; }
                             emit this->dataRead(validInCacheEntities, CEntityFlags::ReadFinished, 0);
                         });
                     }
@@ -234,6 +237,7 @@ namespace BlackCore
             {
                 const QString dataFileData = nwReply->readAll();
                 nwReply->close(); // close asap
+                datastoreResponse.setStringSize(dataFileData.size());
                 if (dataFileData.isEmpty())
                 {
                     datastoreResponse.setMessage(CStatusMessage(this, CStatusMessage::SeverityError, "Empty response, no data"));
@@ -662,6 +666,13 @@ namespace BlackCore
             }
         }
 
+        void CDatabaseReader::logParseMessage(const QString &entity, int size, int msElapsed, const CDatabaseReader::JsonDatastoreResponse &response) const
+        {
+            CLogMessage(this).info("Parsed %1 %2 in %3ms, thread '%4' | '%5'")
+                    << size << entity << msElapsed
+                    << QThread::currentThread()->objectName() << response.toQString();
+        }
+
         QString CDatabaseReader::fileNameForMode(CEntityFlags::Entity entity, CDbFlags::DataRetrievalModeFlag mode)
         {
             Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "needs single entity");
@@ -786,6 +797,12 @@ namespace BlackCore
             m_arraySize = value.size();
         }
 
+        QString CDatabaseReader::JsonDatastoreResponse::toQString() const
+        {
+            static const QString s("DB: %1 | restricted: %2 | array: %3 | string size: %4 | content: %5");
+            return s.arg(boolToYesNo(this->isLoadedFromDb()), boolToYesNo(this->isRestricted())).arg(this->getArraySize()).arg(m_stringSize).arg(this->getContentLengthHeader());
+        }
+
         bool CDatabaseReader::HeaderResponse::isSharedFile() const
         {
             const QString fn(getUrl().getFileName());

src/blackcore/db/databasereader.h

@@ -132,30 +132,37 @@ namespace BlackCore
             private:
                 QJsonArray m_jsonArray;          //!< JSON array data
                 int        m_arraySize  = -1;    //!< size of array, if applicable (copied to member for debugging purposes)
+                int        m_stringSize =  0;    //!< string size of JSON data
                 bool       m_restricted = false; //!< restricted reponse, only changed data
 
             public:
                 //! Any data?
                 bool isEmpty() const { return m_jsonArray.isEmpty(); }
 
-                //! Number of elements
-                int size() const { return m_jsonArray.size(); }
+                //! Is loaded from database
+                bool isLoadedFromDb() const;
 
                 //! Incremental data, restricted by query?
                 bool isRestricted() const { return m_restricted; }
 
-                //! Is loaded from database
-                bool isLoadedFromDb() const;
-
                 //! Mark as restricted
                 void setRestricted(bool restricted) { m_restricted = restricted; }
 
                 //! Get the JSON array
                 QJsonArray getJsonArray() const { return m_jsonArray; }
 
+                //! Number of elements
+                int getArraySize() const { return m_jsonArray.size(); }
+
                 //! Set the JSON array
                 void setJsonArray(const QJsonArray &value);
 
+                //! Set string size
+                void setStringSize(int size) { m_stringSize = size; }
+
+                //! String info
+                QString toQString() const;
+
                 //! Implicit conversion
                 operator QJsonArray() const { return m_jsonArray; }
             };
@@ -411,6 +418,9 @@ namespace BlackCore
             //! Override cache from file
             //! \threadsafe
             bool overrideCacheFromFile(bool overrideNewerOnly, const QFileInfo &fileInfo, BlackMisc::Network::CEntityFlags::Entity entity, BlackMisc::CStatusMessageList &msgs) const;
+
+            //! Parsing info message
+            void logParseMessage(const QString &entity, int size, int msElapsed, const JsonDatastoreResponse &response) const;
         };
     } // ns
 } // ns

src/blackcore/db/icaodatareader.cpp

@@ -281,8 +281,11 @@ namespace BlackCore
             }
             else
             {
-                // normally read from special view which already filters incomplete
+                // normally read from special DB view which already filters incomplete
+                QTime time;
+                time.start();
                 codes  = CAircraftIcaoCodeList::fromDatabaseJson(res, true, &inconsistent);
+                this->logParseMessage("aircraft ICAO", codes.size(), time.elapsed(), res);
             }
 
             if (!inconsistent.isEmpty())
@@ -332,8 +335,11 @@ namespace BlackCore
             }
             else
             {
-                // normally read from special view which already filters incomplete
+                // normally read from special DB view which already filters incomplete
+                QTime time;
+                time.start();
                 codes = CAirlineIcaoCodeList::fromDatabaseJson(res, true, &inconsistent);
+                this->logParseMessage("airline ICAO", codes.size(), time.elapsed(), res);
             }
 
             if (!inconsistent.isEmpty())
@@ -380,8 +386,11 @@ namespace BlackCore
             }
             else
             {
-                // normally read from special view which already filters incomplete
+                // normally read from special DB view which already filters incomplete
+                QTime time;
+                time.start();
                 countries  = CCountryList::fromDatabaseJson(res);
+                this->logParseMessage("countries", countries.size(), time.elapsed(), res);
             }
 
             if (!this->doWorkCheck()) { return; }

src/blackcore/db/modeldatareader.cpp

@@ -299,7 +299,10 @@ namespace BlackCore
             }
             else
             {
+                QTime time;
+                time.start();
                 liveries  = CLiveryList::fromDatabaseJson(res);
+                this->logParseMessage("liveries", liveries.size(), time.elapsed(), res);
             }
 
             if (!this->doWorkCheck()) { return; }
@@ -343,7 +346,10 @@ namespace BlackCore
             }
             else
             {
+                QTime time;
+                time.start();
                 distributors = CDistributorList::fromDatabaseJson(res);
+                this->logParseMessage("distributors", distributors.size(), time.elapsed(), res);
             }
 
             if (!this->doWorkCheck()) { return; }
@@ -391,8 +397,7 @@ namespace BlackCore
                 QTime time;
                 time.start();
                 models = CAircraftModelList::fromDatabaseJson(res);
-                const int elapsed = time.elapsed();
-                CLogMessage(this).info("Parsed %1 models in %2 ms") << models.size() << elapsed;
+                this->logParseMessage("models", models.size(), time.elapsed(), res);
             }
 
             // synchronized update