Sample reg. JSON performance

* DB JSON vs. swift JSON
* Database reader: own function for stringToDatastoreResponse so it can be reused

samples/blackmisc/main.cpp

@@ -34,12 +34,14 @@ int main(int argc, char *argv[])
     // otherwise no QPixmap metadata (metadata sample)
     QCoreApplication qa(argc, argv);
     CApplication a;
-
+    Q_UNUSED(qa);
+    Q_UNUSED(a);
     QTextStream qtout(stdout);
     QTextStream qtin(stdin);
 
     do
     {
+        qtout << endl;
         qtout << "1 .. JSON" << endl;
         qtout << "2 .. Change object" << endl;
         qtout << "3 .. Containers" << endl;
@@ -49,7 +51,8 @@ int main(int argc, char *argv[])
         qtout << "6c .. 25/20 Performance impl. type" << endl;
         qtout << "6d .. 40/20 Interpolator scenario" << endl;
         qtout << "6e .. JSON performance" << endl;
-        qtout << "6f .. string utils vs. regex" << endl;
+        qtout << "6f .. JSON model performance (database vs. own JSON)" << endl;
+        qtout << "6g .. string utils vs.regex" << endl;
         qtout << "7 .. Algorithms" << endl;
         qtout << "-----" << endl;
         qtout << "x .. Bye" << endl;
@@ -64,7 +67,8 @@ int main(int argc, char *argv[])
         else if (s.startsWith("6c")) { CSamplesPerformance::samplesImplementationType(qtout, 25, 20); }
         else if (s.startsWith("6d")) { CSamplesPerformance::interpolatorScenario(qtout, 40, 20); }
         else if (s.startsWith("6e")) { CSamplesPerformance::samplesJson(qtout); }
-        else if (s.startsWith("6f")) { CSamplesPerformance::samplesString(qtout); }
+        else if (s.startsWith("6f")) { CSamplesPerformance::samplesJsonModel(qtout); }
+        else if (s.startsWith("6g")) { CSamplesPerformance::samplesString(qtout); }
         else if (s.startsWith("7")) { CSamplesAlgorithm::samples(); }
         else if (s.startsWith("x")) { break; }
     }

samples/blackmisc/sampleblackmisc.pro

@@ -7,7 +7,7 @@ TEMPLATE = app
 
 CONFIG   += console
 CONFIG   -= app_bundle
-CONFIG   += blackmisc blackcore blackgui
+CONFIG   += blackmisc blackcore blackgui blackconfig
 
 DEPENDPATH += . $$SourceRoot/src/blackmisc
 INCLUDEPATH += . $$SourceRoot/src

samples/blackmisc/samplesperformance.cpp

@@ -11,6 +11,9 @@
 //! \ingroup sampleblackmisc
 
 #include "samplesperformance.h"
+#include "blackcore/db/databasereader.h"
+#include "blackconfig/buildconfig.h"
+#include "blackmisc/simulation/aircraftmodellist.h"
 #include "blackmisc/aviation/aircraftsituation.h"
 #include "blackmisc/aviation/aircraftsituationlist.h"
 #include "blackmisc/aviation/altitude.h"
@@ -41,6 +44,9 @@ using namespace BlackMisc;
 using namespace BlackMisc::Aviation;
 using namespace BlackMisc::Geo;
 using namespace BlackMisc::PhysicalQuantities;
+using namespace BlackMisc::Simulation;
+using namespace BlackConfig;
+using namespace BlackCore::Db;
 
 namespace BlackSample
 {
@@ -342,6 +348,39 @@ namespace BlackSample
         return 0;
     }
 
+    int CSamplesPerformance::samplesJsonModel(QTextStream &out)
+    {
+        const QString dir = CBuildConfig::getSwiftStaticDbFilesDir();
+        const QString file = QDir(dir).filePath("models.json");
+        QFile modelFile(file);
+        Q_ASSERT_X(modelFile.exists(), Q_FUNC_INFO, "Model file does not exist");
+
+        out << "Load DB JSON file " << modelFile.fileName() << endl;
+        const QString data = CFileUtils::readFileToString(modelFile.fileName());
+        Q_ASSERT_X(!data.isEmpty(), Q_FUNC_INFO, "Model file empty");
+
+        // DB format, all models denormalized in DB JSON format
+        CDatabaseReader::JsonDatastoreResponse response = CDatabaseReader::stringToDatastoreResponse(data);
+        QTime timer;
+        timer.start();
+        const CAircraftModelList dbModels = CAircraftModelList::fromDatabaseJson(response);
+        int ms = timer.elapsed();
+        out << "Read via DB JSON format: " << dbModels.size() << " models in " << ms << "ms" << endl;
+
+        // swift JSON format
+        const QJsonObject swiftJsonObject = dbModels.toJson();
+        out << "Converted to swift JSON" << endl;
+
+        CAircraftModelList swiftModels;
+        timer.start();
+        swiftModels.convertFromJson(swiftJsonObject);
+        ms = timer.elapsed();
+        out << "Read via swift JSON format: " << swiftModels.size() << " models in " << ms << "ms" << endl;
+
+        Q_ASSERT_X(swiftModels.size() == dbModels.size(), Q_FUNC_INFO, "Mismatching container size");
+        return 0;
+    }
+
     int CSamplesPerformance::samplesString(QTextStream &out)
     {
         QTime timer;

samples/blackmisc/samplesperformance.h

@@ -38,6 +38,9 @@ namespace BlackSample
         //! JSON (de)serialization
         static int samplesJson(QTextStream &out);
 
+        //! JSON loading (database vs. own format
+        static int samplesJsonModel(QTextStream &out);
+
         //! String manipulation
         static int samplesString(QTextStream &out);
 

src/blackcore/db/databasereader.cpp

@@ -125,13 +125,12 @@ namespace BlackCore
         CDatabaseReader::JsonDatastoreResponse CDatabaseReader::transformReplyIntoDatastoreResponse(QNetworkReply *nwReply) const
         {
             this->threadAssertCheck();
-            static const CLogCategoryList cats(CLogCategoryList(this).join({ CLogCategory::webservice()}));
 
             JsonDatastoreResponse datastoreResponse;
             if (this->isAbandoned())
             {
                 nwReply->abort();
-                datastoreResponse.setMessage(CStatusMessage(cats, CStatusMessage::SeverityError, "Terminated data parsing process"));
+                datastoreResponse.setMessage(CStatusMessage(this, CStatusMessage::SeverityError, "Terminated data parsing process"));
                 return datastoreResponse; // stop, terminate straight away, ending thread
             }
 
@@ -141,35 +140,24 @@ namespace BlackCore
                 nwReply->close(); // close asap
                 if (dataFileData.isEmpty())
                 {
-                    datastoreResponse.setMessage(CStatusMessage(cats, CStatusMessage::SeverityError, "Empty response, no data"));
+                    datastoreResponse.setMessage(CStatusMessage(this, CStatusMessage::SeverityError, "Empty response, no data"));
-                    datastoreResponse.m_updated = QDateTime::currentDateTimeUtc();
-                    return datastoreResponse;
-                }
-
-                QJsonDocument jsonResponse = QJsonDocument::fromJson(dataFileData.toUtf8());
-                if (jsonResponse.isArray())
-                {
-                    // directly an array, no further info
-                    datastoreResponse.setJsonArray(jsonResponse.array());
                     datastoreResponse.m_updated = QDateTime::currentDateTimeUtc();
                 }
                 else
                 {
-                    QJsonObject responseObject(jsonResponse.object());
+                    datastoreResponse = CDatabaseReader::stringToDatastoreResponse(dataFileData);
-                    datastoreResponse.setJsonArray(responseObject["data"].toArray());
-                    QString ts(responseObject["latest"].toString());
-                    datastoreResponse.m_updated = ts.isEmpty() ? QDateTime::currentDateTimeUtc() : CDatastoreUtility::parseTimestamp(ts);
-                    datastoreResponse.m_restricted = responseObject["restricted"].toBool();
                 }
-                return datastoreResponse;
             }
+            else
+            {
 
-            // no valid response
+                // no valid response
-            QString error(nwReply->errorString());
+                const QString error(nwReply->errorString());
-            QString url(nwReply->url().toString());
+                const QString url(nwReply->url().toString());
-            nwReply->abort();
+                nwReply->abort();
-            datastoreResponse.setMessage(CStatusMessage(cats, CStatusMessage::SeverityError,
+                datastoreResponse.setMessage(CStatusMessage(this, CStatusMessage::SeverityError,
-                                         QString("Reading data failed: " + error + " " + url)));
+                                             QString("Reading data failed: " + error + " " + url)));
+            }
             return datastoreResponse;
         }
 
@@ -284,7 +272,7 @@ namespace BlackCore
         {
             static const BlackMisc::CLogCategoryList cats
             (
-                CThreadedReader::getLogCategories().join({ BlackMisc::CLogCategory::swiftDbWebservice(), BlackMisc::CLogCategory::mapping() })
+                CThreadedReader::getLogCategories().join({ BlackMisc::CLogCategory::swiftDbWebservice(), BlackMisc::CLogCategory::webservice() })
             );
             return cats;
         }
@@ -318,6 +306,34 @@ namespace BlackCore
             return CNetworkUtils::canConnect(url);
         }
 
+        CDatabaseReader::JsonDatastoreResponse CDatabaseReader::stringToDatastoreResponse(const QString &jsonContent)
+        {
+            CDatabaseReader::JsonDatastoreResponse datastoreResponse;
+            if (jsonContent.isEmpty())
+            {
+                datastoreResponse.setMessage(CStatusMessage(getLogCategories(), CStatusMessage::SeverityError, "Empty string, no data"));
+                datastoreResponse.m_updated = QDateTime::currentDateTimeUtc();
+                return datastoreResponse;
+            }
+
+            const QJsonDocument jsonResponse = QJsonDocument::fromJson(jsonContent.toUtf8());
+            if (jsonResponse.isArray())
+            {
+                // directly an array, no further info
+                datastoreResponse.setJsonArray(jsonResponse.array());
+                datastoreResponse.m_updated = QDateTime::currentDateTimeUtc();
+            }
+            else
+            {
+                const QJsonObject responseObject(jsonResponse.object());
+                datastoreResponse.setJsonArray(responseObject["data"].toArray());
+                const QString ts(responseObject["latest"].toString());
+                datastoreResponse.m_updated = ts.isEmpty() ? QDateTime::currentDateTimeUtc() : CDatastoreUtility::parseTimestamp(ts);
+                datastoreResponse.m_restricted = responseObject["restricted"].toBool();
+            }
+            return datastoreResponse;
+        }
+
         void CDatabaseReader::JsonDatastoreResponse::setJsonArray(const QJsonArray &value)
         {
             m_jsonArray = value;

src/blackcore/db/databasereader.h

@@ -144,6 +144,9 @@ namespace BlackCore
             //! sift DB server reachable?
             static bool canPingSwiftServer();
 
+            //! Transform JSON data to response struct
+            static JsonDatastoreResponse stringToDatastoreResponse(const QString &jsonContent);
+
         signals:
             //! Combined read signal
             void dataRead(BlackMisc::Network::CEntityFlags::Entity entity, BlackMisc::Network::CEntityFlags::ReadState state, int number);