refs #930, utility conversion functions supporting compression

src/blackcore/db/databasereader.cpp

@@ -9,6 +9,7 @@
 
 #include "blackcore/db/databasereader.h"
 #include "blackcore/db/infodatareader.h"
+#include "blackcore/db/databaseutils.h"
 #include "blackcore/webdataservices.h"
 #include "blackcore/application.h"
 #include "blackmisc/db/datastoreutility.h"
@@ -32,6 +33,7 @@ using namespace BlackMisc::Db;
 using namespace BlackMisc::Network;
 using namespace BlackCore;
 using namespace BlackCore::Data;
+using namespace BlackCore::Db;
 
 namespace BlackCore
 {
@@ -188,7 +190,7 @@ namespace BlackCore
             const bool ok = this->setHeaderInfoPart(datastoreResponse, nwReply);
             if (ok)
             {
-                const QString dataFileData = nwReply->readAll().trimmed();
+                const QString dataFileData = nwReply->readAll();
                 nwReply->close(); // close asap
                 if (dataFileData.isEmpty())
                 {
@@ -337,7 +339,7 @@ namespace BlackCore
                 url.appendPath(fileName);
 
                 const QString entityString = CEntityFlags::flagToString(currentEntity);
-                CLogMessage(this).info("Triggered read of header for shared file of %1") << entityString;
+                CLogMessage(this).info("Triggered read of header for shared file of '%1'") << entityString;
                 const QNetworkReply *reply = sApp->headerFromNetwork(url, { this, &CDatabaseReader::receivedSharedFileHeader });
                 if (reply) { c++; }
                 currentEntity = CEntityFlags::iterateDbEntities(allEntities);
@@ -439,7 +441,7 @@ namespace BlackCore
             // wrap pointer, make sure any exit cleans up reply
             // required to use delete later as object is created in a different thread
             QScopedPointer<QNetworkReply, QScopedPointerDeleteLater> nwReply(nwReplyPtr);
-            if (this->isAbandoned()) { return; }
+            if (this->isShuttingDown()) { return; }
             this->receivedSharedFileHeaderNonClosing(nwReplyPtr);
             nwReply->close();
         }
@@ -585,7 +587,13 @@ namespace BlackCore
                 return;
             }
 
-            const QJsonDocument jsonResponse = QJsonDocument::fromJson(jsonContent.toUtf8());
+            const QJsonDocument jsonResponse = CDatabaseUtils::databaseJsonToQJsonDocument(jsonContent);
+            if (jsonResponse.isEmpty())
+            {
+                datastoreResponse.setMessage(CStatusMessage(getLogCategories(), CStatusMessage::SeverityError, "Empty JSON, no data"));
+                return;
+            }
+
             if (jsonResponse.isArray())
             {
                 // directly an array, no further info

src/blackcore/db/databaseutils.cpp

@@ -11,6 +11,7 @@
 #include "databaseutils.h"
 #include "blackcore/webdataservices.h"
 #include "blackmisc/logmessage.h"
+#include "blackmisc/fileutils.h"
 
 using namespace BlackMisc;
 using namespace BlackMisc::Aviation;
@@ -239,6 +240,68 @@ namespace BlackCore
             return stashModels;
         }
 
+        QJsonDocument CDatabaseUtils::databaseJsonToQJsonDocument(const QString &content)
+        {
+            static const QString compressed("swift:");
+            if (content.isEmpty()) { return QJsonDocument(); }
+            QByteArray byteData;
+            if (Json::looksLikeJson(content))
+            {
+                // uncompressed
+                byteData = content.toUtf8();
+            }
+            else if (content.startsWith(compressed) && content.length() > compressed.length() + 3)
+            {
+                do
+                {
+                    // "swift:1234:base64encoded
+                    const int cl = compressed.length();
+                    const int contentIndex = content.indexOf(':', cl);
+                    if (contentIndex < cl) break; // should not happen, malformed
+                    const QString ls = content.mid(cl, contentIndex - cl); // content length
+                    bool ok;
+                    const qint32 size = ls.toInt(&ok);
+                    if (!ok) break; // malformed size
+                    if (size < 1) break;
+
+                    // Length header, unsigned, big-endian, 32-bit integer
+                    QByteArray lengthHeader;
+                    QDataStream stream(&lengthHeader, QIODevice::WriteOnly);
+                    stream.setByteOrder(QDataStream::BigEndian);
+                    stream << size;
+
+                    QByteArray ba;
+                    ba.append(content.mid(contentIndex));
+                    ba = QByteArray::fromBase64(ba);
+                    ba.insert(0, lengthHeader); // adding 4 bytes length header
+                    byteData = qUncompress(ba);
+                }
+                while (false);
+
+            }
+            if (byteData.isEmpty()) { return QJsonDocument(); }
+            return QJsonDocument::fromJson(byteData);
+        }
+
+        QJsonDocument CDatabaseUtils::readQJsonDocumentFromDatabaseFile(const QString &filename)
+        {
+            const QString raw = CFileUtils::readFileToString(filename);
+            if (raw.isEmpty()) { return QJsonDocument(); }
+            return CDatabaseUtils::databaseJsonToQJsonDocument(raw);
+        }
+
+        QJsonObject CDatabaseUtils::readQJsonObjectFromDatabaseFile(const QString &filename)
+        {
+            const QString raw = CFileUtils::readFileToString(filename);
+            if (raw.isEmpty()) { return QJsonObject(); }
+            return BlackMisc::Json::jsonObjectFromString(raw);
+        }
+
+        QJsonObject CDatabaseUtils::readQJsonObjectFromDatabaseFile(const QString &directory, const QString &filename)
+        {
+            return CDatabaseUtils::readQJsonObjectFromDatabaseFile(CFileUtils::appendFilePaths(directory, filename));
+        }
+
         bool CDatabaseUtils::hasDbAircraftData()
         {
             return sApp && sApp->hasWebDataServices() && sApp->getWebDataServices()->hasDbAircraftData();

src/blackcore/db/databaseutils.h

@@ -55,6 +55,18 @@ namespace BlackCore
             //! Create stash models if the DB models miss that simulator
             static BlackMisc::Simulation::CAircraftModelList updateSimulatorForFsFamily(const BlackMisc::Simulation::CAircraftModelList &ownModels, int maxToStash = -1, BlackCore::IProgressIndicator *progressIndicator = nullptr, bool processEvents = true);
 
+            //! Database JSON from content string, which can be compressed
+            static QJsonDocument databaseJsonToQJsonDocument(const QString &content);
+
+            //! QJsonDocument from database JSON file (normally shared file)
+            static QJsonDocument readQJsonDocumentFromDatabaseFile(const QString &filename);
+
+            //! QJsonObject from database JSON file (normally shared file)
+            static QJsonObject readQJsonObjectFromDatabaseFile(const QString &filename);
+
+            //! QJsonObject from database JSON file (normally shared file)
+            static QJsonObject readQJsonObjectFromDatabaseFile(const QString &directory, const QString &filename);
+
             //! Convenience function
             static bool hasDbAircraftData();
         };