From de80053c7cc8461f9f06c30c3a2cc1e31f840afd Mon Sep 17 00:00:00 2001 From: Klaus Basan Date: Thu, 6 Apr 2017 03:03:05 +0200 Subject: [PATCH] refs #930, utility conversion functions supporting compression --- src/blackcore/db/databasereader.cpp | 16 ++++++-- src/blackcore/db/databaseutils.cpp | 63 +++++++++++++++++++++++++++++ src/blackcore/db/databaseutils.h | 12 ++++++ 3 files changed, 87 insertions(+), 4 deletions(-) diff --git a/src/blackcore/db/databasereader.cpp b/src/blackcore/db/databasereader.cpp index f8b03f67f..672610dd8 100644 --- a/src/blackcore/db/databasereader.cpp +++ b/src/blackcore/db/databasereader.cpp @@ -9,6 +9,7 @@ #include "blackcore/db/databasereader.h" #include "blackcore/db/infodatareader.h" +#include "blackcore/db/databaseutils.h" #include "blackcore/webdataservices.h" #include "blackcore/application.h" #include "blackmisc/db/datastoreutility.h" @@ -32,6 +33,7 @@ using namespace BlackMisc::Db; using namespace BlackMisc::Network; using namespace BlackCore; using namespace BlackCore::Data; +using namespace BlackCore::Db; namespace BlackCore { @@ -188,7 +190,7 @@ namespace BlackCore const bool ok = this->setHeaderInfoPart(datastoreResponse, nwReply); if (ok) { - const QString dataFileData = nwReply->readAll().trimmed(); + const QString dataFileData = nwReply->readAll(); nwReply->close(); // close asap if (dataFileData.isEmpty()) { @@ -337,7 +339,7 @@ namespace BlackCore url.appendPath(fileName); const QString entityString = CEntityFlags::flagToString(currentEntity); - CLogMessage(this).info("Triggered read of header for shared file of %1") << entityString; + CLogMessage(this).info("Triggered read of header for shared file of '%1'") << entityString; const QNetworkReply *reply = sApp->headerFromNetwork(url, { this, &CDatabaseReader::receivedSharedFileHeader }); if (reply) { c++; } currentEntity = CEntityFlags::iterateDbEntities(allEntities); @@ -439,7 +441,7 @@ namespace BlackCore // wrap pointer, make sure any exit cleans up reply // required to use delete later as object is created in a different thread QScopedPointer nwReply(nwReplyPtr); - if (this->isAbandoned()) { return; } + if (this->isShuttingDown()) { return; } this->receivedSharedFileHeaderNonClosing(nwReplyPtr); nwReply->close(); } @@ -585,7 +587,13 @@ namespace BlackCore return; } - const QJsonDocument jsonResponse = QJsonDocument::fromJson(jsonContent.toUtf8()); + const QJsonDocument jsonResponse = CDatabaseUtils::databaseJsonToQJsonDocument(jsonContent); + if (jsonResponse.isEmpty()) + { + datastoreResponse.setMessage(CStatusMessage(getLogCategories(), CStatusMessage::SeverityError, "Empty JSON, no data")); + return; + } + if (jsonResponse.isArray()) { // directly an array, no further info diff --git a/src/blackcore/db/databaseutils.cpp b/src/blackcore/db/databaseutils.cpp index cc4477845..1cc7060fe 100644 --- a/src/blackcore/db/databaseutils.cpp +++ b/src/blackcore/db/databaseutils.cpp @@ -11,6 +11,7 @@ #include "databaseutils.h" #include "blackcore/webdataservices.h" #include "blackmisc/logmessage.h" +#include "blackmisc/fileutils.h" using namespace BlackMisc; using namespace BlackMisc::Aviation; @@ -239,6 +240,68 @@ namespace BlackCore return stashModels; } + QJsonDocument CDatabaseUtils::databaseJsonToQJsonDocument(const QString &content) + { + static const QString compressed("swift:"); + if (content.isEmpty()) { return QJsonDocument(); } + QByteArray byteData; + if (Json::looksLikeJson(content)) + { + // uncompressed + byteData = content.toUtf8(); + } + else if (content.startsWith(compressed) && content.length() > compressed.length() + 3) + { + do + { + // "swift:1234:base64encoded + const int cl = compressed.length(); + const int contentIndex = content.indexOf(':', cl); + if (contentIndex < cl) break; // should not happen, malformed + const QString ls = content.mid(cl, contentIndex - cl); // content length + bool ok; + const qint32 size = ls.toInt(&ok); + if (!ok) break; // malformed size + if (size < 1) break; + + // Length header, unsigned, big-endian, 32-bit integer + QByteArray lengthHeader; + QDataStream stream(&lengthHeader, QIODevice::WriteOnly); + stream.setByteOrder(QDataStream::BigEndian); + stream << size; + + QByteArray ba; + ba.append(content.mid(contentIndex)); + ba = QByteArray::fromBase64(ba); + ba.insert(0, lengthHeader); // adding 4 bytes length header + byteData = qUncompress(ba); + } + while (false); + + } + if (byteData.isEmpty()) { return QJsonDocument(); } + return QJsonDocument::fromJson(byteData); + } + + QJsonDocument CDatabaseUtils::readQJsonDocumentFromDatabaseFile(const QString &filename) + { + const QString raw = CFileUtils::readFileToString(filename); + if (raw.isEmpty()) { return QJsonDocument(); } + return CDatabaseUtils::databaseJsonToQJsonDocument(raw); + } + + QJsonObject CDatabaseUtils::readQJsonObjectFromDatabaseFile(const QString &filename) + { + const QString raw = CFileUtils::readFileToString(filename); + if (raw.isEmpty()) { return QJsonObject(); } + return BlackMisc::Json::jsonObjectFromString(raw); + } + + QJsonObject CDatabaseUtils::readQJsonObjectFromDatabaseFile(const QString &directory, const QString &filename) + { + return CDatabaseUtils::readQJsonObjectFromDatabaseFile(CFileUtils::appendFilePaths(directory, filename)); + } + bool CDatabaseUtils::hasDbAircraftData() { return sApp && sApp->hasWebDataServices() && sApp->getWebDataServices()->hasDbAircraftData(); diff --git a/src/blackcore/db/databaseutils.h b/src/blackcore/db/databaseutils.h index 1e0d58c54..a04bb2f8c 100644 --- a/src/blackcore/db/databaseutils.h +++ b/src/blackcore/db/databaseutils.h @@ -55,6 +55,18 @@ namespace BlackCore //! Create stash models if the DB models miss that simulator static BlackMisc::Simulation::CAircraftModelList updateSimulatorForFsFamily(const BlackMisc::Simulation::CAircraftModelList &ownModels, int maxToStash = -1, BlackCore::IProgressIndicator *progressIndicator = nullptr, bool processEvents = true); + //! Database JSON from content string, which can be compressed + static QJsonDocument databaseJsonToQJsonDocument(const QString &content); + + //! QJsonDocument from database JSON file (normally shared file) + static QJsonDocument readQJsonDocumentFromDatabaseFile(const QString &filename); + + //! QJsonObject from database JSON file (normally shared file) + static QJsonObject readQJsonObjectFromDatabaseFile(const QString &filename); + + //! QJsonObject from database JSON file (normally shared file) + static QJsonObject readQJsonObjectFromDatabaseFile(const QString &directory, const QString &filename); + //! Convenience function static bool hasDbAircraftData(); };