From 4c33e4d7920fe17be6358fd7c21999707993f0fd Mon Sep 17 00:00:00 2001 From: Klaus Basan Date: Tue, 26 Feb 2019 23:04:29 +0100 Subject: [PATCH] Ref T554, publishing function for "auto publish data" * writer * services --- src/blackcore/db/databaseutils.cpp | 6 ++ src/blackcore/db/databaseutils.h | 3 + src/blackcore/db/databasewriter.cpp | 126 ++++++++++++++++++++++------ src/blackcore/db/databasewriter.h | 29 +++++-- src/blackcore/webdataservices.cpp | 9 +- src/blackcore/webdataservices.h | 9 +- 6 files changed, 144 insertions(+), 38 deletions(-) diff --git a/src/blackcore/db/databaseutils.cpp b/src/blackcore/db/databaseutils.cpp index fbc5b82c9..ebaecf7bf 100644 --- a/src/blackcore/db/databaseutils.cpp +++ b/src/blackcore/db/databaseutils.cpp @@ -425,6 +425,12 @@ namespace BlackCore return CDatabaseUtils::getJsonTextMultipart(bytes, compress); } + QHttpPart CDatabaseUtils::getJsonTextMultipart(const QString &json, bool compress) + { + const QByteArray bytes(json.toUtf8()); + return CDatabaseUtils::getJsonTextMultipart(bytes, compress); + } + QHttpPart CDatabaseUtils::getJsonTextMultipart(const QJsonArray &json, bool compress) { const QByteArray bytes(QJsonDocument(json).toJson(QJsonDocument::Compact)); diff --git a/src/blackcore/db/databaseutils.h b/src/blackcore/db/databaseutils.h index cfb53a363..6983659ea 100644 --- a/src/blackcore/db/databaseutils.h +++ b/src/blackcore/db/databaseutils.h @@ -93,6 +93,9 @@ namespace BlackCore //! Multipart for JSON static QHttpPart getJsonTextMultipart(const QJsonObject &json, bool compress); + //! Multipart for JSON + static QHttpPart getJsonTextMultipart(const QString &json, bool compress); + //! Multipart for JSON static QHttpPart getJsonTextMultipart(const QJsonArray &json, bool compress); diff --git a/src/blackcore/db/databasewriter.cpp b/src/blackcore/db/databasewriter.cpp index a10776ae5..95bdab276 100644 --- a/src/blackcore/db/databasewriter.cpp +++ b/src/blackcore/db/databasewriter.cpp @@ -6,14 +6,15 @@ * or distributed except according to the terms contained in the LICENSE file. */ -#include "blackcore/application.h" #include "blackcore/data/globalsetup.h" #include "blackcore/db/databasewriter.h" #include "blackcore/db/databaseutils.h" -#include "blackmisc/db/datastoreutility.h" -#include "blackmisc/logcategory.h" -#include "blackmisc/logcategorylist.h" +#include "blackcore/application.h" +#include "blackmisc/simulation/autopublishdata.h" #include "blackmisc/network/networkutils.h" +#include "blackmisc/db/datastoreutility.h" +#include "blackmisc/logcategorylist.h" +#include "blackmisc/logcategory.h" #include "blackmisc/statusmessage.h" #include @@ -38,7 +39,8 @@ namespace BlackCore { CDatabaseWriter::CDatabaseWriter(const Network::CUrl &baseUrl, QObject *parent) : QObject(parent), - m_modelPublishUrl(getModelPublishUrl(baseUrl)) + m_modelPublishUrl(CDatabaseWriter::getModelPublishUrl(baseUrl)), + m_autoPublishUrl(CDatabaseWriter::getAutoPublishUrl(baseUrl)) { // void } @@ -51,15 +53,15 @@ namespace BlackCore CStatusMessageList CDatabaseWriter::asyncPublishModels(const CAircraftModelList &models) { CStatusMessageList msgs; - if (m_shutdown) + if (m_shutdown || !sApp) { - msgs.push_back(CStatusMessage(CStatusMessage::SeverityWarning, u"Database writer shuts down")); + msgs.push_back(CStatusMessage(CStatusMessage::SeverityWarning, u"Database writer shutting down")); return msgs; } - if (this->isReplyOverdue()) + if (this->isModelReplyOverdue()) { - const bool killed = this->killPendingReply(); + const bool killed = this->killPendingModelReply(); if (killed) { const CStatusMessage msg(CStatusMessage::SeverityWarning, u"Aborted outdated pending reply"); @@ -69,7 +71,7 @@ namespace BlackCore } } - if (m_pendingReply) + if (m_pendingModelPublishReply) { msgs.push_back(CStatusMessage(CStatusMessage::SeverityWarning, u"Another write operation in progress")); return msgs; @@ -88,15 +90,49 @@ namespace BlackCore QNetworkRequest request(url); CNetworkUtils::ignoreSslVerification(request); const int logId = m_writeLog.addPendingUrl(url); - m_pendingReply = sApp->postToNetwork(request, logId, multiPart, { this, &CDatabaseWriter::postedModelsResponse}); - m_replyPendingSince = QDateTime::currentMSecsSinceEpoch(); + m_pendingModelPublishReply = sApp->postToNetwork(request, logId, multiPart, { this, &CDatabaseWriter::postedModelsResponse}); + m_modelReplyPendingSince = QDateTime::currentMSecsSinceEpoch(); + return msgs; + } + + CStatusMessageList CDatabaseWriter::asyncAutoPublish(const CAutoPublishData &data) + { + CStatusMessageList msgs; + if (m_shutdown || !sApp) + { + msgs.push_back(CStatusMessage(CStatusMessage::SeverityWarning, u"Database writer shutting down")); + return msgs; + } + + if (data.isEmpty()) + { + msgs.push_back(CStatusMessage(CStatusMessage::SeverityWarning, u"No auto update data")); + return msgs; + } + + const QString json = data.toDatabaseJson(); + const bool compress = json.size() > 2048; + QHttpMultiPart *multiPart = new QHttpMultiPart(QHttpMultiPart::FormDataType, this); + multiPart->append(CDatabaseUtils::getJsonTextMultipart(json, compress)); + if (sApp->getGlobalSetup().dbDebugFlag()) + { + multiPart->append(CDatabaseUtils::getMultipartWithDebugFlag()); + } + + QUrl url(m_modelPublishUrl.toQUrl()); + if (compress) { url.setQuery(CDatabaseUtils::getCompressedQuery()); } + QNetworkRequest request(url); + CNetworkUtils::ignoreSslVerification(request); + const int logId = m_writeLog.addPendingUrl(url); + m_pendingAutoPublishReply = sApp->postToNetwork(request, logId, multiPart, { this, &CDatabaseWriter::postedAutoPublishResponse}); + m_autoPublishReplyPendingSince = QDateTime::currentMSecsSinceEpoch(); return msgs; } void CDatabaseWriter::gracefulShutdown() { m_shutdown = true; - this->killPendingReply(); + this->killPendingModelReply(); } const QString &CDatabaseWriter::getName() @@ -109,20 +145,20 @@ namespace BlackCore { static const CLogCategoryList cats(CLogCategoryList(this).join({ CLogCategory::swiftDbWebservice()})); QScopedPointer nwReply(nwReplyPtr); - if (m_shutdown) + if (m_shutdown || !sApp) { nwReply->abort(); return; } - m_pendingReply = nullptr; + m_pendingModelPublishReply = nullptr; const QUrl url(nwReply->url()); const QString urlString(url.toString()); if (nwReply->error() == QNetworkReply::NoError) { - const QString dataFileData(nwReply->readAll().trimmed()); + const QString responseData(nwReply->readAll().trimmed()); nwReply->close(); // close asap - if (dataFileData.isEmpty()) + if (responseData.isEmpty()) { const CStatusMessageList msgs({CStatusMessage(cats, CStatusMessage::SeverityError, u"No response data from " % urlString)}); emit this->publishedModels(CAircraftModelList(), CAircraftModelList(), msgs, false, false); @@ -133,7 +169,7 @@ namespace BlackCore CAircraftModelList modelsSkipped; CStatusMessageList msgs; bool directWrite; - const bool sendingSuccessful = CDatastoreUtility::parseSwiftPublishResponse(dataFileData, modelsPublished, modelsSkipped, msgs, directWrite); + const bool sendingSuccessful = CDatastoreUtility::parseSwiftPublishResponse(responseData, modelsPublished, modelsSkipped, msgs, directWrite); const int c = CDatabaseUtils::fillInMissingAircraftAndLiveryEntities(modelsPublished); emit this->publishedModels(modelsPublished, modelsSkipped, msgs, sendingSuccessful, directWrite); if (!modelsPublished.isEmpty()) @@ -146,24 +182,55 @@ namespace BlackCore { const QString error = nwReply->errorString(); nwReply->close(); // close asap - const CStatusMessageList msgs( {CStatusMessage(cats, CStatusMessage::SeverityError, u"HTTP error: " % error)}); + const CStatusMessageList msgs({CStatusMessage(cats, CStatusMessage::SeverityError, u"HTTP error: " % error)}); emit this->publishedModels(CAircraftModelList(), CAircraftModelList(), msgs, false, false); } } - bool CDatabaseWriter::killPendingReply() + void CDatabaseWriter::postedAutoPublishResponse(QNetworkReply *nwReplyPtr) { - if (!m_pendingReply) { return false; } - m_pendingReply->abort(); - m_pendingReply = nullptr; - m_replyPendingSince = -1; + static const CLogCategoryList cats(CLogCategoryList(this).join({ CLogCategory::swiftDbWebservice()})); + QScopedPointer nwReply(nwReplyPtr); + if (m_shutdown || !sApp) + { + nwReply->abort(); + return; + } + + m_pendingAutoPublishReply = nullptr; + const QUrl url(nwReply->url()); + const QString urlString(url.toString()); + if (nwReply->error() == QNetworkReply::NoError) + { + const QString responseData(nwReply->readAll().trimmed()); + nwReply->close(); // close asap + if (responseData.isEmpty()) + { + const CStatusMessageList msgs({CStatusMessage(cats, CStatusMessage::SeverityError, u"No response data from " % urlString)}); + return; + } + } + else + { + const QString error = nwReply->errorString(); + nwReply->close(); // close asap + const CStatusMessageList msgs({CStatusMessage(cats, CStatusMessage::SeverityError, u"HTTP error: " % error)}); + } + } + + bool CDatabaseWriter::killPendingModelReply() + { + if (!m_pendingModelPublishReply) { return false; } + m_pendingModelPublishReply->abort(); + m_pendingModelPublishReply = nullptr; + m_modelReplyPendingSince = -1; return true; } - bool CDatabaseWriter::isReplyOverdue() const + bool CDatabaseWriter::isModelReplyOverdue() const { - if (m_replyPendingSince < 0 || !m_pendingReply) { return false; } - const qint64 ms = QDateTime::currentMSecsSinceEpoch() - m_replyPendingSince; + if (m_modelReplyPendingSince < 0 || !m_pendingModelPublishReply) { return false; } + const qint64 ms = QDateTime::currentMSecsSinceEpoch() - m_modelReplyPendingSince; return ms > 7500; } @@ -172,6 +239,11 @@ namespace BlackCore return baseUrl.withAppendedPath("service/publishmodels.php"); } + CUrl CDatabaseWriter::getAutoPublishUrl(const CUrl &baseUrl) + { + return baseUrl.withAppendedPath("service/publishauto.php"); + } + QList CDatabaseWriter::splitData(const QByteArray &data, int size) { if (data.size() <= size) { return QList({data}); } diff --git a/src/blackcore/db/databasewriter.h b/src/blackcore/db/databasewriter.h index 4ac2699ba..d03e9d949 100644 --- a/src/blackcore/db/databasewriter.h +++ b/src/blackcore/db/databasewriter.h @@ -12,9 +12,9 @@ #define BLACKCORE_DATABASE_WRITER_H #include "blackcore/blackcoreexport.h" -#include "blackmisc/network/url.h" -#include "blackmisc/network/urlloglist.h" #include "blackmisc/simulation/aircraftmodellist.h" +#include "blackmisc/network/urlloglist.h" +#include "blackmisc/network/url.h" #include "blackmisc/statusmessagelist.h" #include @@ -23,6 +23,7 @@ class QNetworkReply; +namespace BlackMisc { namespace Simulation { class CAutoPublishData; }} namespace BlackCore { namespace Db @@ -42,6 +43,9 @@ namespace BlackCore //! Write models to DB BlackMisc::CStatusMessageList asyncPublishModels(const BlackMisc::Simulation::CAircraftModelList &models); + //! Write auto publis data + BlackMisc::CStatusMessageList asyncAutoPublish(const BlackMisc::Simulation::CAutoPublishData &data); + //! Shutdown void gracefulShutdown(); @@ -66,23 +70,32 @@ namespace BlackCore private: BlackMisc::Network::CUrlLogList m_writeLog; - BlackMisc::Network::CUrl m_modelPublishUrl; - QNetworkReply *m_pendingReply = nullptr; - qint64 m_replyPendingSince = -1; + BlackMisc::Network::CUrl m_modelPublishUrl; //!< model publishing + BlackMisc::Network::CUrl m_autoPublishUrl; //!< auto publish data + QNetworkReply *m_pendingModelPublishReply = nullptr; + QNetworkReply *m_pendingAutoPublishReply = nullptr; + qint64 m_modelReplyPendingSince = -1; + qint64 m_autoPublishReplyPendingSince = -1; bool m_shutdown = false; - //! Post response + //! Post response for models void postedModelsResponse(QNetworkReply *nwReplyPtr); + //! Post response for auto publish + void postedAutoPublishResponse(QNetworkReply *nwReplyPtr); + //! Kill the pending reply - bool killPendingReply(); + bool killPendingModelReply(); //! Reply timed out? - bool isReplyOverdue() const; + bool isModelReplyOverdue() const; //! URL model web service static BlackMisc::Network::CUrl getModelPublishUrl(const BlackMisc::Network::CUrl &baseUrl); + //! URL auto publish web service + static BlackMisc::Network::CUrl getAutoPublishUrl(const BlackMisc::Network::CUrl &baseUrl); + //! Split data array static QList splitData(const QByteArray &data, int size); }; diff --git a/src/blackcore/webdataservices.cpp b/src/blackcore/webdataservices.cpp index b5c3bd2bc..93ebcb07b 100644 --- a/src/blackcore/webdataservices.cpp +++ b/src/blackcore/webdataservices.cpp @@ -167,6 +167,12 @@ namespace BlackCore return CStatusMessageList(); } + CStatusMessageList CWebDataServices::asyncAutoPublish(const CAutoPublishData &data) const + { + if (m_databaseWriter) { return m_databaseWriter->asyncAutoPublish(data);} + return CStatusMessageList(); + } + void CWebDataServices::triggerReadOfDbInfoObjects() { initDbInfoObjectReaderAndTriggerRead(); @@ -1242,8 +1248,7 @@ namespace BlackCore void CWebDataServices::initWriters() { - m_databaseWriter = new CDatabaseWriter(sApp->getGlobalSetup().getDbRootDirectoryUrl(), - this); + m_databaseWriter = new CDatabaseWriter(sApp->getGlobalSetup().getDbRootDirectoryUrl(), this); } bool CWebDataServices::signalEntitiesAlreadyRead(CEntityFlags::Entity entities) diff --git a/src/blackcore/webdataservices.h b/src/blackcore/webdataservices.h index 53c4ded37..a282155ae 100644 --- a/src/blackcore/webdataservices.h +++ b/src/blackcore/webdataservices.h @@ -50,7 +50,11 @@ namespace BlackMisc template class Restricted; namespace Aviation { class CCallsign; } - namespace Simulation { class CSimulatedAircraft; } + namespace Simulation + { + class CSimulatedAircraft; + class CAutoPublishData; + } } namespace BlackCore @@ -387,6 +391,9 @@ namespace BlackCore //! Publish models to database BlackMisc::CStatusMessageList asyncPublishModels(const BlackMisc::Simulation::CAircraftModelList &modelsToBePublished) const; + //! Auto publish to database + BlackMisc::CStatusMessageList asyncAutoPublish(const BlackMisc::Simulation::CAutoPublishData &data) const; + //! Trigger read of DB info objects void triggerReadOfDbInfoObjects();