flightsimulatorplatfrom/pilotclient
1
0
Fork 0
mirror of https://github.com/swift-project/pilotclient.git synced 2026-03-31 04:25:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

Ref T149, get URL logs from web data services

Browse Source
This commit is contained in:
Klaus Basan
2017-09-10 23:41:08 +02:00
committed by Mathew Sutcliffe
parent 17d2243e3f
commit dbe66eceb2
5 changed files with 135 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
src/blackcore/application.cpp
View File

@@ -616,20 +616,38 @@ namespace BlackCore
QNetworkReply *CApplication::getFromNetwork(const CUrl &url, const CSlot<void(QNetworkReply *)> &callback, int maxRedirects)
{
return httpRequestImpl(url.toNetworkRequest(), callback, maxRedirects, [ ](QNetworkAccessManager & nam, const QNetworkRequest & request) { return nam.get(request); });
return getFromNetwork(url.toNetworkRequest(), NoLogRequestId, callback, maxRedirects);
}
QNetworkReply *CApplication::getFromNetwork(const CUrl &url, int logId, const CSlot<void(QNetworkReply *)> &callback, int maxRedirects)
{
return getFromNetwork(url.toNetworkRequest(), logId, callback, maxRedirects);
}
QNetworkReply *CApplication::getFromNetwork(const QNetworkRequest &request, const CSlot<void(QNetworkReply *)> &callback, int maxRedirects)
{
return httpRequestImpl(request, callback, maxRedirects, [ ](QNetworkAccessManager & nam, const QNetworkRequest & request) { return nam.get(request); });
return getFromNetwork(request, NoLogRequestId, callback, maxRedirects);
}
QNetworkReply *CApplication::postToNetwork(const QNetworkRequest &request, const QByteArray &data, const CSlot<void(QNetworkReply *)> &callback)
QNetworkReply *CApplication::getFromNetwork(const QNetworkRequest &request, int logId, const CSlot<void (QNetworkReply *)> &callback, int maxRedirects)
{
return httpRequestImpl(request, callback, -1, [ data ](QNetworkAccessManager & nam, const QNetworkRequest & request) { return nam.post(request, data); });
return httpRequestImpl(request, logId, callback, maxRedirects, [](QNetworkAccessManager & qam, const QNetworkRequest & request)
{
QNetworkReply *nr = qam.get(request);
return nr;
});
}
QNetworkReply *CApplication::postToNetwork(const QNetworkRequest &request, QHttpMultiPart *multiPart, const CSlot<void(QNetworkReply *)> &callback)
QNetworkReply *CApplication::postToNetwork(const QNetworkRequest &request, int logId, const QByteArray &data, const CSlot<void(QNetworkReply *)> &callback)
{
return httpRequestImpl(request, logId, callback, NoRedirects, [ data ](QNetworkAccessManager & qam, const QNetworkRequest & request)
{
QNetworkReply *nr = qam.post(request, data);
return nr;
});
}
QNetworkReply *CApplication::postToNetwork(const QNetworkRequest &request, int logId, QHttpMultiPart *multiPart, const CSlot<void(QNetworkReply *)> &callback)
{
if (!this->isNetworkAccessible()) { return nullptr; }
if (QThread::currentThread() != this->m_accessManager->thread())
@@ -637,23 +655,22 @@ namespace BlackCore
multiPart->moveToThread(this->m_accessManager->thread());
}
return httpRequestImpl(request, callback, -1, [ this, multiPart ](QNetworkAccessManager & qam, const QNetworkRequest & request)
return httpRequestImpl(request, logId, callback, NoRedirects, [ this, multiPart ](QNetworkAccessManager & qam, const QNetworkRequest & request)
{
QNetworkReply *reply = qam.post(request, multiPart);
Q_ASSERT(reply);
multiPart->setParent(reply);
return reply;
QNetworkReply *nr = qam.post(request, multiPart);
multiPart->setParent(nr);
return nr;
});
}
QNetworkReply *CApplication::headerFromNetwork(const CUrl &url, const CSlot<void (QNetworkReply *)> &callback, int maxRedirects)
{
return httpRequestImpl(url.toNetworkRequest(), callback, maxRedirects, [ ](QNetworkAccessManager & qam, const QNetworkRequest & request) { return qam.head(request); });
return headerFromNetwork(url.toNetworkRequest(), callback, maxRedirects);
}
QNetworkReply *CApplication::headerFromNetwork(const QNetworkRequest &request, const CSlot<void (QNetworkReply *)> &callback, int maxRedirects)
{
return httpRequestImpl(request, callback, maxRedirects, [ ](QNetworkAccessManager & qam, const QNetworkRequest & request) { return qam.head(request); });
return httpRequestImpl(request, NoLogRequestId, callback, maxRedirects, [ ](QNetworkAccessManager & qam, const QNetworkRequest & request) { return qam.head(request); });
}
QNetworkReply *CApplication::downloadFromNetwork(const CUrl &url, const QString &saveAsFileName, const BlackMisc::CSlot<void (const CStatusMessage &)> &callback, int maxRedirects)
@@ -1310,7 +1327,7 @@ namespace BlackCore
if (!QFileInfo::exists(handler))
{
return CStatusMessage(this).warning("%1 not found. Cannot init crash handler!") << handler;
return CStatusMessage(this).warning("Crashpad handler '%1' not found. Cannot init handler!") << handler;
}
const CUrl serverUrl = this->getGlobalSetup().getCrashReportServerUrl();
@@ -1342,7 +1359,9 @@ namespace BlackCore
#endif
}
QNetworkReply *CApplication::httpRequestImpl(const QNetworkRequest &request, const BlackMisc::CSlot<void (QNetworkReply *)> &callback, int maxRedirects, std::function<QNetworkReply *(QNetworkAccessManager &, const QNetworkRequest &)> requestOrPostMethod)
QNetworkReply *CApplication::httpRequestImpl(
const QNetworkRequest &request, int logId,
const BlackMisc::CSlot<void (QNetworkReply *)> &callback, int maxRedirects, std::function<QNetworkReply *(QNetworkAccessManager &, const QNetworkRequest &)> requestOrPostMethod)
{
if (this->isShuttingDown()) { return nullptr; }
if (!this->isNetworkAccessible()) { return nullptr; }
@@ -1351,7 +1370,7 @@ namespace BlackCore
if (QThread::currentThread() != this->m_accessManager->thread())
{
// run in QAM thread
QTimer::singleShot(0, this->m_accessManager, std::bind(&CApplication::httpRequestImpl, this, request, callback, maxRedirects, requestOrPostMethod));
QTimer::singleShot(0, this->m_accessManager, std::bind(&CApplication::httpRequestImpl, this, request, logId, callback, maxRedirects, requestOrPostMethod));
return nullptr; // not yet started
}
@@ -1360,11 +1379,12 @@ namespace BlackCore
CNetworkUtils::ignoreSslVerification(copiedRequest);
CNetworkUtils::setSwiftUserAgent(copiedRequest);
// If URL is one of the shared urls, add swift client SSL certificate
// If URL is one of the shared URLs, add swift client SSL certificate
CNetworkUtils::setSwiftClientSslCertificate(copiedRequest, getGlobalSetup().getSwiftSharedUrls());
QNetworkReply *reply = requestOrPostMethod(*this->m_accessManager, copiedRequest);
reply->setProperty("started", QVariant(QDateTime::currentMSecsSinceEpoch()));
reply->setProperty(CUrlLog::propertyNameId(), QVariant(logId));
if (callback)
{
Q_ASSERT_X(callback.object(), Q_FUNC_INFO, "Need callback object (to determine thread)");
@@ -1381,7 +1401,7 @@ namespace BlackCore
{
QNetworkRequest redirectRequest(redirectUrl);
const int redirectsLeft = maxRedirects - 1;
QTimer::singleShot(0, this, std::bind(&CApplication::httpRequestImpl, this, redirectRequest, callback, redirectsLeft, requestOrPostMethod));
QTimer::singleShot(0, this, std::bind(&CApplication::httpRequestImpl, this, redirectRequest, logId, callback, redirectsLeft, requestOrPostMethod));
return;
}
}

32
src/blackcore/application.h
View File

@@ -365,42 +365,56 @@ namespace BlackCore
//! Wait for setup data by calling the event loop and waiting until everything is ready
BlackMisc::CStatusMessageList waitForSetup();
public:
static constexpr int NoRedirects = -1; //!< network request not allowing redirects
static constexpr int NoLogRequestId = -1; //!< network request without logging
static constexpr int DefaultMaxRedirects = 2; //!< network request, default for max.redirects
//! Request to get network reply
//! \threadsafe
QNetworkReply *getFromNetwork(const BlackMisc::Network::CUrl &url,
const BlackMisc::CSlot<void(QNetworkReply *)> &callback, int maxRedirects = 2);
const BlackMisc::CSlot<void(QNetworkReply *)> &callback, int maxRedirects = DefaultMaxRedirects);
//! Request to get network reply, supporting BlackMisc::Network::CUrlLog
//! \threadsafe
QNetworkReply *getFromNetwork(const BlackMisc::Network::CUrl &url, int logId,
const BlackMisc::CSlot<void(QNetworkReply *)> &callback, int maxRedirects = DefaultMaxRedirects);
//! Request to get network reply
//! \threadsafe
QNetworkReply *getFromNetwork(const QNetworkRequest &request,
const BlackMisc::CSlot<void(QNetworkReply *)> &callback, int maxRedirects = 2);
const BlackMisc::CSlot<void(QNetworkReply *)> &callback, int maxRedirects = DefaultMaxRedirects);
//! Request to get network reply, supporting BlackMisc::Network::CUrlLog
//! \threadsafe
QNetworkReply *getFromNetwork(const QNetworkRequest &request, int logId,
const BlackMisc::CSlot<void(QNetworkReply *)> &callback, int maxRedirects = DefaultMaxRedirects);
//! Post to network
//! \threadsafe
QNetworkReply *postToNetwork(const QNetworkRequest &request, const QByteArray &data,
QNetworkReply *postToNetwork(const QNetworkRequest &request, int logId, const QByteArray &data,
const BlackMisc::CSlot<void(QNetworkReply *)> &callback);
//! Post to network
//! \note This method takes ownership over \c multiPart.
//! \threadsafe
QNetworkReply *postToNetwork(const QNetworkRequest &request, QHttpMultiPart *multiPart,
QNetworkReply *postToNetwork(const QNetworkRequest &request, int logId, QHttpMultiPart *multiPart,
const BlackMisc::CSlot<void(QNetworkReply *)> &callback);
//! Request to get network repy using HTTP's HEADER method
//! \threadsafe
QNetworkReply *headerFromNetwork(const BlackMisc::Network::CUrl &url,
const BlackMisc::CSlot<void(QNetworkReply *)> &callback, int maxRedirects = -1);
const BlackMisc::CSlot<void(QNetworkReply *)> &callback, int maxRedirects = NoRedirects);
//! Request to get network repy using HTTP's HEADER method
//! \threadsafe
QNetworkReply *headerFromNetwork(const QNetworkRequest &request,
const BlackMisc::CSlot<void(QNetworkReply *)> &callback, int maxRedirects = -1);
const BlackMisc::CSlot<void(QNetworkReply *)> &callback, int maxRedirects = NoRedirects);
public: // downloadFromNetwork no slot to avoid issue with CSlot, see T125
//! Download file from network and store it as passed
//! \threadsafe
QNetworkReply *downloadFromNetwork(const BlackMisc::Network::CUrl &url, const QString &saveAsFileName,
const BlackMisc::CSlot<void(const BlackMisc::CStatusMessage &)> &callback, int maxRedirects = 2);
const BlackMisc::CSlot<void(const BlackMisc::CStatusMessage &)> &callback, int maxRedirects = DefaultMaxRedirects);
signals:
//! Setup available (cache, web load, ..) or failed to load setup
@@ -501,7 +515,9 @@ namespace BlackCore
BlackMisc::CStatusMessageList asyncWebAndContextStart();
//! Implementation for getFromNetwork(), postToNetwork() and headerFromNetwork()
//! \return QNetworkReply reply will only be returned, if the QNetworkAccessManager is in the same thread
QNetworkReply *httpRequestImpl(const QNetworkRequest &request,
int logId,
const BlackMisc::CSlot<void(QNetworkReply *)> &callback,
int maxRedirects,
std::function<QNetworkReply *(QNetworkAccessManager &, const QNetworkRequest &)> requestOrPostMethod);

85
src/blackcore/webdataservices.cpp
View File

@@ -412,6 +412,30 @@ namespace BlackCore
return this->getInfoObjectCount(entity, m_sharedInfoDataReader);
}
QString CWebDataServices::getDbReadersLog(const QString separator) const
{
QStringList report;
if (m_dbInfoDataReader) { report << m_dbInfoDataReader->getName() + ": " + m_dbInfoDataReader->getReadLog().getSummary(); }
if (m_sharedInfoDataReader) { report << m_sharedInfoDataReader->getName() + ": " + m_sharedInfoDataReader->getReadLog().getSummary(); }
if (m_airportDataReader) { report << m_airportDataReader->getName() + ": " + m_airportDataReader->getReadLog().getSummary(); }
if (m_icaoDataReader) { report << m_icaoDataReader->getName() + ": " + m_icaoDataReader->getReadLog().getSummary(); }
if (m_modelDataReader) { report << m_modelDataReader->getName() + ": " + m_modelDataReader->getReadLog().getSummary(); }
if (m_databaseWriter) { report << m_databaseWriter->getName() + ": " + m_databaseWriter->getWriteLog().getSummary(); }
return report.join(separator);
}
QString CWebDataServices::getReadersLog(const QString separator) const
{
const QString db = this->getDbReadersLog(separator);
QStringList report;
if (m_vatsimBookingReader) { report << m_vatsimBookingReader->getName() + ": " + m_vatsimBookingReader->getReadLog().getSummary(); }
if (m_vatsimMetarReader) { report << m_vatsimMetarReader->getName() + ": " + m_vatsimMetarReader->getReadLog().getSummary(); }
if (m_vatsimStatusReader) { report << m_vatsimStatusReader->getName() + ": " + m_vatsimStatusReader->getReadLog().getSummary(); }
if (report.isEmpty()) { return db; }
return report.join(separator) + separator + db;
}
CDistributorList CWebDataServices::getDistributors() const
{
if (m_modelDataReader) { return m_modelDataReader->getDistributors(); }
@@ -1165,7 +1189,7 @@ namespace BlackCore
if (!this->waitForDbInfoObjectsThenRead(entities)) { return; }
}
const bool waitForSharedInfoFile = m_dbReaderConfig.needsSharedInfoFile(entities);
const bool waitForSharedInfoFile = m_dbReaderConfig.needsSharedInfoFile(entities) && !m_sharedInfoDataReader->areAllInfoObjectsRead();
if (waitForSharedInfoFile)
{
// do not read yet, will call this function again after some time
@@ -1181,67 +1205,68 @@ namespace BlackCore
bool CWebDataServices::waitForDbInfoObjectsThenRead(CEntityFlags::Entity entities)
{
Q_ASSERT_X(m_dbInfoDataReader, Q_FUNC_INFO, "need reader");
if (m_dbInfoDataReader->areAllInfoObjectsRead()) { return true; }
if (!m_dbInfoObjectTimeout.isValid()) { m_dbInfoObjectTimeout = QDateTime::currentDateTimeUtc().addMSecs(10 * 1000); }
const bool read = this->waitForInfoObjectsThenRead(entities, "DB", m_dbInfoDataReader, m_dbInfoObjectTimeout);
if (read) { m_dbInfoObjectTimeout = QDateTime(); } // reset to null
return read;
}
bool CWebDataServices::waitForSharedInfoObjectsThenRead(CEntityFlags::Entity entities)
{
Q_ASSERT_X(m_sharedInfoDataReader, Q_FUNC_INFO, "need reader");
if (m_sharedInfoDataReader->areAllInfoObjectsRead()) { return true; }
if (!m_sharedInfoObjectsTimeout.isValid()) { m_sharedInfoObjectsTimeout = QDateTime::currentDateTimeUtc().addMSecs(10 * 1000); }
const bool read = this->waitForInfoObjectsThenRead(entities, "shared", m_sharedInfoDataReader, m_sharedInfoObjectsTimeout);
if (read) { m_sharedInfoObjectsTimeout = QDateTime(); } // reset to null
return read;
}
bool CWebDataServices::waitForInfoObjectsThenRead(CEntityFlags::Entity entities, const QString &info, CInfoDataReader *reader, const QDateTime &timeOut)
bool CWebDataServices::waitForInfoObjectsThenRead(CEntityFlags::Entity entities, const QString &info, CInfoDataReader *infoReader, QDateTime &timeOut)
{
Q_ASSERT_X(reader, Q_FUNC_INFO, "Need info data reader");
Q_ASSERT_X(infoReader, Q_FUNC_INFO, "Need info data reader");
// this will called for each entity readers, i.e. model reader, ICAO reader ...
const int waitForInfoObjectsMs = 1000; // ms
if (infoReader->areAllInfoObjectsRead())
{
// we have all data and carry on
CLogMessage(this).info("Info objects (%1) triggered for '%2' loaded from '%3'") << info << CEntityFlags::flagToString(entities) << infoReader->getInfoObjectsUrl().toQString();
timeOut = QDateTime(); // reset to null
return true; // no need to wait any longer
}
// try to read if not timed out
if (QDateTime::currentDateTimeUtc() > timeOut)
if (timeOut.isValid() && QDateTime::currentDateTimeUtc() > timeOut)
{
const QString timeOutString = timeOut.toString();
CLogMessage(this).warning("Could not read '%1' info objects for '%2' from '%3', time out '%4'. Marking reader '%5' as failed and continue.")
<< info << CEntityFlags::flagToString(entities)
<< reader->getInfoObjectsUrl().toQString() << timeOutString
<< reader->getName();
<< infoReader->getInfoObjectsUrl().toQString() << timeOutString
<< infoReader->getName();
// continue here and read data without info objects
reader->setMarkedAsFailed(true);
infoReader->setMarkedAsFailed(true);
// no timeout reset here
return true; // carry on, regardless of situation
}
else if (reader->hasReceivedFirstReply())
if (infoReader->hasReceivedFirstReply())
{
if (reader->areAllInfoObjectsRead())
// we have received a response, but not all data yet
if (infoReader->hasReceivedOkReply())
{
// we have all data and carry on
CLogMessage(this).info("Info objects (%1) for '%2' loaded from '%3'") << info << CEntityFlags::flagToString(entities) << reader->getInfoObjectsUrl().toQString();
return true; // no need to wait any longer
// ok, this means we are parsing
this->readDeferredInBackground(entities, waitForInfoObjectsMs);
CLogMessage(this).info("Waiting for objects (%1) for '%2' from '%3'") << info << CEntityFlags::flagToString(entities) << infoReader->getInfoObjectsUrl().toQString();
return false; // wait
}
else
{
// we have received a response, but not all data yet
if (reader->hasReceivedOkReply())
{
// ok, this means we are parsing
this->readDeferredInBackground(entities, waitForInfoObjectsMs);
CLogMessage(this).info("Waiting for objects (%1) for '%2' from '%3'") << info << CEntityFlags::flagToString(entities) << reader->getInfoObjectsUrl().toQString();
return false; // wait
}
else
{
// we have a response, but a failure, means server is alive, but responded with error
// such an error (access, ...) normally will not go away
CLogMessage(this).error("Info objects (%1) loading for '%2' failed from '%3', '%4'") << info << CEntityFlags::flagToString(entities) << reader->getInfoObjectsUrl().toQString() << reader->getStatusMessage();
reader->setMarkedAsFailed(true);
return true; // carry on, regardless of situation
}
// we have a response, but a failure, means server is alive, but responded with error
// such an error (access, ...) normally will not go away
CLogMessage(this).error("Info objects (%1) loading for '%2' failed from '%3', '%4'") << info << CEntityFlags::flagToString(entities) << infoReader->getInfoObjectsUrl().toQString() << infoReader->getStatusMessage();
infoReader->setMarkedAsFailed(true);
return true; // carry on, regardless of situation
}
}
else

8
src/blackcore/webdataservices.h
View File

@@ -395,6 +395,12 @@ namespace BlackCore
//! \threadsafe
int getSharedInfoObjectCount(BlackMisc::Network::CEntityFlags::Entity entity) const;
//! For all available DB readers the log info is generated
QString getDbReadersLog(const QString separator = "\n") const;
//! For all available readers the log info is generated
QString getReadersLog(const QString separator = "\n") const;
//! Has already connect swift DB?
bool hasConnectedSwiftDb() const;
@@ -513,7 +519,7 @@ namespace BlackCore
//! Wait for info objects to be read
//! \return true means info objects available
bool waitForInfoObjectsThenRead(BlackMisc::Network::CEntityFlags::Entity entities, const QString &info, BlackCore::Db::CInfoDataReader *reader, const QDateTime &timeOut);
bool waitForInfoObjectsThenRead(BlackMisc::Network::CEntityFlags::Entity entities, const QString &info, BlackCore::Db::CInfoDataReader *infoReader, QDateTime &timeOut);
CWebReaderFlags::WebReader m_readers = CWebReaderFlags::WebReaderFlag::None; //!< which readers are available
BlackMisc::Network::CEntityFlags::Entity m_entitiesPeriodicallyRead = BlackMisc::Network::CEntityFlags::NoEntity; //!< entities permanently updated by timers

12
src/blackgui/guiapplication.cpp
View File

@@ -10,6 +10,7 @@
#include "blackconfig/buildconfig.h"
#include "blackcore/context/contextnetwork.h"
#include "blackcore/data/globalsetup.h"
#include "blackcore/webdataservices.h"
#include "blackgui/components/applicationclosedialog.h"
#include "blackgui/components/downloadandinstalldialog.h"
#include "blackgui/components/aboutdialog.h"
@@ -56,6 +57,7 @@ using namespace BlackMisc;
using namespace BlackMisc::Db;
using namespace BlackMisc::Network;
using namespace BlackGui::Components;
using namespace BlackCore;
using namespace BlackCore::Data;
BlackGui::CGuiApplication *sGui = nullptr; // set by constructor
@@ -457,6 +459,16 @@ namespace BlackGui
});
Q_ASSERT_X(c, Q_FUNC_INFO, "Connect failed");
if (this->hasWebDataServices())
{
a = menu.addAction("Services log");
c = connect(a, &QAction::triggered, this, [a, this]()
{
this->displayTextInConsole(this->getWebDataServices()->getReadersLog());
});
Q_ASSERT_X(c, Q_FUNC_INFO, "Connect failed");
}
a = menu.addAction("Metadata (slow)");
c = connect(a, &QAction::triggered, this, [a, this]()
{