Ref T149, get URL logs from web data services

src/blackcore/application.cpp

@@ -616,20 +616,38 @@ namespace BlackCore
 
     QNetworkReply *CApplication::getFromNetwork(const CUrl &url, const CSlot<void(QNetworkReply *)> &callback, int maxRedirects)
     {
-        return httpRequestImpl(url.toNetworkRequest(), callback, maxRedirects, [ ](QNetworkAccessManager & nam, const QNetworkRequest & request) { return nam.get(request); });
+        return getFromNetwork(url.toNetworkRequest(), NoLogRequestId, callback, maxRedirects);
+    }
+
+    QNetworkReply *CApplication::getFromNetwork(const CUrl &url, int logId, const CSlot<void(QNetworkReply *)> &callback, int maxRedirects)
+    {
+        return getFromNetwork(url.toNetworkRequest(), logId, callback, maxRedirects);
     }
 
     QNetworkReply *CApplication::getFromNetwork(const QNetworkRequest &request, const CSlot<void(QNetworkReply *)> &callback, int maxRedirects)
     {
-        return httpRequestImpl(request, callback, maxRedirects, [ ](QNetworkAccessManager & nam, const QNetworkRequest & request) { return nam.get(request); });
+        return getFromNetwork(request, NoLogRequestId, callback, maxRedirects);
     }
 
-    QNetworkReply *CApplication::postToNetwork(const QNetworkRequest &request, const QByteArray &data, const CSlot<void(QNetworkReply *)> &callback)
+    QNetworkReply *CApplication::getFromNetwork(const QNetworkRequest &request, int logId, const CSlot<void (QNetworkReply *)> &callback, int maxRedirects)
     {
-        return httpRequestImpl(request, callback, -1, [ data ](QNetworkAccessManager & nam, const QNetworkRequest & request) { return nam.post(request, data); });
+        return httpRequestImpl(request, logId, callback, maxRedirects, [](QNetworkAccessManager & qam, const QNetworkRequest & request)
+        {
+            QNetworkReply *nr = qam.get(request);
+            return nr;
+        });
     }
 
-    QNetworkReply *CApplication::postToNetwork(const QNetworkRequest &request, QHttpMultiPart *multiPart, const CSlot<void(QNetworkReply *)> &callback)
+    QNetworkReply *CApplication::postToNetwork(const QNetworkRequest &request, int logId, const QByteArray &data, const CSlot<void(QNetworkReply *)> &callback)
+    {
+        return httpRequestImpl(request, logId, callback, NoRedirects, [ data ](QNetworkAccessManager & qam, const QNetworkRequest & request)
+        {
+            QNetworkReply *nr = qam.post(request, data);
+            return nr;
+        });
+    }
+
+    QNetworkReply *CApplication::postToNetwork(const QNetworkRequest &request, int logId, QHttpMultiPart *multiPart, const CSlot<void(QNetworkReply *)> &callback)
     {
         if (!this->isNetworkAccessible()) { return nullptr; }
         if (QThread::currentThread() != this->m_accessManager->thread())
@@ -637,23 +655,22 @@ namespace BlackCore
             multiPart->moveToThread(this->m_accessManager->thread());
         }
 
-        return httpRequestImpl(request, callback, -1, [ this, multiPart ](QNetworkAccessManager & qam, const QNetworkRequest & request)
+        return httpRequestImpl(request, logId, callback, NoRedirects, [ this, multiPart ](QNetworkAccessManager & qam, const QNetworkRequest & request)
         {
-            QNetworkReply *reply = qam.post(request, multiPart);
+            QNetworkReply *nr = qam.post(request, multiPart);
-            Q_ASSERT(reply);
+            multiPart->setParent(nr);
-            multiPart->setParent(reply);
+            return nr;
-            return reply;
         });
     }
 
     QNetworkReply *CApplication::headerFromNetwork(const CUrl &url, const CSlot<void (QNetworkReply *)> &callback, int maxRedirects)
     {
-        return httpRequestImpl(url.toNetworkRequest(), callback, maxRedirects, [ ](QNetworkAccessManager & qam, const QNetworkRequest & request) { return qam.head(request); });
+        return headerFromNetwork(url.toNetworkRequest(), callback, maxRedirects);
     }
 
     QNetworkReply *CApplication::headerFromNetwork(const QNetworkRequest &request, const CSlot<void (QNetworkReply *)> &callback, int maxRedirects)
     {
-        return httpRequestImpl(request, callback, maxRedirects, [ ](QNetworkAccessManager & qam, const QNetworkRequest & request) { return qam.head(request); });
+        return httpRequestImpl(request, NoLogRequestId, callback, maxRedirects, [ ](QNetworkAccessManager & qam, const QNetworkRequest & request) { return qam.head(request); });
     }
 
     QNetworkReply *CApplication::downloadFromNetwork(const CUrl &url, const QString &saveAsFileName, const BlackMisc::CSlot<void (const CStatusMessage &)> &callback, int maxRedirects)
@@ -1310,7 +1327,7 @@ namespace BlackCore
 
         if (!QFileInfo::exists(handler))
         {
-            return CStatusMessage(this).warning("%1 not found. Cannot init crash handler!") << handler;
+            return CStatusMessage(this).warning("Crashpad handler '%1' not found. Cannot init handler!") << handler;
         }
 
         const CUrl serverUrl = this->getGlobalSetup().getCrashReportServerUrl();
@@ -1342,7 +1359,9 @@ namespace BlackCore
 #endif
     }
 
-    QNetworkReply *CApplication::httpRequestImpl(const QNetworkRequest &request, const BlackMisc::CSlot<void (QNetworkReply *)> &callback, int maxRedirects, std::function<QNetworkReply *(QNetworkAccessManager &, const QNetworkRequest &)> requestOrPostMethod)
+    QNetworkReply *CApplication::httpRequestImpl(
+        const QNetworkRequest &request, int logId,
+        const BlackMisc::CSlot<void (QNetworkReply *)> &callback, int maxRedirects, std::function<QNetworkReply *(QNetworkAccessManager &, const QNetworkRequest &)> requestOrPostMethod)
     {
         if (this->isShuttingDown()) { return nullptr; }
         if (!this->isNetworkAccessible()) { return nullptr; }
@@ -1351,7 +1370,7 @@ namespace BlackCore
         if (QThread::currentThread() != this->m_accessManager->thread())
         {
             // run in QAM thread
-            QTimer::singleShot(0, this->m_accessManager, std::bind(&CApplication::httpRequestImpl, this, request, callback, maxRedirects, requestOrPostMethod));
+            QTimer::singleShot(0, this->m_accessManager, std::bind(&CApplication::httpRequestImpl, this, request, logId, callback, maxRedirects, requestOrPostMethod));
             return nullptr; // not yet started
         }
 
@@ -1360,11 +1379,12 @@ namespace BlackCore
         CNetworkUtils::ignoreSslVerification(copiedRequest);
         CNetworkUtils::setSwiftUserAgent(copiedRequest);
 
-        // If URL is one of the shared urls, add swift client SSL certificate
+        // If URL is one of the shared URLs, add swift client SSL certificate
         CNetworkUtils::setSwiftClientSslCertificate(copiedRequest, getGlobalSetup().getSwiftSharedUrls());
 
         QNetworkReply *reply = requestOrPostMethod(*this->m_accessManager, copiedRequest);
         reply->setProperty("started", QVariant(QDateTime::currentMSecsSinceEpoch()));
+        reply->setProperty(CUrlLog::propertyNameId(), QVariant(logId));
         if (callback)
         {
             Q_ASSERT_X(callback.object(), Q_FUNC_INFO, "Need callback object (to determine thread)");
@@ -1381,7 +1401,7 @@ namespace BlackCore
                     {
                         QNetworkRequest redirectRequest(redirectUrl);
                         const int redirectsLeft = maxRedirects - 1;
-                        QTimer::singleShot(0, this, std::bind(&CApplication::httpRequestImpl, this, redirectRequest, callback, redirectsLeft, requestOrPostMethod));
+                        QTimer::singleShot(0, this, std::bind(&CApplication::httpRequestImpl, this, redirectRequest, logId, callback, redirectsLeft, requestOrPostMethod));
                         return;
                     }
                 }

src/blackcore/application.h

@@ -365,42 +365,56 @@ namespace BlackCore
         //! Wait for setup data by calling the event loop and waiting until everything is ready
         BlackMisc::CStatusMessageList waitForSetup();
 
+    public:
+        static constexpr int NoRedirects = -1;        //!< network request not allowing redirects
+        static constexpr int NoLogRequestId = -1;     //!< network request without logging
+        static constexpr int DefaultMaxRedirects = 2; //!< network request, default for max.redirects
+
         //! Request to get network reply
         //! \threadsafe
         QNetworkReply *getFromNetwork(const BlackMisc::Network::CUrl &url,
-                                      const BlackMisc::CSlot<void(QNetworkReply *)> &callback, int maxRedirects = 2);
+                                      const BlackMisc::CSlot<void(QNetworkReply *)> &callback, int maxRedirects = DefaultMaxRedirects);
+
+        //! Request to get network reply, supporting BlackMisc::Network::CUrlLog
+        //! \threadsafe
+        QNetworkReply *getFromNetwork(const BlackMisc::Network::CUrl &url, int logId,
+                                      const BlackMisc::CSlot<void(QNetworkReply *)> &callback, int maxRedirects = DefaultMaxRedirects);
 
         //! Request to get network reply
         //! \threadsafe
         QNetworkReply *getFromNetwork(const QNetworkRequest &request,
-                                      const BlackMisc::CSlot<void(QNetworkReply *)> &callback, int maxRedirects = 2);
+                                      const BlackMisc::CSlot<void(QNetworkReply *)> &callback, int maxRedirects = DefaultMaxRedirects);
+
+        //! Request to get network reply, supporting BlackMisc::Network::CUrlLog
+        //! \threadsafe
+        QNetworkReply *getFromNetwork(const QNetworkRequest &request, int logId,
+                                      const BlackMisc::CSlot<void(QNetworkReply *)> &callback, int maxRedirects = DefaultMaxRedirects);
 
         //! Post to network
         //! \threadsafe
-        QNetworkReply *postToNetwork(const QNetworkRequest &request, const QByteArray &data,
+        QNetworkReply *postToNetwork(const QNetworkRequest &request, int logId, const QByteArray &data,
                                      const BlackMisc::CSlot<void(QNetworkReply *)> &callback);
 
         //! Post to network
         //! \note This method takes ownership over \c multiPart.
         //! \threadsafe
-        QNetworkReply *postToNetwork(const QNetworkRequest &request, QHttpMultiPart *multiPart,
+        QNetworkReply *postToNetwork(const QNetworkRequest &request, int logId, QHttpMultiPart *multiPart,
                                      const BlackMisc::CSlot<void(QNetworkReply *)> &callback);
 
         //! Request to get network repy using HTTP's HEADER method
         //! \threadsafe
         QNetworkReply *headerFromNetwork(const BlackMisc::Network::CUrl &url,
-                                         const BlackMisc::CSlot<void(QNetworkReply *)> &callback, int maxRedirects = -1);
+                                         const BlackMisc::CSlot<void(QNetworkReply *)> &callback, int maxRedirects = NoRedirects);
 
         //! Request to get network repy using HTTP's HEADER method
         //! \threadsafe
         QNetworkReply *headerFromNetwork(const QNetworkRequest &request,
-                                         const BlackMisc::CSlot<void(QNetworkReply *)> &callback, int maxRedirects = -1);
+                                         const BlackMisc::CSlot<void(QNetworkReply *)> &callback, int maxRedirects = NoRedirects);
 
-    public: // downloadFromNetwork no slot to avoid issue with CSlot, see T125
         //! Download file from network and store it as passed
         //! \threadsafe
         QNetworkReply *downloadFromNetwork(const BlackMisc::Network::CUrl &url, const QString &saveAsFileName,
-                                           const BlackMisc::CSlot<void(const BlackMisc::CStatusMessage &)> &callback, int maxRedirects = 2);
+                                           const BlackMisc::CSlot<void(const BlackMisc::CStatusMessage &)> &callback, int maxRedirects = DefaultMaxRedirects);
 
     signals:
         //! Setup available (cache, web load, ..) or failed to load setup
@@ -501,7 +515,9 @@ namespace BlackCore
         BlackMisc::CStatusMessageList asyncWebAndContextStart();
 
         //! Implementation for getFromNetwork(), postToNetwork() and headerFromNetwork()
+        //! \return QNetworkReply reply will only be returned, if the QNetworkAccessManager is in the same thread
         QNetworkReply *httpRequestImpl(const QNetworkRequest &request,
+                                       int logId,
                                        const BlackMisc::CSlot<void(QNetworkReply *)> &callback,
                                        int maxRedirects,
                                        std::function<QNetworkReply *(QNetworkAccessManager &, const QNetworkRequest &)> requestOrPostMethod);

src/blackcore/webdataservices.cpp

@@ -412,6 +412,30 @@ namespace BlackCore
         return this->getInfoObjectCount(entity, m_sharedInfoDataReader);
     }
 
+    QString CWebDataServices::getDbReadersLog(const QString separator) const
+    {
+        QStringList report;
+        if (m_dbInfoDataReader)     { report << m_dbInfoDataReader->getName() + ": " + m_dbInfoDataReader->getReadLog().getSummary(); }
+        if (m_sharedInfoDataReader) { report << m_sharedInfoDataReader->getName() + ": " + m_sharedInfoDataReader->getReadLog().getSummary(); }
+
+        if (m_airportDataReader) { report << m_airportDataReader->getName() + ": " + m_airportDataReader->getReadLog().getSummary(); }
+        if (m_icaoDataReader)    { report << m_icaoDataReader->getName() + ": " + m_icaoDataReader->getReadLog().getSummary(); }
+        if (m_modelDataReader)   { report << m_modelDataReader->getName() + ": " + m_modelDataReader->getReadLog().getSummary(); }
+        if (m_databaseWriter)    { report << m_databaseWriter->getName() + ": " + m_databaseWriter->getWriteLog().getSummary(); }
+        return report.join(separator);
+    }
+
+    QString CWebDataServices::getReadersLog(const QString separator) const
+    {
+        const QString db = this->getDbReadersLog(separator);
+        QStringList report;
+        if (m_vatsimBookingReader) { report << m_vatsimBookingReader->getName() + ": " + m_vatsimBookingReader->getReadLog().getSummary(); }
+        if (m_vatsimMetarReader)   { report << m_vatsimMetarReader->getName() + ": " + m_vatsimMetarReader->getReadLog().getSummary(); }
+        if (m_vatsimStatusReader)  { report << m_vatsimStatusReader->getName() + ": " + m_vatsimStatusReader->getReadLog().getSummary(); }
+        if (report.isEmpty()) { return db; }
+        return report.join(separator) + separator + db;
+    }
+
     CDistributorList CWebDataServices::getDistributors() const
     {
         if (m_modelDataReader) { return m_modelDataReader->getDistributors(); }
@@ -1165,7 +1189,7 @@ namespace BlackCore
                 if (!this->waitForDbInfoObjectsThenRead(entities)) { return; }
             }
 
-            const bool waitForSharedInfoFile = m_dbReaderConfig.needsSharedInfoFile(entities);
+            const bool waitForSharedInfoFile = m_dbReaderConfig.needsSharedInfoFile(entities) && !m_sharedInfoDataReader->areAllInfoObjectsRead();
             if (waitForSharedInfoFile)
             {
                 // do not read yet, will call this function again after some time
@@ -1181,67 +1205,68 @@ namespace BlackCore
     bool CWebDataServices::waitForDbInfoObjectsThenRead(CEntityFlags::Entity entities)
     {
         Q_ASSERT_X(m_dbInfoDataReader, Q_FUNC_INFO, "need reader");
+        if (m_dbInfoDataReader->areAllInfoObjectsRead()) { return true; }
         if (!m_dbInfoObjectTimeout.isValid()) { m_dbInfoObjectTimeout = QDateTime::currentDateTimeUtc().addMSecs(10 * 1000); }
         const bool read = this->waitForInfoObjectsThenRead(entities, "DB", m_dbInfoDataReader, m_dbInfoObjectTimeout);
-        if (read) { m_dbInfoObjectTimeout = QDateTime(); } // reset to null
         return read;
     }
 
     bool CWebDataServices::waitForSharedInfoObjectsThenRead(CEntityFlags::Entity entities)
     {
         Q_ASSERT_X(m_sharedInfoDataReader, Q_FUNC_INFO, "need reader");
+        if (m_sharedInfoDataReader->areAllInfoObjectsRead()) { return true; }
         if (!m_sharedInfoObjectsTimeout.isValid()) { m_sharedInfoObjectsTimeout = QDateTime::currentDateTimeUtc().addMSecs(10 * 1000); }
         const bool read = this->waitForInfoObjectsThenRead(entities, "shared", m_sharedInfoDataReader, m_sharedInfoObjectsTimeout);
-        if (read) { m_sharedInfoObjectsTimeout = QDateTime(); } // reset to null
         return read;
     }
 
-    bool CWebDataServices::waitForInfoObjectsThenRead(CEntityFlags::Entity entities, const QString &info, CInfoDataReader *reader, const QDateTime &timeOut)
+    bool CWebDataServices::waitForInfoObjectsThenRead(CEntityFlags::Entity entities, const QString &info, CInfoDataReader *infoReader, QDateTime &timeOut)
     {
-        Q_ASSERT_X(reader, Q_FUNC_INFO, "Need info data reader");
+        Q_ASSERT_X(infoReader, Q_FUNC_INFO, "Need info data reader");
 
         // this will called for each entity readers, i.e. model reader, ICAO reader ...
         const int waitForInfoObjectsMs = 1000; // ms
 
+        if (infoReader->areAllInfoObjectsRead())
+        {
+            // we have all data and carry on
+            CLogMessage(this).info("Info objects (%1) triggered for '%2' loaded from '%3'") << info << CEntityFlags::flagToString(entities) << infoReader->getInfoObjectsUrl().toQString();
+            timeOut = QDateTime(); // reset to null
+            return true; // no need to wait any longer
+        }
+
         // try to read if not timed out
-        if (QDateTime::currentDateTimeUtc() > timeOut)
+        if (timeOut.isValid() && QDateTime::currentDateTimeUtc() > timeOut)
         {
             const QString timeOutString = timeOut.toString();
             CLogMessage(this).warning("Could not read '%1' info objects for '%2' from '%3', time out '%4'. Marking reader '%5' as failed and continue.")
                     << info << CEntityFlags::flagToString(entities)
-                    << reader->getInfoObjectsUrl().toQString() << timeOutString
+                    << infoReader->getInfoObjectsUrl().toQString() << timeOutString
-                    << reader->getName();
+                    << infoReader->getName();
 
             // continue here and read data without info objects
-            reader->setMarkedAsFailed(true);
+            infoReader->setMarkedAsFailed(true);
+            // no timeout reset here
             return true; // carry on, regardless of situation
         }
-        else if (reader->hasReceivedFirstReply())
+
+        if (infoReader->hasReceivedFirstReply())
         {
-            if (reader->areAllInfoObjectsRead())
+            // we have received a response, but not all data yet
+            if (infoReader->hasReceivedOkReply())
             {
-                // we have all data and carry on
+                // ok, this means we are parsing
-                CLogMessage(this).info("Info objects (%1) for '%2' loaded from '%3'") << info << CEntityFlags::flagToString(entities) << reader->getInfoObjectsUrl().toQString();
+                this->readDeferredInBackground(entities, waitForInfoObjectsMs);
-                return true; // no need to wait any longer
+                CLogMessage(this).info("Waiting for objects (%1) for '%2' from '%3'") << info << CEntityFlags::flagToString(entities) << infoReader->getInfoObjectsUrl().toQString();
+                return false; // wait
             }
             else
             {
-                // we have received a response, but not all data yet
+                // we have a response, but a failure, means server is alive, but responded with error
-                if (reader->hasReceivedOkReply())
+                // such an error (access, ...) normally will not go away
-                {
+                CLogMessage(this).error("Info objects (%1) loading for '%2' failed from '%3', '%4'") << info << CEntityFlags::flagToString(entities) << infoReader->getInfoObjectsUrl().toQString() << infoReader->getStatusMessage();
-                    // ok, this means we are parsing
+                infoReader->setMarkedAsFailed(true);
-                    this->readDeferredInBackground(entities, waitForInfoObjectsMs);
+                return true; // carry on, regardless of situation
-                    CLogMessage(this).info("Waiting for objects (%1) for '%2' from '%3'") << info << CEntityFlags::flagToString(entities) << reader->getInfoObjectsUrl().toQString();
-                    return false; // wait
-                }
-                else
-                {
-                    // we have a response, but a failure, means server is alive, but responded with error
-                    // such an error (access, ...) normally will not go away
-                    CLogMessage(this).error("Info objects (%1) loading for '%2' failed from '%3', '%4'") << info << CEntityFlags::flagToString(entities) << reader->getInfoObjectsUrl().toQString() << reader->getStatusMessage();
-                    reader->setMarkedAsFailed(true);
-                    return true; // carry on, regardless of situation
-                }
             }
         }
         else

src/blackcore/webdataservices.h

@@ -395,6 +395,12 @@ namespace BlackCore
         //! \threadsafe
         int getSharedInfoObjectCount(BlackMisc::Network::CEntityFlags::Entity entity) const;
 
+        //! For all available DB readers the log info is generated
+        QString getDbReadersLog(const QString separator = "\n") const;
+
+        //! For all available readers the log info is generated
+        QString getReadersLog(const QString separator = "\n") const;
+
         //! Has already connect swift DB?
         bool hasConnectedSwiftDb() const;
 
@@ -513,7 +519,7 @@ namespace BlackCore
 
         //! Wait for info objects to be read
         //! \return true means info objects available
-        bool waitForInfoObjectsThenRead(BlackMisc::Network::CEntityFlags::Entity entities, const QString &info, BlackCore::Db::CInfoDataReader *reader, const QDateTime &timeOut);
+        bool waitForInfoObjectsThenRead(BlackMisc::Network::CEntityFlags::Entity entities, const QString &info, BlackCore::Db::CInfoDataReader *infoReader, QDateTime &timeOut);
 
         CWebReaderFlags::WebReader               m_readers = CWebReaderFlags::WebReaderFlag::None; //!< which readers are available
         BlackMisc::Network::CEntityFlags::Entity m_entitiesPeriodicallyRead = BlackMisc::Network::CEntityFlags::NoEntity; //!< entities permanently updated by timers

src/blackgui/guiapplication.cpp

@@ -10,6 +10,7 @@
 #include "blackconfig/buildconfig.h"
 #include "blackcore/context/contextnetwork.h"
 #include "blackcore/data/globalsetup.h"
+#include "blackcore/webdataservices.h"
 #include "blackgui/components/applicationclosedialog.h"
 #include "blackgui/components/downloadandinstalldialog.h"
 #include "blackgui/components/aboutdialog.h"
@@ -56,6 +57,7 @@ using namespace BlackMisc;
 using namespace BlackMisc::Db;
 using namespace BlackMisc::Network;
 using namespace BlackGui::Components;
+using namespace BlackCore;
 using namespace BlackCore::Data;
 
 BlackGui::CGuiApplication *sGui = nullptr; // set by constructor
@@ -457,6 +459,16 @@ namespace BlackGui
         });
         Q_ASSERT_X(c, Q_FUNC_INFO, "Connect failed");
 
+        if (this->hasWebDataServices())
+        {
+            a = menu.addAction("Services log");
+            c = connect(a, &QAction::triggered, this, [a, this]()
+            {
+                this->displayTextInConsole(this->getWebDataServices()->getReadersLog());
+            });
+            Q_ASSERT_X(c, Q_FUNC_INFO, "Connect failed");
+        }
+
         a = menu.addAction("Metadata (slow)");
         c = connect(a, &QAction::triggered, this, [a, this]()
         {