From 0cf308450e6ec44c3b388ada93e18373af36218f Mon Sep 17 00:00:00 2001
From: Klaus Basan <klaus.basan@klausbasan.de>
Date: Sun, 17 Sep 2017 23:35:50 +0200
Subject: [PATCH] Ref T149, utility functions for pending URLs

* Do not call read twice in info reader
* Threadsafe utility functions
---
 src/blackcore/db/infodatareader.cpp  |  8 ++++++++
 src/blackcore/threadedreader.cpp     | 21 +++++++++++++++++++++
 src/blackcore/threadedreader.h       | 12 ++++++++++++
 src/blackmisc/network/urlloglist.cpp |  6 ++++++
 src/blackmisc/network/urlloglist.h   |  3 +++
 5 files changed, 50 insertions(+)

diff --git a/src/blackcore/db/infodatareader.cpp b/src/blackcore/db/infodatareader.cpp
index cfdc013c4..32dfab1fa 100644
--- a/src/blackcore/db/infodatareader.cpp
+++ b/src/blackcore/db/infodatareader.cpp
@@ -113,7 +113,15 @@ namespace BlackCore
         void CInfoDataReader::read()
         {
             if (!this->doWorkCheck()) { return; }
+
             const CUrl url(this->getInfoObjectsUrl());
+            const CUrlLogList urlLogList(this->getUrlLogList()); // thread safe copy
+
+            if (urlLogList.hasPending())
+            {
+                CLogMessage(this).info("Info data reading still pending, summary: '%1'") << urlLogList.getSummary();
+                return;
+            }
             if (!url.isEmpty())
             {
                 this->getFromNetworkAndLog(url, { this, &CInfoDataReader::parseInfoObjectsData});
diff --git a/src/blackcore/threadedreader.cpp b/src/blackcore/threadedreader.cpp
index 1399429dc..d95b23d24 100644
--- a/src/blackcore/threadedreader.cpp
+++ b/src/blackcore/threadedreader.cpp
@@ -89,6 +89,18 @@ namespace BlackCore
         QTimer::singleShot(0, &m_updateTimer, &QTimer::stop);
     }
 
+    bool CThreadedReader::hasPendingUrls() const
+    {
+        QReadLocker l(&m_lock);
+        return m_urlReadLog.hasPending();
+    }
+
+    CUrlLogList CThreadedReader::getUrlLogList() const
+    {
+        QReadLocker l(&m_lock);
+        return m_urlReadLog;
+    }
+
     bool CThreadedReader::didContentChange(const QString &content, int startPosition)
     {
         uint oldHash = 0;
@@ -162,7 +174,16 @@ namespace BlackCore
     QNetworkReply *CThreadedReader::getFromNetworkAndLog(const CUrl &url, const BlackMisc::CSlot<void (QNetworkReply *)> &callback)
     {
         // returned QNetworkReply normally nullptr since QAM is in different thread
+        QWriteLocker wl(&m_lock);
+        const CUrlLogList outdatedPendingUrls = m_urlReadLog.findOutdatedPending(OutdatedPendingCallMs);
+        if (!outdatedPendingUrls.isEmpty())
+        {
+            CLogMessage(this).warning("Detected outdated pending calls: '%1'") << outdatedPendingUrls.toQString(true);
+            m_urlReadLog.removeOlderThanNowMinusOffset(OutdatedPendingCallMs); // clean up
+        }
+
         const int id = m_urlReadLog.addPendingUrl(url);
+        wl.unlock();
         return sApp->getFromNetwork(url, id, callback);
     }
 
diff --git a/src/blackcore/threadedreader.h b/src/blackcore/threadedreader.h
index fa1c2592a..f78d78a52 100644
--- a/src/blackcore/threadedreader.h
+++ b/src/blackcore/threadedreader.h
@@ -83,6 +83,14 @@ namespace BlackCore
         //! \remark needs to be done before started in different thread
         void markAsUsedInUnitTest() { m_unitTest = true; }
 
+        //! Has pending URLs?
+        //! \threadsafe
+        bool hasPendingUrls() const;
+
+        //! Get the URL log list
+        //! \threadsafe
+        BlackMisc::Network::CUrlLogList getUrlLogList() const;
+
     protected:
         mutable QReadWriteLock m_lock { QReadWriteLock::Recursive }; //!< lock which can be used from the derived classes
 
@@ -106,9 +114,11 @@ namespace BlackCore
         virtual void doWorkImpl() {}
 
         //! Still enabled etc.?
+        //! \threadsafe under normal conditions
         bool doWorkCheck() const;
 
         //! Get request from network, and log with m_urlReadLog
+        //! \threadsafe read log access is thread safe
         QNetworkReply *getFromNetworkAndLog(const BlackMisc::Network::CUrl &url, const BlackMisc::CSlot<void(QNetworkReply *)> &callback);
 
         //! Network reply received, mark in m_urlReadLog
@@ -123,6 +133,8 @@ namespace BlackCore
         //! Trigger doWorkImpl
         void doWork();
 
+        static constexpr int OutdatedPendingCallMs = 30 * 1000; //!< when is a call considered "outdated"
+
         int               m_initialTime = -1;         //!< Initial start delay
         int               m_periodicTime = -1;        //!< Periodic time after which the task is repeated
         QDateTime         m_updateTimestamp;          //!< when file/resource was read
diff --git a/src/blackmisc/network/urlloglist.cpp b/src/blackmisc/network/urlloglist.cpp
index f152a88a9..4e280d3e5 100644
--- a/src/blackmisc/network/urlloglist.cpp
+++ b/src/blackmisc/network/urlloglist.cpp
@@ -41,6 +41,12 @@ namespace BlackMisc
             return this->findBy(&CUrlLog::isPending, true);
         }
 
+        CUrlLogList CUrlLogList::findOutdatedPending(int outdatedOffsetMs) const
+        {
+            if (this->isEmpty()) { return CUrlLogList(); }
+            return this->findPending().findBeforeNowMinusOffset(outdatedOffsetMs);
+        }
+
         CUrlLogList CUrlLogList::findErrors() const
         {
             return this->findBy(&CUrlLog::isPending, false, &CUrlLog::isSuccess, false);
diff --git a/src/blackmisc/network/urlloglist.h b/src/blackmisc/network/urlloglist.h
index b064bacd4..b8c132db7 100644
--- a/src/blackmisc/network/urlloglist.h
+++ b/src/blackmisc/network/urlloglist.h
@@ -48,6 +48,9 @@ namespace BlackMisc
             //! Find pending log entries
             CUrlLogList findPending() const;
 
+            //! Find outdated pending log entries
+            CUrlLogList findOutdatedPending(int outdatedOffsetMs) const;
+
             //! Find log entries with errors (not pending)
             CUrlLogList findErrors() const;