From 0939b1f08f0cdbe0726c87060bef4619725005f8 Mon Sep 17 00:00:00 2001 From: Klaus Basan Date: Sun, 26 Jun 2016 00:13:02 +0200 Subject: [PATCH] refs #687, further performance improvements * avoid redundant reads by excluding entities which will be periodically updated * restart timers when data are received, avoid overlapping requests --- src/blackcore/threadedreader.cpp | 15 +++++++++++++++ src/blackcore/threadedreader.h | 10 +++++++++- src/blackcore/vatsim/vatsimbookingreader.cpp | 6 ++++-- src/blackcore/vatsim/vatsimdatafilereader.cpp | 6 +++--- src/blackcore/vatsim/vatsimmetarreader.cpp | 6 ++++-- src/blackcore/webdataservices.cpp | 14 +++++++++----- src/blackcore/webdataservices.h | 1 + 7 files changed, 45 insertions(+), 13 deletions(-) diff --git a/src/blackcore/threadedreader.cpp b/src/blackcore/threadedreader.cpp index 2d43278ec..556b3abb6 100644 --- a/src/blackcore/threadedreader.cpp +++ b/src/blackcore/threadedreader.cpp @@ -98,6 +98,15 @@ namespace BlackCore }); } + void CThreadedReader::restartTimer(bool onlyWhenActive) + { + const int intervalMs(this->interval()); + if (!onlyWhenActive || this->isTimerActive()) + { + this->setInterval(intervalMs); + } + } + bool CThreadedReader::didContentChange(const QString &content, int startPosition) { uint oldHash = 0; @@ -126,6 +135,12 @@ namespace BlackCore return this->m_updateTimer->interval(); } + bool CThreadedReader::isTimerActive() const + { + QReadLocker rl(&this->m_lock); + return this->m_updateTimer->isActive(); + } + void CThreadedReader::setIntervalFromSettingsAndStart() { this->setInitialTime(); diff --git a/src/blackcore/threadedreader.h b/src/blackcore/threadedreader.h index 671ef8829..790e8839c 100644 --- a/src/blackcore/threadedreader.h +++ b/src/blackcore/threadedreader.h @@ -59,6 +59,10 @@ namespace BlackCore //! \threadsafe int interval() const; + //! Is timer running + //! \threadsafe + bool isTimerActive() const; + //! Set inverval from settings and start void setIntervalFromSettingsAndStart(); @@ -68,7 +72,7 @@ namespace BlackCore void gracefulShutdown(); protected: - QTimer *m_updateTimer = nullptr; //!< update timer + QTimer *m_updateTimer = nullptr; //!< update timer mutable QReadWriteLock m_lock {QReadWriteLock::Recursive}; //!< lock which can be used from the derived classes //! Constructor @@ -94,6 +98,10 @@ namespace BlackCore //! \threadsafe void setInterval(int updatePeriodMs); + //! Restart timer + //! \threadsafe + void restartTimer(bool onlyWhenActive = false); + //! Stores new content hash and returns if content changed (based on hash value //! \threadsafe bool didContentChange(const QString &content, int startPosition = -1); diff --git a/src/blackcore/vatsim/vatsimbookingreader.cpp b/src/blackcore/vatsim/vatsimbookingreader.cpp index 679d4e90c..6585cea93 100644 --- a/src/blackcore/vatsim/vatsimbookingreader.cpp +++ b/src/blackcore/vatsim/vatsimbookingreader.cpp @@ -68,10 +68,11 @@ namespace BlackCore void CVatsimBookingReader::ps_read() { this->threadAssertCheck(); + this->restartTimer(true); // when timer active, restart so we cause no undesired reads + Q_ASSERT_X(sApp, Q_FUNC_INFO, "No application"); const QUrl url(sApp->getGlobalSetup().getVatsimBookingsUrl()); if (url.isEmpty()) { return; } - sApp->getFromNetwork(url, { this, &CVatsimBookingReader::ps_parseBookings}); } @@ -114,10 +115,11 @@ namespace BlackCore if (this->getUpdateTimestamp() == updateTimestamp) return; // nothing to do // save parsing and all follow up actions if nothing changed + this->restartTimer(); // do not consider time for reading bool changed = this->didContentChange(xmlData, xmlData.indexOf("")); if (!changed) { - CLogMessage(this).info("Bookings unchanged, skipped"); + CLogMessage(this).info("Read bookings unchanged, skipped"); return; // stop, terminate straight away, ending thread } } diff --git a/src/blackcore/vatsim/vatsimdatafilereader.cpp b/src/blackcore/vatsim/vatsimdatafilereader.cpp index cb5bd4517..c4fe7be16 100644 --- a/src/blackcore/vatsim/vatsimdatafilereader.cpp +++ b/src/blackcore/vatsim/vatsimdatafilereader.cpp @@ -187,6 +187,7 @@ namespace BlackCore void CVatsimDataFileReader::ps_read() { this->threadAssertCheck(); + this->restartTimer(true); // when timer active, restart so we cause no undesired reads // round robin for load balancing // remark: Don't use QThread to run network operations in the background @@ -196,7 +197,6 @@ namespace BlackCore const QUrl url(urls.obtainNextWorkingUrl(true)); if (url.isEmpty()) { return; } sApp->getFromNetwork(url, { this, &CVatsimDataFileReader::ps_parseVatsimFile}); - } void CVatsimDataFileReader::ps_parseVatsimFile(QNetworkReply *nwReplyPtr) @@ -222,8 +222,8 @@ namespace BlackCore nwReply->close(); // close asap if (dataFileData.isEmpty()) { return; } - // Quick check by hash - if (!this->didContentChange(dataFileData)) + this->restartTimer(); // do not consider time for reading + if (!this->didContentChange(dataFileData)) // Quick check by hash { CLogMessage(this).info("VATSIM file has same content, skipped"); return; diff --git a/src/blackcore/vatsim/vatsimmetarreader.cpp b/src/blackcore/vatsim/vatsimmetarreader.cpp index d579cfd05..6e3b3d280 100644 --- a/src/blackcore/vatsim/vatsimmetarreader.cpp +++ b/src/blackcore/vatsim/vatsimmetarreader.cpp @@ -82,6 +82,8 @@ namespace BlackCore void CVatsimMetarReader::ps_readMetars() { this->threadAssertCheck(); + this->restartTimer(true); // when timer active, restart so we cause no undesired reads + CFailoverUrlList urls(sApp->getVatsimMetarUrls()); const CUrl url(urls.obtainNextWorkingUrl(true)); if (url.isEmpty()) { return; } @@ -110,8 +112,8 @@ namespace BlackCore QString metarData = nwReply->readAll(); nwReply->close(); // close asap - // Quick check by hash - if (!this->didContentChange(metarData)) + this->restartTimer(); // do not consider time for reading + if (!this->didContentChange(metarData)) // Quick check by hash { CLogMessage(this).info("METAR file has same content, skipped"); return; diff --git a/src/blackcore/webdataservices.cpp b/src/blackcore/webdataservices.cpp index e88655b0f..19e273821 100644 --- a/src/blackcore/webdataservices.cpp +++ b/src/blackcore/webdataservices.cpp @@ -61,7 +61,7 @@ namespace BlackCore const bool readFromSwiftDb = dbReaderConfig.possiblyReadsFromSwiftDb(); // only cached? if (!readFromSwiftDb && readers.testFlag(CWebReaderFlags::InfoDataReader)) { - // will remove info reader becaue not needed + // will remove info reader because not needed readers &= ~CWebReaderFlags::InfoDataReader; this->m_readers = readers; CLogMessage(this).info("Remove info object reader because not needed"); @@ -72,7 +72,7 @@ namespace BlackCore if (entities.testFlag(CEntityFlags::InfoObjectEntity)) { Q_ASSERT_X(readers.testFlag(CWebReaderFlags::InfoDataReader), Q_FUNC_INFO, "info object but no reader"); - CLogMessage(this).info("Using info objects for swift DB objects"); + CLogMessage(this).info("Using info objects for swift DB entities"); } this->initReaders(readers); // reads info object if required @@ -80,7 +80,9 @@ namespace BlackCore // make sure this is called in event queue, so pending tasks cam be performed // important so info objects can be read - entities &= ~CEntityFlags::InfoObjectEntity; + entities &= ~CEntityFlags::InfoObjectEntity; // triggered in init readers + entities &= ~CEntityFlags::VatsimStatusFile; // triggered in init readers + entities &= ~this->m_entitiesPeriodicallyRead; // will be triggered by timers this->singleShotReadInBackground(entities, 1000); } @@ -440,10 +442,9 @@ namespace BlackCore Q_ASSERT_X(c, Q_FUNC_INFO, "ICAO info object signals"); c = connect(this->m_infoDataReader, &CInfoDataReader::dataRead, this, &CWebDataServices::dataRead); Q_ASSERT_X(c, Q_FUNC_INFO, "connect failed info data"); + this->m_infoDataReader->start(QThread::LowPriority); // info data reader has a special role, it will not be triggered in triggerRead() - this->m_infoDataReader->start(QThread::LowPriority); - // directly call read QTimer::singleShot(0, [this]() { this->m_infoDataReader->read(CEntityFlags::InfoObjectEntity, QDateTime()); }); } @@ -466,6 +467,7 @@ namespace BlackCore Q_ASSERT_X(c, Q_FUNC_INFO, "VATSIM booking reader signals"); c = connect(this->m_vatsimBookingReader, &CVatsimBookingReader::dataRead, this, &CWebDataServices::dataRead); Q_ASSERT_X(c, Q_FUNC_INFO, "connect failed bookings"); + this->m_entitiesPeriodicallyRead |= CEntityFlags::BookingEntity; this->m_vatsimBookingReader->start(QThread::LowPriority); this->m_vatsimBookingReader->setIntervalFromSettingsAndStart(); } @@ -478,6 +480,7 @@ namespace BlackCore Q_ASSERT_X(c, Q_FUNC_INFO, "VATSIM data reader signals"); c = connect(this->m_vatsimDataFileReader, &CVatsimDataFileReader::dataRead, this, &CWebDataServices::dataRead); Q_ASSERT_X(c, Q_FUNC_INFO, "connect failed VATSIM data file"); + this->m_entitiesPeriodicallyRead |= CEntityFlags::VatsimDataFile; this->m_vatsimDataFileReader->start(QThread::LowPriority); this->m_vatsimDataFileReader->setIntervalFromSettingsAndStart(); } @@ -490,6 +493,7 @@ namespace BlackCore Q_ASSERT_X(c, Q_FUNC_INFO, "VATSIM METAR reader signals"); c = connect(this->m_vatsimMetarReader, &CVatsimMetarReader::dataRead, this, &CWebDataServices::dataRead); Q_ASSERT_X(c, Q_FUNC_INFO, "connect failed VATSIM METAR"); + this->m_entitiesPeriodicallyRead |= CEntityFlags::MetarEntity; this->m_vatsimMetarReader->start(QThread::LowPriority); this->m_vatsimMetarReader->setIntervalFromSettingsAndStart(); } diff --git a/src/blackcore/webdataservices.h b/src/blackcore/webdataservices.h index 84e39ed76..2aa93731a 100644 --- a/src/blackcore/webdataservices.h +++ b/src/blackcore/webdataservices.h @@ -325,6 +325,7 @@ namespace BlackCore CWebReaderFlags::WebReader m_readers = CWebReaderFlags::WebReaderFlag::None; //!< which readers are available BlackCore::Db::CDatabaseReaderConfigList m_dbReaderConfig; //!< how to read DB data + BlackMisc::Network::CEntityFlags::Entity m_entitiesPeriodicallyRead = BlackMisc::Network::CEntityFlags::NoEntity; //!< those entities which are permanently updated by timers bool m_initialRead = false; //!< Initial read started int m_infoObjectTrials = 0; //!< Tried to read info objects