refs #597, use CMD arguments in setup reader.

* Setup reader will now be initialized and handles by CApplication
* Setup reader no longer threaded reader as it will be initialized and loade upfront
This commit is contained in:
Klaus Basan
2016-02-18 19:56:35 +01:00
committed by Mathew Sutcliffe
parent 6a06aa0460
commit 9ff322ae25
8 changed files with 281 additions and 136 deletions

View File

@@ -122,6 +122,11 @@ namespace BlackCore
QString s("timestamp: ");
s.append(this->getFormattedUtcTimestampYmdhms());
s.append(separator);
s.append("Loaded: ");
s.append(boolToYesNo(this->wasLoaded()));
s.append(separator);
s.append("For development: ");
s.append(boolToYesNo(isDevelopment()));
s.append(separator);
@@ -200,6 +205,8 @@ namespace BlackCore
return CVariant::fromValue(this->swiftDbDataFileLocationUrls());
case IndexShared:
return CVariant::fromValue(this->m_sharedUrls);
case IndexWasLoaded:
return CVariant::fromValue(this->m_wasLoaded);
default:
return CValueObject::propertyByIndex(index);
}
@@ -240,6 +247,9 @@ namespace BlackCore
case IndexShared:
this->m_sharedUrls = variant.value<CUrlList>();
break;
case IndexWasLoaded:
this->m_wasLoaded = variant.toBool();
break;
default:
CValueObject::setPropertyByIndex(variant, index);
break;

View File

@@ -43,6 +43,7 @@ namespace BlackCore
IndexSwiftDbFiles,
IndexBootstrap,
IndexUpdateInfo,
IndexWasLoaded,
IndexShared
};
@@ -52,6 +53,12 @@ namespace BlackCore
//! Destructor.
~CGlobalSetup() {}
//! Has data loaded from web
bool wasLoaded() const { return m_wasLoaded; }
//! Mark as loaded
void markAsLoaded(bool loaded) { this->m_wasLoaded = loaded; }
//! Http port
int dbHttpPort() const { return m_dbHttpPort; }
@@ -133,6 +140,7 @@ namespace BlackCore
private:
BLACK_ENABLE_TUPLE_CONVERSION(BlackCore::Data::CGlobalSetup)
bool m_wasLoaded = false; //!< Loaded from web
int m_dbHttpPort = 80; //!< port
int m_dbHttpsPort = 443; //!< SSL port
bool m_development = false; //!< dev. version?
@@ -167,6 +175,7 @@ namespace BlackCore
Q_DECLARE_METATYPE(BlackCore::Data::CGlobalSetup)
BLACK_DECLARE_TUPLE_CONVERSION(BlackCore::Data::CGlobalSetup, (
attr(o.m_wasLoaded),
attr(o.m_timestampMSecsSinceEpoch),
attr(o.m_dbRootDirectoryUrl),
attr(o.m_dbHttpPort),

View File

@@ -30,6 +30,5 @@ namespace BlackCore
BlackCore::Data::CGlobalSetup::registerMetadata();
BlackCore::Data::CUpdateInfo::registerMetadata();
BlackCore::CSetupReader::instance(); //! \todo will go into new runtime
}
} // namespace

View File

@@ -7,9 +7,11 @@
* contained in the LICENSE file.
*/
#include "blackcore/application.h"
#include "blackmisc/network/networkutils.h"
#include "blackmisc/sequence.h"
#include "blackmisc/logmessage.h"
#include "blackmisc/logcategory.h"
#include "blackmisc/json.h"
#include "blackmisc/project.h"
#include "blackmisc/fileutils.h"
@@ -26,93 +28,145 @@ using namespace BlackCore::Data;
namespace BlackCore
{
CSetupReader::CSetupReader(QObject *owner) :
CThreadedReader(owner, "CSetupReader")
CSetupReader::CSetupReader(QObject *parent) :
QObject(parent)
{
connect(this, &CSetupReader::setupSynchronized, this, &CSetupReader::ps_setupSyncronized);
QString localFileName;
if (this->localBootstrapFile(localFileName))
connect(this, &CSetupReader::updateInfoSynchronized, this, &CSetupReader::ps_versionInfoSyncronized);
}
QList<QCommandLineOption> CSetupReader::getCmdLineOptions() const
{
return QList<QCommandLineOption>
{
{
this->m_cmdBootstrapUrl,
this->m_cmdBootstrapMode
}
};
}
CStatusMessage CSetupReader::asyncLoad()
{
if (this->readLocalBootstrapFile(this->m_localSetupFileValue))
{
// initialized by local file for testing
// I do not even need to start in background here
CLogMessage(this).info("Using local bootstrap file: %1") << localFileName;
BlackMisc::singleShot(1000, QThread::currentThread(), [ = ]()
{
emit this->setupSynchronized(true);
});
emit this->setupSynchronized(true);
return CStatusMessage(cats(), CStatusMessage::SeverityInfo, "Using local bootstrap file: " + this->m_localSetupFileValue);
}
else if (this->m_bootstrapMode == CacheOnly)
{
m_setup.synchronize();
CGlobalSetup currentSetup = m_setup.get();
this->m_updateInfoUrls = currentSetup.updateInfoFileUrls();
emit this->setupSynchronized(true);
emit this->updateInfoSynchronized(true);
return CStatusMessage(cats(), CStatusMessage::SeverityInfo, "Cache only setup, using it as it is");
}
else
{
this->m_bootstrapUrls.uniqueWrite()->push_back(m_setup.get().bootstrapUrls());
this->m_updateInfoUrls.uniqueWrite()->push_back(m_setup.get().updateInfoUrls());
// web URL
if (!this->m_bootsrapUrlFileValue.isEmpty())
{
// start with the one from cmd args
this->m_bootstrapUrls.push_front(CUrl(this->m_bootsrapUrlFileValue));
}
this->m_networkManagerBootstrap = new QNetworkAccessManager(this);
this->connect(this->m_networkManagerBootstrap, &QNetworkAccessManager::finished, this, &CSetupReader::ps_parseSetupFile);
// if ever loaded add those URLs
m_setup.synchronize();
CGlobalSetup currentSetup = m_setup.get();
if (currentSetup.wasLoaded())
{
if (this->m_bootstrapMode != Explicit || this->m_bootstrapUrls.isEmpty())
{
// also use previously cached URLS
this->m_bootstrapUrls.push_back(currentSetup.bootstrapFileUrls());
}
}
this->m_networkManagerUpdateInfo = new QNetworkAccessManager(this);
this->connect(this->m_networkManagerUpdateInfo, &QNetworkAccessManager::finished, this, &CSetupReader::ps_parseUpdateInfoFile);
this->start(QThread::LowPriority);
if (this->m_bootstrapUrls.isEmpty())
{
return CStatusMessage(cats(), CStatusMessage::SeverityError, "No bootstrap URLs, cannot load setup");
}
else
{
this->m_bootstrapUrls.removeDuplicates();
this->ps_readSetup(); // start reading
return CStatusMessage(cats(), CStatusMessage::SeverityInfo, "Will start loading setup");
}
}
}
CSetupReader &CSetupReader::instance()
bool CSetupReader::parseCmdLineArguments()
{
static CSetupReader reader(QCoreApplication::instance());
return reader;
this->m_bootsrapUrlFileValue = CGlobalSetup::buildBootstrapFileUrl(
sApp->getParserOptionValue(this->m_cmdBootstrapUrl)
);
this->m_bootstrapMode = stringToEnum(sApp->getParserOptionValue(this->m_cmdBootstrapMode));
QUrl url(this->m_bootsrapUrlFileValue);
// check on local file
if (url.isLocalFile())
{
this->m_localSetupFileValue = url.toLocalFile();
QFile f(this->m_localSetupFileValue);
if (!f.exists())
{
sApp->errorMessage("File " + this->m_localSetupFileValue + " does not exist");
return false;
}
}
// check on explicit URL
if (this->m_bootstrapMode == Explicit)
{
if (!url.isLocalFile())
{
if (!CNetworkUtils::canConnect(url))
{
sApp->errorMessage("URL " + url.toString() + " not reachable");
return false;
}
}
}
return true;
}
void CSetupReader::initialize()
void CSetupReader::gracefulShutdown()
{
// start to read by myself
CThreadedReader::initialize();
QTimer::singleShot(500, this, &CSetupReader::ps_readSetup);
}
void CSetupReader::cleanup()
{
delete this->m_networkManagerBootstrap;
this->m_networkManagerBootstrap = nullptr;
delete this->m_networkManagerUpdateInfo;
this->m_networkManagerUpdateInfo = nullptr;
this->m_shutdown = true;
}
void CSetupReader::ps_readSetup()
{
this->threadAssertCheck();
Q_ASSERT_X(this->m_networkManagerBootstrap, Q_FUNC_INFO, "Missing network manager");
CUrl url(this->m_bootstrapUrls.uniqueWrite()->getNextWorkingUrl());
CUrl url(this->m_bootstrapUrls.getNextWorkingUrl());
if (url.isEmpty())
{
CLogMessage(this).warning("Cannot read setup, failed URLs: %1") << this->m_bootstrapUrls.read()->getFailedUrls();
CLogMessage(this).warning("Cannot read setup, failed URLs: %1") << this->m_bootstrapUrls.getFailedUrls();
emit setupSynchronized(false);
return;
}
QNetworkRequest request(url);
CNetworkUtils::ignoreSslVerification(request);
this->m_networkManagerBootstrap->get(request);
if (m_shutdown) { return; }
sApp->requestNetworkResource(url.toNetworkRequest(), { this, &CSetupReader::ps_parseSetupFile });
}
void CSetupReader::ps_readUpdateInfo()
{
this->threadAssertCheck();
Q_ASSERT_X(this->m_networkManagerUpdateInfo, Q_FUNC_INFO, "Missing network manager");
CUrl url(this->m_updateInfoUrls.uniqueWrite()->getNextWorkingUrl());
CUrl url(this->m_updateInfoUrls.getNextWorkingUrl());
if (url.isEmpty())
{
CLogMessage(this).warning("Cannot read update info, failed URLs: %1") << this->m_updateInfoUrls.read()->getFailedUrls();
emit versionSynchronized(false);
CLogMessage(this).warning("Cannot read update info, failed URLs: %1") << this->m_updateInfoUrls.getFailedUrls();
emit updateInfoSynchronized(false);
return;
}
QNetworkRequest request(url);
CNetworkUtils::ignoreSslVerification(request);
this->m_networkManagerUpdateInfo->get(request);
if (m_shutdown) { return; }
sApp->requestNetworkResource(url.toNetworkRequest(), { this, &CSetupReader::ps_parseUpdateInfoFile});
}
void CSetupReader::ps_setupSyncronized(bool success)
{
// trigger
// trigger consecutive read
this->m_setupSyncronized = success;
if (success)
{
CLogMessage(this).info("Setup synchronized, will trigger read of update information");
@@ -124,19 +178,44 @@ namespace BlackCore
}
}
void CSetupReader::ps_versionInfoSyncronized(bool success)
{
this->m_updateInfoSyncronized = success;
}
void CSetupReader::ps_setupChanged()
{
// settings have changed on disk
}
bool CSetupReader::localBootstrapFile(QString &fileName)
CSetupReader::BootsrapMode CSetupReader::stringToEnum(const QString &s)
{
QString dir(CProject::getSwiftPrivateResourceDir());
if (dir.isEmpty()) { return false; }
const QString bsm(s.toLower().trimmed());
if (bsm.startsWith("expl")) return Explicit;
if (bsm.startsWith("cache")) return CacheOnly;
return Default;
}
// no version for local files, as those come withe the current code
fileName = CFileUtils::appendFilePaths(dir, "bootstrap/bootstrap.json");
QString content(CFileUtils::readFileToString(fileName));
bool CSetupReader::readLocalBootstrapFile(QString &fileName)
{
if (fileName.isEmpty()) { return false; }
QString fn;
QFile file(fileName);
if (!file.exists())
{
// relative name?
QString dir(CProject::getSwiftPrivateResourceDir());
if (dir.isEmpty()) { return false; }
// no version for local files, as those come withe the current code
fn = CFileUtils::appendFilePaths(dir, "bootstrap/bootstrap.json");
}
else
{
fn = fileName;
}
QString content(CFileUtils::readFileToString(fn));
if (content.isEmpty()) { return false; }
CGlobalSetup s;
s.convertFromJson(content);
@@ -155,22 +234,15 @@ namespace BlackCore
// wrap pointer, make sure any exit cleans up reply
// required to use delete later as object is created in a different thread
QScopedPointer<QNetworkReply, QScopedPointerDeleteLater> nwReply(nwReplyPtr);
this->threadAssertCheck();
if (m_shutdown) { return; }
QUrl url(nwReply->url());
QString urlString(url.toString());
QString replyMessage(nwReply->errorString());
if (this->isAbandoned())
{
CLogMessage(this).info("Terminated loading bootstrap files");
nwReply->abort();
emit setupSynchronized(false);
return; // stop, terminate straight away, ending thread
}
if (nwReply->error() == QNetworkReply::NoError)
{
qint64 lastModified = this->lastModifiedMsSinceEpoch(nwReply.data());
qint64 lastModified = CNetworkUtils::lastModifiedMsSinceEpoch(nwReply.data());
QString setupJson(nwReplyPtr->readAll());
nwReplyPtr->close();
if (setupJson.isEmpty())
@@ -180,13 +252,11 @@ namespace BlackCore
}
else
{
CGlobalSetup currentSetup(m_setup.get()); // from cache
CGlobalSetup currentSetup = m_setup.get();
CGlobalSetup loadedSetup;
loadedSetup.convertFromJson(Json::jsonObjectFromString(setupJson));
loadedSetup.setDevelopment(isForDevelopment()); // always update, regardless what persistent setting says
if (loadedSetup.getMSecsSinceEpoch() == 0 && lastModified > 0) { loadedSetup.setMSecsSinceEpoch(lastModified); }
qint64 currentVersionTimestamp = currentSetup.getMSecsSinceEpoch();
qint64 newVersionTimestamp = loadedSetup.getMSecsSinceEpoch();
loadedSetup.markAsLoaded(true);
if (lastModified > 0 && lastModified > loadedSetup.getMSecsSinceEpoch()) { loadedSetup.setMSecsSinceEpoch(lastModified); }
bool sameVersionLoaded = (loadedSetup == currentSetup);
if (sameVersionLoaded)
{
@@ -195,24 +265,27 @@ namespace BlackCore
return; // success
}
bool sameType = loadedSetup.hasSameType(currentSetup);
bool outdatedVersionLoaded = sameType && (newVersionTimestamp < currentVersionTimestamp);
if (outdatedVersionLoaded)
qint64 currentVersionTimestamp = currentSetup.getMSecsSinceEpoch();
qint64 newVersionTimestamp = loadedSetup.getMSecsSinceEpoch();
bool outdatedVersionLoaded = (newVersionTimestamp < currentVersionTimestamp);
if (this->m_bootstrapMode != Explicit && outdatedVersionLoaded)
{
CLogMessage(this).info("Setup loaded from %1 outdated, older than version in data cache %2") << urlString << m_setup.getFilename();
// try next URL
}
else
{
CStatusMessage m = m_setup.set(loadedSetup);
if (!m.isEmpty())
CStatusMessage m = m_setup.set(loadedSetup, loadedSetup.getMSecsSinceEpoch());
if (m.isWarningOrAbove())
{
m.setCategories(cats());
CLogMessage(this).preformatted(m);
emit setupSynchronized(false);
return; // issue with cache
}
else
{
this->m_updateInfoUrls = loadedSetup.updateInfoFileUrls();
CLogMessage(this).info("Setup: Updated data cache in %1") << this->m_setup.getFilename();
emit setupSynchronized(true);
return; // success
@@ -229,7 +302,7 @@ namespace BlackCore
}
// try next one if any
if (this->m_bootstrapUrls.uniqueWrite()->addFailedUrl(url))
if (this->m_bootstrapUrls.addFailedUrl(url))
{
QTimer::singleShot(500, this, &CSetupReader::ps_readSetup);
}
@@ -244,22 +317,15 @@ namespace BlackCore
// wrap pointer, make sure any exit cleans up reply
// required to use delete later as object is created in a different thread
QScopedPointer<QNetworkReply, QScopedPointerDeleteLater> nwReply(nwReplyPtr);
this->threadAssertCheck();
if (m_shutdown) { return; }
QUrl url(nwReply->url());
QString urlString(url.toString());
QString replyMessage(nwReply->errorString());
if (this->isAbandoned())
{
CLogMessage(this).info("Terminated loading of update info");
nwReply->abort();
emit versionSynchronized(false);
return; // stop, terminate straight away, ending thread
}
if (nwReply->error() == QNetworkReply::NoError)
{
qint64 lastModified = this->lastModifiedMsSinceEpoch(nwReply.data());
qint64 lastModified = CNetworkUtils::lastModifiedMsSinceEpoch(nwReply.data());
QString setupJson(nwReplyPtr->readAll());
nwReplyPtr->close();
if (setupJson.isEmpty())
@@ -273,20 +339,18 @@ namespace BlackCore
CUpdateInfo loadedUpdateInfo;
loadedUpdateInfo.convertFromJson(Json::jsonObjectFromString(setupJson));
loadedUpdateInfo.setDevelopment(isForDevelopment()); // always update, regardless what persistent setting says
if (loadedUpdateInfo.getMSecsSinceEpoch() == 0 && lastModified > 0) { loadedUpdateInfo.setMSecsSinceEpoch(lastModified); }
qint64 currentVersionTimestamp = currentUpdateInfo.getMSecsSinceEpoch();
qint64 newVersionTimestamp = loadedUpdateInfo.getMSecsSinceEpoch();
if (lastModified > 0 && lastModified > loadedUpdateInfo.getMSecsSinceEpoch()) { loadedUpdateInfo.setMSecsSinceEpoch(lastModified); }
bool sameVersionLoaded = (loadedUpdateInfo == currentUpdateInfo);
if (sameVersionLoaded)
{
CLogMessage(this).info("Same update info loaded from %1 as already in data cache %2") << urlString << m_updateInfo.getFilename();
this->setUpdateTimestamp();
emit versionSynchronized(true);
CLogMessage(this).info("Same update info version loaded from %1 as already in data cache %2") << urlString << m_setup.getFilename();
emit updateInfoSynchronized(true);
return; // success
}
bool sameType = loadedUpdateInfo.hasSameType(currentUpdateInfo);
bool outdatedVersionLoaded = sameType && (newVersionTimestamp < currentVersionTimestamp);
qint64 currentVersionTimestamp = currentUpdateInfo.getMSecsSinceEpoch();
qint64 newVersionTimestamp = loadedUpdateInfo.getMSecsSinceEpoch();
bool outdatedVersionLoaded = (newVersionTimestamp < currentVersionTimestamp);
if (outdatedVersionLoaded)
{
CLogMessage(this).info("Update info loaded from %1 outdated, older than version in data cache %2") << urlString << m_updateInfo.getFilename();
@@ -294,21 +358,21 @@ namespace BlackCore
}
else
{
CStatusMessage m = m_updateInfo.set(loadedUpdateInfo);
CStatusMessage m = m_updateInfo.set(loadedUpdateInfo, loadedUpdateInfo.getMSecsSinceEpoch());
if (!m.isEmpty())
{
m.setCategories(cats());
CLogMessage(this).preformatted(m);
emit versionSynchronized(false);
emit updateInfoSynchronized(false);
return; // issue with cache
}
else
{
CLogMessage(this).info("Update info: Updated data cache in %1") << m_updateInfo.getFilename();
this->setUpdateTimestamp();
emit versionSynchronized(true);
emit updateInfoSynchronized(true);
return; // success
} // cache
} // outdated?
} // outdated
} // json empty
} // no error
@@ -320,14 +384,20 @@ namespace BlackCore
}
// try next one if any
if (this->m_updateInfoUrls.uniqueWrite()->addFailedUrl(url))
if (this->m_updateInfoUrls.addFailedUrl(url))
{
QTimer::singleShot(500, this, &CSetupReader::ps_readSetup);
}
else
{
emit versionSynchronized(false);
emit updateInfoSynchronized(false);
}
} // method
} // function
const CLogCategoryList &CSetupReader::cats()
{
static const CLogCategoryList cats(CLogCategoryList(this).join({ CLogCategory::webservice()}));
return cats;
}
} // namespace

View File

@@ -21,39 +21,56 @@
#include <QObject>
#include <QTimer>
#include <QNetworkReply>
#include <QCommandLineOption>
namespace BlackCore
{
//! Read the central URLs / locations of our data / setup
class BLACKCORE_EXPORT CSetupReader : public BlackMisc::CThreadedReader
//! Read the central URLs / locations of our data / setup.
//! This should be only used in BlackCore::CApplication
//! \note This class is no(!) BlackCore::CThreadedReader as it will be loaded once during startup
//! and is usually fast.
//! \sa BlackCore::Data::GlobalSetup
//! \sa BlackCore::Data::UpdateInfo
class BLACKCORE_EXPORT CSetupReader : public QObject
{
Q_OBJECT
public:
//! Single instance
static CSetupReader &instance();
friend class CApplication; //!< only using class
protected:
//! Constructor
explicit CSetupReader(QObject *parent);
//! Load the data
BlackMisc::CStatusMessage asyncLoad();
//! Parse cmd line arguments
bool parseCmdLineArguments();
//! Add cmd line arguments to BlackCore::CApplication
QList<QCommandLineOption> getCmdLineOptions() const;
//! Terminate
void gracefulShutdown();
//! Setup loaded?
bool isSetupSyncronized() const { return m_setupSyncronized; }
//! Version info loaded?
bool isUpdateSyncronized() const { return m_updateInfoSyncronized; }
signals:
//! Setup has been read
void setupSynchronized(bool success);
//! Version bas been read
void versionSynchronized(bool success);
protected slots:
//! \copydoc BlackMisc::CThreadedReader::initialize
virtual void initialize() override;
//! \copydoc BlackMisc::CThreadedReader::cleanup
virtual void cleanup() override;
void updateInfoSynchronized(bool success);
private slots:
//! Setup has been read
//! \threadsafe
void ps_parseSetupFile(QNetworkReply *nwReply);
//! Update info has been read
//! \threadsafe
void ps_parseUpdateInfoFile(QNetworkReply *nwReplyPtr);
//! Do reading
@@ -65,25 +82,56 @@ namespace BlackCore
//! Setup has beem syncronized
void ps_setupSyncronized(bool success);
//! Version info has beem syncronized
void ps_versionInfoSyncronized(bool success);
//! Setup has been changed
void ps_setupChanged();
private:
QNetworkAccessManager *m_networkManagerBootstrap = nullptr;
QNetworkAccessManager *m_networkManagerUpdateInfo = nullptr;
BlackMisc::LockFree<BlackMisc::Network::CFailoverUrlList> m_bootstrapUrls;
BlackMisc::LockFree<BlackMisc::Network::CFailoverUrlList> m_updateInfoUrls;
//! Bootstrap mode
enum BootsrapMode
{
Default,
Explicit,
CacheOnly
};
bool m_shutdown = false;
bool m_setupSyncronized = false;
bool m_updateInfoSyncronized = false;
QString m_localSetupFileValue;
QString m_bootsrapUrlFileValue;
BootsrapMode m_bootstrapMode;
BlackMisc::Network::CFailoverUrlList m_bootstrapUrls;
BlackMisc::Network::CFailoverUrlList m_updateInfoUrls;
BlackMisc::CData<BlackCore::Data::GlobalSetup> m_setup {this, &CSetupReader::ps_setupChanged}; //!< data cache setup
BlackMisc::CData<BlackCore::Data::UpdateInfo> m_updateInfo {this}; //!< data cache update info
//! Constructor
explicit CSetupReader(QObject *owner);
QCommandLineOption m_cmdBootstrapUrl
{
{ "url", "bootstrap-url", "bootstrapurl" },
QCoreApplication::translate("application", "bootsrap URL, e.g. datastore.swift-project.org"),
"bootstrapurl"
}; //!< bootstrap URL
QCommandLineOption m_cmdBootstrapMode
{
{ "bmode", "bootstrap-mode", "bootstrapmode" },
QCoreApplication::translate("application", "bootstrap mode: (e)xplicit, d(default), (c)ache-only"),
"bootstrapmode", "default"
}; //!< bootstrap mode
//! Read by local individual file
bool localBootstrapFile(QString &fileName);
bool readLocalBootstrapFile(QString &fileName);
//! Convert string to mode
static BootsrapMode stringToEnum(const QString &s);
//! Read for development environment?
static bool isForDevelopment();
//! Categories
const BlackMisc::CLogCategoryList &cats();
};
} // ns