From 466e0dbae4954826b984f9009e4fd5b11e9f5634 Mon Sep 17 00:00:00 2001 From: Lars Toenning Date: Mon, 27 Feb 2023 17:07:25 +0100 Subject: [PATCH] Read VATSIM servers from servers.json --- src/blackcore/vatsim/vatsimdatafilereader.cpp | 34 +--- src/blackcore/vatsim/vatsimdatafilereader.h | 6 - .../vatsim/vatsimserverfilereader.cpp | 156 ++++++++++++++++++ src/blackcore/vatsim/vatsimserverfilereader.h | 85 ++++++++++ src/blackcore/webdataservices.cpp | 48 ++++-- src/blackcore/webdataservices.h | 26 ++- src/blackcore/webreaderflags.h | 25 +-- 7 files changed, 309 insertions(+), 71 deletions(-) create mode 100644 src/blackcore/vatsim/vatsimserverfilereader.cpp create mode 100644 src/blackcore/vatsim/vatsimserverfilereader.h diff --git a/src/blackcore/vatsim/vatsimdatafilereader.cpp b/src/blackcore/vatsim/vatsimdatafilereader.cpp index 38ea153cd..62883af33 100644 --- a/src/blackcore/vatsim/vatsimdatafilereader.cpp +++ b/src/blackcore/vatsim/vatsimdatafilereader.cpp @@ -88,12 +88,7 @@ namespace BlackCore::Vatsim CServerList CVatsimDataFileReader::getVoiceServers() const { - return m_lastGoodSetup.get().getVoiceServers(); - } - - CServerList CVatsimDataFileReader::getFsdServers() const - { - return m_lastGoodSetup.get().getFsdServers(); + return {}; // TODO: Method not used anymore with AFV. } CUserList CVatsimDataFileReader::getPilotsForCallsigns(const CCallsignSet &callsigns) const @@ -275,16 +270,6 @@ namespace BlackCore::Vatsim } atcStations.push_back(parseController(atis.toObject())); } - for (QJsonValueRef server : jsonDoc["servers"].toArray()) - { - if (!this->doWorkCheck()) - { - CLogMessage(this).info(u"Terminated VATSIM file parsing process"); - return; - } - fsdServers.push_back(parseServer(server.toObject())); - if (!fsdServers.back().hasName()) { fsdServers.pop_back(); } - } // Setup for VATSIM servers and sorting for comparison fsdServers.sortBy(&CServer::getName, &CServer::getDescription); @@ -298,15 +283,6 @@ namespace BlackCore::Vatsim m_flightPlanRemarks = flightPlanRemarksMap; } - // update cache itself is thread safe - CVatsimSetup vs(m_lastGoodSetup.get()); - const bool changedSetup = vs.setServers(fsdServers, {}); - if (changedSetup) - { - vs.setUtcTimestamp(updateTimestampFromFile); - m_lastGoodSetup.set(vs); - } - // warnings, if required if (!illegalEquipmentCodes.isEmpty()) { @@ -368,14 +344,6 @@ namespace BlackCore::Vatsim return CAtcStation(callsign, user, freq, {}, range, true, {}, {}, atis); } - CServer CVatsimDataFileReader::parseServer(const QJsonObject &server) const - { - return CServer(server["name"].toString(), server["location"].toString(), - server["hostname_or_ip"].toString(), 6809, CUser("id", "real name", "email", "password"), - CFsdSetup::vatsimStandard(), CVoiceSetup::vatsimStandard(), CEcosystem::VATSIM, - CServer::FSDServerVatsim, server["clients_connection_allowed"].toInt()); - } - void CVatsimDataFileReader::reloadSettings() { CReaderSettings s = m_settings.get(); diff --git a/src/blackcore/vatsim/vatsimdatafilereader.h b/src/blackcore/vatsim/vatsimdatafilereader.h index f2c5624a4..6b1f981ed 100644 --- a/src/blackcore/vatsim/vatsimdatafilereader.h +++ b/src/blackcore/vatsim/vatsimdatafilereader.h @@ -69,10 +69,6 @@ namespace BlackCore::Vatsim //! \threadsafe BlackMisc::Network::CServerList getVoiceServers() const; - //! Get all VATSIM FSD servers - //! \threadsafe - BlackMisc::Network::CServerList getFsdServers() const; - //! Users for callsign(s) //! \threadsafe BlackMisc::Network::CUserList getUsersForCallsigns(const BlackMisc::Aviation::CCallsignSet &callsigns) const; @@ -146,7 +142,6 @@ namespace BlackCore::Vatsim BlackMisc::Aviation::CAtcStationList m_atcStations; BlackMisc::Simulation::CSimulatedAircraftList m_aircraft; - BlackMisc::CData m_lastGoodSetup { this }; BlackMisc::CSettingReadOnly m_settings { this, &CVatsimDataFileReader::reloadSettings }; QMap m_flightPlanRemarks; //!< cache for flight plan remarks @@ -156,7 +151,6 @@ namespace BlackCore::Vatsim BlackMisc::Simulation::CSimulatedAircraft parsePilot(const QJsonObject &, QStringList &o_illegalEquipmentCodes) const; BlackMisc::Aviation::CFlightPlanRemarks parseFlightPlanRemarks(const QJsonObject &) const; BlackMisc::Aviation::CAtcStation parseController(const QJsonObject &) const; - BlackMisc::Network::CServer parseServer(const QJsonObject &) const; //! Read / re-read data file void read(); diff --git a/src/blackcore/vatsim/vatsimserverfilereader.cpp b/src/blackcore/vatsim/vatsimserverfilereader.cpp new file mode 100644 index 000000000..1ef2deb4f --- /dev/null +++ b/src/blackcore/vatsim/vatsimserverfilereader.cpp @@ -0,0 +1,156 @@ +/* Copyright (C) 2023 + * swift project Community / Contributors + * + * This file is part of swift project. It is subject to the license terms in the LICENSE file found in the top-level + * directory of this distribution. No part of swift project, including this file, may be copied, modified, propagated, + * or distributed except according to the terms contained in the LICENSE file. + */ + +#include "blackcore/vatsim/vatsimserverfilereader.h" +#include "blackcore/application.h" +#include "blackmisc/network/entityflags.h" +#include "blackmisc/network/server.h" +#include "blackmisc/network/user.h" +#include "blackmisc/pq/units.h" +#include "blackmisc/logmessage.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace BlackMisc; +using namespace BlackMisc::Aviation; +using namespace BlackMisc::Audio; +using namespace BlackMisc::Network; +using namespace BlackMisc::Geo; +using namespace BlackMisc::Simulation; +using namespace BlackMisc::PhysicalQuantities; +using namespace BlackCore::Data; + +namespace BlackCore::Vatsim +{ + CVatsimServerFileReader::CVatsimServerFileReader(QObject *owner) : + CThreadedReader(owner, "CVatsimServerFileReader"), + CEcosystemAware(CEcosystemAware::providerIfPossible(owner)) + { } + + CServerList CVatsimServerFileReader::getFsdServers() const + { + return m_lastGoodSetup.get().getFsdServers(); + } + + void CVatsimServerFileReader::readInBackgroundThread() + { + QPointer myself(this); + QTimer::singleShot(0, this, [ = ] + { + if (!myself) { return; } + myself->read(); + }); + } + + void CVatsimServerFileReader::doWorkImpl() + { + this->read(); + } + + void CVatsimServerFileReader::read() + { + this->threadAssertCheck(); + if (!this->doWorkCheck()) { return; } + if (!this->isInternetAccessible("No network/internet access, cannot read VATSIM server file")) { return; } + if (this->isNotVATSIMEcosystem()) { return; } + + Q_ASSERT_X(sApp, Q_FUNC_INFO, "Missing application"); + const QUrl url = sApp->getVatsimServerFileUrl(); + if (url.isEmpty()) { return; } + this->getFromNetworkAndLog(url, { this, &CVatsimServerFileReader::parseVatsimFile}); + } + + void CVatsimServerFileReader::parseVatsimFile(QNetworkReply *nwReplyPtr) + { + // wrap pointer, make sure any exit cleans up reply + // required to use delete later as object is created in a different thread + QScopedPointer nwReply(nwReplyPtr); + this->threadAssertCheck(); + if (this->isNotVATSIMEcosystem()) { return; } + + // Worker thread, make sure to write only synced here! + if (!this->doWorkCheck()) + { + CLogMessage(this).info(u"Terminated VATSIM file parsing process"); + return; // stop, terminate straight away, ending thread + } + + this->logNetworkReplyReceived(nwReplyPtr); + + const QUrl url = nwReply->url(); + const QString urlString = url.toString(); + + if (nwReply->error() == QNetworkReply::NoError) + { + const QString dataFileData = nwReply->readAll(); + nwReply->close(); // close asap + + if (dataFileData.isEmpty()) { return; } + if (!this->didContentChange(dataFileData)) // Quick check by hash + { + CLogMessage(this).info(u"VATSIM file '%1' has same content, skipped") << urlString; + return; + } + auto jsonDoc = QJsonDocument::fromJson(dataFileData.toUtf8()); + if (jsonDoc.isEmpty()) { return; } + + // build on local vars for thread safety + CServerList fsdServers; + + for (QJsonValueRef server : jsonDoc.array()) + { + if (!this->doWorkCheck()) + { + CLogMessage(this).info(u"Terminated VATSIM file parsing process"); + return; + } + fsdServers.push_back(parseServer(server.toObject())); + if (!fsdServers.back().hasName()) { fsdServers.pop_back(); } + } + + // Setup for VATSIM servers and sorting for comparison + fsdServers.sortBy(&CServer::getName, &CServer::getDescription); + + // update cache itself is thread safe + CVatsimSetup vs(m_lastGoodSetup.get()); + const bool changedSetup = vs.setServers(fsdServers, {}); + if (changedSetup) + { + vs.setCurrentUtcTime(); + m_lastGoodSetup.set(vs); + } + + // data read finished + emit this->dataFileRead(dataFileData.size() / 1000); + emit this->dataRead(CEntityFlags::VatsimDataFile, CEntityFlags::ReadFinished, dataFileData.size() / 1000, url); + } + else + { + // network error + CLogMessage(this).warning(u"Reading VATSIM data file failed '%1' '%2'") << nwReply->errorString() << urlString; + nwReply->abort(); + emit this->dataRead(CEntityFlags::VatsimDataFile, CEntityFlags::ReadFailed, 0, url); + } + } + + CServer CVatsimServerFileReader::parseServer(const QJsonObject &server) const + { + return CServer(server["name"].toString(), server["location"].toString(), + server["hostname_or_ip"].toString(), 6809, CUser("id", "real name", "email", "password"), + CFsdSetup::vatsimStandard(), CVoiceSetup::vatsimStandard(), CEcosystem::VATSIM, + CServer::FSDServerVatsim, server["clients_connection_allowed"].toInt()); + } + +} // ns diff --git a/src/blackcore/vatsim/vatsimserverfilereader.h b/src/blackcore/vatsim/vatsimserverfilereader.h new file mode 100644 index 000000000..eee6824ca --- /dev/null +++ b/src/blackcore/vatsim/vatsimserverfilereader.h @@ -0,0 +1,85 @@ +/* Copyright (C) 2023 + * swift project Community / Contributors + * + * This file is part of swift project. It is subject to the license terms in the LICENSE file found in the top-level + * directory of this distribution. No part of swift project, including this file, may be copied, modified, propagated, + * or distributed except according to the terms contained in the LICENSE file. + */ + +//! \file + +#ifndef BLACKCORE_VATSIM_VATSIMSERVERFILEREADER_H +#define BLACKCORE_VATSIM_VATSIMSERVERFILEREADER_H + +#include "blackcore/blackcoreexport.h" +#include "blackcore/data/vatsimsetup.h" +#include "blackmisc/aviation/aircrafticaocode.h" +#include "blackmisc/aviation/airlineicaocode.h" +#include "blackmisc/aviation/atcstationlist.h" +#include "blackmisc/aviation/callsignset.h" +#include "blackmisc/aviation/flightplan.h" +#include "blackmisc/network/entityflags.h" +#include "blackmisc/network/ecosystemprovider.h" +#include "blackmisc/network/serverlist.h" +#include "blackmisc/network/userlist.h" +#include "blackmisc/network/voicecapabilities.h" +#include "blackmisc/simulation/simulatedaircraftlist.h" +#include "blackmisc/datacache.h" +#include "blackcore/threadedreader.h" + +#include +#include +#include +#include + +class QNetworkReply; + +namespace BlackCore::Vatsim +{ + //! Read VATSIM server file + //! \sa https://data.vatsim.net/v3/vatsim-servers.json + class BLACKCORE_EXPORT CVatsimServerFileReader : + public CThreadedReader, + public BlackMisc::Network::CEcosystemAware + { + Q_OBJECT + + public: + //! Constructor + explicit CVatsimServerFileReader(QObject *owner); + + //! Get all VATSIM FSD servers + //! \threadsafe + BlackMisc::Network::CServerList getFsdServers() const; + + //! Start reading in own thread + void readInBackgroundThread(); + + signals: + //! Data have been read + void dataFileRead(int kB); + + //! Data have been read + void dataRead(BlackMisc::Network::CEntityFlags::Entity entity, BlackMisc::Network::CEntityFlags::ReadState state, int number, const QUrl &url); + + protected: + //! \name BlackCore::CThreadedReader overrides + //! @{ + virtual void doWorkImpl() override; + //! @} + + private: + BlackMisc::CData m_lastGoodSetup { this }; + + //! Data have been read, parse VATSIM server file + void parseVatsimFile(QNetworkReply *nwReply); + + BlackMisc::Network::CServer parseServer(const QJsonObject &) const; + + //! Read / re-read data file + void read(); + + }; +} // ns + +#endif // guard diff --git a/src/blackcore/webdataservices.cpp b/src/blackcore/webdataservices.cpp index 2906b09cb..5a25e9434 100644 --- a/src/blackcore/webdataservices.cpp +++ b/src/blackcore/webdataservices.cpp @@ -18,6 +18,7 @@ #include "blackcore/vatsim/vatsimdatafilereader.h" #include "blackcore/vatsim/vatsimmetarreader.h" #include "blackcore/vatsim/vatsimstatusfilereader.h" +#include "blackcore/vatsim/vatsimserverfilereader.h" #include "blackcore/webdataservices.h" #include "blackcore/setupreader.h" #include "blackcore/application.h" @@ -120,7 +121,7 @@ namespace BlackCore CServerList CWebDataServices::getVatsimFsdServers() const { - if (m_vatsimDataFileReader) { return m_vatsimDataFileReader->getFsdServers(); } + if (m_vatsimServerFileReader) { return m_vatsimServerFileReader->getFsdServers(); } return CServerList(); } @@ -971,14 +972,15 @@ namespace BlackCore if (m_shuttingDown) { return; } m_shuttingDown = true; this->disconnect(); // all signals - if (m_vatsimMetarReader) { m_vatsimMetarReader->quitAndWait(); m_vatsimMetarReader = nullptr; } - if (m_vatsimBookingReader) { m_vatsimBookingReader->quitAndWait(); m_vatsimBookingReader = nullptr; } - if (m_vatsimDataFileReader) { m_vatsimDataFileReader->quitAndWait(); m_vatsimDataFileReader = nullptr; } - if (m_vatsimStatusReader) { m_vatsimStatusReader->quitAndWait(); m_vatsimStatusReader = nullptr; } - if (m_modelDataReader) { m_modelDataReader->quitAndWait(); m_modelDataReader = nullptr; } - if (m_airportDataReader) { m_airportDataReader->quitAndWait(); m_airportDataReader = nullptr; } - if (m_icaoDataReader) { m_icaoDataReader->quitAndWait(); m_icaoDataReader = nullptr; } - if (m_dbInfoDataReader) { m_dbInfoDataReader->quitAndWait(); m_dbInfoDataReader = nullptr; } + if (m_vatsimMetarReader) { m_vatsimMetarReader->quitAndWait(); m_vatsimMetarReader = nullptr; } + if (m_vatsimBookingReader) { m_vatsimBookingReader->quitAndWait(); m_vatsimBookingReader = nullptr; } + if (m_vatsimDataFileReader) { m_vatsimDataFileReader->quitAndWait(); m_vatsimDataFileReader = nullptr; } + if (m_vatsimStatusReader) { m_vatsimStatusReader->quitAndWait(); m_vatsimStatusReader = nullptr; } + if (m_vatsimServerFileReader) { m_vatsimServerFileReader->quitAndWait(); m_vatsimServerFileReader = nullptr; } + if (m_modelDataReader) { m_modelDataReader->quitAndWait(); m_modelDataReader = nullptr; } + if (m_airportDataReader) { m_airportDataReader->quitAndWait(); m_airportDataReader = nullptr; } + if (m_icaoDataReader) { m_icaoDataReader->quitAndWait(); m_icaoDataReader = nullptr; } + if (m_dbInfoDataReader) { m_dbInfoDataReader->quitAndWait(); m_dbInfoDataReader = nullptr; } // DB writer is no threaded reader, it has a special role if (m_databaseWriter) { m_databaseWriter->gracefulShutdown(); m_databaseWriter = nullptr; } @@ -1050,7 +1052,7 @@ namespace BlackCore this->initSharedInfoObjectReaderAndTriggerRead(); } - // 2. Status file, updating the VATSIM related caches + // 2. Status and server file, updating the VATSIM related caches // Read as soon as initReaders is done if (readersNeeded.testFlag(CWebReaderFlags::VatsimStatusReader) || readersNeeded.testFlag(CWebReaderFlags::VatsimDataReader) || readersNeeded.testFlag(CWebReaderFlags::VatsimMetarReader)) { @@ -1061,12 +1063,14 @@ namespace BlackCore // run single shot in main loop, so readInBackgroundThread is not called before initReaders completes const QPointer myself(this); - QTimer::singleShot(100, this, [ = ]() + QTimer::singleShot(0, this, [ = ]() { if (!myself || m_shuttingDown) { return; } if (!sApp || sApp->isShuttingDown()) { return; } m_vatsimStatusReader->readInBackgroundThread(); }); + + startVatsimServerFileReader(); } // ---- "normal data", triggerRead will start read, not starting directly @@ -1168,6 +1172,23 @@ namespace BlackCore } } + void CWebDataServices::startVatsimServerFileReader() + { + m_vatsimServerFileReader = new CVatsimServerFileReader(this); + connect(m_vatsimServerFileReader, &CVatsimServerFileReader::dataFileRead, this, &CWebDataServices::vatsimServerFileRead, Qt::QueuedConnection); + CLogMessage(this).info(u"Trigger read of VATSIM server file"); + m_vatsimServerFileReader->start(QThread::LowPriority); + + // run single shot in main loop, so readInBackgroundThread is not called before initReaders completes + const QPointer myself(this); + QTimer::singleShot(0, this, [ = ]() + { + if (!myself || m_shuttingDown) { return; } + if (!sApp || sApp->isShuttingDown()) { return; } + m_vatsimServerFileReader->readInBackgroundThread(); + }); + } + void CWebDataServices::initDbInfoObjectReaderAndTriggerRead() { // run in correct thread @@ -1332,6 +1353,11 @@ namespace BlackCore CLogMessage(this).info(u"Read VATSIM status file, %1 lines") << lines; } + void CWebDataServices::vatsimServerFileRead(int lines) + { + CLogMessage(this).info(u"Read VATSIM server file, %1 lines") << lines; + } + void CWebDataServices::readFromSwiftReader(CEntityFlags::Entity entities, CEntityFlags::ReadState state, int number, const QUrl &url) { if (state == CEntityFlags::ReadStarted) { return; } // just started diff --git a/src/blackcore/webdataservices.h b/src/blackcore/webdataservices.h index 7c604aa7f..2a36129f7 100644 --- a/src/blackcore/webdataservices.h +++ b/src/blackcore/webdataservices.h @@ -67,6 +67,7 @@ namespace BlackCore class CVatsimDataFileReader; class CVatsimMetarReader; class CVatsimStatusFileReader; + class CVatsimServerFileReader; } namespace Db @@ -581,6 +582,12 @@ namespace BlackCore //! VATSIM status file has been read void vatsimStatusFileRead(int lines); + //! VATSIM server file has been read + void vatsimServerFileRead(int lines); + + //! Initialize and start VATSIM server file reader + void startVatsimServerFileReader(); + //! Read finished from reader void readFromSwiftReader(BlackMisc::Network::CEntityFlags::Entity entities, BlackMisc::Network::CEntityFlags::ReadState state, int number, const QUrl &url); @@ -643,15 +650,16 @@ namespace BlackCore QSet m_signalledEntities; //!< remember signalled entites // for reading XML and VATSIM data files - Vatsim::CVatsimStatusFileReader *m_vatsimStatusReader = nullptr; - Vatsim::CVatsimBookingReader *m_vatsimBookingReader = nullptr; - Vatsim::CVatsimDataFileReader *m_vatsimDataFileReader = nullptr; - Vatsim::CVatsimMetarReader *m_vatsimMetarReader = nullptr; - Db::CIcaoDataReader *m_icaoDataReader = nullptr; - Db::CModelDataReader *m_modelDataReader = nullptr; - Db::CAirportDataReader *m_airportDataReader = nullptr; - Db::CInfoDataReader *m_dbInfoDataReader = nullptr; - Db::CInfoDataReader *m_sharedInfoDataReader = nullptr; + Vatsim::CVatsimStatusFileReader *m_vatsimStatusReader = nullptr; + Vatsim::CVatsimBookingReader *m_vatsimBookingReader = nullptr; + Vatsim::CVatsimDataFileReader *m_vatsimDataFileReader = nullptr; + Vatsim::CVatsimMetarReader *m_vatsimMetarReader = nullptr; + Vatsim::CVatsimServerFileReader *m_vatsimServerFileReader = nullptr; + Db::CIcaoDataReader *m_icaoDataReader = nullptr; + Db::CModelDataReader *m_modelDataReader = nullptr; + Db::CAirportDataReader *m_airportDataReader = nullptr; + Db::CInfoDataReader *m_dbInfoDataReader = nullptr; + Db::CInfoDataReader *m_sharedInfoDataReader = nullptr; // writing objects directly into DB Db::CDatabaseWriter *m_databaseWriter = nullptr; diff --git a/src/blackcore/webreaderflags.h b/src/blackcore/webreaderflags.h index 8d6411f7c..4dbd21c62 100644 --- a/src/blackcore/webreaderflags.h +++ b/src/blackcore/webreaderflags.h @@ -28,18 +28,19 @@ namespace BlackCore //! Which readers to init enum WebReaderFlag { - None = 0, //!< no reader at all - VatsimBookingReader = 1 << 0, //!< reader for VATSIM booking data - VatsimDataReader = 1 << 1, //!< reader for VATSIM data - VatsimMetarReader = 1 << 2, //!< reader for VATSIM metar data - VatsimStatusReader = 1 << 3, //!< reader for VATSIM status file - IcaoDataReader = 1 << 4, //!< reader for ICAO data - ModelReader = 1 << 5, //!< reader for model data such as liveries, models, etc - AirportReader = 1 << 6, //!< reader for airport list - DbInfoDataReader = 1 << 7, //!< DB info data (metdata, how many data, when updated) - AllVatsimReaders = VatsimBookingReader | VatsimDataReader | VatsimMetarReader | VatsimStatusReader, //!< all VATSIM readers - AllSwiftDbReaders = IcaoDataReader | ModelReader | DbInfoDataReader | AirportReader, //!< all swift data - AllReaders = AllSwiftDbReaders | AllVatsimReaders //!< everything + None = 0, //!< no reader at all + VatsimBookingReader = 1 << 0, //!< reader for VATSIM booking data + VatsimDataReader = 1 << 1, //!< reader for VATSIM data + VatsimMetarReader = 1 << 2, //!< reader for VATSIM metar data + VatsimStatusReader = 1 << 3, //!< reader for VATSIM status file + VatsimServerFileReader = 1 << 4, //!< reader for VATSIM server file + IcaoDataReader = 1 << 5, //!< reader for ICAO data + ModelReader = 1 << 6, //!< reader for model data such as liveries, models, etc + AirportReader = 1 << 7, //!< reader for airport list + DbInfoDataReader = 1 << 8, //!< DB info data (metdata, how many data, when updated) + AllVatsimReaders = VatsimBookingReader | VatsimDataReader | VatsimMetarReader | VatsimStatusReader | VatsimServerFileReader, //!< all VATSIM readers + AllSwiftDbReaders = IcaoDataReader | ModelReader | DbInfoDataReader | AirportReader, //!< all swift data + AllReaders = AllSwiftDbReaders | AllVatsimReaders //!< everything }; Q_DECLARE_FLAGS(WebReader, WebReaderFlag)