Read VATSIM servers from servers.json

src/blackcore/vatsim/vatsimdatafilereader.cpp

@@ -88,12 +88,7 @@ namespace BlackCore::Vatsim
 
     CServerList CVatsimDataFileReader::getVoiceServers() const
     {
-        return m_lastGoodSetup.get().getVoiceServers();
+        return {}; // TODO: Method not used anymore with AFV.
-    }
-
-    CServerList CVatsimDataFileReader::getFsdServers() const
-    {
-        return m_lastGoodSetup.get().getFsdServers();
     }
 
     CUserList CVatsimDataFileReader::getPilotsForCallsigns(const CCallsignSet &callsigns) const
@@ -275,16 +270,6 @@ namespace BlackCore::Vatsim
                 }
                 atcStations.push_back(parseController(atis.toObject()));
             }
-            for (QJsonValueRef server : jsonDoc["servers"].toArray())
-            {
-                if (!this->doWorkCheck())
-                {
-                    CLogMessage(this).info(u"Terminated VATSIM file parsing process");
-                    return;
-                }
-                fsdServers.push_back(parseServer(server.toObject()));
-                if (!fsdServers.back().hasName()) { fsdServers.pop_back(); }
-            }
 
             // Setup for VATSIM servers and sorting for comparison
             fsdServers.sortBy(&CServer::getName, &CServer::getDescription);
@@ -298,15 +283,6 @@ namespace BlackCore::Vatsim
                 m_flightPlanRemarks = flightPlanRemarksMap;
             }
 
-            // update cache itself is thread safe
-            CVatsimSetup vs(m_lastGoodSetup.get());
-            const bool changedSetup = vs.setServers(fsdServers, {});
-            if (changedSetup)
-            {
-                vs.setUtcTimestamp(updateTimestampFromFile);
-                m_lastGoodSetup.set(vs);
-            }
-
             // warnings, if required
             if (!illegalEquipmentCodes.isEmpty())
             {
@@ -368,14 +344,6 @@ namespace BlackCore::Vatsim
         return CAtcStation(callsign, user, freq, {}, range, true, {}, {}, atis);
     }
 
-    CServer CVatsimDataFileReader::parseServer(const QJsonObject &server) const
-    {
-        return CServer(server["name"].toString(), server["location"].toString(),
-            server["hostname_or_ip"].toString(), 6809, CUser("id", "real name", "email", "password"),
-            CFsdSetup::vatsimStandard(), CVoiceSetup::vatsimStandard(), CEcosystem::VATSIM,
-            CServer::FSDServerVatsim, server["clients_connection_allowed"].toInt());
-    }
-
     void CVatsimDataFileReader::reloadSettings()
     {
         CReaderSettings s = m_settings.get();

src/blackcore/vatsim/vatsimdatafilereader.h

@@ -69,10 +69,6 @@ namespace BlackCore::Vatsim
         //! \threadsafe
         BlackMisc::Network::CServerList getVoiceServers() const;
 
-        //! Get all VATSIM FSD servers
-        //! \threadsafe
-        BlackMisc::Network::CServerList getFsdServers() const;
-
         //! Users for callsign(s)
         //! \threadsafe
         BlackMisc::Network::CUserList getUsersForCallsigns(const BlackMisc::Aviation::CCallsignSet &callsigns) const;
@@ -146,7 +142,6 @@ namespace BlackCore::Vatsim
 
         BlackMisc::Aviation::CAtcStationList m_atcStations;
         BlackMisc::Simulation::CSimulatedAircraftList m_aircraft;
-        BlackMisc::CData<BlackCore::Data::TVatsimSetup> m_lastGoodSetup { this };
         BlackMisc::CSettingReadOnly<BlackCore::Vatsim::TVatsimDataFile> m_settings { this, &CVatsimDataFileReader::reloadSettings };
         QMap<BlackMisc::Aviation::CCallsign, BlackMisc::Aviation::CFlightPlanRemarks> m_flightPlanRemarks; //!< cache for flight plan remarks
 
@@ -156,7 +151,6 @@ namespace BlackCore::Vatsim
         BlackMisc::Simulation::CSimulatedAircraft parsePilot(const QJsonObject &, QStringList &o_illegalEquipmentCodes) const;
         BlackMisc::Aviation::CFlightPlanRemarks parseFlightPlanRemarks(const QJsonObject &) const;
         BlackMisc::Aviation::CAtcStation parseController(const QJsonObject &) const;
-        BlackMisc::Network::CServer parseServer(const QJsonObject &) const;
 
         //! Read / re-read data file
         void read();

src/blackcore/vatsim/vatsimserverfilereader.cpp

@@ -0,0 +1,156 @@
+/* Copyright (C) 2023
+ * swift project Community / Contributors
+ *
+ * This file is part of swift project. It is subject to the license terms in the LICENSE file found in the top-level
+ * directory of this distribution. No part of swift project, including this file, may be copied, modified, propagated,
+ * or distributed except according to the terms contained in the LICENSE file.
+ */
+
+#include "blackcore/vatsim/vatsimserverfilereader.h"
+#include "blackcore/application.h"
+#include "blackmisc/network/entityflags.h"
+#include "blackmisc/network/server.h"
+#include "blackmisc/network/user.h"
+#include "blackmisc/pq/units.h"
+#include "blackmisc/logmessage.h"
+
+#include <QStringBuilder>
+#include <QMetaObject>
+#include <QNetworkReply>
+#include <QScopedPointerDeleteLater>
+#include <QTimer>
+#include <QUrl>
+#include <QtGlobal>
+#include <QPointer>
+
+using namespace BlackMisc;
+using namespace BlackMisc::Aviation;
+using namespace BlackMisc::Audio;
+using namespace BlackMisc::Network;
+using namespace BlackMisc::Geo;
+using namespace BlackMisc::Simulation;
+using namespace BlackMisc::PhysicalQuantities;
+using namespace BlackCore::Data;
+
+namespace BlackCore::Vatsim
+{
+    CVatsimServerFileReader::CVatsimServerFileReader(QObject *owner) :
+            CThreadedReader(owner, "CVatsimServerFileReader"),
+            CEcosystemAware(CEcosystemAware::providerIfPossible(owner))
+    { }
+
+    CServerList CVatsimServerFileReader::getFsdServers() const
+    {
+        return m_lastGoodSetup.get().getFsdServers();
+    }
+
+    void CVatsimServerFileReader::readInBackgroundThread()
+    {
+        QPointer<CVatsimServerFileReader> myself(this);
+        QTimer::singleShot(0, this, [ = ]
+        {
+            if (!myself) { return; }
+            myself->read();
+        });
+    }
+
+    void CVatsimServerFileReader::doWorkImpl()
+    {
+        this->read();
+    }
+
+    void CVatsimServerFileReader::read()
+    {
+        this->threadAssertCheck();
+        if (!this->doWorkCheck()) { return; }
+        if (!this->isInternetAccessible("No network/internet access, cannot read VATSIM server file")) { return; }
+        if (this->isNotVATSIMEcosystem()) { return; }
+
+        Q_ASSERT_X(sApp, Q_FUNC_INFO, "Missing application");
+        const QUrl url = sApp->getVatsimServerFileUrl();
+        if (url.isEmpty()) { return; }
+        this->getFromNetworkAndLog(url, { this, &CVatsimServerFileReader::parseVatsimFile});
+    }
+
+    void CVatsimServerFileReader::parseVatsimFile(QNetworkReply *nwReplyPtr)
+    {
+        // wrap pointer, make sure any exit cleans up reply
+        // required to use delete later as object is created in a different thread
+        QScopedPointer<QNetworkReply, QScopedPointerDeleteLater> nwReply(nwReplyPtr);
+        this->threadAssertCheck();
+        if (this->isNotVATSIMEcosystem()) { return; }
+
+        // Worker thread, make sure to write only synced here!
+        if (!this->doWorkCheck())
+        {
+            CLogMessage(this).info(u"Terminated VATSIM file parsing process");
+            return; // stop, terminate straight away, ending thread
+        }
+
+        this->logNetworkReplyReceived(nwReplyPtr);
+
+        const QUrl url = nwReply->url();
+        const QString urlString = url.toString();
+
+        if (nwReply->error() == QNetworkReply::NoError)
+        {
+            const QString dataFileData = nwReply->readAll();
+            nwReply->close(); // close asap
+
+            if (dataFileData.isEmpty()) { return; }
+            if (!this->didContentChange(dataFileData)) // Quick check by hash
+            {
+                CLogMessage(this).info(u"VATSIM file '%1' has same content, skipped") << urlString;
+                return;
+            }
+            auto jsonDoc = QJsonDocument::fromJson(dataFileData.toUtf8());
+            if (jsonDoc.isEmpty()) { return; }
+
+            // build on local vars for thread safety
+            CServerList fsdServers;
+
+            for (QJsonValueRef server : jsonDoc.array())
+            {
+                if (!this->doWorkCheck())
+                {
+                    CLogMessage(this).info(u"Terminated VATSIM file parsing process");
+                    return;
+                }
+                fsdServers.push_back(parseServer(server.toObject()));
+                if (!fsdServers.back().hasName()) { fsdServers.pop_back(); }
+            }
+
+            // Setup for VATSIM servers and sorting for comparison
+            fsdServers.sortBy(&CServer::getName, &CServer::getDescription);
+
+            // update cache itself is thread safe
+            CVatsimSetup vs(m_lastGoodSetup.get());
+            const bool changedSetup = vs.setServers(fsdServers, {});
+            if (changedSetup)
+            {
+                vs.setCurrentUtcTime();
+                m_lastGoodSetup.set(vs);
+            }
+
+            // data read finished
+            emit this->dataFileRead(dataFileData.size() / 1000);
+            emit this->dataRead(CEntityFlags::VatsimDataFile, CEntityFlags::ReadFinished, dataFileData.size() / 1000, url);
+        }
+        else
+        {
+            // network error
+            CLogMessage(this).warning(u"Reading VATSIM data file failed '%1' '%2'") << nwReply->errorString() << urlString;
+            nwReply->abort();
+            emit this->dataRead(CEntityFlags::VatsimDataFile, CEntityFlags::ReadFailed, 0, url);
+        }
+    }
+
+    CServer CVatsimServerFileReader::parseServer(const QJsonObject &server) const
+    {
+        return CServer(server["name"].toString(), server["location"].toString(),
+                       server["hostname_or_ip"].toString(), 6809, CUser("id", "real name", "email", "password"),
+                       CFsdSetup::vatsimStandard(), CVoiceSetup::vatsimStandard(), CEcosystem::VATSIM,
+                       CServer::FSDServerVatsim, server["clients_connection_allowed"].toInt());
+    }
+
+} // ns

src/blackcore/vatsim/vatsimserverfilereader.h

@@ -0,0 +1,85 @@
+/* Copyright (C) 2023
+ * swift project Community / Contributors
+ *
+ * This file is part of swift project. It is subject to the license terms in the LICENSE file found in the top-level
+ * directory of this distribution. No part of swift project, including this file, may be copied, modified, propagated,
+ * or distributed except according to the terms contained in the LICENSE file.
+ */
+
+//! \file
+
+#ifndef BLACKCORE_VATSIM_VATSIMSERVERFILEREADER_H
+#define BLACKCORE_VATSIM_VATSIMSERVERFILEREADER_H
+
+#include "blackcore/blackcoreexport.h"
+#include "blackcore/data/vatsimsetup.h"
+#include "blackmisc/aviation/aircrafticaocode.h"
+#include "blackmisc/aviation/airlineicaocode.h"
+#include "blackmisc/aviation/atcstationlist.h"
+#include "blackmisc/aviation/callsignset.h"
+#include "blackmisc/aviation/flightplan.h"
+#include "blackmisc/network/entityflags.h"
+#include "blackmisc/network/ecosystemprovider.h"
+#include "blackmisc/network/serverlist.h"
+#include "blackmisc/network/userlist.h"
+#include "blackmisc/network/voicecapabilities.h"
+#include "blackmisc/simulation/simulatedaircraftlist.h"
+#include "blackmisc/datacache.h"
+#include "blackcore/threadedreader.h"
+
+#include <QMap>
+#include <QObject>
+#include <QString>
+#include <QStringList>
+
+class QNetworkReply;
+
+namespace BlackCore::Vatsim
+{
+    //! Read VATSIM server file
+    //! \sa https://data.vatsim.net/v3/vatsim-servers.json
+    class BLACKCORE_EXPORT CVatsimServerFileReader :
+            public CThreadedReader,
+            public BlackMisc::Network::CEcosystemAware
+    {
+    Q_OBJECT
+
+    public:
+        //! Constructor
+        explicit CVatsimServerFileReader(QObject *owner);
+
+        //! Get all VATSIM FSD servers
+        //! \threadsafe
+        BlackMisc::Network::CServerList getFsdServers() const;
+
+        //! Start reading in own thread
+        void readInBackgroundThread();
+
+    signals:
+        //! Data have been read
+        void dataFileRead(int kB);
+
+        //! Data have been read
+        void dataRead(BlackMisc::Network::CEntityFlags::Entity entity, BlackMisc::Network::CEntityFlags::ReadState state, int number, const QUrl &url);
+
+    protected:
+        //! \name BlackCore::CThreadedReader overrides
+        //! @{
+        virtual void doWorkImpl() override;
+        //! @}
+
+    private:
+        BlackMisc::CData<BlackCore::Data::TVatsimSetup> m_lastGoodSetup { this };
+
+        //! Data have been read, parse VATSIM server file
+        void parseVatsimFile(QNetworkReply *nwReply);
+
+        BlackMisc::Network::CServer parseServer(const QJsonObject &) const;
+
+        //! Read / re-read data file
+        void read();
+
+    };
+} // ns
+
+#endif // guard

src/blackcore/webdataservices.cpp

@@ -18,6 +18,7 @@
 #include "blackcore/vatsim/vatsimdatafilereader.h"
 #include "blackcore/vatsim/vatsimmetarreader.h"
 #include "blackcore/vatsim/vatsimstatusfilereader.h"
+#include "blackcore/vatsim/vatsimserverfilereader.h"
 #include "blackcore/webdataservices.h"
 #include "blackcore/setupreader.h"
 #include "blackcore/application.h"
@@ -120,7 +121,7 @@ namespace BlackCore
 
     CServerList CWebDataServices::getVatsimFsdServers() const
     {
-        if (m_vatsimDataFileReader) { return m_vatsimDataFileReader->getFsdServers(); }
+        if (m_vatsimServerFileReader) { return m_vatsimServerFileReader->getFsdServers(); }
         return CServerList();
     }
 
@@ -975,6 +976,7 @@ namespace BlackCore
         if (m_vatsimBookingReader)    { m_vatsimBookingReader->quitAndWait(); m_vatsimBookingReader = nullptr; }
         if (m_vatsimDataFileReader)   { m_vatsimDataFileReader->quitAndWait(); m_vatsimDataFileReader = nullptr; }
         if (m_vatsimStatusReader)     { m_vatsimStatusReader->quitAndWait(); m_vatsimStatusReader = nullptr; }
+        if (m_vatsimServerFileReader) { m_vatsimServerFileReader->quitAndWait(); m_vatsimServerFileReader = nullptr; }
         if (m_modelDataReader)        { m_modelDataReader->quitAndWait(); m_modelDataReader = nullptr; }
         if (m_airportDataReader)      { m_airportDataReader->quitAndWait(); m_airportDataReader = nullptr; }
         if (m_icaoDataReader)         { m_icaoDataReader->quitAndWait(); m_icaoDataReader = nullptr; }
@@ -1050,7 +1052,7 @@ namespace BlackCore
             this->initSharedInfoObjectReaderAndTriggerRead();
         }
 
-        // 2. Status file, updating the VATSIM related caches
+        // 2. Status and server file, updating the VATSIM related caches
         // Read as soon as initReaders is done
         if (readersNeeded.testFlag(CWebReaderFlags::VatsimStatusReader) || readersNeeded.testFlag(CWebReaderFlags::VatsimDataReader) || readersNeeded.testFlag(CWebReaderFlags::VatsimMetarReader))
         {
@@ -1061,12 +1063,14 @@ namespace BlackCore
 
             // run single shot in main loop, so readInBackgroundThread is not called before initReaders completes
             const QPointer<CWebDataServices> myself(this);
-            QTimer::singleShot(100, this, [ = ]()
+            QTimer::singleShot(0, this, [ = ]()
             {
                 if (!myself || m_shuttingDown) { return; }
                 if (!sApp || sApp->isShuttingDown()) { return; }
                 m_vatsimStatusReader->readInBackgroundThread();
             });
+
+            startVatsimServerFileReader();
         }
 
         // ---- "normal data", triggerRead will start read, not starting directly
@@ -1168,6 +1172,23 @@ namespace BlackCore
         }
     }
 
+    void CWebDataServices::startVatsimServerFileReader()
+    {
+        m_vatsimServerFileReader = new CVatsimServerFileReader(this);
+        connect(m_vatsimServerFileReader, &CVatsimServerFileReader::dataFileRead, this, &CWebDataServices::vatsimServerFileRead, Qt::QueuedConnection);
+        CLogMessage(this).info(u"Trigger read of VATSIM server file");
+        m_vatsimServerFileReader->start(QThread::LowPriority);
+
+        // run single shot in main loop, so readInBackgroundThread is not called before initReaders completes
+        const QPointer<CWebDataServices> myself(this);
+        QTimer::singleShot(0, this, [ = ]()
+        {
+            if (!myself || m_shuttingDown) { return; }
+            if (!sApp || sApp->isShuttingDown()) { return; }
+            m_vatsimServerFileReader->readInBackgroundThread();
+        });
+    }
+
     void CWebDataServices::initDbInfoObjectReaderAndTriggerRead()
     {
         // run in correct thread
@@ -1332,6 +1353,11 @@ namespace BlackCore
         CLogMessage(this).info(u"Read VATSIM status file, %1 lines") << lines;
     }
 
+    void CWebDataServices::vatsimServerFileRead(int lines)
+    {
+        CLogMessage(this).info(u"Read VATSIM server file, %1 lines") << lines;
+    }
+
     void CWebDataServices::readFromSwiftReader(CEntityFlags::Entity entities, CEntityFlags::ReadState state, int number, const QUrl &url)
     {
         if (state == CEntityFlags::ReadStarted) { return; } // just started

src/blackcore/webdataservices.h

@@ -67,6 +67,7 @@ namespace BlackCore
         class CVatsimDataFileReader;
         class CVatsimMetarReader;
         class CVatsimStatusFileReader;
+        class CVatsimServerFileReader;
     }
 
     namespace Db
@@ -581,6 +582,12 @@ namespace BlackCore
         //! VATSIM status file has been read
         void vatsimStatusFileRead(int lines);
 
+        //! VATSIM server file has been read
+        void vatsimServerFileRead(int lines);
+
+        //! Initialize and start VATSIM server file reader
+        void startVatsimServerFileReader();
+
         //! Read finished from reader
         void readFromSwiftReader(BlackMisc::Network::CEntityFlags::Entity entities, BlackMisc::Network::CEntityFlags::ReadState state, int number, const QUrl &url);
 
@@ -647,6 +654,7 @@ namespace BlackCore
         Vatsim::CVatsimBookingReader    *m_vatsimBookingReader    = nullptr;
         Vatsim::CVatsimDataFileReader   *m_vatsimDataFileReader   = nullptr;
         Vatsim::CVatsimMetarReader      *m_vatsimMetarReader      = nullptr;
+        Vatsim::CVatsimServerFileReader *m_vatsimServerFileReader = nullptr;
         Db::CIcaoDataReader             *m_icaoDataReader         = nullptr;
         Db::CModelDataReader            *m_modelDataReader        = nullptr;
         Db::CAirportDataReader          *m_airportDataReader      = nullptr;

src/blackcore/webreaderflags.h

@@ -33,11 +33,12 @@ namespace BlackCore
             VatsimDataReader       = 1 << 1,   //!< reader for VATSIM data
             VatsimMetarReader      = 1 << 2,   //!< reader for VATSIM metar data
             VatsimStatusReader     = 1 << 3,   //!< reader for VATSIM status file
-            IcaoDataReader       = 1 << 4,   //!< reader for ICAO data
+            VatsimServerFileReader = 1 << 4,   //!< reader for VATSIM server file
-            ModelReader          = 1 << 5,   //!< reader for model data such as liveries, models, etc
+            IcaoDataReader         = 1 << 5,   //!< reader for ICAO data
-            AirportReader        = 1 << 6,   //!< reader for airport list
+            ModelReader            = 1 << 6,   //!< reader for model data such as liveries, models, etc
-            DbInfoDataReader     = 1 << 7,   //!< DB info data (metdata, how many data, when updated)
+            AirportReader          = 1 << 7,   //!< reader for airport list
-            AllVatsimReaders     = VatsimBookingReader | VatsimDataReader | VatsimMetarReader | VatsimStatusReader,  //!< all VATSIM readers
+            DbInfoDataReader       = 1 << 8,   //!< DB info data (metdata, how many data, when updated)
+            AllVatsimReaders       = VatsimBookingReader | VatsimDataReader | VatsimMetarReader | VatsimStatusReader | VatsimServerFileReader,  //!< all VATSIM readers
             AllSwiftDbReaders      = IcaoDataReader      | ModelReader      | DbInfoDataReader  | AirportReader,       //!< all swift data
             AllReaders             = AllSwiftDbReaders   | AllVatsimReaders                                            //!< everything
         };