refs #418, ICAO data for reader

* value objects and respective lists
* adjusted CAircraftIcao to use new classes
* some smaller fixes, Doxygen
* reader for values from database JSON values
  http://vatrep.vatsim-germany.org/service/allaircrafticao.php?rows=10000&sord=asc
* Related: Slightly improved signals for VATSIM reader

src/blackcore/context_network_impl.cpp

@@ -16,7 +16,8 @@
 #include "network_vatlib.h"
 #include "vatsimbookingreader.h"
 #include "vatsimdatafilereader.h"
-
+#include "icaodatareader.h"
+#include "airspace_monitor.h"
 #include "blackmisc/networkutils.h"
 #include "blackmisc/aviation/atcstationlist.h"
 #include "blackmisc/logmessage.h"
@@ -66,12 +67,21 @@ namespace BlackCore
         this->m_vatsimDataFileReader->readInBackgroundThread(); // first read
         this->m_vatsimDataFileReader->setInterval(90 * 1000);
 
-        // 4. Update timer for data (network data such as frequency)
+        // 4. ICAO data reader
+        this->m_icaoDataReader = new CIcaoDataReader(this,
+                "http://vatrep.vatsim-germany.org/service/allaircrafticao.php?rows=20000&sord=asc",
+                "http://vatrep.vatsim-germany.org/service/allairlineicao.php?rows=20000&sord=asc");
+        connect(this->m_icaoDataReader, &CIcaoDataReader::readAircraftIcaoCodes, this, &CContextNetwork::ps_readAircraftIcaoCodes);
+        connect(this->m_icaoDataReader, &CIcaoDataReader::readAirlinesIcaoCodes, this, &CContextNetwork::ps_readAirlinesIcaoCodes);
+        this->m_icaoDataReader->start();
+        this->m_icaoDataReader->readInBackgroundThread();
+
+        // 5. Update timer for data (network data such as frequency)
         this->m_dataUpdateTimer = new QTimer(this);
         connect(this->m_dataUpdateTimer, &QTimer::timeout, this, &CContextNetwork::requestDataUpdates);
         this->m_dataUpdateTimer->start(30 * 1000);
 
-        // 5. Airspace contents
+        // 6. Airspace contents
         Q_ASSERT_X(this->getRuntime()->getCContextOwnAircraft(), Q_FUNC_INFO, "this and own aircraft context must be local");
         this->m_airspace = new CAirspaceMonitor(this, this->getRuntime()->getCContextOwnAircraft(), this->m_network, this->m_vatsimBookingReader, this->m_vatsimDataFileReader);
         connect(this->m_airspace, &CAirspaceMonitor::changedAtcStationsOnline, this, &CContextNetwork::changedAtcStationsOnline);
@@ -450,13 +460,25 @@ namespace BlackCore
         m_airspace->analyzer()->setSimulatorRenderRestrictionsChanged(restricted, maxAircraft, maxRenderedDistance, maxRenderedBoundary);
     }
 
-    void CContextNetwork::ps_dataFileRead()
+    void CContextNetwork::ps_dataFileRead(int lines)
     {
         if (this->isDebugEnabled()) { CLogMessage(this, CLogCategory::contextSlot()).debug() << Q_FUNC_INFO; }
-        CLogMessage(this).info("Read VATSIM data file");
+        CLogMessage(this).info("Read VATSIM data file, %1 lines") << lines;
         emit vatsimDataFileRead();
     }
 
+    void CContextNetwork::ps_readAircraftIcaoCodes(int number)
+    {
+        if (this->isDebugEnabled()) { CLogMessage(this, CLogCategory::contextSlot()).debug() << Q_FUNC_INFO; }
+        CLogMessage(this).info("Read %1 aircraft ICAO codes") << number;
+    }
+
+    void CContextNetwork::ps_readAirlinesIcaoCodes(int number)
+    {
+        if (this->isDebugEnabled()) { CLogMessage(this, CLogCategory::contextSlot()).debug() << Q_FUNC_INFO; }
+        CLogMessage(this).info("Read %1 airline ICAO codes") << number;
+    }
+
     void CContextNetwork::ps_checkForSupervisiorTextMessage(const CTextMessageList &messages)
     {
         if (messages.containsPrivateMessages())

src/blackcore/context_network_impl.h

@@ -16,24 +16,25 @@
 #include "blackcore/context_network.h"
 #include "blackcore/context_settings.h"
 #include "blackcore/context_runtime.h"
-#include "blackmisc/simulation/remoteaircraftprovider.h"
 #include "blackcore/dbus_server.h"
 #include "blackcore/network.h"
-#include "blackcore/airspace_monitor.h"
+#include "blackmisc/simulation/remoteaircraftprovider.h"
 #include "blackmisc/aviation/atcstationlist.h"
+#include "blackmisc/aviation/aircraftsituationlist.h"
 #include "blackmisc/setnetwork.h"
 #include "blackmisc/network/clientlist.h"
 #include "blackmisc/digestsignal.h"
 #include "blackmisc/logmessage.h"
-#include "blackmisc/aviation/aircraftsituationlist.h"
 
 #include <QMap>
 #include <QTimer>
 
 namespace BlackCore
 {
+    class CAirspaceMonitor;
     class CVatsimBookingReader;
     class CVatsimDataFileReader;
+    class CIcaoDataReader;
 
     //! Network context implementation
     class BLACKCORE_EXPORT CContextNetwork :
@@ -97,7 +98,7 @@ namespace BlackCore
         virtual BlackMisc::Simulation::CAirspaceAircraftSnapshot getLatestAirspaceAircraftSnapshot() const override;
 
         //! Network library
-        INetwork* network() const { return m_network; }
+        INetwork *network() const { return m_network; }
 
     public slots:
         //! \copydoc IContextNetwork::updateAircraftEnabled
@@ -248,6 +249,7 @@ namespace BlackCore
         // for reading XML and VATSIM data files
         CVatsimBookingReader  *m_vatsimBookingReader  = nullptr;
         CVatsimDataFileReader *m_vatsimDataFileReader = nullptr;
+        CIcaoDataReader       *m_icaoDataReader       = nullptr;
         QTimer *m_dataUpdateTimer = nullptr; //!< general updates such as ATIS, frequencies, see requestDataUpdates()
 
         //! Get network settings
@@ -266,7 +268,13 @@ namespace BlackCore
         void ps_receivedBookings(const BlackMisc::Aviation::CAtcStationList &bookedStations);
 
         //! Data file has been read
-        void ps_dataFileRead();
+        void ps_dataFileRead(int lines);
+
+        //! Read ICAO codes
+        void ps_readAircraftIcaoCodes(int number);
+
+        //! Read ICAO codes
+        void ps_readAirlinesIcaoCodes(int number);
 
         //! Check if a supervisor message was received
         void ps_checkForSupervisiorTextMessage(const BlackMisc::Network::CTextMessageList &messages);
@@ -279,6 +287,6 @@ namespace BlackCore
         void ps_simulatorRenderRestrictionsChanged(bool restricted, int maxAircraft, const BlackMisc::PhysicalQuantities::CLength &maxRenderedDistance, const BlackMisc::PhysicalQuantities::CLength &maxRenderedBoundary);
 
     };
-}
+} // ns
 
 #endif // guard

src/blackcore/icaodatareader.cpp

@@ -0,0 +1,135 @@
+/* Copyright (C) 2015
+ * swift project Community / Contributors
+ *
+ * This file is part of swift project. It is subject to the license terms in the LICENSE file found in the top-level
+ * directory of this distribution and at http://www.swift-project.org/license.html. No part of swift project,
+ * including this file, may be copied, modified, propagated, or distributed except according to the terms
+ * contained in the LICENSE file.
+ */
+
+#include "blackmisc/sequence.h"
+#include "blackmisc/logmessage.h"
+#include "icaodatareader.h"
+
+#include <QRegularExpression>
+
+using namespace BlackMisc;
+using namespace BlackMisc::Aviation;
+
+namespace BlackCore
+{
+    CIcaoDataReader::CIcaoDataReader(QObject *owner, const QString &aircraftIcaoUrl, const QString &airlineIcaoUrl) :
+        CThreadedReader(owner, "CIcaoDataReader"),
+        m_urlAircraftIcao(aircraftIcaoUrl), m_urlAirlineIcao(airlineIcaoUrl)
+    {
+        this->m_networkManagerAircraft = new QNetworkAccessManager(this);
+        this->m_networkManagerAirlines = new QNetworkAccessManager(this);
+
+        this->connect(this->m_networkManagerAircraft, &QNetworkAccessManager::finished, this, &CIcaoDataReader::ps_parseAircraftIcaoData);
+        this->connect(this->m_networkManagerAirlines, &QNetworkAccessManager::finished, this, &CIcaoDataReader::ps_parseAirlineIcaoData);
+    }
+
+    CAircraftIcaoCodeList CIcaoDataReader::getAircraftIcaoCodes() const
+    {
+        QReadLocker l(&m_lockAircraft);
+        return m_aircraftIcaos;
+    }
+
+    CAirlineIcaoCodeList CIcaoDataReader::getAirlineIcaoCodes() const
+    {
+        QReadLocker l(&m_lockAirline);
+        return m_airlineIcaos;
+    }
+
+    void CIcaoDataReader::readInBackgroundThread()
+    {
+        bool s = QMetaObject::invokeMethod(this, "ps_read");
+        Q_ASSERT_X(s, Q_FUNC_INFO, "Invoke failed");
+        Q_UNUSED(s);
+    }
+
+    void CIcaoDataReader::ps_read()
+    {
+        this->threadAssertCheck();
+        Q_ASSERT(this->m_networkManagerAircraft);
+        Q_ASSERT(this->m_networkManagerAirlines);
+        Q_ASSERT(!m_urlAircraftIcao.isEmpty());
+        Q_ASSERT(!m_urlAirlineIcao.isEmpty());
+        QNetworkRequest requestAircraft(m_urlAircraftIcao);
+        QNetworkRequest requestAirline(m_urlAirlineIcao);
+        this->m_networkManagerAircraft->get(requestAircraft);
+        this->m_networkManagerAirlines->get(requestAirline);
+    }
+
+    QJsonArray CIcaoDataReader::splitReplyIntoArray(QNetworkReply *nwReply) const
+    {
+        this->threadAssertCheck();
+        QJsonArray array;
+        if (this->isFinished())
+        {
+            CLogMessage(this).debug() << Q_FUNC_INFO;
+            CLogMessage(this).info("Terminated ICAO data parsing process"); // for users
+            return array; // stop, terminate straight away, ending thread
+        }
+
+        if (nwReply->error() == QNetworkReply::NoError)
+        {
+            const QString dataFileData = nwReply->readAll();
+            nwReply->close(); // close asap
+            if (dataFileData.isEmpty()) { return array; }
+
+            QJsonDocument jsonResponse = QJsonDocument::fromJson(dataFileData.toUtf8());
+            QJsonObject jsonObject = jsonResponse.object();
+            QJsonArray jsonArray = jsonObject["rows"].toArray();
+            return jsonArray;
+        }
+        CLogMessage(this).warning("Reading data failed %1 %2") << nwReply->errorString() << nwReply->url().toString();
+        nwReply->abort();
+        return array;
+    }
+
+    bool BlackCore::CIcaoDataReader::checkIfFinished() const
+    {
+        if (!this->isFinished()) { return false; }
+        CLogMessage(this).debug() << Q_FUNC_INFO;
+        CLogMessage(this).info("Terminated ICAO data parsing process"); // for users
+        return true;
+    }
+
+    void CIcaoDataReader::ps_parseAircraftIcaoData(QNetworkReply *nwReplyPtr)
+    {
+        // wrap pointer, make sure any exit cleans up reply
+        // required to use delete later as object is created in a different thread
+        QScopedPointer<QNetworkReply, QScopedPointerDeleteLater> nwReply(nwReplyPtr);
+        QJsonArray array = this->splitReplyIntoArray(nwReply.data());
+        if (array.isEmpty()) { return; }
+        CAircraftIcaoCodeList codes = CAircraftIcaoCodeList::fromDatabaseJson(array);
+
+        // this part needs to be synchronized
+        int n;
+        {
+            QWriteLocker wl(&this->m_lockAircraft);
+            this->m_aircraftIcaos = codes;
+            n = codes.size();
+        }
+        emit readAircraftIcaoCodes(n);
+    }
+
+    void CIcaoDataReader::ps_parseAirlineIcaoData(QNetworkReply *nwReplyPtr)
+    {
+        QScopedPointer<QNetworkReply, QScopedPointerDeleteLater> nwReply(nwReplyPtr);
+        QJsonArray array = this->splitReplyIntoArray(nwReply.data());
+        if (array.isEmpty()) { return; }
+        CAirlineIcaoCodeList codes = CAirlineIcaoCodeList::fromDatabaseJson(array);
+
+        // this part needs to be synchronized
+        int n;
+        {
+            QWriteLocker wl(&this->m_lockAirline);
+            this->m_airlineIcaos = codes;
+            n = codes.size();
+        }
+        emit readAirlinesIcaoCodes(n);
+    }
+
+} // namespace

src/blackcore/icaodatareader.h

@@ -0,0 +1,83 @@
+/* Copyright (C) 2015
+ * swift project Community / Contributors
+ *
+ * This file is part of swift project. It is subject to the license terms in the LICENSE file found in the top-level
+ * directory of this distribution and at http://www.swift-project.org/license.html. No part of swift project,
+ * including this file, may be copied, modified, propagated, or distributed except according to the terms
+ * contained in the LICENSE file.
+ */
+
+//! \file
+
+#ifndef BLACKCORE_ICAODATAREADER_H
+#define BLACKCORE_ICAODATAREADER_H
+
+#include "blackcoreexport.h"
+#include "blackmisc/aviation/aircrafticaocodelist.h"
+#include "blackmisc/aviation/airlineicaocodelist.h"
+#include "blackmisc/threadedreader.h"
+
+#include <QObject>
+#include <QTimer>
+#include <QNetworkReply>
+#include <QReadWriteLock>
+
+namespace BlackCore
+{
+    //! Read bookings from VATSIM
+    class BLACKCORE_EXPORT CIcaoDataReader : public BlackMisc::CThreadedReader
+    {
+        Q_OBJECT
+
+    public:
+        //! Constructor
+        explicit CIcaoDataReader(QObject *owner, const QString &aircraftIcaoUrl, const QString &airlineIcaoUrl);
+
+        //! Get aircraft ICAO information
+        //! \threadsafe
+        BlackMisc::Aviation::CAircraftIcaoCodeList getAircraftIcaoCodes() const;
+
+        //! Get airline ICAO information
+        //! \threadsafe
+        BlackMisc::Aviation::CAirlineIcaoCodeList getAirlineIcaoCodes() const;
+
+        //! Start reading in own thread
+        void readInBackgroundThread();
+
+    signals:
+        //! Codes have been read
+        void readAircraftIcaoCodes(int number);
+
+        //! Codes have been read
+        void readAirlinesIcaoCodes(int number);
+
+    private slots:
+        //! Aircraft have been read
+        void ps_parseAircraftIcaoData(QNetworkReply *nwReply);
+
+        //! Airlines have been read
+        void ps_parseAirlineIcaoData(QNetworkReply *nwReply);
+
+        //! Read / re-read data file
+        void ps_read();
+
+    private:
+        QNetworkAccessManager *m_networkManagerAircraft = nullptr;
+        QNetworkAccessManager *m_networkManagerAirlines = nullptr;
+        BlackMisc::Aviation::CAircraftIcaoCodeList m_aircraftIcaos;
+        BlackMisc::Aviation::CAirlineIcaoCodeList m_airlineIcaos;
+        QString m_urlAircraftIcao;
+        QString m_urlAirlineIcao;
+
+        mutable QReadWriteLock m_lockAirline;
+        mutable QReadWriteLock m_lockAircraft;
+
+        //! Check if terminated or error, otherwise split into array of objects
+        QJsonArray splitReplyIntoArray(QNetworkReply *nwReply) const;
+
+        //! Check if thread has been finished
+        bool checkIfFinished() const;
+    };
+}
+
+#endif // guard

src/blackcore/vatsimbookingreader.h

@@ -23,9 +23,7 @@
 
 namespace BlackCore
 {
-    /*!
-     * Read bookings from VATSIM
-     */
+    //! Read bookings from VATSIM
     class BLACKCORE_EXPORT CVatsimBookingReader : public BlackMisc::CThreadedReader
     {
         Q_OBJECT
@@ -46,10 +44,10 @@ namespace BlackCore
         void ps_read();
 
     private:
-        QString m_serviceUrl; /*!< URL of the service */
+        QString m_serviceUrl; //!< URL of the service
         QNetworkAccessManager *m_networkManager = nullptr;
 
-     signals:
+    signals:
         //! Bookings have been read and converted to BlackMisc::Aviation::CAtcStationList
         void dataRead(const BlackMisc::Aviation::CAtcStationList &bookedStations);
     };

src/blackcore/vatsimdatafilereader.cpp

@@ -123,8 +123,8 @@ namespace BlackCore
     CUserList CVatsimDataFileReader::getUsersForCallsigns(const CCallsignSet &callsigns)
     {
         CUserList users;
-        if (callsigns.isEmpty()) return users;
-        foreach(CCallsign callsign, callsigns)
+        if (callsigns.isEmpty()) { return users; }
+        for (const CCallsign &callsign : callsigns)
         {
             users.push_back(this->getPilotsForCallsign(callsign));
             users.push_back(this->getControllersForCallsign(callsign));
@@ -135,7 +135,7 @@ namespace BlackCore
     void CVatsimDataFileReader::readInBackgroundThread()
     {
         bool s = QMetaObject::invokeMethod(this, "ps_read");
-        Q_ASSERT(s);
+        Q_ASSERT_X(s, Q_FUNC_INFO, "Invoke failed");
         Q_UNUSED(s);
     }
 
@@ -181,7 +181,7 @@ namespace BlackCore
 
             if (dataFileData.isEmpty()) return;
             QStringList lines = dataFileData.split(QRegExp("[\r\n]"), QString::SkipEmptyParts);
-            if (lines.isEmpty()) return;
+            if (lines.isEmpty()) { return; }
 
             // build on local vars for thread safety
             CServerList     voiceServers;
@@ -198,7 +198,7 @@ namespace BlackCore
                 if (this->isFinished())
                 {
                     CLogMessage(this).debug() << Q_FUNC_INFO;
-                    CLogMessage(this).info("terminated booking parsing process"); // for users
+                    CLogMessage(this).info("Terminated booking parsing process"); // for users
                     return; // stop, terminate straight away, ending thread
                 }
 
@@ -290,7 +290,7 @@ namespace BlackCore
                         }
                         else
                         {
-                            Q_ASSERT_X(false, "CVatsimDataFileReader::loadFinished", "Wrong client type");
+                            Q_ASSERT_X(false, Q_FUNC_INFO, "Wrong client type");
                         }
                     }
                     break;
@@ -335,18 +335,19 @@ namespace BlackCore
 
                 } // switch section
 
-                // this part needs to be synchronized
-                {
-                    QWriteLocker wl(&this->m_lock);
-                    this->setUpdateTimestamp(updateTimestampFromFile);
-                    this->m_aircraft = aircraft;
-                    this->m_atcStations = atcStations;
-                    this->m_voiceServers = voiceServers;
-                    this->m_fsdServers = fsdServers;
-                    this->m_voiceCapabilities = voiceCapabilities;
-                }
             } // for each line
 
+            // this part needs to be synchronized
+            {
+                QWriteLocker wl(&this->m_lock);
+                this->setUpdateTimestamp(updateTimestampFromFile);
+                this->m_aircraft = aircraft;
+                this->m_atcStations = atcStations;
+                this->m_voiceServers = voiceServers;
+                this->m_fsdServers = fsdServers;
+                this->m_voiceCapabilities = voiceCapabilities;
+            }
+
             // warnings, if required
             if (!illegalIcaoCodes.isEmpty())
             {
@@ -354,7 +355,7 @@ namespace BlackCore
             }
 
             // data read finished
-            emit this->dataRead();
+            emit this->dataRead(lines.count());
         }
         else
         {

src/blackcore/vatsimdatafilereader.h

@@ -135,7 +135,7 @@ namespace BlackCore
 
     signals:
         //! Data have been read
-        void dataRead();
+        void dataRead(int lines);
     };
 }