refs #452, integrated METAR reader in CWebDataServices

* renamed files without "_"
* added flags
* integrated URL into global reader settings
* added new parameter for connectVatsimDataSignals
* removed unused setting in settings / network.h

src/blackcore/airspace_monitor.cpp

@@ -60,9 +60,13 @@ namespace BlackCore
         this->connect(&m_interimPositionUpdateTimer, &QTimer::timeout, this, &CAirspaceMonitor::ps_sendInterimPositions);
 
         // AutoConnection: this should also avoid race conditions by updating the bookings
-        this->connect(this->m_webDataReader->getBookingReader(), &CVatsimBookingReader::dataRead, this, &CAirspaceMonitor::ps_receivedBookings);
-        this->connect(this->m_webDataReader->getDataFileReader(), &CVatsimDataFileReader::dataRead, this, &CAirspaceMonitor::ps_receivedDataFile);
-        this->connect(this->m_webDataReader->getMetarReader(), &CVatsimMetarReader::metarUpdated, this, &CAirspaceMonitor::ps_updateMetars);
+        Q_ASSERT_X(webDataReader, Q_FUNC_INFO, "Missing data reader");
+        if (webDataReader)
+        {
+            this->connect(webDataReader->getBookingReader(), &CVatsimBookingReader::dataRead, this, &CAirspaceMonitor::ps_receivedBookings);
+            this->connect(webDataReader->getDataFileReader(), &CVatsimDataFileReader::dataRead, this, &CAirspaceMonitor::ps_receivedDataFile);
+            this->connect(webDataReader->getMetarReader(), &CVatsimMetarReader::dataRead, this, &CAirspaceMonitor::ps_updateMetars);
+        }
 
         // Force snapshot in the main event loop
         this->connect(this->m_analyzer, &CAirspaceAnalyzer::airspaceAircraftSnapshot, this, &CAirspaceMonitor::airspaceAircraftSnapshot, Qt::QueuedConnection);

src/blackcore/airspace_monitor.h

@@ -14,7 +14,7 @@
 
 #include "blackcore/blackcoreexport.h"
 #include "blackcore/network.h"
-#include "blackcore/vatsim_metar_reader.h"
+#include "blackcore/vatsimmetarreader.h"
 #include "airspace_analyzer.h"
 #include "blackmisc/simulation/simulatedaircraftlist.h"
 #include "blackmisc/simulation/ownaircraftprovider.h"

src/blackcore/context_network_impl.cpp

@@ -14,7 +14,7 @@
 #include "context_simulator.h"
 #include "context_ownaircraft_impl.h"
 #include "network_vatlib.h"
-#include "vatsim_metar_reader.h"
+#include "vatsimmetarreader.h"
 #include "airspace_monitor.h"
 #include "webdataservices.h"
 #include "blackmisc/networkutils.h"

src/blackcore/settings/global_reader_settings.cpp

@@ -17,13 +17,19 @@ namespace BlackCore
             m_protocolIcaoReader("http"), m_serverIcaoReader("ubuntu12"),  m_baseUrlIcaoReader("vatrep/public"),
             m_protocolModelReader("http"), m_serverModelReader("ubuntu12"), m_baseUrlModelReader("vatrep/public"),
             m_bookingsUrl("http://vatbook.euroutepro.com/xml2.php"),
+            m_metarUrl("http://metar.vatsim.net/metar.php"),
             m_vatsimDataFileUrls({ "http://info.vroute.net/vatsim-data.txt" })
         { }
 
+        QString CGlobalReaderSettings::urlVatsimMetars() const
+        {
+            return m_metarUrl + "?id=all";
+        }
+
         const CGlobalReaderSettings &CGlobalReaderSettings::instance()
         {
             static const CGlobalReaderSettings rs;
             return rs;
         }
-    }
-}
+    } // ns
+} // ns

src/blackcore/settings/global_reader_settings.h

@@ -42,12 +42,18 @@ namespace BlackCore
             //! Model Reader server
             const QString &baseUrlModelReader() const { return m_baseUrlModelReader; }
 
-            //! URL to read bookings
+            //! URL to read VATSIM bookings
             const QString &bookingsUrl() const { return m_bookingsUrl; }
 
             //! VATSIM data file URLs
             const QStringList &vatsimDataFileUrls() const { return m_vatsimDataFileUrls; }
 
+            //! VATSIM METAR URL
+            const QString &baseUrlVatsimMetars() const { return m_metarUrl; }
+
+            //! VATSIM METAR URL (with query string)
+            QString urlVatsimMetars() const;
+
             //! Singleton
             static const CGlobalReaderSettings &instance();
 
@@ -65,6 +71,7 @@ namespace BlackCore
             QString m_serverModelReader;
             QString m_baseUrlModelReader;
             QString m_bookingsUrl;
+            QString m_metarUrl;
             QStringList m_vatsimDataFileUrls;
         };
     }

src/blackcore/settings/network.h

@@ -57,16 +57,6 @@ namespace BlackCore
                 }
             };
 
-            //! Booking service URL
-            struct BookingService : public CSettingTrait<QString>
-            {
-                //! \copydoc BlackCore::CSetting::key
-                static const char *key() { return "network/bookingservice"; }
-
-                //! \copydoc BlackCore::CSetting::defaultValue
-                static const QString &defaultValue() { static const QString dv("http://vatbook.euroutepro.com/xml2.php"); return dv; }
-            };
-
             //! DBus server address
             struct DBusServerAddress : public CSettingTrait<QString>
             {
@@ -76,8 +66,8 @@ namespace BlackCore
                 //! \copydoc BlackCore::CSetting::defaultValue
                 static const QString &defaultValue() { static const QString dv("session"); return dv; }
             };
-        }
-    }
-}
+        } // ns
+    } // ns
+} // ns
 
 #endif

src/blackcore/vatsimmetarreader.cpp

@@ -7,7 +7,7 @@
  * contained in the LICENSE file.
  */
 
-#include "vatsim_metar_reader.h"
+#include "vatsimmetarreader.h"
 #include "blackmisc/sequence.h"
 #include "blackmisc/logmessage.h"
 #include <QTextStream>
@@ -23,18 +23,18 @@ namespace BlackCore
         m_metarUrl(url)
     {
         this->m_networkManager = new QNetworkAccessManager(this);
-        this->connect(this->m_networkManager, &QNetworkAccessManager::finished, this, &CVatsimMetarReader::ps_decodeMetar);
-        this->connect(this->m_updateTimer, &QTimer::timeout, this, &CVatsimMetarReader::ps_readMetar);
+        this->connect(this->m_networkManager, &QNetworkAccessManager::finished, this, &CVatsimMetarReader::ps_decodeMetars);
+        this->connect(this->m_updateTimer, &QTimer::timeout, this, &CVatsimMetarReader::ps_readMetars);
     }
 
     void CVatsimMetarReader::readInBackgroundThread()
     {
         bool s = QMetaObject::invokeMethod(this, "ps_readMetar");
-        Q_ASSERT(s);
+        Q_ASSERT_X(s, Q_FUNC_INFO, "Cannot invoke");
         Q_UNUSED(s);
     }
 
-    void CVatsimMetarReader::ps_readMetar()
+    void CVatsimMetarReader::ps_readMetars()
     {
         this->threadAssertCheck();
         QUrl url(this->m_metarUrl);
@@ -44,7 +44,7 @@ namespace BlackCore
         this->m_networkManager->get(request);
     }
 
-    void CVatsimMetarReader::ps_decodeMetar(QNetworkReply *nwReplyPtr)
+    void CVatsimMetarReader::ps_decodeMetars(QNetworkReply *nwReplyPtr)
     {
         // wrap pointer, make sure any exit cleans up reply
         // required to use delete later as object is created in a different thread
@@ -56,7 +56,7 @@ namespace BlackCore
         if (this->isFinished())
         {
             CLogMessage(this).debug() << Q_FUNC_INFO;
-            CLogMessage(this).info("terminated metar decoding process"); // for users
+            CLogMessage(this).info("terminated METAR decoding process"); // for users
             return; // stop, terminate straight away, ending thread
         }
 
@@ -64,21 +64,14 @@ namespace BlackCore
         {
             QString metarData = nwReply->readAll();
             nwReply->close(); // close asap
-
             CMetarSet metars;
-            QFile outFile ("invalid_metar.txt");
-            if (!outFile.open(QIODevice::WriteOnly | QIODevice::Text)) { return; }
-            QTextStream out(&outFile);
 
+            QString invalidMetars;
+            int invalidLineCount = 0;
             QTextStream lineReader(&metarData);
             while (!lineReader.atEnd())
             {
                 QString line = lineReader.readLine();
-                if (line.startsWith("MSV 291130Z"))
-                {
-                    qDebug() << "now!";
-                }
-
                 CMetar metar = m_metarDecoder.decode(line);
                 if (metar != CMetar())
                 {
@@ -86,18 +79,24 @@ namespace BlackCore
                 }
                 else
                 {
-                    out << line << endl;
+                    invalidMetars += line;
+                    invalidMetars += "\n";
+                    invalidLineCount++;
                 }
             }
-            outFile.flush();
-            outFile.close();
 
-            emit metarUpdated(metars);
+            // I could use those for logging, etc.
+            Q_UNUSED(invalidMetars);
+            if (invalidLineCount > 0)
+            {
+                CLogMessage(this).warning("Reading METARs failed for %1 entries") << invalidLineCount;
+            }
+            emit dataRead(metars);
         }
         else
         {
             // network error
-            CLogMessage(this).warning("Reading METARS failed %1 %2") << nwReply->errorString() << nwReply->url().toString();
+            CLogMessage(this).warning("Reading METARs failed %1 %2") << nwReply->errorString() << nwReply->url().toString();
             nwReply->abort();
         }
     } // method

src/blackcore/vatsimmetarreader.h

@@ -37,16 +37,16 @@ namespace BlackCore
         void readInBackgroundThread();
 
     signals:
-        //! Bookings have been read and converted to BlackMisc::Aviation::CAtcStationList
-        void metarUpdated(const BlackMisc::Weather::CMetarSet &metars);
+        //! METARs have been read and converted to BlackMisc::Weather::CMetarSet
+        void dataRead(const BlackMisc::Weather::CMetarSet &metars);
 
     private slots:
-        //! Decode metar
+        //! Decode METARs
         //! \threadsafe
-        void ps_decodeMetar(QNetworkReply *nwReply);
+        void ps_decodeMetars(QNetworkReply *nwReply);
 
         //! Do reading
-        void ps_readMetar();
+        void ps_readMetars();
 
     private:
         QString m_metarUrl; //!< URL of the service

src/blackcore/webdataservices.cpp

@@ -9,10 +9,11 @@
 
 #include "blackcore/webdataservices.h"
 #include "blackcore/modeldatareader.h"
+#include "blackcore/icaodatareader.h"
 #include "blackcore/databasewriter.h"
-#include "vatsimbookingreader.h"
-#include "vatsimdatafilereader.h"
-#include "icaodatareader.h"
+#include "blackcore/vatsimbookingreader.h"
+#include "blackcore/vatsimdatafilereader.h"
+#include "blackcore/vatsimmetarreader.h"
 #include "settings/global_reader_settings.h"
 #include "blackmisc/logmessage.h"
 #include "blackmisc/worker.h"
@@ -22,6 +23,7 @@ using namespace BlackMisc;
 using namespace BlackMisc::Simulation;
 using namespace BlackMisc::Network;
 using namespace BlackMisc::Aviation;
+using namespace BlackMisc::Weather;
 
 namespace BlackCore
 {
@@ -33,7 +35,10 @@ namespace BlackCore
         this->initWriters();
     }
 
-    QList<QMetaObject::Connection> CWebDataServices::connectVatsimDataSignals(QObject *receiver, std::function<void(int)> bookingsRead, std::function<void(int)> dataFileRead)
+    QList<QMetaObject::Connection> CWebDataServices::connectVatsimDataSignals(QObject *receiver,
+            std::function<void(int)> bookingsRead,
+            std::function<void(int)> dataFileRead,
+            std::function<void(int)> metarsRead)
     {
         // bind does not allow to define connection type
         // so anything in its own thread will be sent with this thread affinity
@@ -43,16 +48,23 @@ namespace BlackCore
         if (m_readerFlags.testFlag(CWebReaderFlags::WebReaderFlags::VatsimBookingReader))
         {
             QMetaObject::Connection c1 = connect(this, &CWebDataServices::vatsimBookingsRead, receiver, bookingsRead);
-            Q_ASSERT_X(c1, Q_FUNC_INFO, "connect failed");
+            Q_ASSERT_X(c1, Q_FUNC_INFO, "connect failed (bookings)");
             cl.append(c1);
         }
 
         if (m_readerFlags.testFlag(CWebReaderFlags::WebReaderFlags::VatsimDataReader))
         {
             QMetaObject::Connection c2 = connect(this, &CWebDataServices::vatsimDataFileRead, receiver, dataFileRead);
-            Q_ASSERT_X(c2, Q_FUNC_INFO, "connect failed");
+            Q_ASSERT_X(c2, Q_FUNC_INFO, "connect failed (data)");
             cl.append(c2);
         }
+
+        if (m_readerFlags.testFlag(CWebReaderFlags::WebReaderFlags::VatsimMetarReader))
+        {
+            QMetaObject::Connection c3 = connect(this, &CWebDataServices::vatsimMetarsRead, receiver, metarsRead);
+            Q_ASSERT_X(c3, Q_FUNC_INFO, "connect failed (METARs)");
+            cl.append(c3);
+        }
         return cl;
     }
 
@@ -66,17 +78,17 @@ namespace BlackCore
 
         if (m_readerFlags.testFlag(CWebReaderFlags::WebReaderFlags::IcaoDataReader))
         {
-            Q_ASSERT_X(this->m_icaoDataReader, Q_FUNC_INFO, "Missing reader");
+            Q_ASSERT_X(this->m_icaoDataReader, Q_FUNC_INFO, "Missing reader ICAO");
             QMetaObject::Connection con = connect(this->m_icaoDataReader, &CIcaoDataReader::readData, receiver, dataRead);
-            Q_ASSERT_X(con, Q_FUNC_INFO, "connect failed");
+            Q_ASSERT_X(con, Q_FUNC_INFO, "connect failed ICAO");
             cl.append(con);
         }
 
         if (m_readerFlags.testFlag(CWebReaderFlags::WebReaderFlags::ModelReader))
         {
-            Q_ASSERT_X(this->m_modelDataReader, Q_FUNC_INFO, "Missing reader");
+            Q_ASSERT_X(this->m_modelDataReader, Q_FUNC_INFO, "Missing reader models");
             QMetaObject::Connection con = connect(this->m_modelDataReader, &CModelDataReader::readData, receiver, dataRead);
-            Q_ASSERT_X(con, Q_FUNC_INFO, "connect failed");
+            Q_ASSERT_X(con, Q_FUNC_INFO, "connect failed models");
             cl.append(con);
         }
         return cl;
@@ -353,9 +365,9 @@ namespace BlackCore
         {
             this->m_vatsimBookingReader = new CVatsimBookingReader(this, CGlobalReaderSettings::instance().bookingsUrl());
             bool c = connect(this->m_vatsimBookingReader, &CVatsimBookingReader::dataRead, this, &CWebDataServices::ps_receivedBookings);
-            Q_ASSERT_X(c, Q_FUNC_INFO, "VATSIM reader signals");
+            Q_ASSERT_X(c, Q_FUNC_INFO, "VATSIM booking reader signals");
             Q_UNUSED(c);
-            this->m_vatsimBookingReader->start();
+            this->m_vatsimBookingReader->start(QThread::LowPriority);
             this->m_vatsimBookingReader->setInterval(3 * 60 * 1000);
         }
 
@@ -364,13 +376,24 @@ namespace BlackCore
         {
             this->m_vatsimDataFileReader = new CVatsimDataFileReader(this, CGlobalReaderSettings::instance().vatsimDataFileUrls());
             bool c = connect(this->m_vatsimDataFileReader, &CVatsimDataFileReader::dataRead, this, &CWebDataServices::ps_dataFileRead);
-            Q_ASSERT_X(c, Q_FUNC_INFO, "VATSIM reader signals");
+            Q_ASSERT_X(c, Q_FUNC_INFO, "VATSIM data reader signals");
             Q_UNUSED(c);
-            this->m_vatsimDataFileReader->start();
+            this->m_vatsimDataFileReader->start(QThread::LowPriority);
             this->m_vatsimDataFileReader->setInterval(90 * 1000);
         }
 
-        // 3. ICAO data reader
+        // 3. VATSIM metar data
+        if (flags.testFlag(CWebReaderFlags::WebReaderFlags::VatsimMetarReader))
+        {
+            this->m_vatsimMetarReader = new CVatsimMetarReader(this, CGlobalReaderSettings::instance().urlVatsimMetars());
+            bool c = connect(this->m_vatsimMetarReader, &CVatsimMetarReader::dataRead, this, &CWebDataServices::ps_receivedMetars);
+            Q_ASSERT_X(c, Q_FUNC_INFO, "VATSIM METAR reader signals");
+            Q_UNUSED(c);
+            this->m_vatsimMetarReader->start(QThread::LowPriority);
+            this->m_vatsimMetarReader->setInterval(90 * 1000);
+        }
+
+        // 4. ICAO data reader
         if (flags.testFlag(CWebReaderFlags::WebReaderFlags::IcaoDataReader))
         {
             bool c;
@@ -381,7 +404,7 @@ namespace BlackCore
             this->m_icaoDataReader->start();
         }
 
-        // 4. Model reader
+        // 5. Model reader
         if (flags.testFlag(CWebReaderFlags::WebReaderFlags::ModelReader))
         {
             this->m_modelDataReader = new CModelDataReader(this, CGlobalReaderSettings::instance().protocolModelReader(), CGlobalReaderSettings::instance().serverModelReader(), CGlobalReaderSettings::instance().baseUrlModelReader());
@@ -403,10 +426,16 @@ namespace BlackCore
 
     void CWebDataServices::ps_receivedBookings(const CAtcStationList &stations)
     {
-        CLogMessage(this).info("Read bookings from network");
+        CLogMessage(this).info("Read %1 ATC bookings from network") << stations.size();
         emit vatsimBookingsRead(stations.size());
     }
 
+    void CWebDataServices::ps_receivedMetars(const CMetarSet &metars)
+    {
+        CLogMessage(this).info("Read %1 METARs") << metars.size();
+        emit vatsimDataFileRead(metars.size());
+    }
+
     void CWebDataServices::ps_dataFileRead(int lines)
     {
         CLogMessage(this).info("Read VATSIM data file, %1 lines") << lines;

src/blackcore/webdataservices.h

@@ -22,6 +22,7 @@
 #include "blackmisc/network/voicecapabilities.h"
 #include "blackmisc/network/webdataservicesprovider.h"
 #include "blackmisc/simulation/distributorlist.h"
+#include "blackmisc/weather/metarset.h"
 #include "blackmisc/logcategorylist.h"
 #include "blackmisc/countrylist.h"
 #include <QObject>
@@ -30,6 +31,7 @@ namespace BlackCore
 {
     class CVatsimBookingReader;
     class CVatsimDataFileReader;
+    class CVatsimMetarReader;
     class CIcaoDataReader;
     class CModelDataReader;
     class CDatabaseWriter;
@@ -60,6 +62,9 @@ namespace BlackCore
         //! Data file reader
         CVatsimDataFileReader *getDataFileReader() const { return m_vatsimDataFileReader; }
 
+        //! Metar reader
+        CVatsimMetarReader *getMetarReader() const { return m_vatsimMetarReader; }
+
         //! Reader flags
         CWebReaderFlags::WebReader getReaderFlags() const { return m_readerFlags; }
 
@@ -72,7 +77,7 @@ namespace BlackCore
         //! \ingroup webdatareaderprovider
         virtual QList<QMetaObject::Connection> connectVatsimDataSignals(
             QObject *receiver,
-            std::function<void(int)> bookingsRead, std::function<void(int)> dataFileRead) override;
+            std::function<void(int)> bookingsRead, std::function<void(int)> dataFileRead, std::function<void(int)> metarsRead) override;
 
         //! \copydoc IWebDataReaderProvider::connectSwiftDatabaseSignals
         //! \ingroup webdatareaderprovider
@@ -227,6 +232,9 @@ namespace BlackCore
         //! Bookings read
         void vatsimBookingsRead(int number);
 
+        //! METARs read
+        void vatsimMetarsRead(int number);
+
         //! Combined read signal
         void readSwiftDbData(BlackMisc::Network::CDbFlags::Entity entity, BlackMisc::Network::CDbFlags::ReadState state, int number);
 
@@ -237,6 +245,9 @@ namespace BlackCore
         //! ATC bookings received
         void ps_receivedBookings(const BlackMisc::Aviation::CAtcStationList &bookedStations);
 
+        //! Received METAR data
+        void ps_receivedMetars(const BlackMisc::Weather::CMetarSet &metars);
+
         //! Data file has been read
         void ps_dataFileRead(int lines);
 
@@ -255,6 +266,7 @@ namespace BlackCore
         // for reading XML and VATSIM data files
         CVatsimBookingReader  *m_vatsimBookingReader  = nullptr;
         CVatsimDataFileReader *m_vatsimDataFileReader = nullptr;
+        CVatsimMetarReader    *m_vatsimMetarReader    = nullptr;
         CIcaoDataReader       *m_icaoDataReader       = nullptr;
         CModelDataReader      *m_modelDataReader      = nullptr;
 

src/blackcore/webreaderflags.h

@@ -31,9 +31,10 @@ namespace BlackCore
             None = 0,  ///< no reader at all
             VatsimBookingReader  = 1 << 0,  ///< reader for VATSIM booking data
             VatsimDataReader     = 1 << 1,  ///< reader for VATSIM data
-            IcaoDataReader       = 1 << 2,  ///< reader for ICAO data
-            ModelReader          = 1 << 3,  ///< reader for model data such as liveries, models, ..
-            AllVatsimReaders     = VatsimBookingReader | VatsimDataReader, ///< all readers
+            VatsimMetarReader    = 1 << 2,  ///< reader for VATSIM metar data
+            IcaoDataReader       = 1 << 3,  ///< reader for ICAO data
+            ModelReader          = 1 << 4,  ///< reader for model data such as liveries, models, ..
+            AllVatsimReaders     = VatsimBookingReader | VatsimDataReader | VatsimMetarReader, ///< all readers
             AllSwiftDbReaders    = IcaoDataReader | ModelReader,           ///< all swift data
             AllReaders           = AllSwiftDbReaders | AllVatsimReaders    ///< everything
         };

src/blackmisc/network/webdataservicesprovider.h

@@ -173,9 +173,13 @@ namespace BlackMisc
             //! Connect signals to slot receiver. As the interface is no QObject, slots can not be connected directly.
             //! In order to disconnect a list of connections is provided, which have to be disconnected manually.
             //! \note receiver is required for connection type
-            virtual QList<QMetaObject::Connection> connectVatsimDataSignals(QObject *receiver, std::function<void(int)> bookingsRead, std::function<void(int)> dataFileRead) = 0;
+            //! \todo currently still used, will be replaced by abstract connection by entity
+            virtual QList<QMetaObject::Connection> connectVatsimDataSignals(
+                QObject *receiver,
+                std::function<void(int)> bookingsRead, std::function<void(int)> dataFileRead, std::function<void(int)> metarsRead) = 0;
 
             //! Relay signals for swift data
+            //! \todo currently still used, will be replaced by abstract connection by entity
             virtual QList<QMetaObject::Connection> connectSwiftDatabaseSignals(
                 QObject *receiver,
                 std::function<void(BlackMisc::Network::CDbFlags::Entity, BlackMisc::Network::CDbFlags::ReadState, int)> dataRead) = 0;