refs #748 Support database format in CAirportDataReader

* CAirport is now database object
* CAirportList is now datastore object list
* CAirportDataReader adjustments

src/blackcore/db/airportdatareader.cpp

@@ -27,13 +27,6 @@ namespace BlackCore
             // void
         }
 
-        void CAirportDataReader::readInBackgroundThread()
-        {
-            bool s = QMetaObject::invokeMethod(this, "ps_readAirports");
-            Q_ASSERT_X(s, Q_FUNC_INFO, "Cannot invoke ");
-            Q_UNUSED(s);
-        }
-
         BlackMisc::Aviation::CAirportList CAirportDataReader::getAirports() const
         {
             return m_airportCache.get();
@@ -61,79 +54,74 @@ namespace BlackCore
 
         bool CAirportDataReader::hasChangedUrl(CEntityFlags::Entity entity) const
         {
-            // TODO
+            Q_UNUSED(entity);
-            return false;
+            return CDatabaseReader::isChangedUrl(this->m_readerUrlCache.get(), this->getAirportsUrl());
         }
 
         CUrl CAirportDataReader::getAirportsUrl() const
         {
-            const CUrl url(sApp->getGlobalSetup().getSwiftAirportUrls().getRandomWorkingUrl());
+            return sApp->getGlobalSetup().getDbRootDirectoryUrl().withAppendedPath("service/jsonairport.php");
-            return url;
         }
 
         void CAirportDataReader::ps_parseAirportData(QNetworkReply *nwReply)
         {
-            QJsonParseError error;
+            CDatabaseReader::JsonDatastoreResponse res = this->setStatusAndTransformReplyIntoDatastoreResponse(nwReply);
-            QByteArray data = nwReply->readAll();
+            if (res.hasErrorMessage())
-            QJsonDocument document = QJsonDocument::fromJson(data, &error);
-            if (error.error != QJsonParseError::NoError)
             {
-                CLogMessage(this).error("Error parsing airport list from JSON (%1)") << error.errorString();
+                CLogMessage::preformatted(res.lastWarningOrAbove());
-                return;
+                emit dataRead(CEntityFlags::AirportEntity, CEntityFlags::ReadFailed, 0);
-            }
-
-            QJsonArray array = document.array();
-            if (array.isEmpty())
-            {
-                CLogMessage(this).error("Error parsing airport list from JSON (document is not an array)");
                 return;
             }
 
             CAirportList airports;
-            airports.convertFromDatabaseJson(array);
-            quint64 timestamp = lastModifiedMsSinceEpoch(nwReply);
 
+            if (res.isRestricted())
             {
-                QWriteLocker wl(&this->m_lock);
+                airports = this->getAirports();
-                m_airportCache.set(airports, timestamp);
+                CAirportList updates;
+                updates.convertFromDatabaseJson(res);
+                airports.replaceOrAddObjectsByKey(updates);
             }
+            else
+            {
+                airports.convertFromDatabaseJson(res);
+            }
+
+            int size = airports.size();
+            qint64 timestamp = lastModifiedMsSinceEpoch(nwReply);
+            if (size > 0 && timestamp < 0)
+            {
+                CLogMessage(this).error("No timestamp in airport list, setting to last modified value");
+                timestamp = lastModifiedMsSinceEpoch(nwReply);
+            }
+
+            m_airportCache.set(airports, timestamp);
 
             emit dataRead(CEntityFlags::AirportEntity, CEntityFlags::ReadFinished, airports.size());
         }
 
-        void CAirportDataReader::ps_parseAirportHeader(QNetworkReply *nwReply)
+        void CAirportDataReader::ps_read(CEntityFlags::Entity entity, const QDateTime &newerThan)
         {
             this->threadAssertCheck();
-            m_lastModified = lastModifiedMsSinceEpoch(nwReply);
+            if (this->isAbandoned()) { return; }
-            ps_readAirports();
-        }
 
-        void CAirportDataReader::ps_readAirports()
+            if (entity.testFlag(CEntityFlags::AirportEntity))
             {
-            this->threadAssertCheck();
+                CUrl url = getAirportsUrl();
-            Q_ASSERT_X(sApp, Q_FUNC_INFO, "No Application");
+                if (!url.isEmpty())
-
-            CFailoverUrlList urls(sApp->getGlobalSetup().getSwiftAirportUrls());
-            const CUrl url(urls.obtainNextWorkingUrl(true));
-            if (url.isEmpty()) { return; }
-
-            if (0 == m_lastModified) {
-                sApp->headerFromNetwork(url, { this, &CAirportDataReader::ps_parseAirportHeader });
-                return;
-            }
-
-            m_airportCache.synchronize();
-
-            int size = m_airportCache.get().size();
-            if (size > 0 &&
-                    m_airportCache.getAvailableTimestamp().toMSecsSinceEpoch() >= static_cast<qint64>(m_lastModified)) // cache is up-to-date
                 {
-                CLogMessage(this).info("Loaded %1 airports from cache") << m_airportCache.get().size();
+                    if (!newerThan.isNull())
-                emit dataRead(CEntityFlags::AirportEntity, CEntityFlags::ReadFinished, size);
+                    {
+                        const QString tss(newerThan.toString(Qt::ISODate));
+                        url.appendQuery(QString(parameterLatestTimestamp() + "=" + tss));
+                    }
+                    sApp->getFromNetwork(url, { this, &CAirportDataReader::ps_parseAirportData });
+                    emit dataRead(CEntityFlags::AirportEntity, CEntityFlags::StartRead, 0);
                 }
                 else
                 {
-                sApp->getFromNetwork(url, { this, &CAirportDataReader::ps_parseAirportData });
+                    CLogMessage(this).error("No URL for %1") << CEntityFlags::flagToString(CEntityFlags::AirportEntity);
+                }
             }
         }
 
@@ -141,5 +129,10 @@ namespace BlackCore
         {
             // void
         }
+
+        void CAirportDataReader::ps_baseUrlCacheChanged()
+        {
+            // void
+        }
     } // ns
 } // ns

src/blackcore/db/airportdatareader.h

@@ -38,9 +38,6 @@ namespace BlackCore
             //! Constructor
             CAirportDataReader(QObject* parent, const CDatabaseReaderConfigList &config);
 
-            //! Read airports
-            void readInBackgroundThread();
-
             //! Returns a list of all airports in the database.
             //! \threadsafe
             BlackMisc::Aviation::CAirportList getAirports() const;
@@ -69,19 +66,20 @@ namespace BlackCore
             //! Parse downloaded JSON file
             void ps_parseAirportData(QNetworkReply *nwReply);
 
-            //! Read Last-Modified header
+            //! Read / re-read data file
-            void ps_parseAirportHeader(QNetworkReply *nwReply);
+            void ps_read(BlackMisc::Network::CEntityFlags::Entity entity = BlackMisc::Network::CEntityFlags::DistributorLiveryModel, const QDateTime &newerThan = QDateTime());
-
-            //! Read airports
-            void ps_readAirports();
 
             //! Airport cache changed
             void ps_airportCacheChanged();
 
+            //! Base url cache changed
+            void ps_baseUrlCacheChanged();
+
         private:
             BlackMisc::CData<BlackCore::Data::TDbAirportCache> m_airportCache {this, &CAirportDataReader::ps_airportCacheChanged};
-            mutable QReadWriteLock m_lock;
+
-            quint64 m_lastModified = 0; //!< When was data file updated, obtained from HTTP Last-Modified header, in ms from epoch
+            //! Reader URL (we read from where?) used to detect changes of location
+            BlackMisc::CData<BlackCore::Data::TDbModelReaderBaseUrl> m_readerUrlCache {this, &CAirportDataReader::ps_baseUrlCacheChanged };
 
         };
     }

src/blackcore/webdataservices.cpp

@@ -212,7 +212,8 @@ namespace BlackCore
         {
             if (whatToRead.testFlag(CEntityFlags::AirportEntity))
             {
-                m_airportDataReader->readInBackgroundThread();
+                CEntityFlags::Entity airportEntities = whatToRead & CEntityFlags::AirportEntity;
+                m_airportDataReader->readInBackgroundThread(airportEntities, newerThan);
                 triggeredRead |= CEntityFlags::AirportEntity;
             }
         }

src/blackmisc/aviation/airport.cpp

@@ -74,6 +74,24 @@ namespace BlackMisc
             setPosition(pos);
         }
 
+        CAirport CAirport::fromDatabaseJson(const QJsonObject &json, const QString &prefix)
+        {
+            CAirport airport(json.value("icao").toString());
+            airport.setDescriptiveName(json.value("name").toString());
+            airport.setElevation(CLength(json.value("altitude").toInt(), CLengthUnit::ft()));
+            CCoordinateGeodetic pos(json.value("latitude").toDouble(), json.value("longitude").toDouble(), 0);
+            airport.setPosition(pos);
+
+            if (json.value("alpha3").isString() && json.value("country").isString())
+            {
+                CCountry country(json.value("alpha3").toString(), json.value("country").toString());
+                airport.setCountry(country);
+            }
+
+            airport.setKeyAndTimestampFromDatabaseJson(json, prefix);
+            return airport;
+        }
+
         CVariant CAirport::propertyByIndex(const BlackMisc::CPropertyIndex &index) const
         {
             if (index.isMyself()) { return CVariant::from(*this); }

src/blackmisc/aviation/airport.h

@@ -14,6 +14,7 @@
 
 #include "blackmisc/aviation/airporticaocode.h"
 #include "blackmisc/blackmiscexport.h"
+#include "blackmisc/db/datastore.h"
 #include "blackmisc/geo/coordinategeodetic.h"
 #include "blackmisc/geo/latitude.h"
 #include "blackmisc/geo/longitude.h"
@@ -34,7 +35,10 @@ namespace BlackMisc
     namespace Aviation
     {
         //! Value object encapsulating information about an airpot.
-        class BLACKMISC_EXPORT CAirport : public CValueObject<CAirport>, public Geo::ICoordinateWithRelativePosition
+        class BLACKMISC_EXPORT CAirport :
+                public CValueObject<CAirport>,
+                public BlackMisc::Db::IDatastoreObjectWithIntegerKey,
+                public Geo::ICoordinateWithRelativePosition
         {
         public:
             //! Properties by index
@@ -134,6 +138,9 @@ namespace BlackMisc
             //! \copydoc BlackMisc::CValueObject::convertFromJson
             void convertFromDatabaseJson(const QJsonObject &json);
 
+            //! Object from JSON
+            static CAirport fromDatabaseJson(const QJsonObject &json, const QString &prefix = QString("apt_"));
+
         private:
             CAirportIcaoCode                    m_icao;
             QString                             m_descriptiveName;

src/blackmisc/aviation/airportlist.h

@@ -16,6 +16,7 @@
 #include "blackmisc/aviation/airporticaocode.h"
 #include "blackmisc/blackmiscexport.h"
 #include "blackmisc/collection.h"
+#include "blackmisc/db/datastoreobjectlist.h"
 #include "blackmisc/geo/geoobjectlist.h"
 #include "blackmisc/sequence.h"
 #include "blackmisc/variant.h"
@@ -29,6 +30,7 @@ namespace BlackMisc
         //! Value object for a list of airports.
         class BLACKMISC_EXPORT CAirportList :
             public CSequence<CAirport>,
+            public BlackMisc::Db::IDatastoreObjectList<CAirport, CAirportList, int>,
             public BlackMisc::Geo::IGeoObjectWithRelativePositionList<CAirport, CAirportList>,
             public BlackMisc::Mixin::MetaType<CAirportList>
         {

src/blackmisc/db/datastoreobjectlist.cpp

@@ -10,6 +10,8 @@
 #include "blackmisc/db/datastoreobjectlist.h"
 #include "blackmisc/predicates.h"
 #include "blackmisc/countrylist.h"
+#include "blackmisc/aviation/airport.h"
+#include "blackmisc/aviation/airportlist.h"
 #include "blackmisc/aviation/liverylist.h"
 #include "blackmisc/aviation/aircrafticaocodelist.h"
 #include "blackmisc/aviation/airlineicaocodelist.h"
@@ -170,6 +172,7 @@ namespace BlackMisc
         template class IDatastoreObjectList<BlackMisc::Simulation::CAircraftModel, BlackMisc::Simulation::CAircraftModelList, int>;
         template class IDatastoreObjectList<BlackMisc::Simulation::CDistributor, BlackMisc::Simulation::CDistributorList, QString>;
         template class IDatastoreObjectList<BlackMisc::CCountry, BlackMisc::CCountryList, QString>;
+        template class IDatastoreObjectList<BlackMisc::Aviation::CAirport, BlackMisc::Aviation::CAirportList, int>;
 
     } // ns
 } // ns

src/blackmisc/db/datastoreobjectlist.h

@@ -66,20 +66,6 @@ namespace BlackMisc
         class CDbInfo;
         class CDbInfoList;
 
-        namespace Aviation
-        {
-            class CLivery;
-            class CLiveryList;
-        }
-
-        namespace Simulation
-        {
-            class CDistributor;
-            class CDistributorList;
-            class CAircraftModel;
-            class CAircraftModelList;
-        }
-
         extern template class BLACKMISC_EXPORT_TEMPLATE IDatastoreObjectList<BlackMisc::Aviation::CLivery, BlackMisc::Aviation::CLiveryList, int>;
         extern template class BLACKMISC_EXPORT_TEMPLATE IDatastoreObjectList<BlackMisc::Aviation::CAircraftIcaoCode, BlackMisc::Aviation::CAircraftIcaoCodeList, int>;
         extern template class BLACKMISC_EXPORT_TEMPLATE IDatastoreObjectList<BlackMisc::Aviation::CAirlineIcaoCode, BlackMisc::Aviation::CAirlineIcaoCodeList, int>;
@@ -87,6 +73,7 @@ namespace BlackMisc
         extern template class BLACKMISC_EXPORT_TEMPLATE IDatastoreObjectList<BlackMisc::Simulation::CAircraftModel, BlackMisc::Simulation::CAircraftModelList, int>;
         extern template class BLACKMISC_EXPORT_TEMPLATE IDatastoreObjectList<BlackMisc::Simulation::CDistributor, BlackMisc::Simulation::CDistributorList, QString>;
         extern template class BLACKMISC_EXPORT_TEMPLATE IDatastoreObjectList<BlackMisc::CCountry, BlackMisc::CCountryList, QString>;
+        extern template class BLACKMISC_EXPORT_TEMPLATE IDatastoreObjectList<BlackMisc::Aviation::CAirport, BlackMisc::Aviation::CAirportList, int>;
         //! \endcond
 
     } // ns

src/blackmisc/timestampobjectlist.cpp

@@ -12,6 +12,8 @@
 #include "blackmisc/aviation/liverylist.h"
 #include "blackmisc/aviation/aircrafticaocodelist.h"
 #include "blackmisc/aviation/airlineicaocodelist.h"
+#include "blackmisc/aviation/airport.h"
+#include "blackmisc/aviation/airportlist.h"
 #include "blackmisc/db/dbinfolist.h"
 #include "blackmisc/network/textmessage.h"
 #include "blackmisc/network/textmessagelist.h"
@@ -208,6 +210,7 @@ namespace BlackMisc
     template class ITimestampObjectList<BlackMisc::Aviation::CLivery, BlackMisc::Aviation::CLiveryList>;
     template class ITimestampObjectList<BlackMisc::Aviation::CAircraftIcaoCode, BlackMisc::Aviation::CAircraftIcaoCodeList>;
     template class ITimestampObjectList<BlackMisc::Aviation::CAirlineIcaoCode, BlackMisc::Aviation::CAirlineIcaoCodeList>;
+    template class ITimestampObjectList<BlackMisc::Aviation::CAirport, BlackMisc::Aviation::CAirportList>;
     template class ITimestampObjectList<BlackMisc::Db::CDbInfo, BlackMisc::Db::CDbInfoList>;
     template class ITimestampObjectList<BlackMisc::Simulation::CAircraftModel, BlackMisc::Simulation::CAircraftModelList>;
     template class ITimestampObjectList<BlackMisc::Simulation::CDistributor, BlackMisc::Simulation::CDistributorList>;

src/blackmisc/timestampobjectlist.h

@@ -104,6 +104,8 @@ namespace BlackMisc
         class CAircraftSituationList;
         class CAircraftParts;
         class CAircraftPartsList;
+        class CAirport;
+        class CAirportList;
         class CLivery;
         class CLiveryList;
         class CAircraftIcaoCode;
@@ -144,6 +146,7 @@ namespace BlackMisc
     extern template class BLACKMISC_EXPORT_TEMPLATE ITimestampObjectList<BlackMisc::Aviation::CLivery, BlackMisc::Aviation::CLiveryList>;
     extern template class BLACKMISC_EXPORT_TEMPLATE ITimestampObjectList<BlackMisc::Aviation::CAircraftIcaoCode, BlackMisc::Aviation::CAircraftIcaoCodeList>;
     extern template class BLACKMISC_EXPORT_TEMPLATE ITimestampObjectList<BlackMisc::Aviation::CAirlineIcaoCode, BlackMisc::Aviation::CAirlineIcaoCodeList>;
+    extern template class BLACKMISC_EXPORT_TEMPLATE ITimestampObjectList<BlackMisc::Aviation::CAirport, BlackMisc::Aviation::CAirportList>;
     extern template class BLACKMISC_EXPORT_TEMPLATE ITimestampObjectList<BlackMisc::Db::CDbInfo, BlackMisc::Db::CDbInfoList>;
     extern template class BLACKMISC_EXPORT_TEMPLATE ITimestampObjectList<BlackMisc::Simulation::CAircraftModel, BlackMisc::Simulation::CAircraftModelList>;
     extern template class BLACKMISC_EXPORT_TEMPLATE ITimestampObjectList<BlackMisc::Simulation::CDistributor, BlackMisc::Simulation::CDistributorList>;

tests/blackcore/testreaders.cpp

@@ -123,7 +123,7 @@ namespace BlackCoreTest
         const CUrl url(sApp->getGlobalSetup().getSwiftAirportUrls().getRandomWorkingUrl());
         if (!this->pingServer(url)) { return; }
         m_airportReader->start();
-        m_airportReader->readInBackgroundThread();
+        m_airportReader->readInBackgroundThread(CEntityFlags::AirportEntity, QDateTime());
 
         for (int i = 0; i < 120; ++i)
         {