Ref T28, consolidated reading

* fixed newer parameter
* unified handling of incremental updates
* skip cache update when there was no incremental data

src/blackcore/db/airportdatareader.cpp

@@ -161,9 +161,10 @@ namespace BlackCore
             CAirportList airports;
             if (res.isRestricted())
             {
+                const CAirportList incAirports(CAirportList::fromDatabaseJson(res));
+                if (incAirports.isEmpty()) { return; } // currently ignored
                 airports = this->getAirports();
-                CAirportList updates(CAirportList::fromDatabaseJson(res));
+                airports.replaceOrAddObjectsByKey(incAirports);
-                airports.replaceOrAddObjectsByKey(updates);
             }
             else
             {
@@ -200,11 +201,7 @@ namespace BlackCore
                 CUrl url = getAirportsUrl(mode);
                 if (!url.isEmpty())
                 {
-                    if (!newerThan.isNull())
+                    url.appendQuery(queryLatestTimestamp(newerThan));
-                    {
-                        const QString tss(newerThan.toString(Qt::ISODate));
-                        url.appendQuery(QString(parameterLatestTimestamp() + "=" + tss));
-                    }
                     sApp->getFromNetwork(url, { this, &CAirportDataReader::ps_parseAirportData });
                     emit dataRead(CEntityFlags::AirportEntity, CEntityFlags::StartRead, 0);
                 }

src/blackcore/db/databasereader.cpp

@@ -574,6 +574,12 @@ namespace BlackCore
             }
         }
 
+        QString CDatabaseReader::dateTimeToDbLatestTs(const QDateTime &ts)
+        {
+            if (!ts.isValid()) return "";
+            return ts.toUTC().toString(Qt::ISODate);
+        }
+
         const CLogCategoryList &CDatabaseReader::getLogCategories()
         {
             static const BlackMisc::CLogCategoryList cats
@@ -589,6 +595,13 @@ namespace BlackCore
             return p;
         }
 
+        QString CDatabaseReader::queryLatestTimestamp(const QDateTime &ts)
+        {
+            if (!ts.isValid()) return "";
+            const QString q = parameterLatestTimestamp() + "=" + dateTimeToDbLatestTs(ts);
+            return q;
+        }
+
         const QString &CDatabaseReader::parameterLatestId()
         {
             static const QString p("latestId");

src/blackcore/db/databasereader.h

@@ -227,12 +227,6 @@ namespace BlackCore
             //! Log categories
             static const BlackMisc::CLogCategoryList &getLogCategories();
 
-            //! Name of latest timestamp
-            static const QString &parameterLatestTimestamp();
-
-            //! Name of parameter for latest id
-            static const QString &parameterLatestId();
-
             //! swift DB server reachable?
             static bool canPingSwiftServer();
 
@@ -300,6 +294,19 @@ namespace BlackCore
             //! File name for given mode, either php service or shared file name
             static QString fileNameForMode(BlackMisc::Network::CEntityFlags::Entity entity, BlackMisc::Db::CDbFlags::DataRetrievalModeFlag mode);
 
+            //! Name of latest timestamp
+            static const QString &parameterLatestTimestamp();
+
+            //! Name of parameter for latest id
+            static const QString &parameterLatestId();
+
+            //! A newer than value understood by swift DB
+            //! \sa CDatabaseReader::parameterLatestTimestamp
+            static QString dateTimeToDbLatestTs(const QDateTime &ts);
+
+            //! Latest timestamp query for DB
+            static QString queryLatestTimestamp(const QDateTime &ts);
+
             //! \name Cache access
             //! @{
             //! Synchronize caches for given entities

src/blackcore/db/icaodatareader.cpp

@@ -146,7 +146,7 @@ namespace BlackCore
                 CUrl url(getAircraftIcaoUrl(mode));
                 if (!url.isEmpty())
                 {
-                    if (!newerThan.isNull()) { url.appendQuery("newer=" + newerThan.toString(Qt::ISODate)); }
+                    url.appendQuery(queryLatestTimestamp(newerThan));
                     sApp->getFromNetwork(url, { this, &CIcaoDataReader::ps_parseAircraftIcaoData });
                     entitiesTriggered |= CEntityFlags::AircraftIcaoEntity;
                 }
@@ -161,7 +161,7 @@ namespace BlackCore
                 CUrl url(getAirlineIcaoUrl(mode));
                 if (!url.isEmpty())
                 {
-                    if (!newerThan.isNull()) { url.appendQuery("newer=" + newerThan.toString(Qt::ISODate)); }
+                    url.appendQuery(queryLatestTimestamp(newerThan));
                     sApp->getFromNetwork(url, { this, &CIcaoDataReader::ps_parseAirlineIcaoData });
                     entitiesTriggered |= CEntityFlags::AirlineIcaoEntity;
                 }
@@ -176,7 +176,7 @@ namespace BlackCore
                 CUrl url(getCountryUrl(mode));
                 if (!url.isEmpty())
                 {
-                    if (!newerThan.isNull()) { url.appendQuery("newer=" + newerThan.toString(Qt::ISODate)); }
+                    url.appendQuery(queryLatestTimestamp(newerThan));
                     sApp->getFromNetwork(url, { this, &CIcaoDataReader::ps_parseCountryData });
                     entitiesTriggered |= CEntityFlags::CountryEntity;
                 }
@@ -238,8 +238,21 @@ namespace BlackCore
                 return;
             }
 
-            // normally read from special view which already filter incomplete
+            CAircraftIcaoCodeList codes;
-            const CAircraftIcaoCodeList codes = CAircraftIcaoCodeList::fromDatabaseJson(res, true);
+            if (res.isRestricted())
+            {
+                // create full list if it was just incremental
+                const CAircraftIcaoCodeList incIcao(CAircraftIcaoCodeList::fromDatabaseJson(res, true));
+                if (incIcao.isEmpty()) { return; } // currently ignored
+                codes = this->getAircraftIcaoCodes();
+                codes.replaceOrAddObjectsByKey(incIcao);
+            }
+            else
+            {
+                // normally read from special view which already filters incomplete
+                codes  = CAircraftIcaoCodeList::fromDatabaseJson(res, true);
+            }
+
             const int n = codes.size();
             qint64 latestTimestamp = codes.latestTimestampMsecsSinceEpoch();
             if (n > 0 && latestTimestamp < 0)
@@ -266,7 +279,22 @@ namespace BlackCore
                 emit dataRead(CEntityFlags::AirlineIcaoEntity, CEntityFlags::ReadFailed, 0);
                 return;
             }
-            const CAirlineIcaoCodeList codes = CAirlineIcaoCodeList::fromDatabaseJson(res, true);
+
+            CAirlineIcaoCodeList codes;
+            if (res.isRestricted())
+            {
+                // create full list if it was just incremental
+                const CAirlineIcaoCodeList incIcao(CAirlineIcaoCodeList::fromDatabaseJson(res, true));
+                if (incIcao.isEmpty()) { return; } // currently ignored
+                codes = this->getAirlineIcaoCodes();
+                codes.replaceOrAddObjectsByKey(incIcao);
+            }
+            else
+            {
+                // normally read from special view which already filters incomplete
+                codes  = CAirlineIcaoCodeList::fromDatabaseJson(res, true);
+            }
+
             const int n = codes.size();
             qint64 latestTimestamp = codes.latestTimestampMsecsSinceEpoch();
             if (n > 0 && latestTimestamp < 0)
@@ -291,7 +319,22 @@ namespace BlackCore
                 emit dataRead(CEntityFlags::CountryEntity, CEntityFlags::ReadFailed, 0);
                 return;
             }
-            const CCountryList countries = CCountryList::fromDatabaseJson(res);
+
+            CCountryList countries;
+            if (res.isRestricted())
+            {
+                // create full list if it was just incremental
+                const CCountryList incCountries(CCountryList::fromDatabaseJson(res));
+                if (incCountries.isEmpty()) { return; } // currently ignored
+                countries = this->getCountries();
+                countries.replaceOrAddObjectsByKey(incCountries);
+            }
+            else
+            {
+                // normally read from special view which already filters incomplete
+                countries  = CCountryList::fromDatabaseJson(res);
+            }
+
             const int n = countries.size();
             qint64 latestTimestamp = countries.latestTimestampMsecsSinceEpoch();
             if (n > 0 && latestTimestamp < 0)

src/blackcore/db/modeldatareader.cpp

@@ -166,11 +166,7 @@ namespace BlackCore
                 CUrl url(getLiveryUrl(mode));
                 if (!url.isEmpty())
                 {
-                    if (!newerThan.isNull())
+                    url.appendQuery(queryLatestTimestamp(newerThan));
-                    {
-                        const QString tss(newerThan.toString(Qt::ISODate));
-                        url.appendQuery(QString(parameterLatestTimestamp() + "=" + tss));
-                    }
                     sApp->getFromNetwork(url, { this, &CModelDataReader::ps_parseLiveryData});
                     triggeredRead |= CEntityFlags::LiveryEntity;
                 }
@@ -185,11 +181,7 @@ namespace BlackCore
                 CUrl url(getDistributorUrl(mode));
                 if (!url.isEmpty())
                 {
-                    if (!newerThan.isNull())
+                    url.appendQuery(queryLatestTimestamp(newerThan));
-                    {
-                        const QString tss(newerThan.toString(Qt::ISODate));
-                        url.appendQuery(QString(parameterLatestTimestamp() + "=" + tss));
-                    }
                     sApp->getFromNetwork(url, { this, &CModelDataReader::ps_parseDistributorData});
                     triggeredRead |= CEntityFlags::DistributorEntity;
                 }
@@ -204,11 +196,7 @@ namespace BlackCore
                 CUrl url(getModelUrl(mode));
                 if (!url.isEmpty())
                 {
-                    if (!newerThan.isNull())
+                    url.appendQuery(queryLatestTimestamp(newerThan));
-                    {
-                        const QString tss(newerThan.toString(Qt::ISODate));
-                        url.appendQuery(QString(parameterLatestTimestamp() + "=" + tss));
-                    }
                     sApp->getFromNetwork(url, { this, &CModelDataReader::ps_parseModelData});
                     triggeredRead |= CEntityFlags::ModelEntity;
                 }
@@ -274,8 +262,10 @@ namespace BlackCore
             if (res.isRestricted())
             {
                 // create full list if it was just incremental
+                const CLiveryList incLiveries(CLiveryList::fromDatabaseJson(res));
+                if (incLiveries.isEmpty()) { return; } // currenty ignored
                 liveries = this->getLiveries();
-                liveries.replaceOrAddObjectsByKey(CLiveryList::fromDatabaseJson(res));
+                liveries.replaceOrAddObjectsByKey(incLiveries);
             }
             else
             {
@@ -314,8 +304,10 @@ namespace BlackCore
             if (res.isRestricted())
             {
                 // create full list if it was just incremental
+                const CDistributorList incDistributors(CDistributorList::fromDatabaseJson(res));
+                if (incDistributors.isEmpty()) { return; } // currently ignored
                 distributors = this->getDistributors();
-                distributors.replaceOrAddObjectsByKey(CDistributorList::fromDatabaseJson(res));
+                distributors.replaceOrAddObjectsByKey(incDistributors);
             }
             else
             {
@@ -354,8 +346,10 @@ namespace BlackCore
             if (res.isRestricted())
             {
                 // create full list if it was just incremental
+                const CAircraftModelList incModels(CAircraftModelList::fromDatabaseJson(res));
+                if (incModels.isEmpty()) { return; } // currently ignored
                 models = this->getModels();
-                models.replaceOrAddObjectsByKey(CAircraftModelList::fromDatabaseJson(res));
+                models.replaceOrAddObjectsByKey(incModels);
             }
             else
             {