From 260585e35c6102b12c7410afb0ba5c6e429a5922 Mon Sep 17 00:00:00 2001 From: Klaus Basan Date: Wed, 29 Aug 2018 02:55:28 +0200 Subject: [PATCH] Ref T308, performance optimized fromDatabaseJsonCaching * maps for id/object * caching fromDatabaseJsonCaching * fixed CAircraftModel::fromDatabaseJson wrong prefix --- src/blackmisc/aviation/aircrafticaocode.h | 4 + src/blackmisc/aviation/airlineicaocode.h | 5 + .../aviation/airlineicaocodelist.cpp | 11 +++ src/blackmisc/aviation/airlineicaocodelist.h | 3 + src/blackmisc/aviation/livery.cpp | 48 +++++++++ src/blackmisc/aviation/livery.h | 8 ++ src/blackmisc/aviation/liverylist.cpp | 12 +++ src/blackmisc/aviation/liverylist.h | 5 + src/blackmisc/simulation/aircraftmodel.cpp | 98 +++++++++++++++++-- src/blackmisc/simulation/aircraftmodel.h | 4 +- .../simulation/aircraftmodellist.cpp | 17 +++- src/blackmisc/simulation/aircraftmodellist.h | 3 + src/blackmisc/simulation/distributor.cpp | 5 +- src/blackmisc/simulation/distributor.h | 5 + 14 files changed, 214 insertions(+), 14 deletions(-) diff --git a/src/blackmisc/aviation/aircrafticaocode.h b/src/blackmisc/aviation/aircrafticaocode.h index 70a8ce429..68679376d 100644 --- a/src/blackmisc/aviation/aircrafticaocode.h +++ b/src/blackmisc/aviation/aircrafticaocode.h @@ -372,6 +372,10 @@ namespace BlackMisc BLACK_METAMEMBER(rank) ); }; + + //! Id/CAircraftIcaoCode map + using AircraftIcaoIdMap = QMap; + } // namespace } // namespace diff --git a/src/blackmisc/aviation/airlineicaocode.h b/src/blackmisc/aviation/airlineicaocode.h index 5773d82ed..961348152 100644 --- a/src/blackmisc/aviation/airlineicaocode.h +++ b/src/blackmisc/aviation/airlineicaocode.h @@ -25,6 +25,7 @@ #include #include #include +#include #include namespace BlackMisc @@ -281,6 +282,10 @@ namespace BlackMisc BLACK_METAMEMBER(isMilitary) ); }; + + //! Airline ICAO/id list + using AirlineIcaoIdMap = QMap; + } // namespace } // namespace diff --git a/src/blackmisc/aviation/airlineicaocodelist.cpp b/src/blackmisc/aviation/airlineicaocodelist.cpp index 949d36506..6300a9d42 100644 --- a/src/blackmisc/aviation/airlineicaocodelist.cpp +++ b/src/blackmisc/aviation/airlineicaocodelist.cpp @@ -307,5 +307,16 @@ namespace BlackMisc if (vDesignator.length() < 4) { return this->containsDesignator(vDesignator); } return this->contains(&CAirlineIcaoCode::getVDesignator, vDesignator.toUpper()); } + + AirlineIcaoIdMap CAirlineIcaoCodeList::toIdMap() const + { + AirlineIcaoIdMap map; + for (const CAirlineIcaoCode &code : *this) + { + if (!code.hasValidDbKey()) { continue; } + map.insert(code.getDbKey(), code); + } + return map; + } } // ns } // ns diff --git a/src/blackmisc/aviation/airlineicaocodelist.h b/src/blackmisc/aviation/airlineicaocodelist.h index f6795121c..5b77b12eb 100644 --- a/src/blackmisc/aviation/airlineicaocodelist.h +++ b/src/blackmisc/aviation/airlineicaocodelist.h @@ -113,6 +113,9 @@ namespace BlackMisc //! Contains given designator? bool containsVDesignator(const QString &vDesignator) const; + //! To id map + AirlineIcaoIdMap toIdMap() const; + //! From our DB JSON static CAirlineIcaoCodeList fromDatabaseJson(const QJsonArray &array, bool ignoreIncomplete = true, CAirlineIcaoCodeList *inconsistent = nullptr); }; diff --git a/src/blackmisc/aviation/livery.cpp b/src/blackmisc/aviation/livery.cpp index 7b0054406..d047d2297 100644 --- a/src/blackmisc/aviation/livery.cpp +++ b/src/blackmisc/aviation/livery.cpp @@ -263,6 +263,54 @@ namespace BlackMisc return livery; } + CLivery CLivery::fromDatabaseJsonCaching(const QJsonObject &json, AirlineIcaoIdMap &airlineIcaos, const QString &prefix) + { + if (!existsKey(json, prefix)) + { + // when using relationship, this can be null + return CLivery(); + } + + const QString combinedCode(json.value(prefix % QStringLiteral("combinedcode")).toString()); + if (combinedCode.isEmpty()) + { + CLivery liveryStub; // only consists of id, maybe key and timestamp + liveryStub.setKeyAndTimestampFromDatabaseJson(json, prefix); + return liveryStub; + } + + const bool isColorLivery = combinedCode.startsWith(colorLiveryMarker()); + const QString description(json.value(prefix % QStringLiteral("description")).toString()); + const CRgbColor colorFuselage(json.value(prefix % QStringLiteral("colorfuselage")).toString()); + const CRgbColor colorTail(json.value(prefix % QStringLiteral("colortail")).toString()); + const bool military = CDatastoreUtility::dbBoolStringToBool(json.value(prefix % QStringLiteral("military")).toString()); + + CAirlineIcaoCode airline; + if (!isColorLivery) + { + static const QString prefixAirline("al_"); + const int idAirlineIcao = json.value(prefixAirline % "id").toInt(-1); + const bool cachedAirlineIcao = idAirlineIcao >= 0 && airlineIcaos.contains(idAirlineIcao); + + airline = cachedAirlineIcao ? + airlineIcaos[idAirlineIcao] : + CAirlineIcaoCode::fromDatabaseJson(json, prefixAirline); + + if (!cachedAirlineIcao && airline.isLoadedFromDb()) + { + airlineIcaos[idAirlineIcao] = airline; + } + } + + CLivery livery(combinedCode, airline, description, colorFuselage, colorTail, military); + livery.setKeyAndTimestampFromDatabaseJson(json, prefix); + + // color liveries must have default ICAO, but airline liveries must have DB airline + BLACK_VERIFY_X((livery.isColorLivery() && !livery.getAirlineIcaoCode().hasValidDbKey()) || (livery.isAirlineLivery() && livery.getAirlineIcaoCode().hasValidDbKey()), Q_FUNC_INFO, "inconsistent data"); + + return livery; + } + bool CLivery::isValidCombinedCode(const QString &candidate) { if (candidate.isEmpty()) { return false; } diff --git a/src/blackmisc/aviation/livery.h b/src/blackmisc/aviation/livery.h index 7fc884145..589c9947b 100644 --- a/src/blackmisc/aviation/livery.h +++ b/src/blackmisc/aviation/livery.h @@ -22,6 +22,7 @@ #include "blackmisc/valueobject.h" #include "blackmisc/variant.h" +#include #include #include #include @@ -198,6 +199,9 @@ namespace BlackMisc //! Object from JSON static CLivery fromDatabaseJson(const QJsonObject &json, const QString &prefix = QString("liv_")); + //! Object from JSON caching + static CLivery fromDatabaseJsonCaching(const QJsonObject &json, AirlineIcaoIdMap &airlineIcaos, const QString &prefix = QString("liv_")); + //! Valid combined code string? static bool isValidCombinedCode(const QString &candidate); @@ -233,6 +237,10 @@ namespace BlackMisc BLACK_METAMEMBER(military) ); }; + + //! Id/CLivery map + using LiveryIdMap = QMap; + } // namespace } // namespace diff --git a/src/blackmisc/aviation/liverylist.cpp b/src/blackmisc/aviation/liverylist.cpp index 9701d2210..b860d1cdb 100644 --- a/src/blackmisc/aviation/liverylist.cpp +++ b/src/blackmisc/aviation/liverylist.cpp @@ -203,5 +203,17 @@ namespace BlackMisc } return CLivery(); } + + CLiveryList CLiveryList::fromDatabaseJsonCaching(const QJsonArray &array, const CAirlineIcaoCodeList &relatedAirlines) + { + AirlineIcaoIdMap airlineIcaos = relatedAirlines.toIdMap(); + + CLiveryList models; + for (const QJsonValue &value : array) + { + models.push_back(CLivery::fromDatabaseJsonCaching(value.toObject(), airlineIcaos)); + } + return models; + } } // namespace } // namespace diff --git a/src/blackmisc/aviation/liverylist.h b/src/blackmisc/aviation/liverylist.h index 88e31e5a7..5f7c65e8b 100644 --- a/src/blackmisc/aviation/liverylist.h +++ b/src/blackmisc/aviation/liverylist.h @@ -80,6 +80,11 @@ namespace BlackMisc //! Find by multiple criteria CLivery smartLiverySelector(const CLivery &liveryPattern) const; + + //! Caching version from DB data + //! \param relatedAirlines passing the airline can skip the parsing from livery + //! \remark without passing related airlines there is not much sense using this function, as most airlines/liveries have a 1:1 ratio + static CLiveryList fromDatabaseJsonCaching(const QJsonArray &array, const CAirlineIcaoCodeList &relatedAirlines = {}); }; } //namespace } // namespace diff --git a/src/blackmisc/simulation/aircraftmodel.cpp b/src/blackmisc/simulation/aircraftmodel.cpp index 9d67e21de..08dce16bb 100644 --- a/src/blackmisc/simulation/aircraftmodel.cpp +++ b/src/blackmisc/simulation/aircraftmodel.cpp @@ -772,19 +772,24 @@ namespace BlackMisc CAircraftModel CAircraftModel::fromDatabaseJson(const QJsonObject &json, const QString &prefix) { - const QString modelString(json.value(prefix + "modelstring").toString()); - const QString modelDescription(json.value(prefix + "description").toString()); - const QString modelName(json.value(prefix + "name").toString()); - const QString modelMode(json.value(prefix + "mode").toString()); + static const QString prefixAircraftIcao("ac_"); + static const QString prefixLivery("liv_"); + static const QString prefixDistributor("dist_"); + + const QString modelString(json.value(prefix % QStringLiteral("modelstring")).toString()); + const QString modelDescription(json.value(prefix % QStringLiteral("description")).toString()); + const QString modelName(json.value(prefix % QStringLiteral("name")).toString()); + const QString modelMode(json.value(prefix % QStringLiteral("mode")).toString()); const CSimulatorInfo simInfo = CSimulatorInfo::fromDatabaseJson(json, prefix); - CDistributor distributor(CDistributor::fromDatabaseJson(json, "dist_")); - CAircraftIcaoCode aircraftIcao(CAircraftIcaoCode::fromDatabaseJson(json, "ac_")); - CLivery livery(CLivery::fromDatabaseJson(json, "liv_")); + CDistributor distributor(CDistributor::fromDatabaseJson(json, prefixDistributor)); + CAircraftIcaoCode aircraftIcao(CAircraftIcaoCode::fromDatabaseJson(json, prefixAircraftIcao)); + CLivery livery(CLivery::fromDatabaseJson(json, prefixLivery)); + //! \todo KB 2018-08 "idaircrafticao" seem not to be used anymore (remove???) in "vAircraftModelDenormalized", but was used in "vAircraftModelComplete" if (!aircraftIcao.isLoadedFromDb()) { - const int idAircraftIcao = json.value(prefix + "idaircrafticao").toInt(-1); + const int idAircraftIcao = json.value(prefix % QStringLiteral("idaircrafticao")).toInt(-1); if (idAircraftIcao >= 0) { aircraftIcao.setDbKey(idAircraftIcao); @@ -793,7 +798,7 @@ namespace BlackMisc if (!livery.isLoadedFromDb()) { - const int idLivery = json.value(prefix + "idlivery").toInt(-1); + const int idLivery = json.value(prefix % QStringLiteral("idlivery")).toInt(-1); if (idLivery >= 0) { livery.setDbKey(idLivery); @@ -802,7 +807,7 @@ namespace BlackMisc if (!distributor.isLoadedFromDb()) { - const QString idDistributor = json.value(prefix + "iddistributor").toString(); + const QString idDistributor = json.value(prefix % QStringLiteral("iddistributor")).toString(); if (!idDistributor.isEmpty()) { distributor.setDbKey(idDistributor); @@ -818,6 +823,79 @@ namespace BlackMisc return model; } + CAircraftModel CAircraftModel::fromDatabaseJsonCaching( + const QJsonObject &json, + AircraftIcaoIdMap &aircraftIcaos, LiveryIdMap &liveries, DistributorIdMap &distributors, + const QString &prefix) + { + static const QString prefixAircraftIcao("ac_"); + static const QString prefixLivery("liv_"); + static const QString prefixDistributor("dist_"); + + const QString modelString(json.value(prefix % QStringLiteral("modelstring")).toString()); + const QString modelDescription(json.value(prefix % QStringLiteral("description")).toString()); + const QString modelName(json.value(prefix % QStringLiteral("name")).toString()); + const QString modelMode(json.value(prefix % QStringLiteral("mode")).toString()); + + const CSimulatorInfo simInfo = CSimulatorInfo::fromDatabaseJson(json, prefix); + const int idAircraftIcao = json.value(prefixAircraftIcao % QStringLiteral("id")).toInt(-1); + const int idLivery = json.value(prefixLivery % QStringLiteral("id")).toInt(-1); + const QString idDistributor = json.value(prefixDistributor % QStringLiteral("id")).toString(); + const bool cachedAircraftIcao = (idAircraftIcao >= 0) && aircraftIcaos.contains(idAircraftIcao); + const bool cachedLivery = (idLivery >= 0) && liveries.contains(idLivery); + const bool cachedDistributor = !idDistributor.isEmpty() && distributors.contains(idDistributor); + + CAircraftIcaoCode aircraftIcao(cachedAircraftIcao ? + aircraftIcaos[idAircraftIcao] : + CAircraftIcaoCode::fromDatabaseJson(json, prefixAircraftIcao)); + + CLivery livery(cachedLivery ? + liveries[idLivery] : + CLivery::fromDatabaseJson(json, prefixLivery)); + + CDistributor distributor(cachedDistributor ? + distributors[idDistributor] : + CDistributor::fromDatabaseJson(json, prefixDistributor)); + + if (!aircraftIcao.isLoadedFromDb()) + { + if (idAircraftIcao >= 0) + { + aircraftIcao.setDbKey(idAircraftIcao); + } + } + + if (!livery.isLoadedFromDb()) + { + const int idLivery = json.value(prefix % QStringLiteral("idlivery")).toInt(-1); + if (idLivery >= 0) + { + livery.setDbKey(idLivery); + } + } + + if (!distributor.isLoadedFromDb()) + { + const QString idDistributor = json.value(prefix % QStringLiteral("iddistributor")).toString(); + if (!idDistributor.isEmpty()) + { + distributor.setDbKey(idDistributor); + } + } + + if (!cachedAircraftIcao && aircraftIcao.isLoadedFromDb()) { aircraftIcaos[aircraftIcao.getDbKey()] = aircraftIcao; } + if (!cachedLivery && livery.isLoadedFromDb()) { liveries[livery.getDbKey()] = livery; } + if (!cachedDistributor && distributor.isLoadedFromDb()) { distributors[distributor.getDbKey()] = distributor; } + + CAircraftModel model( + modelString, CAircraftModel::TypeDatabaseEntry, simInfo, modelName, modelDescription, aircraftIcao, livery + ); + model.setDistributor(distributor); + model.setModelModeAsString(modelMode); + model.setKeyAndTimestampFromDatabaseJson(json, prefix); + return model; + } + QStringList CAircraftModel::splitNetworkLiveryString(const QString &liveryString) { QStringList liveryModelStrings({ "", "" }); diff --git a/src/blackmisc/simulation/aircraftmodel.h b/src/blackmisc/simulation/aircraftmodel.h index dcd4d1f8a..3a6ffed2d 100644 --- a/src/blackmisc/simulation/aircraftmodel.h +++ b/src/blackmisc/simulation/aircraftmodel.h @@ -37,7 +37,6 @@ #include #include #include -#include #include #include #include @@ -437,6 +436,9 @@ namespace BlackMisc //! From swift DB JSON static CAircraftModel fromDatabaseJson(const QJsonObject &json, const QString &prefix = QString("mod_")); + //! From swift DB JSON, caching during this process (faster) + static CAircraftModel fromDatabaseJsonCaching(const QJsonObject &json, Aviation::AircraftIcaoIdMap &aircraftIcaos, Aviation::LiveryIdMap &liveries, DistributorIdMap &distributors, const QString &prefix = QString("mod_")); + //! Split swift network string "DLH._STD [modelname]" //! \return QStringList [0] livery code , [1] model string //! \sa getSwiftLiveryString diff --git a/src/blackmisc/simulation/aircraftmodellist.cpp b/src/blackmisc/simulation/aircraftmodellist.cpp index 09c0c6e6e..c1d2ee613 100644 --- a/src/blackmisc/simulation/aircraftmodellist.cpp +++ b/src/blackmisc/simulation/aircraftmodellist.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include using namespace BlackMisc::Network; @@ -1070,7 +1071,7 @@ namespace BlackMisc for (const CAircraftModel &model : *this) { CAircraftModel copy(model); - copy.normalizeFileNameForDb(); + copy.normalizeFileNameForDb(); // strip full path QJsonValue v(copy.toDatabaseJson()); array.append(v); } @@ -1163,5 +1164,19 @@ namespace BlackMisc } return stats; } + + CAircraftModelList CAircraftModelList::fromDatabaseJsonCaching(const QJsonArray &array) + { + AircraftIcaoIdMap aircraftIcaos; + LiveryIdMap liveries; + DistributorIdMap distributors; + + CAircraftModelList models; + for (const QJsonValue &value : array) + { + models.push_back(CAircraftModel::fromDatabaseJsonCaching(value.toObject(), aircraftIcaos, liveries, distributors)); + } + return models; + } } // namespace } // namespace diff --git a/src/blackmisc/simulation/aircraftmodellist.h b/src/blackmisc/simulation/aircraftmodellist.h index 0348f9ab1..d8f3834c7 100644 --- a/src/blackmisc/simulation/aircraftmodellist.h +++ b/src/blackmisc/simulation/aircraftmodellist.h @@ -363,6 +363,9 @@ namespace BlackMisc //! A HTML summary of the data in the list QString htmlStatistics(bool aircraftStats, bool airlineStats) const; + + //! Newer version + static CAircraftModelList fromDatabaseJsonCaching(const QJsonArray &array); }; //! Model per callsign diff --git a/src/blackmisc/simulation/distributor.cpp b/src/blackmisc/simulation/distributor.cpp index ef510208e..17fa17ead 100644 --- a/src/blackmisc/simulation/distributor.cpp +++ b/src/blackmisc/simulation/distributor.cpp @@ -14,6 +14,7 @@ #include #include +#include #include namespace BlackMisc @@ -174,8 +175,8 @@ namespace BlackMisc } const CSimulatorInfo simulator = CSimulatorInfo::fromDatabaseJson(json, prefix); - const QString alias1(json.value(prefix + "alias1").toString()); - const QString alias2(json.value(prefix + "alias2").toString()); + const QString alias1(json.value(prefix % QStringLiteral("alias1")).toString()); + const QString alias2(json.value(prefix % QStringLiteral("alias2")).toString()); Q_ASSERT_X(!description.isEmpty(), Q_FUNC_INFO, "Missing description"); CDistributor distributor("", description, alias1, alias2, simulator); distributor.setKeyAndTimestampFromDatabaseJson(json, prefix); diff --git a/src/blackmisc/simulation/distributor.h b/src/blackmisc/simulation/distributor.h index 37ea8abbe..09064d0e8 100644 --- a/src/blackmisc/simulation/distributor.h +++ b/src/blackmisc/simulation/distributor.h @@ -24,6 +24,7 @@ #include #include +#include #include namespace BlackMisc @@ -142,6 +143,10 @@ namespace BlackMisc BLACK_METAMEMBER(simulator) ); }; + + //! Distributor/id map + using DistributorIdMap = QMap; + } // namespace } // namespace