From 299859646ab268a299a903cac468a7552c03657d Mon Sep 17 00:00:00 2001 From: Klaus Basan Date: Wed, 1 Apr 2020 18:08:20 +0200 Subject: [PATCH] VATSIM data file parser "fixes" - Limit the number of parser errors going to the log - stop parsing if no attributes can be obtained (otherwise we see too many parser messages) --- src/blackcore/vatsim/vatsimdatafilereader.cpp | 35 +++++++++++++++---- src/blackcore/vatsim/vatsimdatafilereader.h | 2 +- src/blackcore/webdataservices.cpp | 4 +-- 3 files changed, 32 insertions(+), 9 deletions(-) diff --git a/src/blackcore/vatsim/vatsimdatafilereader.cpp b/src/blackcore/vatsim/vatsimdatafilereader.cpp index db60a43ce..f5e510bd9 100644 --- a/src/blackcore/vatsim/vatsimdatafilereader.cpp +++ b/src/blackcore/vatsim/vatsimdatafilereader.cpp @@ -245,6 +245,7 @@ namespace BlackCore QStringList clientSectionAttributes; Section section = SectionNone; + int invalidSections = 0; QString currentLine; // declared outside of the for loop, to amortize the cost of allocation for (const QStringRef &clRef : lines) @@ -267,6 +268,15 @@ namespace BlackCore const QVector attributes = currentLine.midRef(i).trimmed().split(':', QString::SkipEmptyParts); for (const QStringRef &attr : attributes) { clientSectionAttributes.push_back(attr.toString().trimmed().toLower()); } section = SectionNone; // reset + + // consistency check to avoid tons of parsing errors afterwards + // normally we have 40 attributes + if (attributes.size() < 10) + { + CLogMessage(this).warning(u"Too few (%1) attributes in VATSIM file, CANCEL parsing. Line: '%2'") << attributes.size() << currentLine; + return; + } + } continue; } @@ -280,9 +290,14 @@ namespace BlackCore { case SectionClients: { - const QMap clientPartsMap = clientPartsToMap(currentLine, clientSectionAttributes); + const bool logInconsistencies = invalidSections < 5; // flood protection + const QMap clientPartsMap = clientPartsToMap(currentLine, clientSectionAttributes, logInconsistencies); const CCallsign callsign = CCallsign(clientPartsMap["callsign"]); - if (callsign.isEmpty()) { break; } + if (callsign.isEmpty()) + { + invalidSections++; + break; + } const CUser user(clientPartsMap["cid"], clientPartsMap["realname"], callsign); const QString clientType = clientPartsMap["clienttype"].toLower(); if (clientType.isEmpty()) { break; } // sometimes type is empty @@ -458,7 +473,7 @@ namespace BlackCore setInitialAndPeriodicTime(s.getInitialTime().toMs(), s.getPeriodicTime().toMs()); } - const QMap CVatsimDataFileReader::clientPartsToMap(const QString ¤tLine, const QStringList &clientSectionAttributes) + const QMap CVatsimDataFileReader::clientPartsToMap(const QString ¤tLine, const QStringList &clientSectionAttributes, bool logInconsistency) { QMap parts; if (currentLine.isEmpty()) { return parts; } @@ -466,10 +481,18 @@ namespace BlackCore // remove last empty item if required if (currentLine.endsWith(':')) { clientParts.removeLast(); } - if (clientParts.size() != clientSectionAttributes.size()) + const int noParts = clientParts.size(); + const int noAttributes = clientSectionAttributes.size(); + const bool valid = (noParts == noAttributes); + + // valid data? + if (!valid) { - logInconsistentData( - CStatusMessage(static_cast(nullptr), CStatusMessage::SeverityInfo, u"VATSIM data file client parts: %1 attributes: %2 line: '%3'") << clientParts.size() << clientSectionAttributes.size() << currentLine); + if (logInconsistency) + { + logInconsistentData( + CStatusMessage(static_cast(nullptr), CStatusMessage::SeverityInfo, u"VATSIM data file client parts: %1 attributes: %2 line: '%3'") << clientParts.size() << clientSectionAttributes.size() << currentLine); + } return parts; } diff --git a/src/blackcore/vatsim/vatsimdatafilereader.h b/src/blackcore/vatsim/vatsimdatafilereader.h index b0aac8fb2..0cd2cddc2 100644 --- a/src/blackcore/vatsim/vatsimdatafilereader.h +++ b/src/blackcore/vatsim/vatsimdatafilereader.h @@ -163,7 +163,7 @@ namespace BlackCore //! Split line and assign values to their corresponding attribute names //! \remark attributes expected as lower case - static const QMap clientPartsToMap(const QString ¤tLine, const QStringList &clientSectionAttributes); + static const QMap clientPartsToMap(const QString ¤tLine, const QStringList &clientSectionAttributes, bool logInconsistency); //! Get current section static Section currentLineToSection(const QString ¤tLine); diff --git a/src/blackcore/webdataservices.cpp b/src/blackcore/webdataservices.cpp index eb112a002..0fb24430d 100644 --- a/src/blackcore/webdataservices.cpp +++ b/src/blackcore/webdataservices.cpp @@ -1155,7 +1155,7 @@ namespace BlackCore Q_ASSERT_X(c, Q_FUNC_INFO, "Cannot connect Model reader signals"); m_airportDataReader->start(QThread::LowPriority); } - Q_UNUSED(c); // signal connect flag + Q_UNUSED(c) // signal connect flag const QDateTime threshold = QDateTime::currentDateTimeUtc().addDays(-365); // country and airports are "semi static" const CEntityFlags::Entity cachedDbEntities = this->getDbEntitiesWithCachedData(); // those caches are already read @@ -1196,7 +1196,7 @@ namespace BlackCore c = connect(m_dbInfoDataReader, &CInfoDataReader::dataRead, this, &CWebDataServices::dataRead); Q_ASSERT_X(c, Q_FUNC_INFO, "Info reader connect failed"); c = connect(m_dbInfoDataReader, &CInfoDataReader::databaseReaderMessages, this, &CWebDataServices::databaseReaderMessages); - Q_UNUSED(c); + Q_UNUSED(c) // start in own thread m_dbInfoDataReader->start(QThread::LowPriority);