flightsimulatorplatfrom/pilotclient
1
0
Fork 0
mirror of https://github.com/swift-project/pilotclient.git synced 2026-04-02 06:35:52 +08:00
Code Issues Packages Projects Releases Wiki Activity
Files
299859646ab268a299a903cac468a7552c03657d
pilotclient/src/blackcore/vatsim/vatsimdatafilereader.cpp
Klaus Basan 299859646a VATSIM data file parser "fixes" 
- Limit the number of parser errors going to the log
- stop parsing if no attributes can be obtained (otherwise we see too many parser messages)
2020-06-12 18:42:42 +01:00

518 lines
24 KiB
C++
Raw Blame History

/* Copyright (C) 2013
* swift project Community / Contributors
*
* This file is part of swift project. It is subject to the license terms in the LICENSE file found in the top-level
* directory of this distribution. No part of swift project, including this file, may be copied, modified, propagated,
* or distributed except according to the terms contained in the LICENSE file.
*/
#include "blackcore/vatsim/vatsimdatafilereader.h"
#include "blackcore/application.h"
#include "blackmisc/aviation/aircraftsituation.h"
#include "blackmisc/aviation/altitude.h"
#include "blackmisc/aviation/atcstation.h"
#include "blackmisc/geo/coordinategeodetic.h"
#include "blackmisc/network/entityflags.h"
#include "blackmisc/network/server.h"
#include "blackmisc/network/url.h"
#include "blackmisc/network/urllist.h"
#include "blackmisc/network/user.h"
#include "blackmisc/pq/frequency.h"
#include "blackmisc/pq/length.h"
#include "blackmisc/pq/speed.h"
#include "blackmisc/pq/units.h"
#include "blackmisc/logmessage.h"
#include "blackmisc/compare.h"
#include "blackmisc/predicates.h"
#include "blackmisc/range.h"
#include "blackmisc/simulation/simulatedaircraft.h"
#include "blackmisc/statusmessage.h"
#include "blackmisc/verify.h"
#include <QStringBuilder>
#include <QByteArray>
#include <QDateTime>
#include <QMetaObject>
#include <QNetworkReply>
#include <QReadLocker>
#include <QRegularExpression>
#include <QScopedPointer>
#include <QScopedPointerDeleteLater>
#include <QTimer>
#include <QUrl>
#include <QWriteLocker>
#include <Qt>
#include <QtGlobal>
#include <QPointer>
using namespace BlackMisc;
using namespace BlackMisc::Aviation;
using namespace BlackMisc::Audio;
using namespace BlackMisc::Network;
using namespace BlackMisc::Geo;
using namespace BlackMisc::Simulation;
using namespace BlackMisc::PhysicalQuantities;
using namespace BlackCore::Data;
namespace BlackCore
{
namespace Vatsim
{
CVatsimDataFileReader::CVatsimDataFileReader(QObject *owner) :
CThreadedReader(owner, "CVatsimDataFileReader"),
CEcosystemAware(CEcosystemAware::providerIfPossible(owner))
{
this->reloadSettings();
}
CSimulatedAircraftList CVatsimDataFileReader::getAircraft() const
{
QReadLocker rl(&m_lock);
return m_aircraft;
}
CAtcStationList CVatsimDataFileReader::getAtcStations() const
{
QReadLocker rl(&m_lock);
return m_atcStations;
}
CAtcStationList CVatsimDataFileReader::getAtcStationsForCallsign(const CCallsign &callsign) const
{
const CCallsignSet cs({callsign});
return this->getAtcStationsForCallsigns(cs);
}
CAtcStationList CVatsimDataFileReader::getAtcStationsForCallsigns(const CCallsignSet &callsigns) const
{
return this->getAtcStations().findByCallsigns(callsigns);
}
CServerList CVatsimDataFileReader::getVoiceServers() const
{
return m_lastGoodSetup.get().getVoiceServers();
}
CServerList CVatsimDataFileReader::getFsdServers() const
{
return m_lastGoodSetup.get().getFsdServers();
}
CUserList CVatsimDataFileReader::getPilotsForCallsigns(const CCallsignSet &callsigns) const
{
return this->getAircraft().findByCallsigns(callsigns).transform(Predicates::MemberTransform(&CSimulatedAircraft::getPilot));
}
CUserList CVatsimDataFileReader::getPilotsForCallsign(const CCallsign &callsign) const
{
const CCallsignSet callsigns({callsign});
return this->getPilotsForCallsigns(callsigns);
}
CAirlineIcaoCode CVatsimDataFileReader::getAirlineIcaoCode(const CCallsign &callsign) const
{
const CSimulatedAircraft aircraft = this->getAircraft().findFirstByCallsign(callsign);
return aircraft.getAirlineIcaoCode();
}
CAircraftIcaoCode CVatsimDataFileReader::getAircraftIcaoCode(const CCallsign &callsign) const
{
const CSimulatedAircraft aircraft = this->getAircraft().findFirstByCallsign(callsign);
return aircraft.getAircraftIcaoCode();
}
CVoiceCapabilities CVatsimDataFileReader::getVoiceCapabilityForCallsign(const CCallsign &callsign) const
{
if (callsign.isEmpty()) { return CVoiceCapabilities(); }
QReadLocker rl(&m_lock);
return m_flightPlanRemarks.value(callsign).getVoiceCapabilities();
}
CFlightPlanRemarks CVatsimDataFileReader::getFlightPlanRemarksForCallsign(const CCallsign &callsign) const
{
if (callsign.isEmpty()) { return QString(); }
QReadLocker rl(&m_lock);
return m_flightPlanRemarks.value(callsign);
}
void CVatsimDataFileReader::updateWithVatsimDataFileData(CSimulatedAircraft &aircraftToBeUdpated) const
{
this->getAircraft().updateWithVatsimDataFileData(aircraftToBeUdpated);
}
CUserList CVatsimDataFileReader::getControllersForCallsign(const CCallsign &callsign) const
{
const CCallsignSet cs({callsign});
return this->getControllersForCallsigns(cs);
}
CUserList CVatsimDataFileReader::getControllersForCallsigns(const CCallsignSet &callsigns) const
{
return this->getAtcStations().findByCallsigns(callsigns).transform(Predicates::MemberTransform(&CAtcStation::getController));
}
CUserList CVatsimDataFileReader::getUsersForCallsign(const CCallsign &callsign) const
{
const CCallsignSet callsigns({callsign});
return this->getUsersForCallsigns(callsigns);
}
CUserList CVatsimDataFileReader::getUsersForCallsigns(const CCallsignSet &callsigns) const
{
CUserList users;
if (callsigns.isEmpty()) { return users; }
for (const CCallsign &callsign : callsigns)
{
users.push_back(this->getPilotsForCallsign(callsign));
users.push_back(this->getControllersForCallsign(callsign));
}
return users;
}
void CVatsimDataFileReader::readInBackgroundThread()
{
QPointer<CVatsimDataFileReader> myself(this);
QTimer::singleShot(0, this, [ = ]
{
if (!myself) { return; }
myself->read();
});
}
void CVatsimDataFileReader::doWorkImpl()
{
this->read();
}
void CVatsimDataFileReader::read()
{
this->threadAssertCheck();
if (!this->doWorkCheck()) { return; }
if (!this->isInternetAccessible("No network/internet access, cannot read VATSIM data file")) { return; }
if (this->isNotVATSIMEcosystem()) { return; }
// round robin for load balancing
// remark: Don't use QThread to run network operations in the background
// see http://qt-project.org/doc/qt-4.7/qnetworkaccessmanager.html
Q_ASSERT_X(sApp, Q_FUNC_INFO, "Missing application");
CFailoverUrlList urls(sApp->getVatsimDataFileUrls());
const QUrl url(urls.obtainNextWorkingUrl(true));
if (url.isEmpty()) { return; }
this->getFromNetworkAndLog(url, { this, &CVatsimDataFileReader::parseVatsimFile});
}
void CVatsimDataFileReader::parseVatsimFile(QNetworkReply *nwReplyPtr)
{
// wrap pointer, make sure any exit cleans up reply
// required to use delete later as object is created in a different thread
QScopedPointer<QNetworkReply, QScopedPointerDeleteLater> nwReply(nwReplyPtr);
this->threadAssertCheck();
if (this->isNotVATSIMEcosystem()) { return; }
// Worker thread, make sure to write only synced here!
if (!this->doWorkCheck())
{
CLogMessage(this).info(u"Terminated VATSIM file parsing process");
return; // stop, terminate straight away, ending thread
}
this->logNetworkReplyReceived(nwReplyPtr);
QStringList illegalEquipmentCodes;
const QUrl url = nwReply->url();
const QString urlString = url.toString();
if (nwReply->error() == QNetworkReply::NoError)
{
const QString dataFileData = nwReply->readAll();
nwReply->close(); // close asap
if (dataFileData.isEmpty()) { return; }
if (!this->didContentChange(dataFileData)) // Quick check by hash
{
CLogMessage(this).info(u"VATSIM file '%1' has same content, skipped") << urlString;
return;
}
const QList<QStringRef> lines = splitLinesRefs(dataFileData);
if (lines.isEmpty()) { return; }
// build on local vars for thread safety
CServerList voiceServers;
CServerList fsdServers;
CAtcStationList atcStations;
CSimulatedAircraftList aircraft;
QMap<CCallsign, CFlightPlanRemarks> flightPlanRemarksMap;
QDateTime updateTimestampFromFile;
QStringList clientSectionAttributes;
Section section = SectionNone;
int invalidSections = 0;
QString currentLine; // declared outside of the for loop, to amortize the cost of allocation
for (const QStringRef &clRef : lines)
{
if (!this->doWorkCheck())
{
CLogMessage(this).info(u"Terminated VATSIM file parsing process"); // for users
return; // stop, terminate straight away, ending thread
}
// parse lines
currentLine = clRef.toString().trimmed();
if (currentLine.isEmpty()) continue;
if (currentLine.startsWith(";"))
{
if (clientSectionAttributes.isEmpty() && currentLine.contains("!CLIENTS SECTION", Qt::CaseInsensitive))
{
// ; !CLIENTS section
const int i = currentLine.lastIndexOf(' ');
const QVector<QStringRef> attributes = currentLine.midRef(i).trimmed().split(':', QString::SkipEmptyParts);
for (const QStringRef &attr : attributes) { clientSectionAttributes.push_back(attr.toString().trimmed().toLower()); }
section = SectionNone; // reset
// consistency check to avoid tons of parsing errors afterwards
// normally we have 40 attributes
if (attributes.size() < 10)
{
CLogMessage(this).warning(u"Too few (%1) attributes in VATSIM file, CANCEL parsing. Line: '%2'") << attributes.size() << currentLine;
return;
}
}
continue;
}
else if (currentLine.startsWith("!"))
{
section = currentLineToSection(currentLine);
continue;
}
switch (section)
{
case SectionClients:
{
const bool logInconsistencies = invalidSections < 5; // flood protection
const QMap<QString, QString> clientPartsMap = clientPartsToMap(currentLine, clientSectionAttributes, logInconsistencies);
const CCallsign callsign = CCallsign(clientPartsMap["callsign"]);
if (callsign.isEmpty())
{
invalidSections++;
break;
}
const CUser user(clientPartsMap["cid"], clientPartsMap["realname"], callsign);
const QString clientType = clientPartsMap["clienttype"].toLower();
if (clientType.isEmpty()) { break; } // sometimes type is empty
bool ok;
bool validPos = true;
QStringList posMsg;
const double lat = clientPartsMap["latitude"].toDouble(&ok);
if (!ok) { validPos = false; posMsg << QStringLiteral("latitude: '%1'").arg(clientPartsMap["latitude"]); }
const double lng = clientPartsMap["longitude"].toDouble(&ok);
if (!ok) { validPos = false; posMsg << QStringLiteral("longitude: '%1'").arg(clientPartsMap["longitude"]); }
const double alt = clientPartsMap["altitude"].toDouble(&ok);
if (!ok) { validPos = false; posMsg << QStringLiteral("altitude: '%1'").arg(clientPartsMap["altitude"]); }
const CCoordinateGeodetic position = validPos ? CCoordinateGeodetic(lat, lng, alt) : CCoordinateGeodetic::null();
Q_ASSERT_X((validPos && posMsg.isEmpty()) || (!validPos && !posMsg.isEmpty()), Q_FUNC_INFO, "Inconsistent data");
if (!posMsg.isEmpty())
{
// Only info not to flood lof with warning
CLogMessage(this).validationInfo(u"Callsign '%1' %2 (VATSIM data file)") << callsign << posMsg.join(", ");
}
const CFrequency frequency = CFrequency(clientPartsMap["frequency"].toDouble(), CFrequencyUnit::MHz());
const QString flightPlanRemarks = clientPartsMap["planned_remarks"].trimmed();
// Voice capabilities
if (!flightPlanRemarks.isEmpty())
{
// CFlightPlanRemarks contains voice capabilities and other parsed values
flightPlanRemarksMap[callsign] = CFlightPlanRemarks(flightPlanRemarks);
}
// set as per ATC/pilot
if (clientType.startsWith('p'))
{
// Pilot section
const double groundSpeedKts = clientPartsMap["groundspeed"].toDouble();
CAircraftSituation situation(position);
situation.setGroundSpeed(CSpeed(groundSpeedKts, CSpeedUnit::kts()));
CSimulatedAircraft currentAircraft(user.getCallsign().getStringAsSet(), user, situation);
const QString equipmentCodeAndAircraft = clientPartsMap["planned_aircraft"].trimmed();
if (!equipmentCodeAndAircraft.isEmpty())
{
const QString aircraftIcaoCode = CFlightPlan::aircraftIcaoCodeFromEquipmentCode(equipmentCodeAndAircraft);
if (CAircraftIcaoCode::isValidDesignator(aircraftIcaoCode))
{
currentAircraft.setAircraftIcaoDesignator(aircraftIcaoCode);
}
else
{
illegalEquipmentCodes.append(equipmentCodeAndAircraft);
}
}
aircraft.push_back(currentAircraft);
}
else if (clientType.startsWith('a'))
{
// ATC section
CLength range;
// should be alread have alt/height position.setGeodeticHeight(altitude);
// the altitude is elevation for a station
CAtcStation station(user.getCallsign().getStringAsSet(), user, frequency, position, range);
station.setOnline(true);
atcStations.push_back(station);
}
else
{
BLACK_VERIFY_X(false, Q_FUNC_INFO, "Wrong client type");
break;
}
}
break;
case SectionGeneral:
{
if (currentLine.contains("UPDATE"))
{
const QStringList updateParts = currentLine.replace(" ", "").split('=');
if (updateParts.length() < 2) { break; }
const QString dts = updateParts.at(1).trimmed();
updateTimestampFromFile = fromStringUtc(dts, "yyyyMMddHHmmss");
const bool alreadyRead = (updateTimestampFromFile == this->getUpdateTimestamp());
if (alreadyRead)
{
CLogMessage(this).info(u"VATSIM file has same timestamp, skipped");
return;
}
}
}
break;
case SectionFsdServers:
{
// ident:hostname_or_IP:location:name:clients_connection_allowed:
const QStringList fsdServerParts = currentLine.split(':');
if (fsdServerParts.size() < 5) { break; }
if (!fsdServerParts.at(4).trimmed().contains('1')) { break; } // allowed?
const QString description(fsdServerParts.at(2)); // part(3) could be added
const CServer fsdServer(fsdServerParts.at(0), description, fsdServerParts.at(1), 6809,
CUser("id", "real name", "email", "password"),
CFsdSetup::vatsimStandard(), CVoiceSetup::vatsimStandard(),
CEcosystem(CEcosystem::VATSIM), CServer::FSDServerVatsim);
fsdServers.push_back(fsdServer);
}
break;
case SectionVoiceServers:
{
// hostname_or_IP:location:name:clients_connection_allowed:type_of_voice_server:
const QStringList voiceServerParts = currentLine.split(':');
if (voiceServerParts.size() < 4) { break; }
if (!voiceServerParts.at(3).trimmed().contains('1')) { break; } // allowed?
const CServer voiceServer(voiceServerParts.at(1), voiceServerParts.at(2), voiceServerParts.at(0), -1,
CUser(),
CFsdSetup(), CVoiceSetup::vatsimStandard(),
CEcosystem(CEcosystem::VATSIM), CServer::VoiceServerVatsim);
voiceServers.push_back(voiceServer);
}
break;
case SectionNone:
default:
break;
} // switch section
} // for each line
// Setup for VATSIM servers and sorting for comparison
fsdServers.sortBy(&CServer::getName, &CServer::getDescription);
voiceServers.sortBy(&CServer::getName, &CServer::getDescription);
// this part needs to be synchronized
{
QWriteLocker wl(&m_lock);
this->setUpdateTimestamp(updateTimestampFromFile);
m_aircraft = aircraft;
m_atcStations = atcStations;
m_flightPlanRemarks = flightPlanRemarksMap;
}
// update cache itself is thread safe
CVatsimSetup vs(m_lastGoodSetup.get());
const bool changedSetup = vs.setServers(fsdServers, voiceServers);
if (changedSetup)
{
vs.setUtcTimestamp(updateTimestampFromFile);
m_lastGoodSetup.set(vs);
}
// warnings, if required
if (!illegalEquipmentCodes.isEmpty())
{
CVatsimDataFileReader::logInconsistentData(
CStatusMessage(this, CStatusMessage::SeverityInfo, u"Illegal / ignored equipment code(s) in VATSIM data file: %1") << illegalEquipmentCodes.join(", ")
);
}
// data read finished
emit this->dataFileRead(lines.count());
emit this->dataRead(CEntityFlags::VatsimDataFile, CEntityFlags::ReadFinished, lines.count(), url);
}
else
{
// network error
CLogMessage(this).warning(u"Reading VATSIM data file failed '%1' '%2'") << nwReply->errorString() << urlString;
nwReply->abort();
emit this->dataRead(CEntityFlags::VatsimDataFile, CEntityFlags::ReadFailed, 0, url);
}
}
void CVatsimDataFileReader::reloadSettings()
{
CReaderSettings s = m_settings.get();
setInitialAndPeriodicTime(s.getInitialTime().toMs(), s.getPeriodicTime().toMs());
}
const QMap<QString, QString> CVatsimDataFileReader::clientPartsToMap(const QString &currentLine, const QStringList &clientSectionAttributes, bool logInconsistency)
{
QMap<QString, QString> parts;
if (currentLine.isEmpty()) { return parts; }
QStringList clientParts = currentLine.split(':');
// remove last empty item if required
if (currentLine.endsWith(':')) { clientParts.removeLast(); }
const int noParts = clientParts.size();
const int noAttributes = clientSectionAttributes.size();
const bool valid = (noParts == noAttributes);
// valid data?
if (!valid)
{
if (logInconsistency)
{
logInconsistentData(
CStatusMessage(static_cast<CVatsimDataFileReader *>(nullptr), CStatusMessage::SeverityInfo, u"VATSIM data file client parts: %1 attributes: %2 line: '%3'") << clientParts.size() << clientSectionAttributes.size() << currentLine);
}
return parts;
}
for (int i = 0; i < clientSectionAttributes.size(); i++)
{
// section attributes are the column names
const QString attribute(clientSectionAttributes.at(i));
parts.insert(attribute, clientParts.at(i));
}
return parts;
}
CVatsimDataFileReader::Section CVatsimDataFileReader::currentLineToSection(const QString &currentLine)
{
if (currentLine.contains("!GENERAL", Qt::CaseInsensitive)) { return SectionGeneral; }
if (currentLine.contains("!VOICE SERVERS", Qt::CaseInsensitive)) { return SectionVoiceServers; }
if (currentLine.contains("!SERVERS", Qt::CaseInsensitive)) { return SectionFsdServers; }
if (currentLine.contains("!CLIENTS", Qt::CaseInsensitive)) { return SectionClients; }
return SectionNone;
}
} // ns
} // ns
Reference in New Issue View Git Blame Copy Permalink