flightsimulatorplatfrom/pilotclient
1
0
Fork 0
mirror of https://github.com/swift-project/pilotclient.git synced 2026-03-22 14:55:36 +08:00
Code Issues Packages Projects Releases Wiki Activity

Update VATSIM data file reader to read new JSON data file format

Browse Source
This commit is contained in:
Mat Sutcliffe
2020-12-05 15:41:06 +00:00
parent 9014a673f1
commit 50eebb799d
8 changed files with 138 additions and 240 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
resources/share/shared/bootstrap/bootstrap.json
View File

@@ -65,7 +65,7 @@
"vatsimDataFileUrls": {
"containerbase": [
{
"url": "http://info.vroute.net/vatsim-data.txt"
"url": "https://data.vatsim.net/v3/vatsim-data.json"
}
]
},

2
src/blackcore/data/globalsetup.cpp
View File

@@ -50,7 +50,7 @@ namespace BlackCore
m_vatsimBookingsUrl = CUrl("http://vatbook.euroutepro.com/xml2.php");
m_vatsimMetarsUrls = CUrlList{"http://metar.vatsim.net/metar.php"};
m_vatsimStatusFileUrls = CUrlList{ "https://status.vatsim.net" };
m_vatsimDataFileUrls = CUrlList{ "http://info.vroute.net/vatsim-data.txt" };
m_vatsimDataFileUrls = CUrlList{ "https://data.vatsim.net/v3/vatsim-data.json" };
m_sharedUrls = CUrlList
{
"https://datastore.swift-project.net/shared/",

308
src/blackcore/vatsim/vatsimdatafilereader.cpp
View File

@@ -232,201 +232,64 @@ namespace BlackCore
CLogMessage(this).info(u"VATSIM file '%1' has same content, skipped") << urlString;
return;
}
const QList<QStringRef> lines = splitLinesRefs(dataFileData);
if (lines.isEmpty()) { return; }
auto jsonDoc = QJsonDocument::fromJson(dataFileData.toUtf8());
if (jsonDoc.isEmpty()) { return; }
// build on local vars for thread safety
CServerList voiceServers;
CServerList fsdServers;
CAtcStationList atcStations;
CSimulatedAircraftList aircraft;
QMap<CCallsign, CFlightPlanRemarks> flightPlanRemarksMap;
QDateTime updateTimestampFromFile;
auto updateTimestampFromFile = QDateTime::fromString(jsonDoc["general"]["update_timestamp"].toString(), Qt::ISODateWithMs);
QStringList clientSectionAttributes;
Section section = SectionNone;
int invalidSections = 0;
const bool alreadyRead = (updateTimestampFromFile == this->getUpdateTimestamp());
if (alreadyRead)
{
CLogMessage(this).info(u"VATSIM file has same timestamp, skipped");
return;
}
QString currentLine; // declared outside of the for loop, to amortize the cost of allocation
for (const QStringRef &clRef : lines)
for (QJsonValueRef pilot : jsonDoc["pilots"].toArray())
{
if (!this->doWorkCheck())
{
CLogMessage(this).info(u"Terminated VATSIM file parsing process"); // for users
return; // stop, terminate straight away, ending thread
CLogMessage(this).info(u"Terminated VATSIM file parsing process");
return;
}
// parse lines
currentLine = clRef.toString().trimmed();
if (currentLine.isEmpty()) continue;
if (currentLine.startsWith(";"))
aircraft.push_back(parsePilot(pilot.toObject(), illegalEquipmentCodes));
flightPlanRemarksMap.insert(aircraft.back().getCallsign(), parseFlightPlanRemarks(pilot.toObject()));
}
for (QJsonValueRef controller : jsonDoc["controllers"].toArray())
{
if (!this->doWorkCheck())
{
if (clientSectionAttributes.isEmpty() && currentLine.contains("!CLIENTS SECTION", Qt::CaseInsensitive))
{
// ; !CLIENTS section
const int i = currentLine.lastIndexOf(' ');
const QVector<QStringRef> attributes = currentLine.midRef(i).trimmed().split(':', Qt::SkipEmptyParts);
for (const QStringRef &attr : attributes) { clientSectionAttributes.push_back(attr.toString().trimmed().toLower()); }
section = SectionNone; // reset
// consistency check to avoid tons of parsing errors afterwards
// normally we have 40 attributes
if (attributes.size() < 10)
{
CLogMessage(this).warning(u"Too few (%1) attributes in VATSIM file, CANCEL parsing. Line: '%2'") << attributes.size() << currentLine;
return;
}
}
continue;
CLogMessage(this).info(u"Terminated VATSIM file parsing process");
return;
}
else if (currentLine.startsWith("!"))
atcStations.push_back(parseController(controller.toObject()));
}
for (QJsonValueRef atis : jsonDoc["atis"].toArray())
{
if (!this->doWorkCheck())
{
section = currentLineToSection(currentLine);
continue;
CLogMessage(this).info(u"Terminated VATSIM file parsing process");
return;
}
switch (section)
atcStations.push_back(parseController(atis.toObject()));
}
for (QJsonValueRef server : jsonDoc["servers"].toArray())
{
if (!this->doWorkCheck())
{
case SectionClients:
{
const bool logInconsistencies = invalidSections < 5; // flood protection
const QMap<QString, QString> clientPartsMap = clientPartsToMap(currentLine, clientSectionAttributes, logInconsistencies);
const CCallsign callsign = CCallsign(clientPartsMap["callsign"]);
if (callsign.isEmpty())
{
invalidSections++;
break;
}
const CUser user(clientPartsMap["cid"], clientPartsMap["realname"], callsign);
const QString clientType = clientPartsMap["clienttype"].toLower();
if (clientType.isEmpty()) { break; } // sometimes type is empty
bool ok;
bool validPos = true;
QStringList posMsg;
const double lat = clientPartsMap["latitude"].toDouble(&ok);
if (!ok) { validPos = false; posMsg << QStringLiteral("latitude: '%1'").arg(clientPartsMap["latitude"]); }
const double lng = clientPartsMap["longitude"].toDouble(&ok);
if (!ok) { validPos = false; posMsg << QStringLiteral("longitude: '%1'").arg(clientPartsMap["longitude"]); }
const double alt = clientPartsMap["altitude"].toDouble(&ok);
if (!ok) { validPos = false; posMsg << QStringLiteral("altitude: '%1'").arg(clientPartsMap["altitude"]); }
const CCoordinateGeodetic position = validPos ? CCoordinateGeodetic(lat, lng, alt) : CCoordinateGeodetic::null();
Q_ASSERT_X((validPos && posMsg.isEmpty()) || (!validPos && !posMsg.isEmpty()), Q_FUNC_INFO, "Inconsistent data");
if (!posMsg.isEmpty())
{
// Only info not to flood lof with warning
CLogMessage(this).validationInfo(u"Callsign '%1' %2 (VATSIM data file)") << callsign << posMsg.join(", ");
}
const CFrequency frequency = CFrequency(clientPartsMap["frequency"].toDouble(), CFrequencyUnit::MHz());
const QString flightPlanRemarks = clientPartsMap["planned_remarks"].trimmed();
// Voice capabilities
if (!flightPlanRemarks.isEmpty())
{
// CFlightPlanRemarks contains voice capabilities and other parsed values
flightPlanRemarksMap[callsign] = CFlightPlanRemarks(flightPlanRemarks);
}
// set as per ATC/pilot
if (clientType.startsWith('p'))
{
// Pilot section
const double groundSpeedKts = clientPartsMap["groundspeed"].toDouble();
CAircraftSituation situation(position);
situation.setGroundSpeed(CSpeed(groundSpeedKts, CSpeedUnit::kts()));
CSimulatedAircraft currentAircraft(user.getCallsign().getStringAsSet(), user, situation);
const QString equipmentCodeAndAircraft = clientPartsMap["planned_aircraft"].trimmed();
if (!equipmentCodeAndAircraft.isEmpty())
{
const QString aircraftIcaoCode = CFlightPlan::aircraftIcaoCodeFromEquipmentCode(equipmentCodeAndAircraft);
if (CAircraftIcaoCode::isValidDesignator(aircraftIcaoCode))
{
currentAircraft.setAircraftIcaoDesignator(aircraftIcaoCode);
}
else
{
illegalEquipmentCodes.append(equipmentCodeAndAircraft);
}
}
aircraft.push_back(currentAircraft);
}
else if (clientType.startsWith('a'))
{
// ATC section
CLength range;
// should be alread have alt/height position.setGeodeticHeight(altitude);
// the altitude is elevation for a station
CAtcStation station(user.getCallsign().getStringAsSet(), user, frequency, position, range);
station.setOnline(true);
atcStations.push_back(station);
}
else
{
BLACK_VERIFY_X(false, Q_FUNC_INFO, "Wrong client type");
break;
}
}
break;
case SectionGeneral:
{
if (currentLine.contains("UPDATE"))
{
const QStringList updateParts = currentLine.replace(" ", "").split('=');
if (updateParts.length() < 2) { break; }
const QString dts = updateParts.at(1).trimmed();
updateTimestampFromFile = fromStringUtc(dts, "yyyyMMddHHmmss");
const bool alreadyRead = (updateTimestampFromFile == this->getUpdateTimestamp());
if (alreadyRead)
{
CLogMessage(this).info(u"VATSIM file has same timestamp, skipped");
return;
}
}
}
break;
case SectionFsdServers:
{
// ident:hostname_or_IP:location:name:clients_connection_allowed:
const QStringList fsdServerParts = currentLine.split(':');
if (fsdServerParts.size() < 5) { break; }
if (!fsdServerParts.at(4).trimmed().contains('1')) { break; } // allowed?
const QString description(fsdServerParts.at(2)); // part(3) could be added
const CServer fsdServer(fsdServerParts.at(0), description, fsdServerParts.at(1), 6809,
CUser("id", "real name", "email", "password"),
CFsdSetup::vatsimStandard(), CVoiceSetup::vatsimStandard(),
CEcosystem(CEcosystem::VATSIM), CServer::FSDServerVatsim);
fsdServers.push_back(fsdServer);
}
break;
case SectionVoiceServers:
{
// hostname_or_IP:location:name:clients_connection_allowed:type_of_voice_server:
const QStringList voiceServerParts = currentLine.split(':');
if (voiceServerParts.size() < 4) { break; }
if (!voiceServerParts.at(3).trimmed().contains('1')) { break; } // allowed?
const CServer voiceServer(voiceServerParts.at(1), voiceServerParts.at(2), voiceServerParts.at(0), -1,
CUser(),
CFsdSetup(), CVoiceSetup::vatsimStandard(),
CEcosystem(CEcosystem::VATSIM), CServer::VoiceServerVatsim);
voiceServers.push_back(voiceServer);
}
break;
case SectionNone:
default:
break;
} // switch section
} // for each line
CLogMessage(this).info(u"Terminated VATSIM file parsing process");
return;
}
fsdServers.push_back(parseServer(server.toObject()));
if (!fsdServers.back().hasName()) { fsdServers.pop_back(); }
}
// Setup for VATSIM servers and sorting for comparison
fsdServers.sortBy(&CServer::getName, &CServer::getDescription);
voiceServers.sortBy(&CServer::getName, &CServer::getDescription);
// this part needs to be synchronized
{
@@ -439,7 +302,7 @@ namespace BlackCore
// update cache itself is thread safe
CVatsimSetup vs(m_lastGoodSetup.get());
const bool changedSetup = vs.setServers(fsdServers, voiceServers);
const bool changedSetup = vs.setServers(fsdServers, {});
if (changedSetup)
{
vs.setUtcTimestamp(updateTimestampFromFile);
@@ -455,8 +318,8 @@ namespace BlackCore
}
// data read finished
emit this->dataFileRead(lines.count());
emit this->dataRead(CEntityFlags::VatsimDataFile, CEntityFlags::ReadFinished, lines.count(), url);
emit this->dataFileRead(dataFileData.size() / 1000);
emit this->dataRead(CEntityFlags::VatsimDataFile, CEntityFlags::ReadFinished, dataFileData.size() / 1000, url);
}
else
{
@@ -467,51 +330,58 @@ namespace BlackCore
}
}
CSimulatedAircraft CVatsimDataFileReader::parsePilot(const QJsonObject &pilot, QStringList &o_illegalEquipmentCodes) const
{
const CCallsign callsign(pilot["callsign"].toString());
const CUser user(pilot["cid"].toString(), pilot["name"].toString(), callsign);
const CCoordinateGeodetic position(pilot["latitude"].toDouble(), pilot["longitude"].toDouble(), pilot["altitude"].toInt());
const CHeading heading(pilot["heading"].toInt(), CAngleUnit::deg());
const CSpeed groundspeed(pilot["groundspeed"].toInt(), CSpeedUnit::kts());
const CAircraftSituation situation(callsign, position, heading, {}, {}, groundspeed);
CSimulatedAircraft aircraft(callsign, user, situation);
const QString icaoAndEquipment(pilot["flight_plan"]["aircraft"].toString().trimmed());
const QString icao(CFlightPlan::aircraftIcaoCodeFromEquipmentCode(icaoAndEquipment));
if (CAircraftIcaoCode::isValidDesignator(icao))
{
aircraft.setAircraftIcaoCode(icao);
}
else if (!icaoAndEquipment.isEmpty())
{
o_illegalEquipmentCodes.push_back(icaoAndEquipment);
}
aircraft.setTransponderCode(pilot["transponder"].toString().toInt());
return aircraft;
}
CFlightPlanRemarks CVatsimDataFileReader::parseFlightPlanRemarks(const QJsonObject &pilot) const
{
return CFlightPlanRemarks(pilot["flight_plan"]["remarks"].toString().trimmed());
}
CAtcStation CVatsimDataFileReader::parseController(const QJsonObject &controller) const
{
const CCallsign callsign(controller["callsign"].toString());
const CUser user(controller["cid"].toString(), controller["name"].toString(), callsign);
const CFrequency freq(controller["frequency"].toString().toDouble(), CFrequencyUnit::kHz());
const CLength range(controller["visual_range"].toInt(), CLengthUnit::NM());
const QJsonArray atisLines = controller["text_atis"].toArray();
const auto atisText = makeRange(atisLines).transform([](auto line) { return line.toString(); });
const CInformationMessage atis(CInformationMessage::ATIS, atisText.to<QStringList>().join('\n'));
return CAtcStation(callsign, user, freq, {}, range, true, {}, {}, atis);
}
CServer CVatsimDataFileReader::parseServer(const QJsonObject &server) const
{
return CServer(server["name"].toString(), server["location"].toString(),
server["hostname_or_ip"].toString(), 6809, CUser("id", "real name", "email", "password"),
CFsdSetup::vatsimStandard(), CVoiceSetup::vatsimStandard(), CEcosystem::VATSIM,
CServer::FSDServerVatsim, server["clients_connection_allowed"].toInt());
}
void CVatsimDataFileReader::reloadSettings()
{
CReaderSettings s = m_settings.get();
setInitialAndPeriodicTime(s.getInitialTime().toMs(), s.getPeriodicTime().toMs());
}
const QMap<QString, QString> CVatsimDataFileReader::clientPartsToMap(const QString &currentLine, const QStringList &clientSectionAttributes, bool logInconsistency)
{
QMap<QString, QString> parts;
if (currentLine.isEmpty()) { return parts; }
QStringList clientParts = currentLine.split(':');
// remove last empty item if required
if (currentLine.endsWith(':')) { clientParts.removeLast(); }
const int noParts = clientParts.size();
const int noAttributes = clientSectionAttributes.size();
const bool valid = (noParts == noAttributes);
// valid data?
if (!valid)
{
if (logInconsistency)
{
logInconsistentData(
CStatusMessage(static_cast<CVatsimDataFileReader *>(nullptr), CStatusMessage::SeverityInfo, u"VATSIM data file client parts: %1 attributes: %2 line: '%3'") << clientParts.size() << clientSectionAttributes.size() << currentLine);
}
return parts;
}
for (int i = 0; i < clientSectionAttributes.size(); i++)
{
// section attributes are the column names
const QString attribute(clientSectionAttributes.at(i));
parts.insert(attribute, clientParts.at(i));
}
return parts;
}
CVatsimDataFileReader::Section CVatsimDataFileReader::currentLineToSection(const QString &currentLine)
{
if (currentLine.contains("!GENERAL", Qt::CaseInsensitive)) { return SectionGeneral; }
if (currentLine.contains("!VOICE SERVERS", Qt::CaseInsensitive)) { return SectionVoiceServers; }
if (currentLine.contains("!SERVERS", Qt::CaseInsensitive)) { return SectionFsdServers; }
if (currentLine.contains("!CLIENTS", Qt::CaseInsensitive)) { return SectionClients; }
return SectionNone;
}
} // ns
} // ns

14
src/blackcore/vatsim/vatsimdatafilereader.h
View File

@@ -124,7 +124,7 @@ namespace BlackCore
signals:
//! Data have been read
void dataFileRead(int lines);
void dataFileRead(int kB);
//! Data have been read
void dataRead(BlackMisc::Network::CEntityFlags::Entity entity, BlackMisc::Network::CEntityFlags::ReadState state, int number, const QUrl &url);
@@ -155,18 +155,16 @@ namespace BlackCore
//! Data have been read, parse VATSIM file
void parseVatsimFile(QNetworkReply *nwReply);
BlackMisc::Simulation::CSimulatedAircraft parsePilot(const QJsonObject &, QStringList &o_illegalEquipmentCodes) const;
BlackMisc::Aviation::CFlightPlanRemarks parseFlightPlanRemarks(const QJsonObject &) const;
BlackMisc::Aviation::CAtcStation parseController(const QJsonObject &) const;
BlackMisc::Network::CServer parseServer(const QJsonObject &) const;
//! Read / re-read data file
void read();
//! Reload the reader settings
void reloadSettings();
//! Split line and assign values to their corresponding attribute names
//! \remark attributes expected as lower case
static const QMap<QString, QString> clientPartsToMap(const QString &currentLine, const QStringList &clientSectionAttributes, bool logInconsistency);
//! Get current section
static Section currentLineToSection(const QString &currentLine);
};
} // ns
} // ns

2
src/blackcore/vatsim/vatsimstatusfilereader.cpp
View File

@@ -147,7 +147,7 @@ namespace BlackCore
const QString key(parts[0].trimmed().toLower());
const QString value(parts[1].trimmed());
const CUrl url(value);
if (key.startsWith("url0"))
if (key.startsWith("json3"))
{
dataFileUrls.push_back(url);
}

4
src/blackcore/webdataservices.cpp
View File

@@ -1323,9 +1323,9 @@ namespace BlackCore
CLogMessage(this).info(u"Read %1 METARs") << metars.size();
}
void CWebDataServices::vatsimDataFileRead(int lines)
void CWebDataServices::vatsimDataFileRead(int kB)
{
CLogMessage(this).info(u"Read VATSIM data file, %1 lines") << lines;
CLogMessage(this).info(u"Read VATSIM data file, %1 kB") << kB;
}
void CWebDataServices::vatsimStatusFileRead(int lines)

2
src/blackcore/webdataservices.h
View File

@@ -576,7 +576,7 @@ namespace BlackCore
void receivedMetars(const BlackMisc::Weather::CMetarList &metars);
//! VATSIM data file has been read
void vatsimDataFileRead(int lines);
void vatsimDataFileRead(int kB);
//! VATSIM status file has been read
void vatsimStatusFileRead(int lines);

44
src/blackmisc/aviation/flightplan.cpp
View File

@@ -745,20 +745,49 @@ namespace BlackMisc
QString CFlightPlan::aircraftIcaoCodeFromEquipmentCode(const QString &equipmentCodeAndAircraft)
{
// http://uk.flightaware.com/about/faq_aircraft_flight_plan_suffix.rvt
// we expect something like H/B772/F B773 B773/F
thread_local const QRegularExpression reg("/.");
QString aircraftIcaoCode(equipmentCodeAndAircraft);
aircraftIcaoCode = aircraftIcaoCode.replace(reg, "").trimmed().toUpper();
return aircraftIcaoCode;
return splitEquipmentCode(equipmentCodeAndAircraft)[1].trimmed().toUpper();
}
QStringList CFlightPlan::splitEquipmentCode(const QString &equipmentCodeAndAircraft)
{
static const QStringList empty({"", "", ""});
if (empty.isEmpty()) { return empty; }
QStringList firstSplit = equipmentCodeAndAircraft.split('-');
if (firstSplit.size() >= 2)
{
// format like B789/H-SDE1E2E3FGHIJ2J3J4J5M1RWXY/LB1D1
QString equipment = firstSplit.size() >= 2 ? firstSplit[1] : "";
QStringList split = firstSplit[0].split('/');
if (split.size() >= 3)
{
return { split[2], split[1], equipment.isEmpty() ? split[0] : equipment }; // "F/B789/H"
}
else if (split.size() >= 2)
{
if (split[0].size() <= 1) // "H/B789"
{
return { split[0], split[1], equipment };
}
else // "B789/H"
{
return { split[1], split[0], equipment };
}
}
else // "B789"
{
return { {}, split[0], equipment };
}
}
QStringList split = equipmentCodeAndAircraft.split('/');
if (split.length() == 3) { return split; } // "H/B738/F"
if (split.length() >= 3)
{
if (split[1].size() == 1 && CAircraftIcaoCode::isValidDesignator(split[0]))
{
using std::swap;
swap(split[0], split[1]); // "A359/H/L"
}
return split; // "H/B738/F"
}
if (split.length() == 2)
{
if (split[0].length() == 1)
@@ -766,6 +795,7 @@ namespace BlackMisc
// we assume prefix + ICAO
// e.g. "H/B748"
split.push_back("");
return split;
}
else
{