mirror of
https://github.com/OpenSquawk/OpenSquawk
synced 2026-07-02 00:15:50 +08:00
get rid of ptt and decide difference and unite into one endpoint
This commit is contained in:
114
app/pages/pm.vue
114
app/pages/pm.vue
@@ -484,14 +484,6 @@
|
||||
<span class="text-xs uppercase tracking-[0.3em] text-cyan-300">Letzte Übertragung</span>
|
||||
</div>
|
||||
<p class="text-sm text-white font-mono">{{ lastTransmission }}</p>
|
||||
<div v-if="lastEvaluation" class="flex items-center gap-2">
|
||||
<v-chip size="x-small" :color="getScoreColor(lastEvaluation.score)" variant="flat">
|
||||
Score: {{ lastEvaluation.score }}%
|
||||
</v-chip>
|
||||
<v-chip size="x-small" color="grey" variant="outlined">
|
||||
{{ lastEvaluation.recommendation }}
|
||||
</v-chip>
|
||||
</div>
|
||||
</v-card-text>
|
||||
</v-card>
|
||||
|
||||
@@ -570,7 +562,6 @@ const loading = ref(false)
|
||||
const error = ref('')
|
||||
const pilotInput = ref('')
|
||||
const lastTransmission = ref('')
|
||||
const lastEvaluation = ref<any>(null)
|
||||
const radioMode = ref<'atc' | 'intercom'>('atc')
|
||||
const isRecording = ref(false)
|
||||
const micPermission = ref(false)
|
||||
@@ -611,13 +602,6 @@ const normalizeExpectedText = (text: string): string => {
|
||||
return normalizeATCText(text, { ...vars.value, ...flags.value })
|
||||
}
|
||||
|
||||
const getScoreColor = (score: number): string => {
|
||||
if (score >= 90) return 'green'
|
||||
if (score >= 75) return 'cyan'
|
||||
if (score >= 50) return 'orange'
|
||||
return 'red'
|
||||
}
|
||||
|
||||
// VATSIM Integration
|
||||
const loadFlightPlans = async () => {
|
||||
if (!vatsimId.value) return
|
||||
@@ -670,7 +654,6 @@ const backToSetup = () => {
|
||||
currentScreen.value = 'login'
|
||||
selectedPlan.value = null
|
||||
lastTransmission.value = ''
|
||||
lastEvaluation.value = null
|
||||
resetCommunications()
|
||||
}
|
||||
|
||||
@@ -745,30 +728,59 @@ const processTransmission = async (audioBlob: Blob, isIntercom: boolean) => {
|
||||
const arrayBuffer = await audioBlob.arrayBuffer()
|
||||
const base64Audio = btoa(String.fromCharCode(...new Uint8Array(arrayBuffer)))
|
||||
|
||||
const result = await $fetch('/api/atc/ptt', {
|
||||
method: 'POST',
|
||||
body: {
|
||||
audio: base64Audio,
|
||||
expectedText: currentStep.value?.pilot || 'radio communication',
|
||||
moduleId: 'pilot-monitoring',
|
||||
lessonId: currentState.value?.id || 'general',
|
||||
format: 'webm'
|
||||
}
|
||||
})
|
||||
if (isIntercom) {
|
||||
// Handle intercom separately - simple transcription only
|
||||
const result = await $fetch('/api/atc/ptt', {
|
||||
method: 'POST',
|
||||
body: {
|
||||
audio: base64Audio,
|
||||
context: { // Minimal context for intercom
|
||||
state_id: currentState.value?.id || 'INTERCOM',
|
||||
state: {},
|
||||
candidates: [],
|
||||
variables: { callsign: vars.value.callsign },
|
||||
flags: {}
|
||||
},
|
||||
moduleId: 'pilot-monitoring-intercom',
|
||||
lessonId: 'intercom',
|
||||
format: 'webm'
|
||||
}
|
||||
})
|
||||
|
||||
if (result.success) {
|
||||
lastTransmission.value = result.transcription
|
||||
lastEvaluation.value = result.evaluation
|
||||
|
||||
if (isIntercom) {
|
||||
// Handle intercom (for checklists, etc.)
|
||||
if (result.success) {
|
||||
lastTransmission.value = `INTERCOM: ${result.transcription}`
|
||||
const transcription = result.transcription.toLowerCase()
|
||||
if (transcription.includes('checklist') || transcription.includes('check list')) {
|
||||
setTimeout(() => speakWithRadioEffects('Checklist functionality available in advanced mode'), 1000)
|
||||
}
|
||||
} else {
|
||||
// Handle ATC communication through Decision Tree
|
||||
await processPilotAndLLM(result.transcription)
|
||||
}
|
||||
} else {
|
||||
// Handle ATC communication - full PTT + Decision in one call
|
||||
const ctx = buildLLMContext('') // Build context first
|
||||
|
||||
const result = await $fetch('/api/atc/ptt', {
|
||||
method: 'POST',
|
||||
body: {
|
||||
audio: base64Audio,
|
||||
context: ctx, // Full LLM context
|
||||
moduleId: 'pilot-monitoring',
|
||||
lessonId: currentState.value?.id || 'general',
|
||||
format: 'webm'
|
||||
}
|
||||
})
|
||||
|
||||
if (result.success) {
|
||||
lastTransmission.value = result.transcription
|
||||
|
||||
// Apply the decision directly from PTT response
|
||||
applyLLMDecision(result.decision)
|
||||
|
||||
// If ATC should respond, speak it
|
||||
if (result.decision.controller_say_tpl && !result.decision.radio_check) {
|
||||
setTimeout(async () => {
|
||||
await speakWithRadioEffects(result.decision.controller_say_tpl!)
|
||||
}, 1000 + Math.random() * 2000)
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
@@ -812,7 +824,35 @@ const sendPilotText = async () => {
|
||||
pilotInput.value = ''
|
||||
lastTransmission.value = text
|
||||
|
||||
await processPilotAndLLM(text)
|
||||
// Process text input directly through decision engine
|
||||
const quickResponse = processPilotTransmission(text)
|
||||
|
||||
if (quickResponse) {
|
||||
// Radio check or emergency was handled directly
|
||||
return
|
||||
}
|
||||
|
||||
// Build context and get LLM decision
|
||||
const ctx = buildLLMContext(text)
|
||||
|
||||
try {
|
||||
const decision = await $fetch('/api/llm/decide', {
|
||||
method: 'POST',
|
||||
body: ctx
|
||||
})
|
||||
|
||||
applyLLMDecision(decision)
|
||||
|
||||
// If ATC should respond, speak it
|
||||
if (decision.controller_say_tpl && !decision.radio_check) {
|
||||
setTimeout(async () => {
|
||||
await speakWithRadioEffects(decision.controller_say_tpl!)
|
||||
}, 1000 + Math.random() * 2000)
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('LLM decision failed', e)
|
||||
lastTransmission.value += ' (LLM failed - logged only)'
|
||||
}
|
||||
}
|
||||
|
||||
const speakWithRadioEffects = async (text: string) => {
|
||||
|
||||
@@ -1,37 +1,36 @@
|
||||
// server/api/atc/ptt.post.ts
|
||||
import { createError, readBody } from "h3";
|
||||
import { writeFile, readFile, mkdir, rm } from "node:fs/promises";
|
||||
import { existsSync } from "node:fs";
|
||||
import { writeFile, rm } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { execFile } from "node:child_process";
|
||||
import { openaiOld, LLM_MODEL, normalizeATC } from "../../utils/openaiOld";
|
||||
import { createReadStream } from "node:fs"; // ← Diesen Import hinzufügen!
|
||||
import { openai, routeDecision } from "../../utils/openai";
|
||||
import { createReadStream } from "node:fs";
|
||||
|
||||
interface PTTRequest {
|
||||
audio: string; // Base64 encoded audio
|
||||
expectedText: string; // Was der Nutzer wiederholen sollte
|
||||
context: {
|
||||
state_id: string;
|
||||
state: any;
|
||||
candidates: Array<{ id: string; state: any }>;
|
||||
variables: Record<string, any>;
|
||||
flags: Record<string, any>;
|
||||
};
|
||||
moduleId: string;
|
||||
lessonId: string;
|
||||
format?: 'wav' | 'mp3' | 'ogg' | 'webm'; // Audio format
|
||||
format?: 'wav' | 'mp3' | 'ogg' | 'webm';
|
||||
}
|
||||
|
||||
interface PTTResponse {
|
||||
success: boolean;
|
||||
transcription: string;
|
||||
normalized: string;
|
||||
expectedNormalized: string;
|
||||
evaluation: {
|
||||
score: number; // 0-100
|
||||
accuracy: number; // Text similarity
|
||||
keywordMatch: number; // Keyword matching
|
||||
recommendation: 'excellent' | 'good' | 'retry' | 'listen_again';
|
||||
atcResponse?: string; // Antwort des ATC (vom LLM generiert)
|
||||
feedback: string;
|
||||
mistakes?: string[];
|
||||
decision: {
|
||||
next_state: string;
|
||||
controller_say_tpl?: string;
|
||||
off_schema?: boolean;
|
||||
radio_check?: boolean;
|
||||
};
|
||||
playAgain: boolean; // Ob der ursprüngliche Funkspruch nochmal abgespielt werden soll
|
||||
}
|
||||
|
||||
async function sh(cmd: string, args: string[]) {
|
||||
@@ -53,170 +52,13 @@ async function convertToWav(inputPath: string, outputPath: string) {
|
||||
]);
|
||||
}
|
||||
|
||||
// Levenshtein Distance für Text-Ähnlichkeit
|
||||
function levenshteinDistance(str1: string, str2: string): number {
|
||||
const matrix = Array(str2.length + 1).fill(null).map(() => Array(str1.length + 1).fill(null));
|
||||
|
||||
for (let i = 0; i <= str1.length; i++) matrix[0][i] = i;
|
||||
for (let j = 0; j <= str2.length; j++) matrix[j][0] = j;
|
||||
|
||||
for (let j = 1; j <= str2.length; j++) {
|
||||
for (let i = 1; i <= str1.length; i++) {
|
||||
const indicator = str1[i - 1] === str2[j - 1] ? 0 : 1;
|
||||
matrix[j][i] = Math.min(
|
||||
matrix[j][i - 1] + 1, // deletion
|
||||
matrix[j - 1][i] + 1, // insertion
|
||||
matrix[j - 1][i - 1] + indicator // substitution
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return matrix[str2.length][str1.length];
|
||||
}
|
||||
|
||||
function calculateSimilarity(str1: string, str2: string): number {
|
||||
const distance = levenshteinDistance(str1.toLowerCase(), str2.toLowerCase());
|
||||
const maxLength = Math.max(str1.length, str2.length);
|
||||
return maxLength === 0 ? 1 : (maxLength - distance) / maxLength;
|
||||
}
|
||||
|
||||
// Extrahiere Keywords für ATC-spezifische Bewertung
|
||||
function extractATCKeywords(text: string): string[] {
|
||||
const atcKeywords = [
|
||||
// Callsigns
|
||||
'lufthansa', 'speedbird', 'air france', 'klm', 'american', 'united', 'delta', 'ryanair', 'easy',
|
||||
// Numbers/Letters
|
||||
'alpha', 'bravo', 'charlie', 'delta', 'echo', 'foxtrot', 'golf', 'hotel', 'india', 'juliett',
|
||||
'kilo', 'lima', 'mike', 'november', 'oscar', 'papa', 'quebec', 'romeo', 'sierra', 'tango',
|
||||
'uniform', 'victor', 'whiskey', 'x-ray', 'yankee', 'zulu',
|
||||
'zero', 'wun', 'too', 'tree', 'fower', 'fife', 'six', 'seven', 'eight', 'niner',
|
||||
// ATC Phraseology
|
||||
'runway', 'taxi', 'contact', 'ground', 'tower', 'approach', 'departure',
|
||||
'cleared', 'hold short', 'line up', 'wait', 'takeoff', 'landing', 'vacate',
|
||||
'via', 'squawk', 'heading', 'altitude', 'flight level', 'decimal',
|
||||
'roger', 'affirm', 'negative', 'unable', 'standby', 'say again'
|
||||
];
|
||||
|
||||
const normalized = text.toLowerCase().replace(/[^\w\s]/g, ' ').replace(/\s+/g, ' ').trim();
|
||||
return atcKeywords.filter(keyword => normalized.includes(keyword));
|
||||
}
|
||||
|
||||
async function evaluateReadback(transcription: string, expected: string, moduleId: string, lessonId: string) {
|
||||
const normalizedTranscription = normalizeATC(transcription);
|
||||
const normalizedExpected = normalizeATC(expected);
|
||||
|
||||
// Text-Ähnlichkeit
|
||||
const accuracy = calculateSimilarity(normalizedTranscription, normalizedExpected);
|
||||
|
||||
// Keyword-Matching für ATC-spezifische Begriffe
|
||||
const expectedKeywords = extractATCKeywords(normalizedExpected);
|
||||
const transcribedKeywords = extractATCKeywords(normalizedTranscription);
|
||||
const keywordMatch = expectedKeywords.length > 0
|
||||
? transcribedKeywords.filter(k => expectedKeywords.includes(k)).length / expectedKeywords.length
|
||||
: 1;
|
||||
|
||||
// Gewichtete Gesamtbewertung
|
||||
const score = Math.round((accuracy * 0.6 + keywordMatch * 0.4) * 100);
|
||||
|
||||
// LLM für detailliertes Feedback
|
||||
const prompt = `You are an ATC instructor evaluating a pilot's readback.
|
||||
|
||||
Expected: "${expected}"
|
||||
Pilot said: "${transcription}"
|
||||
|
||||
Module: ${moduleId}, Lesson: ${lessonId}
|
||||
|
||||
Evaluate the readback and provide:
|
||||
1. Brief feedback (max 2 sentences)
|
||||
2. Specific mistakes if any
|
||||
3. Overall assessment
|
||||
4. A short ATC-style response to the pilot's readback (if appropriate)
|
||||
|
||||
Be constructive and educational. Focus on aviation safety and standard phraseology.
|
||||
|
||||
Response format:
|
||||
FEEDBACK: [your feedback]
|
||||
ATC_RESPONSE: [your response via ATC phraseology to the pilot]
|
||||
MISTAKES: [list mistakes or "None"]
|
||||
ASSESSMENT: [excellent/good/needs_improvement/unacceptable]`;
|
||||
|
||||
try {
|
||||
const completion = await openaiOld.chat.completions.create({
|
||||
model: LLM_MODEL,
|
||||
messages: [{ role: "user", content: prompt }],
|
||||
});
|
||||
|
||||
const response = completion.choices[0]?.message?.content || "";
|
||||
console.log("LLM Evaluation Response:", response);
|
||||
|
||||
const feedback = response.match(/FEEDBACK: (.+?)(?=MISTAKES:|$)/)?.[1]?.trim() || "Good attempt.";
|
||||
const atcResponse = response.match(/ATC_RESPONSE: (.+?)(?=MISTAKES:|$)/)?.[1]?.trim() || "Noted.";
|
||||
const mistakes = response.match(/MISTAKES: (.+?)(?=ASSESSMENT:|$)/)?.[1]?.trim();
|
||||
const assessment = response.match(/ASSESSMENT: (.+)/)?.[1]?.trim() || "good";
|
||||
|
||||
let recommendation: PTTResponse['evaluation']['recommendation'] = 'good';
|
||||
let playAgain = false;
|
||||
|
||||
if (score >= 90) {
|
||||
recommendation = 'excellent';
|
||||
} else if (score >= 75) {
|
||||
recommendation = 'good';
|
||||
} else if (score >= 50) {
|
||||
recommendation = 'retry';
|
||||
playAgain = true;
|
||||
} else {
|
||||
recommendation = 'listen_again';
|
||||
playAgain = true;
|
||||
}
|
||||
|
||||
return {
|
||||
score,
|
||||
accuracy,
|
||||
keywordMatch,
|
||||
recommendation,
|
||||
atcResponse,
|
||||
feedback,
|
||||
mistakes: mistakes && mistakes !== "None" ? [mistakes] : undefined,
|
||||
playAgain
|
||||
};
|
||||
|
||||
} catch (error) {
|
||||
console.error("LLM evaluation failed:", error);
|
||||
|
||||
// Fallback ohne LLM
|
||||
let recommendation: PTTResponse['evaluation']['recommendation'] = 'good';
|
||||
let playAgain = false;
|
||||
|
||||
if (score >= 90) {
|
||||
recommendation = 'excellent';
|
||||
} else if (score >= 75) {
|
||||
recommendation = 'good';
|
||||
} else if (score >= 50) {
|
||||
recommendation = 'retry';
|
||||
playAgain = true;
|
||||
} else {
|
||||
recommendation = 'listen_again';
|
||||
playAgain = true;
|
||||
}
|
||||
|
||||
return {
|
||||
score,
|
||||
accuracy,
|
||||
keywordMatch,
|
||||
recommendation,
|
||||
feedback: score >= 75 ? "Good readback!" : "Try to match the phraseology more closely.",
|
||||
playAgain
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export default defineEventHandler(async (event) => {
|
||||
const body = await readBody<PTTRequest>(event);
|
||||
|
||||
if (!body.audio || !body.expectedText || !body.moduleId || !body.lessonId) {
|
||||
if (!body.audio || !body.context || !body.moduleId || !body.lessonId) {
|
||||
throw createError({
|
||||
statusCode: 400,
|
||||
statusMessage: "audio, expectedText, moduleId, and lessonId are required"
|
||||
statusMessage: "audio, context, moduleId, and lessonId are required"
|
||||
});
|
||||
}
|
||||
|
||||
@@ -229,19 +71,23 @@ export default defineEventHandler(async (event) => {
|
||||
const audioBuffer = Buffer.from(body.audio, 'base64');
|
||||
await writeFile(tmpAudioInput, audioBuffer);
|
||||
|
||||
// 2. Zu WAV konvertieren falls nötig
|
||||
// 2. Zu WAV konvertieren falls nötig (nur wenn FFmpeg verfügbar)
|
||||
let audioFileForWhisper = tmpAudioInput;
|
||||
if (body.format !== 'wav') {
|
||||
await convertToWav(tmpAudioInput, tmpAudioWav);
|
||||
try {
|
||||
await convertToWav(tmpAudioInput, tmpAudioWav);
|
||||
audioFileForWhisper = tmpAudioWav;
|
||||
} catch (err) {
|
||||
console.warn('FFmpeg conversion failed, using original audio:', err);
|
||||
}
|
||||
}
|
||||
|
||||
const audioFileForWhisper = body.format === 'wav' ? tmpAudioInput : tmpAudioWav;
|
||||
|
||||
// 3. OpenAI Whisper für Transkription
|
||||
const transcription = await openaiOld.audio.transcriptions.create({
|
||||
file: createReadStream(audioFileForWhisper), // ReadStream
|
||||
const transcription = await openai.audio.transcriptions.create({
|
||||
file: createReadStream(audioFileForWhisper),
|
||||
model: "whisper-1",
|
||||
language: "en", // ATC ist standardmäßig auf Englisch
|
||||
prompt: "This is ATC radio communication with aviation phraseology including callsigns, runway numbers, and standard procedures.",
|
||||
language: "en",
|
||||
prompt: "This is ATC radio communication with aviation phraseology including callsigns, runway numbers, and standard procedures."
|
||||
});
|
||||
|
||||
const transcribedText = transcription.text.trim();
|
||||
@@ -253,32 +99,32 @@ export default defineEventHandler(async (event) => {
|
||||
});
|
||||
}
|
||||
|
||||
// 4. Evaluation der Readback
|
||||
const evaluation = await evaluateReadback(
|
||||
transcribedText,
|
||||
body.expectedText,
|
||||
body.moduleId,
|
||||
body.lessonId
|
||||
);
|
||||
// 4. Direkt LLM Decision aufrufen mit transkribiertem Text
|
||||
const decisionInput = {
|
||||
...body.context,
|
||||
pilot_utterance: transcribedText
|
||||
};
|
||||
|
||||
const decision = await routeDecision(decisionInput);
|
||||
|
||||
// 5. Cleanup
|
||||
await rm(tmpAudioInput).catch(() => {});
|
||||
if (body.format !== 'wav') {
|
||||
if (audioFileForWhisper !== tmpAudioInput) {
|
||||
await rm(tmpAudioWav).catch(() => {});
|
||||
}
|
||||
|
||||
return {
|
||||
const result: PTTResponse = {
|
||||
success: true,
|
||||
transcription: transcribedText,
|
||||
normalized: normalizeATC(transcribedText),
|
||||
expectedNormalized: normalizeATC(body.expectedText),
|
||||
evaluation,
|
||||
playAgain: evaluation.playAgain
|
||||
} as PTTResponse;
|
||||
decision
|
||||
};
|
||||
|
||||
} catch (error) {
|
||||
return result;
|
||||
|
||||
} catch (error: any) {
|
||||
// Cleanup bei Fehler
|
||||
await rm(tmpAudioInput).catch(() => {});
|
||||
await rm(tmpAudioWav).catch(() => {});
|
||||
|
||||
if (error.statusCode) {
|
||||
throw error;
|
||||
@@ -286,7 +132,7 @@ export default defineEventHandler(async (event) => {
|
||||
|
||||
throw createError({
|
||||
statusCode: 500,
|
||||
statusMessage: `PTT processing failed: ${error}`
|
||||
statusMessage: `PTT processing failed: ${error.message || error}`
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user