get rid of ptt and decide difference and unite into one endpoint

This commit is contained in:
itsrubberduck
2025-09-16 13:39:46 +02:00
parent 3924bb0b91
commit 329bb5dec5
2 changed files with 123 additions and 237 deletions

View File

@@ -484,14 +484,6 @@
<span class="text-xs uppercase tracking-[0.3em] text-cyan-300">Letzte Übertragung</span>
</div>
<p class="text-sm text-white font-mono">{{ lastTransmission }}</p>
<div v-if="lastEvaluation" class="flex items-center gap-2">
<v-chip size="x-small" :color="getScoreColor(lastEvaluation.score)" variant="flat">
Score: {{ lastEvaluation.score }}%
</v-chip>
<v-chip size="x-small" color="grey" variant="outlined">
{{ lastEvaluation.recommendation }}
</v-chip>
</div>
</v-card-text>
</v-card>
@@ -570,7 +562,6 @@ const loading = ref(false)
const error = ref('')
const pilotInput = ref('')
const lastTransmission = ref('')
const lastEvaluation = ref<any>(null)
const radioMode = ref<'atc' | 'intercom'>('atc')
const isRecording = ref(false)
const micPermission = ref(false)
@@ -611,13 +602,6 @@ const normalizeExpectedText = (text: string): string => {
return normalizeATCText(text, { ...vars.value, ...flags.value })
}
const getScoreColor = (score: number): string => {
if (score >= 90) return 'green'
if (score >= 75) return 'cyan'
if (score >= 50) return 'orange'
return 'red'
}
// VATSIM Integration
const loadFlightPlans = async () => {
if (!vatsimId.value) return
@@ -670,7 +654,6 @@ const backToSetup = () => {
currentScreen.value = 'login'
selectedPlan.value = null
lastTransmission.value = ''
lastEvaluation.value = null
resetCommunications()
}
@@ -745,30 +728,59 @@ const processTransmission = async (audioBlob: Blob, isIntercom: boolean) => {
const arrayBuffer = await audioBlob.arrayBuffer()
const base64Audio = btoa(String.fromCharCode(...new Uint8Array(arrayBuffer)))
const result = await $fetch('/api/atc/ptt', {
method: 'POST',
body: {
audio: base64Audio,
expectedText: currentStep.value?.pilot || 'radio communication',
moduleId: 'pilot-monitoring',
lessonId: currentState.value?.id || 'general',
format: 'webm'
}
})
if (isIntercom) {
// Handle intercom separately - simple transcription only
const result = await $fetch('/api/atc/ptt', {
method: 'POST',
body: {
audio: base64Audio,
context: { // Minimal context for intercom
state_id: currentState.value?.id || 'INTERCOM',
state: {},
candidates: [],
variables: { callsign: vars.value.callsign },
flags: {}
},
moduleId: 'pilot-monitoring-intercom',
lessonId: 'intercom',
format: 'webm'
}
})
if (result.success) {
lastTransmission.value = result.transcription
lastEvaluation.value = result.evaluation
if (isIntercom) {
// Handle intercom (for checklists, etc.)
if (result.success) {
lastTransmission.value = `INTERCOM: ${result.transcription}`
const transcription = result.transcription.toLowerCase()
if (transcription.includes('checklist') || transcription.includes('check list')) {
setTimeout(() => speakWithRadioEffects('Checklist functionality available in advanced mode'), 1000)
}
} else {
// Handle ATC communication through Decision Tree
await processPilotAndLLM(result.transcription)
}
} else {
// Handle ATC communication - full PTT + Decision in one call
const ctx = buildLLMContext('') // Build context first
const result = await $fetch('/api/atc/ptt', {
method: 'POST',
body: {
audio: base64Audio,
context: ctx, // Full LLM context
moduleId: 'pilot-monitoring',
lessonId: currentState.value?.id || 'general',
format: 'webm'
}
})
if (result.success) {
lastTransmission.value = result.transcription
// Apply the decision directly from PTT response
applyLLMDecision(result.decision)
// If ATC should respond, speak it
if (result.decision.controller_say_tpl && !result.decision.radio_check) {
setTimeout(async () => {
await speakWithRadioEffects(result.decision.controller_say_tpl!)
}, 1000 + Math.random() * 2000)
}
}
}
} catch (err) {
@@ -812,7 +824,35 @@ const sendPilotText = async () => {
pilotInput.value = ''
lastTransmission.value = text
await processPilotAndLLM(text)
// Process text input directly through decision engine
const quickResponse = processPilotTransmission(text)
if (quickResponse) {
// Radio check or emergency was handled directly
return
}
// Build context and get LLM decision
const ctx = buildLLMContext(text)
try {
const decision = await $fetch('/api/llm/decide', {
method: 'POST',
body: ctx
})
applyLLMDecision(decision)
// If ATC should respond, speak it
if (decision.controller_say_tpl && !decision.radio_check) {
setTimeout(async () => {
await speakWithRadioEffects(decision.controller_say_tpl!)
}, 1000 + Math.random() * 2000)
}
} catch (e) {
console.error('LLM decision failed', e)
lastTransmission.value += ' (LLM failed - logged only)'
}
}
const speakWithRadioEffects = async (text: string) => {

View File

@@ -1,37 +1,36 @@
// server/api/atc/ptt.post.ts
import { createError, readBody } from "h3";
import { writeFile, readFile, mkdir, rm } from "node:fs/promises";
import { existsSync } from "node:fs";
import { writeFile, rm } from "node:fs/promises";
import { join } from "node:path";
import { tmpdir } from "node:os";
import { randomUUID } from "node:crypto";
import { execFile } from "node:child_process";
import { openaiOld, LLM_MODEL, normalizeATC } from "../../utils/openaiOld";
import { createReadStream } from "node:fs"; // ← Diesen Import hinzufügen!
import { openai, routeDecision } from "../../utils/openai";
import { createReadStream } from "node:fs";
interface PTTRequest {
audio: string; // Base64 encoded audio
expectedText: string; // Was der Nutzer wiederholen sollte
context: {
state_id: string;
state: any;
candidates: Array<{ id: string; state: any }>;
variables: Record<string, any>;
flags: Record<string, any>;
};
moduleId: string;
lessonId: string;
format?: 'wav' | 'mp3' | 'ogg' | 'webm'; // Audio format
format?: 'wav' | 'mp3' | 'ogg' | 'webm';
}
interface PTTResponse {
success: boolean;
transcription: string;
normalized: string;
expectedNormalized: string;
evaluation: {
score: number; // 0-100
accuracy: number; // Text similarity
keywordMatch: number; // Keyword matching
recommendation: 'excellent' | 'good' | 'retry' | 'listen_again';
atcResponse?: string; // Antwort des ATC (vom LLM generiert)
feedback: string;
mistakes?: string[];
decision: {
next_state: string;
controller_say_tpl?: string;
off_schema?: boolean;
radio_check?: boolean;
};
playAgain: boolean; // Ob der ursprüngliche Funkspruch nochmal abgespielt werden soll
}
async function sh(cmd: string, args: string[]) {
@@ -53,170 +52,13 @@ async function convertToWav(inputPath: string, outputPath: string) {
]);
}
// Levenshtein Distance für Text-Ähnlichkeit
function levenshteinDistance(str1: string, str2: string): number {
const matrix = Array(str2.length + 1).fill(null).map(() => Array(str1.length + 1).fill(null));
for (let i = 0; i <= str1.length; i++) matrix[0][i] = i;
for (let j = 0; j <= str2.length; j++) matrix[j][0] = j;
for (let j = 1; j <= str2.length; j++) {
for (let i = 1; i <= str1.length; i++) {
const indicator = str1[i - 1] === str2[j - 1] ? 0 : 1;
matrix[j][i] = Math.min(
matrix[j][i - 1] + 1, // deletion
matrix[j - 1][i] + 1, // insertion
matrix[j - 1][i - 1] + indicator // substitution
);
}
}
return matrix[str2.length][str1.length];
}
function calculateSimilarity(str1: string, str2: string): number {
const distance = levenshteinDistance(str1.toLowerCase(), str2.toLowerCase());
const maxLength = Math.max(str1.length, str2.length);
return maxLength === 0 ? 1 : (maxLength - distance) / maxLength;
}
// Extrahiere Keywords für ATC-spezifische Bewertung
function extractATCKeywords(text: string): string[] {
const atcKeywords = [
// Callsigns
'lufthansa', 'speedbird', 'air france', 'klm', 'american', 'united', 'delta', 'ryanair', 'easy',
// Numbers/Letters
'alpha', 'bravo', 'charlie', 'delta', 'echo', 'foxtrot', 'golf', 'hotel', 'india', 'juliett',
'kilo', 'lima', 'mike', 'november', 'oscar', 'papa', 'quebec', 'romeo', 'sierra', 'tango',
'uniform', 'victor', 'whiskey', 'x-ray', 'yankee', 'zulu',
'zero', 'wun', 'too', 'tree', 'fower', 'fife', 'six', 'seven', 'eight', 'niner',
// ATC Phraseology
'runway', 'taxi', 'contact', 'ground', 'tower', 'approach', 'departure',
'cleared', 'hold short', 'line up', 'wait', 'takeoff', 'landing', 'vacate',
'via', 'squawk', 'heading', 'altitude', 'flight level', 'decimal',
'roger', 'affirm', 'negative', 'unable', 'standby', 'say again'
];
const normalized = text.toLowerCase().replace(/[^\w\s]/g, ' ').replace(/\s+/g, ' ').trim();
return atcKeywords.filter(keyword => normalized.includes(keyword));
}
async function evaluateReadback(transcription: string, expected: string, moduleId: string, lessonId: string) {
const normalizedTranscription = normalizeATC(transcription);
const normalizedExpected = normalizeATC(expected);
// Text-Ähnlichkeit
const accuracy = calculateSimilarity(normalizedTranscription, normalizedExpected);
// Keyword-Matching für ATC-spezifische Begriffe
const expectedKeywords = extractATCKeywords(normalizedExpected);
const transcribedKeywords = extractATCKeywords(normalizedTranscription);
const keywordMatch = expectedKeywords.length > 0
? transcribedKeywords.filter(k => expectedKeywords.includes(k)).length / expectedKeywords.length
: 1;
// Gewichtete Gesamtbewertung
const score = Math.round((accuracy * 0.6 + keywordMatch * 0.4) * 100);
// LLM für detailliertes Feedback
const prompt = `You are an ATC instructor evaluating a pilot's readback.
Expected: "${expected}"
Pilot said: "${transcription}"
Module: ${moduleId}, Lesson: ${lessonId}
Evaluate the readback and provide:
1. Brief feedback (max 2 sentences)
2. Specific mistakes if any
3. Overall assessment
4. A short ATC-style response to the pilot's readback (if appropriate)
Be constructive and educational. Focus on aviation safety and standard phraseology.
Response format:
FEEDBACK: [your feedback]
ATC_RESPONSE: [your response via ATC phraseology to the pilot]
MISTAKES: [list mistakes or "None"]
ASSESSMENT: [excellent/good/needs_improvement/unacceptable]`;
try {
const completion = await openaiOld.chat.completions.create({
model: LLM_MODEL,
messages: [{ role: "user", content: prompt }],
});
const response = completion.choices[0]?.message?.content || "";
console.log("LLM Evaluation Response:", response);
const feedback = response.match(/FEEDBACK: (.+?)(?=MISTAKES:|$)/)?.[1]?.trim() || "Good attempt.";
const atcResponse = response.match(/ATC_RESPONSE: (.+?)(?=MISTAKES:|$)/)?.[1]?.trim() || "Noted.";
const mistakes = response.match(/MISTAKES: (.+?)(?=ASSESSMENT:|$)/)?.[1]?.trim();
const assessment = response.match(/ASSESSMENT: (.+)/)?.[1]?.trim() || "good";
let recommendation: PTTResponse['evaluation']['recommendation'] = 'good';
let playAgain = false;
if (score >= 90) {
recommendation = 'excellent';
} else if (score >= 75) {
recommendation = 'good';
} else if (score >= 50) {
recommendation = 'retry';
playAgain = true;
} else {
recommendation = 'listen_again';
playAgain = true;
}
return {
score,
accuracy,
keywordMatch,
recommendation,
atcResponse,
feedback,
mistakes: mistakes && mistakes !== "None" ? [mistakes] : undefined,
playAgain
};
} catch (error) {
console.error("LLM evaluation failed:", error);
// Fallback ohne LLM
let recommendation: PTTResponse['evaluation']['recommendation'] = 'good';
let playAgain = false;
if (score >= 90) {
recommendation = 'excellent';
} else if (score >= 75) {
recommendation = 'good';
} else if (score >= 50) {
recommendation = 'retry';
playAgain = true;
} else {
recommendation = 'listen_again';
playAgain = true;
}
return {
score,
accuracy,
keywordMatch,
recommendation,
feedback: score >= 75 ? "Good readback!" : "Try to match the phraseology more closely.",
playAgain
};
}
}
export default defineEventHandler(async (event) => {
const body = await readBody<PTTRequest>(event);
if (!body.audio || !body.expectedText || !body.moduleId || !body.lessonId) {
if (!body.audio || !body.context || !body.moduleId || !body.lessonId) {
throw createError({
statusCode: 400,
statusMessage: "audio, expectedText, moduleId, and lessonId are required"
statusMessage: "audio, context, moduleId, and lessonId are required"
});
}
@@ -229,19 +71,23 @@ export default defineEventHandler(async (event) => {
const audioBuffer = Buffer.from(body.audio, 'base64');
await writeFile(tmpAudioInput, audioBuffer);
// 2. Zu WAV konvertieren falls nötig
// 2. Zu WAV konvertieren falls nötig (nur wenn FFmpeg verfügbar)
let audioFileForWhisper = tmpAudioInput;
if (body.format !== 'wav') {
await convertToWav(tmpAudioInput, tmpAudioWav);
try {
await convertToWav(tmpAudioInput, tmpAudioWav);
audioFileForWhisper = tmpAudioWav;
} catch (err) {
console.warn('FFmpeg conversion failed, using original audio:', err);
}
}
const audioFileForWhisper = body.format === 'wav' ? tmpAudioInput : tmpAudioWav;
// 3. OpenAI Whisper für Transkription
const transcription = await openaiOld.audio.transcriptions.create({
file: createReadStream(audioFileForWhisper), // ReadStream
const transcription = await openai.audio.transcriptions.create({
file: createReadStream(audioFileForWhisper),
model: "whisper-1",
language: "en", // ATC ist standardmäßig auf Englisch
prompt: "This is ATC radio communication with aviation phraseology including callsigns, runway numbers, and standard procedures.",
language: "en",
prompt: "This is ATC radio communication with aviation phraseology including callsigns, runway numbers, and standard procedures."
});
const transcribedText = transcription.text.trim();
@@ -253,32 +99,32 @@ export default defineEventHandler(async (event) => {
});
}
// 4. Evaluation der Readback
const evaluation = await evaluateReadback(
transcribedText,
body.expectedText,
body.moduleId,
body.lessonId
);
// 4. Direkt LLM Decision aufrufen mit transkribiertem Text
const decisionInput = {
...body.context,
pilot_utterance: transcribedText
};
const decision = await routeDecision(decisionInput);
// 5. Cleanup
await rm(tmpAudioInput).catch(() => {});
if (body.format !== 'wav') {
if (audioFileForWhisper !== tmpAudioInput) {
await rm(tmpAudioWav).catch(() => {});
}
return {
const result: PTTResponse = {
success: true,
transcription: transcribedText,
normalized: normalizeATC(transcribedText),
expectedNormalized: normalizeATC(body.expectedText),
evaluation,
playAgain: evaluation.playAgain
} as PTTResponse;
decision
};
} catch (error) {
return result;
} catch (error: any) {
// Cleanup bei Fehler
await rm(tmpAudioInput).catch(() => {});
await rm(tmpAudioWav).catch(() => {});
if (error.statusCode) {
throw error;
@@ -286,7 +132,7 @@ export default defineEventHandler(async (event) => {
throw createError({
statusCode: 500,
statusMessage: `PTT processing failed: ${error}`
statusMessage: `PTT processing failed: ${error.message || error}`
});
}
});