emily/video2document
1
0
Fork 0
mirror of https://gitlab.rlp.net/proj-wise2526-video2document/video2document.git synced 2026-06-15 18:01:52 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'develop' into 'feature/ui-test'

Browse Source 
Getting the ui-test up to date

See merge request proj-wise2526-video2document/video2document!69
This commit is contained in:
Minning, Eric
2025-12-17 14:09:01 +01:00
parent 292b893ed2 0e746749e6
commit 2778626e4b
11 changed files with 148 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitlab-ci.yml
+1
View File
@@ -24,4 +24,5 @@ job-test:
- npm install
- echo "ASSEMBLYAI_API_KEY=$apikey_assembly" > .env
- echo "GOOGLE_API_KEY=$apikey_gemini" >> .env
- echo "SAIA_API_KEY=$apikey_saia" >> .env
- npm test
main.js
+18 -35
View File
@@ -132,7 +132,7 @@ electron.ipcMain.on("file_submit", async (event, args) => {
globalArgs = args
let curstep = 0
let totalsteps = 4
const TEMPLATE_MAP = {
"followup-report": "followup_report.txt",
"agenda": "agenda.txt",
@@ -140,9 +140,9 @@ electron.ipcMain.on("file_submit", async (event, args) => {
"sprint-planning": "sprint_planning_note.txt",
"custom": "custom_document.txt"
};
const templateFile = TEMPLATE_MAP[args.document.type];
if (!templateFile) {
throw new Error("Unknown document type: " + args.document.type);
}
@@ -196,42 +196,25 @@ electron.ipcMain.on("file_submit", async (event, args) => {
// This code handles the Text to Document processing module call
console.log(`\n\n Running the LLM for Document Style ${args.document.type}`);
await mapFunctions.get("module-handler").function(args.document.module, { inputTranscriptPath: transcriptpath, documentTypePath: "./storage/documentType/" + templateFile, language: "en" }).then(resp => {
console.log(resp);
globalFinalHtmlPath = resp
curstep++
mainWindow.webContents.send("progress", {curstep:curstep, totalsteps:totalsteps})
}).catch(err => {
mainWindow.webContents.send("error", err)
return
})
await mapFunctions.get("extract-speaker-snippets").function({audioPath: audiopath, jsonPath: transcriptpath }).then(resp => {
mainWindow.webContents.send("submitSpeaker", resp)
console.log(resp)
await mapFunctions.get("module-handler").function(args.document.module, { inputTranscriptPath: transcriptpath, documentTypePath: "./storage/documentType/" + templateFile, language: "en" }).then(resp => {
console.log(resp);
globalFinalHtmlPath = resp
curstep++
mainWindow.webContents.send("progress", {curstep:curstep, totalsteps:totalsteps})
}).catch(err => {
mainWindow.webContents.send("error", err)
return
})
// TODO actually implement this functionality
// Module to get the first few lines for each speaker to send to the frontend
// await mapFunctions.get("speaker-getter-idfk").function(transcriptpath).then(resp => {
// console.log(resp);
// transcriptpath = resp
// curstep++
// mainWindow.webContents.send("progress", {curstep:curstep, totalsteps:totalsteps})
// // {
// // speakerA: {source: "Pfad zur Audio File"},
// // speakerB:.....
// // }
// mainWindow.webContents.send("speakers", {speakerA:"pfad1", speakerB:"pfad2"})
// }).catch(err => {
// mainWindow.webContents.send("error", err)
// return
// })
await mapFunctions.get("extract-speaker-snippets").function({audioPath: audiopath, jsonPath: transcriptpath }).then(resp => {
mainWindow.webContents.send("submitSpeaker", resp)
console.log(resp)
}).catch(err => {
mainWindow.webContents.send("error", err)
return
})
} catch (error) {
console.log(error);
}
services/modules/audioSnippets/extract-speaker-snippets.js
+31 -24
View File
@@ -14,7 +14,7 @@ module.exports = {
let output = {}
console.log("Extract Speaker Snippets\n");
// console.log("Extract Speaker Snippets\n");
// Pfade
const AUDIO_PATH = parameter.audioPath; // Gesamt-Audio
@@ -23,7 +23,8 @@ module.exports = {
if (!AUDIO_PATH || !JSON_PATH) {
console.error("no audioPath or jsonPath available");
// console.error("no audioPath or jsonPath available");
reject(new Error("no audioPath or jsonPath available"));
return;
}
@@ -37,12 +38,14 @@ module.exports = {
try {
entries = JSON.parse(fs.readFileSync(JSON_PATH, "utf8"));
} catch (err) {
console.error("JSON reading failed", err);
// console.error("JSON reading failed", err);
reject(new Error(err));
return;
}
if (!Array.isArray(entries)) {
console.error("JSON is not an Array");
// console.error("JSON is not an Array");
reject(new Error("JSON is not an Array"));
return;
}
@@ -64,32 +67,36 @@ module.exports = {
const durationSec = (data.end - data.start) / 1000;
if (durationSec <= 0) {
console.log(`invalid times for Speaker ${speaker}`);
// console.log(`invalid times for Speaker ${speaker}`);
continue;
}
const outFile = path.join(OUTPUT_DIR, `speaker_${speaker}.wav`);
await new Promise((res, rej) => {
ffmpeg(AUDIO_PATH)
.setStartTime(startSec)
.setDuration(durationSec)
.output(outFile)
.on("end", () => {
output[`speaker${speaker}`] = {src: outFile, name: `speaker${speaker}`}
console.log(`Snippet erstellt: speaker_${speaker}.wav`);
res();
})
.on("error", (err) => {
console.error(`FFmpeg Fehler (${speaker})`, err.message);
rej();
})
.run();
});
try {
await new Promise((res, rej) => {
ffmpeg(AUDIO_PATH)
.setStartTime(startSec)
.setDuration(durationSec)
.output(outFile)
.on("end", () => {
output[`speaker${speaker}`] = {src: outFile, name: `speaker${speaker}`}
// console.log(`Snippet erstellt: speaker_${speaker}.wav`);
res();
})
.on("error", (err) => {
// console.error(`FFmpeg Fehler (${speaker})`, err.message);
rej(err);
return
})
.run();
});
} catch (error) {
reject(error)
return
}
}
resolve(output)
console.log("\nAlle Speaker-Snippets erstellt\n");
// console.log("\nAlle Speaker-Snippets erstellt\n");
})
}
};
services/modules/llm-chat_gpt/chatgpt.js
+3 -2
View File
@@ -1,5 +1,5 @@
const fs = require('fs');
const path = require('path');
// const fs = require('fs');
// const path = require('path');
const outputDir = path.join(__dirname, "../../../storage/documents"); // path for output directory
@@ -43,6 +43,7 @@ const module_exports = {
const documentType = await fs.promises.readFile(documentTypePath, "utf-8"); //read document type from Path
const promptText = `${documentType}, in language ${language}, transcript:\n\n${transcript}`; //combine doc type, language and transcript - Change prompt here if needed
// return
// --- REST CALL ---
const response = await fetch(SAIA_URL, { //safe model response in variable
method: "POST",
services/modules/quen3/quen3.js → services/modules/quen3/qwen3.js
View File
storage/documentType/agenda.txt
+1
View File
@@ -21,5 +21,6 @@ STIL:
- Klar, kompakt
- Business-orientiert
- Keine Sprecher- oder Zeitangaben
- Namen aus dem Transkript speakerA, speakerB etc. sollen weiterhin bestehen bleiben wie sie sind und nicht im Dokument ersetzt werden
TRANSKRIPT:
storage/documentType/custom_document.txt
+1
View File
@@ -10,6 +10,7 @@ WICHTIG:
- Nutze das Transkript als Wissensquelle
- Struktur, Tonalität und Detailgrad anpassen
- Inhalte logisch zusammenführen
- Namen aus dem Transkript speakerA, speakerB etc. sollen weiterhin bestehen bleiben wie sie sind und nicht im Dokument ersetzt werden
FORMAT:
- Passe Struktur und Stil an den Nutzerwunsch an
storage/documentType/standard_meeting_report.txt → storage/documentType/followup_report.txt
+1
View File
@@ -11,6 +11,7 @@ ANFORDERUNGEN:
- Keine Zeitstempel oder Sprecher-Namen zitieren
- Leite Entscheidungen und Aufgaben logisch ab, wenn sie implizit sind
- Markiere offene Punkte klar
- Namen aus dem Transkript speakerA, speakerB etc. sollen weiterhin bestehen bleiben wie sie sind und nicht im Dokument ersetzt werden
STRUKTUR DES DOKUMENTS:
1. Titel & Metadaten
storage/documentType/result_protocol.txt
+2
View File
@@ -21,5 +21,7 @@ REGELN:
- Keine Meinungen oder Spekulationen
- Keine Zeit- oder Sprecherangaben
- Sachlich, formal
- Namen aus dem Transkript speakerA, speakerB etc. sollen weiterhin bestehen bleiben wie sie sind und nicht im Dokument ersetzt werden
TRANSKRIPT:
storage/documentType/sprint_planning_note.txt
+1
View File
@@ -30,5 +30,6 @@ STIL:
- Agile-konform
- Klar & umsetzungsorientiert
- Bullet Points bevorzugen
- Namen aus dem Transkript speakerA, speakerB etc. sollen weiterhin bestehen bleiben wie sie sind und nicht im Dokument ersetzt werden
TRANSKRIPT:
test/unit/test.js
+89 -16
View File
@@ -21,6 +21,7 @@ let audiopath
let transcriptPath
let summarizePath
let llmpath
let speakers
describe("Unit Tests", function() {
@@ -34,7 +35,7 @@ describe("Unit Tests", function() {
// console.log(resp);
done()
}).catch(err => {
throw err;
done(err);
})
})
it('Extract .mp4 to .flac', function (done) {
@@ -43,7 +44,7 @@ describe("Unit Tests", function() {
// console.log(resp);
done()
}).catch(err => {
throw err;
done(err);
})
})
it('Extracting to a nonexistant format', function (done) {
@@ -86,7 +87,7 @@ describe("Unit Tests", function() {
transcriptPath = resp
done()
}).catch(err => {
throw err
done(err)
})
})
@@ -111,7 +112,7 @@ describe("Unit Tests", function() {
mapFunctions.get("summarize-transcription").function(transcriptPath).then(resp => {
done()
}).catch(err => {
throw err
done(err)
})
})
@@ -128,7 +129,7 @@ describe("Unit Tests", function() {
summarizePath = resp
done()
}).catch(err => {
throw err
done(err)
})
})
@@ -145,20 +146,92 @@ describe("Unit Tests", function() {
this.slow(30000)
this.timeout(120000)
// it("ChatGPT", function (done){
// mapFunctions.get("chatgpt").function({inputTranscriptPath: summarizePath, documentTypePath: "./storage/documentType/meetingReport.json", language: "en"}).then(resp => {
// done()
// }).catch(err => {
// throw err
// })
// })
it("Gemini", function (done){
mapFunctions.get("llm-gemini").function({inputTranscriptPath: summarizePath, documentTypePath: "./storage/documentType/meetingReport.json", language: "en"}).then(resp => {
it("ChatGPT", function (done){
mapFunctions.get("llm-saia_openai_gpt").function({inputTranscriptPath: summarizePath, documentTypePath: "./storage/documentType/followup_report.txt", language: "en"}).then(resp => {
llmpath = resp
done()
}).catch(err => {
throw err
done(err)
})
})
it("Gemini", function (done){
mapFunctions.get("llm-gemini").function({inputTranscriptPath: summarizePath, documentTypePath: "./storage/documentType/followup_report.txt", language: "en"}).then(resp => {
done()
}).catch(err => {
if(err.includes("(503)")){done()} // Error 503 is gemini overload, so an Error that they can at any time throw at us which would crash the pipeline, so we just ignore it and we just imagine that the test passed
else{
console.log(err);
done(err)
}
})
})
it("Qwen3", function (done){
mapFunctions.get("qwen3-235b-a22b").function({inputTranscriptPath: summarizePath, documentTypePath: "./storage/documentType/followup_report.txt", language: "en"}).then(resp => {
done()
}).catch(err => {
done(err)
})
})
it("ChatGPT (Nonexistant Type File)", function (done){
mapFunctions.get("llm-saia_openai_gpt").function({inputTranscriptPath: summarizePath, documentTypePath: "a", language: "en"}).then(resp => {
done("Didnt crash")
}).catch(err => {
done()
})
})
it("Gemini (Nonexistant Type File)", function (done){
mapFunctions.get("llm-gemini").function({inputTranscriptPath: summarizePath, documentTypePath: "a", language: "en"}).then(resp => {
done("Didnt crash")
}).catch(err => {
done()
})
})
it("Qwen3 (Nonexistant Type File)", function (done){
mapFunctions.get("qwen3-235b-a22b").function({inputTranscriptPath: summarizePath, documentTypePath: "a", language: "en"}).then(resp => {
done("Didnt crash")
}).catch(err => {
done()
})
})
})
describe("Audio Snippet", function() {
this.slow(1000)
this.timeout(5000)
// transcriptPath = "A:\\programing\\@projects\\video2document\\storage\\transcriptionSummaries\\testvideo-1765900665001.json"
// audiopath = "A:\\programing\\@projects\\video2document\\storage\\audio\\testvideo.mp3"
it("Audio Snipper Generator", function (done){
mapFunctions.get("extract-speaker-snippets").function({audioPath: audiopath, jsonPath: summarizePath }).then(resp => {
speakers = resp
done()
}).catch(err => {
done(err)
})
})
it("Audio Snipper Generator (Nonexistant Transcript File)", function (done){
mapFunctions.get("extract-speaker-snippets").function({audioPath: audiopath, jsonPath: "a" }).then(resp => {
speakers = resp
done("Didnt crash")
}).catch(err => {
done()
})
})
it("Audio Snipper Generator (Nonexistant Audio File)", function (done){
mapFunctions.get("extract-speaker-snippets").function({audioPath: "a", jsonPath: summarizePath }).then(resp => {
speakers = resp
done("Didnt crash")
}).catch(err => {
done()
})
})
})