Summarizer und weiters angepasst

electron/main/preload.js

@@ -12,6 +12,9 @@ try {
     contextBridge.exposeInMainWorld("electronAPI", {
         getFilePath: (file) => {return webUtils.getPathForFile(file)}
     })
+    contextBridge.exposeInMainWorld("summarizer", {
+        runFile: (file) => ipcRenderer.send("summarize-transcription", file)
+    });
 } catch (error) {
     console.log("Error in preload.js");
 }

electron/main/renderer.js

@@ -16,15 +16,21 @@ uploadContainer.addEventListener("drop", (e) => {
         e.preventDefault()
         const files = e.dataTransfer.files
         const filePath = window.explorer.onFileDrop(files[0])
-        var holdy = filePath + "";
-        if(holdy.endsWith(".mp4") || holdy.endsWith(".mov") || holdy.endsWith(".avi") || holdy.endsWith( ".mkv")){
-            console.log(filePath)
 
+        var holdy = String(filePath);
+        const lower = holdy.toLowerCase();
+        const validExt = [".mp4", ".mov", ".avi", ".mkv"];
+        
+       if(validExt.some(ext => lower.endsWith(ext))){
+            console.log(filePath);
             const files1 = e.dataTransfer.files;
             handleFiles(files1);
+        }else{
+            console.log('Video format invalid!');
         }
     } catch (error) {
         console.log("Error in renderer.js with the listerner for the drop function");
+        console.log(error);
     }
     
     

electron/main/script.js

@@ -6,7 +6,7 @@ manualUploadBtn.addEventListener('click', () => {
   } catch (error) {
     console.log("Error in manualBtn EventListener click");
   }
-  
+
 });
 
 //function to check if one checkbox is at least klicked
@@ -15,18 +15,24 @@ function checkBoxes() {
     const checkboxes = document.querySelectorAll('input[name="docFormat"]');
     let isChecked = false;
 
-    checkboxes.forEach(function(checkbox){
+    checkboxes.forEach(function (checkbox) {
-      if(checkbox.checked){
+      if (checkbox.checked) {
         isChecked = true;
       }
     });
 
-    if(isChecked){
+    if (isChecked) {
       //Code to submit the video
       var pathTest = window.electronAPI.getFilePath(videoUpload.files[0]);
-      if(pathTest.endsWith(".mp4") || holdy.endsWith(".mov") || holdy.endsWith(".avi") || holdy.endsWith( ".mkv")){
+
-        window.extractor.extract({inputVideoPath: pathTest, outputType:"wav"})
+      const lower = pathTest.toLowerCase();
+      const validExt = [".mp4", ".mov", ".avi", ".mkv"];
+
+      if(validExt.some(ext => lower.endsWith(ext))){
+        window.extractor.extract({ inputVideoPath: pathTest, outputType: "wav" });
       }
+
+
     } else {
       //language only english at the moment
       alert('Please select at least one document type.');
@@ -34,43 +40,43 @@ function checkBoxes() {
   } catch (error) {
     console.log(error)
   }
-  
+
-// mapFunctions.get("extraction-video-to-audio").function({inputVideoPath:"./a.mp4", outputType:"wav"})
+  // mapFunctions.get("extraction-video-to-audio").function({inputVideoPath:"./a.mp4", outputType:"wav"})
 }
 
 //language changing feature
 function changeLanguage(language) {
-    if (language === 'en') {
+  if (language === 'en') {
-        document.getElementById('title').textContent = 'Video to document';
+    document.getElementById('title').textContent = 'Video to document';
-        document.getElementById('h1').textContent = 'Video to document';
+    document.getElementById('h1').textContent = 'Video to document';
-        document.getElementById('p1').textContent = 'Drag and drop video file';
+    document.getElementById('p1').textContent = 'Drag and drop video file';
-        document.getElementById('fileName').textContent = 'No video chosen';
+    document.getElementById('fileName').textContent = 'No video chosen';
-        document.getElementById('manualUploadBtn').textContent = 'Search video';
+    document.getElementById('manualUploadBtn').textContent = 'Search video';
-        document.getElementById('checkbox_group').textContent = 'Choose prefered document style:';
+    document.getElementById('checkbox_group').textContent = 'Choose prefered document style:';
-        document.getElementById('label_format').textContent = 'Meeting report';
+    document.getElementById('label_format').textContent = 'Meeting report';
-        document.getElementById('label_summary').textContent = 'Summary with timestamps';
+    document.getElementById('label_summary').textContent = 'Summary with timestamps';
-        document.getElementById('submitButton').textContent = 'Submit';
+    document.getElementById('submitButton').textContent = 'Submit';
-    } else if (language === 'de') {
+  } else if (language === 'de') {
-        document.getElementById('title').textContent = 'Video zu Dokument';
+    document.getElementById('title').textContent = 'Video zu Dokument';
-        document.getElementById('h1').textContent = 'Video zu Dokument';
+    document.getElementById('h1').textContent = 'Video zu Dokument';
-        document.getElementById('p1').textContent = 'Video per Drag & Drop ablegen';
+    document.getElementById('p1').textContent = 'Video per Drag & Drop ablegen';
-        document.getElementById('fileName').textContent = 'Kein Video ausgewaehlt';
+    document.getElementById('fileName').textContent = 'Kein Video ausgewaehlt';
-        document.getElementById('manualUploadBtn').textContent = 'Video suchen';
+    document.getElementById('manualUploadBtn').textContent = 'Video suchen';
-        document.getElementById('checkbox_group').textContent = 'Bevorzugte Dokumentvarianten:';
+    document.getElementById('checkbox_group').textContent = 'Bevorzugte Dokumentvarianten:';
-        document.getElementById('label_format').textContent = 'Meeting Bericht';
+    document.getElementById('label_format').textContent = 'Meeting Bericht';
-        document.getElementById('label_summary').textContent = 'Zusammenfassung mit Zeitstempeln';
+    document.getElementById('label_summary').textContent = 'Zusammenfassung mit Zeitstempeln';
-        document.getElementById('submitButton').textContent = 'Absenden';
+    document.getElementById('submitButton').textContent = 'Absenden';
-    } else if(language == "in") {
+  } else if (language == "in") {
-        document.getElementById('title').textContent = 'दस्तावेज़ के लिए वीडियो';
+    document.getElementById('title').textContent = 'दस्तावेज़ के लिए वीडियो';
-        document.getElementById('h1').textContent = 'दस्तावेज़ के लिए वीडियो';
+    document.getElementById('h1').textContent = 'दस्तावेज़ के लिए वीडियो';
-        document.getElementById('p1').textContent = 'वीडियो फ़ाइल खींचें और छोड़ें';
+    document.getElementById('p1').textContent = 'वीडियो फ़ाइल खींचें और छोड़ें';
-        document.getElementById('fileName').textContent = 'कोई वीडियो नहीं चुना गया';
+    document.getElementById('fileName').textContent = 'कोई वीडियो नहीं चुना गया';
-        document.getElementById('manualUploadBtn').textContent = 'वीडियो खोजें';
+    document.getElementById('manualUploadBtn').textContent = 'वीडियो खोजें';
-        document.getElementById('checkbox_group').textContent = 'पसंदीदा दस्तावेज़ शैली चुनें:';
+    document.getElementById('checkbox_group').textContent = 'पसंदीदा दस्तावेज़ शैली चुनें:';
-        document.getElementById('label_format').textContent = 'बैठक रिपोर्ट';
+    document.getElementById('label_format').textContent = 'बैठक रिपोर्ट';
-        document.getElementById('label_summary').textContent = 'टाइमस्टैम्प के साथ सारांश';
+    document.getElementById('label_summary').textContent = 'टाइमस्टैम्प के साथ सारांश';
-        document.getElementById('submitButton').textContent = 'जमा करना';
+    document.getElementById('submitButton').textContent = 'जमा करना';
-    }
+  }
 }
 
 
@@ -81,7 +87,7 @@ videoUpload.addEventListener('change', () => {
   } catch (error) {
     console.log("Error in manualBtn EventListener change");
   }
-  
+
 });
 
 
@@ -99,11 +105,11 @@ function handleFiles(files) {
   } catch (error) {
     console.log("Error in script.js handleFiles function");
   }
-  
+
 }
 
 //function to regulate the progress on the progressbar
-function updateProgressBar(bar, value){
+function updateProgressBar(bar, value) {
   try {
     value = Math.round(value);
     bar.querySelector(".progress_fill").style.width = `${value}%`;
@@ -111,5 +117,5 @@ function updateProgressBar(bar, value){
   } catch (error) {
     console.log("Error in scripts.js updateProgressBar function");
   }
-  
+
 }

main.js

@@ -76,4 +76,9 @@ electron.app.whenReady().then(createWindow);
 
 electron.ipcMain.on("extract", (event, args) => {
     mapFunctions.get("extraction-video-to-audio").function(args)
-})
+})
+
+electron.ipcMain.on("summarize-transcription", (event, args) => {
+    mapFunctions.get("summarize-transcription").function(args);
+});
+

services/modules/jsonTools/transcriptionSummarizer.js

@@ -0,0 +1,138 @@
+const fs = require("fs");
+const path = require("path");
+
+// Prepare output directory (always storage/transcriptionSummaries under project root)
+const outputDir = `${__dirname}/../../../storage/transcriptionSummaries`;
+if (!fs.existsSync(outputDir)) {
+    fs.mkdirSync(outputDir, { recursive: true });
+}
+
+//Speaker, ALL-Sentences, Start, End
+
+module.exports = {
+    name: "summarize-transcription", // Unique name for our function that will later be used to get the function from the map via "mapFunctions.get("example").function()"
+    type: "summarizer", // value used to differentiate each module to order them in the UI
+    displayname: "Summarizer",  // The displayname used within the UI
+    async function(args) {
+        let inputJson = args.json;
+
+        //JSON Path
+        if (args.jsonPath) {
+            try {
+                const raw = fs.readFileSync(args.jsonPath, "utf-8");
+                inputJson = JSON.parse(raw);
+            } catch (e) {
+                console.error("Failed to load JSON from file:", e);
+                return { error: "Could not read JSON from file path." };
+            }
+        }
+        // JSON parsen
+        if (typeof inputJson === "string") {
+            try {
+                inputJson = JSON.parse(inputJson);
+            } catch (e) {
+                console.log("Invalid JSON in summarize-transcription");
+                return { error: "Invalid JSON" };
+            }
+        }
+
+        const words = inputJson.words;
+        if (!Array.isArray(words)) {
+            return { error: "No words Array found" };
+        }
+
+        const ENDINGS = [".", "!", "?"]; // '...' auch als Satzende ?
+        const ABBREVIATIONS = new Set(["z.B.", "bzw.", "u.a.", "Dr.", "Mr.", "Mrs.", "Prof.", "etc."]); //TODO weitere Ergaenzen
+
+        const result = [];
+        let currentSentence = "";
+        let currentSpeaker = null;
+        let startTime = null;
+        let endTime = null;
+
+        for (const w of words) {
+            if (!currentSpeaker) currentSpeaker = w.speaker;
+            if (startTime === null) startTime = w.start;
+            endTime = w.end;
+    
+            //speaker changing
+            if (currentSpeaker !== w.speaker && currentSentence) { 
+                const lastEntry = result[result.length - 1];
+                if (lastEntry && lastEntry.speaker === currentSpeaker) {
+                    lastEntry.sentence += " " + currentSentence;
+                    lastEntry.end = endTime;
+                } else {
+                    result.push({
+                        speaker: currentSpeaker,
+                        sentence: currentSentence,
+                        start: startTime,
+                        end: endTime
+                    });
+                }
+                currentSentence = "";
+                startTime = w.start;
+            }
+            currentSpeaker = w.speaker;
+            currentSentence += (currentSentence ? " " : "") + w.text; //sentence beginning or not
+            const lastWord = w.text.trim();
+            const lastChar = lastWord.slice(-1);
+            const isAbbreviation = ABBREVIATIONS.has(lastWord);
+
+            //sentence ending
+            if (ENDINGS.includes(lastChar) && !isAbbreviation) {
+                const lastEntry = result[result.length - 1];
+                if (lastEntry && lastEntry.speaker === currentSpeaker) {
+                    lastEntry.sentence += " " + currentSentence;
+                    lastEntry.end = endTime;
+                } else {
+                    result.push({
+                        speaker: currentSpeaker,
+                        sentence: currentSentence,
+                        start: startTime,
+                        end: endTime
+                    });
+                }
+                currentSentence = "";
+                startTime = null;
+                endTime = null;
+                currentSpeaker = null;
+            }
+        }
+
+        // safe last sentence 
+        if (currentSentence) {
+            const lastEntry = result[result.length - 1];
+            if (lastEntry && lastEntry.speaker === currentSpeaker) {
+                lastEntry.sentence += " " + currentSentence;
+                lastEntry.end = endTime;
+            } else {
+                result.push({
+                    speaker: currentSpeaker,
+                    sentence: currentSentence,
+                    start: startTime,
+                    end: endTime
+                });
+            }
+        }
+
+        // Output as Text
+        const output = result.map(r => 
+            `Sprecher ${r.speaker} [${r.start.toFixed(2)} - ${r.end.toFixed(2)}]: ${r.sentence}`
+        );
+
+        // Output on cosole
+        //console.log("\n------------\nMerged Transcription Result:\n", output, "\n------------\n");
+
+        try {
+            const jsonPath = path.join(outputDir, "transcription_result.json");
+            fs.writeFileSync(jsonPath, JSON.stringify(result, null, 2), "utf-8");
+
+            const txtPath = path.join(outputDir, "transcription_result.txt");
+            fs.writeFileSync(txtPath, output.join("\n"), "utf-8");
+
+            console.log(`Summary successfully saved:\n- ${jsonPath}\n- ${txtPath}`);
+        } catch (err) {
+            console.error("Error saving Summary:", err);
+        }
+    }
+}

services/modules/jsonTools/transcriptionSummarizer2.js

@@ -0,0 +1,121 @@
+const fs = require("fs");
+const path = require("path");
+
+// Prepare output directory (always storage/transcriptionSummaries under project root)
+const outputDir = `${__dirname}/../../../storage/transcriptionSummaries`;
+if (!fs.existsSync(outputDir)) {
+    fs.mkdirSync(outputDir, { recursive: true });
+}
+
+//Speaker, Sentence, Start, End
+
+module.exports = {
+    name: "summarize-transcription2", // Unique name for our function that will later be used to get the function from the map via "mapFunctions.get("example").function()"
+    type: "summarizer", // value used to differentiate each module to order them in the UI
+    displayname: "Summarizer",  // The displayname used within the UI
+    async function(args) {
+        let inputJson = args.json;
+
+        //JSON Path
+        if (args.jsonPath) {
+            try {
+                const raw = fs.readFileSync(args.jsonPath, "utf-8");
+                inputJson = JSON.parse(raw);
+            } catch (e) {
+                console.error("Failed to load JSON from file:", e);
+                return { error: "Could not read JSON from file path." };
+            }
+        }
+        // JSON parsen
+        if (typeof inputJson === "string") {
+            try {
+                inputJson = JSON.parse(inputJson);
+            } catch (e) {
+                console.log("Invalid JSON in summarize-transcription");
+                return { error: "Invalid JSON" };
+            }
+        }
+
+        const words = inputJson.words;
+        if (!Array.isArray(words)) {
+            return { error: "No words Array found" };
+        }
+
+        const ENDINGS = [".", "!", "?"]; // '...' auch als Satzende ?
+        const ABBREVIATIONS = new Set(["z.B.", "bzw.", "u.a.", "Dr.", "Mr.", "Mrs.", "Prof.", "etc."]); //TODO weitere Ergaenzen
+
+        const result = [];
+        let currentSentence = "";
+        let currentSpeaker = null;
+        let startTime = null;
+        let endTime = null;
+
+        for (const w of words) {
+            if (!currentSpeaker) currentSpeaker = w.speaker;
+            if (startTime === null) startTime = w.start;
+            endTime = w.end;
+    
+            //speaker changing
+             if (currentSpeaker !== w.speaker && currentSentence) {
+                result.push({
+                    speaker: currentSpeaker,
+                    sentence: currentSentence,
+                    start: startTime,
+                    end: endTime
+                });
+                currentSentence = "";
+                startTime = w.start;
+            }
+            currentSpeaker = w.speaker;
+            currentSentence += (currentSentence ? " " : "") + w.text; //sentence beginning or not
+            const lastWord = w.text.trim();
+            const lastChar = lastWord.slice(-1);
+            const isAbbreviation = ABBREVIATIONS.has(lastWord);
+
+            //sentence ending
+            if (ENDINGS.includes(lastChar) && !isAbbreviation) {
+                result.push({
+                    speaker: currentSpeaker,
+                    sentence: currentSentence,
+                    start: startTime,
+                    end: endTime
+                });
+                currentSentence = "";
+                startTime = null;
+                endTime = null;
+                currentSpeaker = null;
+            }
+        }
+
+        // safe last sentence 
+        if (currentSentence) {
+            result.push({
+                speaker: currentSpeaker,
+                sentence: currentSentence,
+                start: startTime,
+                end: endTime
+            });
+        }
+
+
+        // Output as Text
+        const output = result.map(r => 
+            `Sprecher ${r.speaker} [${r.start.toFixed(2)} - ${r.end.toFixed(2)}]: ${r.sentence}`
+        );
+
+        // Output on cosole
+        //console.log("\n------------\nMerged Transcription Result:\n", output, "\n------------\n");
+
+        try {
+            const jsonPath = path.join(outputDir, "transcription_result.json");
+            fs.writeFileSync(jsonPath, JSON.stringify(result, null, 2), "utf-8");
+
+            const txtPath = path.join(outputDir, "transcription_result.txt");
+            fs.writeFileSync(txtPath, output.join("\n"), "utf-8");
+
+            console.log(`Summary successfully saved:\n- ${jsonPath}\n- ${txtPath}`);
+        } catch (err) {
+            console.error("Error saving Summary:", err);
+        }
+    }
+}

services/modules/transcription-remote/assembly.js

@@ -1,8 +1,123 @@
-module.exports = {
+require('dotenv').config();
-    name:"assembly", // Unique name for our function that will later be used to get the function from the map via "mapFunctions.get("example").function()"
+
-    type:"transcription", // value used to differentiate each module to order them in the UI
+const API_KEY = process.env.API_KEY;
-    displayname:"Assembly", // The displayname used within the UI
+const BASE_URL = 'https://api.assemblyai.com/v2';
-    async function(parameter){
+
-        // TODO add code to actually process the audio file
+//---------------------------------------------------Upload audio---------------------------------------------------
+
+async function uploadAudio(audioPath) {
+  const audioData = fs.readFileSync(audioPath);
+
+  const response = await axios.post(`${BASE_URL}/upload`, audioData, {
+    headers: {
+      authorization: API_KEY,
+      'content-type': 'application/octet-stream'
     }
-}
+  });
+
+  return response.data.upload_url;
+}
+
+////---------------------------------------------------Extract session id---------------------------------------------------
+
+function getSessionId(inputPath) {
+  try {
+    const parsed = new URL(inputPath);
+    const base = path.basename(parsed.pathname);
+    return base.replace(/\.[^.]+$/, '');
+  } catch {
+    return path.basename(inputPath, path.extname(inputPath));
+  }
+}
+
+//---------------------------------------------------Create transcript---------------------------------------------------
+
+async function createTranscript(audioUrl) {
+  const response = await axios.post(
+    `${BASE_URL}/transcript`,
+    {
+      audio_url: audioUrl,
+      speaker_labels: true,
+      language_detection: true
+    },
+    {
+      headers: {
+        authorization: API_KEY,
+        'content-type': 'application/json'
+      }
+    }
+  );
+
+  return response.data.id;
+}
+
+//---------------------------------------------------Poll transcript---------------------------------------------------
+
+async function pollTranscript(transcriptId) {
+  while (true) {
+    const response = await axios.get(`${BASE_URL}/transcript/${transcriptId}`, {
+      headers: { authorization: API_KEY }
+    });
+
+    const status = response.data.status;
+
+    if (status === 'completed') return response.data;
+    if (status === 'error') throw new Error(`Transcription failed: ${response.data.error}`);
+
+    await new Promise(res => setTimeout(res, 3000));
+  }
+}
+
+//---------------------------------------------------Save transcript---------------------------------------------------
+
+function saveTranscript(transcript, sessionId) {
+  const outputDir = path.join(__dirname, '../../../storage/transcripts');
+
+  if (!fs.existsSync(outputDir)) {
+    fs.mkdirSync(outputDir, { recursive: true });
+  }
+
+  const outputPath = path.join(outputDir, `${sessionId}.json`);
+  fs.writeFileSync(outputPath, JSON.stringify(transcript, null, 2));
+
+  console.log(`Transcript saved: ${outputPath}`);
+}
+
+//---------------------------------------------------Modul---------------------------------------------------
+
+module.exports = {
+  name: 'assembly',
+  type: 'transcription',
+  displayname: 'AssemblyAI',
+
+  async function(audioFileName) {
+    try {
+      // audioFileName ist nur "datei.mp3"
+      const audioPath = path.join(
+        __dirname,
+        '../../../storage/audio',
+        audioFileName
+      );
+
+      let audioUrl;
+
+      if (/^https?:\/\//i.test(audioFileName)) {
+        audioUrl = audioFileName;
+      } else {
+        if (!fs.existsSync(audioPath)) {
+          throw new Error(`Audio file not found: ${audioPath}`);
+        }
+        audioUrl = await uploadAudio(audioPath);
+      }
+
+      const transcriptId = await createTranscript(audioUrl);
+      const transcript = await pollTranscript(transcriptId);
+
+      const sessionId = getSessionId(audioFileName);
+      saveTranscript(transcript, sessionId);
+
+    } catch (error) {
+      console.error('Transcription error:', error.message);
+    }
+  }
+};

services/modules/utility/@startup.js

@@ -6,6 +6,13 @@ module.exports = {
         // We are now calling the example function from the example folder
         mapFunctions.get("example").function("Startup")
 
+       let transcript = await mapFunctions.get("assembly").function('../../storage/audio/IMG_2978.wav');
+
+       let summary = await mapFunctions.get("summarize-transcription").function({jsonPath:'/Users/santa/Proj25/video2document/storage/transcripts/IMG_2978.json'});
+
+
+        
+
         // mapFunctions.get("extraction-video-to-audio").function({inputVideoPath:"./a.mp4", outputType:"wav"})
         // mapFunctions.get("extraction-video-to-audio").function({inputVideoPath:"./b.mp4", outputType:"wav"})
         // mapFunctions.get("extraction-video-to-audio").function({inputVideoPath:"./b.mp4", outputType:"flac"})

test/integration/assemblyTest.js

@@ -0,0 +1,18 @@
+require('dotenv').config();
+const path = require('path');
+const assemblyModule = require('../../services/modules/transcription-remote/assembly.js');
+
+// Audio-Datei oder URL aus Kommandozeile, Standard: test.wav
+const audioPath = process.argv[2] || './storage/audio/IMG_2978.wav';
+
+(async () => {
+  try {
+    const transcript = await assemblyModule.run(audioPath);
+
+    console.log('Transcription succesful');
+    console.log('Transcript ID:', transcript?.id);
+    console.log('Speaker labels:', transcript?.utterances?.length || 0);
+  } catch (error) {
+    console.error('Error in Transcription:', error?.message || error);
+  }
+})();

test/integration/transcriptionSummarizerTest.js

@@ -0,0 +1,12 @@
+const fs = require('fs');
+const transSummarizer = require("../../services/modules/jsonTools/transcriptionSummarizer.js");
+
+// JSON-Datei laden
+const inputJson = JSON.parse(fs.readFileSync("./testFile.json", "utf8"));
+
+// Übergabe an den Summarizer
+transSummarizer.function({
+    json: inputJson
+});
+
+