mirror of
https://gitlab.rlp.net/proj-wise2526-video2document/video2document.git
synced 2026-06-15 18:01:52 +02:00
Remade replace Speaker module which now takes the premade speaker mapping json and directly replaces speakers in the document. Also refined the Standard documents so consistency is improved
This commit is contained in:
@@ -1,81 +1,72 @@
|
||||
// Loading required packages
|
||||
require("./requires.js")
|
||||
require("./requires.js");
|
||||
console.log(start);
|
||||
|
||||
|
||||
// Initialising map to be used to store the functionality later on for reloadability
|
||||
mapFunctions = new Map()
|
||||
|
||||
mapFunctions = new Map();
|
||||
|
||||
// Loading the Function Map
|
||||
var path = `${mainDir}/services/modules`
|
||||
var path = `${mainDir}/services/modules`;
|
||||
var folders = fs.readdirSync(path).filter(function (file) {
|
||||
return fs.statSync(path+'/'+file).isDirectory();
|
||||
return fs.statSync(path + "/" + file).isDirectory();
|
||||
});
|
||||
folders.forEach(element => {
|
||||
var commandFiles = fs.readdirSync(`${path}/${element}`).filter(file => file.endsWith('.js') && !file.startsWith("index"));
|
||||
for (const file of commandFiles) {
|
||||
delete require.cache[require.resolve(`${path}/${element}/${file}`)];
|
||||
const command = require(`${path}/${element}/${file}`);
|
||||
mapFunctions.set(command.name, command);
|
||||
}
|
||||
folders.forEach((element) => {
|
||||
var commandFiles = fs
|
||||
.readdirSync(`${path}/${element}`)
|
||||
.filter((file) => file.endsWith(".js") && !file.startsWith("index"));
|
||||
for (const file of commandFiles) {
|
||||
delete require.cache[require.resolve(`${path}/${element}/${file}`)];
|
||||
const command = require(`${path}/${element}/${file}`);
|
||||
mapFunctions.set(command.name, command);
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
|
||||
// The startup information for the project, here you can add stuff that might be nice to see when the app starts
|
||||
mapFunctions.get("Startup_function").function()
|
||||
console.log("------------------------------------ Status ------------------------------------");
|
||||
mapFunctions.get("Startup_function").function();
|
||||
console.log(
|
||||
"------------------------------------ Status ------------------------------------"
|
||||
);
|
||||
console.log(__dirname);
|
||||
console.log(platform);
|
||||
console.log(`The Startup took ${new Date() - start}ms`)
|
||||
console.log(`The Startup took ${new Date() - start}ms`);
|
||||
console.log(`${mapFunctions.size} Function modules loaded`);
|
||||
console.log("--------------------------------------------------------------------------------");
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
console.log(
|
||||
"--------------------------------------------------------------------------------"
|
||||
);
|
||||
|
||||
// --------------------------------------------------------- CLI COMMANDS --------------------------------------------------------- //
|
||||
|
||||
const rl = readline.createInterface({
|
||||
input: process.stdin,
|
||||
output: process.stdout
|
||||
input: process.stdin,
|
||||
output: process.stdout,
|
||||
});
|
||||
|
||||
|
||||
rl.on("line", data =>{
|
||||
const args = data.trim().split(" ");
|
||||
const command = args.shift().toLowerCase();
|
||||
mapFunctions.get("cliCommands").function(command, args)
|
||||
})
|
||||
|
||||
|
||||
|
||||
|
||||
rl.on("line", (data) => {
|
||||
const args = data.trim().split(" ");
|
||||
const command = args.shift().toLowerCase();
|
||||
mapFunctions.get("cliCommands").function(command, args);
|
||||
});
|
||||
|
||||
// ----------------------------------------------------------- ELECTRON ----------------------------------------------------------- //
|
||||
|
||||
let mainWindow;
|
||||
|
||||
function createWindow() {
|
||||
mainWindow = new electron.BrowserWindow({
|
||||
width: 1200,
|
||||
height: 800,
|
||||
webPreferences: {
|
||||
nodeIntegration: false,
|
||||
contextIsolation: true,
|
||||
preload: `${mainDir}/electron/main/preload.js`
|
||||
}
|
||||
});
|
||||
mainWindow = new electron.BrowserWindow({
|
||||
width: 1200,
|
||||
height: 800,
|
||||
webPreferences: {
|
||||
nodeIntegration: false,
|
||||
contextIsolation: true,
|
||||
preload: `${mainDir}/electron/main/preload.js`,
|
||||
},
|
||||
});
|
||||
|
||||
mainWindow.loadFile('./electron/main/index.html');
|
||||
mainWindow.loadFile("./electron/main/index.html");
|
||||
}
|
||||
|
||||
electron.app.whenReady().then(createWindow);
|
||||
|
||||
|
||||
// electron.ipcMain.on("extract", (event, args) => {
|
||||
// mapFunctions.get("extraction-video-to-audio").function(args)
|
||||
// })
|
||||
@@ -84,26 +75,31 @@ electron.app.whenReady().then(createWindow);
|
||||
// mainWindow.webContents.send("fuck", "worked uwu")
|
||||
// }, 5000);
|
||||
|
||||
electron.ipcMain.handle('get-module-names', async () => {
|
||||
let module_array = {
|
||||
"ai_modules":[],
|
||||
"transcription_modules":[]
|
||||
electron.ipcMain.handle("get-module-names", async () => {
|
||||
let module_array = {
|
||||
ai_modules: [],
|
||||
transcription_modules: [],
|
||||
};
|
||||
mapFunctions.forEach((e) => {
|
||||
switch (e.type) {
|
||||
case "llm":
|
||||
module_array.ai_modules.push({
|
||||
name: e.name,
|
||||
displayname: e.displayname,
|
||||
});
|
||||
break;
|
||||
case "transcription":
|
||||
module_array.transcription_modules.push({
|
||||
name: e.name,
|
||||
displayname: e.displayname,
|
||||
});
|
||||
break;
|
||||
}
|
||||
mapFunctions.forEach(e => {
|
||||
switch(e.type){
|
||||
case "llm":
|
||||
module_array.ai_modules.push({"name": e.name, "displayname": e.displayname})
|
||||
break;
|
||||
case "transcription":
|
||||
module_array.transcription_modules.push({"name": e.name, "displayname": e.displayname})
|
||||
break;
|
||||
}
|
||||
})
|
||||
// console.log(module_array);
|
||||
return module_array
|
||||
});
|
||||
// console.log(module_array);
|
||||
return module_array;
|
||||
});
|
||||
|
||||
|
||||
// electron.ipcMain.on("get_modules", async (event, args) => {
|
||||
// let module_array = {
|
||||
// "ai_modules":[],
|
||||
@@ -124,165 +120,219 @@ electron.ipcMain.handle('get-module-names', async () => {
|
||||
// mainWindow.webContents.send("modules", module_array)
|
||||
// })
|
||||
|
||||
var globalArgs = {}
|
||||
var globalFinalHtmlPath = ""
|
||||
var globalArgs = {};
|
||||
var globalFinalHtmlPath = "";
|
||||
|
||||
electron.ipcMain.on("file_submit", async (event, args) => {
|
||||
try {
|
||||
globalArgs = args
|
||||
let curstep = 0
|
||||
let totalsteps = 4
|
||||
try {
|
||||
globalArgs = args;
|
||||
let curstep = 0;
|
||||
let totalsteps = 4;
|
||||
|
||||
const TEMPLATE_MAP = {
|
||||
"followup-report": "followup_report.txt",
|
||||
"agenda": "agenda.txt",
|
||||
"result-protocol": "result_protocol.txt",
|
||||
"sprint-planning": "sprint_planning_note.txt",
|
||||
"custom": "custom_document.txt"
|
||||
};
|
||||
const TEMPLATE_MAP = {
|
||||
"followup-report": "followup_report.txt",
|
||||
agenda: "agenda.txt",
|
||||
"result-protocol": "result_protocol.txt",
|
||||
"sprint-planning": "sprint_planning_note.txt",
|
||||
custom: "custom_document.txt",
|
||||
};
|
||||
|
||||
const templateFile = TEMPLATE_MAP[args.document.type];
|
||||
const templateFile = TEMPLATE_MAP[args.document.type];
|
||||
|
||||
if (!templateFile) {
|
||||
throw new Error("Unknown document type: " + args.document.type);
|
||||
}
|
||||
|
||||
console.log(args);
|
||||
let audiopath = ""
|
||||
let transcriptpath = ""
|
||||
|
||||
console.log("\n\n Running the Video to Audio Extractor");
|
||||
// This code handles the Video to Audio extraction module call
|
||||
await mapFunctions.get("module-handler").function(args.video.module, {inputVideoPath: args.video.inputVideoPath, outputType: mapFunctions.get(args.transcription.module).audioformat}).then(resp => {
|
||||
console.log(resp);
|
||||
audiopath = resp
|
||||
curstep++
|
||||
mainWindow.webContents.send("progress", {curstep:curstep, totalsteps:totalsteps})
|
||||
}).catch(err => {
|
||||
mainWindow.webContents.send("error", err)
|
||||
console.log(err);
|
||||
return
|
||||
})
|
||||
|
||||
|
||||
console.log("\n\n Running the Audio to Transcription module");
|
||||
// TODO implement transcription module
|
||||
// This code handles the Audio to Text transcription module call
|
||||
await mapFunctions.get("module-handler").function(args.transcription.module, audiopath).then(resp => {
|
||||
console.log(resp);
|
||||
transcriptpath = resp
|
||||
curstep++
|
||||
mainWindow.webContents.send("progress", {curstep:curstep, totalsteps:totalsteps})
|
||||
}).catch(err => {
|
||||
mainWindow.webContents.send("error", err)
|
||||
console.log(err);
|
||||
return
|
||||
})
|
||||
|
||||
|
||||
console.log("\n\n Running the Transcription Summarizer module");
|
||||
// This code summarises the transcript, so that it can be used by an llm
|
||||
// await mapFunctions.get("summarize-transcription").function('A:\\programing\\@projects\\video2document\\storage\\transcripts\\IMG_2978.json').then(resp => {
|
||||
await mapFunctions.get("summarize-transcription2").function(transcriptpath).then(resp => {
|
||||
console.log(resp);
|
||||
transcriptpath = resp
|
||||
curstep++
|
||||
mainWindow.webContents.send("progress", {curstep:curstep, totalsteps:totalsteps})
|
||||
}).catch(err => {
|
||||
mainWindow.webContents.send("error", err)
|
||||
console.log(err);
|
||||
return
|
||||
})
|
||||
|
||||
console.log("\n\n Running the LLM module");
|
||||
// TODO implement documentation module
|
||||
// This code handles the Text to Document processing module call
|
||||
|
||||
console.log(`\n\n Running the LLM for Document Style ${args.document.type}`);
|
||||
|
||||
await mapFunctions.get("module-handler").function(args.document.module, { inputTranscriptPath: transcriptpath, documentTypePath: "./storage/documentType/" + templateFile, language: "en" }).then(resp => {
|
||||
console.log(resp);
|
||||
globalFinalHtmlPath = resp
|
||||
curstep++
|
||||
mainWindow.webContents.send("progress", {curstep:curstep, totalsteps:totalsteps})
|
||||
}).catch(err => {
|
||||
mainWindow.webContents.send("error", err)
|
||||
console.log(err);
|
||||
return
|
||||
})
|
||||
|
||||
|
||||
await mapFunctions.get("extract-speaker-snippets").function({audioPath: audiopath, jsonPath: transcriptpath }).then(resp => {
|
||||
mainWindow.webContents.send("speakerAudios", resp)
|
||||
}).catch(err => {
|
||||
mainWindow.webContents.send("error", err)
|
||||
console.log(err);
|
||||
return
|
||||
})
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
if (!templateFile) {
|
||||
throw new Error("Unknown document type: " + args.document.type);
|
||||
}
|
||||
})
|
||||
|
||||
electron.ipcMain.on("file_download", async() => {
|
||||
await mapFunctions.get("htmlDocumentConverter").convert({inputPath:globalFinalHtmlPath, format: globalArgs.document.outputType, showDialog: true});
|
||||
})
|
||||
console.log(args);
|
||||
let audiopath = "";
|
||||
let transcriptpath = "";
|
||||
|
||||
electron.ipcMain.on("speaker_submit", async() => {
|
||||
console.log("\n\n\nJa also hier kam was an \n\n\n");
|
||||
})
|
||||
console.log("\n\n Running the Video to Audio Extractor");
|
||||
// This code handles the Video to Audio extraction module call
|
||||
await mapFunctions
|
||||
.get("module-handler")
|
||||
.function(args.video.module, {
|
||||
inputVideoPath: args.video.inputVideoPath,
|
||||
outputType: mapFunctions.get(args.transcription.module).audioformat,
|
||||
})
|
||||
.then((resp) => {
|
||||
console.log(resp);
|
||||
audiopath = resp;
|
||||
curstep++;
|
||||
mainWindow.webContents.send("progress", {
|
||||
curstep: curstep,
|
||||
totalsteps: totalsteps,
|
||||
});
|
||||
})
|
||||
.catch((err) => {
|
||||
mainWindow.webContents.send("error", err);
|
||||
console.log(err);
|
||||
return;
|
||||
});
|
||||
|
||||
let q =
|
||||
{
|
||||
video: {
|
||||
module: "String", // The name of the module, idk if we ever implement other extraction modules, the default one is extraction-video-to-audio
|
||||
inputVideoPath: "String", // See script.js on line 27 for an example of what this should look like
|
||||
outputType: "String" // The file format to be used for the audio output file, such as wav, mp3, flac and so on
|
||||
},
|
||||
transcription:{
|
||||
module: "String" // The module name of the transcription model you want to use
|
||||
},
|
||||
document:{
|
||||
module: "String", // The module name of the AI model you want to use to create the document
|
||||
styles: [ // An array of all the document styles/prompts you want to have the document be processed with
|
||||
{
|
||||
prompt: "String",
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
console.log("\n\n Running the Audio to Transcription module");
|
||||
// TODO implement transcription module
|
||||
// This code handles the Audio to Text transcription module call
|
||||
await mapFunctions
|
||||
.get("module-handler")
|
||||
.function(args.transcription.module, audiopath)
|
||||
.then((resp) => {
|
||||
console.log(resp);
|
||||
transcriptpath = resp;
|
||||
curstep++;
|
||||
mainWindow.webContents.send("progress", {
|
||||
curstep: curstep,
|
||||
totalsteps: totalsteps,
|
||||
});
|
||||
})
|
||||
.catch((err) => {
|
||||
mainWindow.webContents.send("error", err);
|
||||
console.log(err);
|
||||
return;
|
||||
});
|
||||
|
||||
console.log("\n\n Running the Transcription Summarizer module");
|
||||
// This code summarises the transcript, so that it can be used by an llm
|
||||
// await mapFunctions.get("summarize-transcription").function('A:\\programing\\@projects\\video2document\\storage\\transcripts\\IMG_2978.json').then(resp => {
|
||||
await mapFunctions
|
||||
.get("summarize-transcription2")
|
||||
.function(transcriptpath)
|
||||
.then((resp) => {
|
||||
console.log(resp);
|
||||
transcriptpath = resp;
|
||||
curstep++;
|
||||
mainWindow.webContents.send("progress", {
|
||||
curstep: curstep,
|
||||
totalsteps: totalsteps,
|
||||
});
|
||||
})
|
||||
.catch((err) => {
|
||||
mainWindow.webContents.send("error", err);
|
||||
console.log(err);
|
||||
return;
|
||||
});
|
||||
|
||||
console.log("\n\n Running the LLM module");
|
||||
// TODO implement documentation module
|
||||
// This code handles the Text to Document processing module call
|
||||
|
||||
console.log(
|
||||
`\n\n Running the LLM for Document Style ${args.document.type}`
|
||||
);
|
||||
|
||||
await mapFunctions
|
||||
.get("module-handler")
|
||||
.function(args.document.module, {
|
||||
inputTranscriptPath: transcriptpath,
|
||||
documentTypePath: "./storage/documentType/" + templateFile,
|
||||
language: "en",
|
||||
})
|
||||
.then((resp) => {
|
||||
console.log(resp);
|
||||
globalFinalHtmlPath = resp;
|
||||
curstep++;
|
||||
mainWindow.webContents.send("progress", {
|
||||
curstep: curstep,
|
||||
totalsteps: totalsteps,
|
||||
});
|
||||
})
|
||||
.catch((err) => {
|
||||
mainWindow.webContents.send("error", err);
|
||||
console.log(err);
|
||||
return;
|
||||
});
|
||||
|
||||
await mapFunctions
|
||||
.get("extract-speaker-snippets")
|
||||
.function({ audioPath: audiopath, jsonPath: transcriptpath })
|
||||
.then((resp) => {
|
||||
mainWindow.webContents.send("speakerAudios", resp);
|
||||
})
|
||||
.catch((err) => {
|
||||
mainWindow.webContents.send("error", err);
|
||||
console.log(err);
|
||||
return;
|
||||
});
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
}
|
||||
});
|
||||
|
||||
electron.ipcMain.on("file_download", async () => {
|
||||
await mapFunctions
|
||||
.get("htmlDocumentConverter")
|
||||
.convert({
|
||||
inputPath: globalFinalHtmlPath,
|
||||
format: globalArgs.document.outputType,
|
||||
showDialog: true,
|
||||
});
|
||||
});
|
||||
|
||||
electron.ipcMain.on("speaker_submit", async (event, args) => {
|
||||
console.log("\n\n\nJa also hier kam was an \n\n\n");
|
||||
console.log(args);
|
||||
|
||||
try {
|
||||
await mapFunctions.get("replace_speaker").function(args);
|
||||
event.reply("speaker_submit_response", { success: true });
|
||||
} catch (error) {
|
||||
console.error("Error:", error);
|
||||
event.reply("speaker_submit_response", {
|
||||
success: false,
|
||||
error: error.message,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
let q = {
|
||||
video: {
|
||||
module: "String", // The name of the module, idk if we ever implement other extraction modules, the default one is extraction-video-to-audio
|
||||
inputVideoPath: "String", // See script.js on line 27 for an example of what this should look like
|
||||
outputType: "String", // The file format to be used for the audio output file, such as wav, mp3, flac and so on
|
||||
},
|
||||
transcription: {
|
||||
module: "String", // The module name of the transcription model you want to use
|
||||
},
|
||||
document: {
|
||||
module: "String", // The module name of the AI model you want to use to create the document
|
||||
styles: [
|
||||
// An array of all the document styles/prompts you want to have the document be processed with
|
||||
{
|
||||
prompt: "String",
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
let q1 = {
|
||||
"ai_modules": [
|
||||
{name:"abc", displayname:"ABC"},
|
||||
{name:"qeg", displayname:"aqghegahu"}
|
||||
],
|
||||
"transcription_modules": [
|
||||
{name:"abc", displayname:"ABC"},
|
||||
{name:"qeg", displayname:"aqghegahu"}
|
||||
]
|
||||
}
|
||||
ai_modules: [
|
||||
{ name: "abc", displayname: "ABC" },
|
||||
{ name: "qeg", displayname: "aqghegahu" },
|
||||
],
|
||||
transcription_modules: [
|
||||
{ name: "abc", displayname: "ABC" },
|
||||
{ name: "qeg", displayname: "aqghegahu" },
|
||||
],
|
||||
};
|
||||
|
||||
//gibt Documentfiles an preload zurück
|
||||
|
||||
electron.ipcMain.handle('get-txt-files', () => {
|
||||
const storagePath = `${mainDir}/storage/documentType`
|
||||
electron.ipcMain.handle("get-txt-files", () => {
|
||||
const storagePath = `${mainDir}/storage/documentType`;
|
||||
|
||||
return fs.readdirSync(storagePath)
|
||||
.filter(f => f.endsWith('.txt'))
|
||||
return fs.readdirSync(storagePath).filter((f) => f.endsWith(".txt"));
|
||||
});
|
||||
|
||||
//speichern neuer document types
|
||||
|
||||
electron.ipcMain.handle('save-txt-file', (event, fileName, content) => {
|
||||
electron.ipcMain.handle("save-txt-file", (event, fileName, content) => {
|
||||
const filePath = `${mainDir}/storage/documentType/${fileName}.txt`;
|
||||
fs.writeFileSync(filePath, content, 'utf8');
|
||||
fs.writeFileSync(filePath, content, "utf8");
|
||||
return true;
|
||||
});
|
||||
|
||||
//
|
||||
electron.ipcMain.handle('read-txt-file', (event, fileName) => {
|
||||
electron.ipcMain.handle("read-txt-file", (event, fileName) => {
|
||||
const filePath = `${mainDir}/storage/documentType/${fileName}`;
|
||||
return fs.readFileSync(filePath, 'utf8');
|
||||
});
|
||||
return fs.readFileSync(filePath, "utf8");
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user