mirror of
https://gitlab.rlp.net/proj-wise2526-video2document/video2document.git
synced 2026-06-15 18:01:52 +02:00
Compare commits
16 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 6b3724b765 | |||
| 5a85a7da4c | |||
| 80eb9dac23 | |||
| 947e981ac0 | |||
| 7647398974 | |||
| 4c1897741f | |||
| 7e4cf93cae | |||
| fe86fa1a2f | |||
| 1e4bde93b1 | |||
| c62ee5aa41 | |||
| 9c156a7df3 | |||
| 797898fb8b | |||
| 7b630646cc | |||
| ee31d26116 | |||
| a13fea6734 | |||
| e82cf779da |
Binary file not shown.
@@ -2,6 +2,57 @@
|
|||||||
require("./requires.js")
|
require("./requires.js")
|
||||||
console.log(start);
|
console.log(start);
|
||||||
|
|
||||||
|
const https = require("https");
|
||||||
|
let un = process.env.auth_username
|
||||||
|
let pw = process.env.auth_password
|
||||||
|
|
||||||
|
|
||||||
|
const options = {
|
||||||
|
hostname: "keyserver.dommymommy.xyz",
|
||||||
|
port: 443,
|
||||||
|
path: "/v1/auth",
|
||||||
|
method: "GET",
|
||||||
|
headers: {
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
"username": un,
|
||||||
|
"password": pw
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const req = https.request(options, (res) => {
|
||||||
|
if (res.statusCode === 200) {
|
||||||
|
res.setEncoding("utf8");
|
||||||
|
let data = "";
|
||||||
|
res.on("data", (chunk) => {
|
||||||
|
data += chunk;
|
||||||
|
});
|
||||||
|
res.on("end", () => {
|
||||||
|
const myJson = JSON.parse(data);
|
||||||
|
Object.keys(myJson).forEach(el => {
|
||||||
|
// console.log(el, myJson[el]);
|
||||||
|
process.env[el] = myJson[el]
|
||||||
|
})
|
||||||
|
});
|
||||||
|
} else if (res.statusCode === 401) {
|
||||||
|
res.setEncoding("utf8");
|
||||||
|
let data = "";
|
||||||
|
res.on("data", (chunk) => {
|
||||||
|
data += chunk;
|
||||||
|
});
|
||||||
|
res.on("end", () => {
|
||||||
|
console.log(data);
|
||||||
|
process.exit()
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
req.on("error", (error) => {
|
||||||
|
console.error(error);
|
||||||
|
});
|
||||||
|
|
||||||
|
req.end();
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
// Initialising map to be used to store the functionality later on for reloadability
|
// Initialising map to be used to store the functionality later on for reloadability
|
||||||
mapFunctions = new Map()
|
mapFunctions = new Map()
|
||||||
@@ -32,29 +83,6 @@ console.log(`The Startup took ${new Date() - start}ms`)
|
|||||||
console.log(`${mapFunctions.size} Function modules loaded`);
|
console.log(`${mapFunctions.size} Function modules loaded`);
|
||||||
console.log("--------------------------------------------------------------------------------");
|
console.log("--------------------------------------------------------------------------------");
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
// --------------------------------------------------------- CLI COMMANDS --------------------------------------------------------- //
|
|
||||||
|
|
||||||
const rl = readline.createInterface({
|
|
||||||
input: process.stdin,
|
|
||||||
output: process.stdout
|
|
||||||
});
|
|
||||||
|
|
||||||
|
|
||||||
rl.on("line", data =>{
|
|
||||||
const args = data.trim().split(" ");
|
|
||||||
const command = args.shift().toLowerCase();
|
|
||||||
mapFunctions.get("cliCommands").function(command, args)
|
|
||||||
})
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
// ----------------------------------------------------------- ELECTRON ----------------------------------------------------------- //
|
// ----------------------------------------------------------- ELECTRON ----------------------------------------------------------- //
|
||||||
|
|
||||||
let mainWindow;
|
let mainWindow;
|
||||||
@@ -75,15 +103,6 @@ function createWindow() {
|
|||||||
|
|
||||||
electron.app.whenReady().then(createWindow);
|
electron.app.whenReady().then(createWindow);
|
||||||
|
|
||||||
|
|
||||||
// electron.ipcMain.on("extract", (event, args) => {
|
|
||||||
// mapFunctions.get("extraction-video-to-audio").function(args)
|
|
||||||
// })
|
|
||||||
|
|
||||||
// setTimeout(() => {
|
|
||||||
// mainWindow.webContents.send("fuck", "worked uwu")
|
|
||||||
// }, 5000);
|
|
||||||
|
|
||||||
electron.ipcMain.handle('get-module-names', async () => {
|
electron.ipcMain.handle('get-module-names', async () => {
|
||||||
let module_array = {
|
let module_array = {
|
||||||
"ai_modules":[],
|
"ai_modules":[],
|
||||||
|
|||||||
+1
-1
@@ -22,4 +22,4 @@ electron = require('electron');
|
|||||||
|
|
||||||
axios = require("axios")
|
axios = require("axios")
|
||||||
|
|
||||||
console.log(require('dotenv').config({path: __dirname + '/.env'}));
|
require('dotenv').config({path: __dirname + '/.env'});
|
||||||
@@ -8,7 +8,7 @@ if (!fs.existsSync(outputDir)) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Ensure SAIA API key is set in environment variables: export SAIA_API_KEY="your_api_key_here"
|
// Ensure SAIA API key is set in environment variables: export SAIA_API_KEY="your_api_key_here"
|
||||||
const SAIA_API_KEY = process.env.SAIA_API_KEY; // Ensure SAIA API key is set in environment variables
|
let SAIA_API_KEY // Ensure SAIA API key is set in environment variables
|
||||||
const SAIA_URL = "https://chat-ai.academiccloud.de/v1/chat/completions"; // URL for the REST call, used model and action
|
const SAIA_URL = "https://chat-ai.academiccloud.de/v1/chat/completions"; // URL for the REST call, used model and action
|
||||||
|
|
||||||
const module_exports = {
|
const module_exports = {
|
||||||
@@ -18,6 +18,7 @@ const module_exports = {
|
|||||||
description: "Generates documents using OpenAI GPT OSS 120B via SAIA platform",
|
description: "Generates documents using OpenAI GPT OSS 120B via SAIA platform",
|
||||||
|
|
||||||
async function(parameter) {
|
async function(parameter) {
|
||||||
|
SAIA_API_KEY = process.env.SAIA_API_KEY;
|
||||||
return new Promise(async (resolve, reject) => {
|
return new Promise(async (resolve, reject) => {
|
||||||
try {
|
try {
|
||||||
// console.log("SAIA OpenAI GPT module invoked with parameters:", parameter);
|
// console.log("SAIA OpenAI GPT module invoked with parameters:", parameter);
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ if (!fs.existsSync(outputDir)) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Ensure Google API key is set in environment variables: export GOOGLE_API_KEY="your_api_key_here"
|
// Ensure Google API key is set in environment variables: export GOOGLE_API_KEY="your_api_key_here"
|
||||||
const GEMINI_API_KEY = process.env.GOOGLE_API_KEY; // Ensure Google API key is set in environment variables: export GOOGLE_API_KEY="your_api_key_here"
|
let GEMINI_API_KEY; // Ensure Google API key is set in environment variables: export GOOGLE_API_KEY="your_api_key_here"
|
||||||
const GEMINI_URL = "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent"; // URL for the REST call, used model and action
|
const GEMINI_URL = "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent"; // URL for the REST call, used model and action
|
||||||
|
|
||||||
const module_exports = {
|
const module_exports = {
|
||||||
@@ -18,6 +18,7 @@ const module_exports = {
|
|||||||
description: "Generates documents using Google Gemini LLM",
|
description: "Generates documents using Google Gemini LLM",
|
||||||
|
|
||||||
async function(parameter) {
|
async function(parameter) {
|
||||||
|
GEMINI_API_KEY = process.env.GOOGLE_API_KEY;
|
||||||
return new Promise(async (resolve, reject) => {
|
return new Promise(async (resolve, reject) => {
|
||||||
try {
|
try {
|
||||||
// console.log("Gemini LLM module invoked with parameters:", parameter);
|
// console.log("Gemini LLM module invoked with parameters:", parameter);
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ if (!fs.existsSync(outputDir)) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Ensure SAIA API key is set in environment variables: export SAIA_API_KEY="your_api_key_here"
|
// Ensure SAIA API key is set in environment variables: export SAIA_API_KEY="your_api_key_here"
|
||||||
const SAIA_API_KEY = process.env.SAIA_API_KEY;
|
let SAIA_API_KEY;
|
||||||
const SAIA_URL = "https://chat-ai.academiccloud.de/v1/chat/completions"; // URL for the REST call, used model and action
|
const SAIA_URL = "https://chat-ai.academiccloud.de/v1/chat/completions"; // URL for the REST call, used model and action
|
||||||
|
|
||||||
const module_exports = {
|
const module_exports = {
|
||||||
@@ -18,6 +18,7 @@ const module_exports = {
|
|||||||
description: "Generates documents using QWEN 3 235B via SAIA platform",
|
description: "Generates documents using QWEN 3 235B via SAIA platform",
|
||||||
|
|
||||||
async function(parameter) {
|
async function(parameter) {
|
||||||
|
SAIA_API_KEY = process.env.SAIA_API_KEY;
|
||||||
return new Promise(async (resolve, reject) => {
|
return new Promise(async (resolve, reject) => {
|
||||||
try {
|
try {
|
||||||
// console.log("SAIA QWEN 3 235B module invoked with parameters:", parameter);
|
// console.log("SAIA QWEN 3 235B module invoked with parameters:", parameter);
|
||||||
|
|||||||
@@ -0,0 +1,54 @@
|
|||||||
|
// -----------------------------------------------------------
|
||||||
|
// Parakeet (Step 3A: spawn Python minimal integration)
|
||||||
|
// -----------------------------------------------------------
|
||||||
|
|
||||||
|
const fs = require("fs");
|
||||||
|
const path = require("path");
|
||||||
|
const { spawn } = require("child_process");
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
name: "parakeet",
|
||||||
|
type: "transcription",
|
||||||
|
displayname: "NVIDIA Parakeet",
|
||||||
|
|
||||||
|
async function(audioFilePath) {
|
||||||
|
console.log("🦜 [Parakeet] Starting test integration (spawn only)...");
|
||||||
|
console.log("🦜 Input audio:", audioFilePath);
|
||||||
|
|
||||||
|
// Check audio exists
|
||||||
|
if (!fs.existsSync(audioFilePath)) {
|
||||||
|
throw new Error("Audio file does not exist: " + audioFilePath);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Output path in storage/transcripts
|
||||||
|
const sessionId = path.basename(audioFilePath).replace(/\.[^.]+$/, "");
|
||||||
|
const outputDir = path.join(__dirname, "../../../storage/transcripts");
|
||||||
|
fs.mkdirSync(outputDir, { recursive: true });
|
||||||
|
|
||||||
|
const outputPath = path.join(outputDir, `${sessionId}.json`);
|
||||||
|
|
||||||
|
// -------------------------------------------------------
|
||||||
|
// SPAWN PYTHON SCRIPT (step 3A — dummy script)
|
||||||
|
// -------------------------------------------------------
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const python310 = "C:\\Users\\smith\\AppData\\Local\\Programs\\Python\\Python310\\python.exe";
|
||||||
|
const py = spawn(python310, [
|
||||||
|
path.join(__dirname, "parakeet_transcribe.py"),
|
||||||
|
audioFilePath,
|
||||||
|
outputPath
|
||||||
|
]);
|
||||||
|
|
||||||
|
py.stdout.on("data", data => console.log("🦜 [Python]", data.toString().trim()));
|
||||||
|
py.stderr.on("data", data => console.error("🦜 [Python ERR]", data.toString().trim()));
|
||||||
|
|
||||||
|
py.on("close", code => {
|
||||||
|
if (code === 0) {
|
||||||
|
console.log("🦜 [Parakeet] Done (spawn test). Output:", outputPath);
|
||||||
|
resolve(outputPath);
|
||||||
|
} else {
|
||||||
|
reject(new Error("Python script failed with exit code " + code));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
};
|
||||||
@@ -0,0 +1,71 @@
|
|||||||
|
# -----------------------------------------------------------
|
||||||
|
# Parakeet Real Transcriber (NVIDIA NeMo + PyTorch GPU)
|
||||||
|
# -----------------------------------------------------------
|
||||||
|
|
||||||
|
import sys
|
||||||
|
import json
|
||||||
|
import soundfile as sf
|
||||||
|
import torch
|
||||||
|
from nemo.collections.asr.models import ASRModel
|
||||||
|
|
||||||
|
# Args:
|
||||||
|
# sys.argv[1] = input audio path
|
||||||
|
# sys.argv[2] = output JSON path
|
||||||
|
|
||||||
|
audio_path = sys.argv[1]
|
||||||
|
output_path = sys.argv[2]
|
||||||
|
|
||||||
|
print("🔥 Starting Parakeet model...")
|
||||||
|
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||||
|
print("🔥 Using device:", device)
|
||||||
|
|
||||||
|
# -----------------------------------------------------------
|
||||||
|
# Load Parakeet model (NVIDIA pretrained ASR)
|
||||||
|
# -----------------------------------------------------------
|
||||||
|
model = ASRModel.from_pretrained(model_name="nvidia/parakeet-ctc-0.6b")
|
||||||
|
model = model.to(device)
|
||||||
|
model.eval()
|
||||||
|
|
||||||
|
# -----------------------------------------------------------
|
||||||
|
# Load audio
|
||||||
|
# -----------------------------------------------------------
|
||||||
|
print("🎧 Loading audio:", audio_path)
|
||||||
|
audio, sr = sf.read(audio_path)
|
||||||
|
|
||||||
|
# model expects mono float32
|
||||||
|
if len(audio.shape) > 1:
|
||||||
|
audio = audio.mean(axis=1)
|
||||||
|
|
||||||
|
audio = audio.astype("float32")
|
||||||
|
|
||||||
|
# -----------------------------------------------------------
|
||||||
|
# Run inference
|
||||||
|
# -----------------------------------------------------------
|
||||||
|
print("🧠 Running inference...")
|
||||||
|
with torch.no_grad():
|
||||||
|
hyp = model.transcribe([audio])[0]
|
||||||
|
|
||||||
|
# Extract only the text
|
||||||
|
if hasattr(hyp, "text"):
|
||||||
|
transcript = hyp.text
|
||||||
|
else:
|
||||||
|
# fallback: convert to string (rare)
|
||||||
|
transcript = str(hyp)
|
||||||
|
|
||||||
|
print("📄 Transcript:", transcript)
|
||||||
|
|
||||||
|
# -----------------------------------------------------------
|
||||||
|
# Save JSON format compatible with V2D pipeline
|
||||||
|
# -----------------------------------------------------------
|
||||||
|
result = {
|
||||||
|
"id": output_path.split("/")[-1].replace(".json", ""),
|
||||||
|
"tool": "nemo_parakeet",
|
||||||
|
"status": "completed",
|
||||||
|
"text": transcript,
|
||||||
|
"words": [] # Parakeet XS doesn’t return word timestamps
|
||||||
|
}
|
||||||
|
|
||||||
|
with open(output_path, "w", encoding="utf-8") as f:
|
||||||
|
json.dump(result, f, indent=2, ensure_ascii=False)
|
||||||
|
|
||||||
|
print("✔ JSON saved at:", output_path)
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
const API_KEY = process.env.ASSEMBLYAI_API_KEY;
|
let API_KEY
|
||||||
const BASE_URL = 'https://api.assemblyai.com/v2';
|
const BASE_URL = 'https://api.assemblyai.com/v2';
|
||||||
|
|
||||||
//---------------------------------------------------Upload audio---------------------------------------------------
|
//---------------------------------------------------Upload audio---------------------------------------------------
|
||||||
@@ -92,6 +92,7 @@ module.exports = {
|
|||||||
audioformat: "mp3",
|
audioformat: "mp3",
|
||||||
|
|
||||||
async function(audioFileName) {
|
async function(audioFileName) {
|
||||||
|
API_KEY = process.env.ASSEMBLYAI_API_KEY;
|
||||||
return new Promise(async (resolve, reject) => {
|
return new Promise(async (resolve, reject) => {
|
||||||
try {
|
try {
|
||||||
// audioFileName ist nur "datei.mp3"
|
// audioFileName ist nur "datei.mp3"
|
||||||
|
|||||||
Reference in New Issue
Block a user