feat: Adding OpenAI TTS CLI command

.gitignore

@@ -47,3 +47,6 @@ apps/**/dist
 # ENV
 .env
 .vercel
+
+# TEMP
+temp

apps/cli/.eslintrc.js

@@ -0,0 +1,4 @@
+module.exports = {
+  root: true,
+  extends: ["thugga"],
+};

apps/cli/README.md

@@ -0,0 +1 @@
+CLI

apps/cli/jest.config.js

@@ -0,0 +1 @@
+module.exports = require("@thugga/jest");

apps/cli/package.json

@@ -0,0 +1,53 @@
+{
+  "name": "cli",
+  "version": "0.0.1",
+  "license": "MIT",
+  "main": "./src/index.ts",
+  "scripts": {
+    "build": "tsc",
+    "clean": "rm -rf .turbo && rm -rf node_modules && rm -rf dist && rm -rf coverage && rm -rf cdk-out",
+    "cmd": "npx ts-node ./src/index.ts",
+    "lint": "eslint 'src/**/*.{js,ts}'",
+    "lint:fix": "eslint 'src/**/*.{js,ts}' --fix",
+    "test": "jest --forceExit",
+    "test:u": "jest -u --forceExit"
+  },
+  "dependencies": {
+    "@inquirer/prompts": "4.1.0",
+    "@markdoc/markdoc": "0.4.0",
+    "chalk": "4.1.2",
+    "commander": "12.0.0",
+    "date-fns": "3.0.6",
+    "dotenv": "16.3.1",
+    "figlet": "1.7.0",
+    "fluent-ffmpeg": "2.1.3",
+    "html-to-text": "9.0.5",
+    "openai": "4.56.0",
+    "pino": "8.19.0",
+    "ulid": "2.3.0",
+    "zod": "3.23.8",
+    "zx": "7.2.3"
+  },
+  "devDependencies": {
+    "@thugga/jest": "workspace:*",
+    "@types/figlet": "1.5.8",
+    "@types/fluent-ffmpeg": "2.1.25",
+    "@types/html-to-text": "9.0.4",
+    "@types/jest": "29.5.12",
+    "@types/node": "22.2.0",
+    "concurrently": "8.2.2",
+    "eslint": "8.57.0",
+    "eslint-config-thugga": "workspace:*",
+    "jest": "29.7.0",
+    "ts-jest": "29.1.4",
+    "ts-node": "10.9.2",
+    "tsconfig": "workspace:*",
+    "tsx": "3.12.1",
+    "typescript": "5.2.2"
+  },
+  "repository": "[email protected]:meleksomai/website.git",
+  "publishConfig": {
+    "access": "restricted",
+    "registry": "https://npm.pkg.github.com/meleksomai/"
+  }
+}

apps/cli/src/commands/tts.ts

@@ -0,0 +1,106 @@
+import fs from "fs";
+import path from "path";
+
+import { input } from "@inquirer/prompts";
+import Markdoc from "@markdoc/markdoc";
+import chalk from "chalk";
+import Ffmpeg from "fluent-ffmpeg";
+import { convert } from "html-to-text";
+import OpenAI from "openai";
+
+import { chunkText } from "../utils/chunkText";
+
+const TEMP_FOLDER = path.resolve(process.cwd(), "temp");
+const TEMP_SUBFOLDER = new Date().getTime().toString();
+
+const tts = async () => {
+  try {
+    console.log(chalk.green("Generating text-to-speech..."));
+
+    const openai = new OpenAI();
+    const ffmpeg = Ffmpeg();
+
+    // 1. Parameters
+    const source = await input({
+      message: "Enter the source to generate the text-to-speech from",
+    });
+    const output = await input({
+      message: "Enter the destination of the generated audio file",
+    });
+
+    const outputFolder = path.resolve(process.cwd(), output);
+    const sourceFile = path.resolve(process.cwd(), source);
+    const tempFolder = path.join(TEMP_FOLDER, TEMP_SUBFOLDER);
+    const outputFileName = path.basename(sourceFile, path.extname(sourceFile));
+    const outputPath = path.join(outputFolder, `${outputFileName}.mp3`);
+
+    // Create the temporary folder
+    if (!fs.existsSync(tempFolder)) {
+      await fs.promises.mkdir(tempFolder, { recursive: true });
+    }
+
+    // If the sourceFile is test/markdown.md, the speechFile will be `markdown`.
+    const speechFile = path.basename(sourceFile, path.extname(sourceFile));
+
+    // 2. Retrieve the source file content as markdown
+    const file = await fs.promises.readFile(sourceFile, "utf-8");
+    const ast = Markdoc.parse(file);
+    const content = Markdoc.transform(ast /* config */);
+
+    const html = Markdoc.renderers.html(content);
+    const textInput = convert(html, {
+      selectors: [{ selector: "a", format: "inline" }],
+    });
+
+    // 3. Chunk the text.
+    // Note: OpenAI has a limit of 4096 tokens per request.
+    // We need to chunk the text into smaller parts.
+    const chunkSize = 4096;
+    const chunks = chunkText(textInput, chunkSize);
+
+    console.log(chalk.green("Text chunked into", chunks.length, "parts"));
+
+    // 4. Generate the text-to-speech for each chunk
+    const audioFiles: string[] = [];
+    for (let i = 0; i < chunks.length; i++) {
+      const chunk = chunks[i];
+
+      // The temp audio file for the chunk
+      const audioFilePath = path.join(tempFolder, `${speechFile}-${i}.mp3`);
+      audioFiles.push(audioFilePath);
+
+      console.log(chalk.yellow("Generating audio for chunk", i));
+      // 1. Generate the audio file for the chunk
+      const mp3 = await openai.audio.speech.create({
+        model: "tts-1",
+        voice: "alloy",
+        input: chunk,
+      });
+      const buffer = Buffer.from(await mp3.arrayBuffer());
+
+      console.log(chalk.gray("Saving audio file to", audioFilePath));
+      // 2. Save the audio file to the temp folder
+      await fs.promises.writeFile(audioFilePath, buffer);
+
+      // 3. Add the audio file to the ffmpeg input
+      ffmpeg.addInput(audioFilePath);
+
+      console.log(chalk.gray("Speech generated for chunk", i));
+    }
+
+    console.log(chalk.green("Concatenating audio files..."));
+    // 5. Concatenate the audio files
+    ffmpeg.mergeToFile(outputPath, tempFolder).on("end", () => {
+      console.log(chalk.green("Audio file generated at", outputPath));
+    });
+
+    // 6. Cleanup
+    console.log(chalk.green("Cleaning up..."));
+    // Remove the temp folder
+    // await fs.promises.rm(tempFolder, { recursive: true });
+  } catch (error) {
+    console.error(chalk.red("Error: ", error));
+  }
+};
+
+export default tts;

apps/cli/src/index.ts

@@ -0,0 +1,34 @@
+// import chalk from "chalk";
+import path from "path";
+
+import { Command } from "commander";
+import * as dotenv from "dotenv";
+import figlet from "figlet";
+
+import tts from "./commands/tts";
+
+const program = new Command();
+
+dotenv.config({
+  path: path.resolve(__dirname, "../.env"),
+});
+
+console.log(figlet.textSync("Melek Somai CLI"));
+
+program
+  .version("1.0.0")
+  .description("A CLI for managing administrative tasks.");
+
+// =================
+// Generate TTS
+// =================
+program
+  .command("tts", {
+    isDefault: false,
+  })
+  .description("generate text-to-speech using OpenAI's API.")
+  .action(async () => {
+    await tts();
+  });
+
+program.parse();

apps/cli/src/utils/chunkText.ts

@@ -0,0 +1,40 @@
+export function chunkText(text: string, chunkSize: number): string[] {
+  const chunks: string[] = [];
+
+  const paragraphs = text.split("\n\n");
+
+  let currentChunk = "";
+  for (const paragraph of paragraphs) {
+    if ((currentChunk + paragraph).length >= chunkSize) {
+      chunks.push(currentChunk);
+      currentChunk = "";
+
+      if (paragraph.length >= chunkSize) {
+        // Split the paragraph into words and add them to the chunks
+        const words = paragraph.split(" ");
+
+        for (const word of words) {
+          if ((currentChunk + word).length < chunkSize) {
+            currentChunk += (currentChunk ? " " : "") + word;
+          } else {
+            chunks.push(currentChunk);
+            currentChunk = word;
+          }
+        }
+      } else {
+        // The paragraph fits the chunk size. Add it to the chunks.
+        currentChunk = paragraph;
+      }
+    } else {
+      // Add the paragraph to the current chunk if it fits
+      // the chunk size. Otherwise, start a new chunk.
+      currentChunk += (currentChunk ? "\n\n" : "") + paragraph;
+    }
+  }
+  chunks.push(currentChunk);
+
+  // Filter out empty chunks
+  const filteredChunks = chunks.filter((chunk) => chunk !== "");
+
+  return filteredChunks;
+}

apps/cli/src/utils/test/chunkText.test.ts

@@ -0,0 +1,44 @@
+import { chunkText } from "../chunkText";
+
+describe("chunkText", () => {
+  it("should chunk simple text correctly", () => {
+    const text = "This is a simple text for testing.";
+    const chunkSize = 10;
+    const result = chunkText(text, chunkSize);
+    expect(result).toEqual(["This is a", "simple", "text for", "testing."]);
+  });
+
+  it("should chunk text with multiple paragraphs correctly", () => {
+    const text =
+      "This is the first paragraph.\n\nThis is the second paragraph.";
+    const chunkSize = 20;
+    const result = chunkText(text, chunkSize);
+    expect(result).toEqual([
+      "This is the first",
+      "paragraph.",
+      "This is the second",
+      "paragraph.",
+    ]);
+  });
+
+  it("should return the whole text if chunk size is larger than text length", () => {
+    const text = "Short text.";
+    const chunkSize = 50;
+    const result = chunkText(text, chunkSize);
+    expect(result).toEqual(["Short text."]);
+  });
+
+  it("should handle chunk size exactly the length of some words", () => {
+    const text = "Exact length test.";
+    const chunkSize = 5;
+    const result = chunkText(text, chunkSize);
+    expect(result).toEqual(["Exact", "length", "test."]);
+  });
+
+  it("should handle empty text", () => {
+    const text = "";
+    const chunkSize = 10;
+    const result = chunkText(text, chunkSize);
+    expect(result).toEqual([]);
+  });
+});

apps/cli/tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "extends": "tsconfig/library",
+  "compilerOptions": {
+    "outDir": "dist",
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+  },
+  "include": ["src"],
+  "exclude": ["dist", "build", "node_modules"]
+}

apps/somai.me/package.json

@@ -39,7 +39,7 @@
     "eslint": "8.57.0",
     "eslint-config-thugga": "workspace:*",
     "tsconfig": "workspace:*",
-    "typescript": "5.1.3",
+    "typescript": "5.2.2",
     "webpack": "5.93.0"
   },
   "private": true

packages/academic/package.json

@@ -11,19 +11,19 @@
   "dependencies": {
     "@citation-js/date": "0.5.1",
     "citation-js": "0.6.5",
-    "commander": "9.4.1",
+    "commander": "12.0.0",
     "dayjs": "1.11.12",
-    "figlet": "1.5.2",
+    "figlet": "1.7.0",
     "globby": "14.0.2",
     "gray-matter": "4.0.3"
   },
   "devDependencies": {
-    "@types/figlet": "1.5.5",
+    "@types/figlet": "1.5.8",
     "@types/node": "22.2.0",
     "eslint": "8.57.0",
     "eslint-config-thugga": "workspace:*",
     "tsconfig": "workspace:*",
     "tsx": "3.12.1",
-    "typescript": "5.1.3"
+    "typescript": "5.2.2"
   }
 }

packages/eslint-config/package.json

@@ -23,7 +23,7 @@
     "@types/node": "22.2.0",
     "eslint": "8.57.0",
     "prettier": "3.3.3",
-    "typescript": "5.1.3"
+    "typescript": "5.2.2"
   },
   "publishConfig": {
     "access": "public"

packages/jest/jest.config.js

@@ -0,0 +1,17 @@
+module.exports = {
+  testEnvironment: "node",
+  testMatch: ["**/*.test.ts"],
+  transform: {
+    "^.+\\.tsx?$": "ts-jest",
+  },
+  reporters: ["default"],
+  collectCoverageFrom: ["{bin,src}/**/*.{js,ts}"],
+  coverageThreshold: {
+    global: {
+      statements: 80,
+      branches: 80,
+      lines: 80,
+      functions: 80,
+    },
+  },
+};