Build an AI Podcast Generator with Web Scraping, ElevenLabs, and Next.js

SCRAPER_API_KEY=your_scraper_key
ELEVENLABS_API_KEY=your_elevenlabs_key
GROQ_API_KEY=your_groq_key

npx create-next-app@latest podcast-engine
cd podcast-engine
pnpm add openai elevenlabs uuid

import { NextRequest } from "next/server";
import OpenAI from "openai";
import { ElevenLabsClient } from "elevenlabs";
import { v4 as uuidv4 } from "uuid";
import { writeFileSync } from "fs";
import path from "path";

async function scrapeUrl(url: string): Promise<string> {
  const res = await fetch(url);
  const html = await res.text();
  // Use your preferred scraping solution to extract clean text
  return html.replace(/<[^>]*>/g, "").trim();
}

const openai = new OpenAI({
  apiKey: process.env.GROQ_API_KEY!,
  baseURL: "https://api.groq.com/openai/v1",
});

const elevenlabs = new ElevenLabsClient({
  apiKey: process.env.ELEVENLABS_API_KEY!,
});

async function createAudioFromText(text: string): Promise<string> {
  // Limit characters for free tier (adjust as needed)
  const truncatedText = text.substring(0, 800);

  const audio = await elevenlabs.generate({
    voice: "Rachel",
    text: truncatedText,
    model_id: "eleven_monolingual_v1",
  });

  const fileName = `podcast-${Date.now()}.mp3`;
  const filePath = path.join(process.cwd(), "public", "podcasts", fileName);

  const chunks: Buffer[] = [];
  for await (const chunk of audio) {
    chunks.push(Buffer.from(chunk));
  }
  writeFileSync(filePath, Buffer.concat(chunks));

  return `/podcasts/${fileName}`;
}

export async function POST(req: NextRequest) {
  const { urls } = await req.json();

  const stream = new ReadableStream({
    async start(controller) {
      const send = (type: string, data: any) => {
        controller.enqueue(
          new TextEncoder().encode(
            `data: ${JSON.stringify({ type, data })}\n\n`
          )
        );
      };

      try {
        // Step 1: Scrape all URLs in parallel
        send("status", "Scraping websites...");

        const scrapeResults = await Promise.all(
          urls.map(async (url: string) => {
            const markdown = await scrapeUrl(url);
            return markdown;
          })
        );

        const combinedContent = scrapeResults.join("\n\n---\n\n");

        if (!combinedContent.trim()) {
          send("error", "No content could be scraped from the provided URLs.");
          controller.close();
          return;
        }

        // Step 2: Generate podcast script with LLM
        send("status", "Compiling stories...");

        const today = new Date().toLocaleDateString("en-US", {
          weekday: "long",
          year: "numeric",
          month: "long",
          day: "numeric",
        });

        const completion = await openai.chat.completions.create({
          model: "llama-3.2-90b-vision-preview",
          messages: [
            {
              role: "system",
              content:
                "You are a witty tech podcaster. Create a 5-minute script covering the top 5-10 most interesting stories. Summarize each story in 1-4 sentences, keeping the tone funny and entertaining. Aim for a mix of humor.",
            },
            {
              role: "user",
              content: `Create a hilarious and informative 5-minute podcast for ${today}. Here is the source material:\n\n${combinedContent}`,
            },
          ],
          stream: true,
        });

        send("status", "Crafting commentary...");

        let fullScript = "";
        for await (const chunk of completion) {
          const text = chunk.choices[0]?.delta?.content || "";
          fullScript += text;
          send("script", text);
        }

        // Step 3: Generate audio from the script
        send("status", "Generating audio...");
        const audioPath = await createAudioFromText(fullScript);

        send("complete", { audioPath });
      } catch (error) {
        send("error", String(error));
      }

      controller.close();
    },
  });

  return new Response(stream, {
    headers: {
      "Content-Type": "text/event-stream",
      "Cache-Control": "no-cache",
      Connection: "keep-alive",
    },
  });
}

"use client";
import { useState, useRef } from "react";

interface StepStatus {
  label: string;
  complete: boolean;
}

export default function PodcastEngine() {
  const [urls, setUrls] = useState<string[]>([]);
  const [newUrl, setNewUrl] = useState("");
  const [loading, setLoading] = useState(false);
  const [script, setScript] = useState("");
  const [audioSrc, setAudioSrc] = useState("");
  const [statuses, setStatuses] = useState<StepStatus[]>([]);
  const audioRef = useRef<HTMLAudioElement>(null);

  function isValidUrl(str: string): boolean {
    try {
      new URL(str);
      return true;
    } catch {
      return false;
    }
  }

  function addUrl() {
    if (isValidUrl(newUrl) && !urls.includes(newUrl)) {
      setUrls((prev) => [...prev, newUrl]);
      setNewUrl("");
    }
  }

  function removeUrl(url: string) {
    setUrls((prev) => prev.filter((u) => u !== url));
  }

  async function generate() {
    setLoading(true);
    setScript("");
    setAudioSrc("");
    setStatuses([]);

    const response = await fetch("/api/generate-podcast", {
      method: "POST",
      headers: { "Content-Type": "application/json" },
      body: JSON.stringify({ urls }),
    });

    const reader = response.body?.getReader();
    const decoder = new TextDecoder();

    while (reader) {
      const { done, value } = await reader.read();
      if (done) break;

      const text = decoder.decode(value);
      const lines = text.split("\n").filter((l) => l.startsWith("data: "));

      for (const line of lines) {
        const { type, data } = JSON.parse(line.slice(6));

        if (type === "status") {
          setStatuses((prev) => [
            ...prev.map((s) => ({ ...s, complete: true })),
            { label: data, complete: false },
          ]);
        } else if (type === "script") {
          setScript((prev) => prev + data);
        } else if (type === "complete") {
          setAudioSrc(data.audioPath);
          setStatuses((prev) =>
            prev.map((s) => ({ ...s, complete: true }))
          );
        }
      }
    }

    setLoading(false);
  }

  return (
    <div className="max-w-4xl mx-auto p-8">
      <h1 className="text-3xl font-bold mb-6">Podcast Engine</h1>

      <div className="flex gap-2 mb-4">
        <input
          value={newUrl}
          onChange={(e) => setNewUrl(e.target.value)}
          placeholder="https://example.com/article"
          className="flex-1 border rounded px-3 py-2"
          onKeyDown={(e) => e.key === "Enter" && addUrl()}
        />
        <button onClick={addUrl} className="px-4 py-2 bg-black text-white rounded">
          Add
        </button>
      </div>

      {urls.map((url) => (
        <div key={url} className="flex items-center gap-2 mb-2">
          <span className="text-sm truncate flex-1">{url}</span>
          <button onClick={() => removeUrl(url)} className="text-red-500">
            Remove
          </button>
        </div>
      ))}

      <button
        onClick={generate}
        disabled={loading || urls.length === 0}
        className="mt-4 px-6 py-3 bg-black text-white rounded disabled:opacity-50"
      >
        {loading ? "Generating..." : "Generate Podcast"}
      </button>

      {statuses.length > 0 && (
        <div className="mt-6 space-y-2">
          {statuses.map((s, i) => (
            <div key={i} className="flex items-center gap-2">
              <span>{s.complete ? "Done" : "Working..."}</span>
              <span>{s.label}</span>
            </div>
          ))}
        </div>
      )}

      {script && (
        <pre className="mt-6 p-4 bg-gray-50 rounded whitespace-pre-wrap text-sm">
          {script}
        </pre>
      )}

      {audioSrc && (
        <audio ref={audioRef} controls src={audioSrc} className="mt-6 w-full" />
      )}
    </div>
  );
}