Summarize Your Audio Data - AssemblyAI Docs

In this guide, you’ll learn how to use LeMUR to summarize your audio data with key takeaways.

If you want a Quickstart, see Apply LLMs to audio files.

Before you startTo use LeMUR, you need an AssemblyAI account with a credit card set up.

Basic summary example

If you want to send a custom prompt to the LLM, you can use the LeMUR Task and apply the model to your transcribed audio files. To summarize the content in your audio data, define a summarization prompt and call transcript.lemur.task(). The underlying transcript is automatically used as additional context for the model.

import assemblyai as aai

aai.settings.api_key = "YOUR_API_KEY"

# Step 1: Transcribe an audio file.
# audio_file = "./local_file.mp3"
audio_file = "https://assembly.ai/sports_injuries.mp3"

transcriber = aai.Transcriber()
transcript = transcriber.transcribe(audio_file)

# Step 2: Define a summarization prompt.
prompt = "Provide a brief summary of the transcript."

# Step 3: Apply LeMUR.
result = transcript.lemur.task(
    prompt, final_model=aai.LemurModel.claude3_5_sonnet
)

print(result.response)

import { AssemblyAI } from 'assemblyai'

const client = new AssemblyAI({
  apiKey: 'YOUR_API_KEY'
})

const run = async () => {
  // Step 1: Transcribe an audio file.
  //const audioFile = './local_file.mp3'
  const audioFile =
    'https://assembly.ai/sports_injuries.mp3'
  const transcript = await client.transcripts.transcribe({ audio: audioFile })

  // Step 2: Define a summarization prompt.
  const prompt = 'Provide a brief summary of the transcript.'

  // Step 3: Apply LeMUR.
  const { response } = await client.lemur.task({
    transcript_ids: [transcript.id],
    prompt,
    final_model: 'anthropic/claude-3-5-sonnet'
  })

  console.log(response)
}

run()

package main

import (
    "context"
    "fmt"

    aai "github.com/AssemblyAI/assemblyai-go-sdk"
)

func main() {
    ctx := context.Background()

    client := aai.NewClient("YOUR_API_KEY")

    // Step 1: Transcribe an audio file. For local files see our Getting Started guides.
    audioURL := "https://assembly.ai/sports_injuries.mp3"
    transcript, _ := client.Transcripts.TranscribeFromURL(ctx, audioURL, nil)

    // Step 2: Define a summarization prompt.
    prompt := "Provide a brief summary of the transcript."

    // Step 3: Apply LeMUR.
    var params aai.LeMURTaskParams
    params.Prompt = aai.String(prompt)
    params.TranscriptIDs = []string{aai.ToString(transcript.ID)}
    params.FinalModel = "anthropic/claude-3-5-sonnet"

    result, _ := client.LeMUR.Task(ctx, params)

    fmt.Println(*result.Response)
}

import com.assemblyai.api.AssemblyAI;
import com.assemblyai.api.resources.transcripts.types.*;
import com.assemblyai.api.resources.lemur.requests.*;
import java.util.List;

public final class App {
    public static void main(String[] args) {

        AssemblyAI client = AssemblyAI.builder()
                .apiKey("YOUR_API_KEY")
                .build();

        // Step 1: Transcribe an audio file. For local files see our Getting Started guides.
        String audioUrl = "https://assembly.ai/sports_injuries.mp3";
        Transcript transcript = client.transcripts().transcribe(audioUrl);

        // Step 2: Define a summarization prompt.
        String prompt = "Provide a brief summary of the transcript.";

        // Step 3: Apply LeMUR.
        var params = LemurTaskParams.builder()
                .prompt(prompt)
                .transcriptIds(List.of(transcript.getId()))
                .finalModel(LemurModel.ANTHROPIC_CLAUDE3_5_SONNET)
                .build();

        var response = client.lemur().task(params);

        System.out.println(response.getResponse());
    }
}

using AssemblyAI;
using AssemblyAI.Lemur;
using AssemblyAI.Transcripts;

var client = new AssemblyAIClient("YOUR_API_KEY");

// Step 1: Transcribe an audio file. For local files see our Getting Started guides.
var transcript = await client.Transcripts.TranscribeAsync(new TranscriptParams
{
    AudioUrl = "https://assembly.ai/sports_injuries.mp3"
});

// Step 2: Define a summarization prompt.
const string prompt = "Provide a brief summary of the transcript.";

// Step 3: Apply LeMUR.
var lemurTaskParams = new LemurTaskParams
{
    Prompt = prompt,
    TranscriptIds = [transcript.Id],
    FinalModel = LemurModel.AnthropicClaude3_5_Sonnet
};

var response = await client.Lemur.TaskAsync(lemurTaskParams);

Console.WriteLine(response.Response);

require 'assemblyai'

client = AssemblyAI::Client.new(api_key: 'YOUR_API_KEY')

# Step 1: Transcribe an audio file. For local files see our Getting Started guides.
audio_url = 'https://assembly.ai/sports_injuries.mp3'
transcript = client.transcripts.transcribe(audio_url: audio_url)

# Step 2: Define a summarization prompt.
prompt = 'Provide a brief summary of the transcript.'

# Step 3: Apply LeMUR.
response = client.lemur.task(
  prompt: prompt,
  transcript_ids: [transcript.id],
  final_model: AssemblyAI::Lemur::LemurModel::ANTHROPIC_CLAUDE3_5_SONNET
)

Example output

The transcript describes several common sports injuries - runner's knee,
sprained ankle, meniscus tear, rotator cuff tear, and ACL tear. It provides
definitions, causes, and symptoms for each injury. The transcript seems to be
narrating sports footage and describing injuries as they occur to the athletes.
Overall, it provides an overview of these common sports injuries that can result
from overuse or sudden trauma during athletic activities

Summary with specialized endpoint

The LeMUR Summary function requires no prompt engineering and facilitates more deterministic and structured outputs. You can use it with transcript.lemur.summarize(). You can add additional context to provide information that is not explicitly referenced in the audio data, as well as specify an answer format. For this, use the optional parameters context and answer_format.

import assemblyai as aai

aai.settings.api_key = "YOUR_API_KEY"

audio_url = "https://assembly.ai/meeting.mp4"
transcript = aai.Transcriber().transcribe(audio_url)

result = transcript.lemur.summarize(
    final_model=aai.LemurModel.claude3_5_sonnet,
    context="A GitLab meeting to discuss logistics",
    answer_format="TLDR"
)

print(result.response)

import { AssemblyAI } from 'assemblyai'

const client = new AssemblyAI({
  apiKey: 'YOUR_API_KEY'
})

const audioUrl = 'https://assembly.ai/meeting.mp4'

const run = async () => {
  const transcript = await client.transcripts.transcribe({ audio: audioUrl })

  const { response } = await client.lemur.summary({
    transcript_ids: [transcript.id],
    final_model: 'anthropic/claude-3-5-sonnet',
    context: 'A GitLab meeting to discuss logistics',
    answer_format: 'TLDR'
  })

  console.log(response)
}

run()

import com.assemblyai.api.AssemblyAI;
import com.assemblyai.api.resources.lemur.requests.*;
import com.assemblyai.api.resources.lemur.types.*;
import com.assemblyai.api.resources.transcripts.types.*;
import java.util.List;

public final class App {
    public static void main(String[] args) {

        AssemblyAI client = AssemblyAI.builder()
                .apiKey("YOUR_API_KEY")
                .build();

        String audioUrl = "https://assembly.ai/meeting.mp4";

        Transcript transcript = client.transcripts().transcribe(audioUrl);

        var response = client.lemur().summary(LemurSummaryParams.builder()
                .transcriptIds(List.of(transcript.getId()))
                .finalModel(LemurModel.ANTHROPIC_CLAUDE3_5_SONNET)
                .context(LemurBaseParamsContext.of("A GitLab meeting to discuss logistic"))
                .answerFormat("TLDR")
                .build());

        System.out.println(response.getResponse());
    }
}

using AssemblyAI;
using AssemblyAI.Lemur;
using AssemblyAI.Transcripts;

var client = new AssemblyAIClient("YOUR_API_KEY");

var transcript = await client.Transcripts.TranscribeAsync(new TranscriptParams
{
    AudioUrl = "https://assembly.ai/meeting.mp4"
});

var response = await client.Lemur.SummaryAsync(new LemurSummaryParams
{
    TranscriptIds = [transcript.Id],
    FinalModel = LemurModel.AnthropicClaude3_5_Sonnet,
    Context = "A GitLab meeting to discuss logistic",
    AnswerFormat = "TLDR"
});

Console.WriteLine(response.Response);

require 'assemblyai'

client = AssemblyAI::Client.new(api_key: 'YOUR_API_KEY')

audio_url = 'https://assembly.ai/meeting.mp4'

transcript = client.transcripts.transcribe(audio_url: audio_url)

response = client.lemur.summary(
  transcript_ids: [transcript.id],
  final_model: AssemblyAI::Lemur::LemurModel::ANTHROPIC_CLAUDE3_5_SONNET,
  context: 'A GitLab meeting to discuss logistic',
  answer_format: 'TLDR'
)

puts response.response

Custom summary example (Advanced)

In this example, we’ll run a custom LeMUR task with an advanced prompt to create custom summaries: Cookbook: Custom summary with LeMUR Task

More summarization prompt examples

Try any of these prompts to get started:

Use case	Example prompt
Summaries	”Summarize key decisions and important points from the phone call transcript”
Summarize audio segments	”Summarize the key events of each chapter”

For more use cases and prompt examples, see LeMUR examples.

API reference

Improve the results

To improve the results, see the following resources:

Optimize your prompt with the prompt engineering guide.
To alter the outcome, see Change model and parameters.

​Basic summary example

​Example output

​Summary with specialized endpoint

​Custom summary example (Advanced)

​More summarization prompt examples

​API reference

​Improve the results

Basic summary example

Example output

Summary with specialized endpoint

Custom summary example (Advanced)

More summarization prompt examples

API reference

Improve the results