Ask questions about your audio data

If you want a Quickstart, see Apply LLMs to audio files.

To use LeMUR, you need an with a credit card set up.

Basic Q&A example

If you want to send a custom prompt to the LLM, you can use the LeMUR Task and apply the model to your transcribed audio files. To ask question about your audio data, define a prompt with your questions and call transcript.lemur.task(). The underlying transcript is automatically used as additional context for the model.

import assemblyai as aai

aai.settings.api_key = "YOUR_API_KEY"

# Step 1: Transcribe an audio file.
# audio_file = "./local_file.mp3"
audio_file = "https://assembly.ai/sports_injuries.mp3"

transcriber = aai.Transcriber()
transcript = transcriber.transcribe(audio_file)

# Step 2: Define a prompt with your question(s).
prompt = "What is a runner's knee?"

# Step 3: Apply LeMUR.
result = transcript.lemur.task(
    prompt, final_model=aai.LemurModel.claude3_5_sonnet
)

print(result.response)

import { AssemblyAI } from 'assemblyai'

const client = new AssemblyAI({
  apiKey: 'YOUR_API_KEY'
})

const run = async () => {
  // Step 1: Transcribe an audio file.
  //const audioFile = './local_file.mp3'
  const audioFile =
    'https://assembly.ai/sports_injuries.mp3'
  const transcript = await client.transcripts.transcribe({ audio: audioFile })

  // Step 2: Define a prompt with your question(s).
  const prompt = "What is a runner's knee?"

  // Step 3: Apply LeMUR.
  const { response } = await client.lemur.task({
    transcript_ids: [transcript.id],
    prompt,
    final_model: 'anthropic/claude-3-5-sonnet'
  })

  console.log(response)
}

run()

package main

import (
    "context"
    "fmt"

    aai "github.com/AssemblyAI/assemblyai-go-sdk"
)

func main() {
    ctx := context.Background()

    client := aai.NewClient("YOUR_API_KEY")

    // Step 1: Transcribe an audio file. For local files see our Getting Started guides.
    audioURL := "https://assembly.ai/sports_injuries.mp3"
    transcript, _ := client.Transcripts.TranscribeFromURL(ctx, audioURL, nil)

    // Step 2: Define a prompt with your question.
    prompt := "What is a runner's knee?"

    // Step 3: Apply LeMUR.
    var params aai.LeMURTaskParams
    params.Prompt = aai.String(prompt)
    params.TranscriptIDs = []string{aai.ToString(transcript.ID)}
    params.FinalModel = "anthropic/claude-3-5-sonnet"

    result, _ := client.LeMUR.Task(ctx, params)

    fmt.Println(*result.Response)
}

import com.assemblyai.api.AssemblyAI;
import com.assemblyai.api.resources.transcripts.types.*;
import com.assemblyai.api.resources.lemur.requests.*;
import java.util.List;

public final class App {
    public static void main(String[] args) {

        AssemblyAI client = AssemblyAI.builder()
                .apiKey("YOUR_API_KEY")
                .build();

        // Step 1: Transcribe an audio file. For local files see our Getting Started guides.
        String audioUrl = "https://assembly.ai/sports_injuries.mp3";
        Transcript transcript = client.transcripts().transcribe(audioUrl);

        // Step 2: Define a prompt with your question(s).
        String prompt = "What is a runner's knee?";

        // Step 3: Apply LeMUR.
        var params = LemurTaskParams.builder()
                .prompt(prompt)
                .transcriptIds(List.of(transcript.getId()))
                .finalModel(LemurModel.ANTHROPIC_CLAUDE3_5_SONNET)
                .build();

        var response = client.lemur().task(params);

        System.out.println(response.getResponse());
    }
}

using AssemblyAI;
using AssemblyAI.Lemur;
using AssemblyAI.Transcripts;

var client = new AssemblyAIClient("YOUR_API_KEY");

// Step 1: Transcribe an audio file. For local files see our Getting Started guides.
var transcript = await client.Transcripts.TranscribeAsync(new TranscriptParams
{
    AudioUrl = "https://assembly.ai/sports_injuries.mp3"
});

// Step 2: Define a prompt with your question(s).
const string prompt = "What is a runner's knee?";

// Step 3: Apply LeMUR.
var lemurTaskParams = new LemurTaskParams
{
    Prompt = prompt,
    TranscriptIds = [transcript.Id],
    FinalModel = LemurModel.AnthropicClaude3_5_Sonnet
};

var response = await client.Lemur.TaskAsync(lemurTaskParams);

Console.WriteLine(response.Response);

require 'assemblyai'

client = AssemblyAI::Client.new(api_key: 'YOUR_API_KEY')

# Step 1: Transcribe an audio file. For local files see our Getting Started guides.
audio_url = 'https://assembly.ai/sports_injuries.mp3'
transcript = client.transcripts.transcribe(audio_url: audio_url)

# Step 2: Define a prompt with your question(s).
prompt = "What is a runner's knee?"

# Step 3: Apply LeMUR.
response = client.lemur.task(
  prompt: prompt,
  transcript_ids: [transcript.id],
  final_model: AssemblyAI::Lemur::LemurModel::ANTHROPIC_CLAUDE3_5_SONNET
)

puts response.response

Example output

Based on the transcript, runner's knee is a condition characterizedby pain behind or around the kneecap. It is caused by overuse,muscle imbalance and inadequate stretching. Symptoms include painunder or around the kneecap and pain when walking.

Q&A with specialized endpoint

The LeMUR Question & Answer function requires no prompt engineering and facilitates more deterministic and structured outputs. You can use it with transcript.lemur.question(). To use it, define a list of aai.LemurQuestion objects. For each question, you can define additional context and specify either a answer_format or a list of answer_options. Additionally, you can define an overall context.

import assemblyai as aai

aai.settings.api_key = "YOUR_API_KEY"

audio_url = "https://assembly.ai/meeting.mp4"
transcript = aai.Transcriber().transcribe(audio_url)

questions = [
    aai.LemurQuestion(
        question="What are the top level KPIs for engineering?",
        context="KPI stands for key performance indicator",
        answer_format="short sentence"),
    aai.LemurQuestion(
        question="How many days has it been since the data team has gotten updated metrics?",
        answer_options=["1", "2", "3", "4", "5", "6", "7", "more than 7"]),
]

result = transcript.lemur.question(
    final_model=aai.LemurModel.claude3_5_sonnet,
    questions,
    context="A GitLab meeting to discuss logistics"
)

for qa_response in result.response:
    print(f"Question: {qa_response.question}")
    print(f"Answer: {qa_response.answer}")

import { AssemblyAI } from 'assemblyai'

const client = new AssemblyAI({
  apiKey: 'YOUR_API_KEY'
})

const audioUrl = 'https://assembly.ai/meeting.mp4'

const run = async () => {
  const transcript = await client.transcripts.transcribe({ audio: audioUrl })

  const questions = [
    {
      question: 'What are the top level KPIs for engineering?',
      context: 'KPI stands for key performance indicator',
      answer_format: 'short sentence'
    },
    {
      question:
        'How many days has it been since the data team has gotten updated metrics?',
      answer_options: ['1', '2', '3', '4', '5', '6', '7', 'more than 7']
    }
  ]

  const { response: qas } = await client.lemur.questionAnswer({
    transcript_ids: [transcript.id],
    final_model: 'anthropic/claude-3-5-sonnet',
    context: 'A GitLab meeting to discuss logistics',
    questions: questions
  })

  for (const { question, answer } of qas) {
    console.log('Question', question)
    console.log('Answer', answer)
  }
}

run()

import com.assemblyai.api.AssemblyAI;
import com.assemblyai.api.resources.lemur.requests.*;
import com.assemblyai.api.resources.lemur.types.*;
import com.assemblyai.api.resources.transcripts.types.*;
import java.util.List;

public final class App {
    public static void main(String[] args) {

        AssemblyAI client = AssemblyAI.builder()
                .apiKey("YOUR_API_KEY")
                .build();

        String audioUrl = "https://assembly.ai/meeting.mp4";

        Transcript transcript = client.transcripts().transcribe(audioUrl);

        var question1 = LemurQuestion.builder()
                .question("What are the top level KPIs for engineering?")
                .context(LemurQuestionContext.of("KPI stands for key performance indicator"))
                .answerFormat("short sentence").build();

        var question2 = LemurQuestion.builder()
                .question("How many days has it been since the data team has gotten updated metrics?")
                .answerOptions(List.of("1", "2", "3", "4", "5", "6", "7", "more than 7")).build();

        var response = client.lemur().questionAnswer(LemurQuestionAnswerParams.builder()
                .transcriptIds(List.of(transcript.getId()))
                .finalModel(LemurModel.ANTHROPIC_CLAUDE3_5_SONNET)
                .context(LemurBaseParamsContext.of("A GitLab meeting to discuss logistic"))
                .questions(List.of(question1, question2))
                .build());

        for (var qa : response.getResponse()) {
            System.out.println("Question: " + qa.getQuestion());
            System.out.println("Answer: " + qa.getAnswer());
        }
    }
}

using AssemblyAI;
using AssemblyAI.Lemur;
using AssemblyAI.Transcripts;

var client = new AssemblyAIClient("YOUR_API_KEY");

var transcript = await client.Transcripts.TranscribeAsync(new TranscriptParams
{
    AudioUrl = "https://assembly.ai/meeting.mp4"
});

var lemurTaskParams = new LemurQuestionAnswerParams
{
    TranscriptIds = [transcript.Id],
    FinalModel = LemurModel.AnthropicClaude3_5_Sonnet,
    Context = "A GitLab meeting to discuss logistic",
    Questions =
    [
        new LemurQuestion
        {
            Question = "What are the top level KPIs for engineering?",
            Context = "KPI stands for key performance indicator",
            AnswerFormat = "short sentence"
        },
        new LemurQuestion
        {
            Question = "How many days has it been since the data team has gotten updated metrics?",
            Context = "KPI stands for key performance indicator",
            AnswerOptions = ["1", "2", "3", "4", "5", "6", "7", "more than 7"]
        }
    ]
};

var response = await client.Lemur.QuestionAnswerAsync(lemurTaskParams);

foreach (var qa in response.Response)
{
    Console.WriteLine($"Question: {qa.Question}");
    Console.WriteLine($"Answer: {qa.Answer}");
}

require 'assemblyai'

client = AssemblyAI::Client.new(api_key: 'YOUR_API_KEY')

audio_url = 'https://assembly.ai/meeting.mp4'

transcript = client.transcripts.transcribe(audio_url: audio_url)

response = client.lemur.question_answer(
  transcript_ids: [transcript.id],
  final_model: AssemblyAI::Lemur::LemurModel::ANTHROPIC_CLAUDE3_5_SONNET,
  context: 'A GitLab meeting to discuss logistic',
  questions: [
    {
      question: 'What are the top level KPIs for engineering?',
      context: 'KPI stands for key performance indicator',
      answer_format: 'short sentence'
    },
    {
      question: 'How many days has it been since the data team has gotten updated metrics?',
      context: 'KPI stands for key performance indicator',
      answer_options: ['1', '2', '3', '4', '5', '6', '7', 'more than 7']
    }
  ]
)

response.response.each do |qa|
  printf("Question: %<question>s\n", question: qa.question)
  printf("Answer: %<answer>s\n", answer: qa.answer)
end

Custom Q&A example (Advanced)

This example shows how you can run a custom LeMUR task with an advanced prompt to create custom Q&A responses: Cookbook: Custom Q&A with LeMUR Task

More Q&A prompt examples

Try any of these prompts to get started:

Use case	Example prompt
Question and answer	”Identify any patterns or trends based on the transcript”
Closed-ended questions	”Did the customer express a positive sentiment in the phone call?”
Sentiment analysis	”What was the emotional sentiment of the phone call?”

For more use cases and prompt examples, see LeMUR examples.

API reference

Improve the results

To improve the results, see the following resources:

Optimize your prompt with the prompt engineering guide.
To alter the outcome, see Change model and parameters.

​Basic Q&A example

​Example output

​Q&A with specialized endpoint

​Custom Q&A example (Advanced)

​More Q&A prompt examples

​API reference

​Improve the results

Basic Q&A example

Example output

Q&A with specialized endpoint

Custom Q&A example (Advanced)

More Q&A prompt examples

API reference

Improve the results