Using the API

Before starting, make sure you have followed the instructions in the setup page.

Talking to the API

Generating Video
Generating Images

In this example, we’ll use the gen4_turbo model to generate a video from an image using the text prompt “Generate a video”. You’ll want to replace the promptImage with a URL of an image and a promptText with your own text prompt.

First, you’ll want to install the Runway SDK. You can do this with npm:

npm install --save @runwayml/sdk

In your code, you can now import the SDK and start making requests:

import RunwayML from '@runwayml/sdk';

const client = new RunwayML();

async function main() {
  // Create a new image-to-video task using the "gen4_turbo" model
  const imageToVideo = await client.imageToVideo.create({
    model: 'gen4_turbo',
    // Point this at your own image file
    promptImage: 'https://example.com/image.jpg',
    promptText: 'Generate a video',
    ratio: '1280:720',
    duration: 5,
  });

  const taskId = imageToVideo.id;

  // Poll the task until it's complete
  let task: Awaited<ReturnType<typeof client.tasks.retrieve>>;
  do {
    // Wait for ten seconds before polling
    await new Promise(resolve => setTimeout(resolve, 10000));

    task = await client.tasks.retrieve(taskId);
  } while (!['SUCCEEDED', 'FAILED'].includes(task.status));

  console.log('Task complete:', task);
}

main();

First, you’ll want to install the Runway SDK. You can do this with pip:

pip install runwayml

In your code, you can now import the SDK and start making requests:

import time
from runwayml import RunwayML

client = RunwayML()

# Create a new image-to-video task using the "gen4_turbo" model
task = client.image_to_video.create(
  model='gen4_turbo',
  # Point this at your own image file
  prompt_image='https://example.com/image.jpg',
  prompt_text='Generate a video',
  ratio='1280:720',
  duration=5,
)
task_id = task.id

# Poll the task until it's complete
time.sleep(10)  # Wait for ten seconds before polling
task = client.tasks.retrieve(task_id)
while task.status not in ['SUCCEEDED', 'FAILED']:
  time.sleep(10)  # Wait for ten seconds before polling
  task = client.tasks.retrieve(task_id)

print('Task complete:', task)

If you’re not ready to start writing code, you can test the API with cURL.

# Replace the example URL below with your own image URL
curl -X POST https://api.dev.runwayml.com/v1/image_to_video \
  -d '{
    "promptImage": "https://example.com/",
    "promptText": "string",
    "model": "gen4_turbo",
    "ratio": "1280:720",
    "duration": 5
  }' \
  -H "Content-Type: application/json" \
  -H "Authorization: Bearer $RUNWAYML_API_SECRET" \
  -H "X-Runway-Version: 2024-11-06"

This command will start an image-to-video task using the “gen4_turbo” model. You’ll see a JSON response with the task ID printed in your terminal, which you can use to fetch the task status.

Uploading base64 encoded images as data URIs

You can also upload base64 encoded images (as a data URI) instead of pointing to an external URL. This can be useful if you’re working with a local image file and want to avoid an extra network round trip to upload the image.

To do this, simply pass the base64 encoded image string as a data URI in the promptImage field instead of a URL. For more information about file types and size limits, see the Inputs page.

Node
Python

import fs from 'fs';
import path from 'path';
import RunwayML from '@runwayml/sdk';

const client = new RunwayML();

// Sample image path - replace with your image path
const imagePath = 'example.png';

// Read the image file
const imageBuffer = fs.readFileSync(imagePath);

// Convert to base64
const base64String = imageBuffer.toString('base64');

async function main() {
  // Create a new image-to-video task using the "gen4_turbo" model
  const imageToVideo = await client.imageToVideo.create({
    model: 'gen4_turbo',
    // Point this at your own image file
    promptImage: `data:image/${path.extname(imagePath).slice(1)};base64,${base64String}`,
    promptText: 'Generate a video',
    ratio: '1280:720',
    duration: 5,
  });

  const taskId = imageToVideo.id;

  // Poll the task until it's complete
  let task: Awaited<ReturnType<typeof client.tasks.retrieve>>;
  do {
    // Wait for ten seconds before polling
    await new Promise(resolve => setTimeout(resolve, 10000));

    task = await client.tasks.retrieve(taskId);
  } while (!['SUCCEEDED', 'FAILED'].includes(task.status));

  console.log('Task complete:', task);
}

main();

import time, base64
from runwayml import RunwayML

client = RunwayML()

image = './example.png'

# encode image to base64
with open(image, "rb") as f:
    base64_image = base64.b64encode(f.read()).decode("utf-8")

# Create a new image-to-video task using the "gen4_turbo" model
task = client.image_to_video.create(
  model='gen4_turbo',
  # Point this at your own image file
  prompt_image=f"data:image/png;base64,{base64_image}",
  prompt_text='Generate a video',
  ratio='1280:720',
  duration=5,
)
task_id = task.id

# Poll the task until it's complete
time.sleep(10)  # Wait for ten seconds before polling
task = client.tasks.retrieve(task_id)
while task.status not in ['SUCCEEDED', 'FAILED']:
  time.sleep(10)  # Wait for ten seconds before polling
  task = client.tasks.retrieve(task_id)

print('Task complete:', task)

In this example, we’ll use the gen4_image model to generate an image of the Eiffel Tower rendered in the style of the painting Starry Night. To do this, we’ll pass in reference images of the Eiffel Tower and Starry Night, and use the promptText to specify that we want the Eiffel Tower rendered in the style of Starry Night. Reference images can be referenced in the text prompt using at-mention syntax referencing the tag of the reference image.

import RunwayML from '@runwayml/sdk';

const client = new RunwayML();

async function main() {
  // Create a new image generation task using the "gen4_image" model
  let task = await client.textToImage.create({
    model: 'gen4_image',
    ratio: '1920:1080',
    promptText: '@EiffelTower painted in the style of @StarryNight',
    referenceImages: [
      {
        uri: 'https://upload.wikimedia.org/wikipedia/commons/8/85/Tour_Eiffel_Wikimedia_Commons_(cropped).jpg'
        tag: 'EiffelTower',
      },
      {
        uri: 'https://upload.wikimedia.org/wikipedia/commons/thumb/e/ea/Van_Gogh_-_Starry_Night_-_Google_Art_Project.jpg/1513px-Van_Gogh_-_Starry_Night_-_Google_Art_Project.jpg',
        tag: 'StarryNight',
      },
    ],
  });

  do {
    // Wait for 1 second before polling
    await new Promise(resolve => setTimeout(resolve, 1000));

    task = await client.tasks.retrieve(task.id);
  } while (!['SUCCEEDED', 'FAILED'].includes(task.status));

  console.log('Task complete:', task);
  console.log('Image URL:', task.output[0]);
}

main();

import time
from runwayml import RunwayML

client = RunwayML()

task = client.text_to_image.create(
  model='gen4_image',
  ratio='1920:1080',
  prompt_text='@EiffelTower painted in the style of @StarryNight',
  reference_images=[
    {
      'uri': 'https://upload.wikimedia.org/wikipedia/commons/8/85/Tour_Eiffel_Wikimedia_Commons_(cropped).jpg',
      'tag': 'EiffelTower',
    },
    {
      'uri': 'https://upload.wikimedia.org/wikipedia/commons/thumb/e/ea/Van_Gogh_-_Starry_Night_-_Google_Art_Project.jpg/1513px-Van_Gogh_-_Starry_Night_-_Google_Art_Project.jpg',
      'tag': 'StarryNight',
    },
  ],
)
task_id = task.id

# Poll the task until it's complete
time.sleep(1)  # Wait for a second before polling
task = client.tasks.retrieve(task_id)
while task.status not in ['SUCCEEDED', 'FAILED']:
  time.sleep(1)  # Wait for a second before polling
  task = client.tasks.retrieve(task_id)

print('Task complete:', task)
print('Image URL': task.output[0])

If you’re not ready to start writing code, you can test the API with cURL.

# Replace the example URL below with your own image URL
curl -X POST https://api.dev.runwayml.com/v1/text_to_image \
  -d '{
    "promptText": "@EiffelTower painted in the style of @StarryNight",
    "model": "gen4_image",
    "ratio": "1920:1080",
    "referenceImages": [
      {
        "uri": "https://upload.wikimedia.org/wikipedia/commons/8/85/Tour_Eiffel_Wikimedia_Commons_(cropped).jpg",
        "tag": "EiffelTower"
      },
      {
        "uri": "https://upload.wikimedia.org/wikipedia/commons/thumb/e/ea/Van_Gogh_-_Starry_Night_-_Google_Art_Project.jpg/1513px-Van_Gogh_-_Starry_Night_-_Google_Art_Project.jpg",
        "tag": "StarryNight"
      }
    ]
  }' \
  -H "Content-Type: application/json" \
  -H "Authorization: Bearer $RUNWAYML_API_SECRET" \
  -H "X-Runway-Version: 2024-11-06"

Uploading base64 encoded images as data URIs

To do this, simply pass the base64 encoded image string as a data URI in the uri of a referenceImage object instead of a URL. For more information about file types and size limits, see the Inputs page.

In this example, we’ll use the gen4_image model to generate an image of a bunny facing the camera. We provide a reference image of a rabbit that we use at-mention syntax to reference in the promptText. We also provide an untagged reference image that the model will use to style the output image with.

Node
Python

import fs from 'fs';
import path from 'path';
import RunwayML from '@runwayml/sdk';

const client = new RunwayML();

function getBase64Image(imagePath: string) {
  // Read the image file
  const imageBuffer = fs.readFileSync(imagePath);

  // Convert to base64
  const base64String = imageBuffer.toString('base64');

  return `data:image/${path.extname(imagePath).slice(1)};base64,${base64String}`;
}

async function main() {
  // Create a new text-to-image task using the "gen4_image" model
  const textToImage = await client.textToImage.create({
    model: 'gen4_image',
    promptText: '@bunny facing the camera',
    ratio: '1920:1080',
    referenceImages: [
      {
        uri: getBase64Image('rabbit.png'),
        tag: 'bunny',
      },
      {
        uri: getBase64Image('style.png'),
      },
    ],
  });

  const taskId = textToImage.id;

  // Poll the task until it's complete
  let task: Awaited<ReturnType<typeof client.tasks.retrieve>>;
  do {
    // Wait for ten seconds before polling
    await new Promise(resolve => setTimeout(resolve, 10000));

    task = await client.tasks.retrieve(taskId);
  } while (!['SUCCEEDED', 'FAILED'].includes(task.status));

  console.log('Task complete:', task);
}

main();

import time, base64
from runwayml import RunwayML

client = RunwayML()

def get_base64_image(image_path: str) -> str:
    with open(image_path, "rb") as f:
        base64_image = base64.b64encode(f.read()).decode("utf-8")
    return f"data:image/png;base64,{base64_image}"

# Create a new text-to-image task using the "gen4_image" model
task = client.text_to_image.create(
  model='gen4_image',
  ratio='1920:1080',
  prompt_text='@bunny facing the camera',
  reference_images=[
    {
      'uri': get_base64_image('rabbit.png'),
      'tag': 'bunny',
    },
    {
      'uri': get_base64_image('style.png'),
    },
  ],
)
task_id = task.id

# Poll the task until it's complete
time.sleep(10)  # Wait for ten seconds before polling
task = client.tasks.retrieve(task_id)
while task.status not in ['SUCCEEDED', 'FAILED']:
  time.sleep(10)  # Wait for ten seconds before polling
  task = client.tasks.retrieve(task_id)

print('Task complete:', task)