Fetch your collection items from the API

Paginate through every Webflow collection and write items to disk as NDJSON, one record per line, so large collections never accumulate in memory.

With your schema finalized and Studio running locally, fetch your collection items. The script below writes items to disk line-by-line as each page arrives. It doesn't accumulate the full collection in memory before saving. This works the same whether your collection has 500 items or 100,000.

Items are saved as NDJSON (newline-delimited JSON): one JSON object per line, no wrapping array. Your import scripts read and stream these files directly, which is how @sanity/import expects them.

// scripts/fetch-collection-items.js
import fs from 'fs'
import { createWriteStream } from 'fs'

const TOKEN = process.env.WEBFLOW_API_TOKEN
const BASE_URL = 'https://api.webflow.com/v2'
const HEADERS = { Authorization: `Bearer ${TOKEN}`, Accept: 'application/json' }

// Back off on 429 responses using the retry-after header
async function fetchWithBackoff(url, attempt = 0) {
  const res = await fetch(url, { headers: HEADERS })
  if (res.status === 429) {
    const wait = parseInt(res.headers.get('retry-after') ?? '10', 10) * 1000
    const delay = wait * Math.pow(2, attempt)
    console.log(`  Rate limited — waiting ${delay / 1000}s`)
    await new Promise(r => setTimeout(r, delay))
    return fetchWithBackoff(url, attempt + 1)
  }
  if (!res.ok) throw new Error(`${res.status} ${res.statusText} — ${url}`)
  return res.json()
}

// Fetch one collection and write items to NDJSON, one line per item
async function fetchCollectionToNDJSON(collectionId, outputPath) {
  const stream = createWriteStream(outputPath)
  let offset = 0
  const limit = 100
  let total = null
  let written = 0

  while (true) {
    const data = await fetchWithBackoff(
      `${BASE_URL}/collections/${collectionId}/items?limit=${limit}&offset=${offset}`
    )
    if (total === null) total = data.pagination.total

    for (const item of data.items) {
      // Skip drafts and archived items here — keeps output files clean
      if (item.isDraft || item.isArchived) continue
      stream.write(JSON.stringify(item) + '\n')
      written++
    }

    process.stdout.write(`\r  ${Math.min(offset + limit, total)} / ${total}`)
    offset += limit
    if (offset >= total) break
  }

  await new Promise(resolve => stream.end(resolve))
  process.stdout.write('\n')
  return written
}

async function run() {
  const { collections } = JSON.parse(
    fs.readFileSync('webflow/api/collections.json', 'utf8')
  )
  fs.mkdirSync('webflow/api/items', { recursive: true })

  for (const col of collections) {
    const outputPath = `webflow/api/items/${col.id}.ndjson`
    console.log(`Fetching: ${col.displayName}`)
    const count = await fetchCollectionToNDJSON(col.id, outputPath)
    console.log(`Saved ${count} items → ${outputPath}`)
  }
}

run().catch(err => { console.error(err.message); process.exit(1) })

Run it from the project root:

node --env-file=.env scripts/fetch-collection-items.js

Each line in the output file has this shape:

{"id":"64e7f4a8b0e1234567890abc","isDraft":false,"isArchived":false,"fieldData":{"name":"My Post Title","slug":"my-post-title","author":"64e7f4a8b0e1234567890def","tags":["64e7f4a8b0e111","64e7f4a8b0e222"],"body":"<p>Rich text as HTML...</p>"}}

fieldData contains all CMS field values using your collection's field slugs. Reference fields — single and multi — are Webflow item ID strings. That direct ID-to-ID mapping is what makes API-based migration cleaner than CSV exports.

Scale note: Enterprise Webflow plans support 20,000 to 100,000+ items per collection. The NDJSON streaming approach above handles any size. Memory usage stays flat because items are written to disk as each page arrives rather than accumulated first. The rate-limit backoff handles the roughly 1,000 API calls needed to fetch a 100k collection.

The bottleneck at this scale shifts to the import side. Sequential client.create() calls at 100k items would take hours. For collections over roughly 1,000 items, use @sanity/import with a pre-assigned ID strategy instead. See the import lesson for the streaming approach.

Next, you’ll upload all Webflow CDN assets to Sanity before creating any documents.

Mark lesson as complete

You have 1 uncompleted task in this lesson

0 of 1

Write the streaming fetcher