RAG pipeline

POST

crawl

RAG pipeline

curl --request POST \
  --url https://apix.pline.ai/v1/crawl \
  --header 'X-API-Key: <api-key>'

const options = {method: 'POST', headers: {'X-API-Key': '<api-key>'}};

fetch('https://apix.pline.ai/v1/crawl', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

package main

import (
	"fmt"
	"net/http"
	"io"
)

func main() {

	url := "https://apix.pline.ai/v1/crawl"

	req, _ := http.NewRequest("POST", url, nil)

	req.Header.Add("X-API-Key", "<api-key>")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

Turn a documentation site, help center, or blog archive into chunked, embeddable text for a RAG pipeline — without writing a crawler or an HTML parser.

Use case

Build or refresh a vector store from a docs site, knowledge base, or public wiki
Keep an internal assistant’s context current by re-running the crawl on a schedule
Skip boilerplate (nav, footers, ads) so embeddings aren’t diluted with layout noise

Step 1: Crawl the site into Markdown

Crawl discovers every linked page from a starting URL and scrapes each one — request markdown output so every page comes back ready to chunk, with navigation and footers already stripped (only_main_content default).

curl -X POST "https://apix.pline.ai/v1/crawl" \
  -H "X-API-Key: $API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "url": "https://docs.example.com",
    "limit": 1000,
    "output": { "markdown": true },
    "includePaths": ["^/docs/"]
  }'

import os
import httpx

response = httpx.post(
    "https://apix.pline.ai/v1/crawl",
    headers={"X-API-Key": os.environ["API_KEY"]},
    json={
        "url": "https://docs.example.com",
        "limit": 1000,
        "output": {"markdown": True},
        "includePaths": ["^/docs/"],
    },
    timeout=30.0,
)
response.raise_for_status()
job_id = response.json()["id"]
print(f"Crawl started: {job_id}")

const response = await fetch("https://apix.pline.ai/v1/crawl", {
  method: "POST",
  headers: {
    "X-API-Key": process.env.API_KEY,
    "Content-Type": "application/json",
  },
  body: JSON.stringify({
    url: "https://docs.example.com",
    limit: 1000,
    output: { markdown: true },
    includePaths: ["^/docs/"],
  }),
});

const { id } = await response.json();
console.log(`Crawl started: ${id}`);

Step 2: Poll until the crawl completes

Python

import os
import time

import httpx

# job_id comes from Step 1
while True:
    status = httpx.get(
        f"https://apix.pline.ai/v1/crawl/{job_id}",
        headers={"X-API-Key": os.environ["API_KEY"]},
    ).json()
    if status["status"] == "completed":
        break
    if status["status"] in ("failed", "cancelled"):
        raise RuntimeError(f"Crawl {status['status']}: {status.get('error')}")
    time.sleep(10)

pages = status["s3PresignedUrls"]

Step 3: Chunk each page for embedding

When the crawl completes, s3PresignedUrls has one download link per output format you requested. The markdown link is a ZIP archive with one .md file per crawled page, plus a manifest.json that maps each file back to its source URL. This script downloads it, splits every page into overlapping chunks, and writes them to documents.jsonl.

Python

import io
import json
import zipfile

import httpx


def chunk_markdown(text, max_chars=2000, overlap=200):
    """Split text into overlapping chunks (about 500 tokens at 2,000 characters)."""
    chunks, start = [], 0
    while start < len(text):
        end = min(start + max_chars, len(text))
        chunks.append(text[start:end])
        if end == len(text):
            break
        start = end - overlap
    return chunks


archive = httpx.get(status["s3PresignedUrls"]["markdown"]).content

documents = []
with zipfile.ZipFile(io.BytesIO(archive)) as zf:
    manifest = json.loads(zf.read("manifest.json"))
    for page in manifest["pages"]:
        text = zf.read(page["file"]).decode("utf-8")
        for i, chunk in enumerate(chunk_markdown(text)):
            documents.append({"source": page["url"], "chunk": i, "text": chunk})

with open("documents.jsonl", "w") as f:
    for doc in documents:
        f.write(json.dumps(doc) + "\n")

print(f"Wrote {len(documents)} chunks from {len(manifest['pages'])} pages")

Each line of documents.jsonl has the chunk text and the source URL. Pass the text to your embedding model, then store the vector with source and chunk in your vector store so answers can cite the page they came from.

Request highlights

Setting	Value	Why
`output.markdown`	`true`	Markdown chunks cleanly and tokenizes predictably for embeddings
`includePaths`	`["^/docs/"]`	A regex, not a glob: matches paths that start with `/docs/`, keeping the crawl on the knowledge base and off the marketing site
`limit`	`1000`	Caps job size and cost for predictable runs

Tips

Re-run the crawl on a schedule (daily or weekly) and re-embed only pages whose content hash changed, instead of rebuilding the whole index every time.
Use webhook on the crawl request instead of polling if your pipeline runs as a background worker — see Delivery.
Keep tag set per knowledge source so you can trace crawl cost and history per source in Request history.

RAG pipeline

curl --request POST \
  --url https://apix.pline.ai/v1/crawl \
  --header 'X-API-Key: <api-key>'

const options = {method: 'POST', headers: {'X-API-Key': '<api-key>'}};

fetch('https://apix.pline.ai/v1/crawl', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

package main

import (
	"fmt"
	"net/http"
	"io"
)

func main() {

	url := "https://apix.pline.ai/v1/crawl"

	req, _ := http.NewRequest("POST", url, nil)

	req.Header.Add("X-API-Key", "<api-key>")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

​Use case

​Step 1: Crawl the site into Markdown

​Step 2: Poll until the crawl completes

​Step 3: Chunk each page for embedding

​Request highlights

​Tips

​Related

Use case

Step 1: Crawl the site into Markdown

Step 2: Poll until the crawl completes

Step 3: Chunk each page for embedding

Request highlights

Tips

Related