JS Rendering & Browser Control

Learn how to use a render parameter how to define browser instructions in Web Scraper API so you can scrape complex dynamic pages.

JavaScript Rendering

If the page you want to scrape uses JavaScript to load its data into the DOM, add the render parameter to your request. The page is then fully rendered before we return the result, in one of two formats:

render value

You get

html

The raw HTML of the fully rendered page

png

A Base64-encoded screenshot (PNG) of the rendered page

If you want to scrape an image and download it, please refer to this section.

Request sample

curl --user "USERNAME:PASSWORD" \
'https://realtime.oxylabs.io/v1/queries' \
-H "Content-Type: application/json" \
-d '{"source": "universal", "url": "https://www.example.com", "render": "html"}'

import requests
from pprint import pprint

# Structure payload.
payload = {
    'source': 'universal',
    'url': 'https://www.example.com',
    'render': 'html',
}

# Get response.
response = requests.request(
    'POST',
    'https://realtime.oxylabs.io/v1/queries',
    auth=('USERNAME', 'PASSWORD'),
    json=payload,
)

# Instead of response with job status and results url, this will return the
# JSON response with the result.
pprint(response.json())

<?php

$params = [
    'source' => 'universal',
    'url' => 'https://www.example.com',
    'render' => 'html',
];

$ch = curl_init();

curl_setopt($ch, CURLOPT_URL, "https://realtime.oxylabs.io/v1/queries");
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($params));
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_USERPWD, "USERNAME" . ":" . "PASSWORD");

$headers = array();
$headers[] = "Content-Type: application/json";
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);

$result = curl_exec($ch);
echo $result;

if (curl_errno($ch)) {
    echo 'Error:' . curl_error($ch);
}
curl_close($ch);

using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Net.Http.Json;
using System.Threading.Tasks;

namespace OxyApi
{
    class Program
    {
        static async Task Main()
        {
            const string Username = "YOUR_USERNAME";
            const string Password = "YOUR_PASSWORD";

            var parameters = new Dictionary<string, string>()
            {
                { "source", "universal" },
                { "url", "https://www.example.com" },
                { "render" : "html" },
            };


            var client = new HttpClient();

            Uri baseUri = new Uri("https://realtime.oxylabs.io");
            client.BaseAddress = baseUri;

            var requestMessage = new HttpRequestMessage(HttpMethod.Post, "/v1/queries");
            requestMessage.Content = JsonContent.Create(parameters);

            var authenticationString = $"{Username}:{Password}";
            var base64EncodedAuthenticationString = Convert.ToBase64String(System.Text.ASCIIEncoding.UTF8.GetBytes(authenticationString));
            requestMessage.Headers.Add("Authorization", "Basic " + base64EncodedAuthenticationString);

            var response = await client.SendAsync(requestMessage);
            var contents = await response.Content.ReadAsStringAsync();

            Console.WriteLine(contents);
        }
    }
}

package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
)

func main() {
	const Username = "YOUR_USERNAME"
	const Password = "YOUR_PASSWORD"

	payload := map[string]string{
		"source": "universal",
		"url": "https://www.example.com",
        	"render": "html",
	}

	jsonValue, _ := json.Marshal(payload)

	client := &http.Client{}
	request, _ := http.NewRequest("POST",
		"https://realtime.oxylabs.io/v1/queries",
		bytes.NewBuffer(jsonValue),
	)

	request.SetBasicAuth(Username, Password)
	response, _ := client.Do(request)

	responseText, _ := ioutil.ReadAll(response.Body)
	fmt.Println(string(responseText))
}

package org.example;

import okhttp3.*;
import org.json.JSONObject;

public class Main implements Runnable {
    private static final String AUTHORIZATION_HEADER = "Authorization";
    public static final String USERNAME = "YOUR_USERNAME";
    public static final String PASSWORD = "YOUR_PASSWORD";

    public void run() {
        JSONObject jsonObject = new JSONObject();
        jsonObject.put("source", "universal");
        jsonObject.put("url", "https://www.example.com");
        jsonObject.put("render", "html");

        Authenticator authenticator = (route, response) -> {
            String credential = Credentials.basic(USERNAME, PASSWORD);

            return response
                    .request()
                    .newBuilder()
                    .header(AUTHORIZATION_HEADER, credential)
                    .build();
        };

        var client = new OkHttpClient.Builder()
                .authenticator(authenticator)
                .build();

        var mediaType = MediaType.parse("application/json; charset=utf-8");
        var body = RequestBody.create(jsonObject.toString(), mediaType);
        var request = new Request.Builder()
                .url("https://realtime.oxylabs.io/v1/queries")
                .post(body)
                .build();

        try (var response = client.newCall(request).execute()) {
            assert response.body() != null;
            System.out.println(response.body().string());
        } catch (Exception exception) {
            System.out.println("Error: " + exception.getMessage());
        }

        System.exit(0);
    }

    public static void main(String[] args) {
        new Thread(new Main()).start();
    }
}

import fetch from 'node-fetch';

const username = 'YOUR_USERNAME';
const password = 'YOUR_PASSWORD';
const body = {
  'source': 'universal',
  'url': 'https://www.example.com',
  'render': 'html'
};
const response = await fetch('https://realtime.oxylabs.io/v1/queries', {
  method: 'post',
  body: JSON.stringify(body),
  headers: {
    'Content-Type': 'application/json',
    'Authorization': 'Basic ' + Buffer.from(`${username}:${password}`).toString('base64'),
  }
});

console.log(await response.json());

{
    "source": "universal", 
    "url": "https://www.example.com", 
    "render": "html"
}

JavaScript rendering takes more time to scrape the page. Please set timeout on the client side to 180 seconds if using Realtime or Proxy Endpoint integration methods.

To ensure lowest traffic consumption, our system does not load unnecessary assets during page rendering.

Forcing rendering on specific pages

For successful scraping, some page types of specific domains require rendering due to their dynamic content. For these, our system automatically enforces rendering even if you don't set the render parameter, so you always get accurate, reliable data from these otherwise difficult pages.

Please note that rendered jobs consume more traffic compared to non-rendered jobs.

The full list of affected targets is maintained in the file below. We want our users to be fully aware of this when scraping the following pages:

forced-render-domains.csv

3KB

Open

If you wish to disable rendering, you can do so by adding the following parameter to your requests:

"render": ""

Browser instructions

When a page needs interaction, such as clicking a button, entering a search term, or scrolling to load more items, you can define your own browser_instructions that run while the page is rendered.

The easiest way to build browser instructions is the AI-powered visual builder in the Web Scraper API Playground. Read about it here.

Quick start

First of all, browser instructions require the render parameter, either html or png, and are provided as a list in the browser_instructions field. Each item in the list is executed in order.

Say you want to search for pizza boxes on a website — type the term into the search field, click the search button, and wait 5 seconds for the results to load:

{
    "source": "universal",
    "url": "https://www.etsy.com/",
    "render": "html",
    "browser_instructions": [
        {
            "type": "input",
            "value": "pizza boxes",
            "selector": {
                "type": "css",
                "value": "#global-enhancements-search-query"
            }
        },
        {
            "type": "click",
            "selector": {
                "type": "css",
                "value": "button[data-id='gnav-search-submit-button']"
            }
        },
        {
            "type": "wait",
            "wait_time_s": 5
        }
    ]
}

The result contains the HTML after the instructions have run:

{
  "results": [
    {
      "content": "<!DOCTYPE html><html lang=\"en-US\">  Search results for \"pizza boxes\"  </html>",
      "created_at": "2026-08-19 12:23:06",
      "updated_at": "2026-08-19 12:23:24",
      "page": 1,
      "url": "https://www.etsy.com/search?q=pizza+boxes&ref=search_bar&instant_download=false",
      "job_id": "7495817623108602881",
      "is_render_forced": false,
      "type": "raw",
      "status_code": 200
    }
  ]
}

Scraped HTML should look like this:

Fetching browser resources

We provide a standalone browser instruction for fetching browser resources.

The function is defined here:

Using fetch_resource will result in job returning the first occurrence of a Fetch/XHR resource that matches the format provided instead of the HTML that is being targeted.

Let’s say we want to target a GraphQL resource that is fetched when visiting a product page organically in the browser. We will provide job information as such:

{
    "source": "universal",
    "url": "https://www.example.com/product-page/123",
    "render": "html",
    "browser_instructions": [
        {
            "type": "fetch_resource",
            "filter": "/graphql/product-info/123"
        }
    ]
}

These instructions will result in a result as such:

{
  "results": [
    {
      "content": "{'product_id': 123, 'description': '', 'price': 123}",
      "created_at": "2023-10-11 11:35:23",
      "updated_at": "2023-10-11 11:36:08",
      "page": 1,
      "url": "https://example.com/v1/graphql/product-info/123/",
      "job_id": "7117835067442906114",
      "status_code": 200
    }
  ]
}

List of supported browser instructions

General arguments

All the instructions defined below have a consistent set of arguments. The arguments are as follows.

`type`

Type: Enum["click", "input", "scroll", "scroll_to_bottom", "wait", "wait_for_element", "fetch_resource"]
Description: Browser instruction type.
Required: true

`timeout_s`

Type: int
Description: How long until action is skipped if not completed in time.
Restrictions: 0 < timeout_s <= 60
Default value: 5

`wait_time_s`

Type: int
Description: How long to wait before executing next action.
Restrictions: 0 < wait_time_s <= 60
Default value: 0

`on_error`

Type: Enum["error", "skip"]
Description: Indicator what to do with instructions in case this instruction fails:
- "error": Stops the execution of browser instructions.
- "skip": Continues with the next instruction.
Default value: "error"

Example with general arguments

{
    "type": "wait_for_element",
    "selector": {
        "type": "text",
        "value": "Load More Items"
    },
    "timeout_s": 5,
    "wait_time_s": 2,
    "on_error": "skip"

}

Instructions

`click`

Description: Clicks an element and wait a set count of seconds.
Args:
- type: str = "click"
- selector: dict
  - type: Enum["xpath", "css", "text"]
  - value: str

Example:

{
    "type": "click",
    "selector": {
        "type": "xpath",
        "value": "//button"
    }
}

`input`

Description: Enters a text into a selected element.
Args:
- type: str = "input"
- selector: dict
  - type: Enum["xpath", "css", "text"]
  - value: str
- value: str

Example:

{
    "type": "input",
    "selector": {
        "type": "xpath",
        "value": "//input"
    },
    "value": "pizza boxes"
}

`scroll`

Description: Scrolls a set count of pixels.
Args:
- type: str = "scroll"
- x: int
- y: int

Example:

{
    "type": "scroll",
    "x": 0,
    "y": 100
}

`scroll_to_bottom`

Description: Scrolls to bottom for a set count of seconds.
Args:
- type: str = "scroll_to_bottom"

Example:

{
    "type": "scroll_to_bottom",
    "timeout_s": 10
}

`wait`

Description: Waits a set count of seconds.
Args:
- type: str = "wait"

Example:

{
    "type": "wait",
    "wait_time_s": 2
}

`wait_for_element`

Description: Waits for element to load for a set count of seconds.
Args:
- type: str = "wait_for_element"
- selector: dict
  - type: Enum["xpath", "css", "text"]
  - value: str

Example:

{
    "type": "wait_for_element",
    "selector": {
        "type": "text",
        "value": "Load More Items"
    },
    "timeout_s": 5
}

`fetch_resource`

The fetch_resource instruction must be the final instruction in the browser instructions list; any subsequent instructions will not be executed.

Description: Fetches the first occurrence of a Fetch/XHR resource matching the set pattern.
Args:
- type: str = "fetch_resource"
- filter: str(RegEx expression)
- on_error: Enum["error", "skip"]

Example:

{
    "type": "fetch_resource",
    "filter": "/graphql/item/"
}

Instruction validation

Any inconsistency in regards to instruction format will result in a 400 status code and a corresponding error message.

For example, payload as such:

{
    "source": "universal",
    "url": "https://www.example.com/",
    "render": "html",
    "browser_instructions": [
        {
            "type": "unsupported-wait",
            "wait_time_s": 5
        }
    ]
}

Will result in:

{    
    "errors": {
        "message": "Unsupported action type `unsupported-wait`, choose from 'click,fetch_resource,input,scroll,scroll_to_bottom,wait,wait_for_element'"
    }
}

Troubleshooting

Status codes

See our response codes outlined here. Status codes in regards to instructions validation are documented here.

Errors and warnings

If there’s an error or warning resulting from your browsing actions, you’ll find it in the outcome under the keys browser_instructions_error or browser_instructions_warnings. For instance, if you’ve sent the following browser instructions and the expected xpath isn’t located on the page, the result will include a warning.

browser_instructions:

[
    {
        "type": "input", 
        "selector": {
            "type": "xpath",
            "value": "//input[@type='search']"
        },
        "value": "oxylabs"
    }
]

Results:

{
  "results": [
    {
      "content": "<!doctype html><html>
        Content after executing the instructions      
      </html>",
      "created_at": "2023-10-11 11:35:23",
      "updated_at": "2023-10-11 11:36:08",
      "browser_instructions_warnings": [
        {
          "action_type": "click",
          "msg": "Unable to find selector type `xpath` with value `//input[@type=search]` on the page."
        },
      ],
      "page": 1,
      "url": "https://example.com",
      "job_id": "7117835067442906113",
      "status_code": 200
    }
  ]
}

Possible errors and warnings

Unexpected error happened while converting browser instructions to actions.

Unexpected error happened while executing {action.type} browser instructions.

Action {action.type} timed out.

Unable to find selector type {selector.type} with value {selector.value} on the page.

PreviousDomain and Locale

NextCustom Parser

Last updated 23 days ago

Was this helpful?