JavaScript rendering
If the page you wish to scrape requires loading JavaScript to dynamically load all required data into the DOM, instead of setting up and using a headless browser by yourself, you can include the
"X-Oxylabs-Render: html" header with your requests. All requests with this header included will be fully rendered, and all data will be stored in an HTML file or PNG screenshot (depending on the passed parameter).JavaScript rendering takes more time to scrape the page. When using JavaScript rendering, set client side timeout value to 180 seconds.
This parameter has two available values:
html(HTML of a rendered page)png(raw bytes that can be saved as PNG)
cURL
Python
PHP
C#
Golang
Java
Node.js
curl -k -v -x unblock.oxylabs.io:60000 \
-U "user:pass1" "https://ip.oxylabs.io" \
-H "X-Oxylabs-Render: html"
import requests
# Define proxy dict. Don't forget to put your real user and pass here as well.
proxies = {
'http': 'http://user:[email protected]:60000',
'https': 'http://user:[email protected]:60000',
}
headers = {
"X-Oxylabs-Render": "html"
}
response = requests.get(
'https://ip.oxylabs.io',
verify=False, # It is required to ignore certificate
proxies=proxies,
headers=headers,
)
# Print result page to stdout
print(response.text)
# Save returned HTML to result.html file
with open('result.html', 'w') as f:
f.write(response.text)
<?php
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, "https://ip.oxylabs.io");
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_PROXY, 'unblock.oxylabs.io:60000');
curl_setopt($ch, CURLOPT_PROXYUSERPWD, 'YOUR_USERNAME:YOUR_PASSWORD');
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
curl_setopt_array($ch, array(
CURLOPT_HTTPHEADER => array(
'X-Oxylabs-Render: html'
)
));
$result = curl_exec($ch);
echo $result;
if (curl_errno($ch)) {
echo 'Error:' . curl_error($ch);
}
curl_close($ch);
using System;
using System.Net;
using System.Net.Http;
using System.Threading.Tasks;
namespace OxyApi
{
class Program
{
static async Task Main(string[] args)
{
var webProxy = new WebProxy
{
Address = new Uri("http://unblock.oxylabs.io:60000"),
BypassProxyOnLocal = false,
UseDefaultCredentials = false,
Credentials = new NetworkCredential(
userName: "YOUR_USERNAME",
password: "YOUR_PASSWORD"
)
};
var httpClientHandler = new HttpClientHandler
{
Proxy = webProxy,
};
// We recommend accepting our certificate instead of allowing insecure (http) traffic
httpClientHandler.ClientCertificateOptions = ClientCertificateOption.Manual;
httpClientHandler.ServerCertificateCustomValidationCallback =
(httpRequestMessage, cert, cetChain, policyErrors) =>
{
return true;
};
var client = new HttpClient(handler: httpClientHandler, disposeHandler: true);
// Add custom cookies
client.DefaultRequestHeaders.Add("X-Oxylabs-Render", "html");
Uri baseUri = new Uri("https://ip.oxylabs.io");
client.BaseAddress = baseUri;
var requestMessage = new HttpRequestMessage(HttpMethod.Get, "");
var response = await client.SendAsync(requestMessage);
var contents = await response.Content.ReadAsStringAsync();
Console.WriteLine(contents);
}
}
}
package main
import (
"crypto/tls"
"fmt"
"io/ioutil"
"net/http"
"net/url"
)
func main() {
const Username = "YOUR_USERNAME"
const Password = "YOUR_PASSWORD"
proxyUrl, _ := url.Parse(
fmt.Sprintf(
"http://%s:%[email protected]:60000",
Username,
Password,
),
)
customTransport := &http.Transport{Proxy: http.ProxyURL(proxyUrl)}
// We recommend accepting our certificate instead of allowing insecure (http) traffic
customTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
client := &http.Client{Transport: customTransport}
request, _ := http.NewRequest("GET",
"https://ip.oxylabs.io",
nil,
)
// Add custom cookies
request.Header.Add("X-Oxylabs-Render", "html")
request.SetBasicAuth(Username, Password)
response, _ := client.Do(request)
responseText, _ := ioutil.ReadAll(response.Body)
fmt.Println(string(responseText))
}
package org.example;
import okhttp3.Authenticator;
import okhttp3.Credentials;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import javax.net.ssl.*;
import java.net.InetSocketAddress;
import java.net.Proxy;
public class Main implements Runnable {
private static final String AUTHORIZATION_HEADER = "Proxy-Authorization";
public static final String USERNAME = "YOUR_USERNAME";
public static final String PASSWORD = "YOUR_PASSWORD";
public void run() {
Authenticator authenticator = (route, response) -> {
String credential = Credentials.basic(USERNAME, PASSWORD);
return response
.request()
.newBuilder()
.header(AUTHORIZATION_HEADER, credential)
.build();
};
OkHttpClient.Builder builder = new OkHttpClient.Builder();
// We recommend accepting our certificate instead of allowing insecure (http) traffic
this.disableSSLCertificateChecking(builder);
Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress("unblock.oxylabs.io", 60000));
var client = builder
.proxy(proxy)
.proxyAuthenticator(authenticator)
.build();
var request = new Request.Builder()
.url("https://ip.oxylabs.io")
.addHeader("X-Oxylabs-Render", "html")
.get()
.build();
try (var response = client.newCall(request).execute()) {
assert response.body() != null;
System.out.println(response.body().string());
} catch (Exception exception) {
exception.printStackTrace();
System.exit(1);
}
System.exit(0);
}
private void disableSSLCertificateChecking(OkHttpClient.Builder builder) {
TrustManager[] trustManagers = new TrustManager[]{
new X509TrustManager() {
@Override
public java.security.cert.X509Certificate[] getAcceptedIssuers() {
return new java.security.cert.X509Certificate[]{};
}
@Override
public void checkServerTrusted(java.security.cert.X509Certificate[] x509Certificates, String authType) {
}
@Override
public void checkClientTrusted(java.security.cert.X509Certificate[] x509Certificates, String authType) {
}
}
};
try {
HttpsURLConnection.setDefaultHostnameVerifier((s, sslSession) -> true);
SSLContext sslContext = SSLContext.getInstance("TLS");
sslContext.init(null, trustManagers, new java.security.SecureRandom());
HttpsURLConnection.setDefaultSSLSocketFactory(sslContext.getSocketFactory());
builder.sslSocketFactory(sslContext.getSocketFactory(), (X509TrustManager) trustManagers[0]);
} catch (Exception exception) {
exception.printStackTrace();
System.exit(1);
}
builder.hostnameVerifier((hostname, session) -> true);
}
public static void main(String[] args) {
new Thread(new Main()).start();
}
}
import fetch from 'node-fetch';
import createHttpsProxyAgent from 'https-proxy-agent'
const username = 'YOUR_USERNAME';
const password = 'YOUR_PASSWORD';
const agent = createHttpsProxyAgent(
`http://${username}:${password}@unblock.oxylabs.io:60000`
);
// We recommend accepting our certificate instead of allowing insecure (http) traffic
process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = 0;
const headers = {
'X-Oxylabs-Render': 'html',
}
const response = await fetch('https://ip.oxylabs.io', {
method: 'get',
headers: headers,
agent: agent,
});
console.log(await response.text());
Scraping a website HTML
In this example, we will render the YouTube home page and scrape the page content. Normally Youtube homepage would look like this if Web Unblocker is used without Javascript rendering:
Youtube page example without JavaScript rendering
Adding the
"X-Oxylabs-Render: html" header, as provided in the examples below, will enable Javascript rendering and return an HTML of a rendered page:cURL
Python
PHP
C#
Golang
Java
Node.js
curl -k -v -x unblock.oxylabs.io:60000 \
-U "user:pass1" "https://youtube.com" \
-H "X-Oxylabs-Render: html"
import requests
# Define proxy dict. Don't forget to put your real user and pass here as well.
proxies = {
'http': 'http://user:[email protected]:60000',
'https': 'http://user:[email protected]:60000',
}
headers = {
"X-Oxylabs-Render": "html"
}
response = requests.get(
'https://youtube.com',
verify=False, # It is required to ignore certificate
proxies=proxies,
headers=headers,
)
# Print result page to stdout
print(response.text)
# Save returned HTML to result.html file
with open('result.html', 'w') as f:
f.write(response.text)
<?php
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, "https://youtube.com");
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_PROXY, 'unblock.oxylabs.io:60000');
curl_setopt($ch, CURLOPT_PROXYUSERPWD, 'YOUR_USERNAME:YOUR_PASSWORD');
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
curl_setopt_array($ch, [
CURLOPT_HTTPHEADER => [
'X-Oxylabs-Render: html'
]
]);
$result = curl_exec($ch);
echo $result;
if (curl_errno($ch)) {
echo 'Error:' . curl_error($ch);
}
curl_close($ch);
using System;
using System.Net;
using System.Net.Http;
using System.Threading.Tasks;
namespace OxyApi
{
class Program
{
static async Task Main(string[] args)
{
var webProxy = new WebProxy
{
Address = new Uri("http://unblock.oxylabs.io:60000"),
BypassProxyOnLocal = false,
UseDefaultCredentials = false,
Credentials = new NetworkCredential(
userName: "YOUR_USERNAME",
password: "YOUR_PASSWORD"
)
};
var httpClientHandler = new HttpClientHandler
{
Proxy = webProxy,
};
// We recommend accepting our certificate instead of allowing insecure (http) traffic
httpClientHandler.ClientCertificateOptions = ClientCertificateOption.Manual;
httpClientHandler.ServerCertificateCustomValidationCallback =
(httpRequestMessage, cert, cetChain, policyErrors) =>
{
return true;
};
var client = new HttpClient(handler: httpClientHandler, disposeHandler: true);
// Add custom header
client.DefaultRequestHeaders.Add("X-Oxylabs-Render", "html");
Uri baseUri = new Uri("https://youtube.com");
client.BaseAddress = baseUri;
var requestMessage = new HttpRequestMessage(HttpMethod.Get, "");
var response = await client.SendAsync(requestMessage);
var contents = await response.Content.ReadAsStringAsync();
Console.WriteLine(contents);
}
}
}
package main
import (
"crypto/tls"
"fmt"
"io/ioutil"
"net/http"
"net/url"
)
func main() {
const Username = "YOUR_USERNAME"
const Password = "YOUR_PASSWORD"
proxyUrl, _ := url.Parse(
fmt.Sprintf(
"http://%s:%[email protected]:60000",
Username,
Password,
),
)
customTransport := &http.Transport{Proxy: http.ProxyURL(proxyUrl)}
// We recommend accepting our certificate instead of allowing insecure (http) traffic
customTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
client := &http.Client{Transport: customTransport}
request, _ := http.NewRequest("GET",
"https://youtube.com",
nil,
)
// Add custom cookies
request.Header.Add("X-Oxylabs-Render", "html")
request.SetBasicAuth(Username, Password)
response, _ := client.Do(request)
responseText, _ := ioutil.ReadAll(response.Body)
fmt.Println(string(responseText))
}
package org.example;
import okhttp3.Authenticator;
import okhttp3.Credentials;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import javax.net.ssl.*;
import java.net.InetSocketAddress;
import java.net.Proxy;
public class Main implements Runnable {
private static final String AUTHORIZATION_HEADER = "Proxy-Authorization";
public static final String USERNAME = "YOUR_USERNAME";
public static final String PASSWORD = "YOUR_PASSWORD";
public void run() {
Authenticator authenticator = (route, response) -> {
String credential = Credentials.basic(USERNAME, PASSWORD);
return response
.request()
.newBuilder()
.header(AUTHORIZATION_HEADER, credential)
.build();
};
OkHttpClient.Builder builder = new OkHttpClient.Builder();
// We recommend accepting our certificate instead of allowing insecure (http) traffic
this.disableSSLCertificateChecking(builder);
Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress("unblock.oxylabs.io", 60000));
var client = builder
.proxy(proxy)
.proxyAuthenticator(authenticator)
.build();
var request = new Request.Builder()
.url("https://youtube.com")
.addHeader("X-Oxylabs-Render", "html")
.get()
.build();
try (var response = client.newCall(request).execute()) {
assert response.body() != null;
System.out.println(response.body().string());
} catch (Exception exception) {
exception.printStackTrace();
System.exit(1);
}
System.exit(0);
}
private void disableSSLCertificateChecking(OkHttpClient.Builder builder) {
TrustManager[] trustManagers = new TrustManager[]{
new X509TrustManager() {
@Override
public java.security.cert.X509Certificate[] getAcceptedIssuers() {
return new java.security.cert.X509Certificate[]{};
}
@Override
public void checkServerTrusted(java.security.cert.X509Certificate[] x509Certificates, String authType) {
}
@Override
public void checkClientTrusted(java.security.cert.X509Certificate[] x509Certificates, String authType) {
}
}
};
try {
HttpsURLConnection.setDefaultHostnameVerifier((s, sslSession) -> true);
SSLContext sslContext = SSLContext.getInstance("TLS");
sslContext.init(null, trustManagers, new java.security.SecureRandom());
HttpsURLConnection.setDefaultSSLSocketFactory(sslContext.getSocketFactory());
builder.sslSocketFactory(sslContext.getSocketFactory(), (X509TrustManager) trustManagers[0]);
} catch (Exception exception) {
exception.printStackTrace();
System.exit(1);
}
builder.hostnameVerifier((hostname, session) -> true);
}
public static void main(String[] args) {
new Thread(new Main()).start();
}
}
import fetch from 'node-fetch';
import createHttpsProxyAgent from 'https-proxy-agent'
const username = 'YOUR_USERNAME';
const password = 'YOUR_PASSWORD';
const agent = createHttpsProxyAgent(
`http://${username}:${password}@unblock.oxylabs.io:60000`
);
// We recommend accepting our certificate instead of allowing insecure (http) traffic
process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = 0;
const headers = {
'X-Oxylabs-Render': 'html',
}
const response = await fetch('https://youtube.com', {
method: 'get',
headers: headers,
agent: agent,
});
console.log(await response.text());
The HTML file opened in a browser should look like this:
Youtube page example with JavaScript rendering
Getting a screenshot of a fully rendered page
To get a screenshot in PNG format instead of page HTML, it is required to provide the
"X-Oxylabs-Render: png" header. cURL
Python
PHP
C#
Golang
Java
Node.js
curl -k -v -x unblock.oxylabs.io:60000 \
-U "user:pass1" "https://youtube.com" \
-H "X-Oxylabs-Render: png" >> rendered_page.png
import requests
# Define proxy dict. Don't forget to put your real user and pass here as well.
proxies = {
'http': 'http://user:[email protected]:60000',
'https': 'http://user:[email protected]:60000',
}
headers = {
"X-Oxylabs-Render": "png"
}
response = requests.get(
'https://youtube.com',
verify=False, # It is required to ignore certificate
proxies=proxies,
headers=headers,
)
# Save screeenshot as PNG file
with open("rendered_page.png", 'wb') as f:
f.write(response.content)
<?php
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://youtube.com');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_PROXY, 'unblock.oxylabs.io:60000');
curl_setopt($ch, CURLOPT_PROXYUSERPWD, 'YOUR_USERNAME:YOUR_PASSWORD');
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
curl_setopt_array($ch, [
CURLOPT_HTTPHEADER => [
'X-Oxylabs-Render: png'
]
]);
$result = curl_exec($ch);
if (curl_errno($ch)) {
echo 'Error:' . curl_error($ch);
}
curl_close ($ch);
if(file_exists('rendered_page.png')){
unlink('rendered_page.png');
}
$fp = fopen('rendered_page.png','wb');
fwrite($fp, $result);
fclose($fp);
using System;
using System.Net;
using System.Net.Http;
using System.Drawing;
using System.IO;
using System.Threading.Tasks;
namespace OxyApi
{
class Program
{
static async Task Main(string[] args)
{
var webProxy = new WebProxy
{
Address = new Uri("http://unblock.oxylabs.io:60000"),
BypassProxyOnLocal = false,
UseDefaultCredentials = false,
Credentials = new NetworkCredential(
userName: "YOUR_USERNAME",
password: "YOUR_PASSWORD"
)
};
var httpClientHandler = new HttpClientHandler
{
Proxy = webProxy,
};
// We recommend accepting our certificate instead of allowing insecure (http) traffic
httpClientHandler.ClientCertificateOptions = ClientCertificateOption.Manual;
httpClientHandler.ServerCertificateCustomValidationCallback =
(httpRequestMessage, cert, cetChain, policyErrors) =>
{
return true;
};
var client = new HttpClient(handler: httpClientHandler, disposeHandler: true);
// Add custom header
client.DefaultRequestHeaders.Add("X-Oxylabs-Render", "png");
Uri baseUri = new Uri("https://youtube.com");
client.BaseAddress = baseUri;
var requestMessage = new HttpRequestMessage(HttpMethod.Get, "");
var response = await client.SendAsync(requestMessage);
byte[] bytes = await response.Content.ReadAsByteArrayAsync();
using (Image image = Image.FromStream(new MemoryStream(bytes)))
{
image.Save("rendered_page.png");
}
}
}
}
package main
import (
"crypto/tls"
"fmt"
"io/ioutil"
"net/http"
"net/url"
)
func main() {
const Username = "YOUR_USERNAME"
const Password = "YOUR_PASSWORD"
proxyUrl, _ := url.Parse(
fmt.Sprintf(
"http://%s:%[email protected]:60000",
Username,
Password,
),
)
customTransport := &http.Transport{Proxy: http.ProxyURL(proxyUrl)}
// We recommend accepting our certificate instead of allowing insecure (http) traffic
customTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
client := &http.Client{Transport: customTransport}
request, _ := http.NewRequest("GET",
"https://youtube.com",
nil,
)
// Add custom cookies
request.Header.Add("X-Oxylabs-Render", "png")
request.SetBasicAuth(Username, Password)
response, _ := client.Do(request)
responseData, _ := ioutil.ReadAll(response.Body)
response.Body.Close()
ioutil.WriteFile("rendered_page.png", responseData, 0666)
log.Println("Image was saved")
}
package org.example;
import okhttp3.Authenticator;
import okhttp3.Credentials;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import javax.net.ssl.*;
import java.net.InetSocketAddress;
import java.net.Proxy;
import java.util.concurrent.TimeUnit;
import java.io.File;
import org.apache.commons.io.FileUtils;
public class App implements Runnable
{
private static final String AUTHORIZATION_HEADER = "Proxy-Authorization";
public static final String USERNAME = "YOUR_USERNAME";
public static final String PASSWORD = "YOUR_PASSWORD";
public void run() {
Authenticator authenticator = (route, response) -> {
String credential = Credentials.basic(USERNAME, PASSWORD);
return response
.request()
.newBuilder()
.header(AUTHORIZATION_HEADER, credential)
.build();
};
OkHttpClient.Builder builder = new OkHttpClient.Builder()
.readTimeout(180, TimeUnit.SECONDS)
.writeTimeout(180, TimeUnit.SECONDS);
// We recommend accepting our certificate instead of allowing insecure (http) traffic
this.disableSSLCertificateChecking(builder);
Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress("unblock.oxylabs.io", 60000));
OkHttpClient client = builder
.proxy(proxy)
.proxyAuthenticator(authenticator)
.build();
var request = new Request.Builder()