renderização de JavaScript

Aprenda sobre o cabeçalho que você pode incluir nas suas solicitações para obter dados totalmente renderizados, que você pode salvar em um arquivo HTML ou como uma captura de tela PNG.

Se a página que você deseja extrair exigir o carregamento de JavaScript para carregar dinamicamente todos os dados necessários no DOM, em vez de configurar e usar um headless browser por conta própria, você pode incluir o "X-Oxylabs-Render: html" header em suas requisições. Todas as requisições com esse header incluído serão totalmente renderizadas, e todos os dados serão armazenados em um arquivo HTML ou em uma captura de tela PNG (dependendo do parâmetro passado).

A renderização de JavaScript leva mais tempo para extrair a página. Ao usar a renderização de JavaScript, defina o valor de timeout do lado do cliente para 180 segundos.

Para garantir o menor consumo de tráfego possível, nosso sistema não carrega ativos desnecessários durante a renderização da página.

Este parâmetro tem dois valores disponíveis:

html (HTML de uma página renderizada)
png (bytes brutos que podem ser salvos como PNG)

Exemplos de código

curl -k -v -x https://unblock.oxylabs.io:60000 \
-U 'USERNAME:PASSWORD' \
'https://ip.oxylabs.io/location' \
-H 'X-Oxylabs-Render: html'

import requests

# Use suas credenciais do Web Unblocker aqui.
USERNAME, PASSWORD = 'YOUR_USERNAME', 'YOUR_PASSWORD'

# Defina o dicionário de proxy.
proxies = {
  'http': f'http://{USERNAME}:{PASSWORD}@unblock.oxylabs.io:60000',
  'https': f'https://{USERNAME}:{PASSWORD}@unblock.oxylabs.io:60000',
}

headers = {
    'X-Oxylabs-Render': 'html'
}

response = requests.get(
    'https://ip.oxylabs.io/location',
    verify=False,  # É necessário ignorar o certificado
    proxies=proxies,
    headers=headers,
)

# Imprima a página de resultado em stdout
print(response.text)

# Salve o HTML retornado no arquivo result.html
with open('result.html', 'w') as f:
    f.write(response.text)

import fetch from 'node-fetch';
import { HttpsProxyAgent } from 'https-proxy-agent';

const username = 'YOUR_USERNAME';
const password = 'YOUR_PASSWORD';

const agent = new HttpsProxyAgent(
  `https://${username}:${password}@unblock.oxylabs.io:60000`
);

// Recomendamos aceitar nosso certificado em vez de permitir tráfego inseguro (http)
process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = 0;

const headers = {
   'X-Oxylabs-Render': 'html',
}

const response = await fetch('https://ip.oxylabs.io/location', {
  method: 'get',
  headers: headers,
  agent: agent,
});

console.log(await response.text());

<?php
$ch = curl_init();

curl_setopt($ch, CURLOPT_URL, 'https://ip.oxylabs.io/location');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_PROXY, 'https://unblock.oxylabs.io:60000');
curl_setopt($ch, CURLOPT_PROXYUSERPWD, 'YOUR_USERNAME' . ':' . 'YOUR_PASSWORD');
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);

curl_setopt_array($ch, array(
    CURLOPT_HTTPHEADER  => array(
        'X-Oxylabs-Render: html'
    )
));

$result = curl_exec($ch);
echo $result;

if (curl_errno($ch)) {
    echo 'Error:' . curl_error($ch);
}
curl_close($ch);

package main

import (
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
)

func main() {
	const Username = "YOUR_USERNAME"
	const Password = "YOUR_PASSWORD"

	proxyUrl, _ := url.Parse(
		fmt.Sprintf(
			"https://%s:%s@unblock.oxylabs.io:60000",
			Username,
			Password,
		),
	)
	customTransport := &http.Transport{Proxy: http.ProxyURL(proxyUrl)}

	// Recomendamos aceitar nosso certificado em vez de permitir tráfego inseguro (http)
	customTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}

	client := &http.Client{Transport: customTransport}
	request, _ := http.NewRequest("GET",
		"https://ip.oxylabs.io/location",
		nil,
	)
	
	// Adicionar cookies personalizados
        request.Header.Add("X-Oxylabs-Render", "html")
        
	request.SetBasicAuth(Username, Password)
	response, _ := client.Do(request)

	responseText, _ := ioutil.ReadAll(response.Body)
	fmt.Println(string(responseText))
}

using System;
using System.Net;
using System.Net.Http;
using System.Threading.Tasks;

namespace OxyApi
{
    class Program
    {
        static async Task Main(string[] args)
        {
            var webProxy = new WebProxy
            {
                Address = new Uri("https://unblock.oxylabs.io:60000"),
                BypassProxyOnLocal = false,
                UseDefaultCredentials = false,

                Credentials = new NetworkCredential(
                userName: "YOUR_USERNAME",
                password: "YOUR_PASSWORD"
                )
            };

            var httpClientHandler = new HttpClientHandler
            {
                Proxy = webProxy,
            };

            // Recomendamos aceitar nosso certificado em vez de permitir tráfego inseguro (http)
            httpClientHandler.ClientCertificateOptions = ClientCertificateOption.Manual;
            httpClientHandler.ServerCertificateCustomValidationCallback =
                (httpRequestMessage, cert, cetChain, policyErrors) =>
                {
                    return true;
                };


            var client = new HttpClient(handler: httpClientHandler, disposeHandler: true);
            
            // Adicionar cookies personalizados
            client.DefaultRequestHeaders.Add("X-Oxylabs-Render", "html");
            
            Uri baseUri = new Uri("https://ip.oxylabs.io/location");
            client.BaseAddress = baseUri;

            var requestMessage = new HttpRequestMessage(HttpMethod.Get, "");

            var response = await client.SendAsync(requestMessage);
            var contents = await response.Content.ReadAsStringAsync();

            Console.WriteLine(contents);
        }
    }
}

package org.example;

import org.apache.hc.client5.http.auth.AuthScope;
import org.apache.hc.client5.http.auth.CredentialsProvider;
import org.apache.hc.client5.http.classic.methods.HttpGet;
import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.client5.http.impl.auth.CredentialsProviderBuilder;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder;
import org.apache.hc.client5.http.ssl.NoopHostnameVerifier;
import org.apache.hc.client5.http.ssl.SSLConnectionSocketFactoryBuilder;
import org.apache.hc.client5.http.ssl.TrustAllStrategy;
import org.apache.hc.core5.http.HttpHost;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.http.message.StatusLine;
import org.apache.hc.core5.ssl.SSLContextBuilder;

import java.util.Arrays;
import java.util.Properties;


public class Main {

    public static void main(final String[] args)throws Exception {
        final CredentialsProvider credsProvider = CredentialsProviderBuilder.create()
                .add(new AuthScope("unblock.oxylabs.io", 60000), "USERNAME", "PASSWORD".toCharArray())
                .build();
        final HttpHost target = new HttpHost("https", "ip.oxylabs.io", 443);
        final HttpHost proxy = new HttpHost("https", "unblock.oxylabs.io", 60000);
        try (final CloseableHttpClient httpclient = HttpClients.custom()
                .setDefaultCredentialsProvider(credsProvider)
                .setProxy(proxy)
                // Recomendamos aceitar nosso certificado em vez de permitir tráfego inseguro (http)
                .setConnectionManager(PoolingHttpClientConnectionManagerBuilder.create()
                        .setSSLSocketFactory(SSLConnectionSocketFactoryBuilder.create()
                                .setSslContext(SSLContextBuilder.create()
                                        .loadTrustMaterial(TrustAllStrategy.INSTANCE)
                                        .build())
                                .setHostnameVerifier(NoopHostnameVerifier.INSTANCE)
                                .build())
                        .build())
                .build()) {

            final RequestConfig config = RequestConfig.custom()
                    .build();
            final HttpGet request = new HttpGet("/location");
            request.addHeader("X-Oxylabs-Render","html");
            request.setConfig(config);

            System.out.println("Executando requisição " + request.getMethod() + " " + request.getUri() +
                    " via " + proxy + " headers: " + Arrays.toString(request.getHeaders()));

            httpclient.execute(target, request, response -> {
                System.out.println("----------------------------------------");
                System.out.println(request + "->" + new StatusLine(response));
                EntityUtils.consume(response.getEntity());
                return null;
            });
        }
    }
}

Extraindo o HTML de um website

Neste exemplo, vamos renderizar a página inicial do YouTube e extrair o conteúdo da página. Normalmente, a página inicial do Youtube ficaria assim se o Web Unblocker for usado sem renderização de Javascript:

Adicionar o "X-Oxylabs-Render: html" header, como mostrado nos exemplos abaixo, ativará a renderização de Javascript e retornará o HTML de uma página renderizada:

curl -k -v -x https://unblock.oxylabs.io:60000 \
-U 'USERNAME:PASSWORD' \
'https://youtube.com' \
-H 'X-Oxylabs-Render: html'

import requests

# Use suas credenciais do Web Unblocker aqui.
USERNAME, PASSWORD = 'YOUR_USERNAME', 'YOUR_PASSWORD'

# Defina o dicionário de proxy.
proxies = {
  'http': f'http://{USERNAME}:{PASSWORD}@unblock.oxylabs.io:60000',
  'https': f'https://{USERNAME}:{PASSWORD}@unblock.oxylabs.io:60000',
}

headers = {
    'X-Oxylabs-Render': 'html'
}

response = requests.get(
    'https://youtube.com',
    verify=False,  # É necessário ignorar o certificado
    proxies=proxies,
    headers=headers,
)

# Imprima a página de resultado em stdout
print(response.text)

# Salve o HTML retornado no arquivo result.html
with open('result.html', 'w') as f:
    f.write(response.text)

import fetch from 'node-fetch';
import { HttpsProxyAgent } from 'https-proxy-agent';

const username = 'YOUR_USERNAME';
const password = 'YOUR_PASSWORD';

const agent = new HttpsProxyAgent(
  `https://${username}:${password}@unblock.oxylabs.io:60000`
);

// Recomendamos aceitar nosso certificado em vez de permitir tráfego inseguro (http)
process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = 0;

const headers = {
   'X-Oxylabs-Render': 'html',
}

const response = await fetch('https://youtube.com', {
  method: 'get',
  headers: headers,
  agent: agent,
});

console.log(await response.text());

<?php
$ch = curl_init();

curl_setopt($ch, CURLOPT_URL, 'https://youtube.com');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_PROXY, 'https://unblock.oxylabs.io:60000');
curl_setopt($ch, CURLOPT_PROXYUSERPWD, 'YOUR_USERNAME' . ':' . 'YOUR_PASSWORD');
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);

curl_setopt_array($ch, [
    CURLOPT_HTTPHEADER  => [
        'X-Oxylabs-Render: html'
    ]
]);

$result = curl_exec($ch);
echo $result;

if (curl_errno($ch)) {
    echo 'Error:' . curl_error($ch);
}
curl_close($ch);

package main

import (
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
)

func main() {
	const Username = "YOUR_USERNAME"
	const Password = "YOUR_PASSWORD"

	proxyUrl, _ := url.Parse(
		fmt.Sprintf(
			"https://%s:%s@unblock.oxylabs.io:60000",
			Username,
			Password,
		),
	)
	customTransport := &http.Transport{Proxy: http.ProxyURL(proxyUrl)}

	// Recomendamos aceitar nosso certificado em vez de permitir tráfego inseguro (http)
	customTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}

	client := &http.Client{Transport: customTransport}
	request, _ := http.NewRequest("GET",
		"https://youtube.com",
		nil,
	)
	
	// Adicionar cookies personalizados
        request.Header.Add("X-Oxylabs-Render", "html")
        
	request.SetBasicAuth(Username, Password)
	response, _ := client.Do(request)

	responseText, _ := ioutil.ReadAll(response.Body)
	fmt.Println(string(responseText))
}

using System;
using System.Net;
using System.Net.Http;
using System.Threading.Tasks;

namespace OxyApi
{
    class Program
    {
        static async Task Main(string[] args)
        {
            var webProxy = new WebProxy
            {
                Address = new Uri("https://unblock.oxylabs.io:60000"),
                BypassProxyOnLocal = false,
                UseDefaultCredentials = false,

                Credentials = new NetworkCredential(
                userName: "YOUR_USERNAME",
                password: "YOUR_PASSWORD"
                )
            };

            var httpClientHandler = new HttpClientHandler
            {
                Proxy = webProxy,
            };

            // Recomendamos aceitar nosso certificado em vez de permitir tráfego inseguro (http)
            httpClientHandler.ClientCertificateOptions = ClientCertificateOption.Manual;
            httpClientHandler.ServerCertificateCustomValidationCallback =
                (httpRequestMessage, cert, cetChain, policyErrors) =>
                {
                    return true;
                };


            var client = new HttpClient(handler: httpClientHandler, disposeHandler: true);
            
            // Adicionar header personalizado
            client.DefaultRequestHeaders.Add("X-Oxylabs-Render", "html");
            
            Uri baseUri = new Uri("https://youtube.com");
            client.BaseAddress = baseUri;

            var requestMessage = new HttpRequestMessage(HttpMethod.Get, "");

            var response = await client.SendAsync(requestMessage);
            var contents = await response.Content.ReadAsStringAsync();

            Console.WriteLine(contents);
        }
    }
}

package org.example;

import org.apache.hc.client5.http.auth.AuthScope;
import org.apache.hc.client5.http.auth.CredentialsProvider;
import org.apache.hc.client5.http.classic.methods.HttpGet;
import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.client5.http.impl.auth.CredentialsProviderBuilder;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder;
import org.apache.hc.client5.http.ssl.NoopHostnameVerifier;
import org.apache.hc.client5.http.ssl.SSLConnectionSocketFactoryBuilder;
import org.apache.hc.client5.http.ssl.TrustAllStrategy;
import org.apache.hc.core5.http.HttpHost;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.http.message.StatusLine;
import org.apache.hc.core5.ssl.SSLContextBuilder;

import java.util.Arrays;
import java.util.Properties;


public class Main {

    public static void main(final String[] args)throws Exception {
        final CredentialsProvider credsProvider = CredentialsProviderBuilder.create()
                .add(new AuthScope("unblock.oxylabs.io", 60000), "USERNAME", "PASSWORD".toCharArray())
                .build();
        final HttpHost target = new HttpHost("https", "youtube.com", 443);
        final HttpHost proxy = new HttpHost("https", "unblock.oxylabs.io", 60000);
        try (final CloseableHttpClient httpclient = HttpClients.custom()
                .setDefaultCredentialsProvider(credsProvider)
                .setProxy(proxy)
                // Recomendamos aceitar nosso certificado em vez de permitir tráfego inseguro (http)
                .setConnectionManager(PoolingHttpClientConnectionManagerBuilder.create()
                        .setSSLSocketFactory(SSLConnectionSocketFactoryBuilder.create()
                                .setSslContext(SSLContextBuilder.create()
                                        .loadTrustMaterial(TrustAllStrategy.INSTANCE)
                                        .build())
                                .setHostnameVerifier(NoopHostnameVerifier.INSTANCE)
                                .build())
                        .build())
                .build()) {

            final RequestConfig config = RequestConfig.custom()
                    .build();
            final HttpGet request = new HttpGet("/");
            request.addHeader("X-Oxylabs-Render","html");
            request.setConfig(config);

            System.out.println("Executando requisição " + request.getMethod() + " " + request.getUri() +
                    " via " + proxy + " headers: " + Arrays.toString(request.getHeaders()));

            httpclient.execute(target, request, response -> {
                System.out.println("----------------------------------------");
                System.out.println(request + "->" + new StatusLine(response));
                EntityUtils.consume(response.getEntity());
                return null;
            });
        }
    }
}

O arquivo HTML aberto em um navegador deve ficar assim:

Obtendo uma captura de tela de uma página totalmente renderizada

Para obter uma captura de tela em formato PNG em vez do HTML da página, é necessário fornecer o "X-Oxylabs-Render: png" cabeçalho.

curl -k -v -x https://unblock.oxylabs.io:60000 \
-U 'USERNAME:PASSWORD' \
'https://youtube.com' \
-H 'X-Oxylabs-Render: png' > rendered_page.png

import requests

# Use suas credenciais do Web Unblocker aqui.
USERNAME, PASSWORD = 'YOUR_USERNAME', 'YOUR_PASSWORD'

# Defina o dicionário de proxy.
proxies = {
  'http': f'http://{USERNAME}:{PASSWORD}@unblock.oxylabs.io:60000',
  'https': f'https://{USERNAME}:{PASSWORD}@unblock.oxylabs.io:60000',
}

headers = {
    'X-Oxylabs-Render': 'png'
}

response = requests.get(
    'https://youtube.com',
    verify=False,  # É necessário ignorar o certificado
    proxies=proxies,
    headers=headers,
)

# Salvar captura de tela como arquivo PNG
with open('rendered_page.png', 'wb') as f:
  f.write(response.content)

import fs from 'fs';
import fetch from 'node-fetch';
import { HttpsProxyAgent } from 'https-proxy-agent';

const username = 'YOUR_USERNAME';
const password = 'YOUR_PASSWORD';

const agent = new HttpsProxyAgent(
  `https://${username}:${password}@unblock.oxylabs.io:60000`
);

// Recomendamos aceitar nosso certificado em vez de permitir tráfego inseguro (http)
process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = 0;

const headers = {
   'X-Oxylabs-Render': 'png',
}

await fetch('https://youtube.com', {
  method: 'get',
  headers: headers,
  agent: agent,
}).then(res => 
  res.body.pipe(fs.createWriteStream('./rendered_page.png'))
  );

console.log('Image was saved');

<?php
$ch = curl_init();

curl_setopt($ch, CURLOPT_URL, 'https://youtube.com');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_PROXY, 'https://unblock.oxylabs.io:60000');
curl_setopt($ch, CURLOPT_PROXYUSERPWD, 'YOUR_USERNAME' . ':' . 'YOUR_PASSWORD');
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);

curl_setopt_array($ch, [
    CURLOPT_HTTPHEADER  => [
        'X-Oxylabs-Render: png'
    ]
]);

$result = curl_exec($ch);

if (curl_errno($ch)) {
    echo 'Error:' . curl_error($ch);
}
curl_close ($ch);

if(file_exists('rendered_page.png')){
    unlink('rendered_page.png');
}
$fp = fopen('rendered_page.png','wb');
fwrite($fp, $result);
fclose($fp);

package main

import (
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
)

func main() {
	const Username = "YOUR_USERNAME"
	const Password = "YOUR_PASSWORD"

	proxyUrl, _ := url.Parse(
		fmt.Sprintf(
			"https://%s:%s@unblock.oxylabs.io:60000",
			Username,
			Password,
		),
	)
	customTransport := &http.Transport{Proxy: http.ProxyURL(proxyUrl)}

	// Recomendamos aceitar nosso certificado em vez de permitir tráfego inseguro (http)
	customTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}

	client := &http.Client{Transport: customTransport}
	request, _ := http.NewRequest("GET",
		"https://youtube.com",
		nil,
	)
	
	// Adicionar cookies personalizados
        request.Header.Add("X-Oxylabs-Render", "png")
        
	request.SetBasicAuth(Username, Password)
	response, _ := client.Do(request)
	
	responseData, _ := ioutil.ReadAll(response.Body)
	response.Body.Close()
	ioutil.WriteFile("rendered_page.png", responseData, 0666)

	log.Println("Image was saved")
}

using System;
using System.Net;
using System.Net.Http;
using System.Drawing;
using System.IO;
using System.Threading.Tasks;

namespace OxyApi
{
    class Program
    {
        static async Task Main(string[] args)
        {
            var webProxy = new WebProxy
            {
                Address = new Uri("https://unblock.oxylabs.io:60000"),
                BypassProxyOnLocal = false,
                UseDefaultCredentials = false,

                Credentials = new NetworkCredential(
                userName: "YOUR_USERNAME",
                password: "YOUR_PASSWORD"
                )
            };

            var httpClientHandler = new HttpClientHandler
            {
                Proxy = webProxy,
            };

            // Recomendamos aceitar nosso certificado em vez de permitir tráfego inseguro (http)
            httpClientHandler.ClientCertificateOptions = ClientCertificateOption.Manual;
            httpClientHandler.ServerCertificateCustomValidationCallback =
                (httpRequestMessage, cert, cetChain, policyErrors) =>
                {
                    return true;
                };


            var client = new HttpClient(handler: httpClientHandler, disposeHandler: true);
            
            // Adicionar header personalizado
            client.DefaultRequestHeaders.Add("X-Oxylabs-Render", "png");
            
            Uri baseUri = new Uri("https://youtube.com");
            client.BaseAddress = baseUri;

            var requestMessage = new HttpRequestMessage(HttpMethod.Get, "");

            var response = await client.SendAsync(requestMessage);
            byte[] bytes = await response.Content.ReadAsByteArrayAsync();

            using (Image image = Image.FromStream(new MemoryStream(bytes)))
            {
                image.Save("rendered_page.png");
            }
        }
    }
}

package org.example;

import okhttp3.Authenticator;
import okhttp3.Credentials;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import javax.net.ssl.*;
import java.net.InetSocketAddress;
import java.net.Proxy;
import java.util.concurrent.TimeUnit;
import java.io.File;
import org.apache.commons.io.FileUtils;

public class App implements Runnable
{
    private static final String AUTHORIZATION_HEADER = "Proxy-Authorization";
    public static final String USERNAME = "YOUR_USERNAME";
    public static final String PASSWORD = "YOUR_PASSWORD";

    public void run() {
        Authenticator authenticator = (route, response) -> {
            String credential = Credentials.basic(USERNAME, PASSWORD);

            return response
                    .request()
                    .newBuilder()
                    .header(AUTHORIZATION_HEADER, credential)
                    .build();
        };

        OkHttpClient.Builder builder = new OkHttpClient.Builder()
                .readTimeout(180, TimeUnit.SECONDS)
                .writeTimeout(180, TimeUnit.SECONDS);

        // Recomendamos aceitar nosso certificado em vez de permitir tráfego inseguro (http)
        this.disableSSLCertificateChecking(builder);

        Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress("unblock.oxylabs.io", 60000));
        OkHttpClient client = builder
                .proxy(proxy)
                .proxyAuthenticator(authenticator)
                .build();

        var request = new Request.Builder()
                .url("https://youtube.com")
                .addHeader("X-Oxylabs-Render", "png")
                .get()
                .build();

        try (var response = client.newCall(request).execute()) {
            assert response.body() != null;
            okhttp3.ResponseBody body = response.body();
            if (body != null) {
                byte[] bytes = body.bytes();
                FileUtils.writeByteArrayToFile(new File("rendered_page.png"), bytes);
            }
        } catch (Exception exception) {
            exception.printStackTrace();
            System.exit(1);
        }

        System.exit(0);
    }

    private void disableSSLCertificateChecking(OkHttpClient.Builder builder) {
        TrustManager[] trustManagers = new TrustManager[]{
                new X509TrustManager() {
                    @Override
                    public java.security.cert.X509Certificate[] getAcceptedIssuers() {
                        return new java.security.cert.X509Certificate[]{};
                    }

                    @Override
                    public void checkServerTrusted(java.security.cert.X509Certificate[] x509Certificates, String authType) {
                    }

                    @Override
                    public void checkClientTrusted(java.security.cert.X509Certificate[] x509Certificates, String authType) {
                    }
                }
        };

        try {
            HttpsURLConnection.setDefaultHostnameVerifier((s, sslSession) -> true);
            SSLContext sslContext = SSLContext.getInstance("TLS");
            sslContext.init(null, trustManagers, new java.security.SecureRandom());
            HttpsURLConnection.setDefaultSSLSocketFactory(sslContext.getSocketFactory());

            builder.sslSocketFactory(sslContext.getSocketFactory(), (X509TrustManager) trustManagers[0]);
        } catch (Exception exception) {
            exception.printStackTrace();
            System.exit(1);
        }

        builder.hostnameVerifier((hostname, session) -> true);
    }

    public static void main(String[] args) {
        new Thread(new App()).start();
    }
}

A resposta conterá bytes brutos de uma imagem que podem ser salvos em formato PNG e abertos como no exemplo abaixo:

Forçando a renderização em páginas específicas

Para uma extração bem-sucedida, alguns tipos de página de domínios específicos exigem renderização devido ao conteúdo dinâmico. Nosso sistema aplica automaticamente a renderização para essas páginas, mesmo que não seja definida explicitamente pelo usuário.

Observe que trabalhos renderizados consomem mais tráfego em comparação com trabalhos não renderizados.

Queremos que nossos usuários estejam totalmente cientes disso ao extrair as seguintes páginas:

6KB

Force render domains.csv

Abrir

Essa abordagem oferece a melhor experiência de extração possível, garantindo a precisão e a confiabilidade dos dados nessas páginas desafiadoras.

Se você quiser desativar a renderização, envie o header de renderização sem um valor

X-oxylabs-render:

AnteriorInstruções personalizadas do navegador PróximoInstruções do navegador (Beta)

Isto foi útil?

package org.example;

import org.apache.hc.client5.http.auth.AuthScope;
import org.apache.hc.client5.http.auth.CredentialsProvider;
import org.apache.hc.client5.http.classic.methods.HttpGet;
import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.client5.http.impl.auth.CredentialsProviderBuilder;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder;
import org.apache.hc.client5.http.ssl.NoopHostnameVerifier;
import org.apache.hc.client5.http.ssl.SSLConnectionSocketFactoryBuilder;
import org.apache.hc.client5.http.ssl.TrustAllStrategy;
import org.apache.hc.core5.http.HttpHost;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.http.message.StatusLine;
import org.apache.hc.core5.ssl.SSLContextBuilder;

import java.util.Arrays;
import java.util.Properties;


public class Main {

    public static void main(final String[] args)throws Exception {
        final CredentialsProvider credsProvider = CredentialsProviderBuilder.create()
                .add(new AuthScope("unblock.oxylabs.io", 60000), "USERNAME", "PASSWORD".toCharArray())
                .build();
        final HttpHost target = new HttpHost("https", "ip.oxylabs.io", 443);
        final HttpHost proxy = new HttpHost("https", "unblock.oxylabs.io", 60000);
        try (final CloseableHttpClient httpclient = HttpClients.custom()
                .setDefaultCredentialsProvider(credsProvider)
                .setProxy(proxy)
                // Recomendamos aceitar nosso certificado em vez de permitir tráfego inseguro (http)
                .setConnectionManager(PoolingHttpClientConnectionManagerBuilder.create()
                        .setSSLSocketFactory(SSLConnectionSocketFactoryBuilder.create()
                                .setSslContext(SSLContextBuilder.create()
                                        .loadTrustMaterial(TrustAllStrategy.INSTANCE)
                                        .build())
                                .setHostnameVerifier(NoopHostnameVerifier.INSTANCE)
                                .build())
                        .build())
                .build()) {

            final RequestConfig config = RequestConfig.custom()
                    .build();
            final HttpGet request = new HttpGet("/location");
            request.addHeader("X-Oxylabs-Render","html");
            request.setConfig(config);

            System.out.println("Executando requisição " + request.getMethod() + " " + request.getUri() +
                    " via " + proxy + " headers: " + Arrays.toString(request.getHeaders()));

            httpclient.execute(target, request, response -> {
                System.out.println("----------------------------------------");
                System.out.println(request + "->" + new StatusLine(response));
                EntityUtils.consume(response.getEntity());
                return null;
            });
        }
    }
}

package org.example;

import org.apache.hc.client5.http.auth.AuthScope;
import org.apache.hc.client5.http.auth.CredentialsProvider;
import org.apache.hc.client5.http.classic.methods.HttpGet;
import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.client5.http.impl.auth.CredentialsProviderBuilder;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder;
import org.apache.hc.client5.http.ssl.NoopHostnameVerifier;
import org.apache.hc.client5.http.ssl.SSLConnectionSocketFactoryBuilder;
import org.apache.hc.client5.http.ssl.TrustAllStrategy;
import org.apache.hc.core5.http.HttpHost;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.http.message.StatusLine;
import org.apache.hc.core5.ssl.SSLContextBuilder;

import java.util.Arrays;
import java.util.Properties;


public class Main {

    public static void main(final String[] args)throws Exception {
        final CredentialsProvider credsProvider = CredentialsProviderBuilder.create()
                .add(new AuthScope("unblock.oxylabs.io", 60000), "USERNAME", "PASSWORD".toCharArray())
                .build();
        final HttpHost target = new HttpHost("https", "youtube.com", 443);
        final HttpHost proxy = new HttpHost("https", "unblock.oxylabs.io", 60000);
        try (final CloseableHttpClient httpclient = HttpClients.custom()
                .setDefaultCredentialsProvider(credsProvider)
                .setProxy(proxy)
                // Recomendamos aceitar nosso certificado em vez de permitir tráfego inseguro (http)
                .setConnectionManager(PoolingHttpClientConnectionManagerBuilder.create()
                        .setSSLSocketFactory(SSLConnectionSocketFactoryBuilder.create()
                                .setSslContext(SSLContextBuilder.create()
                                        .loadTrustMaterial(TrustAllStrategy.INSTANCE)
                                        .build())
                                .setHostnameVerifier(NoopHostnameVerifier.INSTANCE)
                                .build())
                        .build())
                .build()) {

            final RequestConfig config = RequestConfig.custom()
                    .build();
            final HttpGet request = new HttpGet("/");
            request.addHeader("X-Oxylabs-Render","html");
            request.setConfig(config);

            System.out.println("Executando requisição " + request.getMethod() + " " + request.getUri() +
                    " via " + proxy + " headers: " + Arrays.toString(request.getHeaders()));

            httpclient.execute(target, request, response -> {
                System.out.println("----------------------------------------");
                System.out.println(request + "->" + new StatusLine(response));
                EntityUtils.consume(response.getEntity());
                return null;
            });
        }
    }
}

package org.example;

import okhttp3.Authenticator;
import okhttp3.Credentials;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import javax.net.ssl.*;
import java.net.InetSocketAddress;
import java.net.Proxy;
import java.util.concurrent.TimeUnit;
import java.io.File;
import org.apache.commons.io.FileUtils;

public class App implements Runnable
{
    private static final String AUTHORIZATION_HEADER = "Proxy-Authorization";
    public static final String USERNAME = "YOUR_USERNAME";
    public static final String PASSWORD = "YOUR_PASSWORD";

    public void run() {
        Authenticator authenticator = (route, response) -> {
            String credential = Credentials.basic(USERNAME, PASSWORD);

            return response
                    .request()
                    .newBuilder()
                    .header(AUTHORIZATION_HEADER, credential)
                    .build();
        };

        OkHttpClient.Builder builder = new OkHttpClient.Builder()
                .readTimeout(180, TimeUnit.SECONDS)
                .writeTimeout(180, TimeUnit.SECONDS);

        // Recomendamos aceitar nosso certificado em vez de permitir tráfego inseguro (http)
        this.disableSSLCertificateChecking(builder);

        Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress("unblock.oxylabs.io", 60000));
        OkHttpClient client = builder
                .proxy(proxy)
                .proxyAuthenticator(authenticator)
                .build();

        var request = new Request.Builder()
                .url("https://youtube.com")
                .addHeader("X-Oxylabs-Render", "png")
                .get()
                .build();

        try (var response = client.newCall(request).execute()) {
            assert response.body() != null;
            okhttp3.ResponseBody body = response.body();
            if (body != null) {
                byte[] bytes = body.bytes();
                FileUtils.writeByteArrayToFile(new File("rendered_page.png"), bytes);
            }
        } catch (Exception exception) {
            exception.printStackTrace();
            System.exit(1);
        }

        System.exit(0);
    }

    private void disableSSLCertificateChecking(OkHttpClient.Builder builder) {
        TrustManager[] trustManagers = new TrustManager[]{
                new X509TrustManager() {
                    @Override
                    public java.security.cert.X509Certificate[] getAcceptedIssuers() {
                        return new java.security.cert.X509Certificate[]{};
                    }

                    @Override
                    public void checkServerTrusted(java.security.cert.X509Certificate[] x509Certificates, String authType) {
                    }

                    @Override
                    public void checkClientTrusted(java.security.cert.X509Certificate[] x509Certificates, String authType) {
                    }
                }
        };

        try {
            HttpsURLConnection.setDefaultHostnameVerifier((s, sslSession) -> true);
            SSLContext sslContext = SSLContext.getInstance("TLS");
            sslContext.init(null, trustManagers, new java.security.SecureRandom());
            HttpsURLConnection.setDefaultSSLSocketFactory(sslContext.getSocketFactory());

            builder.sslSocketFactory(sslContext.getSocketFactory(), (X509TrustManager) trustManagers[0]);
        } catch (Exception exception) {
            exception.printStackTrace();
            System.exit(1);
        }

        builder.hostnameVerifier((hostname, session) -> true);
    }

    public static void main(String[] args) {
        new Thread(new App()).start();
    }
}