renderizado de JavaScript

Aprende sobre el encabezado que puedes incluir en tus solicitudes para obtener datos completamente renderizados, que puedes guardar en un archivo HTML o como una captura de pantalla PNG.

Si la página que deseas extraer requiere cargar JavaScript para cargar dinámicamente todos los datos necesarios en el DOM, en lugar de configurar y usar por tu cuenta un Headless Browser, puedes incluir el "X-Oxylabs-Render: html" encabezado en tus solicitudes. Todas las solicitudes con este encabezado incluido se renderizarán por completo, y todos los datos se almacenarán en un archivo HTML o en una captura de pantalla PNG (según el parámetro pasado).

La renderización de JavaScript tarda más en extraer la página. Al usar renderización de JavaScript, establece el valor del tiempo de espera del lado cliente en 180 segundos.

Para garantizar el menor consumo de tráfico, nuestro sistema no carga activos innecesarios durante el renderizado de la página.

Este parámetro tiene dos valores disponibles:

html (HTML de una página renderizada)
png (bytes sin procesar que se pueden guardar como PNG)

Ejemplos de código

curl -k -v -x https://unblock.oxylabs.io:60000 \
-U 'USERNAME:PASSWORD' \
'https://ip.oxylabs.io/location' \
-H 'X-Oxylabs-Render: html'

import requests

# Usa aquí tus credenciales de Web Unblocker.
USERNAME, PASSWORD = 'YOUR_USERNAME', 'YOUR_PASSWORD'

# Define el diccionario de proxies.
proxies = {
  'http': f'http://{USERNAME}:{PASSWORD}@unblock.oxylabs.io:60000',
  'https': f'https://{USERNAME}:{PASSWORD}@unblock.oxylabs.io:60000',
}

headers = {
    'X-Oxylabs-Render': 'html'
}

response = requests.get(
    'https://ip.oxylabs.io/location',
    verify=False,  # Es necesario ignorar el certificado
    proxies=proxies,
    headers=headers,
)

# Imprime la página de resultados en stdout
print(response.text)

# Guarda el HTML devuelto en el archivo result.html
with open('result.html', 'w') as f:
    f.write(response.text)

import fetch from 'node-fetch';
import { HttpsProxyAgent } from 'https-proxy-agent';

const username = 'YOUR_USERNAME';
const password = 'YOUR_PASSWORD';

const agent = new HttpsProxyAgent(
  `https://${username}:${password}@unblock.oxylabs.io:60000`
);

// Recomendamos aceptar nuestro certificado en lugar de permitir tráfico inseguro (http)
process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = 0;

const headers = {
   'X-Oxylabs-Render': 'html',
}

const response = await fetch('https://ip.oxylabs.io/location', {
  method: 'get',
  headers: headers,
  agent: agent,
});

console.log(await response.text());

<?php
$ch = curl_init();

curl_setopt($ch, CURLOPT_URL, 'https://ip.oxylabs.io/location');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_PROXY, 'https://unblock.oxylabs.io:60000');
curl_setopt($ch, CURLOPT_PROXYUSERPWD, 'YOUR_USERNAME' . ':' . 'YOUR_PASSWORD');
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);

curl_setopt_array($ch, array(
    CURLOPT_HTTPHEADER  => array(
        'X-Oxylabs-Render: html'
    )
));

$result = curl_exec($ch);
echo $result;

if (curl_errno($ch)) {
    echo 'Error:' . curl_error($ch);
}
curl_close($ch);

package main

import (
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
)

func main() {
	const Username = "YOUR_USERNAME"
	const Password = "YOUR_PASSWORD"

	proxyUrl, _ := url.Parse(
		fmt.Sprintf(
			"https://%s:%s@unblock.oxylabs.io:60000",
			Username,
			Password,
		),
	)
	customTransport := &http.Transport{Proxy: http.ProxyURL(proxyUrl)}

	// Recomendamos aceptar nuestro certificado en lugar de permitir tráfico inseguro (http)
	customTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}

	client := &http.Client{Transport: customTransport}
	request, _ := http.NewRequest("GET",
		"https://ip.oxylabs.io/location",
		nil,
	)
	
	// Add custom cookies
        request.Header.Add("X-Oxylabs-Render", "html")
        
	request.SetBasicAuth(Username, Password)
	response, _ := client.Do(request)

	responseText, _ := ioutil.ReadAll(response.Body)
	fmt.Println(string(responseText))
}

using System;
using System.Net;
using System.Net.Http;
using System.Threading.Tasks;

namespace OxyApi
{
    class Program
    {
        static async Task Main(string[] args)
        {
            var webProxy = new WebProxy
            {
                Address = new Uri("https://unblock.oxylabs.io:60000"),
                BypassProxyOnLocal = false,
                UseDefaultCredentials = false,

                Credentials = new NetworkCredential(
                userName: "YOUR_USERNAME",
                password: "YOUR_PASSWORD"
                )
            };

            var httpClientHandler = new HttpClientHandler
            {
                Proxy = webProxy,
            };

            // Recomendamos aceptar nuestro certificado en lugar de permitir tráfico inseguro (http)
            httpClientHandler.ClientCertificateOptions = ClientCertificateOption.Manual;
            httpClientHandler.ServerCertificateCustomValidationCallback =
                (httpRequestMessage, cert, cetChain, policyErrors) =>
                {
                    return true;
                };


            var client = new HttpClient(handler: httpClientHandler, disposeHandler: true);
            
            // Add custom cookies
            client.DefaultRequestHeaders.Add("X-Oxylabs-Render", "html");
            
            Uri baseUri = new Uri("https://ip.oxylabs.io/location");
            client.BaseAddress = baseUri;

            var requestMessage = new HttpRequestMessage(HttpMethod.Get, "");

            var response = await client.SendAsync(requestMessage);
            var contents = await response.Content.ReadAsStringAsync();

            Console.WriteLine(contents);
        }
    }
}

package org.example;

import org.apache.hc.client5.http.auth.AuthScope;
import org.apache.hc.client5.http.auth.CredentialsProvider;
import org.apache.hc.client5.http.classic.methods.HttpGet;
import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.client5.http.impl.auth.CredentialsProviderBuilder;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder;
import org.apache.hc.client5.http.ssl.NoopHostnameVerifier;
import org.apache.hc.client5.http.ssl.SSLConnectionSocketFactoryBuilder;
import org.apache.hc.client5.http.ssl.TrustAllStrategy;
import org.apache.hc.core5.http.HttpHost;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.http.message.StatusLine;
import org.apache.hc.core5.ssl.SSLContextBuilder;

import java.util.Arrays;
import java.util.Properties;


public class Main {

    public static void main(final String[] args)throws Exception {
        final CredentialsProvider credsProvider = CredentialsProviderBuilder.create()
                .add(new AuthScope("unblock.oxylabs.io", 60000), "USERNAME", "PASSWORD".toCharArray())
                .build();
        final HttpHost target = new HttpHost("https", "ip.oxylabs.io", 443);
        final HttpHost proxy = new HttpHost("https", "unblock.oxylabs.io", 60000);
        try (final CloseableHttpClient httpclient = HttpClients.custom()
                .setDefaultCredentialsProvider(credsProvider)
                .setProxy(proxy)
                // Recomendamos aceptar nuestro certificado en lugar de permitir tráfico inseguro (http)
                .setConnectionManager(PoolingHttpClientConnectionManagerBuilder.create()
                        .setSSLSocketFactory(SSLConnectionSocketFactoryBuilder.create()
                                .setSslContext(SSLContextBuilder.create()
                                        .loadTrustMaterial(TrustAllStrategy.INSTANCE)
                                        .build())
                                .setHostnameVerifier(NoopHostnameVerifier.INSTANCE)
                                .build())
                        .build())
                .build()) {

            final RequestConfig config = RequestConfig.custom()
                    .build();
            final HttpGet request = new HttpGet("/location");
            request.addHeader("X-Oxylabs-Render","html");
            request.setConfig(config);

            System.out.println("Executing request " + request.getMethod() + " " + request.getUri() +
                    " via " + proxy + " headers: " + Arrays.toString(request.getHeaders()));

            httpclient.execute(target, request, response -> {
                System.out.println("----------------------------------------");
                System.out.println(request + "->" + new StatusLine(response));
                EntityUtils.consume(response.getEntity());
                return null;
            });
        }
    }
}

Extracción del HTML de un sitio web

En este ejemplo, renderizaremos la página de inicio de YouTube y extraeremos el contenido de la página. Normalmente, la página de inicio de Youtube se vería así si se usa Web Unblocker sin renderización de Javascript:

Añadir el "X-Oxylabs-Render: html" encabezado, como se muestra en los ejemplos de abajo, habilitará la renderización de Javascript y devolverá el HTML de una página renderizada:

curl -k -v -x https://unblock.oxylabs.io:60000 \
-U 'USERNAME:PASSWORD' \
'https://youtube.com' \\
-H 'X-Oxylabs-Render: html'

import requests

# Usa aquí tus credenciales de Web Unblocker.
USERNAME, PASSWORD = 'YOUR_USERNAME', 'YOUR_PASSWORD'

# Define el diccionario de proxies.
proxies = {
  'http': f'http://{USERNAME}:{PASSWORD}@unblock.oxylabs.io:60000',
  'https': f'https://{USERNAME}:{PASSWORD}@unblock.oxylabs.io:60000',
}

headers = {
    'X-Oxylabs-Render': 'html'
}

response = requests.get(
    'https://youtube.com',
    verify=False,  # Es necesario ignorar el certificado
    proxies=proxies,
    headers=headers,
)

# Imprime la página de resultados en stdout
print(response.text)

# Guarda el HTML devuelto en el archivo result.html
with open('result.html', 'w') as f:
    f.write(response.text)

import fetch from 'node-fetch';
import { HttpsProxyAgent } from 'https-proxy-agent';

const username = 'YOUR_USERNAME';
const password = 'YOUR_PASSWORD';

const agent = new HttpsProxyAgent(
  `https://${username}:${password}@unblock.oxylabs.io:60000`
);

// Recomendamos aceptar nuestro certificado en lugar de permitir tráfico inseguro (http)
process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = 0;

const headers = {
   'X-Oxylabs-Render': 'html',
}

const response = await fetch('https://youtube.com', {
  method: 'get',
  headers: headers,
  agent: agent,
});

console.log(await response.text());

<?php
$ch = curl_init();

curl_setopt($ch, CURLOPT_URL, 'https://youtube.com');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_PROXY, 'https://unblock.oxylabs.io:60000');
curl_setopt($ch, CURLOPT_PROXYUSERPWD, 'YOUR_USERNAME' . ':' . 'YOUR_PASSWORD');
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);

curl_setopt_array($ch, [
    CURLOPT_HTTPHEADER  => [
        'X-Oxylabs-Render: html'
    ]
]);

$result = curl_exec($ch);
echo $result;

if (curl_errno($ch)) {
    echo 'Error:' . curl_error($ch);
}
curl_close($ch);

package main

import (
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
)

func main() {
	const Username = "YOUR_USERNAME"
	const Password = "YOUR_PASSWORD"

	proxyUrl, _ := url.Parse(
		fmt.Sprintf(
			"https://%s:%s@unblock.oxylabs.io:60000",
			Username,
			Password,
		),
	)
	customTransport := &http.Transport{Proxy: http.ProxyURL(proxyUrl)}

	// Recomendamos aceptar nuestro certificado en lugar de permitir tráfico inseguro (http)
	customTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}

	client := &http.Client{Transport: customTransport}
	request, _ := http.NewRequest("GET",
		"https://youtube.com",
		nil,
	)
	
	// Add custom cookies
        request.Header.Add("X-Oxylabs-Render", "html")
        
	request.SetBasicAuth(Username, Password)
	response, _ := client.Do(request)

	responseText, _ := ioutil.ReadAll(response.Body)
	fmt.Println(string(responseText))
}

using System;
using System.Net;
using System.Net.Http;
using System.Threading.Tasks;

namespace OxyApi
{
    class Program
    {
        static async Task Main(string[] args)
        {
            var webProxy = new WebProxy
            {
                Address = new Uri("https://unblock.oxylabs.io:60000"),
                BypassProxyOnLocal = false,
                UseDefaultCredentials = false,

                Credentials = new NetworkCredential(
                userName: "YOUR_USERNAME",
                password: "YOUR_PASSWORD"
                )
            };

            var httpClientHandler = new HttpClientHandler
            {
                Proxy = webProxy,
            };

            // Recomendamos aceptar nuestro certificado en lugar de permitir tráfico inseguro (http)
            httpClientHandler.ClientCertificateOptions = ClientCertificateOption.Manual;
            httpClientHandler.ServerCertificateCustomValidationCallback =
                (httpRequestMessage, cert, cetChain, policyErrors) =>
                {
                    return true;
                };


            var client = new HttpClient(handler: httpClientHandler, disposeHandler: true);
            
            // Agregar encabezado personalizado
            client.DefaultRequestHeaders.Add("X-Oxylabs-Render", "html");
            
            Uri baseUri = new Uri("https://youtube.com");
            client.BaseAddress = baseUri;

            var requestMessage = new HttpRequestMessage(HttpMethod.Get, "");

            var response = await client.SendAsync(requestMessage);
            var contents = await response.Content.ReadAsStringAsync();

            Console.WriteLine(contents);
        }
    }
}

package org.example;

import org.apache.hc.client5.http.auth.AuthScope;
import org.apache.hc.client5.http.auth.CredentialsProvider;
import org.apache.hc.client5.http.classic.methods.HttpGet;
import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.client5.http.impl.auth.CredentialsProviderBuilder;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder;
import org.apache.hc.client5.http.ssl.NoopHostnameVerifier;
import org.apache.hc.client5.http.ssl.SSLConnectionSocketFactoryBuilder;
import org.apache.hc.client5.http.ssl.TrustAllStrategy;
import org.apache.hc.core5.http.HttpHost;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.http.message.StatusLine;
import org.apache.hc.core5.ssl.SSLContextBuilder;

import java.util.Arrays;
import java.util.Properties;


public class Main {

    public static void main(final String[] args)throws Exception {
        final CredentialsProvider credsProvider = CredentialsProviderBuilder.create()
                .add(new AuthScope("unblock.oxylabs.io", 60000), "USERNAME", "PASSWORD".toCharArray())
                .build();
        final HttpHost target = new HttpHost("https", "youtube.com", 443);
        final HttpHost proxy = new HttpHost("https", "unblock.oxylabs.io", 60000);
        try (final CloseableHttpClient httpclient = HttpClients.custom()
                .setDefaultCredentialsProvider(credsProvider)
                .setProxy(proxy)
                // Recomendamos aceptar nuestro certificado en lugar de permitir tráfico inseguro (http)
                .setConnectionManager(PoolingHttpClientConnectionManagerBuilder.create()
                        .setSSLSocketFactory(SSLConnectionSocketFactoryBuilder.create()
                                .setSslContext(SSLContextBuilder.create()
                                        .loadTrustMaterial(TrustAllStrategy.INSTANCE)
                                        .build())
                                .setHostnameVerifier(NoopHostnameVerifier.INSTANCE)
                                .build())
                        .build())
                .build()) {

            final RequestConfig config = RequestConfig.custom()
                    .build();
            final HttpGet request = new HttpGet("/");
            request.addHeader("X-Oxylabs-Render","html");
            request.setConfig(config);

            System.out.println("Executing request " + request.getMethod() + " " + request.getUri() +
                    " via " + proxy + " headers: " + Arrays.toString(request.getHeaders()));

            httpclient.execute(target, request, response -> {
                System.out.println("----------------------------------------");
                System.out.println(request + "->" + new StatusLine(response));
                EntityUtils.consume(response.getEntity());
                return null;
            });
        }
    }
}

El archivo HTML abierto en un navegador debería verse así:

Obtener una captura de pantalla de una página completamente renderizada

Para obtener una captura de pantalla en formato PNG en lugar del HTML de la página, es necesario proporcionar el "X-Oxylabs-Render: png" encabezado.

curl -k -v -x https://unblock.oxylabs.io:60000 \
-U 'USERNAME:PASSWORD' \
'https://youtube.com' \\
-H 'X-Oxylabs-Render: png' > rendered_page.png

import requests

# Usa aquí tus credenciales de Web Unblocker.
USERNAME, PASSWORD = 'YOUR_USERNAME', 'YOUR_PASSWORD'

# Define el diccionario de proxies.
proxies = {
  'http': f'http://{USERNAME}:{PASSWORD}@unblock.oxylabs.io:60000',
  'https': f'https://{USERNAME}:{PASSWORD}@unblock.oxylabs.io:60000',
}

headers = {
    'X-Oxylabs-Render': 'png'
}

response = requests.get(
    'https://youtube.com',
    verify=False,  # Es necesario ignorar el certificado
    proxies=proxies,
    headers=headers,
)

# Guardar captura de pantalla como archivo PNG
with open('rendered_page.png', 'wb') as f:
  f.write(response.content)

import fs from 'fs';
import fetch from 'node-fetch';
import { HttpsProxyAgent } from 'https-proxy-agent';

const username = 'YOUR_USERNAME';
const password = 'YOUR_PASSWORD';

const agent = new HttpsProxyAgent(
  `https://${username}:${password}@unblock.oxylabs.io:60000`
);

// Recomendamos aceptar nuestro certificado en lugar de permitir tráfico inseguro (http)
process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = 0;

const headers = {
   'X-Oxylabs-Render': 'png',
}

await fetch('https://youtube.com', {
  method: 'get',
  headers: headers,
  agent: agent,
}).then(res => 
  res.body.pipe(fs.createWriteStream('./rendered_page.png'))
  );

console.log('Image was saved');

<?php
$ch = curl_init();

curl_setopt($ch, CURLOPT_URL, 'https://youtube.com');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_PROXY, 'https://unblock.oxylabs.io:60000');
curl_setopt($ch, CURLOPT_PROXYUSERPWD, 'YOUR_USERNAME' . ':' . 'YOUR_PASSWORD');
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);

curl_setopt_array($ch, [
    CURLOPT_HTTPHEADER  => [
        'X-Oxylabs-Render: png'
    ]
]);

$result = curl_exec($ch);

if (curl_errno($ch)) {
    echo 'Error:' . curl_error($ch);
}
curl_close ($ch);

if(file_exists('rendered_page.png')){
    unlink('rendered_page.png');
}
$fp = fopen('rendered_page.png','wb');
fwrite($fp, $result);
fclose($fp);

package main

import (
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
)

func main() {
	const Username = "YOUR_USERNAME"
	const Password = "YOUR_PASSWORD"

	proxyUrl, _ := url.Parse(
		fmt.Sprintf(
			"https://%s:%s@unblock.oxylabs.io:60000",
			Username,
			Password,
		),
	)
	customTransport := &http.Transport{Proxy: http.ProxyURL(proxyUrl)}

	// Recomendamos aceptar nuestro certificado en lugar de permitir tráfico inseguro (http)
	customTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}

	client := &http.Client{Transport: customTransport}
	request, _ := http.NewRequest("GET",
		"https://youtube.com",
		nil,
	)
	
	// Add custom cookies
        request.Header.Add("X-Oxylabs-Render", "png")
        
	request.SetBasicAuth(Username, Password)
	response, _ := client.Do(request)
	
	responseData, _ := ioutil.ReadAll(response.Body)
	response.Body.Close()
	ioutil.WriteFile("rendered_page.png", responseData, 0666)

	log.Println("Image was saved")
}

using System;
using System.Net;
using System.Net.Http;
using System.Drawing;
using System.IO;
using System.Threading.Tasks;

namespace OxyApi
{
    class Program
    {
        static async Task Main(string[] args)
        {
            var webProxy = new WebProxy
            {
                Address = new Uri("https://unblock.oxylabs.io:60000"),
                BypassProxyOnLocal = false,
                UseDefaultCredentials = false,

                Credentials = new NetworkCredential(
                userName: "YOUR_USERNAME",
                password: "YOUR_PASSWORD"
                )
            };

            var httpClientHandler = new HttpClientHandler
            {
                Proxy = webProxy,
            };

            // Recomendamos aceptar nuestro certificado en lugar de permitir tráfico inseguro (http)
            httpClientHandler.ClientCertificateOptions = ClientCertificateOption.Manual;
            httpClientHandler.ServerCertificateCustomValidationCallback =
                (httpRequestMessage, cert, cetChain, policyErrors) =>
                {
                    return true;
                };


            var client = new HttpClient(handler: httpClientHandler, disposeHandler: true);
            
            // Agregar encabezado personalizado
            client.DefaultRequestHeaders.Add("X-Oxylabs-Render", "png");
            
            Uri baseUri = new Uri("https://youtube.com");
            client.BaseAddress = baseUri;

            var requestMessage = new HttpRequestMessage(HttpMethod.Get, "");

            var response = await client.SendAsync(requestMessage);
            byte[] bytes = await response.Content.ReadAsByteArrayAsync();

            using (Image image = Image.FromStream(new MemoryStream(bytes)))
            {
                image.Save("rendered_page.png");
            }
        }
    }
}

package org.example;

import okhttp3.Authenticator;
import okhttp3.Credentials;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import javax.net.ssl.*;
import java.net.InetSocketAddress;
import java.net.Proxy;
import java.util.concurrent.TimeUnit;
import java.io.File;
import org.apache.commons.io.FileUtils;

public class App implements Runnable
{
    private static final String AUTHORIZATION_HEADER = "Proxy-Authorization";
    public static final String USERNAME = "YOUR_USERNAME";
    public static final String PASSWORD = "YOUR_PASSWORD";

    public void run() {
        Authenticator authenticator = (route, response) -> {
            String credential = Credentials.basic(USERNAME, PASSWORD);

            return response
                    .request()
                    .newBuilder()
                    .header(AUTHORIZATION_HEADER, credential)
                    .build();
        };

        OkHttpClient.Builder builder = new OkHttpClient.Builder()
                .readTimeout(180, TimeUnit.SECONDS)
                .writeTimeout(180, TimeUnit.SECONDS);

        // Recomendamos aceptar nuestro certificado en lugar de permitir tráfico inseguro (http)
        this.disableSSLCertificateChecking(builder);

        Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress("unblock.oxylabs.io", 60000));
        OkHttpClient client = builder
                .proxy(proxy)
                .proxyAuthenticator(authenticator)
                .build();

        var request = new Request.Builder()
                .url("https://youtube.com")
                .addHeader("X-Oxylabs-Render", "png")
                .get()
                .build();

        try (var response = client.newCall(request).execute()) {
            assert response.body() != null;
            okhttp3.ResponseBody body = response.body();
            if (body != null) {
                byte[] bytes = body.bytes();
                FileUtils.writeByteArrayToFile(new File("rendered_page.png"), bytes);
            }
        } catch (Exception exception) {
            exception.printStackTrace();
            System.exit(1);
        }

        System.exit(0);
    }

    private void disableSSLCertificateChecking(OkHttpClient.Builder builder) {
        TrustManager[] trustManagers = new TrustManager[]{
                new X509TrustManager() {
                    @Override
                    public java.security.cert.X509Certificate[] getAcceptedIssuers() {
                        return new java.security.cert.X509Certificate[]{};
                    }

                    @Override
                    public void checkServerTrusted(java.security.cert.X509Certificate[] x509Certificates, String authType) {
                    }

                    @Override
                    public void checkClientTrusted(java.security.cert.X509Certificate[] x509Certificates, String authType) {
                    }
                }
        };

        try {
            HttpsURLConnection.setDefaultHostnameVerifier((s, sslSession) -> true);
            SSLContext sslContext = SSLContext.getInstance("TLS");
            sslContext.init(null, trustManagers, new java.security.SecureRandom());
            HttpsURLConnection.setDefaultSSLSocketFactory(sslContext.getSocketFactory());

            builder.sslSocketFactory(sslContext.getSocketFactory(), (X509TrustManager) trustManagers[0]);
        } catch (Exception exception) {
            exception.printStackTrace();
            System.exit(1);
        }

        builder.hostnameVerifier((hostname, session) -> true);
    }

    public static void main(String[] args) {
        new Thread(new App()).start();
    }
}

La respuesta contendrá bytes sin procesar de una imagen que se puede guardar en formato PNG y abrir como en el siguiente ejemplo:

Forzar el renderizado en páginas específicas

Para una extracción exitosa, algunos tipos de páginas de dominios específicos requieren renderizado debido a su contenido dinámico. Nuestro sistema aplica automáticamente el renderizado para estas páginas, incluso si el usuario no lo establece explícitamente.

Tenga en cuenta que los trabajos renderizados consumen más tráfico que los trabajos no renderizados.

Queremos que nuestros usuarios sean plenamente conscientes de esto al extraer las siguientes páginas:

6KB

Force render domains.csv

Abrir

Este enfoque ofrece la mejor experiencia posible de extracción, garantizando la precisión y fiabilidad de los datos de estas páginas exigentes.

Si deseas desactivar la renderización, envía el encabezado de renderización sin un valor

X-oxylabs-render:

AnteriorInstrucciones de navegador personalizadas SiguienteInstrucciones del navegador (Beta)

¿Te fue útil?

package org.example;

import org.apache.hc.client5.http.auth.AuthScope;
import org.apache.hc.client5.http.auth.CredentialsProvider;
import org.apache.hc.client5.http.classic.methods.HttpGet;
import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.client5.http.impl.auth.CredentialsProviderBuilder;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder;
import org.apache.hc.client5.http.ssl.NoopHostnameVerifier;
import org.apache.hc.client5.http.ssl.SSLConnectionSocketFactoryBuilder;
import org.apache.hc.client5.http.ssl.TrustAllStrategy;
import org.apache.hc.core5.http.HttpHost;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.http.message.StatusLine;
import org.apache.hc.core5.ssl.SSLContextBuilder;

import java.util.Arrays;
import java.util.Properties;


public class Main {

    public static void main(final String[] args)throws Exception {
        final CredentialsProvider credsProvider = CredentialsProviderBuilder.create()
                .add(new AuthScope("unblock.oxylabs.io", 60000), "USERNAME", "PASSWORD".toCharArray())
                .build();
        final HttpHost target = new HttpHost("https", "ip.oxylabs.io", 443);
        final HttpHost proxy = new HttpHost("https", "unblock.oxylabs.io", 60000);
        try (final CloseableHttpClient httpclient = HttpClients.custom()
                .setDefaultCredentialsProvider(credsProvider)
                .setProxy(proxy)
                // Recomendamos aceptar nuestro certificado en lugar de permitir tráfico inseguro (http)
                .setConnectionManager(PoolingHttpClientConnectionManagerBuilder.create()
                        .setSSLSocketFactory(SSLConnectionSocketFactoryBuilder.create()
                                .setSslContext(SSLContextBuilder.create()
                                        .loadTrustMaterial(TrustAllStrategy.INSTANCE)
                                        .build())
                                .setHostnameVerifier(NoopHostnameVerifier.INSTANCE)
                                .build())
                        .build())
                .build()) {

            final RequestConfig config = RequestConfig.custom()
                    .build();
            final HttpGet request = new HttpGet("/location");
            request.addHeader("X-Oxylabs-Render","html");
            request.setConfig(config);

            System.out.println("Executing request " + request.getMethod() + " " + request.getUri() +
                    " via " + proxy + " headers: " + Arrays.toString(request.getHeaders()));

            httpclient.execute(target, request, response -> {
                System.out.println("----------------------------------------");
                System.out.println(request + "->" + new StatusLine(response));
                EntityUtils.consume(response.getEntity());
                return null;
            });
        }
    }
}

package org.example;

import org.apache.hc.client5.http.auth.AuthScope;
import org.apache.hc.client5.http.auth.CredentialsProvider;
import org.apache.hc.client5.http.classic.methods.HttpGet;
import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.client5.http.impl.auth.CredentialsProviderBuilder;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder;
import org.apache.hc.client5.http.ssl.NoopHostnameVerifier;
import org.apache.hc.client5.http.ssl.SSLConnectionSocketFactoryBuilder;
import org.apache.hc.client5.http.ssl.TrustAllStrategy;
import org.apache.hc.core5.http.HttpHost;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.http.message.StatusLine;
import org.apache.hc.core5.ssl.SSLContextBuilder;

import java.util.Arrays;
import java.util.Properties;


public class Main {

    public static void main(final String[] args)throws Exception {
        final CredentialsProvider credsProvider = CredentialsProviderBuilder.create()
                .add(new AuthScope("unblock.oxylabs.io", 60000), "USERNAME", "PASSWORD".toCharArray())
                .build();
        final HttpHost target = new HttpHost("https", "youtube.com", 443);
        final HttpHost proxy = new HttpHost("https", "unblock.oxylabs.io", 60000);
        try (final CloseableHttpClient httpclient = HttpClients.custom()
                .setDefaultCredentialsProvider(credsProvider)
                .setProxy(proxy)
                // Recomendamos aceptar nuestro certificado en lugar de permitir tráfico inseguro (http)
                .setConnectionManager(PoolingHttpClientConnectionManagerBuilder.create()
                        .setSSLSocketFactory(SSLConnectionSocketFactoryBuilder.create()
                                .setSslContext(SSLContextBuilder.create()
                                        .loadTrustMaterial(TrustAllStrategy.INSTANCE)
                                        .build())
                                .setHostnameVerifier(NoopHostnameVerifier.INSTANCE)
                                .build())
                        .build())
                .build()) {

            final RequestConfig config = RequestConfig.custom()
                    .build();
            final HttpGet request = new HttpGet("/");
            request.addHeader("X-Oxylabs-Render","html");
            request.setConfig(config);

            System.out.println("Executing request " + request.getMethod() + " " + request.getUri() +
                    " via " + proxy + " headers: " + Arrays.toString(request.getHeaders()));

            httpclient.execute(target, request, response -> {
                System.out.println("----------------------------------------");
                System.out.println(request + "->" + new StatusLine(response));
                EntityUtils.consume(response.getEntity());
                return null;
            });
        }
    }
}

package org.example;

import okhttp3.Authenticator;
import okhttp3.Credentials;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import javax.net.ssl.*;
import java.net.InetSocketAddress;
import java.net.Proxy;
import java.util.concurrent.TimeUnit;
import java.io.File;
import org.apache.commons.io.FileUtils;

public class App implements Runnable
{
    private static final String AUTHORIZATION_HEADER = "Proxy-Authorization";
    public static final String USERNAME = "YOUR_USERNAME";
    public static final String PASSWORD = "YOUR_PASSWORD";

    public void run() {
        Authenticator authenticator = (route, response) -> {
            String credential = Credentials.basic(USERNAME, PASSWORD);

            return response
                    .request()
                    .newBuilder()
                    .header(AUTHORIZATION_HEADER, credential)
                    .build();
        };

        OkHttpClient.Builder builder = new OkHttpClient.Builder()
                .readTimeout(180, TimeUnit.SECONDS)
                .writeTimeout(180, TimeUnit.SECONDS);

        // Recomendamos aceptar nuestro certificado en lugar de permitir tráfico inseguro (http)
        this.disableSSLCertificateChecking(builder);

        Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress("unblock.oxylabs.io", 60000));
        OkHttpClient client = builder
                .proxy(proxy)
                .proxyAuthenticator(authenticator)
                .build();

        var request = new Request.Builder()
                .url("https://youtube.com")
                .addHeader("X-Oxylabs-Render", "png")
                .get()
                .build();

        try (var response = client.newCall(request).execute()) {
            assert response.body() != null;
            okhttp3.ResponseBody body = response.body();
            if (body != null) {
                byte[] bytes = body.bytes();
                FileUtils.writeByteArrayToFile(new File("rendered_page.png"), bytes);
            }
        } catch (Exception exception) {
            exception.printStackTrace();
            System.exit(1);
        }

        System.exit(0);
    }

    private void disableSSLCertificateChecking(OkHttpClient.Builder builder) {
        TrustManager[] trustManagers = new TrustManager[]{
                new X509TrustManager() {
                    @Override
                    public java.security.cert.X509Certificate[] getAcceptedIssuers() {
                        return new java.security.cert.X509Certificate[]{};
                    }

                    @Override
                    public void checkServerTrusted(java.security.cert.X509Certificate[] x509Certificates, String authType) {
                    }

                    @Override
                    public void checkClientTrusted(java.security.cert.X509Certificate[] x509Certificates, String authType) {
                    }
                }
        };

        try {
            HttpsURLConnection.setDefaultHostnameVerifier((s, sslSession) -> true);
            SSLContext sslContext = SSLContext.getInstance("TLS");
            sslContext.init(null, trustManagers, new java.security.SecureRandom());
            HttpsURLConnection.setDefaultSSLSocketFactory(sslContext.getSocketFactory());

            builder.sslSocketFactory(sslContext.getSocketFactory(), (X509TrustManager) trustManagers[0]);
        } catch (Exception exception) {
            exception.printStackTrace();
            System.exit(1);
        }

        builder.hostnameVerifier((hostname, session) -> true);
    }

    public static void main(String[] args) {
        new Thread(new App()).start();
    }
}