Proxy Endpoint

通过 Oxylabs 网页爬虫API Proxy Endpoint 发送和接收数据。通过简单的基于 URL 的集成直接访问目标页面。

如果您曾使用常规代理进行数据抓取，集成 Proxy Endpoint 交付方式将轻而易举。您只需将我们的入口节点用作代理，使用网页爬虫API凭据进行授权，并忽略证书。在 cURL，它是 -k 或 --insecure。您的数据将通过开放连接传输给您。

Proxy Endpoint 仅适用于基于 URL 的数据源，其中提供了完整 URL。因此，它只接受少量额外的任务参数，这些参数 应作为标头发送.

Proxy Endpoint 不是为无头浏览器（例如 Chromium、PhantomJS、Splash）或其自动化库（例如 Playwright、Selenium、Puppeteer）驱动而设计的。

端点

GET realtime.oxylabs.io:60000

输入

请参见下面的请求示例。

cURL

curl -k -x https://realtime.oxylabs.io:60000 \
-U 'USERNAME:PASSWORD' \
-H 'x-oxylabs-user-agent-type: desktop' \
-H 'x-oxylabs-geo-location: Germany' \
'https://www.example.com'

Python

import requests
from pprint import pprint

# 在此使用你的网页爬虫API凭据。
USERNAME, PASSWORD = 'YOUR_USERNAME', 'YOUR_PASSWORD'

# 定义代理字典。
代理 = {
  'http': f'http://{USERNAME}:{PASSWORD}@realtime.oxylabs.io:60000',
  'https': f'https://{USERNAME}:{PASSWORD}@realtime.oxylabs.io:60000'
}

# 要设置特定的地理位置、User-Agent 或渲染 JavaScript
# 需要将参数作为请求头发送。
headers = {
    'x-oxylabs-user-agent-type': 'desktop',
    'x-oxylabs-geo-location': 'Germany',
    #'X-Oxylabs-Render': 'html', # 如果你想在页面中渲染 JavaScript，请取消注释。
}

response = requests.request(
    'GET',
    'https://www.example.com',
    headers = headers, # 传递已定义的请求头。
    verify=False,  # 接受我们的证书。
    代理=代理,
)

# 将结果页面打印到标准输出。
pprint(response.text)

# 将返回的 HTML 保存到 'result.html' 文件。
with open('result.html', 'w') as f:
    f.write(response.text)

Node.js

import fetch from 'node-fetch';
import { HttpsProxyAgent } from 'https-proxy-agent';

const username = 'YOUR_USERNAME';
const password = 'YOUR_PASSWORD';

const agent = new HttpsProxyAgent(
  `https://${username}:${password}@realtime.oxylabs.io:60000`
);

// The Proxy Endpoint presents its own certificate for the target site, so TLS verification must be disabled
process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = 0;

const headers = {
  'x-oxylabs-user-agent-type': 'desktop',
  'x-oxylabs-geo-location': 'Germany',
}

const response = await fetch('https://www.example.com', {
  method: 'get',
  headers: headers,
  agent: agent,
});

console.log(await response.text());

PHP

<?php
$ch = curl_init();

curl_setopt($ch, CURLOPT_URL, 'https://www.example.com/');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_PROXY, 'https://realtime.oxylabs.io:60000');
curl_setopt($ch, CURLOPT_PROXYUSERPWD, 'YOUR_USERNAME' . ':' . 'YOUR_PASSWORD');
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
// To set a specific geo-location, user-agent or to render Javascript
// it is required to send parameters as request headers.
curl_setopt_array($ch, array(
    CURLOPT_HTTPHEADER  => array(
        'x-oxylabs-user-agent-type: desktop',
        'x-oxylabs-geo-location: Germany',
        //'X-Oxylabs-Render: html', // Uncomment if you want to render JavaScript within the page.
    )
));

$result = curl_exec($ch);
echo $result;

if (curl_errno($ch)) {
    echo 'Error:' . curl_error($ch);
}
curl_close($ch);
?>

Golang

package main

import (
    "crypto/tls"
    "fmt"
    "io/ioutil"
    "net/http"
    "net/url"
)

func main() {
    const Username = "YOUR_USERNAME"
    const Password = "YOUR_PASSWORD"

    proxyUrl, _ := url.Parse(
        fmt.Sprintf(
            "https://%s:[e-mail]:60000",
            Username,
            Password,
        ),
    )
    customTransport := &http.Transport{Proxy: http.ProxyURL(proxyUrl)}

    // The Proxy Endpoint presents its own certificate for the target site, so TLS verification must be disabled
    customTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}

    client := &http.Client{Transport: customTransport}
    request, _ := http.NewRequest("GET",
        "https://www.example.com",
        nil,
    )

    request.Header.Add("x-oxylabs-user-agent-type", "desktop")
    request.Header.Add("x-oxylabs-geo-location", "Germany")
    request.SetBasicAuth(Username, Password)
    response, _ := client.Do(request)

    responseText, _ := ioutil.ReadAll(response.Body)
    fmt.Println(string(responseText))
}

C#

using System;
using System.Net;
using System.Net.Http;
using System.Threading.Tasks;

namespace OxyApi
{
    class Program
    {
        static async Task Main(string[] args)
        {
            var webProxy = new WebProxy
            {
                Address = new Uri($"https://realtime.oxylabs.io:60000"),
                BypassProxyOnLocal = false,
                UseDefaultCredentials = false,

                Credentials = new NetworkCredential(
                userName: "YOUR_USERNAME",
                password: "YOUR_PASSWORD"
                )
            };

            var httpClientHandler = new HttpClientHandler
            {
                Proxy = webProxy,
            };

            // The Proxy Endpoint presents its own certificate for the target site, so TLS verification must be disabled
            httpClientHandler.ClientCertificateOptions = ClientCertificateOption.Manual;
            httpClientHandler.ServerCertificateCustomValidationCallback =
                (httpRequestMessage, cert, cetChain, policyErrors) =>
                {
                    return true;
                };


            var client = new HttpClient(handler: httpClientHandler, disposeHandler: true);

            client.DefaultRequestHeaders.Add("x-oxylabs-user-agent-type", "desktop");
            client.DefaultRequestHeaders.Add("x-oxylabs-geo-location", "Germany");

            Uri baseUri = new Uri("https://www.example.com");
            client.BaseAddress = baseUri;

            var requestMessage = new HttpRequestMessage(HttpMethod.Get, "");

            var response = await client.SendAsync(requestMessage);
            var contents = await response.Content.ReadAsStringAsync();

            Console.WriteLine(contents);
        }
    }
}

Java

package org.example;

import org.apache.hc.client5.http.auth.AuthScope;
import org.apache.hc.client5.http.auth.CredentialsProvider;
import org.apache.hc.client5.http.classic.methods.HttpGet;
import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.client5.http.impl.auth.CredentialsProviderBuilder;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder;
import org.apache.hc.client5.http.ssl.NoopHostnameVerifier;
import org.apache.hc.client5.http.ssl.SSLConnectionSocketFactoryBuilder;
import org.apache.hc.client5.http.ssl.TrustAllStrategy;
import org.apache.hc.core5.http.HttpHost;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.http.message.StatusLine;
import org.apache.hc.core5.ssl.SSLContextBuilder;

import java.util.Arrays;
import java.util.Properties;


public class Main {

    public static void main(final String[] args)throws Exception {
        final CredentialsProvider credsProvider = CredentialsProviderBuilder.create()
                .add(new AuthScope("realtime.oxylabs.io", 60000), "YOUR_USERNAME", "YOUR_PASSWORD".toCharArray())
                .build();
        final HttpHost target = new HttpHost("https", "example.com", 443);
        final HttpHost proxy = new HttpHost("https", "realtime.oxylabs.io", 60000);
        try (final CloseableHttpClient httpclient = HttpClients.custom()
                .setDefaultCredentialsProvider(credsProvider)
                .setProxy(proxy)
                // The Proxy Endpoint presents its own certificate for the target site, so TLS verification must be disabled
                .setConnectionManager(PoolingHttpClientConnectionManagerBuilder.create()
                        .setSSLSocketFactory(SSLConnectionSocketFactoryBuilder.create()
                                .setSslContext(SSLContextBuilder.create()
                                        .loadTrustMaterial(TrustAllStrategy.INSTANCE)
                                        .build())
                                .setHostnameVerifier(NoopHostnameVerifier.INSTANCE)
                                .build())
                        .build())
                .build()) {

            final RequestConfig config = RequestConfig.custom()
                    .build();
            final HttpGet request = new HttpGet("/");
            request.addHeader("x-oxylabs-user-agent-type","desktop");
            request.addHeader("x-oxylabs-geo-location","Germany");
            request.setConfig(config);

            System.out.println("Executing request " + request.getMethod() + " " + request.getUri() +
                    " via " + proxy + " headers: " + Arrays.toString(request.getHeaders()));

            httpclient.execute(target, request, response -> {
                System.out.println("----------------------------------------");
                System.out.println(request + "->" + new StatusLine(response));
                EntityUtils.consume(response.getEntity());
                return null;
            });
        }
    }
}

输出

下面你会找到来自 https://example.com:

示例响应

<!doctype html>
<html lang="en">
<head>
<title>Example Domain</title>
<link rel="icon" href="data:,">
<meta name="viewport" content="width=device-width, initial-scale=1">
<style>body{background:#eee;width:60vw;margin:15vh auto;font-family:system-ui,sans-serif}h1{font-size:1.5em}div{opacity:0.8}a:link,a:visited{color:#348}</style>
</head>
<body>
<div>
<h1>示例域名</h1>
<p>此域名用于文档示例，无需许可。请避免用于生产操作。</p>
<p>
<a href="https://iana.org/domains/example">了解更多</a>
</p>
</div>
</body>
</html>

可接受的参数

在发起请求时，除了 URL 之外，你还可以向我们发送一些作业参数，我们会在执行作业时使用这些参数。作业参数应在请求头中发送——请看示例 这里.

以下是你可以随 Proxy Endpoint 请求发送的作业参数列表：

参数	说明
x-oxylabs-user-agent-type	无法指定具体的 User-Agent，但你可以告诉我们希望使用哪种 user-agent 类型。支持的 User-Agent 类型列表可在 这里.
x-oxylabs-geo-location	在某些情况下，你可能需要指定结果应适配的地理位置。此参数对应于 geo_location 参数，已在源码级文档中单独说明。可接受的值取决于你希望我们抓取的 URL。了解更多 这里.
x-oxylabs-render	JavaScript 执行。可接受的值： html 和 png。了解更多 这里.
x-oxylabs-parse	设置为 true 可获取结构化 JSON 输出，而不是原始页面内容。仅适用于由 专用解析器.
x-oxylabs-parser-type	显式选择一个解析器。必须与 x-oxylabs-parse: true一起发送，并且必须是支持你正在抓取的 URL 的解析器。
x-oxylabs-browser-instructions	以 JSON 字符串形式提供的浏览器指令。需要 x-oxylabs-render。了解更多 这里.