Push-Pull

了解 Oxylabs 网页爬虫 API 的 Push-Pull 集成方法。提交任务，然后稍后使用 JSON 格式轮询结果端点。

Push-Pull 是我们推荐的集成方法，用于可靠地处理大量数据。

访问 Oxylabs 的 GitHub 仓库以获取完整的工作示例，演示 Python 中的 Push-Pull 集成.

Push-Pull 是一种异步集成方法。提交任务后，您将立即收到一个 JSON 响应，其中包含所有任务详情，包括任务参数、ID 以及用于结果下载和状态检查的 URL。任务处理完毕后，如果您提供了 JSON 有效载荷，我们会将其发送到您的服务器，前提是您提供了回调 URL。结果在完成后至少可以 检索 24 小时 以上。

通过 Push-Pull，您可以将结果直接上传到您的 云存储 （Google Cloud Storage、AWS S3、Alibaba Cloud OSS 或其他兼容 S3 的存储）。

如果您不想为接收回调通知设置服务，您也可以定期检索结果（轮询).

您还可以使用以下工具探索 Push-Pull 的工作方式： Postman.

单个任务

端点

此端点仅接受单个 query 或 URL 值分隔。

POST https://data.oxylabs.io/v1/queries

输入

请在 JSON 有效负载中提供任务参数，如下示例所示。Python 和 PHP 示例包含注释以便说明。

curl --user "user:pass1" \
'https://data.oxylabs.io/v1/queries' \
-H "Content-Type: application/json" \
 -d '{"source": "ENTER_SOURCE_HERE", "url": "https://www.example.com", "geo_location": "United States", "callback_url": "https://your.callback.url", "storage_type": "s3", "storage_url": "s3://your.storage.bucket.url"}'

import requests
from pprint import pprint


# 构建负载。
payload = {
    "source": "ENTER_SOURCE_HERE", # 选择的来源，例如 "universal"
    "url": "https://www.example.com", # 检查具体来源应使用 "url" 还是 "query"
    "geo_location": "United States", # 某些来源接受邮编或坐标
    #"render" : "html", # 如果要在页面内渲染 JavaScript，则取消注释
    #"render" : "png", # 如果要对抓取的网页截图，则取消注释
    #"parse" : true, # 检查哪些来源支持解析数据
    #"callback_url": "https://your.callback.url", # 使用回调监听时为必需项
    "callback_url": "https://your.callback.url",
    "storage_type": "s3", 
    "storage_url": "s3://your.storage.bucket.url"
}

# 获取响应。
response = requests.request(
    'POST',
    'https://data.oxylabs.io/v1/queries',
    auth=('YOUR_USERNAME', 'YOUR_PASSWORD'), # 在此填写您的凭据
    json=payload,
)

# 将美化后的响应打印到标准输出。
pprint(response.json())

<?php

$params = array(
    'source' => 'ENTER_SOURCE_HERE', // 选择的来源，例如 "universal"
    'url' => 'https://www.example.com', // 检查具体来源应使用 "url" 还是 "query"
    'geo_location' => 'United States', // 某些来源接受邮编或坐标
    //'render' : 'html', // 如果要在页面内渲染 JavaScript，请取消注释
    //'render' : 'png', // 如果要对抓取的网页截图，请取消注释
    //'parse' : TRUE, // 检查哪些来源支持解析数据
    //'callback_url' => 'https://your.callback.url', // 使用回调监听时为必需项
    'callback_url': 'https://your.callback.url',
    'storage_type' => 's3',
    'storage_url' => 's3://your.storage.bucket.url'
);

$ch = curl_init();

curl_setopt($ch, CURLOPT_URL, "https://data.oxylabs.io/v1/queries");
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($params));
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_USERPWD, "YOUR_USERNAME" . ":" . "YOUR_PASSWORD"); // 在此填写您的凭据

$headers = array();
$headers[] = "Content-Type: application/json";
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);

$result = curl_exec($ch);
echo $result;

if (curl_errno($ch)) {
    echo 'Error:' . curl_error($ch);
}
curl_close ($ch);
?>

using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Net.Http.Json;
using System.Threading.Tasks;

namespace OxyApi
{
    class Program
    {
        static async Task Main()
        {
            const string Username = "YOUR_USERNAME";
            const string Password = "YOUR_PASSWORD";

            var parameters = new Dictionary<string, string>()
            {
                { "source", "ENTER_SOURCE_HERE" },
                { "url", "https://example.com" },
                { "geo_location", "United States" },
                { "callback_url", "https://your.callback.url" },
            };


            var client = new HttpClient();

            Uri baseUri = new Uri("https://data.oxylabs.io");
            client.BaseAddress = baseUri;

            var requestMessage = new HttpRequestMessage(HttpMethod.Post, "/v1/queries");
            requestMessage.Content = JsonContent.Create(parameters);

            var authenticationString = $"{Username}:{Password}";
            var base64EncodedAuthenticationString = Convert.ToBase64String(System.Text.ASCIIEncoding.UTF8.GetBytes(authenticationString));
            requestMessage.Headers.Add("Authorization", "Basic " + base64EncodedAuthenticationString);

            var response = await client.SendAsync(requestMessage);
            var contents = await response.Content.ReadAsStringAsync();

            Console.WriteLine(contents);
        }
    }
}

package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
)

func main() {
	const Username = "YOUR_USERNAME"
	const Password = "YOUR_PASSWORD"

	payload := map[string]string{
		"source": "ENTER_SOURCE_HERE",
		"url": "https://example.com",
		"geo_location": "美国",
		"callback_url": "https://your.callback.url",
	}

	jsonValue, _ := json.Marshal(payload)

	client := &http.Client{}
	request, _ := http.NewRequest("POST",
		"https://data.oxylabs.io/v1/queries",
		bytes.NewBuffer(jsonValue),
	)

	request.Header.Add("Content-type", "application/json")
	request.SetBasicAuth(Username, Password)
	response, _ := client.Do(request)

	responseText, _ := ioutil.ReadAll(response.Body)
	fmt.Println(string(responseText))
}

package org.example;

import okhttp3.*;
import org.json.JSONObject;

public class Main implements Runnable {
    private static final String AUTHORIZATION_HEADER = "Authorization";
    public static final String USERNAME = "YOUR_USERNAME";
    public static final String PASSWORD = "YOUR_PASSWORD";

    public void run() {
        JSONObject jsonObject = new JSONObject();
        jsonObject.put("source", "ENTER_SOURCE_HERE");
        jsonObject.put("url", "https://example.com");
        jsonObject.put("geo_location", "United States");
        jsonObject.put("callback_url", "https://your.callback.url");

        Authenticator authenticator = (route, response) -> {
            String credential = Credentials.basic(USERNAME, PASSWORD);

            return response
                    .request()
                    .newBuilder()
                    .header(AUTHORIZATION_HEADER, credential)
                    .build();
        };

        var client = new OkHttpClient.Builder()
                .authenticator(authenticator)
                .build();

        var mediaType = MediaType.parse("application/json; charset=utf-8");
        var body = RequestBody.create(jsonObject.toString(), mediaType);
        var request = new Request.Builder()
                .url("https://data.oxylabs.io/v1/queries")
                .post(body)
                .build();

        try (var response = client.newCall(request).execute()) {
            assert response.body() != null;
            System.out.println(response.body().string());
        } catch (Exception exception) {
            System.out.println("Error: " + exception.getMessage());
        }

        System.exit(0);
    }

    public static void main(String[] args) {
        new Thread(new Main()).start();
    }
}

import fetch from 'node-fetch';

const username = 'YOUR_USERNAME';
const password = 'YOUR_PASSWORD';
const body = {
  source: 'ENTER_SOURCE_HERE',
  url: 'https://www.example.com',
  geo_location: 'United States',
  callback_url: 'https://your.callback.url',
};
const response = await fetch('https://data.oxylabs.io/v1/queries', {
  method: 'post',
  body: JSON.stringify(body),
  headers: {
    'Content-Type': 'application/json',
    'Authorization': 'Basic ' + Buffer.from(`${username}:${password}`).toString('base64'),
  }
});

console.log(await response.json());

输出

该 API 将以 JSON 响应任务信息，类似如下：

{
  "callback_url": "https://your.callback.url",
  "client_id": 5,
  "context": [
    {
      "key": "results_language",
      "value": null
    },
    {
      "key": "safe_search",
      "value": null
    },
    {
      "key": "tbm",
      "value": null
    },
    {
      "key": "cr",
      "value": null
    },
    {
      "key": "filter",
      "value": null
    }
  ],
  "created_at": "2024-06-26 00:00:01",
  "domain": "com",
  "geo_location": "美国",
  "id": "12345678900987654321",
  "limit": 10,
  "locale": null,
  "pages": 1,
  "parse": false,
  "render": null,
  "url": "https://www.example.com",
  "source": "universal",
  "start_page": 1,
  "status": "pending",
  "storage_type": "s3",
  "storage_url": "YOUR_BUCKET_NAME/12345678900987654321.json",
  "subdomain": "www",
  "updated_at": "2024-06-26 00:00:01",
  "user_agent_type": "desktop",
  "_links": [
    {
      "rel": "self",
      "href": "http://data.oxylabs.io/v1/queries/12345678900987654321",
      "method": "GET"
    },
    {
      "rel": "results",
      "href": "http://data.oxylabs.io/v1/queries/12345678900987654321/results",
      "method": "GET"
    }
  ]
}

数据字典

有关任务输入参数的详细说明，请查阅下表或参考您感兴趣的爬虫的具体文档页面。

键

说明

类型

created_at

任务创建的日期时间。

字符串

client_id

与发起请求的客户端用户名关联的数值 ID。

字符串

client_notes

客户端在发送任务时提交的备注。

字符串

content_encoding

如果您要下载图像，请添加此参数。了解更多此处.

字符串

id

任务的唯一 ID。

字符串

statuses

抓取或解析任务的状态码。您可以在此处查看状态码描述此处.

整数

status

任务的状态。 pending 表示任务仍在处理中。 done 表示我们已完成任务。 faulted 表示我们在尝试完成任务时遇到错误并放弃了。

字符串

subdomain

网站的子域名。

字符串

updated_at

任务最后更新时间的日期时间。对于已完成的任务（status is done 或 faulted），此日期时间表示任务完成的时间。

字符串

链接

与所提供输入相关的链接列表。

JSON 数组

链接:rel

链接类型。 self URL 包含任务的元数据，而 results URL 包含任务结果。

字符串

链接:href

资源的 URL。

字符串

链接:方法

应使用的 HTTP 方法来与给定 URL 交互。

字符串

回调

回调是我们发送到您机器上的一个 POST 请求，用于通知数据提取任务已完成并提供用于下载抓取内容的 URL。这意味着您无需 手动检查任务状态 。一旦数据到达，我们会通知您，您只需 检索它.

输入

# 这是一个简单的 Sanic Web 服务器，具有在 localhost:8080 上监听回调的路由。
# 它会将任务结果打印到 stdout。
import requests
from pprint import pprint
from sanic import Sanic, response


AUTH_TUPLE = ('user', 'pass1')

app = Sanic()


# 定义接受 POST 请求的 /job_listener 端点。
@app.route('/job_listener', methods=['POST'])
async def job_listener(request):
    try:
        res = request.json
        links = res.get('_links', [])
        for link in links:
            if link['rel'] == 'results':
                # Sanic 是异步的，但 requests 是同步的；若要充分利用
                # Sanic，请使用 aiohttp。
                res_response = requests.request(
                    method='GET',
                    url=link['href'],
                    auth=AUTH_TUPLE,
                )
                pprint(res_response.json())
                break
    except Exception as e:
        print("Listener exception: {}".format(e))
    return response.json(status=200, body={'status': 'ok'})


if __name__ == '__main__':
    app.run(host='0.0.0.0', port=8080)

<?php
$stdout = fopen('php://stdout', 'w');

if (isset($_POST)) {
    $result = array_merge($_POST, (array) json_decode(file_get_contents('php://input')));

    $ch = curl_init();

    curl_setopt($ch, CURLOPT_URL, "https://data.oxylabs.io/v1/queries/".$result['id'].'/results');
    curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
    curl_setopt($ch, CURLOPT_CUSTOMREQUEST, "GET");
    curl_setopt($ch, CURLOPT_USERPWD, "user" . ":" . "pass1");

    $result = curl_exec($ch);
    fwrite($stdout, $result);

    if (curl_errno($ch)) {
        echo 'Error:' . curl_error($ch);
    }
    curl_close ($ch);
}
?>

using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using System;
using System.Collections.Generic;
using System.Net.Http;

namespace OxyApiWeb
{
    public class Callback
    {
        public Link[] _links { get; set; }
    }

    public class Link
    {
        public string rel { get; set; }
        public string href { get; set; }
    }

    public class Startup
    {
        private const string USERNAME = "YOUR_USERNAME";
        private const string PASSWORD = "YOUR_PASSWORD";

        public Startup(IConfiguration configuration)
        {
            Configuration = configuration;

            client = new HttpClient();
        }

        public IConfiguration Configuration { get; }

        private HttpClient client;

        public void ConfigureServices(IServiceCollection services)
        {
            services.AddControllers();
        }

        public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
        {
            if (env.IsDevelopment())
            {
                app.UseDeveloperExceptionPage();
            }

            app.UseRouting();

            app.UseAuthorization();

            app.UseEndpoints(endpoints =>
            {
                endpoints.MapPost("/job_listener", async context =>
                {
                    var callback = await System.Text.Json.JsonSerializer.DeserializeAsync<Callback>(context.Request.Body);

                    foreach (var link in callback._links)
                    {
                        if (link.rel != "results")
                        {
                            continue;
                        }

                        var requestMessage = new HttpRequestMessage(HttpMethod.Get, new Uri(link.href));

                        var authenticationString = $"{USERNAME}:{PASSWORD}";
                        var base64EncodedAuthenticationString = Convert.ToBase64String(System.Text.ASCIIEncoding.UTF8.GetBytes(authenticationString));
                        requestMessage.Headers.Add("Authorization", "Basic " + base64EncodedAuthenticationString);

                        var response = await client.SendAsync(requestMessage);
                        var contents = await response.Content.ReadAsStringAsync();

                        Console.WriteLine(contents);

                    }

                    var okMessage = new Dictionary<string, string>()
                    {
                        { "message", "ok" }
                    };

                    await System.Text.Json.JsonSerializer.SerializeAsync(context.Response.Body, okMessage);
                });
            });
        }
    }
}

package main

import (
	"fmt"
	"github.com/labstack/echo/v4"
	"io/ioutil"
	"net/http"
)

const Username = "YOUR_USERNAME"
const Password = "YOUR_PASSWORD"

type Callback struct {
	Links []Link `json:"_links"`
}

type Link struct {
	Href string `json:"href"`
	Method string `json:"method"`
	Rel string `json:"rel"`
}

func main() {
	echoServer := echo.New()
	client := &http.Client{}

	echoServer.POST("/job_listener", func(context echo.Context) error {
		callback := new(Callback)
		if err := context.Bind(callback); err != nil {
			return err
		}

		for _, link := range callback.Links {
			if link.Rel != "results" {
				continue
			}

			request, _ := http.NewRequest("GET",
				link.Href,
				nil,
			)

			request.Header.Add("Content-type", "application/json")
			request.SetBasicAuth(Username, Password)
			response, _ := client.Do(request)

			responseText, _ := ioutil.ReadAll(response.Body)
			fmt.Println(string(responseText))
		}
		return context.JSON(http.StatusOK, map[string]string { "status": "ok" })
	})

	echoServer.Logger.Fatal(echoServer.Start(":8080"))
}

package org.example;

import okhttp3.*;
import com.sun.net.httpserver.HttpServer;
import org.apache.commons.io.IOUtils;
import org.json.JSONArray;
import org.json.JSONObject;
import java.io.IOException;
import java.io.OutputStream;
import java.net.InetSocketAddress;
import java.nio.charset.StandardCharsets;
import java.util.Map;
import java.util.Objects;

public class Main implements Runnable {
    private static final String AUTHORIZATION_HEADER = "Authorization";
    public static final String USERNAME = "YOUR_USERNAME";
    public static final String PASSWORD = "YOUR_PASSWORD";

    public void run() {
        HttpServer server = null;
        try {
            server = HttpServer.create(new InetSocketAddress("0.0.0.0", 8080), 0);
        } catch (IOException exception) {
            exception.printStackTrace();
            System.exit(1);
        }

        Authenticator authenticator = (route, response) -> {
            String credential = Credentials.basic(USERNAME, PASSWORD);

            return response
                    .request()
                    .newBuilder()
                    .header(AUTHORIZATION_HEADER, credential)
                    .build();
        };

        var client = new OkHttpClient.Builder()
                .authenticator(authenticator)
                .build();

        server.createContext("/job_listener", exchange -> {
            var requestBody = IOUtils.toString(exchange.getRequestBody(), StandardCharsets.UTF_8);
            JSONObject requestJson = new JSONObject(requestBody);
            JSONArray links = requestJson.getJSONArray("_links");
            for (var link : links.toList()) {
                var linkMap = (Map<?, ?>)link;
                if (!Objects.equals(linkMap.get("rel"), "results")) {
                    continue;
                }

                var request = new Request.Builder()
                        .url((String) linkMap.get("href"))
                        .get()
                        .build();

                try (var response = client.newCall(request).execute()) {
                    assert response.body() != null;
                    System.out.println(response.body().string());
                } catch (Exception exception) {
                    System.out.println("Error: " + exception.getMessage());
                }
            }

            var responseJson = new JSONObject();
            responseJson.put("status", "ok");
            exchange.sendResponseHeaders(200, responseJson.toString().length());

            OutputStream responseBody = exchange.getResponseBody();
            responseBody.write(responseJson.toString().getBytes());
            responseBody.flush();
            responseBody.close();

            exchange.close();
        });
        server.setExecutor(null);
        server.start();
    }

    public static void main(String[] args) {
        new Thread(new Main()).start();
    }
}

import express from 'express'
import fetch from 'node-fetch';

const username = 'YOUR_USERNAME';
const password = 'YOUR_PASSWORD';

const app = express();
app.use(express.json());

app.post('/job_listener', async(request, response) => {
  for (const index in request.body._links) {
    const link = request.body._links[index];
    if (link.rel !== 'results') {
      continue;
    }

    const jobResultResponse = await fetch(link.href, {
      method: 'get',
      headers: {
        'Content-Type': 'application/json',
        'Authorization': 'Basic ' + Buffer.from(`${username}:${password}`).toString('base64'),
      }
    });

    console.log(await jobResultResponse.json());
  }

  response.send({status: 'ok'});
});

app.listen(8080);

输出

{  
   "created_at":"2019-10-01 00:00:01",
   "updated_at":"2019-10-01 00:00:15",
   "locale":null,
   "client_id":163,
   "user_agent_type":"desktop",
   "source":"google_shopping_search",
   "pages":1,
   "subdomain":"www",
   "status":"done",
   "start_page":1,
   "parse":0,
   "render":null,
   "priority":0,
   "ttl":0,
   "origin":"api",
   "persist":true,
   "id":"12345678900987654321",
   "callback_url":"http://your.callback.url/",
   "query":"adidas",
   "domain":"com",
   "limit":10,
   "geo_location":null,
   {...}
   "_links":[
      {  
         "href":"https://data.oxylabs.io/v1/queries/12345678900987654321",
         "method":"GET",
         "rel":"self"
      },
      {  
         "href":"https://data.oxylabs.io/v1/queries/12345678900987654321/results",
         "method":"GET",
         "rel":"results"
      }
   ],
}

检查任务状态

如果您在提交任务时提供了有效的回调 URL，我们将在任务完成时通过向指定的回调 URL 发送一个 JSON 有效载荷来通知您。该有效载荷将表明任务已完成并且其状态设置为 done.

但是，如果您提交任务时未使用 回调服务，您可以手动检查任务状态。从 href 字段中检索 URL，该字段位于提交任务后收到的响应消息的 rel:self 部分。用于检查任务状态的 URL 将类似于以下形式： http://data.oxylabs.io/v1/queries/12345678900987654321。查询此 URL 将返回任务信息，包括其当前 status.

端点

GET https://data.oxylabs.io/v1/queries/{id}

输入

curl --user "user:pass1" \
'http://data.oxylabs.io/v1/queries/12345678900987654321'

import requests
from pprint import pprint

# 从统计端点获取响应。
response = requests.request(
    method='GET',
    url='http://data.oxylabs.io/v1/queries/12345678900987654321',
    auth=('user', 'pass1'),
)

# 将美化的 JSON 响应打印到 stdout。
pprint(response.json())

<?php

$ch = curl_init();

curl_setopt($ch, CURLOPT_URL, "http://data.oxylabs.io/v1/queries/12345678900987654321");
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, "GET");
curl_setopt($ch, CURLOPT_USERPWD, "user" . ":" . "pass1");

$result = curl_exec($ch);
echo $result;

if (curl_errno($ch)) {
    echo 'Error:' . curl_error($ch);
}
curl_close ($ch);
?>

using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Net.Http.Json;
using System.Threading.Tasks;

namespace OxyApi
{
    class Program
    {
        static async Task Main()
        {
            const string JobId = "12345678900987654321";
            const string Username = "YOUR_USERNAME";
            const string Password = "YOUR_PASSWORD";

            var client = new HttpClient();

            Uri baseUri = new Uri("https://data.oxylabs.io");
            client.BaseAddress = baseUri;

            var requestMessage = new HttpRequestMessage(HttpMethod.Get, $"/v1/queries/{JobId}");

            var authenticationString = $"{Username}:{Password}";
            var base64EncodedAuthenticationString = Convert.ToBase64String(System.Text.ASCIIEncoding.UTF8.GetBytes(authenticationString));
            requestMessage.Headers.Add("Authorization", "Basic " + base64EncodedAuthenticationString);

            var response = await client.SendAsync(requestMessage);
            var contents = await response.Content.ReadAsStringAsync();

            Console.WriteLine(contents);
        }
    }
}

package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
)

func main() {
	const JobId = "12345678900987654321"
	const Username = "YOUR_USERNAME"
	const Password = "YOUR_PASSWORD"

	client := &http.Client{}
	request, _ := http.NewRequest("GET",
		fmt.Sprintf("https://data.oxylabs.io/v1/queries/%s", JobId),
		nil,
	)

	request.Header.Add("Content-type", "application/json")
	request.SetBasicAuth(Username, Password)
	response, _ := client.Do(request)

	responseText, _ := ioutil.ReadAll(response.Body)
	fmt.Println(string(responseText))
}

package org.example;

import okhttp3.*;

public class Main implements Runnable {
    private static final String AUTHORIZATION_HEADER = "Authorization";
    private static final String JOB_ID = "12345678900987654321";
    public static final String USERNAME = "YOUR_USERNAME";
    public static final String PASSWORD = "YOUR_PASSWORD";

    public void run() {
        Authenticator authenticator = (route, response) -> {
            String credential = Credentials.basic(USERNAME, PASSWORD);

            return response
                    .request()
                    .newBuilder()
                    .header(AUTHORIZATION_HEADER, credential)
                    .build();
        };

        var client = new OkHttpClient.Builder()
                .authenticator(authenticator)
                .build();

        var request = new Request.Builder()
                .url(String.format("https://data.oxylabs.io/v1/queries/%s", JOB_ID))
                .get()
                .build();

        try (var response = client.newCall(request).execute()) {
            assert response.body() != null;
            System.out.println(response.body().string());
        } catch (Exception exception) {
            System.out.println("Error: " + exception.getMessage());
        }

        System.exit(0);
    }

    public static void main(String[] args) {
        new Thread(new Main()).start();
    }
}

import fetch from 'node-fetch';

const jobId = '12345678900987654321';
const username = 'YOUR_USERNAME';
const password = 'YOUR_PASSWORD';
const response = await fetch(`https://data.oxylabs.io/v1/queries/${jobId}`, {
  method: 'get',
  headers: {
    'Content-Type': 'application/json',
    'Authorization': 'Basic ' + Buffer.from(`${username}:${password}`).toString('base64'),
  }
});

console.log(await response.json());

输出

任务完成后，API 将以 JSON 格式响应查询信息。任务状态将更改为 done，表示任务已完成。您可以通过查询提供的某个链接来检索内容。此外，响应还将包含任务最后更新时间的时间戳，以便您跟踪其处理时间。

{
    "client_id": 5,
    "context": [
        {
            "key": "results_language",
            "value": null
        },
        {
            "key": "safe_search",
            "value": null
        },
        {
            "key": "tbm",
            "value": null
        },
        {
            "key": "cr",
            "value": null
        },
        {
            "key": "filter",
            "value": null
        }
    ],
    "created_at": "2019-10-01 00:00:01",
    "domain": "com",
    "geo_location": null,
    "id": "7173957294344910849",
    "limit": 10,
    "locale": null,
    "pages": 1,
    "parse": false,
    "render": null,
    "query": "adidas",
    "source": "google_shopping_search",
    "start_page": 1,
    "status": "完成",
    "subdomain": "www",
    "updated_at": "2019-10-01 00:00:15",
    "user_agent_type": "desktop",
    "_links": [
        {
            "rel": "self",
            "href": "http://data.oxylabs.io/v1/queries/7173957294344910849",
            "method": "GET"
        },
        {
            "rel": "results",
            "href": "http://data.oxylabs.io/v1/queries/7173957294344910849/results",
            "method": "GET"
        },
        {
            "rel": "results-html",
            "href": "http://data.oxylabs.io/v1/queries/7173957294344910849/results?type=raw",
            "method": "GET"
        },
        {
            "rel": "results-parsed",
            "href": "http://data.oxylabs.io/v1/queries/7173957294344910849/results?type=parsed",
            "method": "GET"
        },
        {
            "rel": "results-parsed",
            "href": "http://data.oxylabs.io/v1/queries/7173957294344910849/results?type=png",
            "method": "GET"
        }
    ]
}

状态值

参数

说明

pending

任务仍在处理中，尚未完成。

done

任务已完成。您可以通过查询响应中 href 字段下提供的 URL 来检索结果，位于 rel:results 部分，例如： http://data.oxylabs.io/v1/queries/12345678900987654321/results.

faulted

任务存在问题，未能完成。对于任何此类 faulted 任务您不会被收费。

检索任务内容

一旦任务准备好检索，您可以使用响应中在 rel:results 部分下提供的 URL。该 URL 看起来如下： http://data.oxylabs.io/v1/queries/7173957294344910849/results.

端点

您可以使用以下端点检索不同类型的结果：

GET https://data.oxylabs.io/v1/queries/{job_id}/results

GET https://data.oxylabs.io/v1/queries/{job_id}/results?type=raw

GET https://data.oxylabs.io/v1/queries/{job_id}/results?type=parsed

GET https://data.oxylabs.io/v1/queries/{job_id}/results?type=png

GET https://data.oxylabs.io/v1/queries/{job_id}/results?type=xhr

GET https://data.oxylabs.io/v1/queries/{job_id}/results?type=markdown

您也可以检索多种结果类型在单个响应中，例如：

GET https://data.oxylabs.io/v1/queries/{job_id}/results?type=parsed,raw

输入

下面是演示如何使用 /results 端点的代码示例：

curl --user "user:pass1" \
'http://data.oxylabs.io/v1/queries/12345678900987654321/results'

import requests
from pprint import pprint

# 从统计端点获取响应。
response = requests.request(
    method='GET',
    url='http://data.oxylabs.io/v1/queries/12345678900987654321/results',
    auth=('user', 'pass1'),
)

# 将美化的 JSON 响应打印到 stdout。
pprint(response.json())

<?php

$ch = curl_init();

curl_setopt($ch, CURLOPT_URL, "http://data.oxylabs.io/v1/queries/12345678900987654321/results");
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, "GET");
curl_setopt($ch, CURLOPT_USERPWD, "user" . ":" . "pass1");

$result = curl_exec($ch);
echo $result;

if (curl_errno($ch)) {
    echo 'Error:' . curl_error($ch);
}
curl_close ($ch);
?>

using System;
using System.Net.Http;
using System.Threading.Tasks;

namespace OxyApi
{
    class Program
    {
        static async Task Main()
        {
            const string JobId = "12345678900987654321";
            const string Username = "YOUR_USERNAME";
            const string Password = "YOUR_PASSWORD";

            var client = new HttpClient();

            Uri baseUri = new Uri("https://data.oxylabs.io");
            client.BaseAddress = baseUri;

            var requestMessage = new HttpRequestMessage(HttpMethod.Get, $"/v1/queries/{JobId}/results");

            var authenticationString = $"{Username}:{Password}";
            var base64EncodedAuthenticationString = Convert.ToBase64String(System.Text.ASCIIEncoding.UTF8.GetBytes(authenticationString));
            requestMessage.Headers.Add("Authorization", "Basic " + base64EncodedAuthenticationString);

            var response = await client.SendAsync(requestMessage);
            var contents = await response.Content.ReadAsStringAsync();

            Console.WriteLine(contents);
        }
    }
}

package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
)

func main() {
	const JobId = "12345678900987654321"
	const Username = "YOUR_USERNAME"
	const Password = "YOUR_PASSWORD"

	client := &http.Client{}
	request, _ := http.NewRequest("GET",
		fmt.Sprintf("https://data.oxylabs.io/v1/queries/%s/results", JobId),
		nil,
	)

	request.Header.Add("Content-type", "application/json")
	request.SetBasicAuth(Username, Password)
	response, _ := client.Do(request)

	responseText, _ := ioutil.ReadAll(response.Body)
	fmt.Println(string(responseText))
}

package org.example;

import okhttp3.*;

public class Main implements Runnable {
    private static final String AUTHORIZATION_HEADER = "Authorization";
    private static final String JOB_ID = "12345678900987654321";
    public static final String USERNAME = "YOUR_USERNAME";
    public static final String PASSWORD = "YOUR_PASSWORD";

    public void run() {
        Authenticator authenticator = (route, response) -> {
            String credential = Credentials.basic(USERNAME, PASSWORD);

            return response
                    .request()
                    .newBuilder()
                    .header(AUTHORIZATION_HEADER, credential)
                    .build();
        };

        var client = new OkHttpClient.Builder()
                .authenticator(authenticator)
                .build();

        var request = new Request.Builder()
                .url(String.format("https://data.oxylabs.io/v1/queries/%s/results", JOB_ID))
                .get()
                .build();

        try (var response = client.newCall(request).execute()) {
            assert response.body() != null;
            System.out.println(response.body().string());
        } catch (Exception exception) {
            System.out.println("Error: " + exception.getMessage());
        }

        System.exit(0);
    }

    public static void main(String[] args) {
        new Thread(new Main()).start();
    }
}

import fetch from 'node-fetch';

const jobId = '12345678900987654321';
const username = 'YOUR_USERNAME';
const password = 'YOUR_PASSWORD';
const response = await fetch(`https://data.oxylabs.io/v1/queries/${jobId}/results`, {
  method: 'get',
  headers: {
    'Content-Type': 'application/json',
    'Authorization': 'Basic ' + Buffer.from(`${username}:${password}`).toString('base64'),
  }
});

console.log(await response.json());

输出

下表解释了基于 API 请求有效负载中包含的头部，默认和其他可用的结果类型。

渲染参数

解析参数

XHR 参数

默认输出

可用输出

html

html

html

html

true

xhr

html, xhr

html

true

parsed

html, xhr, parsed

png

png

html, png

true

parsed

html, parsed

html

true

parsed

html, parsed

png

true

png

html, parsed, png

下面是一个示例响应，来自 /results 端点的代码示例：

{
  "results": [
    {
      "content": "<!doctype html><html>
        内容      
      </html>",
      "created_at": "2019-10-01 00:00:01",
      "updated_at": "2019-10-01 00:00:15",
      "page": 1,
      "url": "https://www.google.com/search?q=adidas&hl=en&gl=US",
      "job_id": "12345678900987654321",
      "status_code": 200
    }
  ]
}

结果可以通过设置回调服务来自动检索，而无需周期性地检查任务状态。为此，在提交任务时指定能够接受传入 HTTP(S) 请求的服务器的 URL。当我们的系统完成任务时，它将 POST 向提供的 URL 发送一个 JSON 有效载荷，回调服务将按 回调实现示例.

批量查询

Scraper APIs 支持在单个批量请求中提交最多 5,000 个 query 或 url 参数值。

Batch requests are currently not supported 用于 chatgpt 和 perplexity 来源。

端点

POST https://data.oxylabs.io/v1/queries/batch

系统将把每个 query 或 url 作为单独的任务处理。如果您提供回调 URL，则每个关键字都会收到单独的回调。否则，我们的初始响应将包含所有关键字的任务 id信息。例如，如果您发送了 50 个关键字，我们将返回 50 个唯一的任务 id。

重要： 使用 /batch 端点时，您只能提交 query或 url参数值的列表（取决于您使用的 source ）。所有其他参数应为单一值。

输入

您需要将查询参数作为 JSON 有效负载发布。以下是提交批量任务的方法：

curl --user "user:pass1" \
'https://data.oxylabs.io/v1/queries/batch' \
-H 'Content-Type: application/json' \
-d '@keywords.json'

import requests
import json
from pprint import pprint


# 从文件获取有效负载。
with open('keywords.json', 'r') as f:
    payload = json.loads(f.read())

response = requests.request(
    'POST',
    'https://data.oxylabs.io/v1/queries/batch',
    auth=('user', 'pass1'),
    json=payload,
)

# 打印美化的响应。
pprint(response.json())

<?php

$paramsFile = file_get_contents(realpath("keywords.json"));
$ch = curl_init();

curl_setopt($ch, CURLOPT_URL, "https://data.oxylabs.io/v1/queries/batch");
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, $paramsFile);
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_USERPWD, "user" . ":" . "pass1");

$headers = array();
$headers[] = "Content-Type: application/json";
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);

$result = curl_exec($ch);
echo $result;

if (curl_errno($ch)) {
    echo 'Error:' . curl_error($ch);
}
curl_close ($ch);
?>

using System;
using System.IO;
using System.Net.Http;
using System.Text;
using System.Threading.Tasks;

namespace OxyApi
{
    class Program
    {
        static async Task Main()
        {
            const string Username = "YOUR_USERNAME";
            const string Password = "YOUR_PASSWORD";


            var content = File.ReadAllText(@"C:\path\to\keywords.json");
                
            var client = new HttpClient();

            var requestMessage = new HttpRequestMessage(HttpMethod.Post, new Uri("https://data.oxylabs.io/v1/queries/batch"));
            requestMessage.Content = new StringContent(content, Encoding.UTF8, "application/json");


            var authenticationString = $"{Username}:{Password}";
            var base64EncodedAuthenticationString = Convert.ToBase64String(ASCIIEncoding.UTF8.GetBytes(authenticationString));
            requestMessage.Headers.Add("Authorization", "Basic " + base64EncodedAuthenticationString);

            var response = await client.SendAsync(requestMessage);
            var contents = await response.Content.ReadAsStringAsync();

            Console.WriteLine(contents);
        }
    }
}

package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
)

func main() {
	const Username = "YOUR_USERNAME"
	const Password = "YOUR_PASSWORD"

	content, err := os.ReadFile("keywords.json")
	if err != nil {
		panic(err)
	}

	client := &http.Client{}
	request, _ := http.NewRequest("POST",
		"https://data.oxylabs.io/v1/queries/batch",
		bytes.NewBuffer(content),
	)

	request.Header.Add("Content-type", "application/json")
	request.SetBasicAuth(Username, Password)
	response, _ := client.Do(request)

	responseText, _ := ioutil.ReadAll(response.Body)
	fmt.Println(string(responseText))
}

import okhttp3.*;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;

public class Main implements Runnable {
    private static final String AUTHORIZATION_HEADER = "Authorization";
    public static final String USERNAME = "YOUR_USERNAME";
    public static final String PASSWORD = "YOUR_PASSWORD";

    public void run() {
        Path filePath = Path.of("/path/to/keywords.json");
        String jsonContent = null;
        try {
            jsonContent = Files.readString(filePath);
        } catch (IOException e) {
            throw new RuntimeException(e);
        }

        Authenticator authenticator = (route, response) -> {
            String credential = Credentials.basic(USERNAME, PASSWORD);

            return response
                    .request()
                    .newBuilder()
                    .header(AUTHORIZATION_HEADER, credential)
                    .build();
        };

        var client = new OkHttpClient.Builder()
                .authenticator(authenticator)
                .build();

        var mediaType = MediaType.parse("application/json; charset=utf-8");
        var body = RequestBody.create(jsonContent, mediaType);
        var request = new Request.Builder()
                .url("https://data.oxylabs.io/v1/queries/batch")
                .post(body)
                .build();

        try (var response = client.newCall(request).execute()) {
            assert response.body() != null;
            System.out.println(response.body().string());
        } catch (Exception exception) {
            System.out.println("Error: " + exception.getMessage());
        }

        System.exit(0);
    }

    public static void main(String[] args) {
        new Thread(new Main()).start();
    }
}

import fetch from 'node-fetch';
import fs from 'fs'

const username = 'YOUR_USERNAME';
const password = 'YOUR_PASSWORD';

const payload = fs.readFileSync('keywords.json').toString();

const response = await fetch('https://data.oxylabs.io/v1/queries/batch', {
  method: 'post',
  body: payload,
  headers: {
    'Content-Type': 'application/json',
    'Authorization': 'Basic ' + Buffer.from(`${username}:${password}`).toString('base64'),
  }
});

console.log(await response.json());

您可能注意到上面的代码示例没有说明 JSON 有何格式，并且引用了一个预先制作的 JSON 文件。下面是 keywords.json 文件，包含多个 query 参数值：

{  
   "query":[  
      "adidas",
      "nike",
      "reebok"
   ],
   "source": "google_shopping_search",
   "domain": "com",
   "callback_url": "https://your.callback.url"
}

...下面是一个 keywords.json 批量输入文件，包含多个 URL：

{  
   "url":[  
      "https://example.com/url1.html",
      "https://example.com/url2.html",
      "https://example.com/url3.html"
   ],
   "source": "universal",
   "callback_url": "https://your.callback.url"
}

输出

该 API 将返回一个 JSON 对象，包含为每个创建的任务生成的任务信息。响应将类似于：

{
  "queries": [
    {
      "callback_url": "https://your.callback.url",
      {...}
      "created_at": "2019-10-01 00:00:01",
      "domain": "com",
      "id": "12345678900987654321",
      {...}
      "query": "adidas",
      "source": "google_shopping_search",
      {...}
          "rel": "results",
          "href": "http://data.oxylabs.io/v1/queries/12345678900987654321/results",
          "method": "GET"
        }
      ]
    },
    {
      "callback_url": "https://your.callback.url",
      {...}
      "created_at": "2019-10-01 00:00:01",
      "domain": "com",
      "id": "12345678901234567890",
      {...}
      "query": "nike",
      "source": "google_shopping_search",
      {...}
          "rel": "results",
          "href": "http://data.oxylabs.io/v1/queries/12345678901234567890/results",
          "method": "GET"
        }
      ]
    },
    {
      "callback_url": "https://your.callback.url",
      {...}
      "created_at": "2019-10-01 00:00:01",
      "domain": "com",
      "id": "01234567899876543210",
	  {...}
      "query": "reebok",
      "source": "google_shopping_search",
	  {...}
          "rel": "results",
          "href": "http://data.oxylabs.io/v1/queries/01234567899876543210/results",
          "method": "GET"
        }
      ]
    }
  ]
}

获取回调器 IP 地址列表

您可能想把向您发送回调消息的 IP 加入白名单，或为其他用途获取这些 IP 列表。您可以通过 GET访问此端点：

端点

GET https://data.oxylabs.io/v1/info/callbacker_ips

输入

下面的代码示例演示如何访问 /callbacker_ips 端点的代码示例：

curl --user "user:pass1" \
'https://data.oxylabs.io/v1/info/callbacker_ips'

import requests
from pprint import pprint

# 从回调 IP 端点获取响应。
response = requests.request(
    method='GET',
    url='https://data.oxylabs.io/v1/info/callbacker_ips',
    auth=('user', 'pass1'),
)

# 将美化的 JSON 响应打印到 stdout。
pprint(response.json())

<?php

$ch = curl_init();

curl_setopt($ch, CURLOPT_URL, "https://data.oxylabs.io/v1/info/callbacker_ips");
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, "GET");
curl_setopt($ch, CURLOPT_USERPWD, "user" . ":" . "pass1");

$result = curl_exec($ch);
echo $result;

if (curl_errno($ch)) {
    echo 'Error:' . curl_error($ch);
}
curl_close ($ch);
?>

using System;
using System.Net.Http;
using System.Threading.Tasks;

namespace OxyApi
{
    class Program
    {
        static async Task Main()
        {
            const string Username = "YOUR_USERNAME";
            const string Password = "YOUR_PASSWORD";

            var client = new HttpClient();

            Uri baseUri = new Uri("https://data.oxylabs.io");
            client.BaseAddress = baseUri;

            var requestMessage = new HttpRequestMessage(HttpMethod.Get, "/v1/info/callbacker_ips");

            var authenticationString = $"{Username}:{Password}";
            var base64EncodedAuthenticationString = Convert.ToBase64String(System.Text.ASCIIEncoding.UTF8.GetBytes(authenticationString));
            requestMessage.Headers.Add("Authorization", "Basic " + base64EncodedAuthenticationString);

            var response = await client.SendAsync(requestMessage);
            var contents = await response.Content.ReadAsStringAsync();

            Console.WriteLine(contents);
        }
    }
}

package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
)

func main() {
	const Username = "YOUR_USERNAME"
	const Password = "YOUR_PASSWORD"

	client := &http.Client{}
	request, _ := http.NewRequest("GET",
		"https://data.oxylabs.io/v1/info/callbacker_ips",
		nil,
	)

	request.Header.Add("Content-type", "application/json")
	request.SetBasicAuth(Username, Password)
	response, _ := client.Do(request)

	responseText, _ := ioutil.ReadAll(response.Body)
	fmt.Println(string(responseText))
}

package org.example;

import okhttp3.*;

public class Main implements Runnable {
    private static final String AUTHORIZATION_HEADER = "Authorization";
    public static final String USERNAME = "YOUR_USERNAME";
    public static final String PASSWORD = "YOUR_PASSWORD";

    public void run() {
        Authenticator authenticator = (route, response) -> {
            String credential = Credentials.basic(USERNAME, PASSWORD);

            return response
                    .request()
                    .newBuilder()
                    .header(AUTHORIZATION_HEADER, credential)
                    .build();
        };

        var client = new OkHttpClient.Builder()
                .authenticator(authenticator)
                .build();

        var request = new Request.Builder()
                .url("https://data.oxylabs.io/v1/info/callbacker_ips")
                .get()
                .build();

        try (var response = client.newCall(request).execute()) {
            assert response.body() != null;
            System.out.println(response.body().string());
        } catch (Exception exception) {
            System.out.println("Error: " + exception.getMessage());
        }

        System.exit(0);
    }

    public static void main(String[] args) {
        new Thread(new Main()).start();
    }
}

import fetch from 'node-fetch';

const username = 'YOUR_USERNAME';
const password = 'YOUR_PASSWORD';
const response = await fetch('https://data.oxylabs.io/v1/info/callbacker_ips', {
  method: 'get',
  headers: {
    'Content-Type': 'application/json',
    'Authorization': 'Basic ' + Buffer.from(`${username}:${password}`).toString('base64'),
  }
});

console.log(await response.json());

输出

该 API 将返回向您系统发送回调请求的 IP 列表：

{
    "ips": [
        "x.x.x.x",
        "y.y.y.y"
    ]
}

Scheduler

Scheduler 是一个可用于安排定期爬取任务的服务。

它扩展了 Push-Pull 集成的功能，且最好与 Cloud integration 功能一起使用。

访问此页面以了解如何使用 Scheduler 功能：

任务管家

上一页Realtime 下一页Proxy Endpoint

最后更新于13天前

这有帮助吗？

早上好

单个任务

端点

输入

输出

数据字典

回调

输入

输出

检查任务状态

端点

输入

输出

状态值

检索任务内容

端点

输入

输出

批量查询

端点

输入

输出

获取回调器 IP 地址列表

端点

输入

输出

Scheduler