Push-Pull

了解 Oxylabs 网页爬虫API 的 Push-Pull 集成方法。提交任务后，稍后使用 JSON 格式数据轮询结果端点。

Push-Pull 是我们推荐的集成方式，用于可靠处理大量数据。

访问 Oxylabs GitHub 仓库，查看完整可运行示例： Python 中的 Push-Pull 集成.

Push-Pull 是一种异步集成方式。提交任务后，您将立即收到一个 JSON 包含所有任务详情的响应，其中包括任务参数、ID 以及用于下载结果和检查状态的 URL。任务处理完成后，我们将通过一个 JSON 发送到您服务器的负载，如果您提供了一个回调 URL。结果可供检索的时间为 至少 24 小时 完成后。

使用 Push-Pull，您可以将结果直接上传到您的 云存储 (Google Cloud Storage, AWS S3, Alibaba Cloud OSS，或其他兼容 S3 的存储)。

如果您不想为接收回调通知搭建服务，您可以定期简单地检索结果（轮询).

您还可以使用以下工具了解 Push-Pull 的工作方式： Postman.

单个任务

端点

此端点只接受单个 查询 或 URL 值。

POST https://data.oxylabs.io/v1/queries

输入

请按照下面的示例，在 JSON 负载中提供任务参数。Python 和 PHP 示例包含注释以便说明。

curl --user "USERNAME:PASSWORD" \
'https://data.oxylabs.io/v1/queries' \
-H "Content-Type: application/json" \
 -d '{"source": "universal", "url": "https://www.example.com", "geo_location": "United States", "callback_url": "https://your.callback.url", "storage_type": "s3", "storage_url": "s3://your.storage.bucket.url"}'

import requests
from pprint import pprint


# Structure payload.
payload = {
    "source": "universal", # Source you choose e.g. "universal"
    "url": "https://www.example.com", # Check the docs of the specific source to see whether you should use "url" or "query"
    "geo_location": "United States", # Some sources accept zip codes or coordinates
    #"render" : "html", # Uncomment if you want to render JavaScript within the page
    #"render" : "png", # Uncomment if you want to take a screenshot of a scraped web page
    #"parse" : True, # Check what sources support parsed data
    "callback_url": "https://your.callback.url", # Required if you use the callback listener
    "storage_type": "s3", 
    "storage_url": "s3://your.storage.bucket.url"
}

# Get response.
response = requests.request(
    'POST',
    'https://data.oxylabs.io/v1/queries',
    auth=('YOUR_USERNAME', 'YOUR_PASSWORD'), #Your credentials go here
    json=payload,
)

# Print prettified response to stdout.
pprint(response.json())

<?php

$params = array(
    'source' => 'universal', //Source you choose e.g. "universal"
    'url' => 'https://www.example.com', // Check the docs of the specific source to see whether you should use "url" or "query"
    'geo_location' => 'United States', //Some sources accept zip codes or coordinates
    //'render' => 'html', // Uncomment if you want to render JavaScript within the page
    //'render' => 'png', // Uncomment if you want to take a screenshot of a scraped web page
    //'parse' => 'TRUE', // Check what sources support parsed data
    'callback_url' => 'https://your.callback.url', //Required if you use the callback listener
    'storage_type' => 's3',
    'storage_url' => 's3://your.storage.bucket.url'
);

$ch = curl_init();

curl_setopt($ch, CURLOPT_URL, "https://data.oxylabs.io/v1/queries");
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($params));
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_USERPWD, "YOUR_USERNAME" . ":" . "YOUR_PASSWORD"); //Your credentials go here

$headers = array();
$headers[] = "Content-Type: application/json";
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);

$result = curl_exec($ch);
echo $result;

if (curl_errno($ch)) {
    echo 'Error:' . curl_error($ch);
}
curl_close($ch);
?>

using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Net.Http.Json;
using System.Threading.Tasks;

namespace OxyApi
{
    class Program
    {
        static async Task Main()
        {
            const string Username = "YOUR_USERNAME";
            const string Password = "YOUR_PASSWORD";

            var parameters = new Dictionary<string, string>()
            {
                { "source", "universal" },
                { "url", "https://example.com" },
                { "geo_location", "United States" },
                { "callback_url", "https://your.callback.url" },
            };


            var client = new HttpClient();

            Uri baseUri = new Uri("https://data.oxylabs.io");
            client.BaseAddress = baseUri;

            var requestMessage = new HttpRequestMessage(HttpMethod.Post, "/v1/queries");
            requestMessage.Content = JsonContent.Create(parameters);

            var authenticationString = $"{Username}:{Password}";
            var base64EncodedAuthenticationString = Convert.ToBase64String(System.Text.ASCIIEncoding.UTF8.GetBytes(authenticationString));
            requestMessage.Headers.Add("Authorization", "Basic " + base64EncodedAuthenticationString);

            var response = await client.SendAsync(requestMessage);
            var contents = await response.Content.ReadAsStringAsync();

            Console.WriteLine(contents);
        }
    }
}

package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
)

func main() {
	const Username = "YOUR_USERNAME"
	const Password = "YOUR_PASSWORD"

	payload := map[string]string{
		"source": "universal",
		"url": "https://example.com",
		"geo_location": "United States",
		"callback_url": "https://your.callback.url",
	}

	jsonValue, _ := json.Marshal(payload)

	client := &http.Client{}
	request, _ := http.NewRequest("POST",
		"https://data.oxylabs.io/v1/queries",
		bytes.NewBuffer(jsonValue),
	)

	request.Header.Add("Content-type", "application/json")
	request.SetBasicAuth(Username, Password)
	response, _ := client.Do(request)

	responseText, _ := ioutil.ReadAll(response.Body)
	fmt.Println(string(responseText))
}

package org.example;

import okhttp3.*;
import org.json.JSONObject;

public class Main implements Runnable {
    private static final String AUTHORIZATION_HEADER = "Authorization";
    public static final String USERNAME = "YOUR_USERNAME";
    public static final String PASSWORD = "YOUR_PASSWORD";

    public void run() {
        JSONObject jsonObject = new JSONObject();
        jsonObject.put("source", "universal");
        jsonObject.put("url", "https://example.com");
        jsonObject.put("geo_location", "United States");
        jsonObject.put("callback_url", "https://your.callback.url");

        Authenticator authenticator = (route, response) -> {
            String credential = Credentials.basic(USERNAME, PASSWORD);

            return response
                    .request()
                    .newBuilder()
                    .header(AUTHORIZATION_HEADER, credential)
                    .build();
        };

        var client = new OkHttpClient.Builder()
                .authenticator(authenticator)
                .build();

        var mediaType = MediaType.parse("application/json; charset=utf-8");
        var body = RequestBody.create(jsonObject.toString(), mediaType);
        var request = new Request.Builder()
                .url("https://data.oxylabs.io/v1/queries")
                .post(body)
                .build();

        try (var response = client.newCall(request).execute()) {
            assert response.body() != null;
            System.out.println(response.body().string());
        } catch (Exception exception) {
            System.out.println("Error: " + exception.getMessage());
        }

        System.exit(0);
    }

    public static void main(String[] args) {
        new Thread(new Main()).start();
    }
}

import fetch from 'node-fetch';

const username = 'YOUR_USERNAME';
const password = 'YOUR_PASSWORD';
const body = {
  source: 'universal',
  url: 'https://www.example.com',
  geo_location: 'United States',
  callback_url: 'https://your.callback.url',
};
const response = await fetch('https://data.oxylabs.io/v1/queries', {
  method: 'post',
  body: JSON.stringify(body),
  headers: {
    'Content-Type': 'application/json',
    'Authorization': 'Basic ' + Buffer.from(`${username}:${password}`).toString('base64'),
  }
});

console.log(await response.json());

输出

API 将返回一个包含任务信息的 JSON，类似如下：

{
  "callback_url": "https://your.callback.url",
  "client_id": 5,
  "context": [
    {...}
  ],
  "created_at": "2026-09-03 00:00:01",
  "domain": "com",
  "geo_location": "United States",
  "id": "12345678900987654321",
  "limit": 10,
  "locale": null,
  "pages": 1,
  "parse": false,
  "parser_type": null,
  "parser_preset": null,
  "parsing_instructions": null,
  "browser_instructions": null,
  "render": null,
  "xhr": false,
  "markdown": false,
  "url": "https://www.example.com",
  "query": "",
  "source": "universal",
  "start_page": 1,
  "status": "pending",
  "storage_type": "s3",
  "storage_url": "YOUR_BUCKET_NAME/12345678900987654321.json",
  "aggregate_name": null,
  "subdomain": "www",
  "content_encoding": "utf-8",
  "updated_at": "2026-09-03 00:00:01",
  "user_agent_type": "desktop",
  "session_info": null,
  "statuses": [],
  "client_notes": null,
  "_links": [
    {
      "rel": "self",
      "href": "http://data.oxylabs.io/v1/queries/12345678900987654321",
      "method": "GET"
    },
    {
      "rel": "results",
      "href": "http://data.oxylabs.io/v1/queries/12345678900987654321/results",
      "method": "GET"
    },
    {
      "rel": "results-content",
      "href_list": [
        "http://data.oxylabs.io/v1/queries/12345678900987654321/results/1/content"
      ],
      "method": "GET"
    },
    {
      "rel": "results-html",
      "href": "http://data.oxylabs.io/v1/queries/12345678900987654321/results?type=raw",
      "method": "GET"
    },
    {
      "rel": "results-content-html",
      "href_list": [
        "http://data.oxylabs.io/v1/queries/12345678900987654321/results/1/content?type=raw"
      ],
      "method": "GET"
    }
  ]
}

数据字典

下表描述了任务响应中返回的字段。对于您可发送的输入参数，请参阅您所使用来源的文档页面。

键

描述

类型

created_at

任务创建时间。

字符串

client_id

与发起请求的客户端用户名相关联的数字 ID。

字符串

client_notes

客户端发送任务时提交的备注。

字符串

content_encoding

返回内容的编码。在以下情况下，请在负载中设置它： 下载图片；它会在任务响应中原样返回。

字符串

id

任务的唯一 ID。

字符串

statuses

爬取或解析任务的状态代码。您可以在这里.

整数

status

任务状态。 pending 表示任务仍在处理中。 done 表示我们已完成任务。 faulted 表示我们在尝试完成任务时遇到错误，并已放弃。

字符串

subdomain

网站的子域名。

字符串

updated_at

任务最后更新时间。对于已完成的任务（status is done 或 faulted），此时间表示任务完成的时间。

字符串

_links

与所提供输入相关的链接列表。

JSON 数组

_links:rel

链接类型。 self URL 包含任务的元数据，而 results URL 包含任务结果。

字符串

_links:href

资源的 URL。

字符串

_links:href_list

资源 URL 列表，每页一个。返回于 而不是 href 在其 rel 以 results-content开头的条目上。参见 检索任务内容.

JSON 数组

_links:method

与给定 URL 交互时应使用的 HTTP 方法。

字符串

回调

回调是一个 POST 我们发送到您机器上的请求，用于通知数据提取任务已完成，并提供一个用于下载爬取内容的 URL。这意味着您不需要 手动检查任务状态 。一旦数据到达，我们会通知您，您现在只需要 检索它.

输入

# This is a simple Sanic web server with a route listening for callbacks on localhost:8080.
# It will print job results to stdout.
import requests
from pprint import pprint
from sanic import Sanic, response


AUTH_TUPLE = ('user', 'pass1')

app = Sanic()


# Define /job_listener endpoint that accepts POST requests.
@app.route('/job_listener', methods=['POST'])
async def job_listener(request):
    try:
        res = request.json
        links = res.get('_links', [])
        for link in links:
            if link['rel'] == 'results':
                # Sanic is async, but requests are synchronous, to fully take
                # advantage of Sanic, use aiohttp.
                res_response = requests.request(
                    method='GET',
                    url=link['href'],
                    auth=AUTH_TUPLE,
                )
                pprint(res_response.json())
                break
    except Exception as e:
        print("Listener exception: {}".format(e))
    return response.json(status=200, body={'status': 'ok'})


if __name__ == '__main__':
    app.run(host='0.0.0.0', port=8080)

<?php
$stdout = fopen('php://stdout', 'w');

if (isset($_POST)) {
    $result = array_merge($_POST, (array) json_decode(file_get_contents('php://input')));

    $ch = curl_init();

    curl_setopt($ch, CURLOPT_URL, "https://data.oxylabs.io/v1/queries/".$result['id'].'/results');
    curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
    curl_setopt($ch, CURLOPT_CUSTOMREQUEST, "GET");
    curl_setopt($ch, CURLOPT_USERPWD, "user" . ":" . "pass1");

    $result = curl_exec($ch);
    fwrite($stdout, $result);

    if (curl_errno($ch)) {
        echo 'Error:' . curl_error($ch);
    }
    curl_close($ch);
}
?>

using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using System;
using System.Collections.Generic;
using System.Net.Http;

namespace OxyApiWeb
{
    public class Callback
    {
        public Link[] _links { get; set; }
    }

    public class Link
    {
        public string rel { get; set; }
        public string href { get; set; }
    }

    public class Startup
    {
        private const string USERNAME = "YOUR_USERNAME";
        private const string PASSWORD = "YOUR_PASSWORD";

        public Startup(IConfiguration configuration)
        {
            Configuration = configuration;

            client = new HttpClient();
        }

        public IConfiguration Configuration { get; }

        private HttpClient client;

        public void ConfigureServices(IServiceCollection services)
        {
            services.AddControllers();
        }

        public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
        {
            if (env.IsDevelopment())
            {
                app.UseDeveloperExceptionPage();
            }

            app.UseRouting();

            app.UseAuthorization();

            app.UseEndpoints(endpoints =>
            {
                endpoints.MapPost("/job_listener", async context =>
                {
                    var callback = await System.Text.Json.JsonSerializer.DeserializeAsync<Callback>(context.Request.Body);

                    foreach (var link in callback._links)
                    {
                        if (link.rel != "results")
                        {
                            continue;
                        }

                        var requestMessage = new HttpRequestMessage(HttpMethod.Get, new Uri(link.href));

                        var authenticationString = $"{USERNAME}:{PASSWORD}";
                        var base64EncodedAuthenticationString = Convert.ToBase64String(System.Text.ASCIIEncoding.UTF8.GetBytes(authenticationString));
                        requestMessage.Headers.Add("Authorization", "Basic " + base64EncodedAuthenticationString);

                        var response = await client.SendAsync(requestMessage);
                        var contents = await response.Content.ReadAsStringAsync();

                        Console.WriteLine(contents);

                    }

                    var okMessage = new Dictionary<string, string>()
                    {
                        { "message", "ok" }
                    };

                    await System.Text.Json.JsonSerializer.SerializeAsync(context.Response.Body, okMessage);
                });
            });
        }
    }
}

package main

import (
	"fmt"
	"github.com/labstack/echo/v4"
	"io/ioutil"
	"net/http"
)

const Username = "YOUR_USERNAME"
const Password = "YOUR_PASSWORD"

type Callback struct {
	Links []Link `json:"_links"`
}

type Link struct {
	Href string `json:"href"`
	Method string `json:"method"`
	Rel string `json:"rel"`
}

func main() {
	echoServer := echo.New()
	client := &http.Client{}

	echoServer.POST("/job_listener", func(context echo.Context) error {
		callback := new(Callback)
		if err := context.Bind(callback); err != nil {
			return err
		}

		for _, link := range callback.Links {
			if link.Rel != "results" {
				continue
			}

			request, _ := http.NewRequest("GET",
				link.Href,
				nil,
			)

			request.Header.Add("Content-type", "application/json")
			request.SetBasicAuth(Username, Password)
			response, _ := client.Do(request)

			responseText, _ := ioutil.ReadAll(response.Body)
			fmt.Println(string(responseText))
		}
		return context.JSON(http.StatusOK, map[string]string { "status": "ok" })
	})

	echoServer.Logger.Fatal(echoServer.Start(":8080"))
}

package org.example;

import okhttp3.*;
import com.sun.net.httpserver.HttpServer;
import org.apache.commons.io.IOUtils;
import org.json.JSONArray;
import org.json.JSONObject;
import java.io.IOException;
import java.io.OutputStream;
import java.net.InetSocketAddress;
import java.nio.charset.StandardCharsets;
import java.util.Map;
import java.util.Objects;

public class Main implements Runnable {
    private static final String AUTHORIZATION_HEADER = "Authorization";
    public static final String USERNAME = "YOUR_USERNAME";
    public static final String PASSWORD = "YOUR_PASSWORD";

    public void run() {
        HttpServer server = null;
        try {
            server = HttpServer.create(new InetSocketAddress("0.0.0.0", 8080), 0);
        } catch (IOException exception) {
            exception.printStackTrace();
            System.exit(1);
        }

        Authenticator authenticator = (route, response) -> {
            String credential = Credentials.basic(USERNAME, PASSWORD);

            return response
                    .request()
                    .newBuilder()
                    .header(AUTHORIZATION_HEADER, credential)
                    .build();
        };

        var client = new OkHttpClient.Builder()
                .authenticator(authenticator)
                .build();

        server.createContext("/job_listener", exchange -> {
            var requestBody = IOUtils.toString(exchange.getRequestBody(), StandardCharsets.UTF_8);
            JSONObject requestJson = new JSONObject(requestBody);
            JSONArray links = requestJson.getJSONArray("_links");
            for (var link : links.toList()) {
                var linkMap = (Map<?, ?>)link;
                if (!Objects.equals(linkMap.get("rel"), "results")) {
                    continue;
                }

                var request = new Request.Builder()
                        .url((String) linkMap.get("href"))
                        .get()
                        .build();

                try (var response = client.newCall(request).execute()) {
                    assert response.body() != null;
                    System.out.println(response.body().string());
                } catch (Exception exception) {
                    System.out.println("Error: " + exception.getMessage());
                }
            }

            var responseJson = new JSONObject();
            responseJson.put("status", "ok");
            exchange.sendResponseHeaders(200, responseJson.toString().length());

            OutputStream responseBody = exchange.getResponseBody();
            responseBody.write(responseJson.toString().getBytes());
            responseBody.flush();
            responseBody.close();

            exchange.close();
        });
        server.setExecutor(null);
        server.start();
    }

    public static void main(String[] args) {
        new Thread(new Main()).start();
    }
}

import express from 'express'
import fetch from 'node-fetch';

const username = 'YOUR_USERNAME';
const password = 'YOUR_PASSWORD';

const app = express();
app.use(express.json());

app.post('/job_listener', async(request, response) => {
  for (const index in request.body._links) {
    const link = request.body._links[index];
    if (link.rel !== 'results') {
      continue;
    }

    const jobResultResponse = await fetch(link.href, {
      method: 'get',
      headers: {
        'Content-Type': 'application/json',
        'Authorization': 'Basic ' + Buffer.from(`${username}:${password}`).toString('base64'),
      }
    });

    console.log(await jobResultResponse.json());
  }

  response.send({status: 'ok'});
});

app.listen(8080);

输出

{
  "callback_url": "https://your.callback.url",
  "client_id": 5,
  "context": [
    {...}
  ],
  "created_at": "2026-09-03 00:00:01",
  "domain": "com",
  "geo_location": "United States",
  "id": "12345678900987654321",
  "limit": 10,
  "locale": null,
  "pages": 1,
  "parse": false,
  "parser_type": null,
  "parser_preset": null,
  "parsing_instructions": null,
  "browser_instructions": null,
  "render": null,
  "xhr": false,
  "markdown": false,
  "url": "https://www.example.com",
  "query": "",
  "source": "universal",
  "start_page": 1,
  "status": "done",
  "storage_type": null,
  "storage_url": null,
  "aggregate_name": null,
  "subdomain": "www",
  "content_encoding": "utf-8",
  "updated_at": "2026-09-03 00:00:15",
  "user_agent_type": "desktop",
  "session_info": null,
  "statuses": [],
  "client_notes": null,
  "_links": [
    {
      "rel": "self",
      "href": "http://data.oxylabs.io/v1/queries/12345678900987654321",
      "method": "GET"
    },
    {
      "rel": "results",
      "href": "http://data.oxylabs.io/v1/queries/12345678900987654321/results",
      "method": "GET"
    },
    {
      "rel": "results-content",
      "href_list": [
        "http://data.oxylabs.io/v1/queries/12345678900987654321/results/1/content"
      ],
      "method": "GET"
    },
    {
      "rel": "results-html",
      "href": "http://data.oxylabs.io/v1/queries/12345678900987654321/results?type=raw",
      "method": "GET"
    },
    {
      "rel": "results-content-html",
      "href_list": [
        "http://data.oxylabs.io/v1/queries/12345678900987654321/results/1/content?type=raw"
      ],
      "method": "GET"
    }
  ]
}

检查任务状态

如果您在提交任务时提供了有效的回调 URL，我们将在任务完成时发送 JSON 负载到指定的回调 URL。该负载将表明任务已完成，状态设为 done.

但是，如果您在提交任务时未使用 回调服务，您可以手动检查任务状态。从 href 字段中的 rel:self 部分获取 URL。该 URL 将类似于以下内容： http://data.oxylabs.io/v1/queries/12345678900987654321。查询此 URL 将返回任务信息，包括其当前 status.

端点

GET https://data.oxylabs.io/v1/queries/{id}

输入

curl --user "USERNAME:PASSWORD" \
'https://data.oxylabs.io/v1/queries/12345678900987654321'

import requests
from pprint import pprint

# Get response from stats endpoint.
response = requests.request(
    method='GET',
    url='https://data.oxylabs.io/v1/queries/12345678900987654321',
    auth=('user', 'pass1'),
)

# Print prettified JSON response to stdout.
pprint(response.json())

<?php

$ch = curl_init();

curl_setopt($ch, CURLOPT_URL, "https://data.oxylabs.io/v1/queries/12345678900987654321");
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, "GET");
curl_setopt($ch, CURLOPT_USERPWD, "user" . ":" . "pass1");

$result = curl_exec($ch);
echo $result;

if (curl_errno($ch)) {
    echo 'Error:' . curl_error($ch);
}
curl_close($ch);
?>

using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Net.Http.Json;
using System.Threading.Tasks;

namespace OxyApi
{
    class Program
    {
        static async Task Main()
        {
            const string JobId = "12345678900987654321";
            const string Username = "YOUR_USERNAME";
            const string Password = "YOUR_PASSWORD";

            var client = new HttpClient();

            Uri baseUri = new Uri("https://data.oxylabs.io");
            client.BaseAddress = baseUri;

            var requestMessage = new HttpRequestMessage(HttpMethod.Get, $"/v1/queries/{JobId}");

            var authenticationString = $"{Username}:{Password}";
            var base64EncodedAuthenticationString = Convert.ToBase64String(System.Text.ASCIIEncoding.UTF8.GetBytes(authenticationString));
            requestMessage.Headers.Add("Authorization", "Basic " + base64EncodedAuthenticationString);

            var response = await client.SendAsync(requestMessage);
            var contents = await response.Content.ReadAsStringAsync();

            Console.WriteLine(contents);
        }
    }
}

package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
)

func main() {
	const JobId = "12345678900987654321"
	const Username = "YOUR_USERNAME"
	const Password = "YOUR_PASSWORD"

	client := &http.Client{}
	request, _ := http.NewRequest("GET",
		fmt.Sprintf("https://data.oxylabs.io/v1/queries/%s", JobId),
		nil,
	)

	request.Header.Add("Content-type", "application/json")
	request.SetBasicAuth(Username, Password)
	response, _ := client.Do(request)

	responseText, _ := ioutil.ReadAll(response.Body)
	fmt.Println(string(responseText))
}

package org.example;

import okhttp3.*;

public class Main implements Runnable {
    private static final String AUTHORIZATION_HEADER = "Authorization";
    private static final String JOB_ID = "12345678900987654321";
    public static final String USERNAME = "YOUR_USERNAME";
    public static final String PASSWORD = "YOUR_PASSWORD";

    public void run() {
        Authenticator authenticator = (route, response) -> {
            String credential = Credentials.basic(USERNAME, PASSWORD);

            return response
                    .request()
                    .newBuilder()
                    .header(AUTHORIZATION_HEADER, credential)
                    .build();
        };

        var client = new OkHttpClient.Builder()
                .authenticator(authenticator)
                .build();

        var request = new Request.Builder()
                .url(String.format("https://data.oxylabs.io/v1/queries/%s", JOB_ID))
                .get()
                .build();

        try (var response = client.newCall(request).execute()) {
            assert response.body() != null;
            System.out.println(response.body().string());
        } catch (Exception exception) {
            System.out.println("Error: " + exception.getMessage());
        }

        System.exit(0);
    }

    public static void main(String[] args) {
        new Thread(new Main()).start();
    }
}

import fetch from 'node-fetch';

const jobId = '12345678900987654321';
const username = 'YOUR_USERNAME';
const password = 'YOUR_PASSWORD';
const response = await fetch(`https://data.oxylabs.io/v1/queries/${jobId}`, {
  method: 'get',
  headers: {
    'Content-Type': 'application/json',
    'Authorization': 'Basic ' + Buffer.from(`${username}:${password}`).toString('base64'),
  }
});

console.log(await response.json());

输出

任务完成后，API 会以 JSON 格式返回查询信息。任务状态将变为 done，表示任务已完成。您可以通过查询提供的链接之一来获取内容。此外，响应还将包含任务上次更新时间戳，使您能够跟踪处理时间。

{
    "callback_url": null,
    "client_id": 5,
    "context": [
        {...}
    ],
    "created_at": "2026-09-03 00:00:01",
    "domain": "com",
    "geo_location": "United States",
    "id": "12345678900987654321",
    "limit": 10,
    "locale": null,
    "pages": 1,
    "parse": false,
    "parser_type": null,
    "parser_preset": null,
    "parsing_instructions": null,
    "browser_instructions": null,
    "render": null,
    "xhr": false,
    "markdown": false,
    "url": "https://www.example.com",
    "query": "",
    "source": "universal",
    "start_page": 1,
    "status": "done",
    "storage_type": null,
    "storage_url": null,
    "aggregate_name": null,
    "subdomain": "www",
    "content_encoding": "utf-8",
    "updated_at": "2026-09-03 00:00:15",
    "user_agent_type": "desktop",
    "session_info": null,
    "statuses": [],
    "client_notes": null,
    "_links": [
        {
            "rel": "self",
            "href": "http://data.oxylabs.io/v1/queries/12345678900987654321",
            "method": "GET"
        },
        {
            "rel": "results",
            "href": "http://data.oxylabs.io/v1/queries/12345678900987654321/results",
            "method": "GET"
        },
        {
            "rel": "results-content",
            "href_list": [
                "http://data.oxylabs.io/v1/queries/12345678900987654321/results/1/content"
            ],
            "method": "GET"
        },
        {
            "rel": "results-html",
            "href": "http://data.oxylabs.io/v1/queries/12345678900987654321/results?type=raw",
            "method": "GET"
        },
        {
            "rel": "results-content-html",
            "href_list": [
                "http://data.oxylabs.io/v1/queries/12345678900987654321/results/1/content?type=raw"
            ],
            "method": "GET"
        }
    ]
}

状态值

参数

描述

pending

任务仍在处理中，尚未完成。

done

任务已完成。您可以通过查询以下内容中提供的 URL 来获取结果 href 字段下的 rel:results 部分，例如： http://data.oxylabs.io/v1/queries/12345678900987654321/results.

faulted

任务出现了问题，我们无法完成它。您不会因任何 faulted 任务而收费。

检索任务内容

一旦任务准备好可供获取，您可以使用响应中 rel:results 部分提供的 URL。该 URL 看起来如下： http://data.oxylabs.io/v1/queries/12345678900987654321/results.

这些 _links URL 以 http:// 方案返回。请将其替换为 https:// 后再调用它们，以免凭据未加密传输。

端点

您可以使用以下端点获取不同的结果类型：

GET https://data.oxylabs.io/v1/queries/{job_id}/results

GET https://data.oxylabs.io/v1/queries/{job_id}/results?type=raw

GET https://data.oxylabs.io/v1/queries/{job_id}/results?type=parsed

GET https://data.oxylabs.io/v1/queries/{job_id}/results?type=png

GET https://data.oxylabs.io/v1/queries/{job_id}/results?type=xhr

GET https://data.oxylabs.io/v1/queries/{job_id}/results?type=markdown

您还可以获取多个结果类型，并在单个响应中返回，例如：

GET https://data.oxylabs.io/v1/queries/{job_id}/results?type=parsed,raw

若要单独获取某一页的原始内容，而不是放在 JSON 对象中，请使用 content 端点，其中 {n} 表示页面编号，起始于 1。相同的 类型 值也适用。

GET https://data.oxylabs.io/v1/queries/{job_id}/results/{n}/content

输入

以下是演示如何使用 /results 端点的代码示例：

curl --user "USERNAME:PASSWORD" \
'https://data.oxylabs.io/v1/queries/12345678900987654321/results'

import requests
from pprint import pprint

# Get response from stats endpoint.
response = requests.request(
    method='GET',
    url='https://data.oxylabs.io/v1/queries/12345678900987654321/results',
    auth=('user', 'pass1'),
)

# Print prettified JSON response to stdout.
pprint(response.json())

<?php

$ch = curl_init();

curl_setopt($ch, CURLOPT_URL, "https://data.oxylabs.io/v1/queries/12345678900987654321/results");
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, "GET");
curl_setopt($ch, CURLOPT_USERPWD, "user" . ":" . "pass1");

$result = curl_exec($ch);
echo $result;

if (curl_errno($ch)) {
    echo 'Error:' . curl_error($ch);
}
curl_close($ch);
?>

using System;
using System.Net.Http;
using System.Threading.Tasks;

namespace OxyApi
{
    class Program
    {
        static async Task Main()
        {
            const string JobId = "12345678900987654321";
            const string Username = "YOUR_USERNAME";
            const string Password = "YOUR_PASSWORD";

            var client = new HttpClient();

            Uri baseUri = new Uri("https://data.oxylabs.io");
            client.BaseAddress = baseUri;

            var requestMessage = new HttpRequestMessage(HttpMethod.Get, $"/v1/queries/{JobId}/results");

            var authenticationString = $"{Username}:{Password}";
            var base64EncodedAuthenticationString = Convert.ToBase64String(System.Text.ASCIIEncoding.UTF8.GetBytes(authenticationString));
            requestMessage.Headers.Add("Authorization", "Basic " + base64EncodedAuthenticationString);

            var response = await client.SendAsync(requestMessage);
            var contents = await response.Content.ReadAsStringAsync();

            Console.WriteLine(contents);
        }
    }
}

package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
)

func main() {
	const JobId = "12345678900987654321"
	const Username = "YOUR_USERNAME"
	const Password = "YOUR_PASSWORD"

	client := &http.Client{}
	request, _ := http.NewRequest("GET",
		fmt.Sprintf("https://data.oxylabs.io/v1/queries/%s/results", JobId),
		nil,
	)

	request.Header.Add("Content-type", "application/json")
	request.SetBasicAuth(Username, Password)
	response, _ := client.Do(request)

	responseText, _ := ioutil.ReadAll(response.Body)
	fmt.Println(string(responseText))
}

package org.example;

import okhttp3.*;

public class Main implements Runnable {
    private static final String AUTHORIZATION_HEADER = "Authorization";
    private static final String JOB_ID = "12345678900987654321";
    public static final String USERNAME = "YOUR_USERNAME";
    public static final String PASSWORD = "YOUR_PASSWORD";

    public void run() {
        Authenticator authenticator = (route, response) -> {
            String credential = Credentials.basic(USERNAME, PASSWORD);

            return response
                    .request()
                    .newBuilder()
                    .header(AUTHORIZATION_HEADER, credential)
                    .build();
        };

        var client = new OkHttpClient.Builder()
                .authenticator(authenticator)
                .build();

        var request = new Request.Builder()
                .url(String.format("https://data.oxylabs.io/v1/queries/%s/results", JOB_ID))
                .get()
                .build();

        try (var response = client.newCall(request).execute()) {
            assert response.body() != null;
            System.out.println(response.body().string());
        } catch (Exception exception) {
            System.out.println("Error: " + exception.getMessage());
        }

        System.exit(0);
    }

    public static void main(String[] args) {
        new Thread(new Main()).start();
    }
}

import fetch from 'node-fetch';

const jobId = '12345678900987654321';
const username = 'YOUR_USERNAME';
const password = 'YOUR_PASSWORD';
const response = await fetch(`https://data.oxylabs.io/v1/queries/${jobId}/results`, {
  method: 'get',
  headers: {
    'Content-Type': 'application/json',
    'Authorization': 'Basic ' + Buffer.from(`${username}:${password}`).toString('base64'),
  }
});

console.log(await response.json());

输出

该表根据 API 请求负载中包含的参数解释默认结果类型。

渲染参数

解析参数

XHR 参数

默认输出

html

html

html

html

true

xhr

html

true

parsed

png

png

true

parsed

html

true

parsed

png

true

png

默认情况下， /results 端点只返回该负载的默认输出。若要接收其他结果类型，请添加 类型 下方显示的查询参数端点。每个请求的类型都会作为单独条目返回到 results 数组中。这是 多个结果类型 功能，并且在 Push-Pull 和 Realtime.

以下是 /results 端点的示例响应。除了 results之外，还会返回一个 任务 对象，其中包含与下文所述相同的任务详情 数据字典，此处已缩写。一些来源还会返回 _request, _response 和 session_info 到每个结果中。

{
  "results": [
    {
      "content": "<!doctype html><html>CONTENT</html>",
      "created_at": "2026-09-03 00:00:01",
      "updated_at": "2026-09-03 00:00:15",
      "page": 1,
      "url": "https://www.example.com",
      "job_id": "12345678900987654321",
      "is_render_forced": false,
      "status_code": 200,
      "type": "raw",
      "_request": {
        "Cookie": [],
        "headers": {...}
      },
      "_response": {
        "Cookie": [],
        "headers": {...}
      },
      "session_info": {
        "expires_at": null,
        "id": null,
        "remaining": null
      }
    }
  ],
  "job": {...}
}

结果可以通过设置回调回调服务自动获取。为此，在提交任务时指定一个能够接受传入 HTTP(S) 请求的服务器 URL。当我们的系统完成任务时，它将 POST 向提供的 URL 发送一个 JSON 负载，Callback 服务将按 Callback 实现示例中的说明下载结果.

批量查询

Scraper APIs 支持提交多达 5,000 查询 或 URL 个参数值到单个批量请求中。

端点

POST https://data.oxylabs.io/v1/queries/batch

系统会将每个 查询 或 URL 提交项作为单独任务处理。如果您提供回调 URL，则每个关键词都会收到单独的调用。否则，我们的初始响应将包含任务 idID，适用于所有关键词。例如，如果您发送了 50 个关键词，我们将返回 50 个唯一的任务 idID。

重要： 使用 /batch 端点时，您只能提交 查询或 URL参数值列表（取决于您使用的 源 ）。所有其他参数都应使用单个值。

输入

您需要将查询参数作为 JSON 负载发送。以下是提交批量任务的方法：

curl --user "USERNAME:PASSWORD" \
'https://data.oxylabs.io/v1/queries/batch' \
-H 'Content-Type: application/json' \
-d '@keywords.json'

import requests
import json
from pprint import pprint


# Get payload from file.
with open('keywords.json', 'r') as f:
    payload = json.loads(f.read())

response = requests.request(
    'POST',
    'https://data.oxylabs.io/v1/queries/batch',
    auth=('user', 'pass1'),
    json=payload,
)

# Print prettified response.
pprint(response.json())

<?php

$paramsFile = file_get_contents(realpath("keywords.json"));
$ch = curl_init();

curl_setopt($ch, CURLOPT_URL, "https://data.oxylabs.io/v1/queries/batch");
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, $paramsFile);
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_USERPWD, "user" . ":" . "pass1");

$headers = array();
$headers[] = "Content-Type: application/json";
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);

$result = curl_exec($ch);
echo $result;

if (curl_errno($ch)) {
    echo 'Error:' . curl_error($ch);
}
curl_close($ch);
?>

using System;
using System.IO;
using System.Net.Http;
using System.Text;
using System.Threading.Tasks;

namespace OxyApi
{
    class Program
    {
        static async Task Main()
        {
            const string Username = "YOUR_USERNAME";
            const string Password = "YOUR_PASSWORD";


            var content = File.ReadAllText(@"C:\path\to\keywords.json");
                
            var client = new HttpClient();

            var requestMessage = new HttpRequestMessage(HttpMethod.Post, new Uri("https://data.oxylabs.io/v1/queries/batch"));
            requestMessage.Content = new StringContent(content, Encoding.UTF8, "application/json");


            var authenticationString = $"{Username}:{Password}";
            var base64EncodedAuthenticationString = Convert.ToBase64String(ASCIIEncoding.UTF8.GetBytes(authenticationString));
            requestMessage.Headers.Add("Authorization", "Basic " + base64EncodedAuthenticationString);

            var response = await client.SendAsync(requestMessage);
            var contents = await response.Content.ReadAsStringAsync();

            Console.WriteLine(contents);
        }
    }
}

package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
)

func main() {
	const Username = "YOUR_USERNAME"
	const Password = "YOUR_PASSWORD"

	content, err := os.ReadFile("keywords.json")
	if err != nil {
		panic(err)
	}

	client := &http.Client{}
	request, _ := http.NewRequest("POST",
		"https://data.oxylabs.io/v1/queries/batch",
		bytes.NewBuffer(content),
	)

	request.Header.Add("Content-type", "application/json")
	request.SetBasicAuth(Username, Password)
	response, _ := client.Do(request)

	responseText, _ := ioutil.ReadAll(response.Body)
	fmt.Println(string(responseText))
}

import okhttp3.*;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;

public class Main implements Runnable {
    private static final String AUTHORIZATION_HEADER = "Authorization";
    public static final String USERNAME = "YOUR_USERNAME";
    public static final String PASSWORD = "YOUR_PASSWORD";

    public void run() {
        Path filePath = Path.of("/path/to/keywords.json");
        String jsonContent = null;
        try {
            jsonContent = Files.readString(filePath);
        } catch (IOException e) {
            throw new RuntimeException(e);
        }

        Authenticator authenticator = (route, response) -> {
            String credential = Credentials.basic(USERNAME, PASSWORD);

            return response
                    .request()
                    .newBuilder()
                    .header(AUTHORIZATION_HEADER, credential)
                    .build();
        };

        var client = new OkHttpClient.Builder()
                .authenticator(authenticator)
                .build();

        var mediaType = MediaType.parse("application/json; charset=utf-8");
        var body = RequestBody.create(jsonContent, mediaType);
        var request = new Request.Builder()
                .url("https://data.oxylabs.io/v1/queries/batch")
                .post(body)
                .build();

        try (var response = client.newCall(request).execute()) {
            assert response.body() != null;
            System.out.println(response.body().string());
        } catch (Exception exception) {
            System.out.println("Error: " + exception.getMessage());
        }

        System.exit(0);
    }

    public static void main(String[] args) {
        new Thread(new Main()).start();
    }
}

import fetch from 'node-fetch';
import fs from 'fs'

const username = 'YOUR_USERNAME';
const password = 'YOUR_PASSWORD';

const payload = fs.readFileSync('keywords.json').toString();

const response = await fetch('https://data.oxylabs.io/v1/queries/batch', {
  method: 'post',
  body: payload,
  headers: {
    'Content-Type': 'application/json',
    'Authorization': 'Basic ' + Buffer.from(`${username}:${password}`).toString('base64'),
  }
});

console.log(await response.json());

你可能会注意到，上面的代码示例没有说明 JSON 负载应如何格式化，并指向一个预先准备好的 JSON 文件。下面是 keywords.json 文件，包含多个 查询 参数值：

{  
   "query":[  
      "adidas",
      "nike",
      "reebok"
   ],
   "source": "google_shopping_search",
   "callback_url": "https://your.callback.url"
}

……这里是一个 keywords.json 批量输入文件，包含多个 URL：

{  
   "url":[  
      "https://example.com/url1.html",
      "https://example.com/url2.html",
      "https://example.com/url3.html"
   ],
   "source": "universal",
   "callback_url": "https://your.callback.url"
}

输出

API 将返回一个 JSON 对象，其中包含每个已创建任务的任务信息。响应将类似于以下内容：

{
  "queries": [
    {
      "callback_url": "https://your.callback.url",
      "created_at": "2026-09-03 00:00:01",
      "id": "12345678900987654321",
      "query": "adidas",
      "source": "google_shopping_search",
      "status": "pending",
      {...}
      "_links": [
        {
          "rel": "self",
          "href": "http://data.oxylabs.io/v1/queries/12345678900987654321",
          "method": "GET"
        },
        {
          "rel": "results",
          "href": "http://data.oxylabs.io/v1/queries/12345678900987654321/results",
          "method": "GET"
        },
        {...}
      ]
    },
    {
      "callback_url": "https://your.callback.url",
      "created_at": "2026-09-03 00:00:01",
      "id": "12345678901234567890",
      "query": "nike",
      "source": "google_shopping_search",
      "status": "pending",
      {...}
      "_links": [
        {
          "rel": "self",
          "href": "http://data.oxylabs.io/v1/queries/12345678901234567890",
          "method": "GET"
        },
        {
          "rel": "results",
          "href": "http://data.oxylabs.io/v1/queries/12345678901234567890/results",
          "method": "GET"
        },
        {...}
      ]
    },
    {
      "callback_url": "https://your.callback.url",
      "created_at": "2026-09-03 00:00:01",
      "id": "01234567899876543210",
      "query": "reebok",
      "source": "google_shopping_search",
      "status": "pending",
      {...}
      "_links": [
        {
          "rel": "self",
          "href": "http://data.oxylabs.io/v1/queries/01234567899876543210",
          "method": "GET"
        },
        {
          "rel": "results",
          "href": "http://data.oxylabs.io/v1/queries/01234567899876543210/results",
          "method": "GET"
        },
        {...}
      ]
    }
  ]
}

每个条目都包含与单个任务响应相同的字段； {...} 表示此处为简洁起见省略的字段。

获取通知器 IP 地址列表

您可能希望将向您发送回调消息的 IP 加入白名单，或出于其他目的获取这些 IP 的列表。您可以通过以下方式： GET调用此端点：

端点

GET https://data.oxylabs.io/v1/info/callbacker_ips

输入

下面的代码示例展示了如何访问 /callbacker_ips 端点的代码示例：

curl --user "USERNAME:PASSWORD" \
'https://data.oxylabs.io/v1/info/callbacker_ips'

import requests
from pprint import pprint

# Get response from the callback IPs endpoint.
response = requests.request(
    method='GET',
    url='https://data.oxylabs.io/v1/info/callbacker_ips',
    auth=('user', 'pass1'),
)

# Print prettified JSON response to stdout.
pprint(response.json())

<?php

$ch = curl_init();

curl_setopt($ch, CURLOPT_URL, "https://data.oxylabs.io/v1/info/callbacker_ips");
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, "GET");
curl_setopt($ch, CURLOPT_USERPWD, "user" . ":" . "pass1");

$result = curl_exec($ch);
echo $result;

if (curl_errno($ch)) {
    echo 'Error:' . curl_error($ch);
}
curl_close($ch);
?>

using System;
using System.Net.Http;
using System.Threading.Tasks;

namespace OxyApi
{
    class Program
    {
        static async Task Main()
        {
            const string Username = "YOUR_USERNAME";
            const string Password = "YOUR_PASSWORD";

            var client = new HttpClient();

            Uri baseUri = new Uri("https://data.oxylabs.io");
            client.BaseAddress = baseUri;

            var requestMessage = new HttpRequestMessage(HttpMethod.Get, "/v1/info/callbacker_ips");

            var authenticationString = $"{Username}:{Password}";
            var base64EncodedAuthenticationString = Convert.ToBase64String(System.Text.ASCIIEncoding.UTF8.GetBytes(authenticationString));
            requestMessage.Headers.Add("Authorization", "Basic " + base64EncodedAuthenticationString);

            var response = await client.SendAsync(requestMessage);
            var contents = await response.Content.ReadAsStringAsync();

            Console.WriteLine(contents);
        }
    }
}

package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
)

func main() {
	const Username = "YOUR_USERNAME"
	const Password = "YOUR_PASSWORD"

	client := &http.Client{}
	request, _ := http.NewRequest("GET",
		"https://data.oxylabs.io/v1/info/callbacker_ips",
		nil,
	)

	request.Header.Add("Content-type", "application/json")
	request.SetBasicAuth(Username, Password)
	response, _ := client.Do(request)

	responseText, _ := ioutil.ReadAll(response.Body)
	fmt.Println(string(responseText))
}

package org.example;

import okhttp3.*;

public class Main implements Runnable {
    private static final String AUTHORIZATION_HEADER = "Authorization";
    public static final String USERNAME = "YOUR_USERNAME";
    public static final String PASSWORD = "YOUR_PASSWORD";

    public void run() {
        Authenticator authenticator = (route, response) -> {
            String credential = Credentials.basic(USERNAME, PASSWORD);

            return response
                    .request()
                    .newBuilder()
                    .header(AUTHORIZATION_HEADER, credential)
                    .build();
        };

        var client = new OkHttpClient.Builder()
                .authenticator(authenticator)
                .build();

        var request = new Request.Builder()
                .url("https://data.oxylabs.io/v1/info/callbacker_ips")
                .get()
                .build();

        try (var response = client.newCall(request).execute()) {
            assert response.body() != null;
            System.out.println(response.body().string());
        } catch (Exception exception) {
            System.out.println("Error: " + exception.getMessage());
        }

        System.exit(0);
    }

    public static void main(String[] args) {
        new Thread(new Main()).start();
    }
}

import fetch from 'node-fetch';

const username = 'YOUR_USERNAME';
const password = 'YOUR_PASSWORD';
const response = await fetch('https://data.oxylabs.io/v1/info/callbacker_ips', {
  method: 'get',
  headers: {
    'Content-Type': 'application/json',
    'Authorization': 'Basic ' + Buffer.from(`${username}:${password}`).toString('base64'),
  }
});

console.log(await response.json());

输出

API 将返回向您的系统发起回调请求的 IP 列表：

{
    "ips": [
        "x.x.x.x",
        "y.y.y.y"
    ]
}

任务管家

任务管家是一项可用于安排重复抓取任务的服务。

它扩展了 Push-Pull 集成的功能，最好与 云集成 功能一起使用。

访问此页面，了解如何使用任务管家功能：

任务管家

上一页Realtime

下一页Proxy Endpoint

最后更新于 2小时前

这有帮助吗？