Realtime

Oxylabs 的网页爬虫API 的 Realtime 集成。从任务提交到返回结果或错误，始终保持 HTTPS 连接开启，并使用 JSON 格式的负载。

Realtime 是一种同步集成方法。它用于可靠地处理大规模数据提取任务，而无需你管理请求分发或基础设施。它 需要保持连接打开 直到任务成功完成或返回错误。

任务提交

端点

用于任务提交的 Realtime API 端点是：

POST https://realtime.oxylabs.io/v1/queries

输入

请按照下面示例，将任务参数放在 JSON 有效负载中。Python 和 PHP 示例包含注释以便说明。

curl --user "USERNAME:PASSWORD" \
'https://realtime.oxylabs.io/v1/queries' \
-H "Content-Type: application/json" \
-d '{"source": "universal", "url": "https://example.com", "geo_location": "United States"}'

import requests
from pprint import pprint


# Structure payload.
payload = {
    "source": "universal", # 你选择的来源，例如 "universal"
    "url": "https://example.com", # 查看你所使用的特定来源的文档，以确定应使用 "url" 还是 "query"
    "geo_location": "United States", # 某些来源接受邮政编码和/或坐标
    #"render" : "html", # 如果你想在页面上渲染 JavaScript，请取消注释
    #"render" : "png", # Uncomment if you want to take a screenshot of a scraped web page
    #"parse" : true, # Check what sources support parsed data
}

# Get response.
response = requests.request(
    'POST',
    'https://realtime.oxylabs.io/v1/queries',
    auth=('YOUR_USERNAME', 'YOUR_PASSWORD'), #Your credentials go here
    json=payload,
)

# 不返回作业状态和结果 URL 的响应，而是返回
# 带有结果的 JSON 响应。
pprint(response.json())

<?php

$params = array(
    'source' => 'universal', // 你选择的来源，例如 "universal"
    'url' => 'https://example.com', // 查看你所使用的特定来源的文档，以确定应使用 "url" 还是 "query"
    'geo_location' => 'United States', // 某些来源接受邮政编码或坐标
    //'render' : 'html', // 如果你想在页面内渲染 JavaScript，请取消注释
    //'render' : 'png', // Uncomment if you want to take a screenshot of a scraped web page
    //'parse' : TRUE, // Check what sources support parsed data
);

$ch = curl_init();

curl_setopt($ch, CURLOPT_URL, "https://realtime.oxylabs.io/v1/queries");
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($params));
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_USERPWD, "YOUR_USERNAME" . ":" . "YOUR_PASSWORD"); //Your credentials go here

$headers = array();
$headers[] = "Content-Type: application/json";
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);

$result = curl_exec($ch);
echo $result;

if (curl_errno($ch)) {
    echo 'Error:' . curl_error($ch);
}
curl_close ($ch);PHP

using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Net.Http.Json;
using System.Threading.Tasks;

namespace OxyApi
{
    class Program
    {
        static async Task Main()
        {
            const string Username = "YOUR_USERNAME";
            const string Password = "YOUR_PASSWORD";

            var parameters = new Dictionary<string, string>()
            {
                { "source", "universal" },
                { "url", "https://example.com" },
                { "geo_location", "United States" },
            };


            var client = new HttpClient();

            Uri baseUri = new Uri("https://realtime.oxylabs.io");
            client.BaseAddress = baseUri;

            var requestMessage = new HttpRequestMessage(HttpMethod.Post, "/v1/queries");
            requestMessage.Content = JsonContent.Create(parameters);

            var authenticationString = $"{Username}:{Password}";
            var base64EncodedAuthenticationString = Convert.ToBase64String(System.Text.ASCIIEncoding.UTF8.GetBytes(authenticationString));
            requestMessage.Headers.Add("Authorization", "Basic " + base64EncodedAuthenticationString);

            var response = await client.SendAsync(requestMessage);
            var contents = await response.Content.ReadAsStringAsync();

            Console.WriteLine(contents);
        }
    }
}

package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
)

func main() {
	const Username = "YOUR_USERNAME"
	const Password = "YOUR_PASSWORD"

	payload := map[string]string{
		"source": "universal",
		"url": "https://example.com",
		"geo_location": "United States",
	}

	jsonValue, _ := json.Marshal(payload)

	client := &http.Client{}
	request, _ := http.NewRequest("POST",
		"https://realtime.oxylabs.io/v1/queries",
		bytes.NewBuffer(jsonValue),
	)

	request.SetBasicAuth(Username, Password)
	response, _ := client.Do(request)

	responseText, _ := ioutil.ReadAll(response.Body)
	fmt.Println(string(responseText))
}

package org.example;

import okhttp3.*;
import org.json.JSONObject;

public class Main implements Runnable {
    private static final String AUTHORIZATION_HEADER = "Authorization";
    public static final String USERNAME = "YOUR_USERNAME";
    public static final String PASSWORD = "YOUR_PASSWORD";

    public void run() {
        JSONObject jsonObject = new JSONObject();
        jsonObject.put("source", "universal");
        jsonObject.put("url", "https://example.com");
        jsonObject.put("geo_location", "United States");

        Authenticator authenticator = (route, response) -> {
            String credential = Credentials.basic(USERNAME, PASSWORD);

            return response
                    .request()
                    .newBuilder()
                    .header(AUTHORIZATION_HEADER, credential)
                    .build();
        };

        var client = new OkHttpClient.Builder()
                .authenticator(authenticator)
                .build();

        var mediaType = MediaType.parse("application/json; charset=utf-8");
        var body = RequestBody.create(jsonObject.toString(), mediaType);
        var request = new Request.Builder()
                .url("https://realtime.oxylabs.io/v1/queries")
                .post(body)
                .build();

        try (var response = client.newCall(request).execute()) {
            assert response.body() != null;
            System.out.println(response.body().string());
        } catch (Exception exception) {
            System.out.println("Error: " + exception.getMessage());
        }

        System.exit(0);
    }

    public static void main(String[] args) {
        new Thread(new Main()).start();
    }
}

import fetch from 'node-fetch';

const username = 'YOUR_USERNAME';
const password = 'YOUR_PASSWORD';
const body = {
  source: 'universal',
  url: 'https://example.com',
  geo_location: 'United States'
};
const response = await fetch('https://realtime.oxylabs.io/v1/queries', {
  method: 'post',
  body: JSON.stringify(body),
  headers: {
    'Content-Type': 'application/json',
    'Authorization': 'Basic ' + Buffer.from(`${username}:${password}`).toString('base64'),
  }
});

console.log(await response.json());

输出

Realtime API 在输出中支持以下结果类型：

HTML：从目标网页抓取的原始 HTML 内容；
JSON：从 HTML 内容解析出的结构化数据，采用 JSON 格式；
PNG：渲染后页面的 PNG 格式 Base64 编码截图。
XHR： XHR 请求在页面加载期间发出的。
Markdown： Markdown 网页的。

你还可以检索多个结果类型，并在单个 API 响应中获取。

此表根据 API 请求载荷中包含的参数说明默认和其他可用的结果类型：

渲染参数

解析参数

默认输出

可用输出

html

html

html

png

png

html, png

true

json

html, json

html

true

json

html, json

png

true

png

html, json, png

Realtime API 始终返回默认输出。要获取其他可用输出，请使用 {Push-Pull} 集成方法。

输出示例：

{
  "results": [
    {
      "content": "<html>
      CONTENT
      </html>",
      "created_at": "2024-06-26 13:13:06",
      "updated_at": "2024-06-26 13:13:07",
      "id": null,
      "page": 1,
      "url": "https://www.example.com/", 
      "job_id": "12345678900987654321",
      "status_code": 200
    }
  ]
}

最后更新于 1个月前

这有帮助吗？

package org.example;

import okhttp3.*;
import org.json.JSONObject;

public class Main implements Runnable {
    private static final String AUTHORIZATION_HEADER = "Authorization";
    public static final String USERNAME = "YOUR_USERNAME";
    public static final String PASSWORD = "YOUR_PASSWORD";

    public void run() {
        JSONObject jsonObject = new JSONObject();
        jsonObject.put("source", "universal");
        jsonObject.put("url", "https://example.com");
        jsonObject.put("geo_location", "United States");

        Authenticator authenticator = (route, response) -> {
            String credential = Credentials.basic(USERNAME, PASSWORD);

            return response
                    .request()
                    .newBuilder()
                    .header(AUTHORIZATION_HEADER, credential)
                    .build();
        };

        var client = new OkHttpClient.Builder()
                .authenticator(authenticator)
                .build();

        var mediaType = MediaType.parse("application/json; charset=utf-8");
        var body = RequestBody.create(jsonObject.toString(), mediaType);
        var request = new Request.Builder()
                .url("https://realtime.oxylabs.io/v1/queries")
                .post(body)
                .build();

        try (var response = client.newCall(request).execute()) {
            assert response.body() != null;
            System.out.println(response.body().string());
        } catch (Exception exception) {
            System.out.println("Error: " + exception.getMessage());
        }

        System.exit(0);
    }

    public static void main(String[] args) {
        new Thread(new Main()).start();
    }
}