URL

Google URL 抓取器可轻松检索任何 Google 页面内容。提供多语言的简单集成和代码示例。

该 google source 被设计用于从各种 Google URL 检索内容。这意味着您可以提供所需 Google 页面的直接 URL，而不是发送多个参数。我们不会剥离任何参数或以任何其他方式更改您的 URL。

请求示例

在下面的示例中，API 将检索 Google 学术的搜索页面。

curl 'https://realtime.oxylabs.io/v1/queries' \
--user 'USERNAME:PASSWORD' \
-H 'Content-Type: application/json' \
-d '{
        "source": "google",
        "url": "https://scholar.google.com/scholar?hl=en&q=newton&btnG=&as_sdt=1%2C5&as_sdtp="
    }'

import requests
from pprint import pprint

# 构建负载（payload）。
payload = {
    'source': 'google',
    'url': 'https://scholar.google.com/scholar?hl=en&q=newton&btnG=&as_sdt=1%2C5&as_sdtp='
}

# 获取响应。
response = requests.request(
    'POST',
    'https://realtime.oxylabs.io/v1/queries',
    auth=('USERNAME', 'PASSWORD'),
    json=payload,
)

# 与返回作业状态和结果 URL 的响应不同，此调用将直接返回
# 带有结果的 JSON 响应。
pprint(response.json())

const https = require("https");

const username = "USERNAME";
const password = "PASSWORD";
const body = {
    source: "google",
    query: "https://scholar.google.com/scholar?hl=en&q=newton&btnG=&as_sdt=1%2C5&as_sdtp=",
};

const options = {
    hostname: "realtime.oxylabs.io",
    path: "/v1/queries",
    method: "POST",
    headers: {
        "Content-Type": "application/json",
        Authorization:
            "Basic " + Buffer.from(`${username}:${password}`).toString("base64"),
    },
};

const request = https.request(options, (response) => {
    let data = "";

    response.on("data", (chunk) => {
        data += chunk;
    });

    response.on("end", () => {
        const responseData = JSON.parse(data);
        console.log(JSON.stringify(responseData, null, 2));
    });
});

request.on("error", (error) => {
    console.error("Error:", error);
});

request.write(JSON.stringify(body));
request.end();

# 需要对 URL 进行编码以转义 `&` 和 `=` 字符：
# URL: https://scholar.google.com/scholar?hl=en&q=newton&btnG=&as_sdt=1%2C5&as_sdtp=
# Encoded URL: https%3A%2F%2Fscholar.google.com%2Fscholar%3Fhl%3Den%26q%3Dnewton%26btnG%3D%26as_sdt%3D1%252C5%26as_sdtp%3D

https://realtime.oxylabs.io/v1/queries?source=google&url=https%3A%2F%2Fscholar.google.com%2Fscholar%3Fhl%3Den%26q%3Dnewton%26btnG%3D%26as_sdt%3D1%252C5%26as_sdtp%3D&access_token=12345abcde

<?php

$params = array(
    'source' => 'google',
    'url' => 'https://scholar.google.com/scholar?hl=en&q=newton&btnG=&as_sdt=1%2C5&as_sdtp='
);

$ch = curl_init();

curl_setopt($ch, CURLOPT_URL, "https://realtime.oxylabs.io/v1/queries");
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($params));
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_USERPWD, "USERNAME" . ":" . "PASSWORD");


$headers = array();
$headers[] = "Content-Type: application/json";
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);

$result = curl_exec($ch);
echo $result;

if (curl_errno($ch)) {
    echo 'Error:' . curl_error($ch);
}
curl_close($ch);

package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
)

func main() {
	const Username = "USERNAME"
	const Password = "PASSWORD"

	payload := map[string]interface{}{
		"source": "google",
		"url":    "https://scholar.google.com/scholar?hl=en&q=newton&btnG=&as_sdt=1%2C5&as_sdtp=",
	}

	jsonValue, _ := json.Marshal(payload)

	client := &http.Client{}
	request, _ := http.NewRequest("POST",
		"https://realtime.oxylabs.io/v1/queries",
		bytes.NewBuffer(jsonValue),
	)

	request.SetBasicAuth(Username, Password)
	response, _ := client.Do(request)

	responseText, _ := ioutil.ReadAll(response.Body)
	fmt.Println(string(responseText))
}

using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Net.Http.Json;
using System.Threading.Tasks;

namespace OxyApi
{
    class Program
    {
        static async Task Main()
        {
            const string Username = "USERNAME";
            const string Password = "PASSWORD";

            var parameters = new {
                source = "google",
                query = "https://scholar.google.com/scholar?hl=en&q=newton&btnG=&as_sdt=1%2C5&as_sdtp="
            };

            var client = new HttpClient();

            Uri baseUri = new Uri("https://realtime.oxylabs.io");
            client.BaseAddress = baseUri;

            var requestMessage = new HttpRequestMessage(HttpMethod.Post, "/v1/queries");
            requestMessage.Content = JsonContent.Create(parameters);

            var authenticationString = $"{Username}:{Password}";
            var base64EncodedAuthenticationString = Convert.ToBase64String(System.Text.ASCIIEncoding.UTF8.GetBytes(authenticationString));
            requestMessage.Headers.Add("Authorization", "Basic " + base64EncodedAuthenticationString);

            var response = await client.SendAsync(requestMessage);
            var contents = await response.Content.ReadAsStringAsync();

            Console.WriteLine(contents);
        }
    }
}

package org.example;

import okhttp3.*;
import org.json.JSONObject;
import java.util.concurrent.TimeUnit;

public class Main implements Runnable {
    private static final String AUTHORIZATION_HEADER = "Authorization";
    public static final String USERNAME = "USERNAME";
    public static final String PASSWORD = "PASSWORD";

    public void run() {
        JSONObject jsonObject = new JSONObject();
        jsonObject.put("source", "google");
        jsonObject.put("url", "https://scholar.google.com/scholar?hl=en&q=newton&btnG=&as_sdt=1%2C5&as_sdtp=");

        Authenticator authenticator = (route, response) -> {
            String credential = Credentials.basic(USERNAME, PASSWORD);
            return response
                    .request()
                    .newBuilder()
                    .header(AUTHORIZATION_HEADER, credential)
                    .build();
        };

        var client = new OkHttpClient.Builder()
                .authenticator(authenticator)
                .readTimeout(180, TimeUnit.SECONDS)
                .build();

        var mediaType = MediaType.parse("application/json; charset=utf-8");
        var body = RequestBody.create(jsonObject.toString(), mediaType);
        var request = new Request.Builder()
                .url("https://realtime.oxylabs.io/v1/queries")
                .post(body)
                .build();

        try (var response = client.newCall(request).execute()) {
            if (response.body() != null) {
                try (var responseBody = response.body()) {
                    System.out.println(responseBody.string());
                }
            }
        } catch (Exception exception) {
            System.out.println("Error: " + exception.getMessage());
        }

        System.exit(0);
    }

    public static void main(String[] args) {
        new Thread(new Main()).start();
    }
}

在我们的示例中，我们使用同步的 Realtime 集成方法。如果您想使用 Proxy Endpoint 或异步的 Push-Pull 集成，请参阅 集成方法 部分。

请求参数值

通用

针对抓取 Google URL 的基本设置和自定义选项。

参数

说明

默认值

source

设置抓取器。

google

url

指向该 Google 页面的直接 URL（链接）。

render

启用 JavaScript 渲染，当设置为 html. 更多信息.

parse

当设置为以下值时返回解析后的数据 true. 仅限特定类型的 URL Google 页面类型.

callback_url

回调端点的 URL。 更多信息.

user_agent_type

设备类型和浏览器。完整列表可在此处.

desktop

geo_location

应为其适配结果的地理位置。正确使用此参数对于获取正确数据非常重要。有关更多信息，请阅读我们建议的 geo_location 参数结构此处.

- 必填参数

输出示例

{
    "results": [
        {
            "content":"<!doctype html>\n<html lang=\"en\">\n<head>
            ...
            </script></body>\n</html>\n",            
            "created_at": "2024-05-27 12:28:27",
            "updated_at": "2024-05-27 12:28:30",
            "page": 1,
            "url": "https://scholar.google.com/scholar?hl=en&q=newton&btnG=&as_sdt=1%2C5&as_sdtp=",
            "job_id": "7200835246005180417",
            "status_code": 200
        }
    ]
}

最后更新于2个月前

这有帮助吗？

早上好

请求示例

请求参数值

通用

输出示例