URL

Google URL scraper retrieves content from any Google page with ease. Simple integration with code examples for multiple languages.

The google source is designed to retrieve content from various Google URLs. This means that instead of sending multiple parameters, you can provide us with a direct URL to the required Google page. We do not strip any parameters or alter your URLs in any other way.

Request samples

In the example below, the API will retrieve a Google Scholar search page.

cURL

curl 'https://data.oxylabs.io/v1/queries' \
--user 'USERNAME:PASSWORD' \
-H 'Content-Type: application/json' \
-d '{
        "source": "google",
        "url": "https://scholar.google.com/scholar?hl=en&q=newton&btnG=&as_sdt=1%2C5&as_sdtp="
    }'

Python

import requests
from pprint import pprint

# Structure payload.
payload = {
    'source': 'google',
    'url': 'https://scholar.google.com/scholar?hl=en&q=newton&btnG=&as_sdt=1%2C5&as_sdtp='
}

# Get response.
response = requests.request(
    'POST',
    'https://data.oxylabs.io/v1/queries',
    auth=('USERNAME', 'PASSWORD'),
    json=payload,
)

# The response contains the job details, including the job ID.
# Use the ID to check the job status and retrieve the results.
pprint(response.json())

Node.js

const https = require("https");

const username = "USERNAME";
const password = "PASSWORD";
const body = {
    source: "google",
    query: "https://scholar.google.com/scholar?hl=en&q=newton&btnG=&as_sdt=1%2C5&as_sdtp=",
};

const options = {
    hostname: "data.oxylabs.io",
    path: "/v1/queries",
    method: "POST",
    headers: {
        "Content-Type": "application/json",
        Authorization:
            "Basic " + Buffer.from(`${username}:${password}`).toString("base64"),
    },
};

const request = https.request(options, (response) => {
    let data = "";

    response.on("data", (chunk) => {
        data += chunk;
    });

    response.on("end", () => {
        const responseData = JSON.parse(data);
        console.log(JSON.stringify(responseData, null, 2));
    });
});

request.on("error", (error) => {
    console.error("Error:", error);
});

request.write(JSON.stringify(body));
request.end();

PHP

<?php

$params = array(
    'source' => 'google',
    'url' => 'https://scholar.google.com/scholar?hl=en&q=newton&btnG=&as_sdt=1%2C5&as_sdtp='
);

$ch = curl_init();

curl_setopt($ch, CURLOPT_URL, "https://data.oxylabs.io/v1/queries");
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($params));
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_USERPWD, "USERNAME" . ":" . "PASSWORD");


$headers = array();
$headers[] = "Content-Type: application/json";
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);

$result = curl_exec($ch);
echo $result;

if (curl_errno($ch)) {
    echo 'Error:' . curl_error($ch);
}
curl_close($ch);

Golang

package main

import (
    "bytes"
    "encoding/json"
    "fmt"
    "io/ioutil"
    "net/http"
)

func main() {
    const Username = "USERNAME"
    const Password = "PASSWORD"

    payload := map[string]interface{}{
        "source": "google",
        "url":    "https://scholar.google.com/scholar?hl=en&q=newton&btnG=&as_sdt=1%2C5&as_sdtp=",
    }

    jsonValue, _ := json.Marshal(payload)

    client := &http.Client{}
    request, _ := http.NewRequest("POST",
        "https://data.oxylabs.io/v1/queries",
        bytes.NewBuffer(jsonValue),
    )

    request.SetBasicAuth(Username, Password)
    response, _ := client.Do(request)

    responseText, _ := ioutil.ReadAll(response.Body)
    fmt.Println(string(responseText))
}

C#

using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Net.Http.Json;
using System.Threading.Tasks;

namespace OxyApi
{
    class Program
    {
        static async Task Main()
        {
            const string Username = "USERNAME";
            const string Password = "PASSWORD";

            var parameters = new {
                source = "google",
                query = "https://scholar.google.com/scholar?hl=en&q=newton&btnG=&as_sdt=1%2C5&as_sdtp="
            };

            var client = new HttpClient();

            Uri baseUri = new Uri("https://data.oxylabs.io");
            client.BaseAddress = baseUri;

            var requestMessage = new HttpRequestMessage(HttpMethod.Post, "/v1/queries");
            requestMessage.Content = JsonContent.Create(parameters);

            var authenticationString = $"{Username}:{Password}";
            var base64EncodedAuthenticationString = Convert.ToBase64String(System.Text.ASCIIEncoding.UTF8.GetBytes(authenticationString));
            requestMessage.Headers.Add("Authorization", "Basic " + base64EncodedAuthenticationString);

            var response = await client.SendAsync(requestMessage);
            var contents = await response.Content.ReadAsStringAsync();

            Console.WriteLine(contents);
        }
    }
}

Java

package org.example;

import okhttp3.*;
import org.json.JSONObject;
import java.util.concurrent.TimeUnit;

public class Main implements Runnable {
    private static final String AUTHORIZATION_HEADER = "Authorization";
    public static final String USERNAME = "USERNAME";
    public static final String PASSWORD = "PASSWORD";

    public void run() {
        JSONObject jsonObject = new JSONObject();
        jsonObject.put("source", "google");
        jsonObject.put("url", "https://scholar.google.com/scholar?hl=en&q=newton&btnG=&as_sdt=1%2C5&as_sdtp=");

        Authenticator authenticator = (route, response) -> {
            String credential = Credentials.basic(USERNAME, PASSWORD);
            return response
                    .request()
                    .newBuilder()
                    .header(AUTHORIZATION_HEADER, credential)
                    .build();
        };

        var client = new OkHttpClient.Builder()
                .authenticator(authenticator)
                .build();

        var mediaType = MediaType.parse("application/json; charset=utf-8");
        var body = RequestBody.create(jsonObject.toString(), mediaType);
        var request = new Request.Builder()
                .url("https://data.oxylabs.io/v1/queries")
                .post(body)
                .build();

        try (var response = client.newCall(request).execute()) {
            if (response.body() != null) {
                try (var responseBody = response.body()) {
                    System.out.println(responseBody.string());
                }
            }
        } catch (Exception exception) {
            System.out.println("Error: " + exception.getMessage());
        }

        System.exit(0);
    }

    public static void main(String[] args) {
        new Thread(new Main()).start();
    }
}

JSON

{
    "source": "google", 
    "url": "https://scholar.google.com/scholar?hl=en&q=newton&btnG=&as_sdt=1%2C5&as_sdtp="
}

We use the Push-Pull integration method in our examples. Submitting a job returns the job ID immediately, not the result. Once the job status is done, retrieve the results with GET https://data.oxylabs.io/v1/queries/{job_id}/results. See Push-Pull for checking the job status, callbacks and batch queries.

Request parameter values

Generic

Basic setup and customization options for scraping Google URLs.

Parameter	Description	Default Value
source	Sets the scraper when set to google.	–
url	Direct URL (link) to the Google page. Supports Google Advanced Search Operators.	–
render	Enables JavaScript rendering when set to html. More info.	–
parse	Returns parsed data when set to true. Limited to URLs of specific Google page types.	–
callback_url	URL to your callback endpoint. More info	–
user_agent_type	Device type and browser. The full list can be found here.	desktop
geo_location	The geographical location that the result should be adapted for. Using this parameter correctly is extremely important to get the right data. For more information, read about our suggested geo_location parameter structures here.	–

- mandatory parameter

Output example

{
    "results": [
        {
            "content":"<!doctype html>\n<html lang=\"en\">\n<head>
            ...
            </script></body>\n</html>\n",            
            "created_at": "2024-05-27 12:28:27",
            "updated_at": "2024-05-27 12:28:30",
            "page": 1,
            "url": "https://scholar.google.com/scholar?hl=en&q=newton&btnG=&as_sdt=1%2C5&as_sdtp=",
            "job_id": "7200835246005180417",
            "status_code": 200
        }
    ]
}