Web Search

The google_search source is designed to retrieve Google Search results (SERPs). This sub-page specifically presents information related to Google Web Search. To explore other result types, read Image Search or News Search.

Request samples

In the samples below, we make a request to get 2 results pages, from number 11 to number 12 , for search term adidas on google.nl domain. The SERP will be filtered to contain French-language results only. API will return parsed results.

cURL

curl 'https://realtime.oxylabs.io/v1/queries' \
--user 'USERNAME:PASSWORD' \
-H 'Content-Type: application/json' \
-d '{
        "source": "google_search",
        "domain": "nl",
        "query": "adidas",
        "start_page": 11,
        "pages": 2,
        "parse": true,
        "context": [
            {
                "key": "results_language",
                "value": "fr"
            }
        ]
    }'

Python

import requests
from pprint import pprint


# Structure payload.
payload = {
    'source': 'google_search',
    'domain': 'nl',
    'query': 'adidas',
    'start_page': 11,
    'pages': 2,
    'parse': True,
    'context': [
        {'key': 'results_language', 'value': 'fr'},
    ],
}


# Get response.
response = requests.request(
    'POST',
    'https://realtime.oxylabs.io/v1/queries',
    auth=('USERNAME', 'PASSWORD'),
    json=payload,
)

# Print prettified response to stdout.
pprint(response.json())

Node.js

const https = require("https");

const username = "USERNAME";
const password = "PASSWORD";
const body = {
    source: "google_search",
    domain: "nl",
    query: "adidas",
    start_page: 11,
    pages: 2,
    parse: true,
    context: [
        { key: "results_language", value: "fr" },
    ],
};

const options = {
    hostname: "realtime.oxylabs.io",
    path: "/v1/queries",
    method: "POST",
    headers: {
        "Content-Type": "application/json",
        Authorization:
            "Basic " + Buffer.from(`${username}:${password}`).toString("base64"),
    },
};

const request = https.request(options, (response) => {
    let data = "";

    response.on("data", (chunk) => {
        data += chunk;
    });

    response.on("end", () => {
        const responseData = JSON.parse(data);
        console.log(JSON.stringify(responseData, null, 2));
    });
});

request.on("error", (error) => {
    console.error("Error:", error);
});

request.write(JSON.stringify(body));
request.end();

HTTP

https://realtime.oxylabs.io/v1/queries?source=google_search&domain=nl&query=adidas&start_page=11&pages=2&parse=true&context[0][key]=results_language&context[0][value]=fr&access_token=12345abcde

PHP

<?php

$params = array(
    'source' => 'google_search',
    'domain' => 'nl',
    'query' => 'adidas',
    'start_page' => 11,
    'pages' => 2,
    'parse' => true,
    'context' => [
        [
            'key' => 'results_language',
            'value' => 'fr',
        ]
    ]
);

$ch = curl_init();

curl_setopt($ch, CURLOPT_URL, "https://realtime.oxylabs.io/v1/queries");
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($params));
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_USERPWD, "USERNAME" . ":" . "PASSWORD");


$headers = array();
$headers[] = "Content-Type: application/json";
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);

$result = curl_exec($ch);
echo $result;

if (curl_errno($ch)) {
    echo 'Error:' . curl_error($ch);
}
curl_close($ch);

Golang

package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
)

func main() {
	const Username = "USERNAME"
	const Password = "PASSWORD"

	payload := map[string]interface{}{
		"source":     "google_search",
		"domain":     "nl",
		"query":      "adidas",
		"start_page": 11,
		"pages":      2,
		"parse":      true,
		"context": []map[string]interface{}{
			{"key": "results_language", "value": "fr"},
		},
	}

	jsonValue, _ := json.Marshal(payload)

	client := &http.Client{}
	request, _ := http.NewRequest("POST",
		"https://realtime.oxylabs.io/v1/queries",
		bytes.NewBuffer(jsonValue),
	)

	request.SetBasicAuth(Username, Password)
	response, _ := client.Do(request)

	responseText, _ := ioutil.ReadAll(response.Body)
	fmt.Println(string(responseText))
}

C#

using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Net.Http.Json;
using System.Threading.Tasks;

namespace OxyApi
{
    class Program
    {
        static async Task Main()
        {
            const string Username = "USERNAME";
            const string Password = "PASSWORD";

            var parameters = new {
                source = "google_search",
                domain = "nl",
                query = "adidas",
                start_page = 11,
                pages = 2,
                parse = true,
                context = new dynamic [] {
                    new { key = "results_language", value = "fr" },
                }
            };

            var client = new HttpClient();

            Uri baseUri = new Uri("https://realtime.oxylabs.io");
            client.BaseAddress = baseUri;

            var requestMessage = new HttpRequestMessage(HttpMethod.Post, "/v1/queries");
            requestMessage.Content = JsonContent.Create(parameters);

            var authenticationString = $"{Username}:{Password}";
            var base64EncodedAuthenticationString = Convert.ToBase64String(System.Text.ASCIIEncoding.UTF8.GetBytes(authenticationString));
            requestMessage.Headers.Add("Authorization", "Basic " + base64EncodedAuthenticationString);

            var response = await client.SendAsync(requestMessage);
            var contents = await response.Content.ReadAsStringAsync();

            Console.WriteLine(contents);
        }
    }
}

Java

package org.example;

import okhttp3.*;
import org.json.JSONArray;
import org.json.JSONObject;
import java.util.concurrent.TimeUnit;

public class Main implements Runnable {
    private static final String AUTHORIZATION_HEADER = "Authorization";
    public static final String USERNAME = "USERNAME";
    public static final String PASSWORD = "PASSWORD";

    public void run() {
        JSONObject jsonObject = new JSONObject();
        jsonObject.put("source", "google_search");
        jsonObject.put("domain", "nl");
        jsonObject.put("query", "adidas");
        jsonObject.put("start_page", 11);
        jsonObject.put("pages", 2);
        jsonObject.put("parse", true);
        jsonObject.put("context", new JSONArray()
                .put(new JSONObject()
                        .put("key", "results_language")
                        .put("value", "fr"))
        );

        Authenticator authenticator = (route, response) -> {
            String credential = Credentials.basic(USERNAME, PASSWORD);
            return response
                    .request()
                    .newBuilder()
                    .header(AUTHORIZATION_HEADER, credential)
                    .build();
        };

        var client = new OkHttpClient.Builder()
                .authenticator(authenticator)
                .readTimeout(180, TimeUnit.SECONDS)
                .build();

        var mediaType = MediaType.parse("application/json; charset=utf-8");
        var body = RequestBody.create(jsonObject.toString(), mediaType);
        var request = new Request.Builder()
                .url("https://realtime.oxylabs.io/v1/queries")
                .post(body)
                .build();

        try (var response = client.newCall(request).execute()) {
            if (response.body() != null) {
                try (var responseBody = response.body()) {
                    System.out.println(responseBody.string());
                }
            }
        } catch (Exception exception) {
            System.out.println("Error: " + exception.getMessage());
        }

        System.exit(0);
    }

    public static void main(String[] args) {
        new Thread(new Main()).start();
    }
}

JSON

{
    "source": "google_search",
    "domain": "nl",
    "query": "adidas",
    "start_page": 11,
    "pages": 2,
    "parse": true,
    "context": [
        {
            "key": "results_language",
            "value": "fr"
        }
    ]
}

We use synchronous Realtime integration method in our examples. If you would like to use Proxy Endpoint or asynchronous Push-Pull integration, refer to the integration methods section.

Request parameter values

Generic

Basic setup and customization options for scraping Google Web search results.

Parameter	Description	Default Value
source	Sets the scraper.	google_search
query	UTF-encoded keyword.	-
render		-
parse		false
callback_url		-
user_agent_type		desktop

- mandatory parameter

Google Advanced Search Operators

When scraping, you might find it useful to combine Google advanced search operators with your query. It enables you to customize the scope of the search, ensuring that the results are more relevant and focused. Explore these special commands here and here. See an example below.

{
    "source": "google_search",
    "query": "iphone 15 launch inurl:apple",
}

Localization

Adapt search results to specific geographical locations, domains, and languages.

Parameter	Description	Default Value
geo_location		-
domain		com
locale		-
context: results_language		-

Pagination

Controls for managing the pagination and retrieval of search results.

Parameter	Description	Default Value
start_page	Starting page number.	1
pages	Number of pages to retrieve.	1
limit	Number of results to retrieve in each page.	10
context:limit_per_page		-

Continuous scroll support

Web Scraper API fully supports Google Search continuous scroll. It automatically detects continuous scrolling layouts, efficiently loading the requested organic results without any extra parameters required.

Limit per page

To use this feature, include a JSON array with JSON objects containing the following data:

Parameter	Description	Example
page	The number of the page you would like to scrape. Any integer value greater than 0 will work	1
limit	The number of results on the page in question. Any integer value between 1 and 100 (inclusive) will work.	90

Request sample

{
    "source": "google_search",
    "query": "adidas",
    "parse": true,
    "context": [
        {
            "key": "limit_per_page",
            "value": [
                {"page": 1, "limit": 10},
                {"page": 2, "limit": 90}
                    ]
        }]
}

Filtering

Options to filter and refine search results based on various criteria. Learn how to use context parameters here.

Parameter	Description	Default Value
context: filter	Setting the value of this parameter to 0 lets you see results that would otherwise be excluded due to similarity to other results.	1
context: safe_search	Safe search. Set to true to enable it.	false
context: udm		-
context: tbm	To-be-matched or tbm parameter. Accepted values are: app, blg, bks, dsc, isch, nws, pts, plcs, rcp, lcl	-
context: tbs		-

udm and tbm context parameters cannot be used together in a single scraping request; please select only one. Using both simultaneously may lead to conflicts or unexpected behavior.

Other

Additional advanced settings and controls for specialized requirements.

Parameter	Description	Default Value
context: fpstate	Setting the fpstate value to aig will make Google load more apps. This parameter is only useful if used together with the render parameter.	-
context: nfpr	true will turn off spelling auto-correction	false

Context parameters

All context parameters should be added to the context array as objects with key and value pairs, e.g.:

...
"context": [
    {
        "key": "filter",
        "value": "0"
    }
]
...

Structured data

Web Scraper API is capable of extracting either an HTML or JSON object that contains Google search results, offering structured data on various elements of the results page.

google_search structured output

{
    "results": [
        {
            "content": {
                "url": "https://www.google.com/search?q=adidas&uule=w+CAIQICINdW5pdGVkIHN0YXRlcw&gl=us&hl=en",
                "page": 1,
                "results": {
                    "pla": {
                        "items": [
                            {
                                "pos": 1,
                                "url": "https://www.adidas.com/us/nmd_r1-primeblue-shoes/GZ9257.html?dfw_tracker=24819-GZ9257-0010",
                                "price": "$150.00",
                                "title": "NMD_R1 Primeblue Shoes Black M 8.5 / W 9.5 - Mens Originals Shoes",
                                "seller": "adidas",
                                "url_image": "https://encrypted-tbn1.gstatic.com/shopping?q=tbn:ANd9GcSZALnENHKXERktsOCnUSscRd4v0cSUw9E0XPaZwgyruM0Wuw-MhWcchAVzZRBdS-OP3_167R3jCg&usqp=CAc",
                                "image_data": "iVBORw0KGgoAAAANSUhE...JRU5ErkJggg=="
                            },
                            ...
                            {
                                "pos": 22,
                                "url": "https://www.adidas.com/us/ultraboost-5.0-dna-shoes/GY6452.html?dfw_tracker=24819-GY6452-0006",
                                "price": "$90.00",
                                "title": "Ultraboost 5.0 DNA Shoes Magic Grey 13K - Kids Originals Shoes",
                                "seller": "adidas",
                                "url_image": "https://encrypted-tbn3.gstatic.com/shopping?q=tbn:ANd9GcSTCgTZ1im_6ZX9YSCrjQfjVEnTmpmiqqYAHAhXzA9VhJdrokOeOnWsbEOxqA8zPkaos192xPF5ogA&usqp=CAc",
                                "image_data": "iVBORw0KGgoAAAANSUhEU...kJggg=="
                            }
                        ],
                        "pos_overall": 1
                    },
                    "paid": [],
                    "images": {
                        "items": [
                            {
                                "alt": "Image result for adidas",
                                "pos": 1,
                                "url": "/search?q=adidas&gl=us&h...E#imgrc=Dw-CBW17i8BqsM",
                                "source": "https://www.britannica.com/topic/Adidas-AG"
                            },
                            ...
                            {
                                "alt": "Image result for adidas",
                                "pos": 10,
                                "url": "/search?q=adidas&gl=us&hl=en&tbm=isch&sourXRM%252Cvz-MIh...QF6BAhLEAE#imgrc=_oOH_rR4W_-X-M",
                                "source": "https://www.shutterstock.com/search/adidas"
                            }
                        ],
                        "pos_overall": 11
                    },
                    "organic": [
                        {
                            "pos": 1,
                            "url": "https://www.adidas.com/us",
                            "desc": "adidas is about more than sportswear and workout clothes. We partner with the best in the industry to co-create. This way we offer our fans the sporting goods, ...",
                            "title": "adidas Official Website | adidas US",
                            "sitelinks": {
                                "expanded": [
                                    {
                                        "url": "https://www.adidas.com/us/women",
                                        "title": "Women"
                                    },
                                    ...
                                    {
                                        "url": "https://www.adidas.com/us/sale",
                                        "title": "Sale"
                                    }
                                ]
                            },
                            "url_shown": "https://www.adidas.com› ...",
                            "pos_overall": 2
                        },
                        ...
                        {
                            "pos": 5,
                            "url": "https://play.google.com/store/apps/details?id=com.adidas.app&hl=en_US&gl=US",
                            "desc": "YOUR ADIDAS - MORE THAN A SHOPPING APP The home of sport and sneakers, the adidas app puts you closer to the action with access to sneaker drops, ...Size: 115MUpdated: May 3, 2022 Rating: 4.8 · ‎322,391 votes · ‎Free · ‎Android · ‎Sports",
                            "title": "adidas - Apps on Google Play",
                            "url_shown": "https://play.google.com› store › apps › details › id=com....",
                            "pos_overall": 12
                        }
                    ],
                    "twitter": {
                        "pos": 1,
                        "url": "https://twitter.com/adidas",
                        "items": [
                            {
                                "pos": 1,
                                "url": "https://twitter.com/adidas/status/1521901157064531968?ref_src=twsrc%5Egoogle%7Ctwcamp%5Eserp%7Ctwgr%5Etweet",
                                "content": "What do you like to listen to most during workouts?",
                                "timeframe": "5 days ago"
                            },
                            ...
                            {
                                "pos": 3,
                                "url": "https://twitter.com/adidas/status/1519345069366652928?ref_src=twsrc%5Egoogle%7Ctwcamp%5Eserp%7Ctwgr%5Etweet",
                                "content": "How would you describe your favorite sport to an alien?",
                                "timeframe": "Apr 27, 2022"
                            }
                        ],
                        "title": "",
                        "pos_overall": 7
                    },
                    "knowledge": {
                        "title": "Adidas",
                        "factoids": [
                            {
                                "title": "Customer service chat",
                                "content": "Online Chat"
                            },
                            ...
                            {
                                "title": "Website",
                                "content": "http://www.adidas-group.com/"
                            }
                        ],
                        "profiles": [
                            {
                                "url": "https://www.instagram.com/adidas",
                                "title": "Instagram"
                            },
                            ...
                            {
                                "url": "https://www.youtube.com/user/adidas",
                                "title": "YouTube"
                            }
                        ],
                        "subtitle": "Design company",
                        "description": "Adidas AG is a German multinational corporation, founded and headquartered in Herzogenaurach, Bavaria, that designs and manufactures shoes, clothing and accessories. It is the largest sportswear manufacturer in Europe, and the second largest in the world, after Nike.",
                        "related_searches": [
                            {
                                "url": "/search?gl=us&hl=en&q=Nike&si=AC1wQDCwN61-ebmuwbQCO5QCrgOvEq5bkWeIzJ5JczItzAKNdbIxoHfFJ8UFY3e1CorwNJXI6gTWniM6aVu3nEoseblyAI-5N39_1F3DTWWABELjIdJa9ZgI_g5n-c9ZUZdzEZeV2VGOkDNPl5DwiBeyifKPveoCnBsiT3wVpKdr52FPqK5rzQSzspA-sjztjVcZy2WfOwFITo5EfSXyZ5AsknO1X-dzXUCOlIIGCvFWqi9OKgK5PaWp7EcXls7n8Ag_L7HC3Pgg2k5FWR631X7Hvi1268shAg%3D%3D&sa=X&ved=2ahUKEwiLlsjE8dH3AhUQkGoFHT2IBUQQxA16BAhdEAU",
                                "title": "Nike"
                            },
                            ...
                            {
                                "url": "/search?gl=us&hl=en&q=Asics&si=AC1wQDDagiMg03ncxeOQZbwVe-CJxRCchC-jr2hCPTxjc9wbgNZ1pCnWqLo_0N5RPhArBCY1qCgzESLe6Y9hB2HKnzquQNjJW9iLV6gQvDXzTwkl6LZHjq2aOuZII0W5uCWjpc0oOVU5JzyOjRUNEYoe_N_KsbL4xn4A0Fl2S4JMYAeaBZmu8jPX8EFVl_C3gerS4CRTyrfcL2bL-VhUNLEXVUlkFxrLlAyeZfUMVYSDo4WqV1c8-Bgp-rBwBzqW73Q7s1kOZHBtCQ8aBNfbfulPCljSG9vXBrcj9RqQRA8pD8TlH1rz8qE%3D&sa=X&ved=2ahUKEwiLlsjE8dH3AhUQkGoFHT2IBUQQxA16BAhdEAs",
                                "title": "Asics"
                            }
                        ]
                    },
                    "top_stories": {
                        "items": [
                            {
                                "pos": 1,
                                "url": "https://www.adidas.com/us/adilette-comfort-slides/GZ5898.html",
                                "title": "adidas Adilette Comfort Slides - Beige | women swim | adidas US",
                                "source": "",
                                "timeframe": "8 hours ago"
                            },
                            ...
                            },
                            {
                                "pos": 3,
                                "url": "https://www.adidas.com/us/women-clothing-sale",
                                "title": "Women's Clothing Sale Up to 50% Off | adidas US",
                                &qu