Google

See all types of Google pages that you can scrape and parse.

With Web Scraper API, you can scrape and parse various types of Google pages; below is an overview of all supported scrapers and their respective source values.

Source	Description	Dedicated parser
google_search	SERPs (Web, Image, News, AI Overviews).	✓
google_ads	SERPs, optimized for maximum ad rate.	✓
google_ai_mode	Google AI Mode conversational responses.	✓
google_lens	Google Lens service.	✓
google_maps	Local search service.	–
google_scholar	Google Scholar service.	✓
google_shopping_search	Search results for a search term of your choice.	✓
google_shopping_product	Product page of a product token of your choice.	✓
google_trends_explore	Trends.	–
google_travel_hotels	Travel: Hotels service.	✓
google	Submit any Google URL you like.	Only specific Google page types.

Although we do not have dedicated parsers for each Google source, you can write your own parsing instructions with Custom Parser feature and get structured data.

Getting started

Create your API user credentials: Sign up for a free trial or purchase the product in the Oxylabs dashboard to create your API user credentials (USERNAME and PASSWORD).

If you need more than one API user for your account, please contact our customer support or message our 24/7 live chat support.

Request sample

cURL

curl 'https://realtime.oxylabs.io/v1/queries' \
--user 'USERNAME:PASSWORD' \
-H 'Content-Type: application/json' \
-d '{
        "source": "google_search",
        "query": "adidas",
        "geo_location": "California,United States",
        "parse": true
    }'

Python

import requests
from pprint import pprint

# Structure payload.
payload = {
    'source': 'google_search',
    'query': 'adidas',
    'geo_location': 'California,United States',
    'parse': True
}

# Get response.
response = requests.request(
    'POST',
    'https://realtime.oxylabs.io/v1/queries',
    auth=('USERNAME', 'PASSWORD'),
    json=payload,
)

# Print prettified response to stdout.
pprint(response.json())

Node.js

const https = require("https");

const username = "USERNAME";
const password = "PASSWORD";
const body = {
    source: "google_search",
    query: "adidas",
    geo_location: "California,United States",
    parse: true,
};

const options = {
    hostname: "realtime.oxylabs.io",
    path: "/v1/queries",
    method: "POST",
    headers: {
        "Content-Type": "application/json",
        Authorization:
            "Basic " + Buffer.from(`${username}:${password}`).toString("base64"),
    },
};

const request = https.request(options, (response) => {
    let data = "";

    response.on("data", (chunk) => {
        data += chunk;
    });

    response.on("end", () => {
        const responseData = JSON.parse(data);
        console.log(JSON.stringify(responseData, null, 2));
    });
});

request.on("error", (error) => {
    console.error("Error:", error);
});

request.write(JSON.stringify(body));
request.end();

PHP

<?php

$params = array(
    'source' => 'google_search',
    'query' => 'adidas',
    'geo_location' => 'California,United States',
    'parse' => true
);

$ch = curl_init();

curl_setopt($ch, CURLOPT_URL, "https://realtime.oxylabs.io/v1/queries");
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($params));
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_USERPWD, "USERNAME" . ":" . "PASSWORD");

$headers = array();
$headers[] = "Content-Type: application/json";
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);

$result = curl_exec($ch);
echo $result;

if (curl_errno($ch)) {
    echo 'Error:' . curl_error($ch);
}
curl_close($ch);

C#

using System;
using System.Net.Http;
using System.Net.Http.Json;
using System.Threading.Tasks;

namespace OxyApi
{
    class Program
    {
        static async Task Main()
        {
            const string Username = "USERNAME";
            const string Password = "PASSWORD";

            var parameters = new {
                source = "google_search",
                query = "adidas",
                geo_location = "California,United States",
                parse = true
            };

            var client = new HttpClient();

            Uri baseUri = new Uri("https://realtime.oxylabs.io");
            client.BaseAddress = baseUri;

            var requestMessage = new HttpRequestMessage(HttpMethod.Post, "/v1/queries");
            requestMessage.Content = JsonContent.Create(parameters);

            var authenticationString = $"{Username}:{Password}";
            var base64EncodedAuthenticationString = Convert.ToBase64String(System.Text.Encoding.ASCII.GetBytes(authenticationString));
            requestMessage.Headers.Add("Authorization", "Basic " + base64EncodedAuthenticationString);

            var response = await client.SendAsync(requestMessage);
            var contents = await response.Content.ReadAsStringAsync();

            Console.WriteLine(contents);
        }
    }
}

Golang

package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
)

func main() {
	const Username = "USERNAME"
	const Password = "PASSWORD"

	payload := map[string]interface{}{
		"source":       "google_search",
		"query":        "adidas",
		"geo_location": "California,United States",
		"parse":        true,
	}

	jsonValue, _ := json.Marshal(payload)

	client := &http.Client{}
	request, _ := http.NewRequest("POST",
		"https://realtime.oxylabs.io/v1/queries",
		bytes.NewBuffer(jsonValue),
	)

	request.SetBasicAuth(Username, Password)
	request.Header.Set("Content-Type", "application/json")
	response, _ := client.Do(request)

	responseText, _ := ioutil.ReadAll(response.Body)
	fmt.Println(string(responseText))
}

HTTP

https://realtime.oxylabs.io/v1/queries?source=google_search&query=adidas&geo_location=California%2CUnited%20States&parse=true&access_token=12345abcde

Java

package org.example;

import okhttp3.*;
import org.json.JSONObject;
import java.util.concurrent.TimeUnit;

public class Main implements Runnable {
    private static final String AUTHORIZATION_HEADER = "Authorization";
    public static final String USERNAME = "USERNAME";
    public static final String PASSWORD = "PASSWORD";

    public void run() {
        JSONObject jsonObject = new JSONObject();
        jsonObject.put("source", "google_search");
        jsonObject.put("query", "adidas");
        jsonObject.put("geo_location", "California,United States");
        jsonObject.put("parse", true);

        Authenticator authenticator = (route, response) -> {
            String credential = Credentials.basic(USERNAME, PASSWORD);
            return response
                    .request()
                    .newBuilder()
                    .header(AUTHORIZATION_HEADER, credential)
                    .build();
        };

        var client = new OkHttpClient.Builder()
                .authenticator(authenticator)
                .readTimeout(180, TimeUnit.SECONDS)
                .build();

        var mediaType = MediaType.parse("application/json; charset=utf-8");
        var body = RequestBody.create(jsonObject.toString(), mediaType);
        var request = new Request.Builder()
                .url("https://realtime.oxylabs.io/v1/queries")
                .post(body)
                .build();

        try (var response = client.newCall(request).execute()) {
            if (response.body() != null) {
                try (var responseBody = response.body()) {
                    System.out.println(responseBody.string());
                }
            }
        } catch (Exception exception) {
            System.out.println("Error: " + exception.getMessage());
        }

        System.exit(0);
    }

    public static void main(String[] args) {
        new Thread(new Main()).start();
    }
}

JSON

{
    "source": "google_search",
    "query": "adidas",
    "geo_location": "California,United States",
    "parse": true
}

google_search structured output

{
    "results": [
        {
            "content": {
                "url": "https://www.google.com/search?q=adidas&uule=w+CAIQICINdW5pdGVkIHN0YXRlcw&gl=us&hl=en",
                "page": 1,
                "results": {
                    "pla": {
                        "items": [
                            {
                                "pos": 1,
                                "url": "https://www.adidas.com/us/nmd_r1-primeblue-shoes/GZ9257.html?dfw_tracker=24819-GZ9257-0010",
                                "price": "$150.00",
                                "title": "NMD_R1 Primeblue Shoes Black M 8.5 / W 9.5 - Mens Originals Shoes",
                                "seller": "adidas",
                                "url_image": "https://encrypted-tbn1.gstatic.com/shopping?q=tbn:ANd9GcSZALnENHKXERktsOCnUSscRd4v0cSUw9E0XPaZwgyruM0Wuw-MhWcchAVzZRBdS-OP3_167R3jCg&usqp=CAc",
                                "image_data": "iVBORw0KGgoAAAANSUhEU...JRU5ErkJggg=="
                            },
                            {
                                "pos": 22,
                                "url": "https://www.adidas.com/us/ultraboost-5.0-dna-shoes/GY6452.html?dfw_tracker=24819-GY6452-0006",
                                "price": "$90.00",
                                "title": "Ultraboost 5.0 DNA Shoes Magic Grey 13K - Kids Originals Shoes",
                                "seller": "adidas",
                                "url_image": "https://encrypted-tbn3.gstatic.com/shopping?q=tbn:ANd9GcSTCgTZ1im_6ZX9YSCrjQfjVEnTmpmiqqYAHAhXzA9VhJdrokOeOnWsbEOxqA8zPkaos192xPF5ogA&usqp=CAc",
                                "image_data": "iVBORw0KGgoAAAANSUhEU...kJggg=="
                            }
                        ],
                        "pos_overall": 1
                    },
                    "paid": [],
                    "images": {
                        "items": [
                            {
                                "alt": "Image result for adidas",
                                "pos": 1,
                                "url": "/search?q=adidas&gl=us&h...E#imgrc=Dw-CBW17i8BqsM",
                                "source": "https://www.britannica.com/topic/Adidas-AG"
                            },
                            {
                                "alt": "Image result for adidas",
                                "pos": 10,
                                "url": "/search?q=adidas&gl=us&hl=en&tbm=isch&sourXRM%252Cvz-MIh...QF6BAhLEAE#imgrc=_oOH_rR4W_-X-M",
                                "source": "https://www.shutterstock.com/search/adidas"
                            }
                        ],
                        "pos_overall": 11
                    },
                    "organic": [
                        {
                            "pos": 1,
                            "url": "https://www.adidas.com/us",
                            "desc": "adidas is about more than sportswear and workout clothes. We partner with the best in the industry to co-create. This way we offer our fans the sporting goods, ...",
                            "title": "adidas Official Website | adidas US",
                            "sitelinks": {
                                "expanded": [
                                    {
                                        "url": "https://www.adidas.com/us/women",
                                        "title": "Women"
                                    },
                                    {
                                        "url": "https://www.adidas.com/us/sale",
                                        "title": "Sale"
                                    }
                                ]
                            },
                            "url_shown": "https://www.adidas.com› ...",
                            "pos_overall": 2
                        },
                        {
                            "pos": 5,
                            "url": "https://play.google.com/store/apps/details?id=com.adidas.app&hl=en_US&gl=US",
                            "desc": "YOUR ADIDAS - MORE THAN A SHOPPING APP The home of sport and sneakers, the adidas app puts you closer to the action with access to sneaker drops, ...Size: 115MUpdated: May 3, 2022 Rating: 4.8 · ‎322,391 votes · ‎Free · ‎Android · ‎Sports",
                            "title": "adidas - Apps on Google Play",
                            "url_shown": "https://play.google.com› store › apps › details › id=com....",
                            "pos_overall": 12
                        }
                    ],
                    "twitter": {
                        "pos": 1,
                        "url": "https://twitter.com/adidas",
                        "items": [
                            {
                                "pos": 1,
                                "url": "https://twitter.com/adidas/status/1521901157064531968?ref_src=twsrc%5Egoogle%7Ctwcamp%5Eserp%7Ctwgr%5Etweet",
                                "content": "What do you like to listen to most during workouts?",
                                "timeframe": "5 days ago"
                            },
                            {
                                "pos": 3,
                                "url": "https://twitter.com/adidas/status/1519345069366652928?ref_src=twsrc%5Egoogle%7Ctwcamp%5Eserp%7Ctwgr%5Etweet",
                                "content": "How would you describe your favorite sport to an alien?",
                                "timeframe": "Apr 27, 2022"
                            }
                        ],
                        "title": "",
                        "pos_overall": 7
                    },
                    "knowledge": {
                        "title": "Adidas",
                        "factoids": [
                            {
                                "title": "Customer service chat",
                                "content": "Online Chat"
                            },
                            {
                                "title": "Website",
                                "content": "http://www.adidas-group.com/"
                            }
                        ],
                        "profiles": [
                            {
                                "url": "https://www.instagram.com/adidas",
                                "title": "Instagram"
                            },
                            {
                                "url": "https://www.youtube.com/user/adidas",
                                "title": "YouTube"
                            }
                        ],
                        "subtitle": "Design company",
                        "description": "Adidas AG is a German multinational corporation, founded and headquartered in Herzogenaurach, Bavaria, that designs and manufactures shoes, clothing and accessories. It is the largest sportswear manufacturer in Europe, and the second largest in the world, after Nike.",
                        "related_searches": [
                            {
                                "url": "/search?gl=us&hl=en&q=Nike&si=AC1wQDCwN61-ebmuwbQCO5QCrgOvEq5bkWeIzJ5JczItzAKNdbIxoHfFJ8UFY3e1CorwNJXI6gTWniM6aVu3nEoseblyAI-5N39_1F3DTWWABELjIdJa9ZgI_g5n-c9ZUZdzEZeV2VGOkDNPl5DwiBeyifKPveoCnBsiT3wVpKdr52FPqK5rzQSzspA-sjztjVcZy2WfOwFITo5EfSXyZ5AsknO1X-dzXUCOlIIGCvFWqi9OKgK5PaWp7EcXls7n8Ag_L7HC3Pgg2k5FWR631X7Hvi1268shAg%3D%3D&sa=X&ved=2ahUKEwiLlsjE8dH3AhUQkGoFHT2IBUQQxA16BAhdEAU",
                                "title": "Nike"
                            },
                            {
                                "url": "/search?gl=us&hl=en&q=Asics&si=AC1wQDDagiMg03ncxeOQZbwVe-CJxRCchC-jr2hCPTxjc9wbgNZ1pCnWqLo_0N5RPhArBCY1qCgzESLe6Y9hB2HKnzquQNjJW9iLV6gQvDXzTwkl6LZHjq2aOuZII0W5uCWjpc0oOVU5JzyOjRUNEYoe_N_KsbL4xn4A0Fl2S4JMYAeaBZmu8jPX8EFVl_C3gerS4CRTyrfcL2bL-VhUNLEXVUlkFxrLlAyeZfUMVYSDo4WqV1c8-Bgp-rBwBzqW73Q7s1kOZHBtCQ8aBNfbfulPCljSG9vXBrcj9RqQRA8pD8TlH1rz8qE%3D&sa=X&ved=2ahUKEwiLlsjE8dH3AhUQkGoFHT2IBUQQxA16BAhdEAs",
                                "title": "Asics"
                            }
                        ]
                    },
                    "top_stories": {
                        "items": [
                            {
                                "pos": 1,
                                "url": "https://www.adidas.com/us/adilette-comfort-slides/GZ5898.html",
                                "title": "adidas Adilette Comfort Slides - Beige | women swim | adidas US",
                                "source": "",
                                "timeframe": "8 hours ago"
                            },
                            {
                                "pos": 3,
                                "url": "https://www.adidas.com/us/women-clothing-sale",
                                "title": "Women's Clothing Sale Up to 50% Off | adidas US",
                                "source": "",
                                "timeframe": "Sep 30, 2021"
                            }
                        ],
                        "pos_overall": 3
                    },
                    "instant_answers": [
                        {
                            "type": "unknown",
                            "_parsed": false,
                            "pos_overall": 5
                        }
                    ],
                    "popular_products": {
                        "items": [
                            {
                                "pos": 1,
                                "title": "Adidas Yeezy Kids Foam Runner",
                                "image_data": "UklGRhwJAABXRUJQVlA...rAZNVIAAA"
                            },
                            {
                                "pos": 10,
                                "title": "Adidas Men's Kaptir 2.0",
                                "image_data": "UklGRhQNAAB...+wTh1cLB/uQAAAAAAAAA="
                            }
                        ],
                        "pos_overall": 4
                    },
                    "related_searches": {
                        "pos_overall": 14,
                        "related_searches": [
                            "adidas shoes",
                            "adidas yeezy",
                            "adidas outlet",
                            "adidas pants",
                            "adidas sneakers",
                            "adidas superstar",
                            "adidas wikipedia"
                        ]
                    },
                    "search_information": {
                        "query": "adidas",
                        "showing_results_for": "adidas",
                        "total_results_count": 1440000000
                    },
                    "total_results_count": 1440000000,
                },
                "last_visible_page": 10,
                "parse_status_code": 12000
            },
            "created_at": "2022-05-09 07:24:16",
            "updated_at": "2022-05-09 07:24:20",
            "page": 1,
            "url": "https://www.google.com/search?q=adidas&uule=w+CAIQICINdW5pdGVkIHN0YXRlcw&gl=us&hl=en",
            "job_id": "6929330182268395521",
            "status_code": 200,
            "parser_type": "v2"
        }
    ]
}

We use synchronous Realtime integration method in our examples. If you would like to use Proxy Endpoint or asynchronous Push-Pull integration, refer to the integration methods section.

Optionally, you can include additional parameters such as geo_location, user_agent_type, parse, render and more to customize your scraping request. Read more:

Features