URL

The google_shopping source is designed to retrieve content from various Google Shopping URLs. This means that instead of sending multiple parameters, you can provide us with a direct URL to the required Google Shopping page. We do not strip any parameters or alter your URLs in any other way.

Request samples

In this example, we send a request with the specified URL to retrieve search results from Google Shopping for the keyword adidas. The search results are retrieved as if the search was conducted in New York, USA, as indicated by the geo_location parameter.

curl 'https://realtime.oxylabs.io/v1/queries' \
--user 'USERNAME:PASSWORD' \
-H 'Content-Type: application/json' \
-d '{
        "source": "google_shopping", 
        "url": "https://www.google.com/search?tbm=shop&q=adidas&hl=en", 
        "geo_location": "New York,New York,United States"
    }'

import requests
from pprint import pprint


# Structure payload.
payload = {
    'source': 'google_shopping',
    'url': 'https://www.google.com/search?tbm=shop&q=adidas&hl=en',
    'geo_location': 'New York,New York,United States'
}

# Get response.
response = requests.request(
    'POST',
    'https://realtime.oxylabs.io/v1/queries',
    auth=('USERNAME', 'PASSWORD'),
    json=payload,
)

# Instead of response with job status and results url, this will return the
# JSON response with results.
pprint(response.json())

const https = require("https");

const username = "USERNAME";
const password = "PASSWORD";
const body = {
    source: "google_shopping",
    url: "https://www.google.com/search?tbm=shop&q=adidas&hl=en",
    geo_location: "New York,New York,United States",
};

const options = {
    hostname: "realtime.oxylabs.io",
    path: "/v1/queries",
    method: "POST",
    headers: {
        "Content-Type": "application/json",
        Authorization:
            "Basic " + Buffer.from(`${username}:${password}`).toString("base64"),
    },
};

const request = https.request(options, (response) => {
    let data = "";

    response.on("data", (chunk) => {
        data += chunk;
    });

    response.on("end", () => {
        const responseData = JSON.parse(data);
        console.log(JSON.stringify(responseData, null, 2));
    });
});

request.on("error", (error) => {
    console.error("Error:", error);
});

request.write(JSON.stringify(body));
request.end();

# Parameters have to be encoded to escape special characters:
# URL: https://www.google.com/search?tbm=shop&q=adidas&hl=en
# geo_location: New York,New York,United States
# Encoded URL: https%3A%2F%2Fwww.google.com%2Fsearch%3Ftbm%3Dshop%26q%3Dadidas%26hl%3Den
# Encoded geo_location: New%20York%2CNew%20York%2CUnited%20States

https://realtime.oxylabs.io/v1/queries?source=google_shopping&url=https%3A%2F%2Fwww.google.com%2Fsearch%3Ftbm%3Dshop%26q%3Dadidas%26hl%3Den&geo_location=New%20York%2CNew%20York%2CUnited%20States&access_token=12345abcde

<?php

$params = array(
    'source' => 'google_shopping',
    'url' => 'https://www.google.com/search?tbm=shop&q=adidas&hl=en',
    'geo_location' => 'New York,New York,United States'
);

$ch = curl_init();

curl_setopt($ch, CURLOPT_URL, "https://realtime.oxylabs.io/v1/queries");
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($params));
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_USERPWD, "USERNAME" . ":" . "PASSWORD");

$headers = array();
$headers[] = "Content-Type: application/json";
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);

$result = curl_exec($ch);
echo $result;

if (curl_errno($ch)) {
    echo 'Error:' . curl_error($ch);
}
curl_close($ch);

package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
)

func main() {
	const Username = "USERNAME"
	const Password = "PASSWORD"

	payload := map[string]interface{}{
		"source":       "google_shopping",
		"url":          "https://www.google.com/search?tbm=shop&q=adidas&hl=en",
		"geo_location": "New York,New York,United States",
	}

	jsonValue, _ := json.Marshal(payload)

	client := &http.Client{}
	request, _ := http.NewRequest("POST",
		"https://realtime.oxylabs.io/v1/queries",
		bytes.NewBuffer(jsonValue),
	)

	request.SetBasicAuth(Username, Password)
	response, _ := client.Do(request)

	responseText, _ := ioutil.ReadAll(response.Body)
	fmt.Println(string(responseText))
}

using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Net.Http.Json;
using System.Threading.Tasks;

namespace OxyApi
{
    class Program
    {
        static async Task Main()
        {
            const string Username = "USERNAME";
            const string Password = "PASSWORD";

            var parameters = new {
                source = "google_shopping",
                url = "https://www.google.com/search?tbm=shop&q=adidas&hl=en",
                geo_location = "New York,New York,United States"
            };

            var client = new HttpClient();

            Uri baseUri = new Uri("https://realtime.oxylabs.io");
            client.BaseAddress = baseUri;

            var requestMessage = new HttpRequestMessage(HttpMethod.Post, "/v1/queries");
            requestMessage.Content = JsonContent.Create(parameters);

            var authenticationString = $"{Username}:{Password}";
            var base64EncodedAuthenticationString = Convert.ToBase64String(System.Text.ASCIIEncoding.UTF8.GetBytes(authenticationString));
            requestMessage.Headers.Add("Authorization", "Basic " + base64EncodedAuthenticationString);

            var response = await client.SendAsync(requestMessage);
            var contents = await response.Content.ReadAsStringAsync();

            Console.WriteLine(contents);
        }
    }
}

package org.example;

import okhttp3.*;
import org.json.JSONObject;
import java.util.concurrent.TimeUnit;

public class Main implements Runnable {
    private static final String AUTHORIZATION_HEADER = "Authorization";
    public static final String USERNAME = "USERNAME";
    public static final String PASSWORD = "PASSWORD";

    public void run() {
        JSONObject jsonObject = new JSONObject();
        jsonObject.put("source", "google_shopping");
        jsonObject.put("url", "https://www.google.com/search?tbm=shop&q=adidas&hl=en");
        jsonObject.put("geo_location", "New York,New York,United States");

        Authenticator authenticator = (route, response) -> {
            String credential = Credentials.basic(USERNAME, PASSWORD);
            return response
                    .request()
                    .newBuilder()
                    .header(AUTHORIZATION_HEADER, credential)
                    .build();
        };

        var client = new OkHttpClient.Builder()
                .authenticator(authenticator)
                .readTimeout(180, TimeUnit.SECONDS)
                .build();

        var mediaType = MediaType.parse("application/json; charset=utf-8");
        var body = RequestBody.create(jsonObject.toString(), mediaType);
        var request = new Request.Builder()
                .url("https://realtime.oxylabs.io/v1/queries")
                .post(body)
                .build();

        try (var response = client.newCall(request).execute()) {
            if (response.body() != null) {
                try (var responseBody = response.body()) {
                    System.out.println(responseBody.string());
                }
            }
        } catch (Exception exception) {
            System.out.println("Error: " + exception.getMessage());
        }

        System.exit(0);
    }

    public static void main(String[] args) {
        new Thread(new Main()).start();
    }
}

{
    "source": "google_shopping", 
    "url": "https://www.google.com/search?tbm=shop&q=adidas&hl=en", 
    "geo_location": "New York,New York,United States"
}

Output example

{
    "results": [
        {
            "content": "<!doctype html>
            CONTENT
            </html>",
            "created_at": "2024-07-01 10:57:20",
            "updated_at": "2024-07-01 10:57:26",
            "page": 1,
            "url": "https://www.google.com/search?tbm=shop&q=adidas&hl=en",
            "job_id": "7213495890735035393",
            "status_code": 200
        }
    ]
}

We use synchronous Realtime integration method in our examples. If you would like to use Proxy Endpoint or asynchronous Push-Pull integration, refer to the integration methods section.

Request parameter values

Generic

Basic setup and customization for scraping Google Shopping pages with offers for a chosen product ID.

Parameter

Description

Default Value

source

Sets the scraper.

google_shopping

url

Direct URL (link) to Google page.

render

Enables JavaScript rendering when set to html. More info.

parse

Returns parsed data when set to true. Limited to URLs of specific Google page types.

false

callback_url

URL to your callback endpoint. More info.

user_agent_type

Device type and browser. The full list can be found here.

desktop

geo_location

The geographical location that the result should be adapted for. Using this parameter correctly is extremely important to get the right data. For more information, read about our suggested geo_location parameter structures here.

- mandatory parameter

PreviousProduct Pricing NextWalmart

Last updated 5 months ago