The google_shopping_search
source is designed to retrieve Google Shopping search results.
Request samples
In this example, we make a request to retrieve the first 4
pages of Google Shopping search for the search term adidas
, sorted by descending price and minimum price of $20
.
cURL Python Node.js HTTP PHP Golang C# Java JSON
Copy curl 'https://realtime.oxylabs.io/v1/queries' \
--user 'USERNAME:PASSWORD' \
-H 'Content-Type: application/json' \
-d '{
"source": "google_shopping_search",
"domain": "com",
"query": "adidas",
"pages": 4,
"parse": true,
"context": [
{
"key": "sort_by",
"value": "pd"
},
{
"key": "min_price",
"value": 20
}
]
}'
Copy import requests
from pprint import pprint
# Structure payload.
payload = {
'source' : 'google_shopping_search' ,
'domain' : 'com' ,
'query' : 'adidas' ,
'pages' : 4 ,
'parse' : True ,
'context' : [
{ 'key' : 'sort_by' , 'value' : 'pd' },
{ 'key' : 'min_price' , 'value' : 20 },
] ,
}
# Get response.
response = requests . request (
'POST' ,
'https://realtime.oxylabs.io/v1/queries' ,
auth = ( 'USERNAME' , 'PASSWORD' ),
json = payload,
)
# Print prettified response to stdout.
pprint (response. json ())
Copy const https = require ( "https" );
const username = "USERNAME" ;
const password = "PASSWORD" ;
const body = {
source : "google_shopping_search" ,
domain : "com" ,
query : "adidas" ,
pages : 4 ,
parse : true ,
context : [
{
key : "sort_by" ,
value : "pd"
} ,
{
key : "min_price" ,
value : 20
} ,
] ,
};
const options = {
hostname : "realtime.oxylabs.io" ,
path : "/v1/queries" ,
method : "POST" ,
headers : {
"Content-Type" : "application/json" ,
Authorization :
"Basic " + Buffer .from ( ` ${ username } : ${ password } ` ) .toString ( "base64" ) ,
} ,
};
const request = https .request (options , (response) => {
let data = "" ;
response .on ( "data" , (chunk) => {
data += chunk;
});
response .on ( "end" , () => {
const responseData = JSON .parse (data);
console .log ( JSON .stringify (responseData , null , 2 ));
});
});
request .on ( "error" , (error) => {
console .error ( "Error:" , error);
});
request .write ( JSON .stringify (body));
request .end ();
Copy https://realtime.oxylabs.io/v1/queries?source=google_shopping_search&domain=com&query=adidas&pages=4&parse=true&context[0][key]=sort_by&context[0][value]=pd&context[1][key]=min_price&context[1][value]=20&access_token=12345abcde
Copy <? php
$params = array (
'source' => 'google_shopping_search' ,
'domain' => 'com' ,
'query' => 'adidas' ,
'pages' => 4 ,
'parse' => true ,
'context' => [
[ 'key' => 'sort_by' , 'value' => 'pd' ] ,
[ 'key' => 'min_price' , 'value' => 20 ]
]
);
$ch = curl_init () ;
curl_setopt ( $ch , CURLOPT_URL , "https://realtime.oxylabs.io/v1/queries" ) ;
curl_setopt ( $ch , CURLOPT_RETURNTRANSFER , 1 ) ;
curl_setopt ( $ch , CURLOPT_POSTFIELDS , json_encode ( $params )) ;
curl_setopt ( $ch , CURLOPT_POST , 1 ) ;
curl_setopt ( $ch , CURLOPT_USERPWD , "USERNAME" . ":" . "PASSWORD" ) ;
$headers = array ();
$headers[] = "Content-Type: application/json" ;
curl_setopt ( $ch , CURLOPT_HTTPHEADER , $headers ) ;
$result = curl_exec ( $ch ) ;
echo $result;
if ( curl_errno ( $ch ) ) {
echo 'Error:' . curl_error ( $ch ) ;
}
curl_close ( $ch ) ;
Copy package main
import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
)
func main () {
const Username = "USERNAME"
const Password = "PASSWORD"
payload := map [ string ] interface {}{
"source" : "google_shopping_search" ,
"domain" : "com" ,
"query" : "adidas" ,
"pages" : 4 ,
"parse" : true ,
"context" : [] map [ string ] interface {}{
{ "key" : "sort_by" , "value" : "pd" },
{ "key" : "min_price" , "value" : 20 },
},
}
jsonValue, _ := json. Marshal (payload)
client := & http . Client {}
request, _ := http. NewRequest ( "POST" ,
"https://realtime.oxylabs.io/v1/queries" ,
bytes. NewBuffer (jsonValue),
)
request. SetBasicAuth (Username, Password)
response, _ := client. Do (request)
responseText, _ := ioutil. ReadAll (response.Body)
fmt. Println ( string (responseText))
}
Copy using System ;
using System . Collections . Generic ;
using System . Net . Http ;
using System . Net . Http . Json ;
using System . Threading . Tasks ;
namespace OxyApi
{
class Program
{
static async Task Main ()
{
const string Username = "USERNAME" ;
const string Password = "PASSWORD" ;
var parameters = new {
source = "google_shopping_search" ,
domain = "com" ,
query = "adidas" ,
pages = 4 ,
parse = true ,
context = new dynamic [] {
new { key = "sort_by" , value = "pd" } ,
new { key = "min_price" , value = "20" }
}
};
var client = new HttpClient ();
Uri baseUri = new Uri ( "https://realtime.oxylabs.io" );
client . BaseAddress = baseUri;
var requestMessage = new HttpRequestMessage ( HttpMethod . Post , "/v1/queries" );
requestMessage . Content = JsonContent . Create (parameters);
var authenticationString = $"{Username}:{Password}" ;
var base64EncodedAuthenticationString = Convert.ToBase64String(System.Text.ASCIIEncoding.UTF8.GetBytes(authenticationString));
requestMessage . Headers . Add ( "Authorization" , "Basic " + base64EncodedAuthenticationString);
var response = await client . SendAsync (requestMessage);
var contents = await response . Content . ReadAsStringAsync ();
Console . WriteLine (contents);
}
}
}
Copy package org . example ;
import okhttp3 . * ;
import org . json . JSONArray ;
import org . json . JSONObject ;
import java . util . concurrent . TimeUnit ;
public class Main implements Runnable {
private static final String AUTHORIZATION_HEADER = "Authorization" ;
public static final String USERNAME = "USERNAME" ;
public static final String PASSWORD = "PASSWORD" ;
public void run () {
JSONObject jsonObject = new JSONObject() ;
jsonObject . put ( "source" , "google_shopping_search" );
jsonObject . put ( "domain" , "com" );
jsonObject . put ( "query" , "adidas" );
jsonObject . put ( "pages" , 4 );
jsonObject . put ( "parse" , true );
jsonObject . put ( "context" , new JSONArray()
. put ( new JSONObject()
. put ( "key" , "sort_by" )
. put ( "value" , "pd" ))
. put ( new JSONObject()
. put ( "key" , "min_price" )
. put ( "value" , 20 ))
);
Authenticator authenticator = (route , response) -> {
String credential = Credentials . basic (USERNAME , PASSWORD);
return response
. request ()
. newBuilder ()
. header (AUTHORIZATION_HEADER , credential)
. build ();
};
var client = new OkHttpClient . Builder ()
. authenticator (authenticator)
. readTimeout ( 180 , TimeUnit . SECONDS )
. build ();
var mediaType = MediaType . parse ( "application/json; charset=utf-8" );
var body = RequestBody . create ( jsonObject . toString () , mediaType);
var request = new Request . Builder ()
. url ( "https://realtime.oxylabs.io/v1/queries" )
. post (body)
. build ();
try ( var response = client . newCall (request) . execute ()) {
if ( response . body () != null ) {
try ( var responseBody = response . body ()) {
System . out . println ( responseBody . string ());
}
}
} catch ( Exception exception) {
System . out . println ( "Error: " + exception . getMessage ());
}
System . exit ( 0 );
}
public static void main ( String [] args) {
new Thread( new Main()) . start ();
}
}
Copy {
"source" : "google_shopping_search" ,
"domain" : "com" ,
"query" : "adidas" ,
"pages" : 4 ,
"parse" : true ,
"context" : [
{
"key" : "sort_by" ,
"value" : "pd"
} ,
{
"key" : "min_price" ,
"value" : 20
}
]
}
We use synchronous Realtime integration method in our examples. If you would like to use Proxy Endpoint or asynchronous Push-Pull integration, refer to the integration methods section.
Request parameter values
Generic
Basic setup and customization options for scraping Google Shopping search results.
Parameter Description Default Value Enables JavaScript rendering when set to html
. More info .
Device type and browser. The full list can be found here .
- mandatory parameter
Localization
Adapt results to specific geographical locations, domains, and languages.
Parameter Description Default Value The geographical location that the result should be adapted for. Using this parameter correctly is extremely important to get the right data. For more information, read about our suggested geo_location
parameter structures here .
Domain localization for Google. The full list of available domains can be found here .
Accept-Language
header value which changes your Google search page web interface language. More info .
context
:
results_language
Results language. List of supported Google languages can be found here .
Controls for managing the pagination and retrieval of search results.
Parameter Description Default Value Number of pages to retrieve.
Filtering
Advanced options for tailoring and refining the search context.
Parameter Description Default Value Sort product list by a given criteria. r
applies default Google sorting, rv
- by review score, p
- by price ascending, pd
- by price descending.
Minimum price of products to filter.
Maximum price of products to filter.
Other
Additional advanced settings and controls for specialized requirements.
Parameter Description Default Value true
will turn off spelling auto-correction.
Structured data
Below you can find a structured output example for google_shopping_search
.
Output data dictionary
HTML example
JSON structure
The table below presents a detailed list of each search page element we parse, along with its description and data type. The table also includes some metadata.
Key Description Type The URL to the Google Shopping search page for the query.
The current page number of the search results.
An object containing detailed search results.
A list of product listing ads with their respective details.
A list of unpaid listings with their respective details.
A list of details for the submitted search query.
The original search term.
search_information.showing_results_for
The search term the search results are shown for. `query` and `showing_results_for` may differ if Google auto-corrected the provided search term.
Value identifying the maximum page number visible in the search query results page. (-1 when loading of more results is initiated by scrolling).
The status code of the parsing job. You can see the parser status codes described here .
The timestamp when the scraping job was created.
The timestamp when the scraping job was finished.
The status code of the scraping job. You can see the scraper status codes described here .
The ID of the job associated with the scraping job.
In the following sections, parsed JSON code snippets are shortened where more than one item for the result type is available.
Paid Listing Ads
An array of objects containing Product Listing Ads (PLA) for the product.
Copy ...
"pla" : [
{
"items" : [
{
"pos" : 1 ,
"url": "/aclk?sa=l&ai=DChcSEwiY8fLUi9OGAxVtj1AGHYnVBj0YABABGgJkZw&gclid=EAIaIQobChMImPHy1IvThgMVbY9QBh2J1QY9EAQYASABEgKpS_D_BwE&sig=AOD64_2DguiyFTR4GRY6Ww9o__l9HgJC_A&ctype=5&q=&ved=0ahUKEwj-6ezUi9OGAxWiWUEAHdbxAgsQww8I2xA&adurl=",
"price" : "$2,199.00" ,
"title" : "Polycade Sente: Black" ,
"seller" : "Polycade" ,
"thumbnail": "https://encrypted-tbn0.gstatic.com/shopping?q=tbn:ANd9GcS59ZNOrZH96cy_cOgzxL52VoJYq9iPl7q8g26f9odcuG8pY8ZRxe9YMhkZDPnFAZDyP04lu29gy57ObwsKpWHb_pzQBja34tkErnSAz3nw&usqp=CAE"
} ,
{
"pos" : 2 ,
"url": "/aclk?sa=l&ai=DChcSEwiY8fLUi9OGAxVtj1AGHYnVBj0YABADGgJkZw&gclid=EAIaIQobChMImPHy1IvThgMVbY9QBh2J1QY9EAQYAiABEgJwHvD_BwE&sig=AOD64_0LFB8jrHwNdEkmOdjcjGOdhQ9ZVg&ctype=5&q=&ved=0ahUKEwj-6ezUi9OGAxWiWUEAHdbxAgsQww8I3hA&adurl=",
"price" : "$2,199.00" ,
"title" : "Polycade Sente: White" ,
"seller" : "Polycade" ,
"thumbnail": "https://encrypted-tbn2.gstatic.com/shopping?q=tbn:ANd9GcQ2onFg_aXbg8LTX3qJT9f9XdiFrl_SNLXlpKhSjCQQ2c5EmQcrNXPwCMphjugJUhWctBpRVC0BiS4OUnq0FRAeQ4BXEWI6FuvZvGERsLc&usqp=CAE"
} ,
...
] ,
"pos_overall" : 1
}
],
...
Key (pla) Description Type All PLAs available within the page.
An indicator denoting the position of a given item among PLA results.
The price of the product in the listing ad.
The title of the product in the listing ad.
The rating of the product.
The seller of the product in the listing ad.
The URL of the thumbnail image of the product.
The count of reviews for the product.
An indication of the position of the result within the SERP.
Filters
Copy ...
"filters" : [
{
"name" : "Show only" ,
"values" : [
{
"url": "/search?sca_esv=bbd3241cb3940ce2&sca_upv=1&gl=us&hl=en&tbm=shop&q=adidas&tbs=mr:1,sales:1&sa=X&ved=0ahUKEwikoMX_iNOGAxVvFbkGHV6uDZcQ7KEGCJ4WKAA",
"value" : "On sale"
}
]
} ,
{
"name" : "Price" ,
"values" : [
{
"url": "/search?sca_esv=bbd3241cb3940ce2&sca_upv=1&gl=us&hl=en&tbm=shop&q=adidas&tbs=mr:1,price:1,ppr_max:40&sa=X&ved=0ahUKEwikoMX_iNOGAxVvFbkGHV6uDZcQvSsIohYoAA",
"value" : "Up to $40"
} ,
...
{
"url": "/search?sca_esv=bbd3241cb3940ce2&sca_upv=1&gl=us&hl=en&tbm=shop&q=adidas&tbs=mr:1,price:1,ppr_min:90&sa=X&ved=0ahUKEwikoMX_iNOGAxVvFbkGHV6uDZcQvSsIpRYoAw",
"value" : "Over $90"
}
]
} ,
{
"name" : "Color" ,
"values" : [
{
"url": "/search?sca_esv=bbd3241cb3940ce2&sca_upv=1&gl=us&hl=en&tbm=shop&q=adidas&tbs=mr:1,color:specific,color_val:black&sa=X&ved=0ahUKEwikoMX_iNOGAxVvFbkGHV6uDZcQtSsIrBYoAA",
"value" : "Black"
} ,
...
{
"url": "/search?sca_esv=bbd3241cb3940ce2&sca_upv=1&gl=us&hl=en&tbm=shop&q=adidas&tbs=mr:1,color:specific,color_val:pink&sa=X&ved=0ahUKEwikoMX_iNOGAxVvFbkGHV6uDZcQtSsIshYoBg",
"value" : "Pink"
}
]
} ,
...
]
Key (filters) Description Type The name of the filter category
Filter options available within the category.
The URL representing the filtered search query for this filter option.
The display name of the filter option
values.merchant_id
(optional)
The ID of the merchant associated with this filter option.
Organic
An array of objects containing details of organic search results.
Copy ...
"organic" : [
{
"pos" : 1 ,
"url": "/shopping/product/1503163696221055935?q=adidas&uule=w+CAIQICINdW5pdGVkIHN0YXRlcw&gl=us&hl=en&prds=eto:9260750834573489043_0,pid:17425630667348523786,rsk:PC_12455715925962143981&sa=X&ved=0ahUKEwikoMX_iNOGAxVvFbkGHV6uDZcQ8gIInBgoAA",
"type" : "grid" ,
"price" : 100 ,
"title" : "Adidas Samba OG 'White Black' 9" ,
"rating" : 4.7 ,
"currency" : "USD" ,
"delivery" : "Delivery by Tue, Jun 18 \u00b7 Free 30-day returns" ,
"merchant" : {
"url": "/url?url=https://www.adidas.com/us/samba-og-shoes/B75806.html&rct=j&q=&esrc=s&opi=95576897&sa=U&ved=0ahUKEwikoMX_iNOGAxVvFbkGHV6uDZcQguUECKIY&usg=AOvVaw1uGHW41BiuoKTfeYo2le3u",
"name" : "adidas"
} ,
"price_str" : "$100.00" ,
"thumbnail": "https://encrypted-tbn2.gstatic.com/shopping?q=tbn:ANd9GcSEQPCgQtOW8Swse-SyzsEBKqqzGJO_l9lcKaq7hIeSPfLqrBv5sKJiligHh3eUZ7XoWnvwJfL2&usqp=CAE",
"product_id" : "1503163696221055935" ,
"pos_overall" : 1 ,
"reviews_count" : 5028
} ,
]
...
Key (organic) Description Type The position of the product in the search results.
The URL of the product page.
The type of listing layout.
The price of the product in the specified currency.
The title of the product listing.
The average user rating of the product, typically out of 5.
The currency code for the product price.
Delivery details, including estimated delivery date and return policy.
An object containing details about the merchant selling the product.
The URL of the merchant's page.
The name of the merchant.
The product price as a string, including the currency symbol.
The URL of the product's thumbnail image.
A unique identifier for the product.
The overall position of the product in the search results.
The total number of reviews for the product.
Last updated 4 months ago