Adaptive parsing
Adaptive parsing is capable of parsing the most important fields from any e-commerce product page. Enable adaptive parser for e-commerce product pages by sending two additional headers: X-Oxylabs-Parser-Type: ecommerce_product to select the parser type and X-Oxylabs-Parse: 1 to parse the retrieved page.
Fields that can be parsed by Next-Gen Residential Proxies Adaptive Parsing for e-commerce product pages:
  • Title
  • Regular price
  • Offer price
  • Currency
  • Product description
  • Product IDs
  • Image URLs
  • URLs from the page

Code examples

cURL
Python
PHP
C#
Golang
Java
Node.js
curl -v -k -x ngrp.oxylabs.io:60000 \
-U user:pass1 "https://ip.oxylabs.io" \
-H "X-Oxylabs-Parser-Type: ecommerce_product" \
-H "X-Oxylabs-Parse: 1"
import requests
from pprint import pprint
# Define proxy dict. Don't forget to put your real user and pass here as well.
proxies = {
'http': 'http://user:[email protected]:60000',
'https': 'https://user:[email protected]:60000',
}
headers = {
"X-Oxylabs-Parser-Type": "ecommerce_product",
"X-Oxylabs-Parse": "1",
}
response = requests.get(
'https://ip.oxylabs.io', #E-commerce product page
verify=False, # It is required to ignore certificate
proxies=proxies,
headers=headers,
)
# Print result page to stdout
pprint(response.json())
<?php
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, "https://ip.oxylabs.io");
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_PROXY, 'ngrp.oxylabs.io:60000');
curl_setopt($ch, CURLOPT_PROXYUSERPWD, "YOUR_USERNAME" . ":" . "YOUR_PASSWORD");
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
curl_setopt_array($ch, array(
CURLOPT_HTTPHEADER => array(
'X-Oxylabs-Parser-Type: ecommerce_product',
'X-Oxylabs-Parse: 1',
)
));
$result = curl_exec($ch);
echo $result;
if (curl_errno($ch)) {
echo 'Error:' . curl_error($ch);
}
curl_close ($ch);
?>
using System;
using System.Net;
using System.Net.Http;
using System.Threading.Tasks;
namespace OxyApi
{
class Program
{
static async Task Main(string[] args)
{
var webProxy = new WebProxy
{
Address = new Uri(quot;http://ngrp.oxylabs.io:60000"),
BypassProxyOnLocal = false,
UseDefaultCredentials = false,
Credentials = new NetworkCredential(
userName: "YOUR_USERNAME",
password: "YOUR_PASSWORD"
)
};
var httpClientHandler = new HttpClientHandler
{
Proxy = webProxy,
};
// We recommend accepting our certificate instead of allowing insecure (http) traffic
httpClientHandler.ClientCertificateOptions = ClientCertificateOption.Manual;
httpClientHandler.ServerCertificateCustomValidationCallback =
(httpRequestMessage, cert, cetChain, policyErrors) =>
{
return true;
};
var client = new HttpClient(handler: httpClientHandler, disposeHandler: true);
client.DefaultRequestHeaders.Add("X-Oxylabs-Parser-Type", "ecommerce_product");
client.DefaultRequestHeaders.Add("X-Oxylabs-Parse", "1");
Uri baseUri = new Uri("https://ip.oxylabs.io");
client.BaseAddress = baseUri;
var requestMessage = new HttpRequestMessage(HttpMethod.Get, "");
var response = await client.SendAsync(requestMessage);
var contents = await response.Content.ReadAsStringAsync();
Console.WriteLine(contents);
}
}
}
package main
import (
"crypto/tls"
"fmt"
"io/ioutil"
"net/http"
"net/url"
)
func main() {
const Username = "YOUR_USERNAME"
const Password = "YOUR_PASSWORD"
proxyUrl, _ := url.Parse(
fmt.Sprintf(
"http://%s:%[email protected]:60000",
Username,
Password,
),
)
customTransport := &http.Transport{Proxy: http.ProxyURL(proxyUrl)}
// We recommend accepting our certificate instead of allowing insecure (http) traffic
customTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
client := &http.Client{Transport: customTransport}
request, _ := http.NewRequest("GET",
"https://ip.oxylabs.io",
nil,
)
request.Header.Add("X-Oxylabs-Parser-Type", "ecommerce_product")
request.Header.Add("X-Oxylabs-Parse", "1")
request.SetBasicAuth(Username, Password)
response, _ := client.Do(request)
responseText, _ := ioutil.ReadAll(response.Body)
fmt.Println(string(responseText))
}
import okhttp3.Authenticator;
import okhttp3.Credentials;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import javax.net.ssl.*;
import java.net.InetSocketAddress;
import java.net.Proxy;
public class Main implements Runnable {
private static final String AUTHORIZATION_HEADER = "Proxy-Authorization";
public static final String USERNAME = "YOUR_USERNAME";
public static final String PASSWORD = "YOUR_PASSWORD";
public void run() {
Authenticator authenticator = (route, response) -> {
String credential = Credentials.basic(USERNAME, PASSWORD);
return response
.request()
.newBuilder()
.header(AUTHORIZATION_HEADER, credential)
.build();
};
OkHttpClient.Builder builder = new OkHttpClient.Builder();
// We recommend accepting our certificate instead of allowing insecure (http) traffic
this.disableSSLCertificateChecking(builder);
Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress("ngrp.oxylabs.io", 60000));
var client = builder
.proxy(proxy)
.proxyAuthenticator(authenticator)
.build();
var request = new Request.Builder()
.url("https://ip.oxylabs.io")
.addHeader("X-Oxylabs-Parser-Type", "ecommerce_product")
.addHeader("X-Oxylabs-Parse", "1")
.get()
.build();
try (var response = client.newCall(request).execute()) {
assert response.body() != null;
System.out.println(response.body().string());
} catch (Exception exception) {
exception.printStackTrace();
System.exit(1);
}
System.exit(0);
}
private void disableSSLCertificateChecking(OkHttpClient.Builder builder) {
TrustManager[] trustManagers = new TrustManager[]{
new X509TrustManager() {
@Override
public java.security.cert.X509Certificate[] getAcceptedIssuers() {
return new java.security.cert.X509Certificate[]{};
}
@Override
public void checkServerTrusted(java.security.cert.X509Certificate[] x509Certificates, String authType) {
}
@Override
public void checkClientTrusted(java.security.cert.X509Certificate[] x509Certificates, String authType) {
}
}
};
try {
HttpsURLConnection.setDefaultHostnameVerifier((s, sslSession) -> true);
SSLContext sslContext = SSLContext.getInstance("TLS");
sslContext.init(null, trustManagers, new java.security.SecureRandom());
HttpsURLConnection.setDefaultSSLSocketFactory(sslContext.getSocketFactory());
builder.sslSocketFactory(sslContext.getSocketFactory(), (X509TrustManager) trustManagers[0]);
} catch (Exception exception) {
exception.printStackTrace();
System.exit(1);
}
builder.hostnameVerifier((hostname, session) -> true);
}
public static void main(String[] args) {
new Thread(new Main()).start();
}
}
import fetch from 'node-fetch';
import createHttpsProxyAgent from 'https-proxy-agent'
const username = 'YOUR_USERNAME';
const password = 'YOUR_PASSWORD';
const agent = createHttpsProxyAgent(
`http://${username}:${password}@ngrp.oxylabs.io:60000`
);
// We recommend accepting our certificate instead of allowing insecure (http) traffic
process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = 0;
const headers = {
'X-Oxylabs-Parser-Type': 'ecommerce_product',
'X-Oxylabs-Parse': '1',
}
const response = await fetch('https://ip.oxylabs.io', {
method: 'get',
headers: headers,
agent: agent,
});
console.log(await response.text());

Sample output

{
"results": [
{
"content": {
"url": "https://ip.oxylabs.io/product/product_example.html",
"body": {
"price": 11.99,
"title": "Example product title",
"currency": "quot;,
"old_price": 15.99,
"description": null,
"image_links": [
"https://ip.oxylabs.io/img/product_example.png"
],
"ids_from_url": [],
"simple_links": [
{
"link": "https://ip.oxylabs.io/product/another_example.html",
"description": "Another product example"
}
],
"ids_from_html": [
{
"Product number": "14158288"
}
],
"price_range_lower": null,
"price_range_upper": null
},
"meta": {
"title": "Example product meta title",
"keywords": [],
"description": "Example product description"
},
"parse_status_code": 12000
},
"created_at": "2020-01-01 10:00:00",
"updated_at": "2020-01-01 10:00:07",
"id": 29964797,
"page": 1,
"url": "https://ip.oxylabs.io/product/product_example.html",
"job_id": "6699272813062145025",
"status_code": 200
}
]
}
Copy link
On this page