Getting Started
Last updated
Was this helpful?
Last updated
Was this helpful?
Integrating is easy, especially if you have previously used regular for web scraping. The only difference is that we require you to ignore the SSL certificate using the -k
or --insecure
cURL flags (or an equivalent expression in the language of your choice).
To make a request using Web Unblocker, you need to use the unblock.oxylabs.io:60000
proxy endpoint. See a cURL example below. You can find code samples in other languages or complete code examples on our .
Use to check the parameters of your IPs—this domain delivers information from four geolocation databases: MaxMind, IP2Location, DB-IP, and IPinfo.io. The parameters include IP address, provider, country, city, ZIP code, ASN, organization name, time zone, and meta (when disclosed by database).
curl -k -x https://unblock.oxylabs.io:60000 \
-U 'USERNAME:PASSWORD' \
'https://ip.oxylabs.io/location'
import requests
# Use your Web Unblocker credentials here.
USERNAME, PASSWORD = 'YOUR_USERNAME', 'YOUR_PASSWORD'
# Define proxy dict.
proxies = {
'http': f'http://{USERNAME}:{PASSWORD}@unblock.oxylabs.io:60000',
'https': f'https://{USERNAME}:{PASSWORD}@unblock.oxylabs.io:60000',
}
response = requests.request(
'GET',
'https://ip.oxylabs.io/location',
verify=False, # Ignore the SSL certificate
proxies=proxies,
)
# Print result page to stdout
print(response.text)
# Save returned HTML to result.html file
with open('result.html', 'w') as f:
f.write(response.text)
import fetch from 'node-fetch';
import { HttpsProxyAgent } from 'https-proxy-agent';
const username = 'YOUR_USERNAME';
const password = 'YOUR_PASSWORD';
const agent = new HttpsProxyAgent(
`https://${username}:${password}@unblock.oxylabs.io:60000`
);
// Ignore the certificate
process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = 0;
const response = await fetch('https://ip.oxylabs.io/location', {
method: 'get',
agent: agent,
});
console.log(await response.text());
<?php
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://ip.oxylabs.io/location');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_PROXY, 'https://unblock.oxylabs.io:60000');
curl_setopt($ch, CURLOPT_PROXYUSERPWD, 'YOUR_USERNAME' . ':' . 'YOUR_PASSWORD');
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
$result = curl_exec($ch);
echo $result;
if (curl_errno($ch)) {
echo 'Error:' . curl_error($ch);
}
curl_close($ch);
package main
import (
"crypto/tls"
"fmt"
"io/ioutil"
"net/http"
"net/url"
)
func main() {
const Username = "YOUR_USERNAME"
const Password = "YOUR_PASSWORD"
proxyUrl, _ := url.Parse(
fmt.Sprintf(
"https://%s:%s@unblock.oxylabs.io:60000",
Username,
Password,
),
)
customTransport := &http.Transport{Proxy: http.ProxyURL(proxyUrl)}
// Ignore the certificate
customTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
client := &http.Client{Transport: customTransport}
request, _ := http.NewRequest("GET",
"https://ip.oxylabs.io/location",
nil,
)
response, _ := client.Do(request)
responseText, _ := ioutil.ReadAll(response.Body)
fmt.Println(string(responseText))
}
using System;
using System.Net;
using System.Net.Http;
using System.Threading.Tasks;
namespace OxyApi
{
class Program
{
static async Task Main(string[] args)
{
var webProxy = new WebProxy
{
Address = new Uri("https://unblock.oxylabs.io:60000"),
BypassProxyOnLocal = false,
UseDefaultCredentials = false,
Credentials = new NetworkCredential(
userName: "YOUR_USERNAME",
password: "YOUR_PASSWORD"
)
};
var httpClientHandler = new HttpClientHandler
{
Proxy = webProxy,
};
// Ignore the certificate
httpClientHandler.ClientCertificateOptions = ClientCertificateOption.Manual;
httpClientHandler.ServerCertificateCustomValidationCallback =
(httpRequestMessage, cert, cetChain, policyErrors) =>
{
return true;
};
var client = new HttpClient(handler: httpClientHandler, disposeHandler: true);
Uri baseUri = new Uri("https://ip.oxylabs.io/location");
client.BaseAddress = baseUri;
var requestMessage = new HttpRequestMessage(HttpMethod.Get, "");
var response = await client.SendAsync(requestMessage);
var contents = await response.Content.ReadAsStringAsync();
Console.WriteLine(contents);
}
}
}
package org.example;
import org.apache.hc.client5.http.auth.AuthScope;
import org.apache.hc.client5.http.auth.CredentialsProvider;
import org.apache.hc.client5.http.classic.methods.HttpGet;
import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.client5.http.impl.auth.CredentialsProviderBuilder;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder;
import org.apache.hc.client5.http.ssl.NoopHostnameVerifier;
import org.apache.hc.client5.http.ssl.SSLConnectionSocketFactoryBuilder;
import org.apache.hc.client5.http.ssl.TrustAllStrategy;
import org.apache.hc.core5.http.HttpHost;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.http.message.StatusLine;
import org.apache.hc.core5.ssl.SSLContextBuilder;
import java.util.Arrays;
import java.util.Properties;
public class Main {
public static void main(final String[] args)throws Exception {
final CredentialsProvider credsProvider = CredentialsProviderBuilder.create()
.add(new AuthScope("unblock.oxylabs.io", 60000), "USERNAME", "PASSWORD".toCharArray())
.build();
final HttpHost target = new HttpHost("https", "ip.oxylabs.io", 443);
final HttpHost proxy = new HttpHost("https", "unblock.oxylabs.io", 60000);
try (final CloseableHttpClient httpclient = HttpClients.custom()
.setDefaultCredentialsProvider(credsProvider)
.setProxy(proxy)
// We recommend accepting our certificate instead of allowing insecure (http) traffic
.setConnectionManager(PoolingHttpClientConnectionManagerBuilder.create()
.setSSLSocketFactory(SSLConnectionSocketFactoryBuilder.create()
.setSslContext(SSLContextBuilder.create()
.loadTrustMaterial(TrustAllStrategy.INSTANCE)
.build())
.setHostnameVerifier(NoopHostnameVerifier.INSTANCE)
.build())
.build())
.build()) {
final RequestConfig config = RequestConfig.custom()
.build();
final HttpGet request = new HttpGet("/location");
request.setConfig(config);
System.out.println("Executing request " + request.getMethod() + " " + request.getUri() +
" via " + proxy + " headers: " + Arrays.toString(request.getHeaders()));
httpclient.execute(target, request, response -> {
System.out.println("----------------------------------------");
System.out.println(request + "->" + new StatusLine(response));
EntityUtils.consume(response.getEntity());
return null;
});
}
}
}
If you are observing low success rates or retrieve empty content, please try adding additional "x-oxylabs-render: html"
header with your request.
If Web Unblocker is being used to scrape websites dependent on loading data via JavaScript, refer to the section. The product is not designed to be used with headless browsers (e.g., Chromium, PhantomJS, Splash, etc.) and their drivers (e.g., Playwright, Selenium, Puppeteer, etc.) directly.
Watch the video below for an example of scraping difficult target without getting blocked:
If you want to learn more about getting data on a large scale with - we suggest watching this Scraping Experts lesson: