Custom status code
By default, we assume the request is successful as long as it returns a 2xx or 4xx status code. However, sometimes websites return the required content together with a non-standard HTTPS status code. If one of your targets does that, you can indicate which status codes are acceptable and valuable to you. Add the X-Oxylabs-Status-Code
header with all the HTTP response codes that work for you.
2xx and 4xx will still be automatically marked as successful.
Web Unblocker counts only the traffic of successfully extracted data. Failed scraping attempts won’t be included in your traffic usage calculation. Both upstream and downstream traffic is calculated.
Code examples
curl -k -v -x unblock.oxylabs.io:60000 \
-U "user:pass1" "https://ip.oxylabs.io/location" \
-H "X-Oxylabs-Successful-Status-Codes: 500,501,502,503"
import requests
# Define proxy dict. Don't forget to put your real user and pass here as well.
proxies = {
'http': 'http://user:pass1@unblock.oxylabs.io:60000',
'https': 'http://user:pass1@unblock.oxylabs.io:60000',
}
headers = {
"X-Oxylabs-Successful-Status-Codes": "500,501,502,503"
}
response = requests.get(
'https://ip.oxylabs.io/location',
verify=False, # It is required to ignore certificate
proxies=proxies,
headers=headers,
)
# Print result page to stdout
print(response.text)
# Save returned HTML to result.html file
with open('result.html', 'w') as f:
f.write(response.text)
<?php
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, "https://ip.oxylabs.io/headers");
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_PROXY, 'unblock.oxylabs.io:60000');
curl_setopt($ch, CURLOPT_PROXYUSERPWD, 'YOUR_USERNAME:YOUR_PASSWORD');
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
curl_setopt_array($ch, [
CURLOPT_HTTPHEADER => [
'X-Oxylabs-Successful-Status-Codes: 500,501,502,503'
]
]);
$result = curl_exec($ch);
echo $result;
if (curl_errno($ch)) {
echo 'Error:' . curl_error($ch);
}
curl_close($ch);
using System;
using System.Net;
using System.Net.Http;
using System.Threading.Tasks;
namespace OxyApi
{
class Program
{
static async Task Main(string[] args)
{
var webProxy = new WebProxy
{
Address = new Uri("http://unblock.oxylabs.io:60000"),
BypassProxyOnLocal = false,
UseDefaultCredentials = false,
Credentials = new NetworkCredential(
userName: "YOUR_USERNAME",
password: "YOUR_PASSWORD"
)
};
var httpClientHandler = new HttpClientHandler
{
Proxy = webProxy,
};
// We recommend accepting our certificate instead of allowing insecure (http) traffic
httpClientHandler.ClientCertificateOptions = ClientCertificateOption.Manual;
httpClientHandler.ServerCertificateCustomValidationCallback =
(httpRequestMessage, cert, cetChain, policyErrors) =>
{
return true;
};
var client = new HttpClient(handler: httpClientHandler, disposeHandler: true);
// Add custom cookies
client.DefaultRequestHeaders.Add("X-Oxylabs-Successful-Status-Codes", "500,501,502,503");
Uri baseUri = new Uri("https://ip.oxylabs.io/headers");
client.BaseAddress = baseUri;
var requestMessage = new HttpRequestMessage(HttpMethod.Get, "");
var response = await client.SendAsync(requestMessage);
var contents = await response.Content.ReadAsStringAsync();
Console.WriteLine(contents);
}
}
}
package main
import (
"crypto/tls"
"fmt"
"io/ioutil"
"net/http"
"net/url"
)
func main() {
const Username = "YOUR_USERNAME"
const Password = "YOUR_PASSWORD"
proxyUrl, _ := url.Parse(
fmt.Sprintf(
"http://%s:%s@unblock.oxylabs.io:60000",
Username,
Password,
),
)
customTransport := &http.Transport{Proxy: http.ProxyURL(proxyUrl)}
// We recommend accepting our certificate instead of allowing insecure (http) traffic
customTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
client := &http.Client{Transport: customTransport}
request, _ := http.NewRequest("GET",
"https://ip.oxylabs.io/location",
nil,
)
// Add custom cookies
request.Header.Add("X-Oxylabs-Successful-Status-Codes", "500,501,502,503")
request.SetBasicAuth(Username, Password)
response, _ := client.Do(request)
responseText, _ := ioutil.ReadAll(response.Body)
fmt.Println(string(responseText))
}
package org.example;
import okhttp3.Authenticator;
import okhttp3.Credentials;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import javax.net.ssl.*;
import java.net.InetSocketAddress;
import java.net.Proxy;
public class Main implements Runnable {
private static final String AUTHORIZATION_HEADER = "Proxy-Authorization";
public static final String USERNAME = "YOUR_USERNAME";
public static final String PASSWORD = "YOUR_PASSWORD";
public void run() {
Authenticator authenticator = (route, response) -> {
String credential = Credentials.basic(USERNAME, PASSWORD);
return response
.request()
.newBuilder()
.header(AUTHORIZATION_HEADER, credential)
.build();
};
OkHttpClient.Builder builder = new OkHttpClient.Builder();
// We recommend accepting our certificate instead of allowing insecure (http) traffic
this.disableSSLCertificateChecking(builder);
Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress("unblock.oxylabs.io", 60000));
var client = builder
.proxy(proxy)
.proxyAuthenticator(authenticator)
.build();
var request = new Request.Builder()
.url("https://ip.oxylabs.io/location")
.addHeader("X-Oxylabs-Successful-Status-Codes", "500,501,502,503")
.get()
.build();
try (var response = client.newCall(request).execute()) {
assert response.body() != null;
System.out.println(response.body().string());
} catch (Exception exception) {
exception.printStackTrace();
System.exit(1);
}
System.exit(0);
}
private void disableSSLCertificateChecking(OkHttpClient.Builder builder) {
TrustManager[] trustManagers = new TrustManager[]{
new X509TrustManager() {
@Override
public java.security.cert.X509Certificate[] getAcceptedIssuers() {
return new java.security.cert.X509Certificate[]{};
}
@Override
public void checkServerTrusted(java.security.cert.X509Certificate[] x509Certificates, String authType) {
}
@Override
public void checkClientTrusted(java.security.cert.X509Certificate[] x509Certificates, String authType) {
}
}
};
try {
HttpsURLConnection.setDefaultHostnameVerifier((s, sslSession) -> true);
SSLContext sslContext = SSLContext.getInstance("TLS");
sslContext.init(null, trustManagers, new java.security.SecureRandom());
HttpsURLConnection.setDefaultSSLSocketFactory(sslContext.getSocketFactory());
builder.sslSocketFactory(sslContext.getSocketFactory(), (X509TrustManager) trustManagers[0]);
} catch (Exception exception) {
exception.printStackTrace();
System.exit(1);
}
builder.hostnameVerifier((hostname, session) -> true);
}
public static void main(String[] args) {
new Thread(new Main()).start();
}
}
import fetch from 'node-fetch';
import createHttpsProxyAgent from 'https-proxy-agent'
const username = 'YOUR_USERNAME';
const password = 'YOUR_PASSWORD';
const agent = createHttpsProxyAgent(
`http://${username}:${password}@unblock.oxylabs.io:60000`
);
// We recommend accepting our certificate instead of allowing insecure (http) traffic
process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = 0;
const headers = {
'X-Oxylabs-Successful-Status-Codes': '500,501,502,503',
}
const response = await fetch('https://ip.oxylabs.io/location', {
method: 'get',
headers: headers,
agent: agent,
});
console.log(await response.text());
Last updated