Scrape Google Finance at scale without being blocked

                      

import requests


url = "https://scraping.soax.com/api/v1/request?store=AMAZON&nocache=true&param=B071WR7MLW&function=getProduct"


payload={}
headers = {}


response = requests.request("GET", url, headers=headers, data=payload)


print(response.text)

                  

                      $ curl -x "http://username:pw;;@proxy.soax.com:9000" -L http://checker.soax.com/api/ipinfo
                  

                      


                                                        
<?php



                                                        
$auth = base64_encode('username:pw;;');

                                                        
$aContext = array(

                                                        
'http' => array(

                                                        
'proxy' => 'tcp://proxy.soax.com:9000',

                                                        
'request_fulluri' => true,

                                                        
'header' => "Proxy-Authorization: Basic $auth",

                                                        
),

                                                        
);



                                                        
$cxContext = stream_context_create($aContext);

                                                        
$sFile = file_get_contents("http://checker.soax.com/api/ipinfo", False, $cxContext);

                                                        
echo $sFile, " ";



                                                        
?>

                                                        
                                                        
                  

                      


                                                        
using Rebex.Net;

                                                        
using System;

                                                        
using System.Collections.Generic;

                                                        
using System.IO;

                                                        
using System.Linq;

                                                        
using System.Text;

                                                        
using System.Threading.Tasks;



                                                        
namespace ConsoleApp1 {

                                                        
class Program {

                                                        
static void Main(string[] args) {

                                                        
Rebex.Licensing.Key = 'ENTER YOUR LICENSING KEY HERE';

                                                        
var client = new Rebex.Net.HttpRequestCreator();

                                                        
client.Proxy.ProxyType = ProxyType.Socks5;

                                                        
client.Proxy.Host = 'proxy.soax.com';

                                                        
client.Proxy.Port = 9000;

                                                        
client.Proxy.UserName = 'username';

                                                        
client.Proxy.Password = 'pw;;;;';

                                                        
var url = 'http://checker.soax.com/api/ipinfo';

                                                        
var httpRequest = client.Create(url);

                                                        
httpRequest.Headers['Accept'] = 'text/html, application/xhtml+xml, image/jxr, */*';

                                                        
httpRequest.Headers['Accept-Language'] = 'en-US,en;q=0.7,ru;q=0.3';

                                                        
httpRequest.Headers['Accept-Encoding'] = 'gzip, deflate';

                                                        
httpRequest.Headers['Host'] = url;

                                                        
httpRequest.Headers['Connection'] = 'Keep-Alive';

                                                        
httpRequest.Timeout = 30000;



                                                        
try {

                                                        
var response = httpRequest.GetResponse() as Rebex.Net.HttpResponse;

                                                        
using (StreamReader sr = new StreamReader(response.GetResponseStream())) {

                                                        
var content = sr.ReadToEnd();

                                                        
Console.WriteLine('Url: ' + url + ' ' + 'Content length': ' + content.Length + ' ' + 'Response': ' + content);

                                                        
}

                                                        
}

                                                        
catch (Exception e) {

                                                        
Console.WriteLine('Url ' + url + ' is failed. ' + e.Message);

                                                        
}

                                                        
Console.ReadKey();

                                                        
}

                                                        
}

                                                        
}