rotate proxy | set proxy | proxy setting | set proxy in webclient

rotate proxy | set proxy | proxy setting | set proxy in webclient

• SomeTime site can response only particular region only than we can't response on our ip.
• In this case we can setup environment it's look like this region.
• you need to identify only site depends on which region.

• than you can collect those region ip's.

• after collect ip you can setup you ip hardcode in this program,
• otherwise insert in database and setup dynamically in the fillProxy().

Note :-

• Setup your request environment instead of dummy environment.
• Setup your proxy hardcode.
• Otherwise database.

Functionality :- 

• If you site blocked than particular ip than program rotate dynamically to fire request other ip.
• untile that rotate your ips which is setup by you.

• it exit with response only this case:-
  
  • if that found this ip which is setup by you is belong to only this region that site may depend. AND Site given response.

Requrie library :- HtmlUnit

Requrie library :- StringUtil

 

Example :-

import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.HttpMethod;
import com.gargoylesoftware.htmlunit.Page;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.WebRequest;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import com.gargoylesoftware.htmlunit.util.NameValuePair;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import org.apache.commons.lang3.StringUtils;

/**
 *
 * @author vishal.khokhar
 */
public class RoteRequestUsingProxy {

    WebClient webClient = null;
    int currentProxyCount = 0;
    public HashMap proxyMap = new HashMap();

    public static void main(String[] args) {
        new RoteRequestUsingProxy().startRequesting();
    }

    public void startRequesting() {
        try {
            fillProxy();
            if (webClient == null) {
                setWebClient();
            }

            //Here this is dummy url put your url
            String url = "http://scrapemania.blogspot.in/2016/10/";

            if (webClient == null) {
                setWebClient();
            }

            WebRequest webRequest1 = new WebRequest(new URL(url), HttpMethod.POST);

            webRequest1.setAdditionalHeader("Accept", "application/xml, text/xml, */*; q=0.01");
            webRequest1.setAdditionalHeader("Accept-Encoding", "gzip, deflate");
            webRequest1.setAdditionalHeader("Host", "scrapemania.blogspot.in");
            webRequest1.setAdditionalHeader("User-Agent", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:50.0) Gecko/20100101 Firefox/50.0");
            webRequest1.setAdditionalHeader("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");
            webRequest1.setAdditionalHeader("X-Requested-With", "XMLHttpRequest");
            webRequest1.setAdditionalHeader("Accept-Language", "en-US,en;q=0.5");
            webRequest1.setAdditionalHeader("Connection", "keep-alive");
            webRequest1.setAdditionalHeader("Faces-Request", "partial/ajax");
            webRequest1.setAdditionalHeader("Referer", "http://scrapemania.blogspot.in/2016/10/");
            //Here, this is dummy Referer value put your Referer value

            List<NameValuePair> val = new ArrayList();

            val.add(new NameValuePair("asesInputForm", "asesInputForm"));
            val.add(new NameValuePair("asesInputForm:j_idt39", "asesInputForm:j_idt39"));
            val.add(new NameValuePair("asesInputForm:searchCriteria", "2016"));
            val.add(new NameValuePair("asesInputForm:searchCriteria1", "10"));
            val.add(new NameValuePair("javax.faces.partial.ajax", "true"));
            val.add(new NameValuePair("javax.faces.partial.execute", "@all"));
            val.add(new NameValuePair("javax.faces.partial.render", "asesAggregationForm asesResultForm"));
            val.add(new NameValuePair("javax.faces.source", "asesInputForm:j_idt39"));

            webRequest1.setRequestParameters(val);
            Page requestPage1 = null;
            do {
                try {
                    requestPage1 = webClient.getPage(webRequest1);
                } catch (Exception ex) {
                    setWebClient();
                    ex.printStackTrace();
                }
            } while (requestPage1 == null);

            System.out.println(requestPage1.getWebResponse().getContentAsString());
           
            url="http://scrapemania.blogspot.in/2016/09/";
           
            WebRequest webRequest = new WebRequest(new URL(url), HttpMethod.POST);

            webRequest.setAdditionalHeader("Accept", "application/xml, text/xml, */*; q=0.01");
            webRequest.setAdditionalHeader("Accept-Encoding", "gzip, deflate");
            webRequest.setAdditionalHeader("Host", "scrapemania.blogspot.in");
            webRequest.setAdditionalHeader("User-Agent", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:50.0) Gecko/20100101 Firefox/50.0");
            webRequest.setAdditionalHeader("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");
            webRequest.setAdditionalHeader("X-Requested-With", "XMLHttpRequest");
            webRequest.setAdditionalHeader("Accept-Language", "en-US,en;q=0.5");
            webRequest.setAdditionalHeader("Connection", "keep-alive");
            webRequest.setAdditionalHeader("Faces-Request", "partial/ajax");
            webRequest.setAdditionalHeader("Referer", "http://scrapemania.blogspot.in/2016/09/");
            //Here, this is dummy Referer value put your Referer value

            List<NameValuePair> val1 = new ArrayList();

            val1.add(new NameValuePair("asesInputForm", "asesInputForm"));
            val1.add(new NameValuePair("asesInputForm:j_idt39", "asesInputForm:j_idt39"));
            val1.add(new NameValuePair("asesInputForm:searchCriteria", "2016"));
            val1.add(new NameValuePair("asesInputForm:searchCriteria", "09"));
            val1.add(new NameValuePair("javax.faces.partial.ajax", "true"));
            val1.add(new NameValuePair("javax.faces.partial.execute", "@all"));
            val1.add(new NameValuePair("javax.faces.partial.render", "asesAggregationForm asesResultForm"));
            val1.add(new NameValuePair("javax.faces.source", "asesInputForm:j_idt39"));
           
            webRequest.setRequestParameters(val1);
           
            HtmlPage requestPage = null;

            do {
                try {
                    requestPage = webClient.getPage(webRequest);
                } catch (Exception ex) {
                    setWebClient();
                    ex.printStackTrace();
                }
            } while (requestPage == null);
            System.out.println(requestPage.getWebResponse().getContentAsString());

        } catch (Exception ex) {
            ex.printStackTrace();
        }
    }

    public void setWebClient() {
        //Here we increment so be can get each time new Proxy wherever we can get response
        currentProxyCount++;

        //Here set the limit of our hardcode proxy is 5 So,we can do this
        //other wise you collect from your database than set your criteria
        if (currentProxyCount == 6) {
            currentProxyCount = 1;
        }

        String tempProxy = proxyMap.get(currentProxyCount).toString();
        System.out.println(tempProxy);
        String proxy = StringUtils.substringBefore(tempProxy, "~");
        int port = Integer.parseInt(StringUtils.substringAfter(tempProxy, "~"));

        //Here we setup webclient
        webClient = new WebClient(BrowserVersion.FIREFOX_38, proxy, port);
        webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
        webClient.getOptions().setThrowExceptionOnScriptError(false);
        webClient.getOptions().setJavaScriptEnabled(true);
        webClient.getOptions().setCssEnabled(true);
    }

    public void fillProxy() {

        //Here is dummy Hardcode proxy put new proxy
        proxyMap.put(1, "1.1.1.1~8080");
        proxyMap.put(2, "2.2.2.2~8080");
        proxyMap.put(3, "3.3.3.3~8080");
        proxyMap.put(4, "4.4.4.4~8080");
        proxyMap.put(5, "5.5.5.5~8080");

        //Other-Wise you can fill new proxy from your database if you collect.
    }
}