Skip to content

Commit c535d23

Browse files
dekel-holadekel-hola
committed
Added files via upload
1 parent 5bffd49 commit c535d23

15 files changed

+821
-2
lines changed

‎README.md

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
1-
# API
1+
# luminati.io API
22

3-
Luminati.io API
3+
To get CUSTOMER, YOURZONE and YOURPASS, sign up on [luminati.io] (http://luminati.io/?cam=github_readme)
4+
5+
Full API documentation can be found [here] (https://luminati.io/cp/api_example?cam=github_readme)
6+
7+
Ruby implementation can be found [here] (https://github.com/Agiley/Luminati)

‎csharp/high_perf.cs

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
2+
3+
4+
using System;
5+
using System.Threading;
6+
using System.Net;
7+
using System.Collections.Generic;
8+
9+
class Client : WebClient
10+
{
11+
public static string username = "lum-customer-CUSTOMER-zone-YOURZONE";
12+
public static string password = "YOURPASS";
13+
public static int port = 22225;
14+
public static int max_failures = 3;
15+
public string session_id;
16+
public string login;
17+
public string country;
18+
public int fail_count;
19+
public int n_req_for_exit_node;
20+
public Random rng;
21+
public HashSet<ServicePoint> service_points;
22+
23+
public Client(string country = null)
24+
{
25+
this.country = country;
26+
rng = new Random();
27+
service_points = new HashSet<ServicePoint>();
28+
switch_session_id();
29+
}
30+
31+
public void switch_session_id()
32+
{
33+
clean_connection_pool();
34+
session_id = rng.Next().ToString();
35+
n_req_for_exit_node = 0;
36+
update_super_proxy();
37+
}
38+
39+
public void update_super_proxy()
40+
{
41+
Proxy = new WebProxy("session-"+session_id+".zproxy.luminati.io", port);
42+
login = username+(country != null ? "-country-"+country : "")
43+
+"-session-"+session_id;
44+
Proxy.Credentials = new NetworkCredential(login, password);
45+
}
46+
47+
public void clean_connection_pool()
48+
{
49+
foreach (ServicePoint sp in service_points)
50+
sp.CloseConnectionGroup(login);
51+
service_points.Clear();
52+
}
53+
54+
public bool have_good_super_proxy()
55+
{
56+
return fail_count < max_failures;
57+
}
58+
59+
public void handle_response(WebException e = null) {
60+
if (e != null && should_switch_exit_node((HttpWebResponse)e.Response))
61+
{
62+
switch_session_id();
63+
fail_count++;
64+
return;
65+
}
66+
// success or other client/website error like 404...
67+
n_req_for_exit_node++;
68+
fail_count = 0;
69+
}
70+
71+
public bool should_switch_exit_node(HttpWebResponse response)
72+
{
73+
return response == null ||
74+
status_code_requires_exit_node_switch((int)response.StatusCode);
75+
}
76+
77+
public bool status_code_requires_exit_node_switch(int code)
78+
{
79+
return code == 403 || code == 429 || code==502 || code == 503;
80+
}
81+
82+
protected override WebRequest GetWebRequest(Uri address)
83+
{
84+
var request = base.GetWebRequest(address) as HttpWebRequest;
85+
request.ConnectionGroupName = login;
86+
request.PreAuthenticate = true;
87+
return request;
88+
}
89+
90+
protected override WebResponse GetWebResponse(WebRequest request)
91+
{
92+
var response = base.GetWebResponse(request);
93+
ServicePoint sp = ((HttpWebRequest)request).ServicePoint;
94+
service_points.Add(sp);
95+
return response;
96+
}
97+
}
98+
99+
class Example
100+
{
101+
public static int n_parallel_exit_nodes = 100;
102+
public static int n_total_req = 1000;
103+
public static int switch_ip_every_n_req = 20;
104+
public static int at_req = 0;
105+
106+
static void Main()
107+
{
108+
ServicePointManager.DefaultConnectionLimit = int.MaxValue;
109+
for (var i = 0; i < n_parallel_exit_nodes; i++)
110+
{
111+
var t = new Thread(new ThreadStart(Run));
112+
t.Name = ""+i;
113+
t.Start();
114+
}
115+
}
116+
117+
static void Run()
118+
{
119+
var client = new Client();
120+
while (Interlocked.Increment(ref at_req) <= n_total_req)
121+
{
122+
if (!client.have_good_super_proxy())
123+
client.switch_session_id();
124+
if (client.n_req_for_exit_node == switch_ip_every_n_req)
125+
client.switch_session_id();
126+
try {
127+
Console.WriteLine(client.DownloadString("http://lumtest.com/myip.json"));
128+
client.handle_response();
129+
} catch (WebException e) {
130+
Console.WriteLine(e.Message);
131+
client.handle_response(e);
132+
}
133+
}
134+
client.clean_connection_pool();
135+
client.Dispose();
136+
}
137+
}
138+

‎csharp/simple.cs

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
2+
using System;
3+
using System.Net;
4+
5+
class Client : WebClient
6+
{
7+
public static string username = "lum-customer-CUSTOMER-zone-YOURZONE";
8+
public static string password = "YOURPASS";
9+
public static int port = 22225;
10+
public string session_id = new Random().Next().ToString();
11+
12+
public Client(string country = null)
13+
{
14+
this.Proxy = new WebProxy("zproxy.luminati.io", port);
15+
var login = username+(country != null ? "-country-"+country : "")
16+
+"-session-"+session_id;
17+
this.Proxy.Credentials = new NetworkCredential(login, password);
18+
}
19+
20+
protected override WebRequest GetWebRequest(Uri address)
21+
{
22+
var request = base.GetWebRequest(address) as HttpWebRequest;
23+
request.ConnectionGroupName = session_id;
24+
return request;
25+
}
26+
}
27+
28+
class Example
29+
{
30+
static void Main()
31+
{
32+
Console.WriteLine("Performing request(s)");
33+
var client = new Client();
34+
// Put full scraping sequence below:
35+
Console.WriteLine(client.DownloadString("http://lumtest.com/myip.json"));
36+
// client.DownloadString(...second request...);
37+
}
38+
}
39+
40+
41+

‎java/high_perf.java

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
package example;
2+
3+
import java.io.*;
4+
import java.util.Random;
5+
import java.util.concurrent.*;
6+
import java.util.concurrent.atomic.AtomicInteger;
7+
8+
import org.apache.http.HttpHost;
9+
import org.apache.http.HttpResponse;
10+
import org.apache.http.auth.*;
11+
import org.apache.http.client.CredentialsProvider;
12+
import org.apache.http.client.config.RequestConfig;
13+
import org.apache.http.client.fluent.Request;
14+
import org.apache.http.client.methods.*;
15+
import org.apache.http.impl.client.*;
16+
import org.apache.http.impl.conn.BasicHttpClientConnectionManager;
17+
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
18+
import org.apache.http.util.EntityUtils;
19+
20+
class Client {
21+
public static final String username = "lum-customer-CUSTOMER-zone-YOURZONE";
22+
public static final String password = "YOURPASS";
23+
public static final int port = 22225;
24+
public static final int max_failures = 3;
25+
public static final int req_timeout = 60*1000;
26+
public String session_id;
27+
public HttpHost super_proxy;
28+
public CloseableHttpClient client;
29+
public String country;
30+
public int fail_count;
31+
public int n_req_for_exit_node;
32+
public Random rng;
33+
34+
public Client(String country) {
35+
this.country = country;
36+
rng = new Random();
37+
switch_session_id();
38+
}
39+
40+
public void switch_session_id() {
41+
session_id = Integer.toString(rng.nextInt(Integer.MAX_VALUE));
42+
n_req_for_exit_node = 0;
43+
super_proxy = new HttpHost("session-"+session_id+".zproxy.luminati.io", port);
44+
update_client();
45+
}
46+
47+
public void update_client() {
48+
close();
49+
String login = username+(country!=null ? "-country-"+country : "")
50+
+"-session-" + session_id;
51+
CredentialsProvider cred_provider = new BasicCredentialsProvider();
52+
cred_provider.setCredentials(new AuthScope(super_proxy),
53+
new UsernamePasswordCredentials(login, password));
54+
RequestConfig config = RequestConfig.custom()
55+
.setConnectTimeout(req_timeout)
56+
.setConnectionRequestTimeout(req_timeout)
57+
.build();
58+
PoolingHttpClientConnectionManager conn_mgr =
59+
new PoolingHttpClientConnectionManager();
60+
conn_mgr.setDefaultMaxPerRoute(Integer.MAX_VALUE);
61+
conn_mgr.setMaxTotal(Integer.MAX_VALUE);
62+
client = HttpClients.custom()
63+
.setConnectionManager(conn_mgr)
64+
.setProxy(super_proxy)
65+
.setDefaultCredentialsProvider(cred_provider)
66+
.setDefaultRequestConfig(config)
67+
.build();
68+
}
69+
70+
public CloseableHttpResponse request(String url) throws IOException {
71+
try {
72+
HttpGet request = new HttpGet(url);
73+
CloseableHttpResponse response = client.execute(request);
74+
handle_response(response);
75+
return response;
76+
} catch (IOException e) {
77+
handle_response(null);
78+
throw e;
79+
}
80+
}
81+
82+
public void handle_response(HttpResponse response) {
83+
if (response != null && !status_code_requires_exit_node_switch(
84+
response.getStatusLine().getStatusCode())) {
85+
// success or other client/website error like 404...
86+
n_req_for_exit_node++;
87+
fail_count = 0;
88+
return;
89+
}
90+
switch_session_id();
91+
fail_count++;
92+
}
93+
94+
public boolean status_code_requires_exit_node_switch(int code) {
95+
return code == 403 || code == 429 || code==502 || code == 503;
96+
}
97+
98+
public boolean have_good_super_proxy() {
99+
return super_proxy != null && fail_count < max_failures;
100+
}
101+
102+
public void close() {
103+
if (client != null)
104+
try { client.close(); } catch (IOException e) {}
105+
client = null;
106+
}
107+
}
108+
109+
public class Example implements Runnable {
110+
public static final int n_parallel_exit_nodes = 100;
111+
public static final int n_total_req = 1000;
112+
public static final int switch_ip_every_n_req = 40;
113+
public static AtomicInteger at_req = new AtomicInteger(0);
114+
115+
public static void main(String[] args) {
116+
ExecutorService executor =
117+
Executors.newFixedThreadPool(n_parallel_exit_nodes);
118+
for (int i = 0; i < n_parallel_exit_nodes; i++)
119+
executor.execute(new Example());
120+
executor.shutdown();
121+
}
122+
123+
@Override
124+
public void run() {
125+
Client client = new Client(null);
126+
while (at_req.getAndAdd(1) < n_total_req) {
127+
if (!client.have_good_super_proxy())
128+
client.switch_session_id();
129+
if (client.n_req_for_exit_node == switch_ip_every_n_req)
130+
client.switch_session_id();
131+
CloseableHttpResponse response = null;
132+
try {
133+
response = client.request("http://lumtest.com/myip.json");
134+
int code = response.getStatusLine().getStatusCode();
135+
System.out.println(code != 200 ? code :
136+
EntityUtils.toString(response.getEntity()));
137+
} catch (IOException e) {
138+
System.out.println(e.getMessage());
139+
} finally {
140+
try {
141+
if (response != null)
142+
response.close();
143+
} catch (Exception e) {}
144+
}
145+
}
146+
client.close();
147+
}
148+
}
149+

‎java/simple.java

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
package example;
2+
3+
import java.io.*;
4+
import java.util.Random;
5+
import org.apache.http.HttpHost;
6+
import org.apache.http.auth.*;
7+
import org.apache.http.client.CredentialsProvider;
8+
import org.apache.http.client.fluent.Request;
9+
import org.apache.http.client.methods.*;
10+
import org.apache.http.impl.client.*;
11+
import org.apache.http.impl.conn.BasicHttpClientConnectionManager;
12+
import org.apache.http.util.EntityUtils;
13+
14+
class Client {
15+
public static final String username = "lum-customer-CUSTOMER-zone-YOURZONE";
16+
public static final String password = "YOURPASS";
17+
public static final int port = 22225;
18+
public String session_id = Integer.toString(new Random().nextInt(Integer.MAX_VALUE));
19+
public CloseableHttpClient client;
20+
21+
public Client(String country) {
22+
String login = username+(country!=null ? "-country-"+country : "")
23+
+"-session-" + session_id;
24+
HttpHost super_proxy = new HttpHost("zproxy.luminati.io", port);
25+
CredentialsProvider cred_provider = new BasicCredentialsProvider();
26+
cred_provider.setCredentials(new AuthScope(super_proxy),
27+
new UsernamePasswordCredentials(login, password));
28+
client = HttpClients.custom()
29+
.setConnectionManager(new BasicHttpClientConnectionManager())
30+
.setProxy(super_proxy)
31+
.setDefaultCredentialsProvider(cred_provider)
32+
.build();
33+
}
34+
35+
public String request(String url) throws IOException {
36+
HttpGet request = new HttpGet(url);
37+
CloseableHttpResponse response = client.execute(request);
38+
try {
39+
return EntityUtils.toString(response.getEntity());
40+
} finally { response.close(); }
41+
}
42+
43+
public void close() throws IOException { client.close(); }
44+
}
45+
46+
public class Example {
47+
public static void main(String[] args) throws IOException {
48+
System.out.println("Performing request(s)");
49+
Client client = new Client(null);
50+
try {
51+
// Put complete scraping sequence below:
52+
System.out.println(client.request("http://lumtest.com/myip.json"));
53+
// System.out.println(client.request(...second request...));
54+
} finally { client.close(); }
55+
}
56+
}

0 commit comments

Comments
 (0)