Merge pull request #17 from FeeeeK/master

Format code and fix collectors

Author: DevAlone

async_requests.py

@@ -1,45 +1,44 @@
-import aiohttp
 import json
-from aiosocks.connector import ProxyConnector, ProxyClientRequest
+
+import aiohttp
+from aiosocks.connector import ProxyClientRequest, ProxyConnector
 
 
 async def get(url, **kwargs):
-    return await request('get', url, **kwargs)
+    return await request("get", url, **kwargs)
 
 
 async def post(url, data, **kwargs):
     if data is dict or data is str:
-        kwargs['json'] = data
+        kwargs["json"] = data
     else:
-        kwargs['data'] = data
+        kwargs["data"] = data
 
-    return await request('post', url, **kwargs)
+    return await request("post", url, **kwargs)
 
 
 async def request(method, url, **kwargs):
     session_kwargs = {}
-    if 'proxy' in kwargs and kwargs['proxy'].startswith('socks'):
-        session_kwargs['connector'] = ProxyConnector(remote_resolve=False)
-        session_kwargs['request_class'] = ProxyClientRequest
+    if "proxy" in kwargs and kwargs["proxy"].startswith("socks"):
+        session_kwargs["connector"] = ProxyConnector(remote_resolve=False)
+        session_kwargs["request_class"] = ProxyClientRequest
 
-    if 'cookies'in kwargs:
-        session_kwargs['cookies'] = kwargs['cookies']
-        del kwargs['cookies']
+    if "cookies" in kwargs:
+        session_kwargs["cookies"] = kwargs["cookies"]
+        del kwargs["cookies"]
 
-    if 'timeout' not in kwargs:
-        kwargs['timeout'] = 10
+    if "timeout" not in kwargs:
+        kwargs["timeout"] = 10
 
     # headers={'User-Agent': get_random_user_agent()}
-    if 'headers' not in kwargs:
-        kwargs['headers'] = {
-            'User-Agent': get_random_user_agent()
-        }
-    elif 'User-Agent' not in kwargs['headers']:
-            kwargs['headers']['User-Agent'] = get_random_user_agent()
-
-    if 'override_session' in kwargs:
-        session = kwargs['override_session']
-        del kwargs['override_session']
+    if "headers" not in kwargs:
+        kwargs["headers"] = {"User-Agent": get_random_user_agent()}
+    elif "User-Agent" not in kwargs["headers"]:
+        kwargs["headers"]["User-Agent"] = get_random_user_agent()
+
+    if "override_session" in kwargs:
+        session = kwargs["override_session"]
+        del kwargs["override_session"]
         async with session.request(method, url, **kwargs) as response:
             return await Response.from_aiohttp_response(response)
 
@@ -57,22 +56,24 @@ def __init__(self, status, text, aiohttp_response=None):
     @staticmethod
     async def from_aiohttp_response(aiohttp_response):
         return Response(
-                status=aiohttp_response.status,
-                text=await aiohttp_response.text(),
-                aiohttp_response=aiohttp_response
+            status=aiohttp_response.status,
+            text=await aiohttp_response.text(),
+            aiohttp_response=aiohttp_response,
         )
 
     def __str__(self):
-        return json.dumps({
-            'status': self.status,
-            'text': self.text,
-        })
+        return json.dumps(
+            {
+                "status": self.status,
+                "text": self.text,
+            }
+        )
 
     __repr__ = __str__
 
 
 def get_random_user_agent():
-    return 'Mozilla/5.0 (Windows NT;) Gecko/20100101 Firefox/58.0'
+    return "Mozilla/5.0 (Windows NT;) Gecko/20100101 Firefox/58.0"
     # return 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:55.0) Gecko/20100101 Firefox/55.0'
     # TODO: do it
     # return UserAgent().random

check_from_stdin.py

@@ -1,37 +1,33 @@
 """
 just a helper script for testing proxies
 """
-from proxy_py import settings
-from models import Proxy
-from checkers.base_checker import BaseChecker
-
 import asyncio
-import proxy_utils
-import sys
 import re
+import sys
 
+import proxy_utils
+from checkers.base_checker import BaseChecker
+from models import Proxy
+from proxy_py import settings
 
-proxy_find_regex = \
-    r"([0-9]{1,3})[^0-9]+([0-9]{1,3})[^0-9]+([0-9]{1,3})[^0-9]+([0-9]{1,3})"\
+proxy_find_regex = (
+    r"([0-9]{1,3})[^0-9]+([0-9]{1,3})[^0-9]+([0-9]{1,3})[^0-9]+([0-9]{1,3})"
     r"[^0-9]+([0-9]{1,5})"
+)
 semaphore = asyncio.BoundedSemaphore(settings.NUMBER_OF_CONCURRENT_TASKS)
 tasks = []
 
 
 async def check_task(ip, port):
     async with semaphore:
         for raw_protocol in range(len(Proxy.PROTOCOLS)):
-            proxy_url = '{}://{}:{}'.format(
-                Proxy.PROTOCOLS[raw_protocol],
-                ip,
-                port
-            )
+            proxy_url = "{}://{}:{}".format(Proxy.PROTOCOLS[raw_protocol], ip, port)
             check_result, _ = await proxy_utils.check_proxy(proxy_url)
             if check_result:
                 break
         # if check_result:
         #     print('proxy {} works'.format(proxy_url))
-        print('+' if check_result else '-', end='', file=sys.stderr)
+        print("+" if check_result else "-", end="", file=sys.stderr)
         sys.stderr.flush()
 
 
@@ -42,7 +38,7 @@ async def main():
             groups = re.search(proxy_find_regex, line).groups()
         except:
             continue
-        ip = '.'.join(groups[:4])
+        ip = ".".join(groups[:4])
         port = groups[4]
 
         tasks.append(asyncio.ensure_future(check_task(ip, port)))
@@ -52,5 +48,5 @@ async def main():
     BaseChecker.clean()
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     asyncio.get_event_loop().run_until_complete(main())

checkers/__init__.py

@@ -1,4 +1,3 @@
 from checkers.d3d_info_checker import D3DInfoChecker
 from checkers.google_com_checker import GoogleComChecker
 from checkers.ipinfo_io_checker import IPInfoIOChecker
-

checkers/base_checker.py

@@ -1,11 +1,12 @@
-from aiosocks.connector import ProxyConnector, ProxyClientRequest
-from proxy_py import settings
-
+import asyncio
 import ssl
+
 import aiohttp
 import aiosocks
-import asyncio
+from aiosocks.connector import ProxyClientRequest, ProxyConnector
+
 import async_requests
+from proxy_py import settings
 
 
 class CheckerResult:
@@ -46,7 +47,9 @@ def __init__(self, url=None, request_type="GET", timeout=None):
                 limit_per_host=settings.NUMBER_OF_SIMULTANEOUS_REQUESTS_PER_HOST,
             )
         self.request_type = request_type
-        self.timeout = timeout if timeout is not None else settings.PROXY_CHECKING_TIMEOUT
+        self.timeout = (
+            timeout if timeout is not None else settings.PROXY_CHECKING_TIMEOUT
+        )
         self.url = url
 
     @staticmethod
@@ -56,7 +59,6 @@ async def init():
 
         :return:
         """
-        pass
 
     @staticmethod
     def get_aiohttp_connector():
@@ -123,7 +125,11 @@ async def _request(self, proxy_address, timeout) -> tuple:
             connector=conn, connector_owner=False, request_class=ProxyClientRequest
         ) as session:
             async with session.request(
-                self.request_type, self.url, proxy=proxy_address, timeout=timeout, headers=headers
+                self.request_type,
+                self.url,
+                proxy=proxy_address,
+                timeout=timeout,
+                headers=headers,
             ) as response:
                 is_working = await self.validate(response, checker_result)
 

checkers/d3d_info_checker.py

@@ -5,7 +5,11 @@
 
 class D3DInfoChecker(BaseChecker):
     def __init__(self, timeout=None):
-        super(D3DInfoChecker, self).__init__("https://test.d3d.info/ok.html", timeout=timeout)
-
-    async def validate(self, response: aiohttp.ClientResponse, checker_result: CheckerResult):
-        return (await response.text()).strip().lower() == 'ok'
+        super(D3DInfoChecker, self).__init__(
+            "https://test.d3d.info/ok.html", timeout=timeout
+        )
+
+    async def validate(
+        self, response: aiohttp.ClientResponse, checker_result: CheckerResult
+    ):
+        return (await response.text()).strip().lower() == "ok"

checkers/google_com_checker.py

@@ -5,11 +5,15 @@
 
 class GoogleComChecker(BaseChecker):
     def __init__(self, timeout=None):
-        super(GoogleComChecker, self).__init__("https://www.google.com/humans.txt", timeout=timeout)
+        super(GoogleComChecker, self).__init__(
+            "https://www.google.com/humans.txt", timeout=timeout
+        )
 
-    async def validate(self, response: aiohttp.ClientResponse, checker_result: CheckerResult):
-        '''
+    async def validate(
+        self, response: aiohttp.ClientResponse, checker_result: CheckerResult
+    ):
+        """
         We have already done the request and it was successful,
         Google returned something(maybe good response, maybe captcha, we don't care)
-        '''
+        """
         return True

checkers/ipinfo_io_checker.py

@@ -1,27 +1,32 @@
-from checkers.base_checker import BaseChecker, CheckerResult
 import aiohttp
 
+from checkers.base_checker import BaseChecker, CheckerResult
+
 
 class IPInfoIOChecker(BaseChecker):
     def __init__(self, timeout=None):
         super(IPInfoIOChecker, self).__init__("https://ipinfo.io/json", timeout=timeout)
 
-    async def validate(self, response: aiohttp.ClientResponse, checker_result: CheckerResult) -> bool:
+    async def validate(
+        self, response: aiohttp.ClientResponse, checker_result: CheckerResult
+    ) -> bool:
         if response.status != 200:
             return False
 
         json_result = await response.json()
-        if 'ip' in json_result:
-            checker_result.ipv4 = json_result['ip']
-        if 'city' in json_result:
-            checker_result.city = json_result['city']
-        if 'region' in json_result:
-            checker_result.region = json_result['region']
-        if 'country' in json_result:
-            checker_result.country_code = json_result['country']
-        if 'loc' in json_result:
-            checker_result.location_coordinates = tuple(float(x) for x in json_result['loc'].split(','))
-        if 'org' in json_result:
-            checker_result.organization_name = json_result['org']
+        if "ip" in json_result:
+            checker_result.ipv4 = json_result["ip"]
+        if "city" in json_result:
+            checker_result.city = json_result["city"]
+        if "region" in json_result:
+            checker_result.region = json_result["region"]
+        if "country" in json_result:
+            checker_result.country_code = json_result["country"]
+        if "loc" in json_result:
+            checker_result.location_coordinates = tuple(
+                float(x) for x in json_result["loc"].split(",")
+            )
+        if "org" in json_result:
+            checker_result.organization_name = json_result["org"]
 
         return True

collectors/__init__.py

@@ -1,2 +1,2 @@
 from collectors.abstract_collector import AbstractCollector
-from collectors.pages_collector import PagesCollector
+from collectors.pages_collector import PagesCollector

collectors/abstract_collector.py

@@ -1,10 +1,9 @@
 # TODO: add wrapper for doing requests and saving its cookies and UserAgent
 import asyncio
-
-from proxy_py import settings
-
 import json
+
 import models
+from proxy_py import settings
 
 
 class AbstractCollector:
@@ -47,8 +46,9 @@ async def _collect(self):
         """Do not call yourself! It is called on collector's processing automatically"""
         collect = self.collect()
         if asyncio.iscoroutine(collect):
+
             async def wrapper(f):
-                for item in (await f):
+                for item in await f:
                     yield item
 
             collect = wrapper(collect)
@@ -78,10 +78,12 @@ async def load_state(self, state):
         self.last_processing_time = state.last_processing_time
         self.processing_period = state.processing_period
         self.last_processing_proxies_count = state.last_processing_proxies_count
-        self.data = json.loads(state.data) if state.data is not None and state.data else {}
-        if '_variables' in self.data:
-            for var_name in self.data['_variables']:
-                setattr(self, var_name, self.data['_variables'][var_name])
+        self.data = (
+            json.loads(state.data) if state.data is not None and state.data else {}
+        )
+        if "_variables" in self.data:
+            for var_name in self.data["_variables"]:
+                setattr(self, var_name, self.data["_variables"][var_name])
 
     async def save_state(self, state: models.CollectorState):
         """
@@ -93,10 +95,10 @@ async def save_state(self, state: models.CollectorState):
         state.last_processing_proxies_count = self.last_processing_proxies_count
 
         if self.saved_variables is not None:
-            if '_variables' not in self.data:
-                self.data['_variables'] = {}
+            if "_variables" not in self.data:
+                self.data["_variables"] = {}
             for var_name in self.saved_variables:
-                self.data['_variables'][var_name] = getattr(self, var_name)
+                self.data["_variables"][var_name] = getattr(self, var_name)
 
         state.data = json.dumps(self.data)
 

collectors/pages_collector.py

@@ -1,5 +1,5 @@
-from proxy_py import settings
 from collectors.abstract_collector import AbstractCollector
+from proxy_py import settings
 
 
 # TODO: save pages to collector state
@@ -17,14 +17,14 @@ class PagesCollector(AbstractCollector):
     def __init__(self):
         super(PagesCollector, self).__init__()
         self.last_proxies_list = []
-        self.saved_variables.add('current_page')
-        self.saved_variables.add('pages_count')
-        self.saved_variables.add('last_proxies_list')
+        self.saved_variables.add("current_page")
+        self.saved_variables.add("pages_count")
+        self.saved_variables.add("last_proxies_list")
 
     async def collect(self):
-        proxies = list(
-            await self.process_page(self.current_page)
-        )[:settings.COLLECTOR_MAXIMUM_NUMBER_OF_PROXIES_PER_REQUEST]
+        proxies = list(await self.process_page(self.current_page))[
+            : settings.COLLECTOR_MAXIMUM_NUMBER_OF_PROXIES_PER_REQUEST
+        ]
 
         if self.dynamic_pages_count:
             if proxies: