#140 及修改可环境变量配置的参数 (#202)

* 1、设置scheduler.py中dev模式的flask run，不进行启动时的自动重新加载reload

* 1、修改PROXY_SCORE_MAX，PROXY_SCORE_MIN，PROXY_SCORE_INIT三项配置，为可环境变量配置
2、添加可环境变量配置项TEST_DONT_SET_MAX_SCORE，允许设置当tester检测到某个proxy可用时，只是保持原score，而不将其score设置成max。

* 增加获取proxy接口的认证header, API-KEY。可配置，默认不需要

Author: inVains

proxypool/processors/server.py

@@ -1,7 +1,7 @@
-from flask import Flask, g
+from flask import Flask, g, request
 from proxypool.storages.redis import RedisClient
-from proxypool.setting import API_HOST, API_PORT, API_THREADED, IS_DEV
-
+from proxypool.setting import API_HOST, API_PORT, API_THREADED, API_KEY, IS_DEV
+import functools
 
 __all__ = ['app']
 
@@ -10,6 +10,25 @@
     app.debug = True
 
 
+def auth_required(func):
+    @functools.wraps(func)
+    def decorator(*args, **kwargs):
+        # conditional decorator, when setting API_KEY is set, otherwise just ignore this decorator
+        if API_KEY == "":
+            return func(*args, **kwargs)
+        if request.headers.get('API-KEY', None) is not None:
+            api_key = request.headers.get('API-KEY')
+        else:
+            return {"message": "Please provide an API key in header"}, 400
+        # Check if API key is correct and valid
+        if request.method == "GET" and api_key == API_KEY:
+            return func(*args, **kwargs)
+        else:
+            return {"message": "The provided API key is not valid"}, 403
+
+    return decorator
+
+
 def get_conn():
     """
     get redis client object
@@ -21,6 +40,7 @@ def get_conn():
 
 
 @app.route('/')
+@auth_required
 def index():
     """
     get home page, you can define your own templates
@@ -30,6 +50,7 @@ def index():
 
 
 @app.route('/random')
+@auth_required
 def get_proxy():
     """
     get a random proxy
@@ -40,6 +61,7 @@ def get_proxy():
 
 
 @app.route('/all')
+@auth_required
 def get_proxy_all():
     """
     get a random proxy
@@ -56,6 +78,7 @@ def get_proxy_all():
 
 
 @app.route('/count')
+@auth_required
 def get_count():
     """
     get the count of proxies

proxypool/processors/tester.py

@@ -3,11 +3,11 @@
 from loguru import logger
 from proxypool.schemas import Proxy
 from proxypool.storages.redis import RedisClient
-from proxypool.setting import TEST_TIMEOUT, TEST_BATCH, TEST_URL, TEST_VALID_STATUS, TEST_ANONYMOUS
+from proxypool.setting import TEST_TIMEOUT, TEST_BATCH, TEST_URL, TEST_VALID_STATUS, TEST_ANONYMOUS, \
+    TEST_DONT_SET_MAX_SCORE
 from aiohttp import ClientProxyConnectionError, ServerDisconnectedError, ClientOSError, ClientHttpProxyError
 from asyncio import TimeoutError
 
-
 EXCEPTIONS = (
     ClientProxyConnectionError,
     ConnectionRefusedError,
@@ -23,14 +23,14 @@ class Tester(object):
     """
     tester for testing proxies in queue
     """
-    
+
     def __init__(self):
         """
         init redis
         """
         self.redis = RedisClient()
         self.loop = asyncio.get_event_loop()
-    
+
     async def test(self, proxy: Proxy):
         """
         test single proxy
@@ -55,15 +55,18 @@ async def test(self, proxy: Proxy):
                 async with session.get(TEST_URL, proxy=f'http://{proxy.string()}', timeout=TEST_TIMEOUT,
                                        allow_redirects=False) as response:
                     if response.status in TEST_VALID_STATUS:
-                        self.redis.max(proxy)
-                        logger.debug(f'proxy {proxy.string()} is valid, set max score')
+                        if TEST_DONT_SET_MAX_SCORE:
+                            logger.debug(f'proxy {proxy.string()} is valid, remain current score')
+                        else:
+                            self.redis.max(proxy)
+                            logger.debug(f'proxy {proxy.string()} is valid, set max score')
                     else:
                         self.redis.decrease(proxy)
                         logger.debug(f'proxy {proxy.string()} is invalid, decrease score')
             except EXCEPTIONS:
                 self.redis.decrease(proxy)
                 logger.debug(f'proxy {proxy.string()} is invalid, decrease score')
-    
+
     @logger.catch
     def run(self):
         """
@@ -84,14 +87,15 @@ def run(self):
             if not cursor:
                 break
 
+
 def run_tester():
     host = '96.113.165.182'
     port = '3128'
     tasks = [tester.test(Proxy(host=host, port=port))]
     tester.loop.run_until_complete(asyncio.wait(tasks))
 
+
 if __name__ == '__main__':
     tester = Tester()
     tester.run()
     # run_tester()
-

proxypool/scheduler.py

@@ -92,7 +92,7 @@ def run_server(self):
                 logger.error("unsupported APP_PROD_METHOD")
                 return
         else:
-            app.run(host=API_HOST, port=API_PORT, threaded=API_THREADED)
+            app.run(host=API_HOST, port=API_PORT, threaded=API_THREADED, use_reloader=False)
 
     def run(self):
         global tester_process, getter_process, server_process

proxypool/setting.py

@@ -53,9 +53,9 @@
     'REDIS_KEY', 'proxies:universal'))
 
 # definition of proxy scores
-PROXY_SCORE_MAX = 100
-PROXY_SCORE_MIN = 0
-PROXY_SCORE_INIT = 10
+PROXY_SCORE_MAX = env.int('PROXY_SCORE_MAX', 100)
+PROXY_SCORE_MIN = env.int('PROXY_SCORE_MIN', 0)
+PROXY_SCORE_INIT = env.int('PROXY_SCORE_INIT', 10)
 
 # definition of proxy number
 PROXY_NUMBER_MAX = 50000
@@ -77,11 +77,17 @@
 #     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36',
 # })
 TEST_VALID_STATUS = env.list('TEST_VALID_STATUS', [200, 206, 302])
+# whether to set max score when one proxy is tested valid
+TEST_DONT_SET_MAX_SCORE = env.bool('TEST_DONT_SET_MAX_SCORE', False)
 
 # definition of api
 API_HOST = env.str('API_HOST', '0.0.0.0')
 API_PORT = env.int('API_PORT', 5555)
 API_THREADED = env.bool('API_THREADED', True)
+# add an api key to get proxy
+# need a header of `API-KEY` in get request to pass the authenticate
+# API_KEY='', do not need `API-KEY` header
+API_KEY = env.str('API_KEY', '')
 
 # flags of enable
 ENABLE_TESTER = env.bool('ENABLE_TESTER', True)