diff --git a/deploy/connectivityserver.yaml b/deploy/connectivityserver.yaml index c957b45..8388d37 100644 --- a/deploy/connectivityserver.yaml +++ b/deploy/connectivityserver.yaml @@ -54,25 +54,16 @@ spec: requests: memory: 32Mi cpu: 100m - startupProbe: - httpGet: - path: /ready - port: http - failureThreshold: 30 - periodSeconds: 10 livenessProbe: - httpGet: - path: /live + tcpSocket: port: http initialDelaySeconds: 15 periodSeconds: 20 readinessProbe: - httpGet: - path: /ready + tcpSocket: port: http - initialDelaySeconds: 5 - periodSeconds: 10 - failureThreshold: 3 + initialDelaySeconds: 15 + periodSeconds: 20 terminationMessagePath: /dev/termination-log terminationMessagePolicy: File securityContext: diff --git a/src/connectivityserver/connectionflask.py b/src/connectivityserver/connectionflask.py index 16ea84c..b164662 100755 --- a/src/connectivityserver/connectionflask.py +++ b/src/connectivityserver/connectionflask.py @@ -17,372 +17,309 @@ from flask import Flask, abort, make_response, request # Some functions exit with an abort(NNN) instead of return so don't complain! -# ruff: noqa RET503 +#ruff: noqa RET503 -partitions = {} -partlock = Lock() +partitions={} +partlock=Lock() -if "CONNECTION_FLASK_DEBUG" in os.environ: - debug_level = int(os.environ["CONNECTION_FLASK_DEBUG"]) +if 'CONNECTION_FLASK_DEBUG' in os.environ: + debug_level=int(os.environ['CONNECTION_FLASK_DEBUG']) else: - debug_level = 1 - + debug_level=1 def convert_log_level(log_level): - if log_level == 0: - return logging.WARNING - if log_level == 1: - return logging.INFO - if log_level == 2: - return logging.DEBUG + if log_level == 0: + return logging.WARNING + if log_level == 1: return logging.INFO + if log_level == 2: + return logging.DEBUG + return logging.INFO - -logging.basicConfig( - level=convert_log_level(debug_level), - format="%(asctime)s %(levelname)s %(filename)s:%(funcName)s:%(lineno)d %(message)s", -) +logging.basicConfig(level=convert_log_level(debug_level), format='%(asctime)s %(levelname)s %(filename)s:%(funcName)s:%(lineno)d %(message)s') log = logging.getLogger(__name__) -if "ENTRY_TTL" in os.environ: - ttl = int(os.environ["ENTRY_TTL"]) +if 'ENTRY_TTL' in os.environ: + ttl=int(os.environ['ENTRY_TTL']) else: - ttl = 10 -entry_ttl = timedelta(seconds=ttl) - -last_stats = datetime.now() -npublishes = 0 -nlookups = 0 -lookup_time = timedelta(0) -publish_time = timedelta() -maxpartitions = 0 -maxentries = {} - -app = Flask(__name__) -appstarted = False - - -@app.before_first_request -def mark_started(): - global appstarted - appstarted = True + ttl=10 +entry_ttl=timedelta(seconds=ttl) +last_stats=datetime.now() +npublishes=0 +nlookups=0 +lookup_time=timedelta(0) +publish_time=timedelta() +maxpartitions=0 +maxentries={} -@app.route("/live", methods=["GET"]) -def live(): - return "OK", 200 - - -@app.route("/ready", methods=["GET"]) -def ready(): - if appstarted: - return "OK", 200 - return "Not Ready", 503 - +app=Flask(__name__) @app.route("/") def dump(): - now = datetime.now() - dstream = StringIO() - dstream.write("

Dump of configuration dictionary

") - dstream.write("

Active partitions

") - if len(partitions) > 0: - pad = ' style="padding-left: 1em;padding-right: 1em"' - dstream.write( - f'' - f'Partition' - f"Entries" - ) - for p in partitions: - dstream.write( - f"{p}{len(partitions[p])}" - ) - dstream.write("
") - dstream.write(f"

Partitions

") - for p in partitions: - store = partitions[p] - dstream.write(f"

{p}

") - dstream.write( - f'' - f'' - f'Name' - f'Connection' - f"" - f'' - f"uri" - f"data_type" - f"capacity" - f"connection_type" - f"time" - f"" - ) - format_cell = lambda value, strike: ( - f'{value}' if strike else f"{value}" - ) - for k, v in store.items(): - expired = now - v.time >= entry_ttl - dstream.write( - f"{format_cell(k, expired)}" - f"{format_cell(v.uri, expired)}" - f"{format_cell(v.data_type, expired)}" - f"{format_cell(v.capacity, expired)}" - f"{format_cell(v.connection_type, expired)}" - f"{format_cell(v.time, expired)}" - ) - dstream.write("
") - else: - dstream.write("None

") - dstream.write("

Server statistics

") - stats_to_html(dstream) - dstream.seek(0) - return dstream.read() - + now=datetime.now() + dstream=StringIO() + dstream.write('

Dump of configuration dictionary

') + dstream.write("

Active partitions

") + if len(partitions)>0: + pad=' style="padding-left: 1em;padding-right: 1em"' + dstream.write(f'' + f'Partition' + f'Entries') + for p in partitions: + dstream.write(f'{p}' + f'{len(partitions[p])}') + dstream.write("
") + dstream.write(f'

Partitions

') + for p in partitions: + store=partitions[p] + dstream.write(f'

{p}

') + dstream.write(f'' + f'' + f'Name' + f'Connection' + f'' + f'' + f'uri' + f'data_type' + f'capacity' + f'connection_type' + f'time' + f'') + format_cell=lambda value,strike: f'{value}' if strike else f'{value}' + for k,v in store.items(): + expired = now-v.time >= entry_ttl + dstream.write(f'{format_cell(k,expired)}' + f'{format_cell(v.uri,expired)}' + f'{format_cell(v.data_type,expired)}' + f'{format_cell(v.capacity,expired)}' + f'{format_cell(v.connection_type,expired)}' + f'{format_cell(v.time,expired)}') + dstream.write("
") + else: + dstream.write("None

") + dstream.write("

Server statistics

") + stats_to_html(dstream) + dstream.seek(0) + return dstream.read() def stats_to_html(dstream): - dstream.write(f"

Since {last_stats}

") - if npublishes > 0: - avg_pub = publish_time / npublishes - else: - avg_pub = timedelta() - dstream.write( - f"

{npublishes} calls to publish in total time {publish_time} " - f"(average {avg_pub.microseconds} µs per call)

" - ) - if nlookups > 0: - avg_lookup = lookup_time / nlookups - else: - avg_lookup = timedelta() - dstream.write( - f"

{nlookups} calls to lookup in total time {lookup_time} " - f"(average {avg_lookup.microseconds} µs per call)

" - ) - dstream.write(f"

Maximum number of partitions active = {maxpartitions}

") - for part in maxentries: - dstream.write( - f"

Maximum entries in partition {part} = {maxentries[part]}

" - ) - + dstream.write(f"

Since {last_stats}

") + if npublishes>0: + avg_pub=publish_time/npublishes + else: + avg_pub=timedelta() + dstream.write(f"

{npublishes} calls to publish in total time {publish_time} " + f"(average {avg_pub.microseconds} µs per call)

") + if nlookups>0: + avg_lookup=lookup_time/nlookups + else: + avg_lookup=timedelta() + dstream.write(f"

{nlookups} calls to lookup in total time {lookup_time} " + f"(average {avg_lookup.microseconds} µs per call)

") + dstream.write(f"

Maximum number of partitions active = {maxpartitions}

") + for part in maxentries: + dstream.write(f"

Maximum entries in partition {part} = {maxentries[part]}

") @app.route("/stats") def dumpStats(): - dstream = StringIO() - dstream.write("

Connection server statistics

") - stats_to_html(dstream) - dstream.seek(0) - return dstream.read() - + dstream=StringIO() + dstream.write('

Connection server statistics

') + stats_to_html(dstream) + dstream.seek(0) + return dstream.read() @app.route("/resetStats") def resetStats(): - stats = dumpStats() - - global \ - last_stats, \ - npublishes, \ - nlookups, \ - lookup_time, \ - publish_time, \ - maxpartitions, \ - maxentries + stats = dumpStats() - last_stats = datetime.now() - npublishes = 0 - nlookups = 0 - lookup_time = timedelta(0) - publish_time = timedelta() - maxpartitions = 0 - maxentries = {} + global last_stats,npublishes,nlookups,lookup_time,publish_time,maxpartitions,maxentries - return stats + last_stats=datetime.now() + npublishes=0 + nlookups=0 + lookup_time=timedelta(0) + publish_time=timedelta() + maxpartitions=0 + maxentries={} + return stats @app.route("/resetService") def reset(): - global partitions - partitions = {} - return resetStats() + global partitions + partitions={} + return resetStats() -@app.route("/publish", methods=["POST"]) +@app.route("/publish",methods=['POST']) def publish(): - # Store multiple connection ids and corresponding uris in a - # dictionary associated with the appropriate partition. - timestamp = datetime.now() - js = json.loads(request.data) - - log.debug(f"{js=}") - part = js["partition"] - - log.info( - f"{len(js['connections'])} connections in partition {part} from {request.remote_addr} uri={js['connections'][0]['uri']}..." - ) - - partlock.acquire() - if part in partitions: - store = partitions[part] - else: - store = {} - partitions[part] = store - global maxpartitions - if len(partitions) > maxpartitions: - maxpartitions = len(partitions) - if part not in maxentries: - maxentries[part] = 0 - - Connection = namedtuple( - "Connection", ["uri", "data_type", "capacity", "connection_type", "time"] - ) + # Store multiple connection ids and corresponding uris in a + # dictionary associated with the appropriate partition. + timestamp=datetime.now() + js=json.loads(request.data) + + log.debug(f"{js=}") + part=js['partition'] + + log.info( + f"{len(js['connections'])} connections in partition {part} from {request.remote_addr} uri={js['connections'][0]['uri']}..." + ) + + partlock.acquire() + if part in partitions: + store=partitions[part] + else: + store={} + partitions[part]=store + global maxpartitions + if len(partitions)>maxpartitions: + maxpartitions=len(partitions) + if part not in maxentries: + maxentries[part]=0 + + Connection=namedtuple( + 'Connection',['uri','data_type','capacity','connection_type','time']) + + for connection in js['connections']: + + if 'uid' in connection and 'uri' in connection: + uid=connection['uid'] + now=datetime.now() + log.info(f"uid={uid}") + + store[uid]=Connection( + uri=connection['uri'], + connection_type=connection['connection_type'], + data_type=connection['data_type'], + capacity=connection['capacity'] if 'capacity' in connection else 0, # Backwards compatibility + time=timestamp + ) + + now=datetime.now() + elapsed=now-timestamp + + log.debug(f"Took {elapsed.microseconds} us to add {len(js['connections'])} connections") + + global npublishes, publish_time + publish_time+=elapsed + npublishes+=1 + + if len(store)>maxentries[part]: + maxentries[part]=len(store) + + partlock.release() + return 'OK' + +@app.route("/retract-partition",methods=['POST']) +def retract_partition(): + if len(request.data) == 0: + abort(400) - for connection in js["connections"]: - if "uid" in connection and "uri" in connection: - uid = connection["uid"] - now = datetime.now() - log.info(f"uid={uid}") - - store[uid] = Connection( - uri=connection["uri"], - connection_type=connection["connection_type"], - data_type=connection["data_type"], - capacity=( - connection["capacity"] if "capacity" in connection else 0 - ), # Backwards compatibility - time=timestamp, - ) - - now = datetime.now() - elapsed = now - timestamp - - log.debug( - f"Took {elapsed.microseconds} us to add {len(js['connections'])} connections" - ) + js=json.loads(request.data) + log.debug(f"request=[{js}]") - global npublishes, publish_time - publish_time += elapsed - npublishes += 1 + if 'partition' not in js: + abort(400) - if len(store) > maxentries[part]: - maxentries[part] = len(store) + part=js['partition'] + partlock.acquire() + if part in partitions: + partitions.pop(part) partlock.release() - return "OK" + return 'OK' + partlock.release() + abort(404) +@app.route("/retract",methods=['POST']) +def retract(): + if len(request.data) == 0: + abort(400) + + js=json.loads(request.data) + good=True + part=js['partition'] + partlock.acquire() + if part not in partitions: + partlock.release() + return make_response(f"Partition {part} not found", 404) + + store=partitions[part] + if 'connections' not in js: + abort(400) + for con in js['connections']: + id=con['connection_id'] + if id in store: + store.pop(id) + else: + log.info(f"Could not find connection_id <{id}>") + good=False + if len(store)==0: + # We've deleted the last entry in this partition so delete the + # partition as well + partitions.pop(part) + partlock.release() + if good: + return 'OK' + abort(404) + +@app.route("/getconnection/",methods=['POST','GET']) +def get_connection(part): + if len(request.data) == 0: + abort(400) -@app.route("/retract-partition", methods=["POST"]) -def retract_partition(): - if len(request.data) == 0: - abort(400) + # Find connection uris that correspond to the connection id pattern + # in the request. The pattern is treated as a regular expression. - js = json.loads(request.data) - log.debug(f"request=[{js}]") + now=datetime.now() + js=json.loads(request.data) + log.debug(f"{js=}") - if "partition" not in js: - abort(400) + if 'uid_regex' in js and 'data_type' in js: + log.info( + f"Searching for connections matching uid_regex<{js['uid_regex']}> and data_type {js['data_type']}" + ) - part = js["partition"] + result=[] + regex=re.compile(js['uid_regex']) + dt=js['data_type'] partlock.acquire() if part in partitions: - partitions.pop(part) - partlock.release() - return "OK" - partlock.release() - abort(404) - + store=partitions[part] + matched=[] + for uid,con in store.items(): + if regex.fullmatch(uid) and con.data_type==dt and now-con.time") - good = False - if len(store) == 0: - # We've deleted the last entry in this partition so delete the - # partition as well - partitions.pop(part) partlock.release() - if good: - return "OK" + log.info(f"Partition {part} not found") abort(404) + else: + abort(400) - -@app.route("/getconnection/", methods=["POST", "GET"]) -def get_connection(part): - if len(request.data) == 0: - abort(400) - - # Find connection uris that correspond to the connection id pattern - # in the request. The pattern is treated as a regular expression. - - now = datetime.now() - js = json.loads(request.data) - log.debug(f"{js=}") - - if "uid_regex" in js and "data_type" in js: - log.info( - f"Searching for connections matching uid_regex<{js['uid_regex']}> and data_type {js['data_type']}" - ) - - result = [] - regex = re.compile(js["uid_regex"]) - dt = js["data_type"] - partlock.acquire() - - if part in partitions: - store = partitions[part] - matched = [] - for uid, con in store.items(): - if ( - regex.fullmatch(uid) - and con.data_type == dt - and now - con.time < entry_ttl - ): - matched.append((uid, con)) - partlock.release() - # We should now be able to construct JSON string while other threads - # have access to the partition dict - for uid, con in matched: - result.append( - "{" - f'"uid":"{uid}",' - f'"uri":"{con.uri}",' - f'"connection_type":{con.connection_type},' - f'"data_type":"{con.data_type}",' - f'"capacity":{con.capacity}' - "}" - ) - - td = datetime.now() - now - log.debug( - f"Lookup took {td.microseconds} us to find {len(result)} connections" - ) - global nlookups, lookup_time - # Should we have the lock while updating statistics? It doesn't - # really matter if the stats aren't entirely accurate. - nlookups += 1 - lookup_time += td - - return "[" + ",".join(result) + "]" - - partlock.release() - log.info(f"Partition {part} not found") - abort(404) - else: - abort(400) diff --git a/tests/test_connectivityservice.py b/tests/test_connectivityservice.py index 7c8e4a7..ec31065 100644 --- a/tests/test_connectivityservice.py +++ b/tests/test_connectivityservice.py @@ -3,22 +3,22 @@ import connectivityserver.connectionflask as cf - @pytest.fixture() def app(): - yield cf.app + yield cf.app @pytest.fixture() def client(app): return app.test_client() - @pytest.fixture() def runner(app): return app.test_cli_runner() + + con = json.loads("""{ "connections":[ { @@ -38,116 +38,100 @@ def runner(app): }""") -def test_noconnection(client): - resp = client.get("/getconnection/bad/con") - assert resp.status_code == 404 +def test_noconnection(client): + resp = client.get("/getconnection/bad/con") + assert resp.status_code == 404 def test_publish(client): - resp = client.post("/publish", json=con) - assert resp.status_code == 200 - + resp = client.post("/publish", json=con) + assert resp.status_code == 200 def test_lookup(client): - query = json.loads("""{"uid_regex":"DRO.*", "data_type":"TPSet"}""") - resp = client.post("/getconnection/ccTest", json=query) - assert resp.status_code == 200 + query = json.loads("""{"uid_regex":"DRO.*", "data_type":"TPSet"}""") + resp = client.post("/getconnection/ccTest", json=query) + assert resp.status_code == 200 - rjson = json.loads(resp.data) - assert len(rjson) == 2 - assert rjson[0]["uid"] == "DRO-000-tp_to_trigger" - assert rjson[1]["uid"] == "DRO-001-tp_to_trigger" + rjson = json.loads(resp.data) + assert len(rjson) == 2 + assert rjson[0]["uid"] == "DRO-000-tp_to_trigger" + assert rjson[1]["uid"] == "DRO-001-tp_to_trigger" - query = json.loads("""{"uid_regex":"DUMMY.*", "data_type":"TPSet"}""") - resp = client.post("/getconnection/ccTest", json=query) - assert resp.status_code == 200 - rjson = json.loads(resp.data) - assert len(rjson) == 0 + query = json.loads("""{"uid_regex":"DUMMY.*", "data_type":"TPSet"}""") + resp = client.post("/getconnection/ccTest", json=query) + assert resp.status_code == 200 + rjson = json.loads(resp.data) + assert len(rjson) == 0 def test_retract(client): - resp = client.post("/retract") - assert resp.status_code == 400 + resp = client.post("/retract") + assert resp.status_code == 400 - retraction = json.loads("""{"partition":"ccTest", + retraction = json.loads("""{"partition":"ccTest", "connections":[{"connection_id":"DRO-000-tp_to_trigger"}, {"connection_id":"DRO-001-tp_to_trigger"}] }""") - resp = client.post("/retract", json=retraction) - assert resp.status_code == 200 + resp = client.post("/retract", json=retraction) + assert resp.status_code == 200 - query = json.loads("""{"uid_regex":"DRO.*", "data_type":"TPSet"}""") - resp = client.post("/getconnection/ccTest", json=query) - assert resp.status_code == 404 + query = json.loads("""{"uid_regex":"DRO.*", "data_type":"TPSet"}""") + resp = client.post("/getconnection/ccTest", json=query) + assert resp.status_code == 404 - # Second time should fail - resp = client.post("/retract", json=retraction) - assert resp.status_code == 404 + # Second time should fail + resp = client.post("/retract", json=retraction) + assert resp.status_code == 404 def test_retract_partition(client): - resp = client.post("/publish", json=con) - assert resp.status_code == 200 + resp = client.post("/publish", json=con) + assert resp.status_code == 200 - resp = client.post("/retract-partition") - assert resp.status_code == 400 + resp = client.post("/retract-partition") + assert resp.status_code == 400 - retraction = json.loads("""{"partition":"ccTest"}""") - resp = client.post("/retract-partition", json=retraction) - assert resp.status_code == 200 + retraction = json.loads("""{"partition":"ccTest"}""") + resp = client.post("/retract-partition", json=retraction) + assert resp.status_code == 200 - # Second time should fail - resp = client.post("/retract-partition", json=retraction) - assert resp.status_code == 404 + # Second time should fail + resp = client.post("/retract-partition", json=retraction) + assert resp.status_code == 404 def test_dump(client): - resp = client.get("/") - assert b"Dump" in resp.data - + resp = client.get("/") + assert b"Dump" in resp.data def test_stats(client): - resp = client.get("/stats") - assert resp.status_code == 200 - assert b"

Connection server statistics" in resp.data - assert b"

0 calls to publish" not in resp.data - - resp = client.get("/resetStats") - assert resp.status_code == 200 - assert b"

Connection server statistics" in resp.data - assert b"

0 calls to publish" not in resp.data - - resp = client.get("/stats") - assert resp.status_code == 200 - assert b"

Connection server statistics" in resp.data - assert b"

0 calls to publish" in resp.data - + resp = client.get("/stats") + assert resp.status_code == 200 + assert b"

Connection server statistics" in resp.data + assert b"

0 calls to publish" not in resp.data + + resp = client.get("/resetStats") + assert resp.status_code == 200 + assert b"

Connection server statistics" in resp.data + assert b"

0 calls to publish" not in resp.data + + resp = client.get("/stats") + assert resp.status_code == 200 + assert b"

Connection server statistics" in resp.data + assert b"

0 calls to publish" in resp.data + def test_reset(client): - resp = client.post("/publish", json=con) - assert resp.status_code == 200 - - resp = client.get("/stats") - assert resp.status_code == 200 - assert b"

0 calls to publish" not in resp.data - - resp = client.get("/resetService") - assert resp.status_code == 200 - - resp = client.get("/stats") - assert resp.status_code == 200 - assert b"

0 calls to publish" in resp.data - + resp = client.post("/publish", json=con) + assert resp.status_code == 200 -def test_live(client): - resp = client.get("/live") - assert resp.status_code == 200 - assert b"OK" in resp.data + resp = client.get("/stats") + assert resp.status_code == 200 + assert b"

0 calls to publish" not in resp.data + resp = client.get("/resetService") + assert resp.status_code == 200 -def test_ready(client): - resp = client.post("/publish", json=con) - assert resp.status_code == 200 - resp = client.get("/ready") - assert resp.status_code == 200 - assert b"OK" in resp.data + resp = client.get("/stats") + assert resp.status_code == 200 + assert b"

0 calls to publish" in resp.data