diff --git a/deploy/connectivityserver.yaml b/deploy/connectivityserver.yaml
index c957b45..8388d37 100644
--- a/deploy/connectivityserver.yaml
+++ b/deploy/connectivityserver.yaml
@@ -54,25 +54,16 @@ spec:
requests:
memory: 32Mi
cpu: 100m
- startupProbe:
- httpGet:
- path: /ready
- port: http
- failureThreshold: 30
- periodSeconds: 10
livenessProbe:
- httpGet:
- path: /live
+ tcpSocket:
port: http
initialDelaySeconds: 15
periodSeconds: 20
readinessProbe:
- httpGet:
- path: /ready
+ tcpSocket:
port: http
- initialDelaySeconds: 5
- periodSeconds: 10
- failureThreshold: 3
+ initialDelaySeconds: 15
+ periodSeconds: 20
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
securityContext:
diff --git a/src/connectivityserver/connectionflask.py b/src/connectivityserver/connectionflask.py
index 16ea84c..b164662 100755
--- a/src/connectivityserver/connectionflask.py
+++ b/src/connectivityserver/connectionflask.py
@@ -17,372 +17,309 @@
from flask import Flask, abort, make_response, request
# Some functions exit with an abort(NNN) instead of return so don't complain!
-# ruff: noqa RET503
+#ruff: noqa RET503
-partitions = {}
-partlock = Lock()
+partitions={}
+partlock=Lock()
-if "CONNECTION_FLASK_DEBUG" in os.environ:
- debug_level = int(os.environ["CONNECTION_FLASK_DEBUG"])
+if 'CONNECTION_FLASK_DEBUG' in os.environ:
+ debug_level=int(os.environ['CONNECTION_FLASK_DEBUG'])
else:
- debug_level = 1
-
+ debug_level=1
def convert_log_level(log_level):
- if log_level == 0:
- return logging.WARNING
- if log_level == 1:
- return logging.INFO
- if log_level == 2:
- return logging.DEBUG
+ if log_level == 0:
+ return logging.WARNING
+ if log_level == 1:
return logging.INFO
+ if log_level == 2:
+ return logging.DEBUG
+ return logging.INFO
-
-logging.basicConfig(
- level=convert_log_level(debug_level),
- format="%(asctime)s %(levelname)s %(filename)s:%(funcName)s:%(lineno)d %(message)s",
-)
+logging.basicConfig(level=convert_log_level(debug_level), format='%(asctime)s %(levelname)s %(filename)s:%(funcName)s:%(lineno)d %(message)s')
log = logging.getLogger(__name__)
-if "ENTRY_TTL" in os.environ:
- ttl = int(os.environ["ENTRY_TTL"])
+if 'ENTRY_TTL' in os.environ:
+ ttl=int(os.environ['ENTRY_TTL'])
else:
- ttl = 10
-entry_ttl = timedelta(seconds=ttl)
-
-last_stats = datetime.now()
-npublishes = 0
-nlookups = 0
-lookup_time = timedelta(0)
-publish_time = timedelta()
-maxpartitions = 0
-maxentries = {}
-
-app = Flask(__name__)
-appstarted = False
-
-
-@app.before_first_request
-def mark_started():
- global appstarted
- appstarted = True
+ ttl=10
+entry_ttl=timedelta(seconds=ttl)
+last_stats=datetime.now()
+npublishes=0
+nlookups=0
+lookup_time=timedelta(0)
+publish_time=timedelta()
+maxpartitions=0
+maxentries={}
-@app.route("/live", methods=["GET"])
-def live():
- return "OK", 200
-
-
-@app.route("/ready", methods=["GET"])
-def ready():
- if appstarted:
- return "OK", 200
- return "Not Ready", 503
-
+app=Flask(__name__)
@app.route("/")
def dump():
- now = datetime.now()
- dstream = StringIO()
- dstream.write("
Dump of configuration dictionary
")
- dstream.write("Active partitions
")
- if len(partitions) > 0:
- pad = ' style="padding-left: 1em;padding-right: 1em"'
- dstream.write(
- f'
'
- f'| Partition | '
- f"Entries |
"
- )
- for p in partitions:
- dstream.write(
- f"| {p} | {len(partitions[p])} |
"
- )
- dstream.write("
")
- dstream.write(f"Partitions
")
- for p in partitions:
- store = partitions[p]
- dstream.write(f"{p}
")
- dstream.write(
- f''
- f''
- f'| Name | '
- f'Connection | '
- f"
"
- f''
- f"| uri | "
- f"data_type | "
- f"capacity | "
- f"connection_type | "
- f"time | "
- f"
"
- )
- format_cell = lambda value, strike: (
- f'{value}' if strike else f"{value}"
- )
- for k, v in store.items():
- expired = now - v.time >= entry_ttl
- dstream.write(
- f"| {format_cell(k, expired)} | "
- f"{format_cell(v.uri, expired)} | "
- f"{format_cell(v.data_type, expired)} | "
- f"{format_cell(v.capacity, expired)} | "
- f"{format_cell(v.connection_type, expired)} | "
- f"{format_cell(v.time, expired)} |
"
- )
- dstream.write("
")
- else:
- dstream.write("None")
- dstream.write("
Server statistics
")
- stats_to_html(dstream)
- dstream.seek(0)
- return dstream.read()
-
+ now=datetime.now()
+ dstream=StringIO()
+ dstream.write('Dump of configuration dictionary
')
+ dstream.write("Active partitions
")
+ if len(partitions)>0:
+ pad=' style="padding-left: 1em;padding-right: 1em"'
+ dstream.write(f'
'
+ f'| Partition | '
+ f'Entries |
')
+ for p in partitions:
+ dstream.write(f'| {p}'
+ f' | {len(partitions[p])} |
')
+ dstream.write("
")
+ dstream.write(f'Partitions
')
+ for p in partitions:
+ store=partitions[p]
+ dstream.write(f'{p}
')
+ dstream.write(f''
+ f''
+ f'| Name | '
+ f'Connection | '
+ f'
'
+ f''
+ f'| uri | '
+ f'data_type | '
+ f'capacity | '
+ f'connection_type | '
+ f'time | '
+ f'
')
+ format_cell=lambda value,strike: f'{value}' if strike else f'{value}'
+ for k,v in store.items():
+ expired = now-v.time >= entry_ttl
+ dstream.write(f'| {format_cell(k,expired)} | '
+ f'{format_cell(v.uri,expired)} | '
+ f'{format_cell(v.data_type,expired)} | '
+ f'{format_cell(v.capacity,expired)} | '
+ f'{format_cell(v.connection_type,expired)} | '
+ f'{format_cell(v.time,expired)} |
')
+ dstream.write("
")
+ else:
+ dstream.write("None")
+ dstream.write("
Server statistics
")
+ stats_to_html(dstream)
+ dstream.seek(0)
+ return dstream.read()
def stats_to_html(dstream):
- dstream.write(f"Since {last_stats}
")
- if npublishes > 0:
- avg_pub = publish_time / npublishes
- else:
- avg_pub = timedelta()
- dstream.write(
- f"{npublishes} calls to publish in total time {publish_time} "
- f"(average {avg_pub.microseconds} µs per call)
"
- )
- if nlookups > 0:
- avg_lookup = lookup_time / nlookups
- else:
- avg_lookup = timedelta()
- dstream.write(
- f"{nlookups} calls to lookup in total time {lookup_time} "
- f"(average {avg_lookup.microseconds} µs per call)
"
- )
- dstream.write(f"Maximum number of partitions active = {maxpartitions}
")
- for part in maxentries:
- dstream.write(
- f"Maximum entries in partition {part} = {maxentries[part]}
"
- )
-
+ dstream.write(f"Since {last_stats}
")
+ if npublishes>0:
+ avg_pub=publish_time/npublishes
+ else:
+ avg_pub=timedelta()
+ dstream.write(f"{npublishes} calls to publish in total time {publish_time} "
+ f"(average {avg_pub.microseconds} µs per call)
")
+ if nlookups>0:
+ avg_lookup=lookup_time/nlookups
+ else:
+ avg_lookup=timedelta()
+ dstream.write(f"{nlookups} calls to lookup in total time {lookup_time} "
+ f"(average {avg_lookup.microseconds} µs per call)
")
+ dstream.write(f"Maximum number of partitions active = {maxpartitions}
")
+ for part in maxentries:
+ dstream.write(f"Maximum entries in partition {part} = {maxentries[part]}
")
@app.route("/stats")
def dumpStats():
- dstream = StringIO()
- dstream.write("Connection server statistics
")
- stats_to_html(dstream)
- dstream.seek(0)
- return dstream.read()
-
+ dstream=StringIO()
+ dstream.write('Connection server statistics
')
+ stats_to_html(dstream)
+ dstream.seek(0)
+ return dstream.read()
@app.route("/resetStats")
def resetStats():
- stats = dumpStats()
-
- global \
- last_stats, \
- npublishes, \
- nlookups, \
- lookup_time, \
- publish_time, \
- maxpartitions, \
- maxentries
+ stats = dumpStats()
- last_stats = datetime.now()
- npublishes = 0
- nlookups = 0
- lookup_time = timedelta(0)
- publish_time = timedelta()
- maxpartitions = 0
- maxentries = {}
+ global last_stats,npublishes,nlookups,lookup_time,publish_time,maxpartitions,maxentries
- return stats
+ last_stats=datetime.now()
+ npublishes=0
+ nlookups=0
+ lookup_time=timedelta(0)
+ publish_time=timedelta()
+ maxpartitions=0
+ maxentries={}
+ return stats
@app.route("/resetService")
def reset():
- global partitions
- partitions = {}
- return resetStats()
+ global partitions
+ partitions={}
+ return resetStats()
-@app.route("/publish", methods=["POST"])
+@app.route("/publish",methods=['POST'])
def publish():
- # Store multiple connection ids and corresponding uris in a
- # dictionary associated with the appropriate partition.
- timestamp = datetime.now()
- js = json.loads(request.data)
-
- log.debug(f"{js=}")
- part = js["partition"]
-
- log.info(
- f"{len(js['connections'])} connections in partition {part} from {request.remote_addr} uri={js['connections'][0]['uri']}..."
- )
-
- partlock.acquire()
- if part in partitions:
- store = partitions[part]
- else:
- store = {}
- partitions[part] = store
- global maxpartitions
- if len(partitions) > maxpartitions:
- maxpartitions = len(partitions)
- if part not in maxentries:
- maxentries[part] = 0
-
- Connection = namedtuple(
- "Connection", ["uri", "data_type", "capacity", "connection_type", "time"]
- )
+ # Store multiple connection ids and corresponding uris in a
+ # dictionary associated with the appropriate partition.
+ timestamp=datetime.now()
+ js=json.loads(request.data)
+
+ log.debug(f"{js=}")
+ part=js['partition']
+
+ log.info(
+ f"{len(js['connections'])} connections in partition {part} from {request.remote_addr} uri={js['connections'][0]['uri']}..."
+ )
+
+ partlock.acquire()
+ if part in partitions:
+ store=partitions[part]
+ else:
+ store={}
+ partitions[part]=store
+ global maxpartitions
+ if len(partitions)>maxpartitions:
+ maxpartitions=len(partitions)
+ if part not in maxentries:
+ maxentries[part]=0
+
+ Connection=namedtuple(
+ 'Connection',['uri','data_type','capacity','connection_type','time'])
+
+ for connection in js['connections']:
+
+ if 'uid' in connection and 'uri' in connection:
+ uid=connection['uid']
+ now=datetime.now()
+ log.info(f"uid={uid}")
+
+ store[uid]=Connection(
+ uri=connection['uri'],
+ connection_type=connection['connection_type'],
+ data_type=connection['data_type'],
+ capacity=connection['capacity'] if 'capacity' in connection else 0, # Backwards compatibility
+ time=timestamp
+ )
+
+ now=datetime.now()
+ elapsed=now-timestamp
+
+ log.debug(f"Took {elapsed.microseconds} us to add {len(js['connections'])} connections")
+
+ global npublishes, publish_time
+ publish_time+=elapsed
+ npublishes+=1
+
+ if len(store)>maxentries[part]:
+ maxentries[part]=len(store)
+
+ partlock.release()
+ return 'OK'
+
+@app.route("/retract-partition",methods=['POST'])
+def retract_partition():
+ if len(request.data) == 0:
+ abort(400)
- for connection in js["connections"]:
- if "uid" in connection and "uri" in connection:
- uid = connection["uid"]
- now = datetime.now()
- log.info(f"uid={uid}")
-
- store[uid] = Connection(
- uri=connection["uri"],
- connection_type=connection["connection_type"],
- data_type=connection["data_type"],
- capacity=(
- connection["capacity"] if "capacity" in connection else 0
- ), # Backwards compatibility
- time=timestamp,
- )
-
- now = datetime.now()
- elapsed = now - timestamp
-
- log.debug(
- f"Took {elapsed.microseconds} us to add {len(js['connections'])} connections"
- )
+ js=json.loads(request.data)
+ log.debug(f"request=[{js}]")
- global npublishes, publish_time
- publish_time += elapsed
- npublishes += 1
+ if 'partition' not in js:
+ abort(400)
- if len(store) > maxentries[part]:
- maxentries[part] = len(store)
+ part=js['partition']
+ partlock.acquire()
+ if part in partitions:
+ partitions.pop(part)
partlock.release()
- return "OK"
+ return 'OK'
+ partlock.release()
+ abort(404)
+@app.route("/retract",methods=['POST'])
+def retract():
+ if len(request.data) == 0:
+ abort(400)
+
+ js=json.loads(request.data)
+ good=True
+ part=js['partition']
+ partlock.acquire()
+ if part not in partitions:
+ partlock.release()
+ return make_response(f"Partition {part} not found", 404)
+
+ store=partitions[part]
+ if 'connections' not in js:
+ abort(400)
+ for con in js['connections']:
+ id=con['connection_id']
+ if id in store:
+ store.pop(id)
+ else:
+ log.info(f"Could not find connection_id <{id}>")
+ good=False
+ if len(store)==0:
+ # We've deleted the last entry in this partition so delete the
+ # partition as well
+ partitions.pop(part)
+ partlock.release()
+ if good:
+ return 'OK'
+ abort(404)
+
+@app.route("/getconnection/",methods=['POST','GET'])
+def get_connection(part):
+ if len(request.data) == 0:
+ abort(400)
-@app.route("/retract-partition", methods=["POST"])
-def retract_partition():
- if len(request.data) == 0:
- abort(400)
+ # Find connection uris that correspond to the connection id pattern
+ # in the request. The pattern is treated as a regular expression.
- js = json.loads(request.data)
- log.debug(f"request=[{js}]")
+ now=datetime.now()
+ js=json.loads(request.data)
+ log.debug(f"{js=}")
- if "partition" not in js:
- abort(400)
+ if 'uid_regex' in js and 'data_type' in js:
+ log.info(
+ f"Searching for connections matching uid_regex<{js['uid_regex']}> and data_type {js['data_type']}"
+ )
- part = js["partition"]
+ result=[]
+ regex=re.compile(js['uid_regex'])
+ dt=js['data_type']
partlock.acquire()
if part in partitions:
- partitions.pop(part)
- partlock.release()
- return "OK"
- partlock.release()
- abort(404)
-
+ store=partitions[part]
+ matched=[]
+ for uid,con in store.items():
+ if regex.fullmatch(uid) and con.data_type==dt and now-con.time")
- good = False
- if len(store) == 0:
- # We've deleted the last entry in this partition so delete the
- # partition as well
- partitions.pop(part)
partlock.release()
- if good:
- return "OK"
+ log.info(f"Partition {part} not found")
abort(404)
+ else:
+ abort(400)
-
-@app.route("/getconnection/", methods=["POST", "GET"])
-def get_connection(part):
- if len(request.data) == 0:
- abort(400)
-
- # Find connection uris that correspond to the connection id pattern
- # in the request. The pattern is treated as a regular expression.
-
- now = datetime.now()
- js = json.loads(request.data)
- log.debug(f"{js=}")
-
- if "uid_regex" in js and "data_type" in js:
- log.info(
- f"Searching for connections matching uid_regex<{js['uid_regex']}> and data_type {js['data_type']}"
- )
-
- result = []
- regex = re.compile(js["uid_regex"])
- dt = js["data_type"]
- partlock.acquire()
-
- if part in partitions:
- store = partitions[part]
- matched = []
- for uid, con in store.items():
- if (
- regex.fullmatch(uid)
- and con.data_type == dt
- and now - con.time < entry_ttl
- ):
- matched.append((uid, con))
- partlock.release()
- # We should now be able to construct JSON string while other threads
- # have access to the partition dict
- for uid, con in matched:
- result.append(
- "{"
- f'"uid":"{uid}",'
- f'"uri":"{con.uri}",'
- f'"connection_type":{con.connection_type},'
- f'"data_type":"{con.data_type}",'
- f'"capacity":{con.capacity}'
- "}"
- )
-
- td = datetime.now() - now
- log.debug(
- f"Lookup took {td.microseconds} us to find {len(result)} connections"
- )
- global nlookups, lookup_time
- # Should we have the lock while updating statistics? It doesn't
- # really matter if the stats aren't entirely accurate.
- nlookups += 1
- lookup_time += td
-
- return "[" + ",".join(result) + "]"
-
- partlock.release()
- log.info(f"Partition {part} not found")
- abort(404)
- else:
- abort(400)
diff --git a/tests/test_connectivityservice.py b/tests/test_connectivityservice.py
index 7c8e4a7..ec31065 100644
--- a/tests/test_connectivityservice.py
+++ b/tests/test_connectivityservice.py
@@ -3,22 +3,22 @@
import connectivityserver.connectionflask as cf
-
@pytest.fixture()
def app():
- yield cf.app
+ yield cf.app
@pytest.fixture()
def client(app):
return app.test_client()
-
@pytest.fixture()
def runner(app):
return app.test_cli_runner()
+
+
con = json.loads("""{
"connections":[
{
@@ -38,116 +38,100 @@ def runner(app):
}""")
-def test_noconnection(client):
- resp = client.get("/getconnection/bad/con")
- assert resp.status_code == 404
+def test_noconnection(client):
+ resp = client.get("/getconnection/bad/con")
+ assert resp.status_code == 404
def test_publish(client):
- resp = client.post("/publish", json=con)
- assert resp.status_code == 200
-
+ resp = client.post("/publish", json=con)
+ assert resp.status_code == 200
def test_lookup(client):
- query = json.loads("""{"uid_regex":"DRO.*", "data_type":"TPSet"}""")
- resp = client.post("/getconnection/ccTest", json=query)
- assert resp.status_code == 200
+ query = json.loads("""{"uid_regex":"DRO.*", "data_type":"TPSet"}""")
+ resp = client.post("/getconnection/ccTest", json=query)
+ assert resp.status_code == 200
- rjson = json.loads(resp.data)
- assert len(rjson) == 2
- assert rjson[0]["uid"] == "DRO-000-tp_to_trigger"
- assert rjson[1]["uid"] == "DRO-001-tp_to_trigger"
+ rjson = json.loads(resp.data)
+ assert len(rjson) == 2
+ assert rjson[0]["uid"] == "DRO-000-tp_to_trigger"
+ assert rjson[1]["uid"] == "DRO-001-tp_to_trigger"
- query = json.loads("""{"uid_regex":"DUMMY.*", "data_type":"TPSet"}""")
- resp = client.post("/getconnection/ccTest", json=query)
- assert resp.status_code == 200
- rjson = json.loads(resp.data)
- assert len(rjson) == 0
+ query = json.loads("""{"uid_regex":"DUMMY.*", "data_type":"TPSet"}""")
+ resp = client.post("/getconnection/ccTest", json=query)
+ assert resp.status_code == 200
+ rjson = json.loads(resp.data)
+ assert len(rjson) == 0
def test_retract(client):
- resp = client.post("/retract")
- assert resp.status_code == 400
+ resp = client.post("/retract")
+ assert resp.status_code == 400
- retraction = json.loads("""{"partition":"ccTest",
+ retraction = json.loads("""{"partition":"ccTest",
"connections":[{"connection_id":"DRO-000-tp_to_trigger"},
{"connection_id":"DRO-001-tp_to_trigger"}]
}""")
- resp = client.post("/retract", json=retraction)
- assert resp.status_code == 200
+ resp = client.post("/retract", json=retraction)
+ assert resp.status_code == 200
- query = json.loads("""{"uid_regex":"DRO.*", "data_type":"TPSet"}""")
- resp = client.post("/getconnection/ccTest", json=query)
- assert resp.status_code == 404
+ query = json.loads("""{"uid_regex":"DRO.*", "data_type":"TPSet"}""")
+ resp = client.post("/getconnection/ccTest", json=query)
+ assert resp.status_code == 404
- # Second time should fail
- resp = client.post("/retract", json=retraction)
- assert resp.status_code == 404
+ # Second time should fail
+ resp = client.post("/retract", json=retraction)
+ assert resp.status_code == 404
def test_retract_partition(client):
- resp = client.post("/publish", json=con)
- assert resp.status_code == 200
+ resp = client.post("/publish", json=con)
+ assert resp.status_code == 200
- resp = client.post("/retract-partition")
- assert resp.status_code == 400
+ resp = client.post("/retract-partition")
+ assert resp.status_code == 400
- retraction = json.loads("""{"partition":"ccTest"}""")
- resp = client.post("/retract-partition", json=retraction)
- assert resp.status_code == 200
+ retraction = json.loads("""{"partition":"ccTest"}""")
+ resp = client.post("/retract-partition", json=retraction)
+ assert resp.status_code == 200
- # Second time should fail
- resp = client.post("/retract-partition", json=retraction)
- assert resp.status_code == 404
+ # Second time should fail
+ resp = client.post("/retract-partition", json=retraction)
+ assert resp.status_code == 404
def test_dump(client):
- resp = client.get("/")
- assert b"Dump" in resp.data
-
+ resp = client.get("/")
+ assert b"Dump" in resp.data
def test_stats(client):
- resp = client.get("/stats")
- assert resp.status_code == 200
- assert b"Connection server statistics" in resp.data
- assert b"
0 calls to publish" not in resp.data
-
- resp = client.get("/resetStats")
- assert resp.status_code == 200
- assert b"
Connection server statistics" in resp.data
- assert b"
0 calls to publish" not in resp.data
-
- resp = client.get("/stats")
- assert resp.status_code == 200
- assert b"
Connection server statistics" in resp.data
- assert b"
0 calls to publish" in resp.data
-
+ resp = client.get("/stats")
+ assert resp.status_code == 200
+ assert b"
Connection server statistics" in resp.data
+ assert b"
0 calls to publish" not in resp.data
+
+ resp = client.get("/resetStats")
+ assert resp.status_code == 200
+ assert b"
Connection server statistics" in resp.data
+ assert b"
0 calls to publish" not in resp.data
+
+ resp = client.get("/stats")
+ assert resp.status_code == 200
+ assert b"
Connection server statistics" in resp.data
+ assert b"
0 calls to publish" in resp.data
+
def test_reset(client):
- resp = client.post("/publish", json=con)
- assert resp.status_code == 200
-
- resp = client.get("/stats")
- assert resp.status_code == 200
- assert b"
0 calls to publish" not in resp.data
-
- resp = client.get("/resetService")
- assert resp.status_code == 200
-
- resp = client.get("/stats")
- assert resp.status_code == 200
- assert b"
0 calls to publish" in resp.data
-
+ resp = client.post("/publish", json=con)
+ assert resp.status_code == 200
-def test_live(client):
- resp = client.get("/live")
- assert resp.status_code == 200
- assert b"OK" in resp.data
+ resp = client.get("/stats")
+ assert resp.status_code == 200
+ assert b"
0 calls to publish" not in resp.data
+ resp = client.get("/resetService")
+ assert resp.status_code == 200
-def test_ready(client):
- resp = client.post("/publish", json=con)
- assert resp.status_code == 200
- resp = client.get("/ready")
- assert resp.status_code == 200
- assert b"OK" in resp.data
+ resp = client.get("/stats")
+ assert resp.status_code == 200
+ assert b"
0 calls to publish" in resp.data