Merge pull request #460 from NHSDigital/ADZ-3283_handle_company_apps

ADZ-3283 handle company apps

Author: acarriedev

ansible/collections/ansible_collections/nhsd/apigee/plugins/action/add_jwks_resource_url_to_app.py

@@ -1,6 +1,3 @@
-import requests
-import json
-import bisect
 import copy
 
 from ansible_collections.nhsd.apigee.plugins.module_utils.models.ansible.add_jwks_resource_url import (
@@ -13,7 +10,6 @@
 from ansible_collections.nhsd.apigee.plugins.module_utils import constants
 
 ATTRIBUTE_NAME = "jwks-resource-url"
-DEVELOPER_DETAILS = "APIGEE_DEVELOPER_DETAILS"
 
 
 class ActionModule(ApigeeAction):
@@ -39,49 +35,24 @@ def run(self, tmp=None, task_vars=None):
         after["attributes"].append(jwks_attribute)
         after["attributes"] = sorted(after["attributes"], key=lambda attr: attr["name"])
 
-        developer_details = task_vars.get(DEVELOPER_DETAILS)
-        if not developer_details:
-            developer_details = []
-            params = {"expand": True}
-            url = (
-                constants.APIGEE_BASE_URL
-                + f"organizations/{args.organization}/developers"
-            )
-            while True:
-                resp = utils.get(url, args.access_token, params=params)
-                if resp.get("failed"):
-                    return resp
-                devs = resp["response"]["body"]["developer"]
-                developer_details.extend(devs)
-                if len(devs) == 1000:
-                    # last developer's ID as startKey will be included
-                    # in next request, so pop now to de-dupe.
-                    last_dev = developer_details.pop()
-                    params["startKey"] = last_dev["developerId"]
-                else:
-                    break
-
-        try:
-            developer_id = args._app_data["developerId"]
-            developer_ids = [d["developerId"] for d in developer_details]
-            i = bisect.bisect_left(developer_ids, developer_id)
-            if i == len(developer_details):
-                raise RuntimeError(f"Unable to find developer with id {developer_id}")
-        except RuntimeError as e:
-            return {"failed": True, "error": str(e)}
-        
-        developer = developer_details[i]
-
         delta = utils.delta(before, after)
         result = {
             "changed": bool(delta),
-            "app": after,
-            "developer": developer,
-            "ansible_facts": {DEVELOPER_DETAILS: developer_details},
+            "app": after
         }
 
+        company_exists = "companyName" in args._app_data.keys()
+        developer_exists = "developerId" in args._app_data.keys()
+        if developer_exists and not company_exists:
+            owner = args._app_data["developerId"]
+        elif company_exists and not developer_exists:
+            owner = args._app_data["companyName"]
+        else:
+            return {"failed": True, "error": f"Invalid owner for app {args._app_data['appId']}"}
+
+        owner_endpoint = "companies" if company_exists else "developers"
         app_name = args._app_data["name"]
-        app_path = f"organizations/{args.organization}/developers/{developer['email']}/apps/{app_name}/attributes"
+        app_path = f"organizations/{args.organization}/{owner_endpoint}/{owner}/apps/{app_name}/attributes"
 
         if diff_mode:
             result["diff"] = [

ansible/filter_plugins/apigee_helpers.py

@@ -25,11 +25,21 @@ def apigee_apps_to_product_map(apps_list: List[dict], product_filter: str = None
                 if api_product not in result:
                     result[api_product] = []
 
+                company_exists = "companyName" in app.keys()
+                developer_exists = "developerId" in app.keys()
+                if developer_exists and not company_exists:
+                    owner = app["developerId"]
+                elif company_exists and not developer_exists:
+                    owner = app["companyName"]
+                else:
+                    raise RuntimeError(f"Invalid owner for app {app['appId']}")
+
                 result[api_product].append(
                     dict(
                         appId=app["appId"],
                         appName=app["name"],
-                        developerId=app["developerId"],
+                        owner=owner,
+                        ownerEndpoint="companies" if company_exists else "developers",
                         consumerKey=cred["consumerKey"],
                         apiproduct=api_product
                      )
@@ -39,6 +49,13 @@ def apigee_apps_to_product_map(apps_list: List[dict], product_filter: str = None
     return result
 
 
+def product_app_mapping_to_owner_display(dev_id_to_email: dict, product_app: dict):
+    if product_app['ownerEndpoint'] == 'developers':
+        return dev_id_to_email[product_app['owner']]
+    else:
+        return product_app['owner']
+
+
 def apigee_products_to_api_map(products: List[dict], proxy_filter: str = None):
 
     result = dict()
@@ -73,28 +90,34 @@ def apigee_remove_proxy_from_product(product: dict, proxy_to_remove):
     return product
 
 
-def apigee_teams_map(teams: List[dict]):
+def apigee_team_to_admin(team):
+    return next((attr.get('value') for attr in team['attributes'] if attr['name'] == 'ADMIN_EMAIL'), None)
+
+
+def apigee_teams_map(team_members: List[dict], teams: List[dict]):
+    from collections import defaultdict
+    joined = defaultdict(dict)
+    for item in team_members + teams:
+        joined[item['name']].update(item)
+    full_teams = list(joined.values())
 
     return {
-        f"{team['id']}@devteam.apigee.io": {
-            "contact": team['pointOfContact'],
-            "members": list(mem['userId'] for mem in team.get('memberships', []))
+        team['name']: {
+            "contact": apigee_team_to_admin(team),
+            "members": team['members']
         }
-        for team in teams
+        for team in full_teams
     }
 
 
 def apigee_teams_to_point_of_contact(teams: List[dict]):
-
-    return {
-        f"{team['id']}@devteam.apigee.io": team['pointOfContact'] for team in teams
-    }
+    return {apigee_team_to_admin(team) for team in teams}
 
 
 def apigee_teams_to_members(teams: List[dict]):
 
     return {
-        f"{team['id']}@devteam.apigee.io": list(mem['userId'] for mem in team.get('memberships', [])) for team in teams
+        team['owner']: team.get('members', []) for team in teams
     }
 
 
@@ -116,18 +139,16 @@ def apigee_product_developers(
             result[product_name] = []
 
         for app in apps:
-            developer = dev_id_to_email[app["developerId"]]
-
-            team = teams_map.get(developer, {})
-
             entry = {
-                "app": app["appName"],
-                "developer": developer
+                "app": app["appName"]
             }
 
-            if team:
+            if app['ownerEndpoint'] == 'companies':
+                team = teams_map.get(app.get("owner"), {})
                 entry['developer'] = team['contact']
                 entry['team'] = team['members']
+            else:
+                entry['developer'] = dev_id_to_email[app["owner"]]
 
             result[product_name].append(entry)
 
@@ -137,10 +158,6 @@ def apigee_product_developers(
 
     return result
 
-    # return {
-    #     f"{team['id']}@devteam.apigee.io": list(mem['userId'] for mem in team.get('memberships', [])) for team in teams
-    # }
-
 
 class FilterModule:
 
@@ -154,5 +171,6 @@ def filters():
             'apigee_teams_to_point_of_contact': apigee_teams_to_point_of_contact,
             'apigee_teams_to_members': apigee_teams_to_members,
             'apigee_teams_map': apigee_teams_map,
-            'apigee_product_developers': apigee_product_developers
+            'apigee_product_developers': apigee_product_developers,
+            'product_app_mapping_to_owner_display': product_app_mapping_to_owner_display
         }

ansible/get-developer-emails.yml

@@ -1,4 +1,4 @@
-- name: remove old pr portal apis
+- name: get developer emails
   hosts: 127.0.0.1
   connection: local
   gather_facts: yes

ansible/roles/get-developer-emails/tasks/main.yml

@@ -45,15 +45,28 @@
 
 - name: get teams
   uri:
-    url: "https://apigee.com/consumers/api/providers/{{ audience_id }}/teams"
+    url: "{{ companies_uri }}?expand=true"
     headers:
       Authorization: "Bearer {{ APIGEE_ACCESS_TOKEN }}"
     return_content: yes
   register: get_teams
 
+- name: get team members
+  uri:
+    url: "{{ companies_uri }}/{{ item | replace(' ', '%20') }}/developers"
+    headers:
+      Authorization: "Bearer {{ APIGEE_ACCESS_TOKEN }}"
+    return_content: yes
+  register: get_team_members
+  loop: "{{ get_teams.json.company | map(attribute='name') }}"
+
+- name: setup developer team members map
+  set_fact:
+    team_members_map: "{{ get_team_members | json_query('results[*].{members: json.developer, name: item}') }}"
+
 - name: setup developer teams map
   set_fact:
-    teams_map: "{{ get_teams.json.data | apigee_teams_map }}"
+    teams_map: "{{ get_team_members | json_query('results[*].{members: json.developer[*].email, name: item}') | apigee_teams_map(teams=get_teams.json.company) }}"
 
 - name: "get apps"
   uri:

ansible/roles/get-developer-emails/tasks/output-emails.yml

@@ -1,6 +1,6 @@
 
 
-- debug: msg="{{ product_slug }} - {{ item.appName}} - {{ dev_id_to_email[item.developerId] }} - {{ apigee_teams_to_members[dev_id_to_email[item.developerId]] | default(dev_id_to_email[item.developerId]) }}"
+- debug: msg="{{ product_slug }} - {{ item.appName}} - {{ product_app_mapping_to_owner_display(dev_id_to_email, item) }} - {{ apigee_teams_to_members(teams_map)[product_app_mapping_to_owner_display(dev_id_to_email, item)] | default(product_app_mapping_to_owner_display(dev_id_to_email, item)) }}"
   loop: "{{ product_app_map.get(product_slug, []) }}"
   loop_control:
     label: "{{ product_slug }} - {{ item.appName }} - {{ item.appId }}"

ansible/roles/get-developer-emails/vars/main.yml

@@ -2,6 +2,7 @@ org_uri: "https://api.enterprise.apigee.com/v1/organizations/{{ APIGEE_ORGANIZAT
 products_uri: "{{ org_uri }}/apiproducts"
 apps_uri: "{{ org_uri }}/apps"
 developers_uri: "{{ org_uri }}/developers"
+companies_uri: "{{ org_uri }}/companies"
 retain_hours:  "{{ (lookup('env', 'retain_hours') or 72) }}"
 product_filter: "{{ lookup('env', 'PRODUCT_FILTER') }}"
 portals_base_uri: "https://apigee.com/portals/api/sites"

ansible/roles/remove-old-pr-products/tasks/main.yml

@@ -1,16 +1,4 @@
 
-- name: "get developers"
-  uri:
-    url: "{{ developers_uri }}?expand=true"
-    headers:
-      Authorization: "Bearer {{ APIGEE_ACCESS_TOKEN }}"
-    return_content: yes
-  register: get_developers
-
-- name: "map developers"
-  set_fact:
-    dev_id_to_email: "{{ get_developers.json.developer | dict_list_to_map('developerId', 'email') }}"
-
 - name: "get apps"
   uri:
     url: "{{ apps_uri }}?expand=true"

ansible/roles/remove-old-pr-products/tasks/remove-product.yml

@@ -42,7 +42,7 @@
 - block:
   - name: "remove product apps {{ product_slug }}"
     uri:
-      url: "{{ developers_uri }}/{{ dev_id_to_email[item.developerId] }}/apps/{{ item.appName | urlencode }}/keys/{{ item.consumerKey }}/apiproducts/{{ product_slug }}"
+      url: "{{ org_uri }}/{{ item.ownerEndpoint }}/{{ item.owner | urlencode }}/apps/{{ item.appName | urlencode }}/keys/{{ item.consumerKey }}/apiproducts/{{ product_slug }}"
       method: DELETE
       headers:
         Authorization: "Bearer {{ APIGEE_ACCESS_TOKEN }}"

ansible/roles/remove-old-pr-products/vars/main.yml

@@ -1,5 +1,4 @@
 org_uri: "https://api.enterprise.apigee.com/v1/organizations/{{ APIGEE_ORGANIZATION }}"
 products_uri: "{{ org_uri }}/apiproducts"
 apps_uri: "{{ org_uri }}/apps"
-developers_uri: "{{ org_uri }}/developers"
 retain_hours:  "{{ (lookup('env', 'retain_hours') or 72) }}"

ansible/roles/remove-target-products/tasks/main.yml

@@ -1,14 +1,3 @@
-- name: "get developers"
-  uri:
-    url: "{{ developers_uri }}?expand=true"
-    headers:
-      Authorization: "Bearer {{ APIGEE_ACCESS_TOKEN }}"
-    return_content: yes
-  register: get_developers
-
-- name: "map developers"
-  set_fact:
-    dev_id_to_email: "{{ get_developers.json.developer | dict_list_to_map('developerId', 'email') }}"
 
 - name: "get apps"
   uri: