Skip to content

Commit 09969e7

Browse files
sathiya-nhssathiya-nhs
authored
Merge pull request #537 from NHSDigital/master
APIM-6679-Python-Update
2 parents 99772c8 + 3cc4f0d commit 09969e7

25 files changed

Lines changed: 1170 additions & 944 deletions

File tree

‎.github/SBOM-README.md‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
# SBOM & Vulnerability Scanning Automation
2+
3+
This repository uses GitHub Actions to automatically generate a Software Bill of Materials (SBOM), scan for vulnerabilities, and produce package inventory reports.
4+
5+
All reports are named with the repository name for easy identification.
6+
7+
## Features
8+
9+
SBOM Generation: Uses Syft to generate an SPDX JSON SBOM.
10+
SBOM Merging: Merges SBOMs for multiple tools if needed.
11+
SBOM to CSV: Converts SBOM JSON to a CSV report.
12+
Vulnerability Scanning: Uses Grype to scan the SBOM for vulnerabilities and outputs a CSV report.
13+
Package Inventory: Extracts a simple package list (name, type, version) as a CSV.
14+
Artifacts: All reports are uploaded as workflow artifacts with the repository name in the filename.
15+
16+
## Workflow Overview
17+
18+
The main workflow is defined in .github/workflows/sbom.yml
19+
20+
## Scripts
21+
22+
scripts/create-sbom.sh
23+
Generates an SBOM for the repo and for specified tools, merging them as needed.
24+
scripts/update-sbom.py
25+
Merges additional SBOMs into the main SBOM.
26+
.github/scripts/sbom_json_to_csv.py
27+
Converts the SBOM JSON to a detailed CSV report.
28+
.github/scripts/grype_json_to_csv.py
29+
Converts Grype’s vulnerability scan JSON output to a CSV report.
30+
Output columns: REPO, NAME, INSTALLED, FIXED-IN, TYPE, VULNERABILITY, SEVERITY
31+
.github/scripts/sbom_packages_to_csv.py
32+
Extracts a simple package inventory from the SBOM.
33+
Output columns: name, type, version
34+
35+
## Example Reports
36+
37+
Vulnerability Report
38+
grype-report-[RepoName].csv
39+
REPO,NAME,INSTALLED,FIXED-IN,TYPE,VULNERABILITY,SEVERITY
40+
my-repo,Flask,2.1.2,,library,CVE-2022-12345,High
41+
...
42+
43+
Package Inventory
44+
sbom-packages-[RepoName].csv
45+
name,type,version
46+
Flask,library,2.1.2
47+
Jinja2,library,3.1.2
48+
...
49+
50+
## Usage
51+
52+
Push to main branch or run the workflow manually.
53+
Download artifacts from the workflow run summary.
54+
55+
## Customization
56+
57+
Add more tools to scripts/create-sbom.sh as needed.
58+
Modify scripts to adjust report formats or add more metadata.

‎.github/scripts/grype_json_to_csv.py‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
import json
2+
import csv
3+
import sys
4+
5+
input_file = sys.argv[1] if len(sys.argv) > 1 else "grype-report.json"
6+
output_file = sys.argv[2] if len(sys.argv) > 2 else "grype-report.csv"
7+
8+
with open(input_file, "r", encoding="utf-8") as f:
9+
data = json.load(f)
10+
11+
columns = ["NAME", "INSTALLED", "FIXED-IN", "TYPE", "VULNERABILITY", "SEVERITY"]
12+
13+
with open(output_file, "w", newline="", encoding="utf-8") as csvfile:
14+
writer = csv.DictWriter(csvfile, fieldnames=columns)
15+
writer.writeheader()
16+
for match in data.get("matches", []):
17+
pkg = match.get("artifact", {})
18+
vuln = match.get("vulnerability", {})
19+
row = {
20+
"NAME": pkg.get("name", ""),
21+
"INSTALLED": pkg.get("version", ""),
22+
"FIXED-IN": vuln.get("fix", {}).get("versions", [""])[0] if vuln.get("fix", {}).get("versions") else "",
23+
"TYPE": pkg.get("type", ""),
24+
"VULNERABILITY": vuln.get("id", ""),
25+
"SEVERITY": vuln.get("severity", ""),
26+
}
27+
writer.writerow(row)
28+
print(f"CSV export complete: {output_file}")

‎.github/scripts/sbom_json_to_csv.py‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
import json
2+
import csv
3+
import sys
4+
# from pathlib import Path
5+
from tabulate import tabulate
6+
7+
input_file = sys.argv[1] if len(sys.argv) > 1 else "sbom.json"
8+
output_file = sys.argv[2] if len(sys.argv) > 2 else "sbom.csv"
9+
10+
with open(input_file, "r", encoding="utf-8") as f:
11+
sbom = json.load(f)
12+
13+
packages = sbom.get("packages", [])
14+
15+
columns = [
16+
"name",
17+
"versionInfo",
18+
"type",
19+
"supplier",
20+
"downloadLocation",
21+
"licenseConcluded",
22+
"licenseDeclared",
23+
"externalRefs"
24+
]
25+
26+
27+
def get_type(pkg):
28+
spdxid = pkg.get("SPDXID", "")
29+
if "-" in spdxid:
30+
parts = spdxid.split("-")
31+
if len(parts) > 2:
32+
return parts[2]
33+
refs = pkg.get("externalRefs", [])
34+
for ref in refs:
35+
if ref.get("referenceType") == "purl":
36+
return ref.get("referenceLocator", "").split("/")[0]
37+
return ""
38+
39+
40+
def get_external_refs(pkg):
41+
refs = pkg.get("externalRefs", [])
42+
return ";".join([ref.get("referenceLocator", "") for ref in refs])
43+
44+
45+
with open(output_file, "w", newline="", encoding="utf-8") as csvfile:
46+
writer = csv.DictWriter(csvfile, fieldnames=columns)
47+
writer.writeheader()
48+
for pkg in packages:
49+
row = {
50+
"name": pkg.get("name", ""),
51+
"versionInfo": pkg.get("versionInfo", ""),
52+
"type": get_type(pkg),
53+
"supplier": pkg.get("supplier", ""),
54+
"downloadLocation": pkg.get("downloadLocation", ""),
55+
"licenseConcluded": pkg.get("licenseConcluded", ""),
56+
"licenseDeclared": pkg.get("licenseDeclared", ""),
57+
"externalRefs": get_external_refs(pkg)
58+
}
59+
writer.writerow(row)
60+
61+
print(f"CSV export complete: {output_file}")
62+
63+
64+
with open("sbom_table.txt", "w", encoding="utf-8") as f:
65+
table = []
66+
for pkg in packages:
67+
row = [
68+
pkg.get("name", ""),
69+
pkg.get("versionInfo", ""),
70+
get_type(pkg),
71+
pkg.get("supplier", ""),
72+
pkg.get("downloadLocation", ""),
73+
pkg.get("licenseConcluded", ""),
74+
pkg.get("licenseDeclared", ""),
75+
get_external_refs(pkg)
76+
]
77+
table.append(row)
78+
f.write(tabulate(table, columns, tablefmt="grid"))

‎.github/scripts/sbom_packages_to_csv.py‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
import json
2+
import csv
3+
import sys
4+
import os
5+
6+
input_file = sys.argv[1] if len(sys.argv) > 1 else "sbom.json"
7+
repo_name = sys.argv[2] if len(sys.argv) > 2 else os.getenv("GITHUB_REPOSITORY", "unknown-repo").split("/")[-1]
8+
output_file = f"sbom-packages-{repo_name}.csv"
9+
10+
with open(input_file, "r", encoding="utf-8") as f:
11+
sbom = json.load(f)
12+
13+
packages = sbom.get("packages", [])
14+
15+
columns = ["name", "type", "version"]
16+
17+
with open(output_file, "w", newline="", encoding="utf-8") as csvfile:
18+
writer = csv.DictWriter(csvfile, fieldnames=columns)
19+
writer.writeheader()
20+
for pkg in packages:
21+
row = {
22+
"name": pkg.get("name", ""),
23+
"type": pkg.get("type", ""),
24+
"version": pkg.get("versionInfo", "")
25+
}
26+
writer.writerow(row)
27+
28+
print(f"Package list CSV generated: {output_file}")

‎.github/workflows/publish.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,10 @@ jobs:
1212
with:
1313
fetch-depth: 0 # This causes all history to be fetched, which is required for calculate-version to function
1414

15-
- name: Install Python 3.8
15+
- name: Install Python 3.13
1616
uses: actions/setup-python@v5
1717
with:
18-
python-version: 3.8
18+
python-version: 3.13
1919

2020
- name: Install poetry
2121
run: pip install poetry

‎.github/workflows/sbom.yml‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
name: SBOM Vulnerability Scanning
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
environment:
7+
description: "Run SBOM check"
8+
required: true
9+
type: choice
10+
options:
11+
- yes
12+
- no
13+
14+
env:
15+
SYFT_VERSION: "1.27.1"
16+
TF_VERSION: "1.12.2"
17+
18+
jobs:
19+
deploy:
20+
name: Software Bill of Materials
21+
runs-on: ubuntu-latest
22+
permissions:
23+
actions: read
24+
contents: write
25+
steps:
26+
- name: Checkout
27+
uses: actions/checkout@v5
28+
29+
- name: Setup Python 3.13
30+
uses: actions/setup-python@v5
31+
with:
32+
python-version: "3.13"
33+
34+
- name: Setup Terraform
35+
uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd
36+
37+
- uses: terraform-linters/setup-tflint@ae78205cfffec9e8d93fd2b3115c7e9d3166d4b6
38+
name: Setup TFLint
39+
40+
- name: Set architecture variable
41+
id: os-arch
42+
run: |
43+
case "${{ runner.arch }}" in
44+
X64) ARCH="amd64" ;;
45+
ARM64) ARCH="arm64" ;;
46+
esac
47+
echo "arch=${ARCH}" >> $GITHUB_OUTPUT
48+
49+
- name: Download and setup Syft
50+
run: |
51+
DOWNLOAD_URL="https://github.com/anchore/syft/releases/download/v${{ env.SYFT_VERSION }}/syft_${{ env.SYFT_VERSION }}_linux_${{ steps.os-arch.outputs.arch }}.tar.gz"
52+
echo "Downloading: ${DOWNLOAD_URL}"
53+
54+
curl -L -o syft.tar.gz "${DOWNLOAD_URL}"
55+
tar -xzf syft.tar.gz
56+
chmod +x syft
57+
58+
# Add to PATH for subsequent steps
59+
echo "$(pwd)" >> $GITHUB_PATH
60+
61+
- name: Create SBOM
62+
run: bash scripts/create-sbom.sh terraform python tflint
63+
64+
- name: Convert SBOM JSON to CSV
65+
run: |
66+
pip install --upgrade pip
67+
pip install tabulate
68+
REPO_NAME=$(basename $GITHUB_REPOSITORY)
69+
python .github/scripts/sbom_json_to_csv.py sbom.json SBOM_${REPO_NAME}.csv
70+
71+
- name: Upload SBOM CSV as artifact
72+
uses: actions/upload-artifact@v4
73+
with:
74+
name: sbom-csv
75+
path: SBOM_${{ github.event.repository.name }}.csv
76+
77+
- name: Install Grype
78+
run: |
79+
curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin
80+
81+
- name: Scan SBOM for Vulnerabilities (JSON)
82+
run: |
83+
grype sbom:sbom.json -o json > grype-report.json
84+
85+
86+
87+
- name: Convert Grype JSON to CSV
88+
run: |
89+
pip install --upgrade pip
90+
REPO_NAME=$(basename $GITHUB_REPOSITORY)
91+
python .github/scripts/grype_json_to_csv.py grype-report.json grype-report-${REPO_NAME}.csv
92+
93+
94+
- name: Upload Vulnerability Report
95+
uses: actions/upload-artifact@v4
96+
with:
97+
name: grype-report
98+
path: grype-report-${{ github.event.repository.name }}.csv
99+
100+
- name: Generate Package Inventory CSV
101+
run: |
102+
pip install --upgrade pip
103+
REPO_NAME=$(basename $GITHUB_REPOSITORY)
104+
python .github/scripts/sbom_packages_to_csv.py sbom.json $REPO_NAME
105+
106+
- name: Upload Package Inventory CSV
107+
uses: actions/upload-artifact@v4
108+
with:
109+
name: sbom-packages
110+
path: sbom-packages-${{ github.event.repository.name }}.csv

‎README.md‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,25 @@
11
# api-management-utils
22
Scripts and utilities used across API managment platform and services
33

4+
5+
################################################################################
6+
########### Python upgrade to 3.13 ###########
7+
########### Utils Repo has been updated to python 3.13 ###########
8+
################################################################################
9+
10+
########### We are continuing to support python 3.8/9(which are currently out of support) until January 26th 2026 ###########
11+
########### After the deadline your pipelines will fail if you are using python version 3.8/9 ###########
12+
13+
Python upgrade related changes
14+
###############################
15+
Projects using Python versions older than 3.13 and extending their pipeline with the utils repository must update their pipelines to ensure compatibility with the latest changes.
16+
For detailed guidance, please refer to the APIM FAQ page:
17+
https://nhsd-confluence.digital.nhs.uk/spaces/APM/pages/1226682275/Pipeline+Queries
18+
19+
Note: Projects running Python version 3.13 or later do not need any pipeline modifications.
20+
21+
22+
423
## Scripts
524
* `template.py` - cli for basic jinja templating
625
* `test_pull_request_deployments.py` - cli for testing utils against other repositories

‎ansible/collections/ansible_collections/nhsd/apigee/Makefile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,10 @@ update-schema:
44
@PYTHONPATH=../../../ poetry run python scripts/update_schema.py
55

66
unit-test:
7-
@poetry run ansible-test units --python=3.8
7+
@poetry run ansible-test units --python=3.13
88

99
integration-test:
10-
@poetry run ansible-test integration --python=3.8
10+
@poetry run ansible-test integration --python=3.13
1111

1212
test: unit-test integration-test
1313

‎ansible/collections/ansible_collections/nhsd/apigee/plugins/module_utils/utils.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import typing
66

77
from ansible_collections.nhsd.apigee.plugins.module_utils import constants
8+
from ansible.utils.unsafe_proxy import AnsibleUnsafeText
89

910

1011
def exclude_keys(dict_, keys_to_ignore):
@@ -19,7 +20,7 @@ def delta(before, after, keys_to_ignore=None):
1920
before,
2021
after,
2122
ignore_order=True,
22-
ignore_type_in_groups=[(ansible.utils.unsafe_proxy.AnsibleUnsafeText,str)],
23+
ignore_type_in_groups=[(AnsibleUnsafeText,str)],
2324
exclude_paths=[f"root['{key}']" for key in keys_to_ignore],
2425
).to_json()
2526
)

‎ansible/collections/ansible_collections/nhsd/apigee/roles/deploy_manifest/tasks/main.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@
5454
# since products can only be created if they reference proxies
5555
# that exist. This safety check can be removed when the
5656
# manifest manages proxies too.
57-
when: item.name | regex_search('^' + SERVICE_NAME + '-' + PULL_REQUEST | default(APIGEE_ENVIRONMENT))
57+
when: (item.name is match('^' ~ SERVICE_NAME ~ '-' ~ (PULL_REQUEST | default(APIGEE_ENVIRONMENT))))
5858

5959
# - name: deploy apigee specs
6060
# nhsd.apigee.deploy_spec:

0 commit comments

Comments
 (0)