Skip to content

Commit 60f0e37

Browse files
author
Bhautik Vala
committed
[patch] Merge stable into aibroker
2 parents d0f8fe3 + 1276341 commit 60f0e37

8 files changed

Lines changed: 3062 additions & 5 deletions

File tree

README.md

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,3 +37,56 @@ updateTektonDefinitions(pipelinesNamespace, "/mascli/templates/ibm-mas-tekton.ya
3737
pipelineURL = launchUpgradePipeline(self.dynamicClient, instanceId)
3838
print(pipelineURL)
3939
```
40+
41+
42+
mas-devops-create-initial-users
43+
---------------------------------------------
44+
45+
46+
Add to /etc/hosts
47+
```
48+
127.0.0.1 tgk01-masdev.mas-tgk01-manage.svc.cluster.local
49+
127.0.0.1 coreapi.mas-tgk01-core.svc.cluster.local
50+
127.0.0.1 admin-dashboard.mas-tgk01-core.svc.cluster.local
51+
```
52+
53+
```bash
54+
SM_AWS_REGION=""
55+
SM_AWS_ACCESS_KEY_ID=""
56+
SM_AWS_SECRET_ACCESS_KEY=""
57+
58+
aws configure set default.region ${SM_AWS_REGION}
59+
aws configure set aws_access_key_id ${SM_AWS_ACCESS_KEY_ID}
60+
aws configure set aws_secret_access_key ${SM_AWS_SECRET_ACCESS_KEY}
61+
62+
63+
oc login --token=sha256~xxx --server=https://xxx:6443
64+
65+
oc port-forward service/admin-dashboard 8445:443 -n mas-tgk01-core
66+
oc port-forward service/coreapi 8444:443 -n mas-tgk01-core
67+
oc port-forward service/tgk01-masdev 8443:443 -n mas-tgk01-manage
68+
69+
mas-devops-create-initial-users-for-saas \
70+
--mas-instance-id tgk01 \
71+
--mas-workspace-id masdev \
72+
--log-level INFO \
73+
--initial-users-secret-name "aws-dev/noble4/tgk01/initial_users" \
74+
--manage-api-port 8443 \
75+
--coreapi-port 8444 \
76+
--admin-dashboard-port 8445
77+
78+
79+
mas-devops-create-initial-users-for-saas \
80+
--mas-instance-id tgk01 \
81+
--mas-workspace-id masdev \
82+
--log-level INFO \
83+
--initial-users-yaml-file /home/tom/workspaces/notes/mascore3423/example-users-single.yaml \
84+
--manage-api-port 8443 \
85+
--coreapi-port 8444 \
86+
--admin-dashboard-port 8445
87+
```
88+
89+
Example of initial_users secret:
90+
```json
91+
{"john.smith1@example.com":"primary,john1,smith1","john.smith2@example.com":"primary,john2,smith2","john.smith3@example.com":"secondary,john3,smith3"}
92+
```
Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
#!/usr/bin/env python3
2+
3+
# *****************************************************************************
4+
# Copyright (c) 2025 IBM Corporation and other Contributors.
5+
#
6+
# All rights reserved. This program and the accompanying materials
7+
# are made available under the terms of the Eclipse Public License v1.0
8+
# which accompanies this distribution, and is available at
9+
# http://www.eclipse.org/legal/epl-v10.html
10+
#
11+
# *****************************************************************************
12+
13+
from kubernetes import client, config
14+
from kubernetes.config.config_exception import ConfigException
15+
import argparse
16+
import logging
17+
import urllib3
18+
urllib3.disable_warnings()
19+
import yaml
20+
import json
21+
import sys
22+
23+
import boto3
24+
from botocore.exceptions import ClientError
25+
26+
from mas.devops.users import MASUserUtils
27+
28+
29+
30+
if __name__ == "__main__":
31+
parser = argparse.ArgumentParser()
32+
33+
# Primary Options
34+
parser.add_argument("--mas-instance-id", required=True)
35+
parser.add_argument("--mas-workspace-id", required=True)
36+
parser.add_argument("--log-level", required=False, choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], default="INFO")
37+
parser.add_argument("--coreapi-port", required=False, default=443)
38+
parser.add_argument("--admin-dashboard-port", required=False, default=443)
39+
parser.add_argument("--manage-api-port", required=False, default=443)
40+
41+
42+
group = parser.add_mutually_exclusive_group(required=True)
43+
group.add_argument("--initial-users-yaml-file")
44+
group.add_argument("--initial-users-secret-name")
45+
46+
args, unknown = parser.parse_known_args()
47+
48+
log_level = getattr(logging, args.log_level)
49+
50+
logger = logging.getLogger()
51+
logger.setLevel(log_level)
52+
53+
ch = logging.StreamHandler()
54+
ch.setLevel(log_level)
55+
chFormatter = logging.Formatter(
56+
"%(asctime)-25s %(name)-50s [%(threadName)s] %(levelname)-8s %(message)s"
57+
)
58+
ch.setFormatter(chFormatter)
59+
logger.addHandler(ch)
60+
61+
mas_instance_id = args.mas_instance_id
62+
mas_workspace_id = args.mas_workspace_id
63+
initial_users_yaml_file = args.initial_users_yaml_file
64+
initial_users_secret_name = args.initial_users_secret_name
65+
coreapi_port = args.coreapi_port
66+
admin_dashboard_port = args.admin_dashboard_port
67+
manage_api_port = args.manage_api_port
68+
69+
70+
logger.info("Configuration:")
71+
logger.info("--------------")
72+
logger.info(f"mas_instance_id: {mas_instance_id}")
73+
logger.info(f"mas_workspace_id: {mas_workspace_id}")
74+
logger.info(f"initial_users_yaml_file: {initial_users_yaml_file}")
75+
logger.info(f"initial_users_secret_name: {initial_users_secret_name}")
76+
logger.info(f"log_level: {log_level}")
77+
logger.info(f"coreapi_port: {coreapi_port}")
78+
logger.info(f"admin_dashboard_port: {admin_dashboard_port}")
79+
logger.info(f"manage_api_port: {manage_api_port}")
80+
logger.info("")
81+
82+
try:
83+
# Try to load in-cluster configuration
84+
config.load_incluster_config()
85+
logger.debug("Loaded in-cluster configuration")
86+
except ConfigException:
87+
# If that fails, fall back to kubeconfig file
88+
config.load_kube_config()
89+
logger.debug("Loaded kubeconfig file")
90+
91+
92+
user_utils = MASUserUtils(mas_instance_id, mas_workspace_id, client.api_client.ApiClient(), coreapi_port=coreapi_port, admin_dashboard_port=admin_dashboard_port, manage_api_port=manage_api_port)
93+
94+
if initial_users_secret_name is not None:
95+
96+
logger.info(f"Loading initial_users configuration from secret {initial_users_secret_name}")
97+
98+
session = boto3.session.Session()
99+
aws_sm_client = session.client(
100+
service_name='secretsmanager',
101+
)
102+
try:
103+
initial_users_secret = aws_sm_client.get_secret_value( # pragma: allowlist secret
104+
SecretId=initial_users_secret_name
105+
)
106+
except ClientError as e:
107+
if e.response['Error']['Code'] == 'ResourceNotFoundException':
108+
logger.info(f"Secret {initial_users_secret_name} was not found, nothing to do, exiting now.")
109+
sys.exit(0)
110+
111+
raise Exception(f"Failed to fetch secret {initial_users_secret_name}: {str(e)}")
112+
113+
secret_json = json.loads(initial_users_secret['SecretString'])
114+
initial_users = user_utils.parse_initial_users_from_aws_secret_json(secret_json)
115+
elif initial_users_yaml_file is not None:
116+
with open(initial_users_yaml_file, 'r') as file:
117+
initial_users = yaml.safe_load(file)
118+
else:
119+
raise Exception("Something unexpected happened")
120+
121+
122+
result = user_utils.create_initial_users_for_saas(initial_users)
123+
124+
# if user details were sourced from an AWS SM secret, remove the completed entries from the secret
125+
# so we don't try and resync them the next time round (and potentially undo an update made by a customer)
126+
if initial_users_secret_name is not None:
127+
has_updates = False
128+
for completed_user in result["completed"]:
129+
logger.info(f"Removing synced user {completed_user['email']} from {initial_users_secret_name} secret")
130+
secret_json.pop(completed_user["email"])
131+
has_updates = True
132+
133+
if has_updates:
134+
logger.info(f"Updating secret {initial_users_secret_name}")
135+
try:
136+
aws_sm_client.update_secret( # pragma: allowlist secret
137+
SecretId=initial_users_secret_name,
138+
SecretString=json.dumps(secret_json)
139+
)
140+
except ClientError as e:
141+
raise Exception(f"Failed to update secret {initial_users_secret_name}: {str(e)}")
142+
143+
144+
if len(result["failed"]) > 0:
145+
failed_user_ids = list(map(lambda u : u["email"], result["failed"]))
146+
raise Exception(f"Sync failed for the following user IDs {failed_user_ids}")

setup.py

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -60,14 +60,17 @@ def get_version(rel_path):
6060
'kubernetes', # Apache Software License
6161
'kubeconfig', # BSD License
6262
'jinja2', # BSD License
63-
'jinja2-base64-filters' # MIT License
63+
'jinja2-base64-filters', # MIT License
64+
'semver', # BSD License
65+
'boto3' # Apache Software License
6466
],
6567
extras_require={
6668
'dev': [
67-
'build', # MIT License
68-
'flake8', # MIT License
69-
'pytest', # MIT License
70-
'pytest-mock' # MIT License
69+
'build', # MIT License
70+
'flake8', # MIT License
71+
'pytest', # MIT License
72+
'pytest-mock', # MIT License
73+
'requests-mock' # Apache Software License
7174
]
7275
},
7376
classifiers=[
@@ -85,6 +88,7 @@ def get_version(rel_path):
8588
],
8689
scripts=[
8790
'bin/mas-devops-db2-validate-config',
91+
'bin/mas-devops-create-initial-users-for-saas',
8892
'bin/mas-devops-saas-job-cleaner'
8993
]
9094
)

src/mas/devops/mas.py

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
from openshift.dynamic import DynamicClient
1919
from openshift.dynamic.exceptions import NotFoundError, ResourceNotFoundError, UnauthorizedError
2020
from jinja2 import Environment, FileSystemLoader
21+
import semver
2122

2223
from .ocp import getStorageClasses
2324
from .olm import getSubscription
@@ -305,3 +306,41 @@ def patchPendingPVC(dynClient: DynamicClient, namespace: str, pvcName: str, stor
305306
except NotFoundError:
306307
logger.error(f"PVC {pvcName} does not exist")
307308
return False
309+
310+
311+
def isVersionBefore(_compare_to_version, _current_version):
312+
"""
313+
The method does a modified semantic version comparison,
314+
as we want to treat any pre-release as == to the real release
315+
but in strict semantic versioning it is <
316+
ie. '8.6.0-pre.m1dev86' is converted to '8.6.0'
317+
"""
318+
if _current_version is None:
319+
print("Version is not informed. Returning False")
320+
return False
321+
322+
strippedVersion = _current_version.split("-")[0]
323+
if '.x' in strippedVersion:
324+
strippedVersion = strippedVersion.replace('.x', '.0')
325+
current_version = semver.VersionInfo.parse(strippedVersion)
326+
compareToVersion = semver.VersionInfo.parse(_compare_to_version)
327+
return current_version.compare(compareToVersion) < 0
328+
329+
330+
def isVersionEqualOrAfter(_compare_to_version, _current_version):
331+
"""
332+
The method does a modified semantic version comparison,
333+
as we want to treat any pre-release as == to the real release
334+
but in strict semantic versioning it is <
335+
ie. '8.6.0-pre.m1dev86' is converted to '8.6.0'
336+
"""
337+
if _current_version is None:
338+
print("Version is not informed. Returning False")
339+
return False
340+
341+
strippedVersion = _current_version.split("-")[0]
342+
if '.x' in strippedVersion:
343+
strippedVersion = strippedVersion.replace('.x', '.0')
344+
current_version = semver.VersionInfo.parse(strippedVersion)
345+
compareToVersion = semver.VersionInfo.parse(_compare_to_version)
346+
return current_version.compare(compareToVersion) >= 0

src/mas/devops/templates/pipelinerun-install.yml.j2

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -816,6 +816,68 @@ spec:
816816
value: "{{ mas_aibroker_db_secret_value }}"
817817
{%- endif %}
818818

819+
# TODO: Fix type for storage sizes and max conn pool size
820+
{%- if mas_app_channel_facilities is defined and mas_app_channel_facilities != "" %}
821+
822+
# Real Estate anda Facilities Application
823+
# -------------------------------------------------------------------------
824+
- name: mas_app_channel_facilities
825+
value: "{{ mas_app_channel_facilities }}"
826+
827+
{%- if mas_ws_facilities_size is defined and mas_ws_facilities_size != "" %}
828+
- name: mas_ws_facilities_size
829+
value: "{{ mas_ws_facilities_size }}"
830+
{%- endif %}
831+
{%- if mas_ws_facilities_routes_timeout is defined and mas_ws_facilities_routes_timeout != "" %}
832+
- name: mas_ws_facilities_routes_timeout
833+
value: "{{ mas_ws_facilities_routes_timeout }}"
834+
{%- endif %}
835+
{%- if mas_ws_facilities_liberty_extension_XML is defined and mas_ws_facilities_liberty_extension_XML != "" %}
836+
- name: mas_ws_facilities_liberty_extension_XML
837+
value: "{{ mas_ws_facilities_liberty_extension_XML }}"
838+
{%- endif %}
839+
{%- if mas_ws_facilities_vault_secret is defined and mas_ws_facilities_vault_secret != "" %}
840+
- name: mas_ws_facilities_vault_secret
841+
value: "{{ mas_ws_facilities_vault_secret }}"
842+
{%- endif %}
843+
{%- if mas_ws_facilities_pull_policy is defined and mas_ws_facilities_pull_policy != "" %}
844+
- name: mas_ws_facilities_pull_policy
845+
value: "{{ mas_ws_facilities_pull_policy }}"
846+
{%- endif %}
847+
- name: mas_ws_facilities_storage_log_class
848+
value: "{{ mas_ws_facilities_storage_log_class }}"
849+
{%- if mas_ws_facilities_storage_log_mode is defined and mas_ws_facilities_storage_log_mode != "" %}
850+
- name: mas_ws_facilities_storage_log_mode
851+
value: "{{ mas_ws_facilities_storage_log_mode }}"
852+
{%- endif %}
853+
# {%- if mas_ws_facilities_storage_log_size is defined and mas_ws_facilities_storage_log_size != "" %}
854+
# - name: mas_ws_facilities_storage_log_size
855+
# value: "{{ mas_ws_facilities_storage_log_size }}"
856+
# {%- endif %}
857+
- name: mas_ws_facilities_storage_userfiles_class
858+
value: "{{ mas_ws_facilities_storage_userfiles_class }}"
859+
{%- if mas_ws_facilities_storage_userfiles_mode is defined and mas_ws_facilities_storage_userfiles_mode != "" %}
860+
- name: mas_ws_facilities_storage_userfiles_mode
861+
value: "{{ mas_ws_facilities_storage_userfiles_mode }}"
862+
{%- endif %}
863+
# {%- if mas_ws_facilities_storage_userfiles_size is defined and mas_ws_facilities_storage_userfiles_size != "" %}
864+
# - name: mas_ws_facilities_storage_userfiles_size
865+
# value: "{{ mas_ws_facilities_storage_userfiles_size }}"
866+
# {%- endif %}
867+
{%- if mas_ws_facilities_dwfagents is defined and mas_ws_facilities_dwfagents != "" %}
868+
- name: mas_ws_facilities_dwfagents
869+
value: "{{ mas_ws_facilities_dwfagents }}"
870+
{%- endif %}
871+
# {%- if mas_ws_facilities_db_maxconnpoolsize is defined and mas_ws_facilities_db_maxconnpoolsize != "" %}
872+
# - name: mas_ws_facilities_db_maxconnpoolsize
873+
# value: "{{ mas_ws_facilities_db_maxconnpoolsize }}"
874+
# {%- endif %}
875+
876+
- name: db2_action_facilities
877+
value: "{{ db2_action_facilities}}"
878+
879+
{%- endif %}
880+
819881
workspaces:
820882
# The generated configuration files
821883
# -------------------------------------------------------------------------

0 commit comments

Comments
 (0)