LCORE-2035: konflux tls e2e fix

.tekton/integration-tests/pipeline/lightspeed-stack-integration-test.yaml

@@ -167,6 +167,8 @@ spec:
               echo "========== End parameters =========="
     - name: lightspeed-stack-integration-tests
       description: Task to run integration tests from lightspeed-stack repository
+      # Full Behave suite (proxy + tls) can exceed 2h; needs PipelineRun timeouts >= this value.
+      timeout: 3h
       params:
         - name: SNAPSHOT
           value: $(params.SNAPSHOT)

tests/e2e-prow/rhoai/manifests/lightspeed/e2e-mock-tls-inference.yaml

@@ -0,0 +1,104 @@
+# Mock HTTPS OpenAI API for tls-*.feature (Konflux / Prow; no Docker Compose).
+# Llama Stack run.yaml uses https://e2e-mock-tls-inference.<ns>.svc.cluster.local:8443|8444|8445/v1
+apiVersion: v1
+kind: Pod
+metadata:
+  name: e2e-mock-tls-inference
+  labels:
+    app: e2e-mock-tls-inference
+spec:
+  securityContext:
+    runAsNonRoot: true
+    seccompProfile:
+      type: RuntimeDefault
+  containers:
+    - name: e2e-mock-tls-inference
+      image: python:3.12-slim
+      securityContext:
+        allowPrivilegeEscalation: false
+        capabilities:
+          drop: ["ALL"]
+        runAsNonRoot: true
+        runAsUser: 1000
+        seccompProfile:
+          type: RuntimeDefault
+      env:
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: PYTHONPATH
+          value: /app:/tmp/pydeps
+      command:
+        - /bin/sh
+        - -c
+        - |
+          set -e
+          pip install --quiet --no-cache-dir --target /tmp/pydeps 'trustme>=1.2.1' 'cryptography>=42.0.0'
+          NS="${POD_NAMESPACE:-default}"
+          export TLS_CERT_DNS_NAMES="mock-tls-inference,localhost,127.0.0.1,e2e-mock-tls-inference,e2e-mock-tls-inference.${NS}.svc.cluster.local"
+          exec python /app/server.py
+      ports:
+        - containerPort: 8443
+          name: tls
+        - containerPort: 8444
+          name: mtls
+        - containerPort: 8445
+          name: mismatch
+      volumeMounts:
+        - name: server-script
+          mountPath: /app/server.py
+          subPath: server.py
+          readOnly: true
+        - name: certs-work
+          mountPath: /certs
+      readinessProbe:
+        exec:
+          command:
+            - python3
+            - -c
+            - |
+              import ssl, urllib.request
+              ctx = ssl.create_default_context()
+              ctx.check_hostname = False
+              ctx.verify_mode = ssl.CERT_NONE
+              urllib.request.urlopen("https://localhost:8443/health", context=ctx)
+        initialDelaySeconds: 8
+        periodSeconds: 5
+      livenessProbe:
+        exec:
+          command:
+            - python3
+            - -c
+            - |
+              import ssl, urllib.request
+              ctx = ssl.create_default_context()
+              ctx.check_hostname = False
+              ctx.verify_mode = ssl.CERT_NONE
+              urllib.request.urlopen("https://localhost:8443/health", context=ctx)
+        initialDelaySeconds: 15
+        periodSeconds: 20
+  volumes:
+    - name: server-script
+      configMap:
+        name: e2e-mock-tls-inference-script
+    - name: certs-work
+      emptyDir: {}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: e2e-mock-tls-inference
+spec:
+  selector:
+    app: e2e-mock-tls-inference
+  ports:
+    - name: tls
+      port: 8443
+      targetPort: tls
+    - name: mtls
+      port: 8444
+      targetPort: mtls
+    - name: mismatch
+      port: 8445
+      targetPort: mismatch

tests/e2e-prow/rhoai/manifests/lightspeed/llama-stack-openai.yaml

@@ -38,6 +38,28 @@ spec:
         - -c
         - |
           set -e
+          # Fast-path: PVC already has a valid venv from a previous pod creation in this pipeline run.
+          # TLS scenarios delete+recreate this pod up to 16 times; skipping the expensive install
+          # reduces per-restart time from ~6-15 min to ~30-90 s (just RAG seed refresh + chown).
+          if [[ -d /opt/app-root/.venv ]] \
+              && /opt/app-root/.venv/bin/python --version >/dev/null 2>&1 \
+              && [[ -d /opt/app-root/src ]]; then
+            echo "PVC cache hit: app-root already provisioned — skipping full install"
+            mkdir -p /opt/app-root/.e2e-rag-seed /opt/app-root/src/.llama/storage/rag /opt/app-root/src/.llama/storage/files
+            if [[ -f /rag-seed/kv_store.db.gz ]]; then
+              gzip -dc /rag-seed/kv_store.db.gz > /opt/app-root/.e2e-rag-seed/kv_store.db
+              _sz=$(stat -c%s /opt/app-root/.e2e-rag-seed/kv_store.db)
+              if [[ "${_sz}" -lt 1048576 ]]; then
+                echo "FATAL: RAG seed too small (${_sz} bytes); check rag-data ConfigMap"
+                exit 1
+              fi
+              cp -f /opt/app-root/.e2e-rag-seed/kv_store.db /opt/app-root/src/.llama/storage/rag/kv_store.db
+            fi
+            chmod -R 775 /opt/app-root && chown -R 1001:0 /opt/app-root
+            echo "PVC fast-path complete"
+            exit 0
+          fi
+          # Full provisioning (PVC is empty — first pod creation this pipeline run).
           REPO_URL="${REPO_URL:-https://github.com/lightspeed-core/lightspeed-stack.git}"
           REPO_REVISION="${REPO_REVISION:-main}"
           case "$REPO_URL" in git@github.com:*) REPO_URL="https://github.com/${REPO_URL#git@github.com:}"; esac
@@ -201,9 +223,14 @@ spec:
           mountPath: /tmp/interception-proxy-ca.pem
           subPath: ca.pem
           readOnly: true
+        # tls-*.feature: client/CA PEMs from Secret e2e-mock-tls-certs (optional).
+        - name: mock-tls-certs
+          mountPath: /certs
+          readOnly: true
   volumes:
     - name: app-root
-      emptyDir: {}
+      persistentVolumeClaim:
+        claimName: llama-stack-app-root
     - name: config-cm
       configMap:
         name: llama-stack-config
@@ -217,3 +244,7 @@ spec:
       secret:
         secretName: e2e-interception-proxy-ca
         optional: true
+    - name: mock-tls-certs
+      secret:
+        secretName: e2e-mock-tls-certs
+        optional: true

tests/e2e-prow/rhoai/pipeline-konflux.sh

@@ -143,6 +143,25 @@ log "✅ Mock servers deployed"
 #========================================
 progress "Deploying lightspeed-stack and llama-stack"
 
+# PVC for llama-stack app-root: caches dnf/uv/git install so TLS per-scenario pod
+# recreates skip the expensive init (~6-15 min → ~1-2 min). Delete first to guarantee
+# a fresh checkout for this pipeline revision; re-create immediately so the pod can bind.
+log "Recreating llama-stack-app-root PVC (fresh per pipeline run)..."
+oc delete pvc llama-stack-app-root -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true
+cat <<'EOF' | oc apply -n "$NAMESPACE" -f -
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: llama-stack-app-root
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 10Gi
+EOF
+log "✅ llama-stack-app-root PVC created"
+
 # Llama run config: single source with GitHub E2E (tests/e2e/configs/run-ci.yaml).
 # Lightspeed stack: same tree as local/docker E2E (tests/e2e/configuration/server-mode).
 oc create configmap llama-stack-config -n "$NAMESPACE" \

tests/e2e-prow/rhoai/pipeline-services-konflux.sh

@@ -26,9 +26,25 @@ oc create secret generic llama-stack-ip-secret \
   -n "$NAMESPACE" \
   --dry-run=client -o yaml | oc apply -f -
 
+# PVC must exist before the pod (pipeline-konflux.sh creates it; guard here for standalone use).
+oc get pvc llama-stack-app-root -n "$NAMESPACE" >/dev/null 2>&1 || \
+  oc apply -n "$NAMESPACE" -f - <<'PVCEOF'
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: llama-stack-app-root
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 10Gi
+PVCEOF
+
 timeout 120 oc delete pod llama-stack-service -n "$NAMESPACE" --ignore-not-found=true --wait=true 2>/dev/null || true
 oc apply -n "$NAMESPACE" -f "$BASE_DIR/manifests/lightspeed/llama-stack-openai.yaml"
-oc wait pod/llama-stack-service -n "$NAMESPACE" --for=condition=Ready --timeout=600s
+# First boot runs the full init (dnf + git clone + uv sync ≈ 6-15 min); use a generous timeout.
+oc wait pod/llama-stack-service -n "$NAMESPACE" --for=condition=Ready --timeout=900s
 oc label pod llama-stack-service pod=llama-stack-service -n "$NAMESPACE"
 oc expose pod llama-stack-service --name=llama-stack-service-svc --port=8321 --type=ClusterIP -n "$NAMESPACE"